xref: /linux/drivers/gpu/drm/xe/xe_device.c (revision 284fc30e66e602a5df58393860f67477d6a79339)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_device.h"
7 
8 #include <linux/aperture.h>
9 #include <linux/delay.h>
10 #include <linux/fault-inject.h>
11 #include <linux/units.h>
12 
13 #include <drm/drm_atomic_helper.h>
14 #include <drm/drm_client.h>
15 #include <drm/drm_gem_ttm_helper.h>
16 #include <drm/drm_ioctl.h>
17 #include <drm/drm_managed.h>
18 #include <drm/drm_print.h>
19 #include <uapi/drm/xe_drm.h>
20 
21 #include "display/xe_display.h"
22 #include "instructions/xe_gpu_commands.h"
23 #include "regs/xe_gt_regs.h"
24 #include "regs/xe_regs.h"
25 #include "xe_bo.h"
26 #include "xe_bo_evict.h"
27 #include "xe_debugfs.h"
28 #include "xe_devcoredump.h"
29 #include "xe_device_sysfs.h"
30 #include "xe_dma_buf.h"
31 #include "xe_drm_client.h"
32 #include "xe_drv.h"
33 #include "xe_exec.h"
34 #include "xe_exec_queue.h"
35 #include "xe_force_wake.h"
36 #include "xe_ggtt.h"
37 #include "xe_gsc_proxy.h"
38 #include "xe_gt.h"
39 #include "xe_gt_mcr.h"
40 #include "xe_gt_printk.h"
41 #include "xe_gt_sriov_vf.h"
42 #include "xe_guc.h"
43 #include "xe_guc_pc.h"
44 #include "xe_hw_engine_group.h"
45 #include "xe_hwmon.h"
46 #include "xe_i2c.h"
47 #include "xe_irq.h"
48 #include "xe_late_bind_fw.h"
49 #include "xe_mmio.h"
50 #include "xe_module.h"
51 #include "xe_nvm.h"
52 #include "xe_oa.h"
53 #include "xe_observation.h"
54 #include "xe_pat.h"
55 #include "xe_pcode.h"
56 #include "xe_pm.h"
57 #include "xe_pmu.h"
58 #include "xe_psmi.h"
59 #include "xe_pxp.h"
60 #include "xe_query.h"
61 #include "xe_shrinker.h"
62 #include "xe_survivability_mode.h"
63 #include "xe_sriov.h"
64 #include "xe_tile.h"
65 #include "xe_ttm_stolen_mgr.h"
66 #include "xe_ttm_sys_mgr.h"
67 #include "xe_vm.h"
68 #include "xe_vm_madvise.h"
69 #include "xe_vram.h"
70 #include "xe_vram_types.h"
71 #include "xe_vsec.h"
72 #include "xe_wait_user_fence.h"
73 #include "xe_wa.h"
74 
75 #include <generated/xe_device_wa_oob.h>
76 #include <generated/xe_wa_oob.h>
77 
xe_file_open(struct drm_device * dev,struct drm_file * file)78 static int xe_file_open(struct drm_device *dev, struct drm_file *file)
79 {
80 	struct xe_device *xe = to_xe_device(dev);
81 	struct xe_drm_client *client;
82 	struct xe_file *xef;
83 	int ret = -ENOMEM;
84 	struct task_struct *task = NULL;
85 
86 	xef = kzalloc(sizeof(*xef), GFP_KERNEL);
87 	if (!xef)
88 		return ret;
89 
90 	client = xe_drm_client_alloc();
91 	if (!client) {
92 		kfree(xef);
93 		return ret;
94 	}
95 
96 	xef->drm = file;
97 	xef->client = client;
98 	xef->xe = xe;
99 
100 	mutex_init(&xef->vm.lock);
101 	xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
102 
103 	mutex_init(&xef->exec_queue.lock);
104 	xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
105 
106 	file->driver_priv = xef;
107 	kref_init(&xef->refcount);
108 
109 	task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
110 	if (task) {
111 		xef->process_name = kstrdup(task->comm, GFP_KERNEL);
112 		xef->pid = task->pid;
113 		put_task_struct(task);
114 	}
115 
116 	return 0;
117 }
118 
xe_file_destroy(struct kref * ref)119 static void xe_file_destroy(struct kref *ref)
120 {
121 	struct xe_file *xef = container_of(ref, struct xe_file, refcount);
122 
123 	xa_destroy(&xef->exec_queue.xa);
124 	mutex_destroy(&xef->exec_queue.lock);
125 	xa_destroy(&xef->vm.xa);
126 	mutex_destroy(&xef->vm.lock);
127 
128 	xe_drm_client_put(xef->client);
129 	kfree(xef->process_name);
130 	kfree(xef);
131 }
132 
133 /**
134  * xe_file_get() - Take a reference to the xe file object
135  * @xef: Pointer to the xe file
136  *
137  * Anyone with a pointer to xef must take a reference to the xe file
138  * object using this call.
139  *
140  * Return: xe file pointer
141  */
xe_file_get(struct xe_file * xef)142 struct xe_file *xe_file_get(struct xe_file *xef)
143 {
144 	kref_get(&xef->refcount);
145 	return xef;
146 }
147 
148 /**
149  * xe_file_put() - Drop a reference to the xe file object
150  * @xef: Pointer to the xe file
151  *
152  * Used to drop reference to the xef object
153  */
xe_file_put(struct xe_file * xef)154 void xe_file_put(struct xe_file *xef)
155 {
156 	kref_put(&xef->refcount, xe_file_destroy);
157 }
158 
xe_file_close(struct drm_device * dev,struct drm_file * file)159 static void xe_file_close(struct drm_device *dev, struct drm_file *file)
160 {
161 	struct xe_device *xe = to_xe_device(dev);
162 	struct xe_file *xef = file->driver_priv;
163 	struct xe_vm *vm;
164 	struct xe_exec_queue *q;
165 	unsigned long idx;
166 
167 	xe_pm_runtime_get(xe);
168 
169 	/*
170 	 * No need for exec_queue.lock here as there is no contention for it
171 	 * when FD is closing as IOCTLs presumably can't be modifying the
172 	 * xarray. Taking exec_queue.lock here causes undue dependency on
173 	 * vm->lock taken during xe_exec_queue_kill().
174 	 */
175 	xa_for_each(&xef->exec_queue.xa, idx, q) {
176 		if (q->vm && q->hwe->hw_engine_group)
177 			xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
178 		xe_exec_queue_kill(q);
179 		xe_exec_queue_put(q);
180 	}
181 	xa_for_each(&xef->vm.xa, idx, vm)
182 		xe_vm_close_and_put(vm);
183 
184 	xe_file_put(xef);
185 
186 	xe_pm_runtime_put(xe);
187 }
188 
189 static const struct drm_ioctl_desc xe_ioctls[] = {
190 	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
191 	DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
192 	DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
193 			  DRM_RENDER_ALLOW),
194 	DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
195 	DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
196 	DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
197 	DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
198 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
199 			  DRM_RENDER_ALLOW),
200 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
201 			  DRM_RENDER_ALLOW),
202 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
203 			  DRM_RENDER_ALLOW),
204 	DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
205 			  DRM_RENDER_ALLOW),
206 	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
207 	DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW),
208 	DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl,
209 			  DRM_RENDER_ALLOW),
210 };
211 
xe_drm_ioctl(struct file * file,unsigned int cmd,unsigned long arg)212 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
213 {
214 	struct drm_file *file_priv = file->private_data;
215 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
216 	long ret;
217 
218 	if (xe_device_wedged(xe))
219 		return -ECANCELED;
220 
221 	ret = xe_pm_runtime_get_ioctl(xe);
222 	if (ret >= 0)
223 		ret = drm_ioctl(file, cmd, arg);
224 	xe_pm_runtime_put(xe);
225 
226 	return ret;
227 }
228 
229 #ifdef CONFIG_COMPAT
xe_drm_compat_ioctl(struct file * file,unsigned int cmd,unsigned long arg)230 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
231 {
232 	struct drm_file *file_priv = file->private_data;
233 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
234 	long ret;
235 
236 	if (xe_device_wedged(xe))
237 		return -ECANCELED;
238 
239 	ret = xe_pm_runtime_get_ioctl(xe);
240 	if (ret >= 0)
241 		ret = drm_compat_ioctl(file, cmd, arg);
242 	xe_pm_runtime_put(xe);
243 
244 	return ret;
245 }
246 #else
247 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
248 #define xe_drm_compat_ioctl NULL
249 #endif
250 
barrier_open(struct vm_area_struct * vma)251 static void barrier_open(struct vm_area_struct *vma)
252 {
253 	drm_dev_get(vma->vm_private_data);
254 }
255 
barrier_close(struct vm_area_struct * vma)256 static void barrier_close(struct vm_area_struct *vma)
257 {
258 	drm_dev_put(vma->vm_private_data);
259 }
260 
barrier_release_dummy_page(struct drm_device * dev,void * res)261 static void barrier_release_dummy_page(struct drm_device *dev, void *res)
262 {
263 	struct page *dummy_page = (struct page *)res;
264 
265 	__free_page(dummy_page);
266 }
267 
barrier_fault(struct vm_fault * vmf)268 static vm_fault_t barrier_fault(struct vm_fault *vmf)
269 {
270 	struct drm_device *dev = vmf->vma->vm_private_data;
271 	struct vm_area_struct *vma = vmf->vma;
272 	vm_fault_t ret = VM_FAULT_NOPAGE;
273 	pgprot_t prot;
274 	int idx;
275 
276 	prot = vm_get_page_prot(vma->vm_flags);
277 
278 	if (drm_dev_enter(dev, &idx)) {
279 		unsigned long pfn;
280 
281 #define LAST_DB_PAGE_OFFSET 0x7ff001
282 		pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
283 				LAST_DB_PAGE_OFFSET);
284 		ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,
285 					  pgprot_noncached(prot));
286 		drm_dev_exit(idx);
287 	} else {
288 		struct page *page;
289 
290 		/* Allocate new dummy page to map all the VA range in this VMA to it*/
291 		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
292 		if (!page)
293 			return VM_FAULT_OOM;
294 
295 		/* Set the page to be freed using drmm release action */
296 		if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
297 			return VM_FAULT_OOM;
298 
299 		ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),
300 					  prot);
301 	}
302 
303 	return ret;
304 }
305 
306 static const struct vm_operations_struct vm_ops_barrier = {
307 	.open = barrier_open,
308 	.close = barrier_close,
309 	.fault = barrier_fault,
310 };
311 
xe_pci_barrier_mmap(struct file * filp,struct vm_area_struct * vma)312 static int xe_pci_barrier_mmap(struct file *filp,
313 			       struct vm_area_struct *vma)
314 {
315 	struct drm_file *priv = filp->private_data;
316 	struct drm_device *dev = priv->minor->dev;
317 	struct xe_device *xe = to_xe_device(dev);
318 
319 	if (!IS_DGFX(xe))
320 		return -EINVAL;
321 
322 	if (vma->vm_end - vma->vm_start > SZ_4K)
323 		return -EINVAL;
324 
325 	if (is_cow_mapping(vma->vm_flags))
326 		return -EINVAL;
327 
328 	if (vma->vm_flags & (VM_READ | VM_EXEC))
329 		return -EINVAL;
330 
331 	vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
332 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
333 	vma->vm_ops = &vm_ops_barrier;
334 	vma->vm_private_data = dev;
335 	drm_dev_get(vma->vm_private_data);
336 
337 	return 0;
338 }
339 
xe_mmap(struct file * filp,struct vm_area_struct * vma)340 static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
341 {
342 	struct drm_file *priv = filp->private_data;
343 	struct drm_device *dev = priv->minor->dev;
344 
345 	if (drm_dev_is_unplugged(dev))
346 		return -ENODEV;
347 
348 	switch (vma->vm_pgoff) {
349 	case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
350 		return xe_pci_barrier_mmap(filp, vma);
351 	}
352 
353 	return drm_gem_mmap(filp, vma);
354 }
355 
356 static const struct file_operations xe_driver_fops = {
357 	.owner = THIS_MODULE,
358 	.open = drm_open,
359 	.release = drm_release_noglobal,
360 	.unlocked_ioctl = xe_drm_ioctl,
361 	.mmap = xe_mmap,
362 	.poll = drm_poll,
363 	.read = drm_read,
364 	.compat_ioctl = xe_drm_compat_ioctl,
365 	.llseek = noop_llseek,
366 #ifdef CONFIG_PROC_FS
367 	.show_fdinfo = drm_show_fdinfo,
368 #endif
369 	.fop_flags = FOP_UNSIGNED_OFFSET,
370 };
371 
372 static struct drm_driver driver = {
373 	/* Don't use MTRRs here; the Xserver or userspace app should
374 	 * deal with them for Intel hardware.
375 	 */
376 	.driver_features =
377 	    DRIVER_GEM |
378 	    DRIVER_RENDER | DRIVER_SYNCOBJ |
379 	    DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
380 	.open = xe_file_open,
381 	.postclose = xe_file_close,
382 
383 	.gem_prime_import = xe_gem_prime_import,
384 
385 	.dumb_create = xe_bo_dumb_create,
386 	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
387 #ifdef CONFIG_PROC_FS
388 	.show_fdinfo = xe_drm_client_fdinfo,
389 #endif
390 	.ioctls = xe_ioctls,
391 	.num_ioctls = ARRAY_SIZE(xe_ioctls),
392 	.fops = &xe_driver_fops,
393 	.name = DRIVER_NAME,
394 	.desc = DRIVER_DESC,
395 	.major = DRIVER_MAJOR,
396 	.minor = DRIVER_MINOR,
397 	.patchlevel = DRIVER_PATCHLEVEL,
398 };
399 
xe_device_destroy(struct drm_device * dev,void * dummy)400 static void xe_device_destroy(struct drm_device *dev, void *dummy)
401 {
402 	struct xe_device *xe = to_xe_device(dev);
403 
404 	xe_bo_dev_fini(&xe->bo_device);
405 
406 	if (xe->preempt_fence_wq)
407 		destroy_workqueue(xe->preempt_fence_wq);
408 
409 	if (xe->ordered_wq)
410 		destroy_workqueue(xe->ordered_wq);
411 
412 	if (xe->unordered_wq)
413 		destroy_workqueue(xe->unordered_wq);
414 
415 	if (xe->destroy_wq)
416 		destroy_workqueue(xe->destroy_wq);
417 
418 	ttm_device_fini(&xe->ttm);
419 }
420 
xe_device_create(struct pci_dev * pdev,const struct pci_device_id * ent)421 struct xe_device *xe_device_create(struct pci_dev *pdev,
422 				   const struct pci_device_id *ent)
423 {
424 	struct xe_device *xe;
425 	int err;
426 
427 	xe_display_driver_set_hooks(&driver);
428 
429 	err = aperture_remove_conflicting_pci_devices(pdev, driver.name);
430 	if (err)
431 		return ERR_PTR(err);
432 
433 	xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);
434 	if (IS_ERR(xe))
435 		return xe;
436 
437 	err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
438 			      xe->drm.anon_inode->i_mapping,
439 			      xe->drm.vma_offset_manager, false, false);
440 	if (WARN_ON(err))
441 		goto err;
442 
443 	xe_bo_dev_init(&xe->bo_device);
444 	err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
445 	if (err)
446 		goto err;
447 
448 	err = xe_shrinker_create(xe);
449 	if (err)
450 		goto err;
451 
452 	xe->info.devid = pdev->device;
453 	xe->info.revid = pdev->revision;
454 	xe->info.force_execlist = xe_modparam.force_execlist;
455 	xe->atomic_svm_timeslice_ms = 5;
456 
457 	err = xe_irq_init(xe);
458 	if (err)
459 		goto err;
460 
461 	xe_validation_device_init(&xe->val);
462 
463 	init_waitqueue_head(&xe->ufence_wq);
464 
465 	init_rwsem(&xe->usm.lock);
466 
467 	xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
468 
469 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
470 		/* Trigger a large asid and an early asid wrap. */
471 		u32 asid;
472 
473 		BUILD_BUG_ON(XE_MAX_ASID < 2);
474 		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
475 				      XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
476 				      &xe->usm.next_asid, GFP_KERNEL);
477 		drm_WARN_ON(&xe->drm, err);
478 		if (err >= 0)
479 			xa_erase(&xe->usm.asid_to_vm, asid);
480 	}
481 
482 	err = xe_bo_pinned_init(xe);
483 	if (err)
484 		goto err;
485 
486 	xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
487 						       WQ_MEM_RECLAIM);
488 	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
489 	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);
490 	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0);
491 	if (!xe->ordered_wq || !xe->unordered_wq ||
492 	    !xe->preempt_fence_wq || !xe->destroy_wq) {
493 		/*
494 		 * Cleanup done in xe_device_destroy via
495 		 * drmm_add_action_or_reset register above
496 		 */
497 		drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
498 		err = -ENOMEM;
499 		goto err;
500 	}
501 
502 	err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
503 	if (err)
504 		goto err;
505 
506 	return xe;
507 
508 err:
509 	return ERR_PTR(err);
510 }
511 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
512 
xe_driver_flr_disabled(struct xe_device * xe)513 static bool xe_driver_flr_disabled(struct xe_device *xe)
514 {
515 	if (IS_SRIOV_VF(xe))
516 		return true;
517 
518 	if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
519 		drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n");
520 		return true;
521 	}
522 
523 	return false;
524 }
525 
526 /*
527  * The driver-initiated FLR is the highest level of reset that we can trigger
528  * from within the driver. It is different from the PCI FLR in that it doesn't
529  * fully reset the SGUnit and doesn't modify the PCI config space and therefore
530  * it doesn't require a re-enumeration of the PCI BARs. However, the
531  * driver-initiated FLR does still cause a reset of both GT and display and a
532  * memory wipe of local and stolen memory, so recovery would require a full HW
533  * re-init and saving/restoring (or re-populating) the wiped memory. Since we
534  * perform the FLR as the very last action before releasing access to the HW
535  * during the driver release flow, we don't attempt recovery at all, because
536  * if/when a new instance of Xe is bound to the device it will do a full
537  * re-init anyway.
538  */
__xe_driver_flr(struct xe_device * xe)539 static void __xe_driver_flr(struct xe_device *xe)
540 {
541 	const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */
542 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
543 	int ret;
544 
545 	drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
546 
547 	/*
548 	 * Make sure any pending FLR requests have cleared by waiting for the
549 	 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
550 	 * to make sure it's not still set from a prior attempt (it's a write to
551 	 * clear bit).
552 	 * Note that we should never be in a situation where a previous attempt
553 	 * is still pending (unless the HW is totally dead), but better to be
554 	 * safe in case something unexpected happens
555 	 */
556 	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
557 	if (ret) {
558 		drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
559 		return;
560 	}
561 	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
562 
563 	/* Trigger the actual Driver-FLR */
564 	xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR);
565 
566 	/* Wait for hardware teardown to complete */
567 	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
568 	if (ret) {
569 		drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
570 		return;
571 	}
572 
573 	/* Wait for hardware/firmware re-init to complete */
574 	ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
575 			     flr_timeout, NULL, false);
576 	if (ret) {
577 		drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
578 		return;
579 	}
580 
581 	/* Clear sticky completion status */
582 	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
583 }
584 
xe_driver_flr(struct xe_device * xe)585 static void xe_driver_flr(struct xe_device *xe)
586 {
587 	if (xe_driver_flr_disabled(xe))
588 		return;
589 
590 	__xe_driver_flr(xe);
591 }
592 
xe_driver_flr_fini(void * arg)593 static void xe_driver_flr_fini(void *arg)
594 {
595 	struct xe_device *xe = arg;
596 
597 	if (xe->needs_flr_on_fini)
598 		xe_driver_flr(xe);
599 }
600 
xe_device_sanitize(void * arg)601 static void xe_device_sanitize(void *arg)
602 {
603 	struct xe_device *xe = arg;
604 	struct xe_gt *gt;
605 	u8 id;
606 
607 	for_each_gt(gt, xe, id)
608 		xe_gt_sanitize(gt);
609 }
610 
xe_set_dma_info(struct xe_device * xe)611 static int xe_set_dma_info(struct xe_device *xe)
612 {
613 	unsigned int mask_size = xe->info.dma_mask_size;
614 	int err;
615 
616 	dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
617 
618 	err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
619 	if (err)
620 		goto mask_err;
621 
622 	err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
623 	if (err)
624 		goto mask_err;
625 
626 	return 0;
627 
628 mask_err:
629 	drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
630 	return err;
631 }
632 
verify_lmem_ready(struct xe_device * xe)633 static bool verify_lmem_ready(struct xe_device *xe)
634 {
635 	u32 val = xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT;
636 
637 	return !!val;
638 }
639 
wait_for_lmem_ready(struct xe_device * xe)640 static int wait_for_lmem_ready(struct xe_device *xe)
641 {
642 	unsigned long timeout, start;
643 
644 	if (!IS_DGFX(xe))
645 		return 0;
646 
647 	if (IS_SRIOV_VF(xe))
648 		return 0;
649 
650 	if (verify_lmem_ready(xe))
651 		return 0;
652 
653 	drm_dbg(&xe->drm, "Waiting for lmem initialization\n");
654 
655 	start = jiffies;
656 	timeout = start + secs_to_jiffies(60); /* 60 sec! */
657 
658 	do {
659 		if (signal_pending(current))
660 			return -EINTR;
661 
662 		/*
663 		 * The boot firmware initializes local memory and
664 		 * assesses its health. If memory training fails,
665 		 * the punit will have been instructed to keep the GT powered
666 		 * down.we won't be able to communicate with it
667 		 *
668 		 * If the status check is done before punit updates the register,
669 		 * it can lead to the system being unusable.
670 		 * use a timeout and defer the probe to prevent this.
671 		 */
672 		if (time_after(jiffies, timeout)) {
673 			drm_dbg(&xe->drm, "lmem not initialized by firmware\n");
674 			return -EPROBE_DEFER;
675 		}
676 
677 		msleep(20);
678 
679 	} while (!verify_lmem_ready(xe));
680 
681 	drm_dbg(&xe->drm, "lmem ready after %ums",
682 		jiffies_to_msecs(jiffies - start));
683 
684 	return 0;
685 }
686 ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */
687 
vf_update_device_info(struct xe_device * xe)688 static void vf_update_device_info(struct xe_device *xe)
689 {
690 	xe_assert(xe, IS_SRIOV_VF(xe));
691 	/* disable features that are not available/applicable to VFs */
692 	xe->info.probe_display = 0;
693 	xe->info.has_heci_cscfi = 0;
694 	xe->info.has_heci_gscfi = 0;
695 	xe->info.has_late_bind = 0;
696 	xe->info.skip_guc_pc = 1;
697 	xe->info.skip_pcode = 1;
698 }
699 
xe_device_vram_alloc(struct xe_device * xe)700 static int xe_device_vram_alloc(struct xe_device *xe)
701 {
702 	struct xe_vram_region *vram;
703 
704 	if (!IS_DGFX(xe))
705 		return 0;
706 
707 	vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL);
708 	if (!vram)
709 		return -ENOMEM;
710 
711 	xe->mem.vram = vram;
712 	return 0;
713 }
714 
715 /**
716  * xe_device_probe_early: Device early probe
717  * @xe: xe device instance
718  *
719  * Initialize MMIO resources that don't require any
720  * knowledge about tile count. Also initialize pcode and
721  * check vram initialization on root tile.
722  *
723  * Return: 0 on success, error code on failure
724  */
xe_device_probe_early(struct xe_device * xe)725 int xe_device_probe_early(struct xe_device *xe)
726 {
727 	int err;
728 
729 	xe_wa_device_init(xe);
730 	xe_wa_process_device_oob(xe);
731 
732 	err = xe_mmio_probe_early(xe);
733 	if (err)
734 		return err;
735 
736 	xe_sriov_probe_early(xe);
737 
738 	if (IS_SRIOV_VF(xe))
739 		vf_update_device_info(xe);
740 
741 	err = xe_pcode_probe_early(xe);
742 	if (err || xe_survivability_mode_is_requested(xe)) {
743 		int save_err = err;
744 
745 		/*
746 		 * Try to leave device in survivability mode if device is
747 		 * possible, but still return the previous error for error
748 		 * propagation
749 		 */
750 		err = xe_survivability_mode_boot_enable(xe);
751 		if (err)
752 			return err;
753 
754 		return save_err;
755 	}
756 
757 	err = wait_for_lmem_ready(xe);
758 	if (err)
759 		return err;
760 
761 	xe->wedged.mode = xe_modparam.wedged_mode;
762 
763 	err = xe_device_vram_alloc(xe);
764 	if (err)
765 		return err;
766 
767 	return 0;
768 }
769 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
770 
probe_has_flat_ccs(struct xe_device * xe)771 static int probe_has_flat_ccs(struct xe_device *xe)
772 {
773 	struct xe_gt *gt;
774 	unsigned int fw_ref;
775 	u32 reg;
776 
777 	/* Always enabled/disabled, no runtime check to do */
778 	if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
779 		return 0;
780 
781 	gt = xe_root_mmio_gt(xe);
782 
783 	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
784 	if (!fw_ref)
785 		return -ETIMEDOUT;
786 
787 	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
788 	xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
789 
790 	if (!xe->info.has_flat_ccs)
791 		drm_dbg(&xe->drm,
792 			"Flat CCS has been disabled in bios, May lead to performance impact");
793 
794 	xe_force_wake_put(gt_to_fw(gt), fw_ref);
795 
796 	return 0;
797 }
798 
xe_device_probe(struct xe_device * xe)799 int xe_device_probe(struct xe_device *xe)
800 {
801 	struct xe_tile *tile;
802 	struct xe_gt *gt;
803 	int err;
804 	u8 id;
805 
806 	xe_pat_init_early(xe);
807 
808 	err = xe_sriov_init(xe);
809 	if (err)
810 		return err;
811 
812 	xe->info.mem_region_mask = 1;
813 
814 	err = xe_set_dma_info(xe);
815 	if (err)
816 		return err;
817 
818 	err = xe_mmio_probe_tiles(xe);
819 	if (err)
820 		return err;
821 
822 	for_each_gt(gt, xe, id) {
823 		err = xe_gt_init_early(gt);
824 		if (err)
825 			return err;
826 	}
827 
828 	for_each_tile(tile, xe, id) {
829 		err = xe_ggtt_init_early(tile->mem.ggtt);
830 		if (err)
831 			return err;
832 	}
833 
834 	/*
835 	 * From here on, if a step fails, make sure a Driver-FLR is triggereed
836 	 */
837 	err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
838 	if (err)
839 		return err;
840 
841 	err = probe_has_flat_ccs(xe);
842 	if (err)
843 		return err;
844 
845 	err = xe_vram_probe(xe);
846 	if (err)
847 		return err;
848 
849 	for_each_tile(tile, xe, id) {
850 		err = xe_tile_init_noalloc(tile);
851 		if (err)
852 			return err;
853 	}
854 
855 	/*
856 	 * Allow allocations only now to ensure xe_display_init_early()
857 	 * is the first to allocate, always.
858 	 */
859 	err = xe_ttm_sys_mgr_init(xe);
860 	if (err)
861 		return err;
862 
863 	/* Allocate and map stolen after potential VRAM resize */
864 	err = xe_ttm_stolen_mgr_init(xe);
865 	if (err)
866 		return err;
867 
868 	/*
869 	 * Now that GT is initialized (TTM in particular),
870 	 * we can try to init display, and inherit the initial fb.
871 	 * This is the reason the first allocation needs to be done
872 	 * inside display.
873 	 */
874 	err = xe_display_init_early(xe);
875 	if (err)
876 		return err;
877 
878 	for_each_tile(tile, xe, id) {
879 		err = xe_tile_init(tile);
880 		if (err)
881 			return err;
882 	}
883 
884 	err = xe_irq_install(xe);
885 	if (err)
886 		return err;
887 
888 	for_each_gt(gt, xe, id) {
889 		err = xe_gt_init(gt);
890 		if (err)
891 			return err;
892 	}
893 
894 	if (xe->tiles->media_gt &&
895 	    XE_GT_WA(xe->tiles->media_gt, 15015404425_disable))
896 		XE_DEVICE_WA_DISABLE(xe, 15015404425);
897 
898 	err = xe_devcoredump_init(xe);
899 	if (err)
900 		return err;
901 
902 	xe_nvm_init(xe);
903 
904 	err = xe_heci_gsc_init(xe);
905 	if (err)
906 		return err;
907 
908 	err = xe_late_bind_init(&xe->late_bind);
909 	if (err)
910 		return err;
911 
912 	err = xe_oa_init(xe);
913 	if (err)
914 		return err;
915 
916 	err = xe_display_init(xe);
917 	if (err)
918 		return err;
919 
920 	err = xe_pxp_init(xe);
921 	if (err)
922 		return err;
923 
924 	err = xe_psmi_init(xe);
925 	if (err)
926 		return err;
927 
928 	err = drm_dev_register(&xe->drm, 0);
929 	if (err)
930 		return err;
931 
932 	xe_display_register(xe);
933 
934 	err = xe_oa_register(xe);
935 	if (err)
936 		goto err_unregister_display;
937 
938 	err = xe_pmu_register(&xe->pmu);
939 	if (err)
940 		goto err_unregister_display;
941 
942 	err = xe_device_sysfs_init(xe);
943 	if (err)
944 		goto err_unregister_display;
945 
946 	xe_debugfs_register(xe);
947 
948 	err = xe_hwmon_register(xe);
949 	if (err)
950 		goto err_unregister_display;
951 
952 	err = xe_i2c_probe(xe);
953 	if (err)
954 		goto err_unregister_display;
955 
956 	for_each_gt(gt, xe, id)
957 		xe_gt_sanitize_freq(gt);
958 
959 	xe_vsec_init(xe);
960 
961 	err = xe_sriov_init_late(xe);
962 	if (err)
963 		goto err_unregister_display;
964 
965 	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
966 
967 err_unregister_display:
968 	xe_display_unregister(xe);
969 
970 	return err;
971 }
972 
xe_device_remove(struct xe_device * xe)973 void xe_device_remove(struct xe_device *xe)
974 {
975 	xe_display_unregister(xe);
976 
977 	xe_nvm_fini(xe);
978 
979 	drm_dev_unplug(&xe->drm);
980 
981 	xe_bo_pci_dev_remove_all(xe);
982 }
983 
xe_device_shutdown(struct xe_device * xe)984 void xe_device_shutdown(struct xe_device *xe)
985 {
986 	struct xe_gt *gt;
987 	u8 id;
988 
989 	drm_dbg(&xe->drm, "Shutting down device\n");
990 
991 	if (xe_driver_flr_disabled(xe)) {
992 		xe_display_pm_shutdown(xe);
993 
994 		xe_irq_suspend(xe);
995 
996 		for_each_gt(gt, xe, id)
997 			xe_gt_shutdown(gt);
998 
999 		xe_display_pm_shutdown_late(xe);
1000 	} else {
1001 		/* BOOM! */
1002 		__xe_driver_flr(xe);
1003 	}
1004 }
1005 
1006 /**
1007  * xe_device_wmb() - Device specific write memory barrier
1008  * @xe: the &xe_device
1009  *
1010  * While wmb() is sufficient for a barrier if we use system memory, on discrete
1011  * platforms with device memory we additionally need to issue a register write.
1012  * Since it doesn't matter which register we write to, use the read-only VF_CAP
1013  * register that is also marked as accessible by the VFs.
1014  */
xe_device_wmb(struct xe_device * xe)1015 void xe_device_wmb(struct xe_device *xe)
1016 {
1017 	wmb();
1018 	if (IS_DGFX(xe))
1019 		xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);
1020 }
1021 
1022 /*
1023  * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt.
1024  */
tdf_request_sync(struct xe_device * xe)1025 static void tdf_request_sync(struct xe_device *xe)
1026 {
1027 	unsigned int fw_ref;
1028 	struct xe_gt *gt;
1029 	u8 id;
1030 
1031 	for_each_gt(gt, xe, id) {
1032 		if (xe_gt_is_media_type(gt))
1033 			continue;
1034 
1035 		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1036 		if (!fw_ref)
1037 			return;
1038 
1039 		xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
1040 
1041 		/*
1042 		 * FIXME: We can likely do better here with our choice of
1043 		 * timeout. Currently we just assume the worst case, i.e. 150us,
1044 		 * which is believed to be sufficient to cover the worst case
1045 		 * scenario on current platforms if all cache entries are
1046 		 * transient and need to be flushed..
1047 		 */
1048 		if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1049 				   150, NULL, false))
1050 			xe_gt_err_once(gt, "TD flush timeout\n");
1051 
1052 		xe_force_wake_put(gt_to_fw(gt), fw_ref);
1053 	}
1054 }
1055 
xe_device_l2_flush(struct xe_device * xe)1056 void xe_device_l2_flush(struct xe_device *xe)
1057 {
1058 	struct xe_gt *gt;
1059 	unsigned int fw_ref;
1060 
1061 	gt = xe_root_mmio_gt(xe);
1062 
1063 	if (!XE_GT_WA(gt, 16023588340))
1064 		return;
1065 
1066 	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1067 	if (!fw_ref)
1068 		return;
1069 
1070 	spin_lock(&gt->global_invl_lock);
1071 
1072 	xe_mmio_write32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1);
1073 	if (xe_mmio_wait32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 500, NULL, true))
1074 		xe_gt_err_once(gt, "Global invalidation timeout\n");
1075 
1076 	spin_unlock(&gt->global_invl_lock);
1077 
1078 	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1079 }
1080 
1081 /**
1082  * xe_device_td_flush() - Flush transient L3 cache entries
1083  * @xe: The device
1084  *
1085  * Display engine has direct access to memory and is never coherent with L3/L4
1086  * caches (or CPU caches), however KMD is responsible for specifically flushing
1087  * transient L3 GPU cache entries prior to the flip sequence to ensure scanout
1088  * can happen from such a surface without seeing corruption.
1089  *
1090  * Display surfaces can be tagged as transient by mapping it using one of the
1091  * various L3:XD PAT index modes on Xe2.
1092  *
1093  * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
1094  * at the end of each submission via PIPE_CONTROL for compute/render, since SA
1095  * Media is not coherent with L3 and we want to support render-vs-media
1096  * usescases. For other engines like copy/blt the HW internally forces uncached
1097  * behaviour, hence why we can skip the TDF on such platforms.
1098  */
xe_device_td_flush(struct xe_device * xe)1099 void xe_device_td_flush(struct xe_device *xe)
1100 {
1101 	struct xe_gt *root_gt;
1102 
1103 	if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
1104 		return;
1105 
1106 	root_gt = xe_root_mmio_gt(xe);
1107 	if (XE_GT_WA(root_gt, 16023588340)) {
1108 		/* A transient flush is not sufficient: flush the L2 */
1109 		xe_device_l2_flush(xe);
1110 	} else {
1111 		xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc);
1112 		tdf_request_sync(xe);
1113 		xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc);
1114 	}
1115 }
1116 
xe_device_ccs_bytes(struct xe_device * xe,u64 size)1117 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
1118 {
1119 	return xe_device_has_flat_ccs(xe) ?
1120 		DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
1121 }
1122 
1123 /**
1124  * xe_device_assert_mem_access - Inspect the current runtime_pm state.
1125  * @xe: xe device instance
1126  *
1127  * To be used before any kind of memory access. It will splat a debug warning
1128  * if the device is currently sleeping. But it doesn't guarantee in any way
1129  * that the device is going to remain awake. Xe PM runtime get and put
1130  * functions might be added to the outer bound of the memory access, while
1131  * this check is intended for inner usage to splat some warning if the worst
1132  * case has just happened.
1133  */
xe_device_assert_mem_access(struct xe_device * xe)1134 void xe_device_assert_mem_access(struct xe_device *xe)
1135 {
1136 	xe_assert(xe, !xe_pm_runtime_suspended(xe));
1137 }
1138 
xe_device_snapshot_print(struct xe_device * xe,struct drm_printer * p)1139 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
1140 {
1141 	struct xe_gt *gt;
1142 	u8 id;
1143 
1144 	drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
1145 	drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
1146 
1147 	for_each_gt(gt, xe, id) {
1148 		drm_printf(p, "GT id: %u\n", id);
1149 		drm_printf(p, "\tTile: %u\n", gt->tile->id);
1150 		drm_printf(p, "\tType: %s\n",
1151 			   gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
1152 		drm_printf(p, "\tIP ver: %u.%u.%u\n",
1153 			   REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
1154 			   REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
1155 			   REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
1156 		drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
1157 	}
1158 }
1159 
xe_device_canonicalize_addr(struct xe_device * xe,u64 address)1160 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
1161 {
1162 	return sign_extend64(address, xe->info.va_bits - 1);
1163 }
1164 
xe_device_uncanonicalize_addr(struct xe_device * xe,u64 address)1165 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
1166 {
1167 	return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
1168 }
1169 
xe_device_wedged_fini(struct drm_device * drm,void * arg)1170 static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
1171 {
1172 	struct xe_device *xe = arg;
1173 
1174 	xe_pm_runtime_put(xe);
1175 }
1176 
1177 /**
1178  * DOC: Xe Device Wedging
1179  *
1180  * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst.
1181  * When device is in wedged state, every IOCTL will be blocked and GT cannot be
1182  * used. Certain critical errors like gt reset failure, firmware failures can cause
1183  * the device to be wedged. The default recovery method for a wedged state
1184  * is rebind/bus-reset.
1185  *
1186  * Another recovery method is vendor-specific. Below are the cases that send
1187  * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent.
1188  *
1189  * Case: Firmware Flash
1190  * --------------------
1191  *
1192  * Identification Hint
1193  * +++++++++++++++++++
1194  *
1195  * ``WEDGED=vendor-specific`` drm device wedged uevent with
1196  * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify
1197  * admin/userspace consumer about the need for a firmware flash.
1198  *
1199  * Recovery Procedure
1200  * ++++++++++++++++++
1201  *
1202  * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow
1203  * the below steps
1204  *
1205  * - Check Runtime Survivability mode sysfs.
1206  *   If enabled, firmware flash is required to recover the device.
1207  *
1208  *   /sys/bus/pci/devices/<device>/survivability_mode
1209  *
1210  * - Admin/userpsace consumer can use firmware flashing tools like fwupd to flash
1211  *   firmware and restore device to normal operation.
1212  */
1213 
1214 /**
1215  * xe_device_set_wedged_method - Set wedged recovery method
1216  * @xe: xe device instance
1217  * @method: recovery method to set
1218  *
1219  * Set wedged recovery method to be sent in drm wedged uevent.
1220  */
xe_device_set_wedged_method(struct xe_device * xe,unsigned long method)1221 void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method)
1222 {
1223 	xe->wedged.method = method;
1224 }
1225 
1226 /**
1227  * xe_device_declare_wedged - Declare device wedged
1228  * @xe: xe device instance
1229  *
1230  * This is a final state that can only be cleared with the recovery method
1231  * specified in the drm wedged uevent. The method can be set using
1232  * xe_device_set_wedged_method before declaring the device as wedged. If no method
1233  * is set, reprobe (unbind/re-bind) will be sent by default.
1234  *
1235  * In this state every IOCTL will be blocked so the GT cannot be used.
1236  * In general it will be called upon any critical error such as gt reset
1237  * failure or guc loading failure. Userspace will be notified of this state
1238  * through device wedged uevent.
1239  * If xe.wedged module parameter is set to 2, this function will be called
1240  * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
1241  * snapshot capture. In this mode, GT reset won't be attempted so the state of
1242  * the issue is preserved for further debugging.
1243  */
xe_device_declare_wedged(struct xe_device * xe)1244 void xe_device_declare_wedged(struct xe_device *xe)
1245 {
1246 	struct xe_gt *gt;
1247 	u8 id;
1248 
1249 	if (xe->wedged.mode == 0) {
1250 		drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
1251 		return;
1252 	}
1253 
1254 	xe_pm_runtime_get_noresume(xe);
1255 
1256 	if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) {
1257 		drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n");
1258 		return;
1259 	}
1260 
1261 	if (!atomic_xchg(&xe->wedged.flag, 1)) {
1262 		xe->needs_flr_on_fini = true;
1263 		drm_err(&xe->drm,
1264 			"CRITICAL: Xe has declared device %s as wedged.\n"
1265 			"IOCTLs and executions are blocked. Only a rebind may clear the failure\n"
1266 			"Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
1267 			dev_name(xe->drm.dev));
1268 	}
1269 
1270 	for_each_gt(gt, xe, id)
1271 		xe_gt_declare_wedged(gt);
1272 
1273 	if (xe_device_wedged(xe)) {
1274 		/* If no wedge recovery method is set, use default */
1275 		if (!xe->wedged.method)
1276 			xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND |
1277 						    DRM_WEDGE_RECOVERY_BUS_RESET);
1278 
1279 		/* Notify userspace of wedged device */
1280 		drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL);
1281 	}
1282 }
1283