1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include "xe_device.h"
7
8 #include <linux/aperture.h>
9 #include <linux/delay.h>
10 #include <linux/fault-inject.h>
11 #include <linux/units.h>
12
13 #include <drm/drm_atomic_helper.h>
14 #include <drm/drm_client.h>
15 #include <drm/drm_gem_ttm_helper.h>
16 #include <drm/drm_ioctl.h>
17 #include <drm/drm_managed.h>
18 #include <drm/drm_print.h>
19 #include <uapi/drm/xe_drm.h>
20
21 #include "display/xe_display.h"
22 #include "instructions/xe_gpu_commands.h"
23 #include "regs/xe_gt_regs.h"
24 #include "regs/xe_regs.h"
25 #include "xe_bo.h"
26 #include "xe_bo_evict.h"
27 #include "xe_debugfs.h"
28 #include "xe_devcoredump.h"
29 #include "xe_device_sysfs.h"
30 #include "xe_dma_buf.h"
31 #include "xe_drm_client.h"
32 #include "xe_drv.h"
33 #include "xe_exec.h"
34 #include "xe_exec_queue.h"
35 #include "xe_force_wake.h"
36 #include "xe_ggtt.h"
37 #include "xe_gsc_proxy.h"
38 #include "xe_gt.h"
39 #include "xe_gt_mcr.h"
40 #include "xe_gt_printk.h"
41 #include "xe_gt_sriov_vf.h"
42 #include "xe_guc.h"
43 #include "xe_guc_pc.h"
44 #include "xe_hw_engine_group.h"
45 #include "xe_hwmon.h"
46 #include "xe_i2c.h"
47 #include "xe_irq.h"
48 #include "xe_mmio.h"
49 #include "xe_module.h"
50 #include "xe_nvm.h"
51 #include "xe_oa.h"
52 #include "xe_observation.h"
53 #include "xe_pat.h"
54 #include "xe_pcode.h"
55 #include "xe_pm.h"
56 #include "xe_pmu.h"
57 #include "xe_pxp.h"
58 #include "xe_query.h"
59 #include "xe_shrinker.h"
60 #include "xe_survivability_mode.h"
61 #include "xe_sriov.h"
62 #include "xe_tile.h"
63 #include "xe_ttm_stolen_mgr.h"
64 #include "xe_ttm_sys_mgr.h"
65 #include "xe_vm.h"
66 #include "xe_vram.h"
67 #include "xe_vsec.h"
68 #include "xe_wait_user_fence.h"
69 #include "xe_wa.h"
70
71 #include <generated/xe_device_wa_oob.h>
72 #include <generated/xe_wa_oob.h>
73
xe_file_open(struct drm_device * dev,struct drm_file * file)74 static int xe_file_open(struct drm_device *dev, struct drm_file *file)
75 {
76 struct xe_device *xe = to_xe_device(dev);
77 struct xe_drm_client *client;
78 struct xe_file *xef;
79 int ret = -ENOMEM;
80 struct task_struct *task = NULL;
81
82 xef = kzalloc(sizeof(*xef), GFP_KERNEL);
83 if (!xef)
84 return ret;
85
86 client = xe_drm_client_alloc();
87 if (!client) {
88 kfree(xef);
89 return ret;
90 }
91
92 xef->drm = file;
93 xef->client = client;
94 xef->xe = xe;
95
96 mutex_init(&xef->vm.lock);
97 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
98
99 mutex_init(&xef->exec_queue.lock);
100 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
101
102 file->driver_priv = xef;
103 kref_init(&xef->refcount);
104
105 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
106 if (task) {
107 xef->process_name = kstrdup(task->comm, GFP_KERNEL);
108 xef->pid = task->pid;
109 put_task_struct(task);
110 }
111
112 return 0;
113 }
114
xe_file_destroy(struct kref * ref)115 static void xe_file_destroy(struct kref *ref)
116 {
117 struct xe_file *xef = container_of(ref, struct xe_file, refcount);
118
119 xa_destroy(&xef->exec_queue.xa);
120 mutex_destroy(&xef->exec_queue.lock);
121 xa_destroy(&xef->vm.xa);
122 mutex_destroy(&xef->vm.lock);
123
124 xe_drm_client_put(xef->client);
125 kfree(xef->process_name);
126 kfree(xef);
127 }
128
129 /**
130 * xe_file_get() - Take a reference to the xe file object
131 * @xef: Pointer to the xe file
132 *
133 * Anyone with a pointer to xef must take a reference to the xe file
134 * object using this call.
135 *
136 * Return: xe file pointer
137 */
xe_file_get(struct xe_file * xef)138 struct xe_file *xe_file_get(struct xe_file *xef)
139 {
140 kref_get(&xef->refcount);
141 return xef;
142 }
143
144 /**
145 * xe_file_put() - Drop a reference to the xe file object
146 * @xef: Pointer to the xe file
147 *
148 * Used to drop reference to the xef object
149 */
xe_file_put(struct xe_file * xef)150 void xe_file_put(struct xe_file *xef)
151 {
152 kref_put(&xef->refcount, xe_file_destroy);
153 }
154
xe_file_close(struct drm_device * dev,struct drm_file * file)155 static void xe_file_close(struct drm_device *dev, struct drm_file *file)
156 {
157 struct xe_device *xe = to_xe_device(dev);
158 struct xe_file *xef = file->driver_priv;
159 struct xe_vm *vm;
160 struct xe_exec_queue *q;
161 unsigned long idx;
162
163 xe_pm_runtime_get(xe);
164
165 /*
166 * No need for exec_queue.lock here as there is no contention for it
167 * when FD is closing as IOCTLs presumably can't be modifying the
168 * xarray. Taking exec_queue.lock here causes undue dependency on
169 * vm->lock taken during xe_exec_queue_kill().
170 */
171 xa_for_each(&xef->exec_queue.xa, idx, q) {
172 if (q->vm && q->hwe->hw_engine_group)
173 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
174 xe_exec_queue_kill(q);
175 xe_exec_queue_put(q);
176 }
177 xa_for_each(&xef->vm.xa, idx, vm)
178 xe_vm_close_and_put(vm);
179
180 xe_file_put(xef);
181
182 xe_pm_runtime_put(xe);
183 }
184
185 static const struct drm_ioctl_desc xe_ioctls[] = {
186 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
187 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
188 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
189 DRM_RENDER_ALLOW),
190 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
191 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
192 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
193 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
194 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
195 DRM_RENDER_ALLOW),
196 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
197 DRM_RENDER_ALLOW),
198 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
199 DRM_RENDER_ALLOW),
200 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
201 DRM_RENDER_ALLOW),
202 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
203 };
204
xe_drm_ioctl(struct file * file,unsigned int cmd,unsigned long arg)205 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
206 {
207 struct drm_file *file_priv = file->private_data;
208 struct xe_device *xe = to_xe_device(file_priv->minor->dev);
209 long ret;
210
211 if (xe_device_wedged(xe))
212 return -ECANCELED;
213
214 ret = xe_pm_runtime_get_ioctl(xe);
215 if (ret >= 0)
216 ret = drm_ioctl(file, cmd, arg);
217 xe_pm_runtime_put(xe);
218
219 return ret;
220 }
221
222 #ifdef CONFIG_COMPAT
xe_drm_compat_ioctl(struct file * file,unsigned int cmd,unsigned long arg)223 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
224 {
225 struct drm_file *file_priv = file->private_data;
226 struct xe_device *xe = to_xe_device(file_priv->minor->dev);
227 long ret;
228
229 if (xe_device_wedged(xe))
230 return -ECANCELED;
231
232 ret = xe_pm_runtime_get_ioctl(xe);
233 if (ret >= 0)
234 ret = drm_compat_ioctl(file, cmd, arg);
235 xe_pm_runtime_put(xe);
236
237 return ret;
238 }
239 #else
240 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
241 #define xe_drm_compat_ioctl NULL
242 #endif
243
barrier_open(struct vm_area_struct * vma)244 static void barrier_open(struct vm_area_struct *vma)
245 {
246 drm_dev_get(vma->vm_private_data);
247 }
248
barrier_close(struct vm_area_struct * vma)249 static void barrier_close(struct vm_area_struct *vma)
250 {
251 drm_dev_put(vma->vm_private_data);
252 }
253
barrier_release_dummy_page(struct drm_device * dev,void * res)254 static void barrier_release_dummy_page(struct drm_device *dev, void *res)
255 {
256 struct page *dummy_page = (struct page *)res;
257
258 __free_page(dummy_page);
259 }
260
barrier_fault(struct vm_fault * vmf)261 static vm_fault_t barrier_fault(struct vm_fault *vmf)
262 {
263 struct drm_device *dev = vmf->vma->vm_private_data;
264 struct vm_area_struct *vma = vmf->vma;
265 vm_fault_t ret = VM_FAULT_NOPAGE;
266 pgprot_t prot;
267 int idx;
268
269 prot = vm_get_page_prot(vma->vm_flags);
270
271 if (drm_dev_enter(dev, &idx)) {
272 unsigned long pfn;
273
274 #define LAST_DB_PAGE_OFFSET 0x7ff001
275 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
276 LAST_DB_PAGE_OFFSET);
277 ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,
278 pgprot_noncached(prot));
279 drm_dev_exit(idx);
280 } else {
281 struct page *page;
282
283 /* Allocate new dummy page to map all the VA range in this VMA to it*/
284 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
285 if (!page)
286 return VM_FAULT_OOM;
287
288 /* Set the page to be freed using drmm release action */
289 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
290 return VM_FAULT_OOM;
291
292 ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),
293 prot);
294 }
295
296 return ret;
297 }
298
299 static const struct vm_operations_struct vm_ops_barrier = {
300 .open = barrier_open,
301 .close = barrier_close,
302 .fault = barrier_fault,
303 };
304
xe_pci_barrier_mmap(struct file * filp,struct vm_area_struct * vma)305 static int xe_pci_barrier_mmap(struct file *filp,
306 struct vm_area_struct *vma)
307 {
308 struct drm_file *priv = filp->private_data;
309 struct drm_device *dev = priv->minor->dev;
310 struct xe_device *xe = to_xe_device(dev);
311
312 if (!IS_DGFX(xe))
313 return -EINVAL;
314
315 if (vma->vm_end - vma->vm_start > SZ_4K)
316 return -EINVAL;
317
318 if (is_cow_mapping(vma->vm_flags))
319 return -EINVAL;
320
321 if (vma->vm_flags & (VM_READ | VM_EXEC))
322 return -EINVAL;
323
324 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
325 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
326 vma->vm_ops = &vm_ops_barrier;
327 vma->vm_private_data = dev;
328 drm_dev_get(vma->vm_private_data);
329
330 return 0;
331 }
332
xe_mmap(struct file * filp,struct vm_area_struct * vma)333 static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
334 {
335 struct drm_file *priv = filp->private_data;
336 struct drm_device *dev = priv->minor->dev;
337
338 if (drm_dev_is_unplugged(dev))
339 return -ENODEV;
340
341 switch (vma->vm_pgoff) {
342 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
343 return xe_pci_barrier_mmap(filp, vma);
344 }
345
346 return drm_gem_mmap(filp, vma);
347 }
348
349 static const struct file_operations xe_driver_fops = {
350 .owner = THIS_MODULE,
351 .open = drm_open,
352 .release = drm_release_noglobal,
353 .unlocked_ioctl = xe_drm_ioctl,
354 .mmap = xe_mmap,
355 .poll = drm_poll,
356 .read = drm_read,
357 .compat_ioctl = xe_drm_compat_ioctl,
358 .llseek = noop_llseek,
359 #ifdef CONFIG_PROC_FS
360 .show_fdinfo = drm_show_fdinfo,
361 #endif
362 .fop_flags = FOP_UNSIGNED_OFFSET,
363 };
364
365 static struct drm_driver driver = {
366 /* Don't use MTRRs here; the Xserver or userspace app should
367 * deal with them for Intel hardware.
368 */
369 .driver_features =
370 DRIVER_GEM |
371 DRIVER_RENDER | DRIVER_SYNCOBJ |
372 DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
373 .open = xe_file_open,
374 .postclose = xe_file_close,
375
376 .gem_prime_import = xe_gem_prime_import,
377
378 .dumb_create = xe_bo_dumb_create,
379 .dumb_map_offset = drm_gem_ttm_dumb_map_offset,
380 #ifdef CONFIG_PROC_FS
381 .show_fdinfo = xe_drm_client_fdinfo,
382 #endif
383 .ioctls = xe_ioctls,
384 .num_ioctls = ARRAY_SIZE(xe_ioctls),
385 .fops = &xe_driver_fops,
386 .name = DRIVER_NAME,
387 .desc = DRIVER_DESC,
388 .major = DRIVER_MAJOR,
389 .minor = DRIVER_MINOR,
390 .patchlevel = DRIVER_PATCHLEVEL,
391 };
392
xe_device_destroy(struct drm_device * dev,void * dummy)393 static void xe_device_destroy(struct drm_device *dev, void *dummy)
394 {
395 struct xe_device *xe = to_xe_device(dev);
396
397 xe_bo_dev_fini(&xe->bo_device);
398
399 if (xe->preempt_fence_wq)
400 destroy_workqueue(xe->preempt_fence_wq);
401
402 if (xe->ordered_wq)
403 destroy_workqueue(xe->ordered_wq);
404
405 if (xe->unordered_wq)
406 destroy_workqueue(xe->unordered_wq);
407
408 if (xe->destroy_wq)
409 destroy_workqueue(xe->destroy_wq);
410
411 ttm_device_fini(&xe->ttm);
412 }
413
xe_device_create(struct pci_dev * pdev,const struct pci_device_id * ent)414 struct xe_device *xe_device_create(struct pci_dev *pdev,
415 const struct pci_device_id *ent)
416 {
417 struct xe_device *xe;
418 int err;
419
420 xe_display_driver_set_hooks(&driver);
421
422 err = aperture_remove_conflicting_pci_devices(pdev, driver.name);
423 if (err)
424 return ERR_PTR(err);
425
426 xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);
427 if (IS_ERR(xe))
428 return xe;
429
430 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
431 xe->drm.anon_inode->i_mapping,
432 xe->drm.vma_offset_manager, false, false);
433 if (WARN_ON(err))
434 goto err;
435
436 xe_bo_dev_init(&xe->bo_device);
437 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
438 if (err)
439 goto err;
440
441 err = xe_shrinker_create(xe);
442 if (err)
443 goto err;
444
445 xe->info.devid = pdev->device;
446 xe->info.revid = pdev->revision;
447 xe->info.force_execlist = xe_modparam.force_execlist;
448 xe->atomic_svm_timeslice_ms = 5;
449
450 err = xe_irq_init(xe);
451 if (err)
452 goto err;
453
454 init_waitqueue_head(&xe->ufence_wq);
455
456 init_rwsem(&xe->usm.lock);
457
458 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
459
460 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
461 /* Trigger a large asid and an early asid wrap. */
462 u32 asid;
463
464 BUILD_BUG_ON(XE_MAX_ASID < 2);
465 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
466 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
467 &xe->usm.next_asid, GFP_KERNEL);
468 drm_WARN_ON(&xe->drm, err);
469 if (err >= 0)
470 xa_erase(&xe->usm.asid_to_vm, asid);
471 }
472
473 err = xe_bo_pinned_init(xe);
474 if (err)
475 goto err;
476
477 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
478 WQ_MEM_RECLAIM);
479 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
480 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);
481 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0);
482 if (!xe->ordered_wq || !xe->unordered_wq ||
483 !xe->preempt_fence_wq || !xe->destroy_wq) {
484 /*
485 * Cleanup done in xe_device_destroy via
486 * drmm_add_action_or_reset register above
487 */
488 drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
489 err = -ENOMEM;
490 goto err;
491 }
492
493 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
494 if (err)
495 goto err;
496
497 return xe;
498
499 err:
500 return ERR_PTR(err);
501 }
502 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
503
xe_driver_flr_disabled(struct xe_device * xe)504 static bool xe_driver_flr_disabled(struct xe_device *xe)
505 {
506 if (IS_SRIOV_VF(xe))
507 return true;
508
509 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) {
510 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n");
511 return true;
512 }
513
514 return false;
515 }
516
517 /*
518 * The driver-initiated FLR is the highest level of reset that we can trigger
519 * from within the driver. It is different from the PCI FLR in that it doesn't
520 * fully reset the SGUnit and doesn't modify the PCI config space and therefore
521 * it doesn't require a re-enumeration of the PCI BARs. However, the
522 * driver-initiated FLR does still cause a reset of both GT and display and a
523 * memory wipe of local and stolen memory, so recovery would require a full HW
524 * re-init and saving/restoring (or re-populating) the wiped memory. Since we
525 * perform the FLR as the very last action before releasing access to the HW
526 * during the driver release flow, we don't attempt recovery at all, because
527 * if/when a new instance of i915 is bound to the device it will do a full
528 * re-init anyway.
529 */
__xe_driver_flr(struct xe_device * xe)530 static void __xe_driver_flr(struct xe_device *xe)
531 {
532 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */
533 struct xe_mmio *mmio = xe_root_tile_mmio(xe);
534 int ret;
535
536 drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
537
538 /*
539 * Make sure any pending FLR requests have cleared by waiting for the
540 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
541 * to make sure it's not still set from a prior attempt (it's a write to
542 * clear bit).
543 * Note that we should never be in a situation where a previous attempt
544 * is still pending (unless the HW is totally dead), but better to be
545 * safe in case something unexpected happens
546 */
547 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
548 if (ret) {
549 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
550 return;
551 }
552 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
553
554 /* Trigger the actual Driver-FLR */
555 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR);
556
557 /* Wait for hardware teardown to complete */
558 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
559 if (ret) {
560 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
561 return;
562 }
563
564 /* Wait for hardware/firmware re-init to complete */
565 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
566 flr_timeout, NULL, false);
567 if (ret) {
568 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
569 return;
570 }
571
572 /* Clear sticky completion status */
573 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
574 }
575
xe_driver_flr(struct xe_device * xe)576 static void xe_driver_flr(struct xe_device *xe)
577 {
578 if (xe_driver_flr_disabled(xe))
579 return;
580
581 __xe_driver_flr(xe);
582 }
583
xe_driver_flr_fini(void * arg)584 static void xe_driver_flr_fini(void *arg)
585 {
586 struct xe_device *xe = arg;
587
588 if (xe->needs_flr_on_fini)
589 xe_driver_flr(xe);
590 }
591
xe_device_sanitize(void * arg)592 static void xe_device_sanitize(void *arg)
593 {
594 struct xe_device *xe = arg;
595 struct xe_gt *gt;
596 u8 id;
597
598 for_each_gt(gt, xe, id)
599 xe_gt_sanitize(gt);
600 }
601
xe_set_dma_info(struct xe_device * xe)602 static int xe_set_dma_info(struct xe_device *xe)
603 {
604 unsigned int mask_size = xe->info.dma_mask_size;
605 int err;
606
607 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
608
609 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
610 if (err)
611 goto mask_err;
612
613 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
614 if (err)
615 goto mask_err;
616
617 return 0;
618
619 mask_err:
620 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
621 return err;
622 }
623
verify_lmem_ready(struct xe_device * xe)624 static bool verify_lmem_ready(struct xe_device *xe)
625 {
626 u32 val = xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT;
627
628 return !!val;
629 }
630
wait_for_lmem_ready(struct xe_device * xe)631 static int wait_for_lmem_ready(struct xe_device *xe)
632 {
633 unsigned long timeout, start;
634
635 if (!IS_DGFX(xe))
636 return 0;
637
638 if (IS_SRIOV_VF(xe))
639 return 0;
640
641 if (verify_lmem_ready(xe))
642 return 0;
643
644 drm_dbg(&xe->drm, "Waiting for lmem initialization\n");
645
646 start = jiffies;
647 timeout = start + secs_to_jiffies(60); /* 60 sec! */
648
649 do {
650 if (signal_pending(current))
651 return -EINTR;
652
653 /*
654 * The boot firmware initializes local memory and
655 * assesses its health. If memory training fails,
656 * the punit will have been instructed to keep the GT powered
657 * down.we won't be able to communicate with it
658 *
659 * If the status check is done before punit updates the register,
660 * it can lead to the system being unusable.
661 * use a timeout and defer the probe to prevent this.
662 */
663 if (time_after(jiffies, timeout)) {
664 drm_dbg(&xe->drm, "lmem not initialized by firmware\n");
665 return -EPROBE_DEFER;
666 }
667
668 msleep(20);
669
670 } while (!verify_lmem_ready(xe));
671
672 drm_dbg(&xe->drm, "lmem ready after %ums",
673 jiffies_to_msecs(jiffies - start));
674
675 return 0;
676 }
677 ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */
678
sriov_update_device_info(struct xe_device * xe)679 static void sriov_update_device_info(struct xe_device *xe)
680 {
681 /* disable features that are not available/applicable to VFs */
682 if (IS_SRIOV_VF(xe)) {
683 xe->info.probe_display = 0;
684 xe->info.has_heci_cscfi = 0;
685 xe->info.has_heci_gscfi = 0;
686 xe->info.skip_guc_pc = 1;
687 xe->info.skip_pcode = 1;
688 }
689 }
690
691 /**
692 * xe_device_probe_early: Device early probe
693 * @xe: xe device instance
694 *
695 * Initialize MMIO resources that don't require any
696 * knowledge about tile count. Also initialize pcode and
697 * check vram initialization on root tile.
698 *
699 * Return: 0 on success, error code on failure
700 */
xe_device_probe_early(struct xe_device * xe)701 int xe_device_probe_early(struct xe_device *xe)
702 {
703 int err;
704
705 xe_wa_device_init(xe);
706 xe_wa_process_device_oob(xe);
707
708 err = xe_mmio_probe_early(xe);
709 if (err)
710 return err;
711
712 xe_sriov_probe_early(xe);
713
714 sriov_update_device_info(xe);
715
716 err = xe_pcode_probe_early(xe);
717 if (err || xe_survivability_mode_is_requested(xe)) {
718 int save_err = err;
719
720 /*
721 * Try to leave device in survivability mode if device is
722 * possible, but still return the previous error for error
723 * propagation
724 */
725 err = xe_survivability_mode_enable(xe);
726 if (err)
727 return err;
728
729 return save_err;
730 }
731
732 err = wait_for_lmem_ready(xe);
733 if (err)
734 return err;
735
736 xe->wedged.mode = xe_modparam.wedged_mode;
737
738 return 0;
739 }
740 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */
741
probe_has_flat_ccs(struct xe_device * xe)742 static int probe_has_flat_ccs(struct xe_device *xe)
743 {
744 struct xe_gt *gt;
745 unsigned int fw_ref;
746 u32 reg;
747
748 /* Always enabled/disabled, no runtime check to do */
749 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
750 return 0;
751
752 gt = xe_root_mmio_gt(xe);
753
754 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
755 if (!fw_ref)
756 return -ETIMEDOUT;
757
758 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
759 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
760
761 if (!xe->info.has_flat_ccs)
762 drm_dbg(&xe->drm,
763 "Flat CCS has been disabled in bios, May lead to performance impact");
764
765 xe_force_wake_put(gt_to_fw(gt), fw_ref);
766
767 return 0;
768 }
769
xe_device_probe(struct xe_device * xe)770 int xe_device_probe(struct xe_device *xe)
771 {
772 struct xe_tile *tile;
773 struct xe_gt *gt;
774 int err;
775 u8 id;
776
777 xe_pat_init_early(xe);
778
779 err = xe_sriov_init(xe);
780 if (err)
781 return err;
782
783 xe->info.mem_region_mask = 1;
784
785 err = xe_set_dma_info(xe);
786 if (err)
787 return err;
788
789 err = xe_mmio_probe_tiles(xe);
790 if (err)
791 return err;
792
793 for_each_gt(gt, xe, id) {
794 err = xe_gt_init_early(gt);
795 if (err)
796 return err;
797 }
798
799 for_each_tile(tile, xe, id) {
800 err = xe_ggtt_init_early(tile->mem.ggtt);
801 if (err)
802 return err;
803 }
804
805 err = xe_devcoredump_init(xe);
806 if (err)
807 return err;
808
809 /*
810 * From here on, if a step fails, make sure a Driver-FLR is triggereed
811 */
812 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
813 if (err)
814 return err;
815
816 err = probe_has_flat_ccs(xe);
817 if (err)
818 return err;
819
820 err = xe_vram_probe(xe);
821 if (err)
822 return err;
823
824 for_each_tile(tile, xe, id) {
825 err = xe_tile_init_noalloc(tile);
826 if (err)
827 return err;
828 }
829
830 /*
831 * Allow allocations only now to ensure xe_display_init_early()
832 * is the first to allocate, always.
833 */
834 err = xe_ttm_sys_mgr_init(xe);
835 if (err)
836 return err;
837
838 /* Allocate and map stolen after potential VRAM resize */
839 err = xe_ttm_stolen_mgr_init(xe);
840 if (err)
841 return err;
842
843 /*
844 * Now that GT is initialized (TTM in particular),
845 * we can try to init display, and inherit the initial fb.
846 * This is the reason the first allocation needs to be done
847 * inside display.
848 */
849 err = xe_display_init_early(xe);
850 if (err)
851 return err;
852
853 for_each_tile(tile, xe, id) {
854 err = xe_tile_init(tile);
855 if (err)
856 return err;
857 }
858
859 err = xe_irq_install(xe);
860 if (err)
861 return err;
862
863 for_each_gt(gt, xe, id) {
864 err = xe_gt_init(gt);
865 if (err)
866 return err;
867 }
868
869 if (xe->tiles->media_gt &&
870 XE_WA(xe->tiles->media_gt, 15015404425_disable))
871 XE_DEVICE_WA_DISABLE(xe, 15015404425);
872
873 xe_nvm_init(xe);
874
875 err = xe_heci_gsc_init(xe);
876 if (err)
877 return err;
878
879 err = xe_oa_init(xe);
880 if (err)
881 return err;
882
883 err = xe_display_init(xe);
884 if (err)
885 return err;
886
887 err = xe_pxp_init(xe);
888 if (err)
889 return err;
890
891 err = drm_dev_register(&xe->drm, 0);
892 if (err)
893 return err;
894
895 xe_display_register(xe);
896
897 err = xe_oa_register(xe);
898 if (err)
899 goto err_unregister_display;
900
901 err = xe_pmu_register(&xe->pmu);
902 if (err)
903 goto err_unregister_display;
904
905 err = xe_device_sysfs_init(xe);
906 if (err)
907 goto err_unregister_display;
908
909 xe_debugfs_register(xe);
910
911 err = xe_hwmon_register(xe);
912 if (err)
913 goto err_unregister_display;
914
915 err = xe_i2c_probe(xe);
916 if (err)
917 goto err_unregister_display;
918
919 for_each_gt(gt, xe, id)
920 xe_gt_sanitize_freq(gt);
921
922 xe_vsec_init(xe);
923
924 return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
925
926 err_unregister_display:
927 xe_display_unregister(xe);
928
929 return err;
930 }
931
xe_device_remove(struct xe_device * xe)932 void xe_device_remove(struct xe_device *xe)
933 {
934 xe_display_unregister(xe);
935
936 xe_nvm_fini(xe);
937
938 drm_dev_unplug(&xe->drm);
939
940 xe_bo_pci_dev_remove_all(xe);
941 }
942
xe_device_shutdown(struct xe_device * xe)943 void xe_device_shutdown(struct xe_device *xe)
944 {
945 struct xe_gt *gt;
946 u8 id;
947
948 drm_dbg(&xe->drm, "Shutting down device\n");
949
950 if (xe_driver_flr_disabled(xe)) {
951 xe_display_pm_shutdown(xe);
952
953 xe_irq_suspend(xe);
954
955 for_each_gt(gt, xe, id)
956 xe_gt_shutdown(gt);
957
958 xe_display_pm_shutdown_late(xe);
959 } else {
960 /* BOOM! */
961 __xe_driver_flr(xe);
962 }
963 }
964
965 /**
966 * xe_device_wmb() - Device specific write memory barrier
967 * @xe: the &xe_device
968 *
969 * While wmb() is sufficient for a barrier if we use system memory, on discrete
970 * platforms with device memory we additionally need to issue a register write.
971 * Since it doesn't matter which register we write to, use the read-only VF_CAP
972 * register that is also marked as accessible by the VFs.
973 */
xe_device_wmb(struct xe_device * xe)974 void xe_device_wmb(struct xe_device *xe)
975 {
976 wmb();
977 if (IS_DGFX(xe))
978 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);
979 }
980
981 /*
982 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt.
983 */
tdf_request_sync(struct xe_device * xe)984 static void tdf_request_sync(struct xe_device *xe)
985 {
986 unsigned int fw_ref;
987 struct xe_gt *gt;
988 u8 id;
989
990 for_each_gt(gt, xe, id) {
991 if (xe_gt_is_media_type(gt))
992 continue;
993
994 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
995 if (!fw_ref)
996 return;
997
998 xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
999
1000 /*
1001 * FIXME: We can likely do better here with our choice of
1002 * timeout. Currently we just assume the worst case, i.e. 150us,
1003 * which is believed to be sufficient to cover the worst case
1004 * scenario on current platforms if all cache entries are
1005 * transient and need to be flushed..
1006 */
1007 if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1008 150, NULL, false))
1009 xe_gt_err_once(gt, "TD flush timeout\n");
1010
1011 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1012 }
1013 }
1014
xe_device_l2_flush(struct xe_device * xe)1015 void xe_device_l2_flush(struct xe_device *xe)
1016 {
1017 struct xe_gt *gt;
1018 unsigned int fw_ref;
1019
1020 gt = xe_root_mmio_gt(xe);
1021
1022 if (!XE_WA(gt, 16023588340))
1023 return;
1024
1025 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1026 if (!fw_ref)
1027 return;
1028
1029 spin_lock(>->global_invl_lock);
1030
1031 xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1);
1032 if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 500, NULL, true))
1033 xe_gt_err_once(gt, "Global invalidation timeout\n");
1034
1035 spin_unlock(>->global_invl_lock);
1036
1037 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1038 }
1039
1040 /**
1041 * xe_device_td_flush() - Flush transient L3 cache entries
1042 * @xe: The device
1043 *
1044 * Display engine has direct access to memory and is never coherent with L3/L4
1045 * caches (or CPU caches), however KMD is responsible for specifically flushing
1046 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout
1047 * can happen from such a surface without seeing corruption.
1048 *
1049 * Display surfaces can be tagged as transient by mapping it using one of the
1050 * various L3:XD PAT index modes on Xe2.
1051 *
1052 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
1053 * at the end of each submission via PIPE_CONTROL for compute/render, since SA
1054 * Media is not coherent with L3 and we want to support render-vs-media
1055 * usescases. For other engines like copy/blt the HW internally forces uncached
1056 * behaviour, hence why we can skip the TDF on such platforms.
1057 */
xe_device_td_flush(struct xe_device * xe)1058 void xe_device_td_flush(struct xe_device *xe)
1059 {
1060 struct xe_gt *root_gt;
1061
1062 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
1063 return;
1064
1065 root_gt = xe_root_mmio_gt(xe);
1066 if (XE_WA(root_gt, 16023588340)) {
1067 /* A transient flush is not sufficient: flush the L2 */
1068 xe_device_l2_flush(xe);
1069 } else {
1070 xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc);
1071 tdf_request_sync(xe);
1072 xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc);
1073 }
1074 }
1075
xe_device_ccs_bytes(struct xe_device * xe,u64 size)1076 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
1077 {
1078 return xe_device_has_flat_ccs(xe) ?
1079 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
1080 }
1081
1082 /**
1083 * xe_device_assert_mem_access - Inspect the current runtime_pm state.
1084 * @xe: xe device instance
1085 *
1086 * To be used before any kind of memory access. It will splat a debug warning
1087 * if the device is currently sleeping. But it doesn't guarantee in any way
1088 * that the device is going to remain awake. Xe PM runtime get and put
1089 * functions might be added to the outer bound of the memory access, while
1090 * this check is intended for inner usage to splat some warning if the worst
1091 * case has just happened.
1092 */
xe_device_assert_mem_access(struct xe_device * xe)1093 void xe_device_assert_mem_access(struct xe_device *xe)
1094 {
1095 xe_assert(xe, !xe_pm_runtime_suspended(xe));
1096 }
1097
xe_device_snapshot_print(struct xe_device * xe,struct drm_printer * p)1098 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
1099 {
1100 struct xe_gt *gt;
1101 u8 id;
1102
1103 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
1104 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
1105
1106 for_each_gt(gt, xe, id) {
1107 drm_printf(p, "GT id: %u\n", id);
1108 drm_printf(p, "\tTile: %u\n", gt->tile->id);
1109 drm_printf(p, "\tType: %s\n",
1110 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
1111 drm_printf(p, "\tIP ver: %u.%u.%u\n",
1112 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
1113 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
1114 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
1115 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
1116 }
1117 }
1118
xe_device_canonicalize_addr(struct xe_device * xe,u64 address)1119 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
1120 {
1121 return sign_extend64(address, xe->info.va_bits - 1);
1122 }
1123
xe_device_uncanonicalize_addr(struct xe_device * xe,u64 address)1124 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
1125 {
1126 return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
1127 }
1128
xe_device_wedged_fini(struct drm_device * drm,void * arg)1129 static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
1130 {
1131 struct xe_device *xe = arg;
1132
1133 xe_pm_runtime_put(xe);
1134 }
1135
1136 /**
1137 * xe_device_declare_wedged - Declare device wedged
1138 * @xe: xe device instance
1139 *
1140 * This is a final state that can only be cleared with a module
1141 * re-probe (unbind + bind).
1142 * In this state every IOCTL will be blocked so the GT cannot be used.
1143 * In general it will be called upon any critical error such as gt reset
1144 * failure or guc loading failure. Userspace will be notified of this state
1145 * through device wedged uevent.
1146 * If xe.wedged module parameter is set to 2, this function will be called
1147 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
1148 * snapshot capture. In this mode, GT reset won't be attempted so the state of
1149 * the issue is preserved for further debugging.
1150 */
xe_device_declare_wedged(struct xe_device * xe)1151 void xe_device_declare_wedged(struct xe_device *xe)
1152 {
1153 struct xe_gt *gt;
1154 u8 id;
1155
1156 if (xe->wedged.mode == 0) {
1157 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
1158 return;
1159 }
1160
1161 xe_pm_runtime_get_noresume(xe);
1162
1163 if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) {
1164 drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n");
1165 return;
1166 }
1167
1168 if (!atomic_xchg(&xe->wedged.flag, 1)) {
1169 xe->needs_flr_on_fini = true;
1170 drm_err(&xe->drm,
1171 "CRITICAL: Xe has declared device %s as wedged.\n"
1172 "IOCTLs and executions are blocked. Only a rebind may clear the failure\n"
1173 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
1174 dev_name(xe->drm.dev));
1175
1176 /* Notify userspace of wedged device */
1177 drm_dev_wedged_event(&xe->drm,
1178 DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET,
1179 NULL);
1180 }
1181
1182 for_each_gt(gt, xe, id)
1183 xe_gt_declare_wedged(gt);
1184 }
1185