1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_device.h" 7 8 #include <linux/aperture.h> 9 #include <linux/delay.h> 10 #include <linux/fault-inject.h> 11 #include <linux/units.h> 12 13 #include <drm/drm_client.h> 14 #include <drm/drm_gem_ttm_helper.h> 15 #include <drm/drm_ioctl.h> 16 #include <drm/drm_managed.h> 17 #include <drm/drm_pagemap_util.h> 18 #include <drm/drm_print.h> 19 #include <kunit/static_stub.h> 20 #include <uapi/drm/xe_drm.h> 21 22 #include "display/xe_display.h" 23 #include "instructions/xe_gpu_commands.h" 24 #include "regs/xe_gt_regs.h" 25 #include "regs/xe_regs.h" 26 #include "xe_bo.h" 27 #include "xe_bo_evict.h" 28 #include "xe_configfs.h" 29 #include "xe_debugfs.h" 30 #include "xe_defaults.h" 31 #include "xe_devcoredump.h" 32 #include "xe_device_sysfs.h" 33 #include "xe_dma_buf.h" 34 #include "xe_drm_client.h" 35 #include "xe_drv.h" 36 #include "xe_exec.h" 37 #include "xe_exec_queue.h" 38 #include "xe_force_wake.h" 39 #include "xe_ggtt.h" 40 #include "xe_gt.h" 41 #include "xe_gt_mcr.h" 42 #include "xe_gt_printk.h" 43 #include "xe_gt_sriov_vf.h" 44 #include "xe_guc.h" 45 #include "xe_guc_pc.h" 46 #include "xe_hw_engine_group.h" 47 #include "xe_hwmon.h" 48 #include "xe_i2c.h" 49 #include "xe_irq.h" 50 #include "xe_late_bind_fw.h" 51 #include "xe_mmio.h" 52 #include "xe_module.h" 53 #include "xe_nvm.h" 54 #include "xe_oa.h" 55 #include "xe_observation.h" 56 #include "xe_pagefault.h" 57 #include "xe_pat.h" 58 #include "xe_pcode.h" 59 #include "xe_pm.h" 60 #include "xe_pmu.h" 61 #include "xe_psmi.h" 62 #include "xe_pxp.h" 63 #include "xe_query.h" 64 #include "xe_shrinker.h" 65 #include "xe_soc_remapper.h" 66 #include "xe_survivability_mode.h" 67 #include "xe_sriov.h" 68 #include "xe_svm.h" 69 #include "xe_sysctrl.h" 70 #include "xe_tile.h" 71 #include "xe_ttm_stolen_mgr.h" 72 #include "xe_ttm_sys_mgr.h" 73 #include "xe_vm.h" 74 #include "xe_vm_madvise.h" 75 #include "xe_vram.h" 76 #include "xe_vram_types.h" 77 #include "xe_vsec.h" 78 #include "xe_wait_user_fence.h" 79 #include "xe_wa.h" 80 81 #include <generated/xe_device_wa_oob.h> 82 #include <generated/xe_wa_oob.h> 83 84 static int xe_file_open(struct drm_device *dev, struct drm_file *file) 85 { 86 struct xe_device *xe = to_xe_device(dev); 87 struct xe_drm_client *client; 88 struct xe_file *xef; 89 int ret = -ENOMEM; 90 struct task_struct *task = NULL; 91 92 xef = kzalloc_obj(*xef); 93 if (!xef) 94 return ret; 95 96 client = xe_drm_client_alloc(); 97 if (!client) { 98 kfree(xef); 99 return ret; 100 } 101 102 xef->drm = file; 103 xef->client = client; 104 xef->xe = xe; 105 106 mutex_init(&xef->vm.lock); 107 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); 108 109 mutex_init(&xef->exec_queue.lock); 110 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); 111 112 file->driver_priv = xef; 113 kref_init(&xef->refcount); 114 115 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID); 116 if (task) { 117 xef->process_name = kstrdup(task->comm, GFP_KERNEL); 118 xef->pid = task->pid; 119 put_task_struct(task); 120 } 121 122 return 0; 123 } 124 125 static void xe_file_destroy(struct kref *ref) 126 { 127 struct xe_file *xef = container_of(ref, struct xe_file, refcount); 128 129 xa_destroy(&xef->exec_queue.xa); 130 mutex_destroy(&xef->exec_queue.lock); 131 xa_destroy(&xef->vm.xa); 132 mutex_destroy(&xef->vm.lock); 133 134 xe_drm_client_put(xef->client); 135 kfree(xef->process_name); 136 kfree(xef); 137 } 138 139 /** 140 * xe_file_get() - Take a reference to the xe file object 141 * @xef: Pointer to the xe file 142 * 143 * Anyone with a pointer to xef must take a reference to the xe file 144 * object using this call. 145 * 146 * Return: xe file pointer 147 */ 148 struct xe_file *xe_file_get(struct xe_file *xef) 149 { 150 kref_get(&xef->refcount); 151 return xef; 152 } 153 154 /** 155 * xe_file_put() - Drop a reference to the xe file object 156 * @xef: Pointer to the xe file 157 * 158 * Used to drop reference to the xef object 159 */ 160 void xe_file_put(struct xe_file *xef) 161 { 162 kref_put(&xef->refcount, xe_file_destroy); 163 } 164 165 static void xe_file_close(struct drm_device *dev, struct drm_file *file) 166 { 167 struct xe_device *xe = to_xe_device(dev); 168 struct xe_file *xef = file->driver_priv; 169 struct xe_vm *vm; 170 struct xe_exec_queue *q; 171 unsigned long idx; 172 173 guard(xe_pm_runtime)(xe); 174 175 /* 176 * No need for exec_queue.lock here as there is no contention for it 177 * when FD is closing as IOCTLs presumably can't be modifying the 178 * xarray. Taking exec_queue.lock here causes undue dependency on 179 * vm->lock taken during xe_exec_queue_kill(). 180 */ 181 xa_for_each(&xef->exec_queue.xa, idx, q) { 182 if (q->vm && q->hwe->hw_engine_group) 183 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 184 xe_exec_queue_kill(q); 185 xe_exec_queue_put(q); 186 } 187 xa_for_each(&xef->vm.xa, idx, vm) 188 xe_vm_close_and_put(vm); 189 190 xe_file_put(xef); 191 } 192 193 static const struct drm_ioctl_desc xe_ioctls[] = { 194 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 195 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), 196 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, 197 DRM_RENDER_ALLOW), 198 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), 199 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), 200 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), 201 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), 202 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, 203 DRM_RENDER_ALLOW), 204 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, 205 DRM_RENDER_ALLOW), 206 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, 207 DRM_RENDER_ALLOW), 208 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, 209 DRM_RENDER_ALLOW), 210 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 211 DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), 212 DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, 213 DRM_RENDER_ALLOW), 214 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, 215 DRM_RENDER_ALLOW), 216 DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl, 217 DRM_RENDER_ALLOW), 218 }; 219 220 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 221 { 222 struct drm_file *file_priv = file->private_data; 223 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 224 long ret; 225 226 if (xe_device_wedged(xe)) 227 return -ECANCELED; 228 229 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); 230 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); 231 if (ret >= 0) 232 ret = drm_ioctl(file, cmd, arg); 233 234 return ret; 235 } 236 237 #ifdef CONFIG_COMPAT 238 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 239 { 240 struct drm_file *file_priv = file->private_data; 241 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 242 long ret; 243 244 if (xe_device_wedged(xe)) 245 return -ECANCELED; 246 247 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); 248 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); 249 if (ret >= 0) 250 ret = drm_compat_ioctl(file, cmd, arg); 251 252 return ret; 253 } 254 #else 255 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */ 256 #define xe_drm_compat_ioctl NULL 257 #endif 258 259 static void barrier_open(struct vm_area_struct *vma) 260 { 261 drm_dev_get(vma->vm_private_data); 262 } 263 264 static void barrier_close(struct vm_area_struct *vma) 265 { 266 drm_dev_put(vma->vm_private_data); 267 } 268 269 static void barrier_release_dummy_page(struct drm_device *dev, void *res) 270 { 271 struct page *dummy_page = (struct page *)res; 272 273 __free_page(dummy_page); 274 } 275 276 static vm_fault_t barrier_fault(struct vm_fault *vmf) 277 { 278 struct drm_device *dev = vmf->vma->vm_private_data; 279 struct vm_area_struct *vma = vmf->vma; 280 vm_fault_t ret = VM_FAULT_NOPAGE; 281 pgprot_t prot; 282 int idx; 283 284 prot = vm_get_page_prot(vma->vm_flags); 285 286 if (drm_dev_enter(dev, &idx)) { 287 unsigned long pfn; 288 289 #define LAST_DB_PAGE_OFFSET 0x7ff001 290 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) + 291 LAST_DB_PAGE_OFFSET); 292 ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn, 293 pgprot_noncached(prot)); 294 drm_dev_exit(idx); 295 } else { 296 struct page *page; 297 298 /* Allocate new dummy page to map all the VA range in this VMA to it*/ 299 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 300 if (!page) 301 return VM_FAULT_OOM; 302 303 /* Set the page to be freed using drmm release action */ 304 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page)) 305 return VM_FAULT_OOM; 306 307 ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page), 308 prot); 309 } 310 311 return ret; 312 } 313 314 static const struct vm_operations_struct vm_ops_barrier = { 315 .open = barrier_open, 316 .close = barrier_close, 317 .fault = barrier_fault, 318 }; 319 320 static int xe_pci_barrier_mmap(struct file *filp, 321 struct vm_area_struct *vma) 322 { 323 struct drm_file *priv = filp->private_data; 324 struct drm_device *dev = priv->minor->dev; 325 struct xe_device *xe = to_xe_device(dev); 326 327 if (!IS_DGFX(xe)) 328 return -EINVAL; 329 330 if (vma->vm_end - vma->vm_start > SZ_4K) 331 return -EINVAL; 332 333 if (is_cow_mapping(vma->vm_flags)) 334 return -EINVAL; 335 336 if (vma->vm_flags & (VM_READ | VM_EXEC)) 337 return -EINVAL; 338 339 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC); 340 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); 341 vma->vm_ops = &vm_ops_barrier; 342 vma->vm_private_data = dev; 343 drm_dev_get(vma->vm_private_data); 344 345 return 0; 346 } 347 348 static int xe_mmap(struct file *filp, struct vm_area_struct *vma) 349 { 350 struct drm_file *priv = filp->private_data; 351 struct drm_device *dev = priv->minor->dev; 352 353 if (drm_dev_is_unplugged(dev)) 354 return -ENODEV; 355 356 switch (vma->vm_pgoff) { 357 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT: 358 return xe_pci_barrier_mmap(filp, vma); 359 } 360 361 return drm_gem_mmap(filp, vma); 362 } 363 364 static const struct file_operations xe_driver_fops = { 365 .owner = THIS_MODULE, 366 .open = drm_open, 367 .release = drm_release_noglobal, 368 .unlocked_ioctl = xe_drm_ioctl, 369 .mmap = xe_mmap, 370 .poll = drm_poll, 371 .read = drm_read, 372 .compat_ioctl = xe_drm_compat_ioctl, 373 .llseek = noop_llseek, 374 #ifdef CONFIG_PROC_FS 375 .show_fdinfo = drm_show_fdinfo, 376 #endif 377 .fop_flags = FOP_UNSIGNED_OFFSET, 378 }; 379 380 /** 381 * xe_is_xe_file() - Is the file an xe device file? 382 * @file: The file. 383 * 384 * Checks whether the file is opened against 385 * an xe device. 386 * 387 * Return: %true if an xe file, %false if not. 388 */ 389 bool xe_is_xe_file(const struct file *file) 390 { 391 return file->f_op == &xe_driver_fops; 392 } 393 394 static const struct drm_driver regular_driver = { 395 .driver_features = 396 XE_DISPLAY_DRIVER_FEATURES | 397 DRIVER_GEM | 398 DRIVER_RENDER | DRIVER_SYNCOBJ | 399 DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, 400 .open = xe_file_open, 401 .postclose = xe_file_close, 402 403 .gem_prime_import = xe_gem_prime_import, 404 405 .dumb_create = xe_bo_dumb_create, 406 .dumb_map_offset = drm_gem_ttm_dumb_map_offset, 407 #ifdef CONFIG_PROC_FS 408 .show_fdinfo = xe_drm_client_fdinfo, 409 #endif 410 .ioctls = xe_ioctls, 411 .num_ioctls = ARRAY_SIZE(xe_ioctls), 412 .fops = &xe_driver_fops, 413 .name = DRIVER_NAME, 414 .desc = DRIVER_DESC, 415 .major = DRIVER_MAJOR, 416 .minor = DRIVER_MINOR, 417 .patchlevel = DRIVER_PATCHLEVEL, 418 XE_DISPLAY_DRIVER_OPS, 419 }; 420 421 #ifdef CONFIG_PCI_IOV 422 static const struct drm_ioctl_desc xe_ioctls_admin_only[] = { 423 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 424 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 425 }; 426 427 static const struct drm_driver admin_only_driver = { 428 .driver_features = 429 DRIVER_GEM | DRIVER_RENDER | DRIVER_GEM_GPUVA, 430 .open = xe_file_open, 431 .postclose = xe_file_close, 432 .ioctls = xe_ioctls_admin_only, 433 .num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only), 434 .fops = &xe_driver_fops, 435 .name = DRIVER_NAME, 436 .desc = DRIVER_DESC, 437 .major = DRIVER_MAJOR, 438 .minor = DRIVER_MINOR, 439 .patchlevel = DRIVER_PATCHLEVEL, 440 }; 441 442 /** 443 * xe_device_is_admin_only() - Check whether device is admin only or not. 444 * @xe: the &xe_device to check 445 * 446 * Return: true if the device is admin only, false otherwise. 447 */ 448 bool xe_device_is_admin_only(const struct xe_device *xe) 449 { 450 KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe); 451 return xe->drm.driver == &admin_only_driver; 452 } 453 #endif 454 455 static void xe_device_destroy(struct drm_device *dev, void *dummy) 456 { 457 struct xe_device *xe = to_xe_device(dev); 458 459 xe_bo_dev_fini(&xe->bo_device); 460 461 if (xe->preempt_fence_wq) 462 destroy_workqueue(xe->preempt_fence_wq); 463 464 if (xe->ordered_wq) 465 destroy_workqueue(xe->ordered_wq); 466 467 if (xe->unordered_wq) 468 destroy_workqueue(xe->unordered_wq); 469 470 if (xe->destroy_wq) 471 destroy_workqueue(xe->destroy_wq); 472 473 ttm_device_fini(&xe->ttm); 474 } 475 476 /** 477 * xe_device_create() - Create a new &xe_device instance 478 * @pdev: the parent &pci_dev 479 * 480 * Allocate and initialize a device managed Xe device structure. 481 * 482 * Return: pointer to new &xe_device on success, or ERR_PTR on failure. 483 */ 484 struct xe_device *xe_device_create(struct pci_dev *pdev) 485 { 486 const struct drm_driver *driver = ®ular_driver; 487 struct xe_device *xe; 488 int err; 489 490 #ifdef CONFIG_PCI_IOV 491 /* 492 * Since XE device is not initialized yet, read from configfs 493 * directly to decide whether we are in admin-only PF mode or not. 494 */ 495 if (xe_configfs_admin_only_pf(pdev)) 496 driver = &admin_only_driver; 497 #endif 498 499 err = aperture_remove_conflicting_pci_devices(pdev, driver->name); 500 if (err) 501 return ERR_PTR(err); 502 503 xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm); 504 if (IS_ERR(xe)) 505 return xe; 506 507 err = xe_device_init_early(xe); 508 if (err) 509 return ERR_PTR(err); 510 511 return xe; 512 } 513 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ 514 515 /** 516 * xe_device_init_early() - Initialize a new &xe_device instance 517 * @xe: the &xe_device to initialize 518 * 519 * Return: 0 on success or a negative error code on failure. 520 */ 521 int xe_device_init_early(struct xe_device *xe) 522 { 523 int err; 524 525 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, 526 xe->drm.anon_inode->i_mapping, 527 xe->drm.vma_offset_manager, 528 TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M))); 529 if (err) 530 return err; 531 532 xe_bo_dev_init(&xe->bo_device); 533 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); 534 if (err) 535 return err; 536 537 err = xe_shrinker_create(xe); 538 if (err) 539 return err; 540 541 xe->atomic_svm_timeslice_ms = 5; 542 xe->min_run_period_lr_ms = 5; 543 544 err = xe_irq_init(xe); 545 if (err) 546 return err; 547 548 xe_validation_device_init(&xe->val); 549 550 init_waitqueue_head(&xe->ufence_wq); 551 552 init_rwsem(&xe->usm.lock); 553 554 err = xe_pagemap_shrinker_create(xe); 555 if (err) 556 return err; 557 558 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); 559 560 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 561 /* Trigger a large asid and an early asid wrap. */ 562 u32 asid; 563 564 BUILD_BUG_ON(XE_MAX_ASID < 2); 565 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL, 566 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1), 567 &xe->usm.next_asid, GFP_KERNEL); 568 drm_WARN_ON(&xe->drm, err); 569 if (err >= 0) 570 xa_erase(&xe->usm.asid_to_vm, asid); 571 } 572 573 err = xe_bo_pinned_init(xe); 574 if (err) 575 return err; 576 577 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", 578 WQ_MEM_RECLAIM); 579 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); 580 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0); 581 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0); 582 if (!xe->ordered_wq || !xe->unordered_wq || 583 !xe->preempt_fence_wq || !xe->destroy_wq) { 584 /* 585 * Cleanup done in xe_device_destroy via 586 * drmm_add_action_or_reset register above 587 */ 588 drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); 589 return -ENOMEM; 590 } 591 592 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); 593 if (err) 594 return err; 595 596 err = xe_pm_init_early(xe); 597 if (err) 598 return err; 599 600 return 0; 601 } 602 603 static bool xe_driver_flr_disabled(struct xe_device *xe) 604 { 605 if (IS_SRIOV_VF(xe)) 606 return true; 607 608 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) { 609 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n"); 610 return true; 611 } 612 613 return false; 614 } 615 616 /* 617 * The driver-initiated FLR is the highest level of reset that we can trigger 618 * from within the driver. It is different from the PCI FLR in that it doesn't 619 * fully reset the SGUnit and doesn't modify the PCI config space and therefore 620 * it doesn't require a re-enumeration of the PCI BARs. However, the 621 * driver-initiated FLR does still cause a reset of both GT and display and a 622 * memory wipe of local and stolen memory, so recovery would require a full HW 623 * re-init and saving/restoring (or re-populating) the wiped memory. Since we 624 * perform the FLR as the very last action before releasing access to the HW 625 * during the driver release flow, we don't attempt recovery at all, because 626 * if/when a new instance of Xe is bound to the device it will do a full 627 * re-init anyway. 628 */ 629 static void __xe_driver_flr(struct xe_device *xe) 630 { 631 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */ 632 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 633 int ret; 634 635 drm_dbg(&xe->drm, "Triggering Driver-FLR\n"); 636 637 /* 638 * Make sure any pending FLR requests have cleared by waiting for the 639 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS 640 * to make sure it's not still set from a prior attempt (it's a write to 641 * clear bit). 642 * Note that we should never be in a situation where a previous attempt 643 * is still pending (unless the HW is totally dead), but better to be 644 * safe in case something unexpected happens 645 */ 646 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 647 if (ret) { 648 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret); 649 return; 650 } 651 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 652 653 /* Trigger the actual Driver-FLR */ 654 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR); 655 656 /* Wait for hardware teardown to complete */ 657 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 658 if (ret) { 659 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); 660 return; 661 } 662 663 /* Wait for hardware/firmware re-init to complete */ 664 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS, 665 flr_timeout, NULL, false); 666 if (ret) { 667 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); 668 return; 669 } 670 671 /* Clear sticky completion status */ 672 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 673 } 674 675 static void xe_driver_flr(struct xe_device *xe) 676 { 677 if (xe_driver_flr_disabled(xe)) 678 return; 679 680 __xe_driver_flr(xe); 681 } 682 683 static void xe_driver_flr_fini(void *arg) 684 { 685 struct xe_device *xe = arg; 686 687 if (xe->needs_flr_on_fini) 688 xe_driver_flr(xe); 689 } 690 691 static void xe_device_sanitize(void *arg) 692 { 693 struct xe_device *xe = arg; 694 struct xe_gt *gt; 695 u8 id; 696 697 for_each_gt(gt, xe, id) 698 xe_gt_sanitize(gt); 699 } 700 701 static int xe_set_dma_info(struct xe_device *xe) 702 { 703 unsigned int mask_size = xe->info.dma_mask_size; 704 int err; 705 706 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev)); 707 708 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 709 if (err) 710 goto mask_err; 711 712 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 713 if (err) 714 goto mask_err; 715 716 return 0; 717 718 mask_err: 719 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); 720 return err; 721 } 722 723 static void assert_lmem_ready(struct xe_device *xe) 724 { 725 if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) 726 return; 727 728 xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & 729 LMEM_INIT); 730 } 731 732 static void vf_update_device_info(struct xe_device *xe) 733 { 734 xe_assert(xe, IS_SRIOV_VF(xe)); 735 /* disable features that are not available/applicable to VFs */ 736 xe->info.probe_display = 0; 737 xe->info.has_heci_cscfi = 0; 738 xe->info.has_heci_gscfi = 0; 739 xe->info.has_late_bind = 0; 740 xe->info.skip_guc_pc = 1; 741 xe->info.skip_pcode = 1; 742 } 743 744 static int xe_device_vram_alloc(struct xe_device *xe) 745 { 746 struct xe_vram_region *vram; 747 748 if (!IS_DGFX(xe)) 749 return 0; 750 751 vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); 752 if (!vram) 753 return -ENOMEM; 754 755 xe->mem.vram = vram; 756 return 0; 757 } 758 759 /** 760 * xe_device_probe_early: Device early probe 761 * @xe: xe device instance 762 * 763 * Initialize MMIO resources that don't require any 764 * knowledge about tile count. Also initialize pcode and 765 * check vram initialization on root tile. 766 * 767 * Return: 0 on success, error code on failure 768 */ 769 int xe_device_probe_early(struct xe_device *xe) 770 { 771 int err; 772 773 xe_wa_device_init(xe); 774 xe_wa_process_device_oob(xe); 775 776 err = xe_mmio_probe_early(xe); 777 if (err) 778 return err; 779 780 xe_sriov_probe_early(xe); 781 782 if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) { 783 xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n"); 784 return -ENODEV; 785 } 786 787 if (IS_SRIOV_VF(xe)) 788 vf_update_device_info(xe); 789 790 /* 791 * Check for pcode uncore_init status to confirm if the SoC 792 * initialization is complete. Until done, any MMIO or lmem access from 793 * the driver will be blocked 794 */ 795 err = xe_pcode_probe_early(xe); 796 if (err || xe_survivability_mode_is_requested(xe)) { 797 int save_err = err; 798 799 /* 800 * Try to leave device in survivability mode if device is 801 * possible, but still return the previous error for error 802 * propagation 803 */ 804 err = xe_survivability_mode_boot_enable(xe); 805 if (err) 806 return err; 807 808 return save_err; 809 } 810 811 /* 812 * Make sure the lmem is initialized and ready to use. xe_pcode_ready() 813 * is flagged after full initialization is complete. Assert if lmem is 814 * not initialized. 815 */ 816 assert_lmem_ready(xe); 817 818 xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ? 819 XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode; 820 drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n", 821 xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode)); 822 823 err = xe_device_vram_alloc(xe); 824 if (err) 825 return err; 826 827 return 0; 828 } 829 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ 830 831 static int probe_has_flat_ccs(struct xe_device *xe) 832 { 833 struct xe_gt *gt; 834 u32 reg; 835 836 /* Always enabled/disabled, no runtime check to do */ 837 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe)) 838 return 0; 839 840 gt = xe_root_mmio_gt(xe); 841 if (!gt) 842 return 0; 843 844 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 845 if (!fw_ref.domains) 846 return -ETIMEDOUT; 847 848 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 849 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE); 850 851 if (!xe->info.has_flat_ccs) 852 drm_dbg(&xe->drm, 853 "Flat CCS has been disabled in bios, May lead to performance impact"); 854 855 return 0; 856 } 857 858 /* 859 * Detect if the driver is being run on pre-production hardware. We don't 860 * keep workarounds for pre-production hardware long term, so print an 861 * error and add taint if we're being loaded on a pre-production platform 862 * for which the pre-prod workarounds have already been removed. 863 * 864 * The general policy is that we'll remove any workarounds that only apply to 865 * pre-production hardware around the time force_probe restrictions are lifted 866 * for a platform of the next major IP generation (for example, Xe2 pre-prod 867 * workarounds should be removed around the time the first Xe3 platforms have 868 * force_probe lifted). 869 */ 870 static void detect_preproduction_hw(struct xe_device *xe) 871 { 872 struct xe_gt *gt; 873 int id; 874 875 /* 876 * SR-IOV VFs don't have access to the FUSE2 register, so we can't 877 * check pre-production status there. But the host OS will notice 878 * and report the pre-production status, which should be enough to 879 * help us catch mistaken use of pre-production hardware. 880 */ 881 if (IS_SRIOV_VF(xe)) 882 return; 883 884 /* 885 * The "SW_CAP" fuse contains a bit indicating whether the device is a 886 * production or pre-production device. This fuse is reflected through 887 * the GT "FUSE2" register, even though the contents of the fuse are 888 * not GT-specific. Every GT's reflection of this fuse should show the 889 * same value, so we'll just use the first available GT for lookup. 890 */ 891 for_each_gt(gt, xe, id) 892 break; 893 894 if (!gt) 895 return; 896 897 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 898 if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) { 899 xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n"); 900 return; 901 } 902 903 if (xe_mmio_read32(>->mmio, FUSE2) & PRODUCTION_HW) 904 return; 905 906 xe_info(xe, "Pre-production hardware detected.\n"); 907 if (!xe->info.has_pre_prod_wa) { 908 xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n"); 909 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); 910 } 911 } 912 913 static void xe_device_wedged_fini(struct drm_device *drm, void *arg) 914 { 915 struct xe_device *xe = arg; 916 917 if (atomic_read(&xe->wedged.flag)) 918 xe_pm_runtime_put(xe); 919 } 920 921 int xe_device_probe(struct xe_device *xe) 922 { 923 struct xe_tile *tile; 924 struct xe_gt *gt; 925 int err; 926 u8 id; 927 928 xe_pat_init_early(xe); 929 930 err = xe_sriov_init(xe); 931 if (err) 932 return err; 933 934 xe->info.mem_region_mask = 1; 935 936 err = xe_set_dma_info(xe); 937 if (err) 938 return err; 939 940 err = xe_mmio_probe_tiles(xe); 941 if (err) 942 return err; 943 944 for_each_gt(gt, xe, id) { 945 err = xe_gt_init_early(gt); 946 if (err) 947 return err; 948 } 949 950 for_each_tile(tile, xe, id) { 951 err = xe_ggtt_init_early(tile->mem.ggtt); 952 if (err) 953 return err; 954 } 955 956 /* 957 * From here on, if a step fails, make sure a Driver-FLR is triggereed 958 */ 959 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe); 960 if (err) 961 return err; 962 963 err = probe_has_flat_ccs(xe); 964 if (err) 965 return err; 966 967 err = xe_vram_probe(xe); 968 if (err) 969 return err; 970 971 for_each_tile(tile, xe, id) { 972 err = xe_tile_init_noalloc(tile); 973 if (err) 974 return err; 975 } 976 977 /* 978 * Allow allocations only now to ensure xe_display_init_early() 979 * is the first to allocate, always. 980 */ 981 err = xe_ttm_sys_mgr_init(xe); 982 if (err) 983 return err; 984 985 /* Allocate and map stolen after potential VRAM resize */ 986 err = xe_ttm_stolen_mgr_init(xe); 987 if (err) 988 return err; 989 990 /* 991 * Now that GT is initialized (TTM in particular), 992 * we can try to init display, and inherit the initial fb. 993 * This is the reason the first allocation needs to be done 994 * inside display. 995 */ 996 err = xe_display_init_early(xe); 997 if (err) 998 return err; 999 1000 for_each_tile(tile, xe, id) { 1001 err = xe_tile_init(tile); 1002 if (err) 1003 return err; 1004 } 1005 1006 err = xe_irq_install(xe); 1007 if (err) 1008 return err; 1009 1010 for_each_gt(gt, xe, id) { 1011 err = xe_gt_init(gt); 1012 if (err) 1013 return err; 1014 } 1015 1016 err = xe_pagefault_init(xe); 1017 if (err) 1018 return err; 1019 1020 if (xe->tiles->media_gt && 1021 XE_GT_WA(xe->tiles->media_gt, 15015404425_disable)) 1022 XE_DEVICE_WA_DISABLE(xe, 15015404425); 1023 1024 err = xe_devcoredump_init(xe); 1025 if (err) 1026 return err; 1027 1028 xe_nvm_init(xe); 1029 1030 err = xe_soc_remapper_init(xe); 1031 if (err) 1032 return err; 1033 1034 err = xe_heci_gsc_init(xe); 1035 if (err) 1036 return err; 1037 1038 err = xe_late_bind_init(&xe->late_bind); 1039 if (err) 1040 return err; 1041 1042 err = xe_oa_init(xe); 1043 if (err) 1044 return err; 1045 1046 err = xe_display_init(xe); 1047 if (err) 1048 return err; 1049 1050 err = xe_pxp_init(xe); 1051 if (err) 1052 return err; 1053 1054 err = xe_psmi_init(xe); 1055 if (err) 1056 return err; 1057 1058 err = drm_dev_register(&xe->drm, 0); 1059 if (err) 1060 return err; 1061 1062 xe_display_register(xe); 1063 1064 err = xe_oa_register(xe); 1065 if (err) 1066 goto err_unregister_display; 1067 1068 err = xe_pmu_register(&xe->pmu); 1069 if (err) 1070 goto err_unregister_display; 1071 1072 err = xe_sysctrl_init(xe); 1073 if (err) 1074 goto err_unregister_display; 1075 1076 err = xe_device_sysfs_init(xe); 1077 if (err) 1078 goto err_unregister_display; 1079 1080 xe_debugfs_register(xe); 1081 1082 err = xe_hwmon_register(xe); 1083 if (err) 1084 goto err_unregister_display; 1085 1086 err = xe_i2c_probe(xe); 1087 if (err) 1088 goto err_unregister_display; 1089 1090 for_each_gt(gt, xe, id) 1091 xe_gt_sanitize_freq(gt); 1092 1093 xe_vsec_init(xe); 1094 1095 err = xe_sriov_init_late(xe); 1096 if (err) 1097 goto err_unregister_display; 1098 1099 detect_preproduction_hw(xe); 1100 1101 err = drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe); 1102 if (err) 1103 goto err_unregister_display; 1104 1105 return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); 1106 1107 err_unregister_display: 1108 xe_display_unregister(xe); 1109 drm_dev_unregister(&xe->drm); 1110 1111 return err; 1112 } 1113 1114 void xe_device_remove(struct xe_device *xe) 1115 { 1116 xe_display_unregister(xe); 1117 1118 drm_dev_unplug(&xe->drm); 1119 1120 xe_bo_pci_dev_remove_all(xe); 1121 } 1122 1123 void xe_device_shutdown(struct xe_device *xe) 1124 { 1125 struct xe_gt *gt; 1126 u8 id; 1127 1128 drm_dbg(&xe->drm, "Shutting down device\n"); 1129 1130 xe_display_pm_shutdown(xe); 1131 1132 xe_irq_suspend(xe); 1133 1134 for_each_gt(gt, xe, id) 1135 xe_gt_shutdown(gt); 1136 1137 xe_display_pm_shutdown_late(xe); 1138 1139 if (!xe_driver_flr_disabled(xe)) { 1140 /* BOOM! */ 1141 __xe_driver_flr(xe); 1142 } 1143 } 1144 1145 /** 1146 * xe_device_wmb() - Device specific write memory barrier 1147 * @xe: the &xe_device 1148 * 1149 * While wmb() is sufficient for a barrier if we use system memory, on discrete 1150 * platforms with device memory we additionally need to issue a register write. 1151 * Since it doesn't matter which register we write to, use the read-only VF_CAP 1152 * register that is also marked as accessible by the VFs. 1153 */ 1154 void xe_device_wmb(struct xe_device *xe) 1155 { 1156 wmb(); 1157 if (IS_DGFX(xe)) 1158 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0); 1159 } 1160 1161 /* 1162 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt. 1163 */ 1164 static void tdf_request_sync(struct xe_device *xe) 1165 { 1166 struct xe_gt *gt; 1167 u8 id; 1168 1169 for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) { 1170 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 1171 if (!fw_ref.domains) 1172 return; 1173 1174 xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); 1175 1176 /* 1177 * FIXME: We can likely do better here with our choice of 1178 * timeout. Currently we just assume the worst case, i.e. 150us, 1179 * which is believed to be sufficient to cover the worst case 1180 * scenario on current platforms if all cache entries are 1181 * transient and need to be flushed.. 1182 */ 1183 if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 1184 300, NULL, false)) 1185 xe_gt_err_once(gt, "TD flush timeout\n"); 1186 } 1187 } 1188 1189 /** 1190 * xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW 1191 * @xe: The device to check. 1192 * 1193 * Return: true if the HW device optimizing L2 flush, false otherwise. 1194 */ 1195 bool xe_device_is_l2_flush_optimized(struct xe_device *xe) 1196 { 1197 /* XA is *always* flushed, like at the end-of-submssion (and maybe other 1198 * places), just that internally as an optimisation hw doesn't need to make 1199 * that a full flush (which will also include XA) when Media is 1200 * off/powergated, since it doesn't need to worry about GT caches vs Media 1201 * coherency, and only CPU vs GPU coherency, so can make that flush a 1202 * targeted XA flush, since stuff tagged with XA now means it's shared with 1203 * the CPU. The main implication is that we now need to somehow flush non-XA before 1204 * freeing system memory pages, otherwise dirty cachelines could be flushed after the free 1205 * (like if Media suddenly turns on and does a full flush) 1206 */ 1207 if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe)) 1208 return true; 1209 return false; 1210 } 1211 1212 void xe_device_l2_flush(struct xe_device *xe) 1213 { 1214 struct xe_gt *gt; 1215 1216 gt = xe_root_mmio_gt(xe); 1217 if (!gt) 1218 return; 1219 1220 if (!XE_GT_WA(gt, 16023588340)) 1221 return; 1222 1223 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 1224 if (!fw_ref.domains) 1225 return; 1226 1227 spin_lock(>->global_invl_lock); 1228 1229 xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1); 1230 if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true)) 1231 xe_gt_err_once(gt, "Global invalidation timeout\n"); 1232 1233 spin_unlock(>->global_invl_lock); 1234 } 1235 1236 /** 1237 * xe_device_td_flush() - Flush transient L3 cache entries 1238 * @xe: The device 1239 * 1240 * Display engine has direct access to memory and is never coherent with L3/L4 1241 * caches (or CPU caches), however KMD is responsible for specifically flushing 1242 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout 1243 * can happen from such a surface without seeing corruption. 1244 * 1245 * Display surfaces can be tagged as transient by mapping it using one of the 1246 * various L3:XD PAT index modes on Xe2. 1247 * 1248 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed 1249 * at the end of each submission via PIPE_CONTROL for compute/render, since SA 1250 * Media is not coherent with L3 and we want to support render-vs-media 1251 * usescases. For other engines like copy/blt the HW internally forces uncached 1252 * behaviour, hence why we can skip the TDF on such platforms. 1253 */ 1254 void xe_device_td_flush(struct xe_device *xe) 1255 { 1256 struct xe_gt *root_gt; 1257 1258 /* 1259 * From Xe3p onward the HW takes care of flush of TD entries also along 1260 * with flushing XA entries, which will be at the usual sync points, 1261 * like at the end of submission, so no manual flush is needed here. 1262 */ 1263 if (GRAPHICS_VER(xe) >= 35) 1264 return; 1265 1266 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20) 1267 return; 1268 1269 root_gt = xe_root_mmio_gt(xe); 1270 if (!root_gt) 1271 return; 1272 1273 if (XE_GT_WA(root_gt, 16023588340)) { 1274 /* A transient flush is not sufficient: flush the L2 */ 1275 xe_device_l2_flush(xe); 1276 } else { 1277 xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc); 1278 tdf_request_sync(xe); 1279 xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc); 1280 } 1281 } 1282 1283 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) 1284 { 1285 return xe_device_has_flat_ccs(xe) ? 1286 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0; 1287 } 1288 1289 /** 1290 * xe_device_assert_mem_access - Inspect the current runtime_pm state. 1291 * @xe: xe device instance 1292 * 1293 * To be used before any kind of memory access. It will splat a debug warning 1294 * if the device is currently sleeping. But it doesn't guarantee in any way 1295 * that the device is going to remain awake. Xe PM runtime get and put 1296 * functions might be added to the outer bound of the memory access, while 1297 * this check is intended for inner usage to splat some warning if the worst 1298 * case has just happened. 1299 */ 1300 void xe_device_assert_mem_access(struct xe_device *xe) 1301 { 1302 xe_assert(xe, !xe_pm_runtime_suspended(xe)); 1303 } 1304 1305 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p) 1306 { 1307 struct xe_gt *gt; 1308 u8 id; 1309 1310 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid); 1311 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid); 1312 1313 for_each_gt(gt, xe, id) { 1314 drm_printf(p, "GT id: %u\n", id); 1315 drm_printf(p, "\tTile: %u\n", gt->tile->id); 1316 drm_printf(p, "\tType: %s\n", 1317 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media"); 1318 drm_printf(p, "\tIP ver: %u.%u.%u\n", 1319 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid), 1320 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid), 1321 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid)); 1322 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock); 1323 } 1324 } 1325 1326 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address) 1327 { 1328 return sign_extend64(address, xe->info.va_bits - 1); 1329 } 1330 1331 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) 1332 { 1333 return address & GENMASK_ULL(xe->info.va_bits - 1, 0); 1334 } 1335 1336 /** 1337 * DOC: Xe Device Wedging 1338 * 1339 * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. 1340 * When device is in wedged state, every IOCTL will be blocked and GT cannot 1341 * be used. The conditions under which the driver declares the device wedged 1342 * depend on the wedged mode configuration (see &enum xe_wedged_mode). The 1343 * default recovery method for a wedged state is rebind/bus-reset. 1344 * 1345 * Another recovery method is vendor-specific. Below are the cases that send 1346 * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. 1347 * 1348 * Case: Firmware Flash 1349 * -------------------- 1350 * 1351 * Identification Hint 1352 * +++++++++++++++++++ 1353 * 1354 * ``WEDGED=vendor-specific`` drm device wedged uevent with 1355 * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify 1356 * admin/userspace consumer about the need for a firmware flash. 1357 * 1358 * Recovery Procedure 1359 * ++++++++++++++++++ 1360 * 1361 * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow 1362 * the below steps 1363 * 1364 * - Check Runtime Survivability mode sysfs. 1365 * If enabled, firmware flash is required to recover the device. 1366 * 1367 * /sys/bus/pci/devices/<device>/survivability_mode 1368 * 1369 * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash 1370 * firmware and restore device to normal operation. 1371 */ 1372 1373 /** 1374 * xe_device_set_wedged_method - Set wedged recovery method 1375 * @xe: xe device instance 1376 * @method: recovery method to set 1377 * 1378 * Set wedged recovery method to be sent in drm wedged uevent. 1379 */ 1380 void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method) 1381 { 1382 xe->wedged.method = method; 1383 } 1384 1385 /** 1386 * xe_device_declare_wedged - Declare device wedged 1387 * @xe: xe device instance 1388 * 1389 * This is a final state that can only be cleared with the recovery method 1390 * specified in the drm wedged uevent. The method can be set using 1391 * xe_device_set_wedged_method before declaring the device as wedged. If no method 1392 * is set, reprobe (unbind/re-bind) will be sent by default. 1393 * 1394 * In this state every IOCTL will be blocked so the GT cannot be used. 1395 * In general it will be called upon any critical error such as gt reset 1396 * failure or guc loading failure. Userspace will be notified of this state 1397 * through device wedged uevent. 1398 * If xe.wedged module parameter is set to 2, this function will be called 1399 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump 1400 * snapshot capture. In this mode, GT reset won't be attempted so the state of 1401 * the issue is preserved for further debugging. 1402 */ 1403 void xe_device_declare_wedged(struct xe_device *xe) 1404 { 1405 struct xe_gt *gt; 1406 u8 id; 1407 1408 if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) { 1409 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); 1410 return; 1411 } 1412 1413 if (!atomic_xchg(&xe->wedged.flag, 1)) { 1414 xe->needs_flr_on_fini = true; 1415 xe_pm_runtime_get_noresume(xe); 1416 drm_err(&xe->drm, 1417 "CRITICAL: Xe has declared device %s as wedged.\n" 1418 "IOCTLs and executions are blocked.\n" 1419 "For recovery procedure, refer to https://docs.kernel.org/gpu/drm-uapi.html#device-wedging\n" 1420 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n", 1421 dev_name(xe->drm.dev)); 1422 } 1423 1424 for_each_gt(gt, xe, id) 1425 xe_gt_declare_wedged(gt); 1426 1427 if (xe_device_wedged(xe)) { 1428 /* 1429 * XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging 1430 * hangs, so wedge the device with 'none' recovery method and have 1431 * it available to the user for debugging. 1432 */ 1433 if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) 1434 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_NONE); 1435 /* If no wedge recovery method is set, use default */ 1436 else if (!xe->wedged.method) 1437 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND | 1438 DRM_WEDGE_RECOVERY_BUS_RESET); 1439 1440 /* Notify userspace of wedged device */ 1441 drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); 1442 } 1443 } 1444 1445 /** 1446 * xe_device_validate_wedged_mode - Check if given mode is supported 1447 * @xe: the &xe_device 1448 * @mode: requested mode to validate 1449 * 1450 * Check whether the provided wedged mode is supported. 1451 * 1452 * Return: 0 if mode is supported, error code otherwise. 1453 */ 1454 int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode) 1455 { 1456 if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { 1457 drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode); 1458 return -EINVAL; 1459 } else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) || 1460 (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) { 1461 drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n", 1462 mode, xe_wedged_mode_to_string(mode), 1463 xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); 1464 return -EPERM; 1465 } 1466 1467 return 0; 1468 } 1469 1470 /** 1471 * xe_wedged_mode_to_string - Convert enum value to string. 1472 * @mode: the &xe_wedged_mode to convert 1473 * 1474 * Returns: wedged mode as a user friendly string. 1475 */ 1476 const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode) 1477 { 1478 switch (mode) { 1479 case XE_WEDGED_MODE_NEVER: 1480 return "never"; 1481 case XE_WEDGED_MODE_UPON_CRITICAL_ERROR: 1482 return "upon-critical-error"; 1483 case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: 1484 return "upon-any-hang-no-reset"; 1485 default: 1486 return "<invalid>"; 1487 } 1488 } 1489 1490 /** 1491 * xe_device_asid_to_vm() - Find VM from ASID 1492 * @xe: the &xe_device 1493 * @asid: Address space ID 1494 * 1495 * Find a VM from ASID and take a reference to VM which caller must drop. 1496 * Reclaim safe. 1497 * 1498 * Return: VM on success, ERR_PTR on failure 1499 */ 1500 struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid) 1501 { 1502 struct xe_vm *vm; 1503 1504 down_read(&xe->usm.lock); 1505 vm = xa_load(&xe->usm.asid_to_vm, asid); 1506 if (vm) 1507 xe_vm_get(vm); 1508 else 1509 vm = ERR_PTR(-EINVAL); 1510 up_read(&xe->usm.lock); 1511 1512 return vm; 1513 } 1514