1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_device.h" 7 8 #include <linux/aperture.h> 9 #include <linux/delay.h> 10 #include <linux/fault-inject.h> 11 #include <linux/units.h> 12 13 #include <drm/drm_atomic_helper.h> 14 #include <drm/drm_client.h> 15 #include <drm/drm_gem_ttm_helper.h> 16 #include <drm/drm_ioctl.h> 17 #include <drm/drm_managed.h> 18 #include <drm/drm_pagemap_util.h> 19 #include <drm/drm_print.h> 20 #include <kunit/static_stub.h> 21 #include <uapi/drm/xe_drm.h> 22 23 #include "display/xe_display.h" 24 #include "instructions/xe_gpu_commands.h" 25 #include "regs/xe_gt_regs.h" 26 #include "regs/xe_regs.h" 27 #include "xe_bo.h" 28 #include "xe_bo_evict.h" 29 #include "xe_configfs.h" 30 #include "xe_debugfs.h" 31 #include "xe_defaults.h" 32 #include "xe_devcoredump.h" 33 #include "xe_device_sysfs.h" 34 #include "xe_dma_buf.h" 35 #include "xe_drm_client.h" 36 #include "xe_drv.h" 37 #include "xe_exec.h" 38 #include "xe_exec_queue.h" 39 #include "xe_force_wake.h" 40 #include "xe_ggtt.h" 41 #include "xe_gt.h" 42 #include "xe_gt_mcr.h" 43 #include "xe_gt_printk.h" 44 #include "xe_gt_sriov_vf.h" 45 #include "xe_guc.h" 46 #include "xe_guc_pc.h" 47 #include "xe_hw_engine_group.h" 48 #include "xe_hwmon.h" 49 #include "xe_i2c.h" 50 #include "xe_irq.h" 51 #include "xe_late_bind_fw.h" 52 #include "xe_mmio.h" 53 #include "xe_module.h" 54 #include "xe_nvm.h" 55 #include "xe_oa.h" 56 #include "xe_observation.h" 57 #include "xe_pagefault.h" 58 #include "xe_pat.h" 59 #include "xe_pcode.h" 60 #include "xe_pm.h" 61 #include "xe_pmu.h" 62 #include "xe_psmi.h" 63 #include "xe_pxp.h" 64 #include "xe_query.h" 65 #include "xe_shrinker.h" 66 #include "xe_soc_remapper.h" 67 #include "xe_survivability_mode.h" 68 #include "xe_sriov.h" 69 #include "xe_svm.h" 70 #include "xe_sysctrl.h" 71 #include "xe_tile.h" 72 #include "xe_ttm_stolen_mgr.h" 73 #include "xe_ttm_sys_mgr.h" 74 #include "xe_vm.h" 75 #include "xe_vm_madvise.h" 76 #include "xe_vram.h" 77 #include "xe_vram_types.h" 78 #include "xe_vsec.h" 79 #include "xe_wait_user_fence.h" 80 #include "xe_wa.h" 81 82 #include <generated/xe_device_wa_oob.h> 83 #include <generated/xe_wa_oob.h> 84 85 static int xe_file_open(struct drm_device *dev, struct drm_file *file) 86 { 87 struct xe_device *xe = to_xe_device(dev); 88 struct xe_drm_client *client; 89 struct xe_file *xef; 90 int ret = -ENOMEM; 91 struct task_struct *task = NULL; 92 93 xef = kzalloc_obj(*xef); 94 if (!xef) 95 return ret; 96 97 client = xe_drm_client_alloc(); 98 if (!client) { 99 kfree(xef); 100 return ret; 101 } 102 103 xef->drm = file; 104 xef->client = client; 105 xef->xe = xe; 106 107 mutex_init(&xef->vm.lock); 108 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); 109 110 mutex_init(&xef->exec_queue.lock); 111 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); 112 113 file->driver_priv = xef; 114 kref_init(&xef->refcount); 115 116 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID); 117 if (task) { 118 xef->process_name = kstrdup(task->comm, GFP_KERNEL); 119 xef->pid = task->pid; 120 put_task_struct(task); 121 } 122 123 return 0; 124 } 125 126 static void xe_file_destroy(struct kref *ref) 127 { 128 struct xe_file *xef = container_of(ref, struct xe_file, refcount); 129 130 xa_destroy(&xef->exec_queue.xa); 131 mutex_destroy(&xef->exec_queue.lock); 132 xa_destroy(&xef->vm.xa); 133 mutex_destroy(&xef->vm.lock); 134 135 xe_drm_client_put(xef->client); 136 kfree(xef->process_name); 137 kfree(xef); 138 } 139 140 /** 141 * xe_file_get() - Take a reference to the xe file object 142 * @xef: Pointer to the xe file 143 * 144 * Anyone with a pointer to xef must take a reference to the xe file 145 * object using this call. 146 * 147 * Return: xe file pointer 148 */ 149 struct xe_file *xe_file_get(struct xe_file *xef) 150 { 151 kref_get(&xef->refcount); 152 return xef; 153 } 154 155 /** 156 * xe_file_put() - Drop a reference to the xe file object 157 * @xef: Pointer to the xe file 158 * 159 * Used to drop reference to the xef object 160 */ 161 void xe_file_put(struct xe_file *xef) 162 { 163 kref_put(&xef->refcount, xe_file_destroy); 164 } 165 166 static void xe_file_close(struct drm_device *dev, struct drm_file *file) 167 { 168 struct xe_device *xe = to_xe_device(dev); 169 struct xe_file *xef = file->driver_priv; 170 struct xe_vm *vm; 171 struct xe_exec_queue *q; 172 unsigned long idx; 173 174 guard(xe_pm_runtime)(xe); 175 176 /* 177 * No need for exec_queue.lock here as there is no contention for it 178 * when FD is closing as IOCTLs presumably can't be modifying the 179 * xarray. Taking exec_queue.lock here causes undue dependency on 180 * vm->lock taken during xe_exec_queue_kill(). 181 */ 182 xa_for_each(&xef->exec_queue.xa, idx, q) { 183 if (q->vm && q->hwe->hw_engine_group) 184 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 185 xe_exec_queue_kill(q); 186 xe_exec_queue_put(q); 187 } 188 xa_for_each(&xef->vm.xa, idx, vm) 189 xe_vm_close_and_put(vm); 190 191 xe_file_put(xef); 192 } 193 194 static const struct drm_ioctl_desc xe_ioctls[] = { 195 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 196 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), 197 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, 198 DRM_RENDER_ALLOW), 199 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), 200 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), 201 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), 202 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), 203 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, 204 DRM_RENDER_ALLOW), 205 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, 206 DRM_RENDER_ALLOW), 207 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, 208 DRM_RENDER_ALLOW), 209 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, 210 DRM_RENDER_ALLOW), 211 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 212 DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), 213 DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, 214 DRM_RENDER_ALLOW), 215 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, 216 DRM_RENDER_ALLOW), 217 DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl, 218 DRM_RENDER_ALLOW), 219 }; 220 221 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 222 { 223 struct drm_file *file_priv = file->private_data; 224 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 225 long ret; 226 227 if (xe_device_wedged(xe)) 228 return -ECANCELED; 229 230 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); 231 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); 232 if (ret >= 0) 233 ret = drm_ioctl(file, cmd, arg); 234 235 return ret; 236 } 237 238 #ifdef CONFIG_COMPAT 239 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 240 { 241 struct drm_file *file_priv = file->private_data; 242 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 243 long ret; 244 245 if (xe_device_wedged(xe)) 246 return -ECANCELED; 247 248 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); 249 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); 250 if (ret >= 0) 251 ret = drm_compat_ioctl(file, cmd, arg); 252 253 return ret; 254 } 255 #else 256 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */ 257 #define xe_drm_compat_ioctl NULL 258 #endif 259 260 static void barrier_open(struct vm_area_struct *vma) 261 { 262 drm_dev_get(vma->vm_private_data); 263 } 264 265 static void barrier_close(struct vm_area_struct *vma) 266 { 267 drm_dev_put(vma->vm_private_data); 268 } 269 270 static void barrier_release_dummy_page(struct drm_device *dev, void *res) 271 { 272 struct page *dummy_page = (struct page *)res; 273 274 __free_page(dummy_page); 275 } 276 277 static vm_fault_t barrier_fault(struct vm_fault *vmf) 278 { 279 struct drm_device *dev = vmf->vma->vm_private_data; 280 struct vm_area_struct *vma = vmf->vma; 281 vm_fault_t ret = VM_FAULT_NOPAGE; 282 pgprot_t prot; 283 int idx; 284 285 prot = vm_get_page_prot(vma->vm_flags); 286 287 if (drm_dev_enter(dev, &idx)) { 288 unsigned long pfn; 289 290 #define LAST_DB_PAGE_OFFSET 0x7ff001 291 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) + 292 LAST_DB_PAGE_OFFSET); 293 ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn, 294 pgprot_noncached(prot)); 295 drm_dev_exit(idx); 296 } else { 297 struct page *page; 298 299 /* Allocate new dummy page to map all the VA range in this VMA to it*/ 300 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 301 if (!page) 302 return VM_FAULT_OOM; 303 304 /* Set the page to be freed using drmm release action */ 305 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page)) 306 return VM_FAULT_OOM; 307 308 ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page), 309 prot); 310 } 311 312 return ret; 313 } 314 315 static const struct vm_operations_struct vm_ops_barrier = { 316 .open = barrier_open, 317 .close = barrier_close, 318 .fault = barrier_fault, 319 }; 320 321 static int xe_pci_barrier_mmap(struct file *filp, 322 struct vm_area_struct *vma) 323 { 324 struct drm_file *priv = filp->private_data; 325 struct drm_device *dev = priv->minor->dev; 326 struct xe_device *xe = to_xe_device(dev); 327 328 if (!IS_DGFX(xe)) 329 return -EINVAL; 330 331 if (vma->vm_end - vma->vm_start > SZ_4K) 332 return -EINVAL; 333 334 if (is_cow_mapping(vma->vm_flags)) 335 return -EINVAL; 336 337 if (vma->vm_flags & (VM_READ | VM_EXEC)) 338 return -EINVAL; 339 340 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC); 341 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); 342 vma->vm_ops = &vm_ops_barrier; 343 vma->vm_private_data = dev; 344 drm_dev_get(vma->vm_private_data); 345 346 return 0; 347 } 348 349 static int xe_mmap(struct file *filp, struct vm_area_struct *vma) 350 { 351 struct drm_file *priv = filp->private_data; 352 struct drm_device *dev = priv->minor->dev; 353 354 if (drm_dev_is_unplugged(dev)) 355 return -ENODEV; 356 357 switch (vma->vm_pgoff) { 358 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT: 359 return xe_pci_barrier_mmap(filp, vma); 360 } 361 362 return drm_gem_mmap(filp, vma); 363 } 364 365 static const struct file_operations xe_driver_fops = { 366 .owner = THIS_MODULE, 367 .open = drm_open, 368 .release = drm_release_noglobal, 369 .unlocked_ioctl = xe_drm_ioctl, 370 .mmap = xe_mmap, 371 .poll = drm_poll, 372 .read = drm_read, 373 .compat_ioctl = xe_drm_compat_ioctl, 374 .llseek = noop_llseek, 375 #ifdef CONFIG_PROC_FS 376 .show_fdinfo = drm_show_fdinfo, 377 #endif 378 .fop_flags = FOP_UNSIGNED_OFFSET, 379 }; 380 381 /** 382 * xe_is_xe_file() - Is the file an xe device file? 383 * @file: The file. 384 * 385 * Checks whether the file is opened against 386 * an xe device. 387 * 388 * Return: %true if an xe file, %false if not. 389 */ 390 bool xe_is_xe_file(const struct file *file) 391 { 392 return file->f_op == &xe_driver_fops; 393 } 394 395 static struct drm_driver regular_driver = { 396 .driver_features = 397 DRIVER_GEM | 398 DRIVER_RENDER | DRIVER_SYNCOBJ | 399 DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, 400 .open = xe_file_open, 401 .postclose = xe_file_close, 402 403 .gem_prime_import = xe_gem_prime_import, 404 405 .dumb_create = xe_bo_dumb_create, 406 .dumb_map_offset = drm_gem_ttm_dumb_map_offset, 407 #ifdef CONFIG_PROC_FS 408 .show_fdinfo = xe_drm_client_fdinfo, 409 #endif 410 .ioctls = xe_ioctls, 411 .num_ioctls = ARRAY_SIZE(xe_ioctls), 412 .fops = &xe_driver_fops, 413 .name = DRIVER_NAME, 414 .desc = DRIVER_DESC, 415 .major = DRIVER_MAJOR, 416 .minor = DRIVER_MINOR, 417 .patchlevel = DRIVER_PATCHLEVEL, 418 }; 419 420 #ifdef CONFIG_PCI_IOV 421 static const struct drm_ioctl_desc xe_ioctls_admin_only[] = { 422 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 423 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 424 }; 425 426 static struct drm_driver admin_only_driver = { 427 .driver_features = 428 DRIVER_GEM | DRIVER_RENDER | DRIVER_GEM_GPUVA, 429 .open = xe_file_open, 430 .postclose = xe_file_close, 431 .ioctls = xe_ioctls_admin_only, 432 .num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only), 433 .fops = &xe_driver_fops, 434 .name = DRIVER_NAME, 435 .desc = DRIVER_DESC, 436 .major = DRIVER_MAJOR, 437 .minor = DRIVER_MINOR, 438 .patchlevel = DRIVER_PATCHLEVEL, 439 }; 440 441 /** 442 * xe_device_is_admin_only() - Check whether device is admin only or not. 443 * @xe: the &xe_device to check 444 * 445 * Return: true if the device is admin only, false otherwise. 446 */ 447 bool xe_device_is_admin_only(const struct xe_device *xe) 448 { 449 KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe); 450 return xe->drm.driver == &admin_only_driver; 451 } 452 #endif 453 454 static void xe_device_destroy(struct drm_device *dev, void *dummy) 455 { 456 struct xe_device *xe = to_xe_device(dev); 457 458 xe_bo_dev_fini(&xe->bo_device); 459 460 if (xe->preempt_fence_wq) 461 destroy_workqueue(xe->preempt_fence_wq); 462 463 if (xe->ordered_wq) 464 destroy_workqueue(xe->ordered_wq); 465 466 if (xe->unordered_wq) 467 destroy_workqueue(xe->unordered_wq); 468 469 if (xe->destroy_wq) 470 destroy_workqueue(xe->destroy_wq); 471 472 ttm_device_fini(&xe->ttm); 473 } 474 475 struct xe_device *xe_device_create(struct pci_dev *pdev, 476 const struct pci_device_id *ent) 477 { 478 struct drm_driver *driver = ®ular_driver; 479 struct xe_device *xe; 480 int err; 481 482 #ifdef CONFIG_PCI_IOV 483 /* 484 * Since XE device is not initialized yet, read from configfs 485 * directly to decide whether we are in admin-only PF mode or not. 486 */ 487 if (xe_configfs_admin_only_pf(pdev)) 488 driver = &admin_only_driver; 489 #endif 490 491 xe_display_driver_set_hooks(driver); 492 493 err = aperture_remove_conflicting_pci_devices(pdev, driver->name); 494 if (err) 495 return ERR_PTR(err); 496 497 xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm); 498 if (IS_ERR(xe)) 499 return xe; 500 501 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, 502 xe->drm.anon_inode->i_mapping, 503 xe->drm.vma_offset_manager, 0); 504 if (WARN_ON(err)) 505 return ERR_PTR(err); 506 507 xe_bo_dev_init(&xe->bo_device); 508 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); 509 if (err) 510 return ERR_PTR(err); 511 512 err = xe_shrinker_create(xe); 513 if (err) 514 return ERR_PTR(err); 515 516 xe->info.devid = pdev->device; 517 xe->info.revid = pdev->revision; 518 xe->info.force_execlist = xe_modparam.force_execlist; 519 xe->atomic_svm_timeslice_ms = 5; 520 xe->min_run_period_lr_ms = 5; 521 522 err = xe_irq_init(xe); 523 if (err) 524 return ERR_PTR(err); 525 526 xe_validation_device_init(&xe->val); 527 528 init_waitqueue_head(&xe->ufence_wq); 529 530 init_rwsem(&xe->usm.lock); 531 532 err = xe_pagemap_shrinker_create(xe); 533 if (err) 534 return ERR_PTR(err); 535 536 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); 537 538 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 539 /* Trigger a large asid and an early asid wrap. */ 540 u32 asid; 541 542 BUILD_BUG_ON(XE_MAX_ASID < 2); 543 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL, 544 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1), 545 &xe->usm.next_asid, GFP_KERNEL); 546 drm_WARN_ON(&xe->drm, err); 547 if (err >= 0) 548 xa_erase(&xe->usm.asid_to_vm, asid); 549 } 550 551 err = xe_bo_pinned_init(xe); 552 if (err) 553 return ERR_PTR(err); 554 555 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", 556 WQ_MEM_RECLAIM); 557 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); 558 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0); 559 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU, 0); 560 if (!xe->ordered_wq || !xe->unordered_wq || 561 !xe->preempt_fence_wq || !xe->destroy_wq) { 562 /* 563 * Cleanup done in xe_device_destroy via 564 * drmm_add_action_or_reset register above 565 */ 566 drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); 567 return ERR_PTR(-ENOMEM); 568 } 569 570 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); 571 if (err) 572 return ERR_PTR(err); 573 574 return xe; 575 } 576 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ 577 578 static bool xe_driver_flr_disabled(struct xe_device *xe) 579 { 580 if (IS_SRIOV_VF(xe)) 581 return true; 582 583 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) { 584 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n"); 585 return true; 586 } 587 588 return false; 589 } 590 591 /* 592 * The driver-initiated FLR is the highest level of reset that we can trigger 593 * from within the driver. It is different from the PCI FLR in that it doesn't 594 * fully reset the SGUnit and doesn't modify the PCI config space and therefore 595 * it doesn't require a re-enumeration of the PCI BARs. However, the 596 * driver-initiated FLR does still cause a reset of both GT and display and a 597 * memory wipe of local and stolen memory, so recovery would require a full HW 598 * re-init and saving/restoring (or re-populating) the wiped memory. Since we 599 * perform the FLR as the very last action before releasing access to the HW 600 * during the driver release flow, we don't attempt recovery at all, because 601 * if/when a new instance of Xe is bound to the device it will do a full 602 * re-init anyway. 603 */ 604 static void __xe_driver_flr(struct xe_device *xe) 605 { 606 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */ 607 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 608 int ret; 609 610 drm_dbg(&xe->drm, "Triggering Driver-FLR\n"); 611 612 /* 613 * Make sure any pending FLR requests have cleared by waiting for the 614 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS 615 * to make sure it's not still set from a prior attempt (it's a write to 616 * clear bit). 617 * Note that we should never be in a situation where a previous attempt 618 * is still pending (unless the HW is totally dead), but better to be 619 * safe in case something unexpected happens 620 */ 621 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 622 if (ret) { 623 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret); 624 return; 625 } 626 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 627 628 /* Trigger the actual Driver-FLR */ 629 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR); 630 631 /* Wait for hardware teardown to complete */ 632 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 633 if (ret) { 634 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); 635 return; 636 } 637 638 /* Wait for hardware/firmware re-init to complete */ 639 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS, 640 flr_timeout, NULL, false); 641 if (ret) { 642 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); 643 return; 644 } 645 646 /* Clear sticky completion status */ 647 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 648 } 649 650 static void xe_driver_flr(struct xe_device *xe) 651 { 652 if (xe_driver_flr_disabled(xe)) 653 return; 654 655 __xe_driver_flr(xe); 656 } 657 658 static void xe_driver_flr_fini(void *arg) 659 { 660 struct xe_device *xe = arg; 661 662 if (xe->needs_flr_on_fini) 663 xe_driver_flr(xe); 664 } 665 666 static void xe_device_sanitize(void *arg) 667 { 668 struct xe_device *xe = arg; 669 struct xe_gt *gt; 670 u8 id; 671 672 for_each_gt(gt, xe, id) 673 xe_gt_sanitize(gt); 674 } 675 676 static int xe_set_dma_info(struct xe_device *xe) 677 { 678 unsigned int mask_size = xe->info.dma_mask_size; 679 int err; 680 681 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev)); 682 683 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 684 if (err) 685 goto mask_err; 686 687 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 688 if (err) 689 goto mask_err; 690 691 return 0; 692 693 mask_err: 694 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); 695 return err; 696 } 697 698 static void assert_lmem_ready(struct xe_device *xe) 699 { 700 if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) 701 return; 702 703 xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & 704 LMEM_INIT); 705 } 706 707 static void vf_update_device_info(struct xe_device *xe) 708 { 709 xe_assert(xe, IS_SRIOV_VF(xe)); 710 /* disable features that are not available/applicable to VFs */ 711 xe->info.probe_display = 0; 712 xe->info.has_heci_cscfi = 0; 713 xe->info.has_heci_gscfi = 0; 714 xe->info.has_late_bind = 0; 715 xe->info.skip_guc_pc = 1; 716 xe->info.skip_pcode = 1; 717 } 718 719 static int xe_device_vram_alloc(struct xe_device *xe) 720 { 721 struct xe_vram_region *vram; 722 723 if (!IS_DGFX(xe)) 724 return 0; 725 726 vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); 727 if (!vram) 728 return -ENOMEM; 729 730 xe->mem.vram = vram; 731 return 0; 732 } 733 734 /** 735 * xe_device_probe_early: Device early probe 736 * @xe: xe device instance 737 * 738 * Initialize MMIO resources that don't require any 739 * knowledge about tile count. Also initialize pcode and 740 * check vram initialization on root tile. 741 * 742 * Return: 0 on success, error code on failure 743 */ 744 int xe_device_probe_early(struct xe_device *xe) 745 { 746 int err; 747 748 xe_wa_device_init(xe); 749 xe_wa_process_device_oob(xe); 750 751 err = xe_mmio_probe_early(xe); 752 if (err) 753 return err; 754 755 xe_sriov_probe_early(xe); 756 757 if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) { 758 xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n"); 759 return -ENODEV; 760 } 761 762 if (IS_SRIOV_VF(xe)) 763 vf_update_device_info(xe); 764 765 /* 766 * Check for pcode uncore_init status to confirm if the SoC 767 * initialization is complete. Until done, any MMIO or lmem access from 768 * the driver will be blocked 769 */ 770 err = xe_pcode_probe_early(xe); 771 if (err || xe_survivability_mode_is_requested(xe)) { 772 int save_err = err; 773 774 /* 775 * Try to leave device in survivability mode if device is 776 * possible, but still return the previous error for error 777 * propagation 778 */ 779 err = xe_survivability_mode_boot_enable(xe); 780 if (err) 781 return err; 782 783 return save_err; 784 } 785 786 /* 787 * Make sure the lmem is initialized and ready to use. xe_pcode_ready() 788 * is flagged after full initialization is complete. Assert if lmem is 789 * not initialized. 790 */ 791 assert_lmem_ready(xe); 792 793 xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ? 794 XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode; 795 drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n", 796 xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode)); 797 798 err = xe_device_vram_alloc(xe); 799 if (err) 800 return err; 801 802 return 0; 803 } 804 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ 805 806 static int probe_has_flat_ccs(struct xe_device *xe) 807 { 808 struct xe_gt *gt; 809 u32 reg; 810 811 /* Always enabled/disabled, no runtime check to do */ 812 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe)) 813 return 0; 814 815 gt = xe_root_mmio_gt(xe); 816 if (!gt) 817 return 0; 818 819 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 820 if (!fw_ref.domains) 821 return -ETIMEDOUT; 822 823 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 824 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE); 825 826 if (!xe->info.has_flat_ccs) 827 drm_dbg(&xe->drm, 828 "Flat CCS has been disabled in bios, May lead to performance impact"); 829 830 return 0; 831 } 832 833 /* 834 * Detect if the driver is being run on pre-production hardware. We don't 835 * keep workarounds for pre-production hardware long term, so print an 836 * error and add taint if we're being loaded on a pre-production platform 837 * for which the pre-prod workarounds have already been removed. 838 * 839 * The general policy is that we'll remove any workarounds that only apply to 840 * pre-production hardware around the time force_probe restrictions are lifted 841 * for a platform of the next major IP generation (for example, Xe2 pre-prod 842 * workarounds should be removed around the time the first Xe3 platforms have 843 * force_probe lifted). 844 */ 845 static void detect_preproduction_hw(struct xe_device *xe) 846 { 847 struct xe_gt *gt; 848 int id; 849 850 /* 851 * SR-IOV VFs don't have access to the FUSE2 register, so we can't 852 * check pre-production status there. But the host OS will notice 853 * and report the pre-production status, which should be enough to 854 * help us catch mistaken use of pre-production hardware. 855 */ 856 if (IS_SRIOV_VF(xe)) 857 return; 858 859 /* 860 * The "SW_CAP" fuse contains a bit indicating whether the device is a 861 * production or pre-production device. This fuse is reflected through 862 * the GT "FUSE2" register, even though the contents of the fuse are 863 * not GT-specific. Every GT's reflection of this fuse should show the 864 * same value, so we'll just use the first available GT for lookup. 865 */ 866 for_each_gt(gt, xe, id) 867 break; 868 869 if (!gt) 870 return; 871 872 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 873 if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) { 874 xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n"); 875 return; 876 } 877 878 if (xe_mmio_read32(>->mmio, FUSE2) & PRODUCTION_HW) 879 return; 880 881 xe_info(xe, "Pre-production hardware detected.\n"); 882 if (!xe->info.has_pre_prod_wa) { 883 xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n"); 884 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); 885 } 886 } 887 888 static void xe_device_wedged_fini(struct drm_device *drm, void *arg) 889 { 890 struct xe_device *xe = arg; 891 892 if (atomic_read(&xe->wedged.flag)) 893 xe_pm_runtime_put(xe); 894 } 895 896 int xe_device_probe(struct xe_device *xe) 897 { 898 struct xe_tile *tile; 899 struct xe_gt *gt; 900 int err; 901 u8 id; 902 903 xe_pat_init_early(xe); 904 905 err = xe_sriov_init(xe); 906 if (err) 907 return err; 908 909 xe->info.mem_region_mask = 1; 910 911 err = xe_set_dma_info(xe); 912 if (err) 913 return err; 914 915 err = xe_mmio_probe_tiles(xe); 916 if (err) 917 return err; 918 919 for_each_gt(gt, xe, id) { 920 err = xe_gt_init_early(gt); 921 if (err) 922 return err; 923 } 924 925 for_each_tile(tile, xe, id) { 926 err = xe_ggtt_init_early(tile->mem.ggtt); 927 if (err) 928 return err; 929 } 930 931 /* 932 * From here on, if a step fails, make sure a Driver-FLR is triggereed 933 */ 934 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe); 935 if (err) 936 return err; 937 938 err = probe_has_flat_ccs(xe); 939 if (err) 940 return err; 941 942 err = xe_vram_probe(xe); 943 if (err) 944 return err; 945 946 for_each_tile(tile, xe, id) { 947 err = xe_tile_init_noalloc(tile); 948 if (err) 949 return err; 950 } 951 952 /* 953 * Allow allocations only now to ensure xe_display_init_early() 954 * is the first to allocate, always. 955 */ 956 err = xe_ttm_sys_mgr_init(xe); 957 if (err) 958 return err; 959 960 /* Allocate and map stolen after potential VRAM resize */ 961 err = xe_ttm_stolen_mgr_init(xe); 962 if (err) 963 return err; 964 965 /* 966 * Now that GT is initialized (TTM in particular), 967 * we can try to init display, and inherit the initial fb. 968 * This is the reason the first allocation needs to be done 969 * inside display. 970 */ 971 err = xe_display_init_early(xe); 972 if (err) 973 return err; 974 975 for_each_tile(tile, xe, id) { 976 err = xe_tile_init(tile); 977 if (err) 978 return err; 979 } 980 981 err = xe_irq_install(xe); 982 if (err) 983 return err; 984 985 for_each_gt(gt, xe, id) { 986 err = xe_gt_init(gt); 987 if (err) 988 return err; 989 } 990 991 err = xe_pagefault_init(xe); 992 if (err) 993 return err; 994 995 if (xe->tiles->media_gt && 996 XE_GT_WA(xe->tiles->media_gt, 15015404425_disable)) 997 XE_DEVICE_WA_DISABLE(xe, 15015404425); 998 999 err = xe_devcoredump_init(xe); 1000 if (err) 1001 return err; 1002 1003 xe_nvm_init(xe); 1004 1005 err = xe_soc_remapper_init(xe); 1006 if (err) 1007 return err; 1008 1009 err = xe_heci_gsc_init(xe); 1010 if (err) 1011 return err; 1012 1013 err = xe_late_bind_init(&xe->late_bind); 1014 if (err) 1015 return err; 1016 1017 err = xe_oa_init(xe); 1018 if (err) 1019 return err; 1020 1021 err = xe_display_init(xe); 1022 if (err) 1023 return err; 1024 1025 err = xe_pxp_init(xe); 1026 if (err) 1027 return err; 1028 1029 err = xe_psmi_init(xe); 1030 if (err) 1031 return err; 1032 1033 err = drm_dev_register(&xe->drm, 0); 1034 if (err) 1035 return err; 1036 1037 xe_display_register(xe); 1038 1039 err = xe_oa_register(xe); 1040 if (err) 1041 goto err_unregister_display; 1042 1043 err = xe_pmu_register(&xe->pmu); 1044 if (err) 1045 goto err_unregister_display; 1046 1047 err = xe_sysctrl_init(xe); 1048 if (err) 1049 goto err_unregister_display; 1050 1051 err = xe_device_sysfs_init(xe); 1052 if (err) 1053 goto err_unregister_display; 1054 1055 xe_debugfs_register(xe); 1056 1057 err = xe_hwmon_register(xe); 1058 if (err) 1059 goto err_unregister_display; 1060 1061 err = xe_i2c_probe(xe); 1062 if (err) 1063 goto err_unregister_display; 1064 1065 for_each_gt(gt, xe, id) 1066 xe_gt_sanitize_freq(gt); 1067 1068 xe_vsec_init(xe); 1069 1070 err = xe_sriov_init_late(xe); 1071 if (err) 1072 goto err_unregister_display; 1073 1074 detect_preproduction_hw(xe); 1075 1076 err = drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe); 1077 if (err) 1078 goto err_unregister_display; 1079 1080 return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); 1081 1082 err_unregister_display: 1083 xe_display_unregister(xe); 1084 drm_dev_unregister(&xe->drm); 1085 1086 return err; 1087 } 1088 1089 void xe_device_remove(struct xe_device *xe) 1090 { 1091 xe_display_unregister(xe); 1092 1093 drm_dev_unplug(&xe->drm); 1094 1095 xe_bo_pci_dev_remove_all(xe); 1096 } 1097 1098 void xe_device_shutdown(struct xe_device *xe) 1099 { 1100 struct xe_gt *gt; 1101 u8 id; 1102 1103 drm_dbg(&xe->drm, "Shutting down device\n"); 1104 1105 xe_display_pm_shutdown(xe); 1106 1107 xe_irq_suspend(xe); 1108 1109 for_each_gt(gt, xe, id) 1110 xe_gt_shutdown(gt); 1111 1112 xe_display_pm_shutdown_late(xe); 1113 1114 if (!xe_driver_flr_disabled(xe)) { 1115 /* BOOM! */ 1116 __xe_driver_flr(xe); 1117 } 1118 } 1119 1120 /** 1121 * xe_device_wmb() - Device specific write memory barrier 1122 * @xe: the &xe_device 1123 * 1124 * While wmb() is sufficient for a barrier if we use system memory, on discrete 1125 * platforms with device memory we additionally need to issue a register write. 1126 * Since it doesn't matter which register we write to, use the read-only VF_CAP 1127 * register that is also marked as accessible by the VFs. 1128 */ 1129 void xe_device_wmb(struct xe_device *xe) 1130 { 1131 wmb(); 1132 if (IS_DGFX(xe)) 1133 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0); 1134 } 1135 1136 /* 1137 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt. 1138 */ 1139 static void tdf_request_sync(struct xe_device *xe) 1140 { 1141 struct xe_gt *gt; 1142 u8 id; 1143 1144 for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) { 1145 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 1146 if (!fw_ref.domains) 1147 return; 1148 1149 xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); 1150 1151 /* 1152 * FIXME: We can likely do better here with our choice of 1153 * timeout. Currently we just assume the worst case, i.e. 150us, 1154 * which is believed to be sufficient to cover the worst case 1155 * scenario on current platforms if all cache entries are 1156 * transient and need to be flushed.. 1157 */ 1158 if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 1159 300, NULL, false)) 1160 xe_gt_err_once(gt, "TD flush timeout\n"); 1161 } 1162 } 1163 1164 /** 1165 * xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW 1166 * @xe: The device to check. 1167 * 1168 * Return: true if the HW device optimizing L2 flush, false otherwise. 1169 */ 1170 bool xe_device_is_l2_flush_optimized(struct xe_device *xe) 1171 { 1172 /* XA is *always* flushed, like at the end-of-submssion (and maybe other 1173 * places), just that internally as an optimisation hw doesn't need to make 1174 * that a full flush (which will also include XA) when Media is 1175 * off/powergated, since it doesn't need to worry about GT caches vs Media 1176 * coherency, and only CPU vs GPU coherency, so can make that flush a 1177 * targeted XA flush, since stuff tagged with XA now means it's shared with 1178 * the CPU. The main implication is that we now need to somehow flush non-XA before 1179 * freeing system memory pages, otherwise dirty cachelines could be flushed after the free 1180 * (like if Media suddenly turns on and does a full flush) 1181 */ 1182 if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe)) 1183 return true; 1184 return false; 1185 } 1186 1187 void xe_device_l2_flush(struct xe_device *xe) 1188 { 1189 struct xe_gt *gt; 1190 1191 gt = xe_root_mmio_gt(xe); 1192 if (!gt) 1193 return; 1194 1195 if (!XE_GT_WA(gt, 16023588340)) 1196 return; 1197 1198 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 1199 if (!fw_ref.domains) 1200 return; 1201 1202 spin_lock(>->global_invl_lock); 1203 1204 xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1); 1205 if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true)) 1206 xe_gt_err_once(gt, "Global invalidation timeout\n"); 1207 1208 spin_unlock(>->global_invl_lock); 1209 } 1210 1211 /** 1212 * xe_device_td_flush() - Flush transient L3 cache entries 1213 * @xe: The device 1214 * 1215 * Display engine has direct access to memory and is never coherent with L3/L4 1216 * caches (or CPU caches), however KMD is responsible for specifically flushing 1217 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout 1218 * can happen from such a surface without seeing corruption. 1219 * 1220 * Display surfaces can be tagged as transient by mapping it using one of the 1221 * various L3:XD PAT index modes on Xe2. 1222 * 1223 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed 1224 * at the end of each submission via PIPE_CONTROL for compute/render, since SA 1225 * Media is not coherent with L3 and we want to support render-vs-media 1226 * usescases. For other engines like copy/blt the HW internally forces uncached 1227 * behaviour, hence why we can skip the TDF on such platforms. 1228 */ 1229 void xe_device_td_flush(struct xe_device *xe) 1230 { 1231 struct xe_gt *root_gt; 1232 1233 /* 1234 * From Xe3p onward the HW takes care of flush of TD entries also along 1235 * with flushing XA entries, which will be at the usual sync points, 1236 * like at the end of submission, so no manual flush is needed here. 1237 */ 1238 if (GRAPHICS_VER(xe) >= 35) 1239 return; 1240 1241 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20) 1242 return; 1243 1244 root_gt = xe_root_mmio_gt(xe); 1245 if (!root_gt) 1246 return; 1247 1248 if (XE_GT_WA(root_gt, 16023588340)) { 1249 /* A transient flush is not sufficient: flush the L2 */ 1250 xe_device_l2_flush(xe); 1251 } else { 1252 xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc); 1253 tdf_request_sync(xe); 1254 xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc); 1255 } 1256 } 1257 1258 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) 1259 { 1260 return xe_device_has_flat_ccs(xe) ? 1261 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0; 1262 } 1263 1264 /** 1265 * xe_device_assert_mem_access - Inspect the current runtime_pm state. 1266 * @xe: xe device instance 1267 * 1268 * To be used before any kind of memory access. It will splat a debug warning 1269 * if the device is currently sleeping. But it doesn't guarantee in any way 1270 * that the device is going to remain awake. Xe PM runtime get and put 1271 * functions might be added to the outer bound of the memory access, while 1272 * this check is intended for inner usage to splat some warning if the worst 1273 * case has just happened. 1274 */ 1275 void xe_device_assert_mem_access(struct xe_device *xe) 1276 { 1277 xe_assert(xe, !xe_pm_runtime_suspended(xe)); 1278 } 1279 1280 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p) 1281 { 1282 struct xe_gt *gt; 1283 u8 id; 1284 1285 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid); 1286 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid); 1287 1288 for_each_gt(gt, xe, id) { 1289 drm_printf(p, "GT id: %u\n", id); 1290 drm_printf(p, "\tTile: %u\n", gt->tile->id); 1291 drm_printf(p, "\tType: %s\n", 1292 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media"); 1293 drm_printf(p, "\tIP ver: %u.%u.%u\n", 1294 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid), 1295 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid), 1296 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid)); 1297 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock); 1298 } 1299 } 1300 1301 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address) 1302 { 1303 return sign_extend64(address, xe->info.va_bits - 1); 1304 } 1305 1306 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) 1307 { 1308 return address & GENMASK_ULL(xe->info.va_bits - 1, 0); 1309 } 1310 1311 /** 1312 * DOC: Xe Device Wedging 1313 * 1314 * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. 1315 * When device is in wedged state, every IOCTL will be blocked and GT cannot 1316 * be used. The conditions under which the driver declares the device wedged 1317 * depend on the wedged mode configuration (see &enum xe_wedged_mode). The 1318 * default recovery method for a wedged state is rebind/bus-reset. 1319 * 1320 * Another recovery method is vendor-specific. Below are the cases that send 1321 * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. 1322 * 1323 * Case: Firmware Flash 1324 * -------------------- 1325 * 1326 * Identification Hint 1327 * +++++++++++++++++++ 1328 * 1329 * ``WEDGED=vendor-specific`` drm device wedged uevent with 1330 * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify 1331 * admin/userspace consumer about the need for a firmware flash. 1332 * 1333 * Recovery Procedure 1334 * ++++++++++++++++++ 1335 * 1336 * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow 1337 * the below steps 1338 * 1339 * - Check Runtime Survivability mode sysfs. 1340 * If enabled, firmware flash is required to recover the device. 1341 * 1342 * /sys/bus/pci/devices/<device>/survivability_mode 1343 * 1344 * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash 1345 * firmware and restore device to normal operation. 1346 */ 1347 1348 /** 1349 * xe_device_set_wedged_method - Set wedged recovery method 1350 * @xe: xe device instance 1351 * @method: recovery method to set 1352 * 1353 * Set wedged recovery method to be sent in drm wedged uevent. 1354 */ 1355 void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method) 1356 { 1357 xe->wedged.method = method; 1358 } 1359 1360 /** 1361 * xe_device_declare_wedged - Declare device wedged 1362 * @xe: xe device instance 1363 * 1364 * This is a final state that can only be cleared with the recovery method 1365 * specified in the drm wedged uevent. The method can be set using 1366 * xe_device_set_wedged_method before declaring the device as wedged. If no method 1367 * is set, reprobe (unbind/re-bind) will be sent by default. 1368 * 1369 * In this state every IOCTL will be blocked so the GT cannot be used. 1370 * In general it will be called upon any critical error such as gt reset 1371 * failure or guc loading failure. Userspace will be notified of this state 1372 * through device wedged uevent. 1373 * If xe.wedged module parameter is set to 2, this function will be called 1374 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump 1375 * snapshot capture. In this mode, GT reset won't be attempted so the state of 1376 * the issue is preserved for further debugging. 1377 */ 1378 void xe_device_declare_wedged(struct xe_device *xe) 1379 { 1380 struct xe_gt *gt; 1381 u8 id; 1382 1383 if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) { 1384 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); 1385 return; 1386 } 1387 1388 if (!atomic_xchg(&xe->wedged.flag, 1)) { 1389 xe->needs_flr_on_fini = true; 1390 xe_pm_runtime_get_noresume(xe); 1391 drm_err(&xe->drm, 1392 "CRITICAL: Xe has declared device %s as wedged.\n" 1393 "IOCTLs and executions are blocked.\n" 1394 "For recovery procedure, refer to https://docs.kernel.org/gpu/drm-uapi.html#device-wedging\n" 1395 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n", 1396 dev_name(xe->drm.dev)); 1397 } 1398 1399 for_each_gt(gt, xe, id) 1400 xe_gt_declare_wedged(gt); 1401 1402 if (xe_device_wedged(xe)) { 1403 /* 1404 * XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging 1405 * hangs, so wedge the device with 'none' recovery method and have 1406 * it available to the user for debugging. 1407 */ 1408 if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) 1409 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_NONE); 1410 /* If no wedge recovery method is set, use default */ 1411 else if (!xe->wedged.method) 1412 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND | 1413 DRM_WEDGE_RECOVERY_BUS_RESET); 1414 1415 /* Notify userspace of wedged device */ 1416 drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); 1417 } 1418 } 1419 1420 /** 1421 * xe_device_validate_wedged_mode - Check if given mode is supported 1422 * @xe: the &xe_device 1423 * @mode: requested mode to validate 1424 * 1425 * Check whether the provided wedged mode is supported. 1426 * 1427 * Return: 0 if mode is supported, error code otherwise. 1428 */ 1429 int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode) 1430 { 1431 if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { 1432 drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode); 1433 return -EINVAL; 1434 } else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) || 1435 (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) { 1436 drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n", 1437 mode, xe_wedged_mode_to_string(mode), 1438 xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); 1439 return -EPERM; 1440 } 1441 1442 return 0; 1443 } 1444 1445 /** 1446 * xe_wedged_mode_to_string - Convert enum value to string. 1447 * @mode: the &xe_wedged_mode to convert 1448 * 1449 * Returns: wedged mode as a user friendly string. 1450 */ 1451 const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode) 1452 { 1453 switch (mode) { 1454 case XE_WEDGED_MODE_NEVER: 1455 return "never"; 1456 case XE_WEDGED_MODE_UPON_CRITICAL_ERROR: 1457 return "upon-critical-error"; 1458 case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: 1459 return "upon-any-hang-no-reset"; 1460 default: 1461 return "<invalid>"; 1462 } 1463 } 1464 1465 /** 1466 * xe_device_asid_to_vm() - Find VM from ASID 1467 * @xe: the &xe_device 1468 * @asid: Address space ID 1469 * 1470 * Find a VM from ASID and take a reference to VM which caller must drop. 1471 * Reclaim safe. 1472 * 1473 * Return: VM on success, ERR_PTR on failure 1474 */ 1475 struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid) 1476 { 1477 struct xe_vm *vm; 1478 1479 down_read(&xe->usm.lock); 1480 vm = xa_load(&xe->usm.asid_to_vm, asid); 1481 if (vm) 1482 xe_vm_get(vm); 1483 else 1484 vm = ERR_PTR(-EINVAL); 1485 up_read(&xe->usm.lock); 1486 1487 return vm; 1488 } 1489