1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_device.h" 7 8 #include <linux/aperture.h> 9 #include <linux/delay.h> 10 #include <linux/fault-inject.h> 11 #include <linux/iopoll.h> 12 #include <linux/units.h> 13 14 #include <drm/drm_atomic_helper.h> 15 #include <drm/drm_client.h> 16 #include <drm/drm_gem_ttm_helper.h> 17 #include <drm/drm_ioctl.h> 18 #include <drm/drm_managed.h> 19 #include <drm/drm_print.h> 20 #include <uapi/drm/xe_drm.h> 21 22 #include "display/xe_display.h" 23 #include "instructions/xe_gpu_commands.h" 24 #include "regs/xe_gt_regs.h" 25 #include "regs/xe_regs.h" 26 #include "xe_bo.h" 27 #include "xe_bo_evict.h" 28 #include "xe_debugfs.h" 29 #include "xe_devcoredump.h" 30 #include "xe_device_sysfs.h" 31 #include "xe_dma_buf.h" 32 #include "xe_drm_client.h" 33 #include "xe_drv.h" 34 #include "xe_exec.h" 35 #include "xe_exec_queue.h" 36 #include "xe_force_wake.h" 37 #include "xe_ggtt.h" 38 #include "xe_gsc_proxy.h" 39 #include "xe_gt.h" 40 #include "xe_gt_mcr.h" 41 #include "xe_gt_printk.h" 42 #include "xe_gt_sriov_vf.h" 43 #include "xe_guc.h" 44 #include "xe_guc_pc.h" 45 #include "xe_hw_engine_group.h" 46 #include "xe_hwmon.h" 47 #include "xe_i2c.h" 48 #include "xe_irq.h" 49 #include "xe_late_bind_fw.h" 50 #include "xe_mmio.h" 51 #include "xe_module.h" 52 #include "xe_nvm.h" 53 #include "xe_oa.h" 54 #include "xe_observation.h" 55 #include "xe_pat.h" 56 #include "xe_pcode.h" 57 #include "xe_pm.h" 58 #include "xe_pmu.h" 59 #include "xe_psmi.h" 60 #include "xe_pxp.h" 61 #include "xe_query.h" 62 #include "xe_shrinker.h" 63 #include "xe_survivability_mode.h" 64 #include "xe_sriov.h" 65 #include "xe_tile.h" 66 #include "xe_ttm_stolen_mgr.h" 67 #include "xe_ttm_sys_mgr.h" 68 #include "xe_vm.h" 69 #include "xe_vm_madvise.h" 70 #include "xe_vram.h" 71 #include "xe_vram_types.h" 72 #include "xe_vsec.h" 73 #include "xe_wait_user_fence.h" 74 #include "xe_wa.h" 75 76 #include <generated/xe_device_wa_oob.h> 77 #include <generated/xe_wa_oob.h> 78 79 static int xe_file_open(struct drm_device *dev, struct drm_file *file) 80 { 81 struct xe_device *xe = to_xe_device(dev); 82 struct xe_drm_client *client; 83 struct xe_file *xef; 84 int ret = -ENOMEM; 85 struct task_struct *task = NULL; 86 87 xef = kzalloc(sizeof(*xef), GFP_KERNEL); 88 if (!xef) 89 return ret; 90 91 client = xe_drm_client_alloc(); 92 if (!client) { 93 kfree(xef); 94 return ret; 95 } 96 97 xef->drm = file; 98 xef->client = client; 99 xef->xe = xe; 100 101 mutex_init(&xef->vm.lock); 102 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); 103 104 mutex_init(&xef->exec_queue.lock); 105 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); 106 107 file->driver_priv = xef; 108 kref_init(&xef->refcount); 109 110 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID); 111 if (task) { 112 xef->process_name = kstrdup(task->comm, GFP_KERNEL); 113 xef->pid = task->pid; 114 put_task_struct(task); 115 } 116 117 return 0; 118 } 119 120 static void xe_file_destroy(struct kref *ref) 121 { 122 struct xe_file *xef = container_of(ref, struct xe_file, refcount); 123 124 xa_destroy(&xef->exec_queue.xa); 125 mutex_destroy(&xef->exec_queue.lock); 126 xa_destroy(&xef->vm.xa); 127 mutex_destroy(&xef->vm.lock); 128 129 xe_drm_client_put(xef->client); 130 kfree(xef->process_name); 131 kfree(xef); 132 } 133 134 /** 135 * xe_file_get() - Take a reference to the xe file object 136 * @xef: Pointer to the xe file 137 * 138 * Anyone with a pointer to xef must take a reference to the xe file 139 * object using this call. 140 * 141 * Return: xe file pointer 142 */ 143 struct xe_file *xe_file_get(struct xe_file *xef) 144 { 145 kref_get(&xef->refcount); 146 return xef; 147 } 148 149 /** 150 * xe_file_put() - Drop a reference to the xe file object 151 * @xef: Pointer to the xe file 152 * 153 * Used to drop reference to the xef object 154 */ 155 void xe_file_put(struct xe_file *xef) 156 { 157 kref_put(&xef->refcount, xe_file_destroy); 158 } 159 160 static void xe_file_close(struct drm_device *dev, struct drm_file *file) 161 { 162 struct xe_device *xe = to_xe_device(dev); 163 struct xe_file *xef = file->driver_priv; 164 struct xe_vm *vm; 165 struct xe_exec_queue *q; 166 unsigned long idx; 167 168 xe_pm_runtime_get(xe); 169 170 /* 171 * No need for exec_queue.lock here as there is no contention for it 172 * when FD is closing as IOCTLs presumably can't be modifying the 173 * xarray. Taking exec_queue.lock here causes undue dependency on 174 * vm->lock taken during xe_exec_queue_kill(). 175 */ 176 xa_for_each(&xef->exec_queue.xa, idx, q) { 177 if (q->vm && q->hwe->hw_engine_group) 178 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 179 xe_exec_queue_kill(q); 180 xe_exec_queue_put(q); 181 } 182 xa_for_each(&xef->vm.xa, idx, vm) 183 xe_vm_close_and_put(vm); 184 185 xe_file_put(xef); 186 187 xe_pm_runtime_put(xe); 188 } 189 190 static const struct drm_ioctl_desc xe_ioctls[] = { 191 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 192 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), 193 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, 194 DRM_RENDER_ALLOW), 195 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), 196 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), 197 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), 198 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), 199 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, 200 DRM_RENDER_ALLOW), 201 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, 202 DRM_RENDER_ALLOW), 203 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, 204 DRM_RENDER_ALLOW), 205 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, 206 DRM_RENDER_ALLOW), 207 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 208 DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), 209 DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, 210 DRM_RENDER_ALLOW), 211 }; 212 213 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 214 { 215 struct drm_file *file_priv = file->private_data; 216 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 217 long ret; 218 219 if (xe_device_wedged(xe)) 220 return -ECANCELED; 221 222 ret = xe_pm_runtime_get_ioctl(xe); 223 if (ret >= 0) 224 ret = drm_ioctl(file, cmd, arg); 225 xe_pm_runtime_put(xe); 226 227 return ret; 228 } 229 230 #ifdef CONFIG_COMPAT 231 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 232 { 233 struct drm_file *file_priv = file->private_data; 234 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 235 long ret; 236 237 if (xe_device_wedged(xe)) 238 return -ECANCELED; 239 240 ret = xe_pm_runtime_get_ioctl(xe); 241 if (ret >= 0) 242 ret = drm_compat_ioctl(file, cmd, arg); 243 xe_pm_runtime_put(xe); 244 245 return ret; 246 } 247 #else 248 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */ 249 #define xe_drm_compat_ioctl NULL 250 #endif 251 252 static void barrier_open(struct vm_area_struct *vma) 253 { 254 drm_dev_get(vma->vm_private_data); 255 } 256 257 static void barrier_close(struct vm_area_struct *vma) 258 { 259 drm_dev_put(vma->vm_private_data); 260 } 261 262 static void barrier_release_dummy_page(struct drm_device *dev, void *res) 263 { 264 struct page *dummy_page = (struct page *)res; 265 266 __free_page(dummy_page); 267 } 268 269 static vm_fault_t barrier_fault(struct vm_fault *vmf) 270 { 271 struct drm_device *dev = vmf->vma->vm_private_data; 272 struct vm_area_struct *vma = vmf->vma; 273 vm_fault_t ret = VM_FAULT_NOPAGE; 274 pgprot_t prot; 275 int idx; 276 277 prot = vm_get_page_prot(vma->vm_flags); 278 279 if (drm_dev_enter(dev, &idx)) { 280 unsigned long pfn; 281 282 #define LAST_DB_PAGE_OFFSET 0x7ff001 283 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) + 284 LAST_DB_PAGE_OFFSET); 285 ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn, 286 pgprot_noncached(prot)); 287 drm_dev_exit(idx); 288 } else { 289 struct page *page; 290 291 /* Allocate new dummy page to map all the VA range in this VMA to it*/ 292 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 293 if (!page) 294 return VM_FAULT_OOM; 295 296 /* Set the page to be freed using drmm release action */ 297 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page)) 298 return VM_FAULT_OOM; 299 300 ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page), 301 prot); 302 } 303 304 return ret; 305 } 306 307 static const struct vm_operations_struct vm_ops_barrier = { 308 .open = barrier_open, 309 .close = barrier_close, 310 .fault = barrier_fault, 311 }; 312 313 static int xe_pci_barrier_mmap(struct file *filp, 314 struct vm_area_struct *vma) 315 { 316 struct drm_file *priv = filp->private_data; 317 struct drm_device *dev = priv->minor->dev; 318 struct xe_device *xe = to_xe_device(dev); 319 320 if (!IS_DGFX(xe)) 321 return -EINVAL; 322 323 if (vma->vm_end - vma->vm_start > SZ_4K) 324 return -EINVAL; 325 326 if (is_cow_mapping(vma->vm_flags)) 327 return -EINVAL; 328 329 if (vma->vm_flags & (VM_READ | VM_EXEC)) 330 return -EINVAL; 331 332 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC); 333 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); 334 vma->vm_ops = &vm_ops_barrier; 335 vma->vm_private_data = dev; 336 drm_dev_get(vma->vm_private_data); 337 338 return 0; 339 } 340 341 static int xe_mmap(struct file *filp, struct vm_area_struct *vma) 342 { 343 struct drm_file *priv = filp->private_data; 344 struct drm_device *dev = priv->minor->dev; 345 346 if (drm_dev_is_unplugged(dev)) 347 return -ENODEV; 348 349 switch (vma->vm_pgoff) { 350 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT: 351 return xe_pci_barrier_mmap(filp, vma); 352 } 353 354 return drm_gem_mmap(filp, vma); 355 } 356 357 static const struct file_operations xe_driver_fops = { 358 .owner = THIS_MODULE, 359 .open = drm_open, 360 .release = drm_release_noglobal, 361 .unlocked_ioctl = xe_drm_ioctl, 362 .mmap = xe_mmap, 363 .poll = drm_poll, 364 .read = drm_read, 365 .compat_ioctl = xe_drm_compat_ioctl, 366 .llseek = noop_llseek, 367 #ifdef CONFIG_PROC_FS 368 .show_fdinfo = drm_show_fdinfo, 369 #endif 370 .fop_flags = FOP_UNSIGNED_OFFSET, 371 }; 372 373 static struct drm_driver driver = { 374 /* Don't use MTRRs here; the Xserver or userspace app should 375 * deal with them for Intel hardware. 376 */ 377 .driver_features = 378 DRIVER_GEM | 379 DRIVER_RENDER | DRIVER_SYNCOBJ | 380 DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, 381 .open = xe_file_open, 382 .postclose = xe_file_close, 383 384 .gem_prime_import = xe_gem_prime_import, 385 386 .dumb_create = xe_bo_dumb_create, 387 .dumb_map_offset = drm_gem_ttm_dumb_map_offset, 388 #ifdef CONFIG_PROC_FS 389 .show_fdinfo = xe_drm_client_fdinfo, 390 #endif 391 .ioctls = xe_ioctls, 392 .num_ioctls = ARRAY_SIZE(xe_ioctls), 393 .fops = &xe_driver_fops, 394 .name = DRIVER_NAME, 395 .desc = DRIVER_DESC, 396 .major = DRIVER_MAJOR, 397 .minor = DRIVER_MINOR, 398 .patchlevel = DRIVER_PATCHLEVEL, 399 }; 400 401 static void xe_device_destroy(struct drm_device *dev, void *dummy) 402 { 403 struct xe_device *xe = to_xe_device(dev); 404 405 xe_bo_dev_fini(&xe->bo_device); 406 407 if (xe->preempt_fence_wq) 408 destroy_workqueue(xe->preempt_fence_wq); 409 410 if (xe->ordered_wq) 411 destroy_workqueue(xe->ordered_wq); 412 413 if (xe->unordered_wq) 414 destroy_workqueue(xe->unordered_wq); 415 416 if (xe->destroy_wq) 417 destroy_workqueue(xe->destroy_wq); 418 419 ttm_device_fini(&xe->ttm); 420 } 421 422 struct xe_device *xe_device_create(struct pci_dev *pdev, 423 const struct pci_device_id *ent) 424 { 425 struct xe_device *xe; 426 int err; 427 428 xe_display_driver_set_hooks(&driver); 429 430 err = aperture_remove_conflicting_pci_devices(pdev, driver.name); 431 if (err) 432 return ERR_PTR(err); 433 434 xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm); 435 if (IS_ERR(xe)) 436 return xe; 437 438 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, 439 xe->drm.anon_inode->i_mapping, 440 xe->drm.vma_offset_manager, false, false); 441 if (WARN_ON(err)) 442 goto err; 443 444 xe_bo_dev_init(&xe->bo_device); 445 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); 446 if (err) 447 goto err; 448 449 err = xe_shrinker_create(xe); 450 if (err) 451 goto err; 452 453 xe->info.devid = pdev->device; 454 xe->info.revid = pdev->revision; 455 xe->info.force_execlist = xe_modparam.force_execlist; 456 xe->atomic_svm_timeslice_ms = 5; 457 458 err = xe_irq_init(xe); 459 if (err) 460 goto err; 461 462 xe_validation_device_init(&xe->val); 463 464 init_waitqueue_head(&xe->ufence_wq); 465 466 init_rwsem(&xe->usm.lock); 467 468 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); 469 470 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 471 /* Trigger a large asid and an early asid wrap. */ 472 u32 asid; 473 474 BUILD_BUG_ON(XE_MAX_ASID < 2); 475 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL, 476 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1), 477 &xe->usm.next_asid, GFP_KERNEL); 478 drm_WARN_ON(&xe->drm, err); 479 if (err >= 0) 480 xa_erase(&xe->usm.asid_to_vm, asid); 481 } 482 483 err = xe_bo_pinned_init(xe); 484 if (err) 485 goto err; 486 487 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", 488 WQ_MEM_RECLAIM); 489 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); 490 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0); 491 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0); 492 if (!xe->ordered_wq || !xe->unordered_wq || 493 !xe->preempt_fence_wq || !xe->destroy_wq) { 494 /* 495 * Cleanup done in xe_device_destroy via 496 * drmm_add_action_or_reset register above 497 */ 498 drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); 499 err = -ENOMEM; 500 goto err; 501 } 502 503 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); 504 if (err) 505 goto err; 506 507 return xe; 508 509 err: 510 return ERR_PTR(err); 511 } 512 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ 513 514 static bool xe_driver_flr_disabled(struct xe_device *xe) 515 { 516 if (IS_SRIOV_VF(xe)) 517 return true; 518 519 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) { 520 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n"); 521 return true; 522 } 523 524 return false; 525 } 526 527 /* 528 * The driver-initiated FLR is the highest level of reset that we can trigger 529 * from within the driver. It is different from the PCI FLR in that it doesn't 530 * fully reset the SGUnit and doesn't modify the PCI config space and therefore 531 * it doesn't require a re-enumeration of the PCI BARs. However, the 532 * driver-initiated FLR does still cause a reset of both GT and display and a 533 * memory wipe of local and stolen memory, so recovery would require a full HW 534 * re-init and saving/restoring (or re-populating) the wiped memory. Since we 535 * perform the FLR as the very last action before releasing access to the HW 536 * during the driver release flow, we don't attempt recovery at all, because 537 * if/when a new instance of Xe is bound to the device it will do a full 538 * re-init anyway. 539 */ 540 static void __xe_driver_flr(struct xe_device *xe) 541 { 542 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */ 543 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 544 int ret; 545 546 drm_dbg(&xe->drm, "Triggering Driver-FLR\n"); 547 548 /* 549 * Make sure any pending FLR requests have cleared by waiting for the 550 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS 551 * to make sure it's not still set from a prior attempt (it's a write to 552 * clear bit). 553 * Note that we should never be in a situation where a previous attempt 554 * is still pending (unless the HW is totally dead), but better to be 555 * safe in case something unexpected happens 556 */ 557 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 558 if (ret) { 559 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret); 560 return; 561 } 562 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 563 564 /* Trigger the actual Driver-FLR */ 565 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR); 566 567 /* Wait for hardware teardown to complete */ 568 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 569 if (ret) { 570 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); 571 return; 572 } 573 574 /* Wait for hardware/firmware re-init to complete */ 575 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS, 576 flr_timeout, NULL, false); 577 if (ret) { 578 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); 579 return; 580 } 581 582 /* Clear sticky completion status */ 583 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 584 } 585 586 static void xe_driver_flr(struct xe_device *xe) 587 { 588 if (xe_driver_flr_disabled(xe)) 589 return; 590 591 __xe_driver_flr(xe); 592 } 593 594 static void xe_driver_flr_fini(void *arg) 595 { 596 struct xe_device *xe = arg; 597 598 if (xe->needs_flr_on_fini) 599 xe_driver_flr(xe); 600 } 601 602 static void xe_device_sanitize(void *arg) 603 { 604 struct xe_device *xe = arg; 605 struct xe_gt *gt; 606 u8 id; 607 608 for_each_gt(gt, xe, id) 609 xe_gt_sanitize(gt); 610 } 611 612 static int xe_set_dma_info(struct xe_device *xe) 613 { 614 unsigned int mask_size = xe->info.dma_mask_size; 615 int err; 616 617 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev)); 618 619 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 620 if (err) 621 goto mask_err; 622 623 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 624 if (err) 625 goto mask_err; 626 627 return 0; 628 629 mask_err: 630 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); 631 return err; 632 } 633 634 static int lmem_initializing(struct xe_device *xe) 635 { 636 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT) 637 return 0; 638 639 if (signal_pending(current)) 640 return -EINTR; 641 642 return 1; 643 } 644 645 static int wait_for_lmem_ready(struct xe_device *xe) 646 { 647 const unsigned long TIMEOUT_SEC = 60; 648 unsigned long prev_jiffies; 649 int initializing; 650 651 if (!IS_DGFX(xe)) 652 return 0; 653 654 if (IS_SRIOV_VF(xe)) 655 return 0; 656 657 if (!lmem_initializing(xe)) 658 return 0; 659 660 drm_dbg(&xe->drm, "Waiting for lmem initialization\n"); 661 prev_jiffies = jiffies; 662 663 /* 664 * The boot firmware initializes local memory and 665 * assesses its health. If memory training fails, 666 * the punit will have been instructed to keep the GT powered 667 * down.we won't be able to communicate with it 668 * 669 * If the status check is done before punit updates the register, 670 * it can lead to the system being unusable. 671 * use a timeout and defer the probe to prevent this. 672 */ 673 poll_timeout_us(initializing = lmem_initializing(xe), 674 initializing <= 0, 675 20 * USEC_PER_MSEC, TIMEOUT_SEC * USEC_PER_SEC, true); 676 if (initializing < 0) 677 return initializing; 678 679 if (initializing) { 680 drm_dbg(&xe->drm, "lmem not initialized by firmware\n"); 681 return -EPROBE_DEFER; 682 } 683 684 drm_dbg(&xe->drm, "lmem ready after %ums", 685 jiffies_to_msecs(jiffies - prev_jiffies)); 686 687 return 0; 688 } 689 ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */ 690 691 static void vf_update_device_info(struct xe_device *xe) 692 { 693 xe_assert(xe, IS_SRIOV_VF(xe)); 694 /* disable features that are not available/applicable to VFs */ 695 xe->info.probe_display = 0; 696 xe->info.has_heci_cscfi = 0; 697 xe->info.has_heci_gscfi = 0; 698 xe->info.has_late_bind = 0; 699 xe->info.skip_guc_pc = 1; 700 xe->info.skip_pcode = 1; 701 } 702 703 static int xe_device_vram_alloc(struct xe_device *xe) 704 { 705 struct xe_vram_region *vram; 706 707 if (!IS_DGFX(xe)) 708 return 0; 709 710 vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); 711 if (!vram) 712 return -ENOMEM; 713 714 xe->mem.vram = vram; 715 return 0; 716 } 717 718 /** 719 * xe_device_probe_early: Device early probe 720 * @xe: xe device instance 721 * 722 * Initialize MMIO resources that don't require any 723 * knowledge about tile count. Also initialize pcode and 724 * check vram initialization on root tile. 725 * 726 * Return: 0 on success, error code on failure 727 */ 728 int xe_device_probe_early(struct xe_device *xe) 729 { 730 int err; 731 732 xe_wa_device_init(xe); 733 xe_wa_process_device_oob(xe); 734 735 err = xe_mmio_probe_early(xe); 736 if (err) 737 return err; 738 739 xe_sriov_probe_early(xe); 740 741 if (IS_SRIOV_VF(xe)) 742 vf_update_device_info(xe); 743 744 err = xe_pcode_probe_early(xe); 745 if (err || xe_survivability_mode_is_requested(xe)) { 746 int save_err = err; 747 748 /* 749 * Try to leave device in survivability mode if device is 750 * possible, but still return the previous error for error 751 * propagation 752 */ 753 err = xe_survivability_mode_boot_enable(xe); 754 if (err) 755 return err; 756 757 return save_err; 758 } 759 760 err = wait_for_lmem_ready(xe); 761 if (err) 762 return err; 763 764 xe->wedged.mode = xe_modparam.wedged_mode; 765 766 err = xe_device_vram_alloc(xe); 767 if (err) 768 return err; 769 770 return 0; 771 } 772 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ 773 774 static int probe_has_flat_ccs(struct xe_device *xe) 775 { 776 struct xe_gt *gt; 777 unsigned int fw_ref; 778 u32 reg; 779 780 /* Always enabled/disabled, no runtime check to do */ 781 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe)) 782 return 0; 783 784 gt = xe_root_mmio_gt(xe); 785 if (!gt) 786 return 0; 787 788 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 789 if (!fw_ref) 790 return -ETIMEDOUT; 791 792 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 793 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE); 794 795 if (!xe->info.has_flat_ccs) 796 drm_dbg(&xe->drm, 797 "Flat CCS has been disabled in bios, May lead to performance impact"); 798 799 xe_force_wake_put(gt_to_fw(gt), fw_ref); 800 801 return 0; 802 } 803 804 int xe_device_probe(struct xe_device *xe) 805 { 806 struct xe_tile *tile; 807 struct xe_gt *gt; 808 int err; 809 u8 id; 810 811 xe_pat_init_early(xe); 812 813 err = xe_sriov_init(xe); 814 if (err) 815 return err; 816 817 xe->info.mem_region_mask = 1; 818 819 err = xe_set_dma_info(xe); 820 if (err) 821 return err; 822 823 err = xe_mmio_probe_tiles(xe); 824 if (err) 825 return err; 826 827 for_each_gt(gt, xe, id) { 828 err = xe_gt_init_early(gt); 829 if (err) 830 return err; 831 } 832 833 for_each_tile(tile, xe, id) { 834 err = xe_ggtt_init_early(tile->mem.ggtt); 835 if (err) 836 return err; 837 } 838 839 /* 840 * From here on, if a step fails, make sure a Driver-FLR is triggereed 841 */ 842 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe); 843 if (err) 844 return err; 845 846 err = probe_has_flat_ccs(xe); 847 if (err) 848 return err; 849 850 err = xe_vram_probe(xe); 851 if (err) 852 return err; 853 854 for_each_tile(tile, xe, id) { 855 err = xe_tile_init_noalloc(tile); 856 if (err) 857 return err; 858 } 859 860 /* 861 * Allow allocations only now to ensure xe_display_init_early() 862 * is the first to allocate, always. 863 */ 864 err = xe_ttm_sys_mgr_init(xe); 865 if (err) 866 return err; 867 868 /* Allocate and map stolen after potential VRAM resize */ 869 err = xe_ttm_stolen_mgr_init(xe); 870 if (err) 871 return err; 872 873 /* 874 * Now that GT is initialized (TTM in particular), 875 * we can try to init display, and inherit the initial fb. 876 * This is the reason the first allocation needs to be done 877 * inside display. 878 */ 879 err = xe_display_init_early(xe); 880 if (err) 881 return err; 882 883 for_each_tile(tile, xe, id) { 884 err = xe_tile_init(tile); 885 if (err) 886 return err; 887 } 888 889 err = xe_irq_install(xe); 890 if (err) 891 return err; 892 893 for_each_gt(gt, xe, id) { 894 err = xe_gt_init(gt); 895 if (err) 896 return err; 897 } 898 899 if (xe->tiles->media_gt && 900 XE_GT_WA(xe->tiles->media_gt, 15015404425_disable)) 901 XE_DEVICE_WA_DISABLE(xe, 15015404425); 902 903 err = xe_devcoredump_init(xe); 904 if (err) 905 return err; 906 907 xe_nvm_init(xe); 908 909 err = xe_heci_gsc_init(xe); 910 if (err) 911 return err; 912 913 err = xe_late_bind_init(&xe->late_bind); 914 if (err) 915 return err; 916 917 err = xe_oa_init(xe); 918 if (err) 919 return err; 920 921 err = xe_display_init(xe); 922 if (err) 923 return err; 924 925 err = xe_pxp_init(xe); 926 if (err) 927 return err; 928 929 err = xe_psmi_init(xe); 930 if (err) 931 return err; 932 933 err = drm_dev_register(&xe->drm, 0); 934 if (err) 935 return err; 936 937 xe_display_register(xe); 938 939 err = xe_oa_register(xe); 940 if (err) 941 goto err_unregister_display; 942 943 err = xe_pmu_register(&xe->pmu); 944 if (err) 945 goto err_unregister_display; 946 947 err = xe_device_sysfs_init(xe); 948 if (err) 949 goto err_unregister_display; 950 951 xe_debugfs_register(xe); 952 953 err = xe_hwmon_register(xe); 954 if (err) 955 goto err_unregister_display; 956 957 err = xe_i2c_probe(xe); 958 if (err) 959 goto err_unregister_display; 960 961 for_each_gt(gt, xe, id) 962 xe_gt_sanitize_freq(gt); 963 964 xe_vsec_init(xe); 965 966 err = xe_sriov_init_late(xe); 967 if (err) 968 goto err_unregister_display; 969 970 return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); 971 972 err_unregister_display: 973 xe_display_unregister(xe); 974 975 return err; 976 } 977 978 void xe_device_remove(struct xe_device *xe) 979 { 980 xe_display_unregister(xe); 981 982 xe_nvm_fini(xe); 983 984 drm_dev_unplug(&xe->drm); 985 986 xe_bo_pci_dev_remove_all(xe); 987 } 988 989 void xe_device_shutdown(struct xe_device *xe) 990 { 991 drm_dbg(&xe->drm, "Shutting down device\n"); 992 993 if (xe_driver_flr_disabled(xe)) { 994 struct xe_gt *gt; 995 u8 id; 996 997 xe_display_pm_shutdown(xe); 998 999 xe_irq_suspend(xe); 1000 1001 for_each_gt(gt, xe, id) 1002 xe_gt_shutdown(gt); 1003 1004 xe_display_pm_shutdown_late(xe); 1005 } else { 1006 /* BOOM! */ 1007 __xe_driver_flr(xe); 1008 } 1009 } 1010 1011 /** 1012 * xe_device_wmb() - Device specific write memory barrier 1013 * @xe: the &xe_device 1014 * 1015 * While wmb() is sufficient for a barrier if we use system memory, on discrete 1016 * platforms with device memory we additionally need to issue a register write. 1017 * Since it doesn't matter which register we write to, use the read-only VF_CAP 1018 * register that is also marked as accessible by the VFs. 1019 */ 1020 void xe_device_wmb(struct xe_device *xe) 1021 { 1022 wmb(); 1023 if (IS_DGFX(xe)) 1024 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0); 1025 } 1026 1027 /* 1028 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt. 1029 */ 1030 static void tdf_request_sync(struct xe_device *xe) 1031 { 1032 unsigned int fw_ref; 1033 struct xe_gt *gt; 1034 u8 id; 1035 1036 for_each_gt(gt, xe, id) { 1037 if (xe_gt_is_media_type(gt)) 1038 continue; 1039 1040 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 1041 if (!fw_ref) 1042 return; 1043 1044 xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); 1045 1046 /* 1047 * FIXME: We can likely do better here with our choice of 1048 * timeout. Currently we just assume the worst case, i.e. 150us, 1049 * which is believed to be sufficient to cover the worst case 1050 * scenario on current platforms if all cache entries are 1051 * transient and need to be flushed.. 1052 */ 1053 if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 1054 150, NULL, false)) 1055 xe_gt_err_once(gt, "TD flush timeout\n"); 1056 1057 xe_force_wake_put(gt_to_fw(gt), fw_ref); 1058 } 1059 } 1060 1061 void xe_device_l2_flush(struct xe_device *xe) 1062 { 1063 struct xe_gt *gt; 1064 unsigned int fw_ref; 1065 1066 gt = xe_root_mmio_gt(xe); 1067 if (!gt) 1068 return; 1069 1070 if (!XE_GT_WA(gt, 16023588340)) 1071 return; 1072 1073 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 1074 if (!fw_ref) 1075 return; 1076 1077 spin_lock(>->global_invl_lock); 1078 1079 xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1); 1080 if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true)) 1081 xe_gt_err_once(gt, "Global invalidation timeout\n"); 1082 1083 spin_unlock(>->global_invl_lock); 1084 1085 xe_force_wake_put(gt_to_fw(gt), fw_ref); 1086 } 1087 1088 /** 1089 * xe_device_td_flush() - Flush transient L3 cache entries 1090 * @xe: The device 1091 * 1092 * Display engine has direct access to memory and is never coherent with L3/L4 1093 * caches (or CPU caches), however KMD is responsible for specifically flushing 1094 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout 1095 * can happen from such a surface without seeing corruption. 1096 * 1097 * Display surfaces can be tagged as transient by mapping it using one of the 1098 * various L3:XD PAT index modes on Xe2. 1099 * 1100 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed 1101 * at the end of each submission via PIPE_CONTROL for compute/render, since SA 1102 * Media is not coherent with L3 and we want to support render-vs-media 1103 * usescases. For other engines like copy/blt the HW internally forces uncached 1104 * behaviour, hence why we can skip the TDF on such platforms. 1105 */ 1106 void xe_device_td_flush(struct xe_device *xe) 1107 { 1108 struct xe_gt *root_gt; 1109 1110 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20) 1111 return; 1112 1113 root_gt = xe_root_mmio_gt(xe); 1114 if (!root_gt) 1115 return; 1116 1117 if (XE_GT_WA(root_gt, 16023588340)) { 1118 /* A transient flush is not sufficient: flush the L2 */ 1119 xe_device_l2_flush(xe); 1120 } else { 1121 xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc); 1122 tdf_request_sync(xe); 1123 xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc); 1124 } 1125 } 1126 1127 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) 1128 { 1129 return xe_device_has_flat_ccs(xe) ? 1130 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0; 1131 } 1132 1133 /** 1134 * xe_device_assert_mem_access - Inspect the current runtime_pm state. 1135 * @xe: xe device instance 1136 * 1137 * To be used before any kind of memory access. It will splat a debug warning 1138 * if the device is currently sleeping. But it doesn't guarantee in any way 1139 * that the device is going to remain awake. Xe PM runtime get and put 1140 * functions might be added to the outer bound of the memory access, while 1141 * this check is intended for inner usage to splat some warning if the worst 1142 * case has just happened. 1143 */ 1144 void xe_device_assert_mem_access(struct xe_device *xe) 1145 { 1146 xe_assert(xe, !xe_pm_runtime_suspended(xe)); 1147 } 1148 1149 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p) 1150 { 1151 struct xe_gt *gt; 1152 u8 id; 1153 1154 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid); 1155 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid); 1156 1157 for_each_gt(gt, xe, id) { 1158 drm_printf(p, "GT id: %u\n", id); 1159 drm_printf(p, "\tTile: %u\n", gt->tile->id); 1160 drm_printf(p, "\tType: %s\n", 1161 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media"); 1162 drm_printf(p, "\tIP ver: %u.%u.%u\n", 1163 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid), 1164 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid), 1165 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid)); 1166 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock); 1167 } 1168 } 1169 1170 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address) 1171 { 1172 return sign_extend64(address, xe->info.va_bits - 1); 1173 } 1174 1175 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) 1176 { 1177 return address & GENMASK_ULL(xe->info.va_bits - 1, 0); 1178 } 1179 1180 static void xe_device_wedged_fini(struct drm_device *drm, void *arg) 1181 { 1182 struct xe_device *xe = arg; 1183 1184 xe_pm_runtime_put(xe); 1185 } 1186 1187 /** 1188 * DOC: Xe Device Wedging 1189 * 1190 * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. 1191 * When device is in wedged state, every IOCTL will be blocked and GT cannot be 1192 * used. Certain critical errors like gt reset failure, firmware failures can cause 1193 * the device to be wedged. The default recovery method for a wedged state 1194 * is rebind/bus-reset. 1195 * 1196 * Another recovery method is vendor-specific. Below are the cases that send 1197 * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. 1198 * 1199 * Case: Firmware Flash 1200 * -------------------- 1201 * 1202 * Identification Hint 1203 * +++++++++++++++++++ 1204 * 1205 * ``WEDGED=vendor-specific`` drm device wedged uevent with 1206 * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify 1207 * admin/userspace consumer about the need for a firmware flash. 1208 * 1209 * Recovery Procedure 1210 * ++++++++++++++++++ 1211 * 1212 * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow 1213 * the below steps 1214 * 1215 * - Check Runtime Survivability mode sysfs. 1216 * If enabled, firmware flash is required to recover the device. 1217 * 1218 * /sys/bus/pci/devices/<device>/survivability_mode 1219 * 1220 * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash 1221 * firmware and restore device to normal operation. 1222 */ 1223 1224 /** 1225 * xe_device_set_wedged_method - Set wedged recovery method 1226 * @xe: xe device instance 1227 * @method: recovery method to set 1228 * 1229 * Set wedged recovery method to be sent in drm wedged uevent. 1230 */ 1231 void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method) 1232 { 1233 xe->wedged.method = method; 1234 } 1235 1236 /** 1237 * xe_device_declare_wedged - Declare device wedged 1238 * @xe: xe device instance 1239 * 1240 * This is a final state that can only be cleared with the recovery method 1241 * specified in the drm wedged uevent. The method can be set using 1242 * xe_device_set_wedged_method before declaring the device as wedged. If no method 1243 * is set, reprobe (unbind/re-bind) will be sent by default. 1244 * 1245 * In this state every IOCTL will be blocked so the GT cannot be used. 1246 * In general it will be called upon any critical error such as gt reset 1247 * failure or guc loading failure. Userspace will be notified of this state 1248 * through device wedged uevent. 1249 * If xe.wedged module parameter is set to 2, this function will be called 1250 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump 1251 * snapshot capture. In this mode, GT reset won't be attempted so the state of 1252 * the issue is preserved for further debugging. 1253 */ 1254 void xe_device_declare_wedged(struct xe_device *xe) 1255 { 1256 struct xe_gt *gt; 1257 u8 id; 1258 1259 if (xe->wedged.mode == 0) { 1260 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); 1261 return; 1262 } 1263 1264 xe_pm_runtime_get_noresume(xe); 1265 1266 if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) { 1267 drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n"); 1268 return; 1269 } 1270 1271 if (!atomic_xchg(&xe->wedged.flag, 1)) { 1272 xe->needs_flr_on_fini = true; 1273 drm_err(&xe->drm, 1274 "CRITICAL: Xe has declared device %s as wedged.\n" 1275 "IOCTLs and executions are blocked. Only a rebind may clear the failure\n" 1276 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n", 1277 dev_name(xe->drm.dev)); 1278 } 1279 1280 for_each_gt(gt, xe, id) 1281 xe_gt_declare_wedged(gt); 1282 1283 if (xe_device_wedged(xe)) { 1284 /* If no wedge recovery method is set, use default */ 1285 if (!xe->wedged.method) 1286 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND | 1287 DRM_WEDGE_RECOVERY_BUS_RESET); 1288 1289 /* Notify userspace of wedged device */ 1290 drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); 1291 } 1292 } 1293