1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_device.h" 7 8 #include <linux/aperture.h> 9 #include <linux/delay.h> 10 #include <linux/fault-inject.h> 11 #include <linux/units.h> 12 13 #include <drm/drm_atomic_helper.h> 14 #include <drm/drm_client.h> 15 #include <drm/drm_gem_ttm_helper.h> 16 #include <drm/drm_ioctl.h> 17 #include <drm/drm_managed.h> 18 #include <drm/drm_print.h> 19 #include <uapi/drm/xe_drm.h> 20 21 #include "display/xe_display.h" 22 #include "instructions/xe_gpu_commands.h" 23 #include "regs/xe_gt_regs.h" 24 #include "regs/xe_regs.h" 25 #include "xe_bo.h" 26 #include "xe_bo_evict.h" 27 #include "xe_debugfs.h" 28 #include "xe_devcoredump.h" 29 #include "xe_device_sysfs.h" 30 #include "xe_dma_buf.h" 31 #include "xe_drm_client.h" 32 #include "xe_drv.h" 33 #include "xe_exec.h" 34 #include "xe_exec_queue.h" 35 #include "xe_force_wake.h" 36 #include "xe_ggtt.h" 37 #include "xe_gsc_proxy.h" 38 #include "xe_gt.h" 39 #include "xe_gt_mcr.h" 40 #include "xe_gt_printk.h" 41 #include "xe_gt_sriov_vf.h" 42 #include "xe_guc.h" 43 #include "xe_guc_pc.h" 44 #include "xe_hw_engine_group.h" 45 #include "xe_hwmon.h" 46 #include "xe_i2c.h" 47 #include "xe_irq.h" 48 #include "xe_mmio.h" 49 #include "xe_module.h" 50 #include "xe_nvm.h" 51 #include "xe_oa.h" 52 #include "xe_observation.h" 53 #include "xe_pat.h" 54 #include "xe_pcode.h" 55 #include "xe_pm.h" 56 #include "xe_pmu.h" 57 #include "xe_pxp.h" 58 #include "xe_query.h" 59 #include "xe_shrinker.h" 60 #include "xe_survivability_mode.h" 61 #include "xe_sriov.h" 62 #include "xe_tile.h" 63 #include "xe_ttm_stolen_mgr.h" 64 #include "xe_ttm_sys_mgr.h" 65 #include "xe_vm.h" 66 #include "xe_vram.h" 67 #include "xe_vsec.h" 68 #include "xe_wait_user_fence.h" 69 #include "xe_wa.h" 70 71 #include <generated/xe_device_wa_oob.h> 72 #include <generated/xe_wa_oob.h> 73 74 static int xe_file_open(struct drm_device *dev, struct drm_file *file) 75 { 76 struct xe_device *xe = to_xe_device(dev); 77 struct xe_drm_client *client; 78 struct xe_file *xef; 79 int ret = -ENOMEM; 80 struct task_struct *task = NULL; 81 82 xef = kzalloc(sizeof(*xef), GFP_KERNEL); 83 if (!xef) 84 return ret; 85 86 client = xe_drm_client_alloc(); 87 if (!client) { 88 kfree(xef); 89 return ret; 90 } 91 92 xef->drm = file; 93 xef->client = client; 94 xef->xe = xe; 95 96 mutex_init(&xef->vm.lock); 97 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); 98 99 mutex_init(&xef->exec_queue.lock); 100 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); 101 102 file->driver_priv = xef; 103 kref_init(&xef->refcount); 104 105 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID); 106 if (task) { 107 xef->process_name = kstrdup(task->comm, GFP_KERNEL); 108 xef->pid = task->pid; 109 put_task_struct(task); 110 } 111 112 return 0; 113 } 114 115 static void xe_file_destroy(struct kref *ref) 116 { 117 struct xe_file *xef = container_of(ref, struct xe_file, refcount); 118 119 xa_destroy(&xef->exec_queue.xa); 120 mutex_destroy(&xef->exec_queue.lock); 121 xa_destroy(&xef->vm.xa); 122 mutex_destroy(&xef->vm.lock); 123 124 xe_drm_client_put(xef->client); 125 kfree(xef->process_name); 126 kfree(xef); 127 } 128 129 /** 130 * xe_file_get() - Take a reference to the xe file object 131 * @xef: Pointer to the xe file 132 * 133 * Anyone with a pointer to xef must take a reference to the xe file 134 * object using this call. 135 * 136 * Return: xe file pointer 137 */ 138 struct xe_file *xe_file_get(struct xe_file *xef) 139 { 140 kref_get(&xef->refcount); 141 return xef; 142 } 143 144 /** 145 * xe_file_put() - Drop a reference to the xe file object 146 * @xef: Pointer to the xe file 147 * 148 * Used to drop reference to the xef object 149 */ 150 void xe_file_put(struct xe_file *xef) 151 { 152 kref_put(&xef->refcount, xe_file_destroy); 153 } 154 155 static void xe_file_close(struct drm_device *dev, struct drm_file *file) 156 { 157 struct xe_device *xe = to_xe_device(dev); 158 struct xe_file *xef = file->driver_priv; 159 struct xe_vm *vm; 160 struct xe_exec_queue *q; 161 unsigned long idx; 162 163 xe_pm_runtime_get(xe); 164 165 /* 166 * No need for exec_queue.lock here as there is no contention for it 167 * when FD is closing as IOCTLs presumably can't be modifying the 168 * xarray. Taking exec_queue.lock here causes undue dependency on 169 * vm->lock taken during xe_exec_queue_kill(). 170 */ 171 xa_for_each(&xef->exec_queue.xa, idx, q) { 172 if (q->vm && q->hwe->hw_engine_group) 173 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 174 xe_exec_queue_kill(q); 175 xe_exec_queue_put(q); 176 } 177 xa_for_each(&xef->vm.xa, idx, vm) 178 xe_vm_close_and_put(vm); 179 180 xe_file_put(xef); 181 182 xe_pm_runtime_put(xe); 183 } 184 185 static const struct drm_ioctl_desc xe_ioctls[] = { 186 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 187 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), 188 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, 189 DRM_RENDER_ALLOW), 190 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), 191 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), 192 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), 193 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), 194 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, 195 DRM_RENDER_ALLOW), 196 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, 197 DRM_RENDER_ALLOW), 198 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, 199 DRM_RENDER_ALLOW), 200 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, 201 DRM_RENDER_ALLOW), 202 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 203 }; 204 205 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 206 { 207 struct drm_file *file_priv = file->private_data; 208 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 209 long ret; 210 211 if (xe_device_wedged(xe)) 212 return -ECANCELED; 213 214 ret = xe_pm_runtime_get_ioctl(xe); 215 if (ret >= 0) 216 ret = drm_ioctl(file, cmd, arg); 217 xe_pm_runtime_put(xe); 218 219 return ret; 220 } 221 222 #ifdef CONFIG_COMPAT 223 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 224 { 225 struct drm_file *file_priv = file->private_data; 226 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 227 long ret; 228 229 if (xe_device_wedged(xe)) 230 return -ECANCELED; 231 232 ret = xe_pm_runtime_get_ioctl(xe); 233 if (ret >= 0) 234 ret = drm_compat_ioctl(file, cmd, arg); 235 xe_pm_runtime_put(xe); 236 237 return ret; 238 } 239 #else 240 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */ 241 #define xe_drm_compat_ioctl NULL 242 #endif 243 244 static void barrier_open(struct vm_area_struct *vma) 245 { 246 drm_dev_get(vma->vm_private_data); 247 } 248 249 static void barrier_close(struct vm_area_struct *vma) 250 { 251 drm_dev_put(vma->vm_private_data); 252 } 253 254 static void barrier_release_dummy_page(struct drm_device *dev, void *res) 255 { 256 struct page *dummy_page = (struct page *)res; 257 258 __free_page(dummy_page); 259 } 260 261 static vm_fault_t barrier_fault(struct vm_fault *vmf) 262 { 263 struct drm_device *dev = vmf->vma->vm_private_data; 264 struct vm_area_struct *vma = vmf->vma; 265 vm_fault_t ret = VM_FAULT_NOPAGE; 266 pgprot_t prot; 267 int idx; 268 269 prot = vm_get_page_prot(vma->vm_flags); 270 271 if (drm_dev_enter(dev, &idx)) { 272 unsigned long pfn; 273 274 #define LAST_DB_PAGE_OFFSET 0x7ff001 275 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) + 276 LAST_DB_PAGE_OFFSET); 277 ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn, 278 pgprot_noncached(prot)); 279 drm_dev_exit(idx); 280 } else { 281 struct page *page; 282 283 /* Allocate new dummy page to map all the VA range in this VMA to it*/ 284 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 285 if (!page) 286 return VM_FAULT_OOM; 287 288 /* Set the page to be freed using drmm release action */ 289 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page)) 290 return VM_FAULT_OOM; 291 292 ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page), 293 prot); 294 } 295 296 return ret; 297 } 298 299 static const struct vm_operations_struct vm_ops_barrier = { 300 .open = barrier_open, 301 .close = barrier_close, 302 .fault = barrier_fault, 303 }; 304 305 static int xe_pci_barrier_mmap(struct file *filp, 306 struct vm_area_struct *vma) 307 { 308 struct drm_file *priv = filp->private_data; 309 struct drm_device *dev = priv->minor->dev; 310 struct xe_device *xe = to_xe_device(dev); 311 312 if (!IS_DGFX(xe)) 313 return -EINVAL; 314 315 if (vma->vm_end - vma->vm_start > SZ_4K) 316 return -EINVAL; 317 318 if (is_cow_mapping(vma->vm_flags)) 319 return -EINVAL; 320 321 if (vma->vm_flags & (VM_READ | VM_EXEC)) 322 return -EINVAL; 323 324 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC); 325 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); 326 vma->vm_ops = &vm_ops_barrier; 327 vma->vm_private_data = dev; 328 drm_dev_get(vma->vm_private_data); 329 330 return 0; 331 } 332 333 static int xe_mmap(struct file *filp, struct vm_area_struct *vma) 334 { 335 struct drm_file *priv = filp->private_data; 336 struct drm_device *dev = priv->minor->dev; 337 338 if (drm_dev_is_unplugged(dev)) 339 return -ENODEV; 340 341 switch (vma->vm_pgoff) { 342 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT: 343 return xe_pci_barrier_mmap(filp, vma); 344 } 345 346 return drm_gem_mmap(filp, vma); 347 } 348 349 static const struct file_operations xe_driver_fops = { 350 .owner = THIS_MODULE, 351 .open = drm_open, 352 .release = drm_release_noglobal, 353 .unlocked_ioctl = xe_drm_ioctl, 354 .mmap = xe_mmap, 355 .poll = drm_poll, 356 .read = drm_read, 357 .compat_ioctl = xe_drm_compat_ioctl, 358 .llseek = noop_llseek, 359 #ifdef CONFIG_PROC_FS 360 .show_fdinfo = drm_show_fdinfo, 361 #endif 362 .fop_flags = FOP_UNSIGNED_OFFSET, 363 }; 364 365 static struct drm_driver driver = { 366 /* Don't use MTRRs here; the Xserver or userspace app should 367 * deal with them for Intel hardware. 368 */ 369 .driver_features = 370 DRIVER_GEM | 371 DRIVER_RENDER | DRIVER_SYNCOBJ | 372 DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, 373 .open = xe_file_open, 374 .postclose = xe_file_close, 375 376 .gem_prime_import = xe_gem_prime_import, 377 378 .dumb_create = xe_bo_dumb_create, 379 .dumb_map_offset = drm_gem_ttm_dumb_map_offset, 380 #ifdef CONFIG_PROC_FS 381 .show_fdinfo = xe_drm_client_fdinfo, 382 #endif 383 .ioctls = xe_ioctls, 384 .num_ioctls = ARRAY_SIZE(xe_ioctls), 385 .fops = &xe_driver_fops, 386 .name = DRIVER_NAME, 387 .desc = DRIVER_DESC, 388 .major = DRIVER_MAJOR, 389 .minor = DRIVER_MINOR, 390 .patchlevel = DRIVER_PATCHLEVEL, 391 }; 392 393 static void xe_device_destroy(struct drm_device *dev, void *dummy) 394 { 395 struct xe_device *xe = to_xe_device(dev); 396 397 xe_bo_dev_fini(&xe->bo_device); 398 399 if (xe->preempt_fence_wq) 400 destroy_workqueue(xe->preempt_fence_wq); 401 402 if (xe->ordered_wq) 403 destroy_workqueue(xe->ordered_wq); 404 405 if (xe->unordered_wq) 406 destroy_workqueue(xe->unordered_wq); 407 408 if (xe->destroy_wq) 409 destroy_workqueue(xe->destroy_wq); 410 411 ttm_device_fini(&xe->ttm); 412 } 413 414 struct xe_device *xe_device_create(struct pci_dev *pdev, 415 const struct pci_device_id *ent) 416 { 417 struct xe_device *xe; 418 int err; 419 420 xe_display_driver_set_hooks(&driver); 421 422 err = aperture_remove_conflicting_pci_devices(pdev, driver.name); 423 if (err) 424 return ERR_PTR(err); 425 426 xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm); 427 if (IS_ERR(xe)) 428 return xe; 429 430 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, 431 xe->drm.anon_inode->i_mapping, 432 xe->drm.vma_offset_manager, false, false); 433 if (WARN_ON(err)) 434 goto err; 435 436 xe_bo_dev_init(&xe->bo_device); 437 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); 438 if (err) 439 goto err; 440 441 err = xe_shrinker_create(xe); 442 if (err) 443 goto err; 444 445 xe->info.devid = pdev->device; 446 xe->info.revid = pdev->revision; 447 xe->info.force_execlist = xe_modparam.force_execlist; 448 xe->atomic_svm_timeslice_ms = 5; 449 450 err = xe_irq_init(xe); 451 if (err) 452 goto err; 453 454 init_waitqueue_head(&xe->ufence_wq); 455 456 init_rwsem(&xe->usm.lock); 457 458 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); 459 460 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 461 /* Trigger a large asid and an early asid wrap. */ 462 u32 asid; 463 464 BUILD_BUG_ON(XE_MAX_ASID < 2); 465 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL, 466 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1), 467 &xe->usm.next_asid, GFP_KERNEL); 468 drm_WARN_ON(&xe->drm, err); 469 if (err >= 0) 470 xa_erase(&xe->usm.asid_to_vm, asid); 471 } 472 473 err = xe_bo_pinned_init(xe); 474 if (err) 475 goto err; 476 477 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", 478 WQ_MEM_RECLAIM); 479 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); 480 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0); 481 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0); 482 if (!xe->ordered_wq || !xe->unordered_wq || 483 !xe->preempt_fence_wq || !xe->destroy_wq) { 484 /* 485 * Cleanup done in xe_device_destroy via 486 * drmm_add_action_or_reset register above 487 */ 488 drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); 489 err = -ENOMEM; 490 goto err; 491 } 492 493 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); 494 if (err) 495 goto err; 496 497 return xe; 498 499 err: 500 return ERR_PTR(err); 501 } 502 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ 503 504 static bool xe_driver_flr_disabled(struct xe_device *xe) 505 { 506 if (IS_SRIOV_VF(xe)) 507 return true; 508 509 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) { 510 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n"); 511 return true; 512 } 513 514 return false; 515 } 516 517 /* 518 * The driver-initiated FLR is the highest level of reset that we can trigger 519 * from within the driver. It is different from the PCI FLR in that it doesn't 520 * fully reset the SGUnit and doesn't modify the PCI config space and therefore 521 * it doesn't require a re-enumeration of the PCI BARs. However, the 522 * driver-initiated FLR does still cause a reset of both GT and display and a 523 * memory wipe of local and stolen memory, so recovery would require a full HW 524 * re-init and saving/restoring (or re-populating) the wiped memory. Since we 525 * perform the FLR as the very last action before releasing access to the HW 526 * during the driver release flow, we don't attempt recovery at all, because 527 * if/when a new instance of i915 is bound to the device it will do a full 528 * re-init anyway. 529 */ 530 static void __xe_driver_flr(struct xe_device *xe) 531 { 532 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */ 533 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 534 int ret; 535 536 drm_dbg(&xe->drm, "Triggering Driver-FLR\n"); 537 538 /* 539 * Make sure any pending FLR requests have cleared by waiting for the 540 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS 541 * to make sure it's not still set from a prior attempt (it's a write to 542 * clear bit). 543 * Note that we should never be in a situation where a previous attempt 544 * is still pending (unless the HW is totally dead), but better to be 545 * safe in case something unexpected happens 546 */ 547 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 548 if (ret) { 549 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret); 550 return; 551 } 552 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 553 554 /* Trigger the actual Driver-FLR */ 555 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR); 556 557 /* Wait for hardware teardown to complete */ 558 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 559 if (ret) { 560 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); 561 return; 562 } 563 564 /* Wait for hardware/firmware re-init to complete */ 565 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS, 566 flr_timeout, NULL, false); 567 if (ret) { 568 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); 569 return; 570 } 571 572 /* Clear sticky completion status */ 573 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 574 } 575 576 static void xe_driver_flr(struct xe_device *xe) 577 { 578 if (xe_driver_flr_disabled(xe)) 579 return; 580 581 __xe_driver_flr(xe); 582 } 583 584 static void xe_driver_flr_fini(void *arg) 585 { 586 struct xe_device *xe = arg; 587 588 if (xe->needs_flr_on_fini) 589 xe_driver_flr(xe); 590 } 591 592 static void xe_device_sanitize(void *arg) 593 { 594 struct xe_device *xe = arg; 595 struct xe_gt *gt; 596 u8 id; 597 598 for_each_gt(gt, xe, id) 599 xe_gt_sanitize(gt); 600 } 601 602 static int xe_set_dma_info(struct xe_device *xe) 603 { 604 unsigned int mask_size = xe->info.dma_mask_size; 605 int err; 606 607 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev)); 608 609 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 610 if (err) 611 goto mask_err; 612 613 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 614 if (err) 615 goto mask_err; 616 617 return 0; 618 619 mask_err: 620 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); 621 return err; 622 } 623 624 static bool verify_lmem_ready(struct xe_device *xe) 625 { 626 u32 val = xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT; 627 628 return !!val; 629 } 630 631 static int wait_for_lmem_ready(struct xe_device *xe) 632 { 633 unsigned long timeout, start; 634 635 if (!IS_DGFX(xe)) 636 return 0; 637 638 if (IS_SRIOV_VF(xe)) 639 return 0; 640 641 if (verify_lmem_ready(xe)) 642 return 0; 643 644 drm_dbg(&xe->drm, "Waiting for lmem initialization\n"); 645 646 start = jiffies; 647 timeout = start + secs_to_jiffies(60); /* 60 sec! */ 648 649 do { 650 if (signal_pending(current)) 651 return -EINTR; 652 653 /* 654 * The boot firmware initializes local memory and 655 * assesses its health. If memory training fails, 656 * the punit will have been instructed to keep the GT powered 657 * down.we won't be able to communicate with it 658 * 659 * If the status check is done before punit updates the register, 660 * it can lead to the system being unusable. 661 * use a timeout and defer the probe to prevent this. 662 */ 663 if (time_after(jiffies, timeout)) { 664 drm_dbg(&xe->drm, "lmem not initialized by firmware\n"); 665 return -EPROBE_DEFER; 666 } 667 668 msleep(20); 669 670 } while (!verify_lmem_ready(xe)); 671 672 drm_dbg(&xe->drm, "lmem ready after %ums", 673 jiffies_to_msecs(jiffies - start)); 674 675 return 0; 676 } 677 ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */ 678 679 static void sriov_update_device_info(struct xe_device *xe) 680 { 681 /* disable features that are not available/applicable to VFs */ 682 if (IS_SRIOV_VF(xe)) { 683 xe->info.probe_display = 0; 684 xe->info.has_heci_gscfi = 0; 685 xe->info.skip_guc_pc = 1; 686 xe->info.skip_pcode = 1; 687 } 688 } 689 690 /** 691 * xe_device_probe_early: Device early probe 692 * @xe: xe device instance 693 * 694 * Initialize MMIO resources that don't require any 695 * knowledge about tile count. Also initialize pcode and 696 * check vram initialization on root tile. 697 * 698 * Return: 0 on success, error code on failure 699 */ 700 int xe_device_probe_early(struct xe_device *xe) 701 { 702 int err; 703 704 xe_wa_device_init(xe); 705 xe_wa_process_device_oob(xe); 706 707 err = xe_mmio_probe_early(xe); 708 if (err) 709 return err; 710 711 xe_sriov_probe_early(xe); 712 713 sriov_update_device_info(xe); 714 715 err = xe_pcode_probe_early(xe); 716 if (err || xe_survivability_mode_is_requested(xe)) { 717 int save_err = err; 718 719 /* 720 * Try to leave device in survivability mode if device is 721 * possible, but still return the previous error for error 722 * propagation 723 */ 724 err = xe_survivability_mode_enable(xe); 725 if (err) 726 return err; 727 728 return save_err; 729 } 730 731 err = wait_for_lmem_ready(xe); 732 if (err) 733 return err; 734 735 xe->wedged.mode = xe_modparam.wedged_mode; 736 737 return 0; 738 } 739 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ 740 741 static int probe_has_flat_ccs(struct xe_device *xe) 742 { 743 struct xe_gt *gt; 744 unsigned int fw_ref; 745 u32 reg; 746 747 /* Always enabled/disabled, no runtime check to do */ 748 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe)) 749 return 0; 750 751 gt = xe_root_mmio_gt(xe); 752 753 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 754 if (!fw_ref) 755 return -ETIMEDOUT; 756 757 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 758 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE); 759 760 if (!xe->info.has_flat_ccs) 761 drm_dbg(&xe->drm, 762 "Flat CCS has been disabled in bios, May lead to performance impact"); 763 764 xe_force_wake_put(gt_to_fw(gt), fw_ref); 765 766 return 0; 767 } 768 769 int xe_device_probe(struct xe_device *xe) 770 { 771 struct xe_tile *tile; 772 struct xe_gt *gt; 773 int err; 774 u8 id; 775 776 xe_pat_init_early(xe); 777 778 err = xe_sriov_init(xe); 779 if (err) 780 return err; 781 782 xe->info.mem_region_mask = 1; 783 784 err = xe_set_dma_info(xe); 785 if (err) 786 return err; 787 788 err = xe_mmio_probe_tiles(xe); 789 if (err) 790 return err; 791 792 for_each_gt(gt, xe, id) { 793 err = xe_gt_init_early(gt); 794 if (err) 795 return err; 796 } 797 798 for_each_tile(tile, xe, id) { 799 err = xe_ggtt_init_early(tile->mem.ggtt); 800 if (err) 801 return err; 802 } 803 804 err = xe_devcoredump_init(xe); 805 if (err) 806 return err; 807 808 /* 809 * From here on, if a step fails, make sure a Driver-FLR is triggereed 810 */ 811 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe); 812 if (err) 813 return err; 814 815 err = probe_has_flat_ccs(xe); 816 if (err) 817 return err; 818 819 err = xe_vram_probe(xe); 820 if (err) 821 return err; 822 823 for_each_tile(tile, xe, id) { 824 err = xe_tile_init_noalloc(tile); 825 if (err) 826 return err; 827 } 828 829 /* 830 * Allow allocations only now to ensure xe_display_init_early() 831 * is the first to allocate, always. 832 */ 833 err = xe_ttm_sys_mgr_init(xe); 834 if (err) 835 return err; 836 837 /* Allocate and map stolen after potential VRAM resize */ 838 err = xe_ttm_stolen_mgr_init(xe); 839 if (err) 840 return err; 841 842 /* 843 * Now that GT is initialized (TTM in particular), 844 * we can try to init display, and inherit the initial fb. 845 * This is the reason the first allocation needs to be done 846 * inside display. 847 */ 848 err = xe_display_init_early(xe); 849 if (err) 850 return err; 851 852 for_each_tile(tile, xe, id) { 853 err = xe_tile_init(tile); 854 if (err) 855 return err; 856 } 857 858 err = xe_irq_install(xe); 859 if (err) 860 return err; 861 862 for_each_gt(gt, xe, id) { 863 err = xe_gt_init(gt); 864 if (err) 865 return err; 866 } 867 868 if (xe->tiles->media_gt && 869 XE_WA(xe->tiles->media_gt, 15015404425_disable)) 870 XE_DEVICE_WA_DISABLE(xe, 15015404425); 871 872 xe_nvm_init(xe); 873 874 err = xe_heci_gsc_init(xe); 875 if (err) 876 return err; 877 878 err = xe_oa_init(xe); 879 if (err) 880 return err; 881 882 err = xe_display_init(xe); 883 if (err) 884 return err; 885 886 err = xe_pxp_init(xe); 887 if (err) 888 return err; 889 890 err = drm_dev_register(&xe->drm, 0); 891 if (err) 892 return err; 893 894 xe_display_register(xe); 895 896 err = xe_oa_register(xe); 897 if (err) 898 goto err_unregister_display; 899 900 err = xe_pmu_register(&xe->pmu); 901 if (err) 902 goto err_unregister_display; 903 904 err = xe_device_sysfs_init(xe); 905 if (err) 906 goto err_unregister_display; 907 908 xe_debugfs_register(xe); 909 910 err = xe_hwmon_register(xe); 911 if (err) 912 goto err_unregister_display; 913 914 err = xe_i2c_probe(xe); 915 if (err) 916 goto err_unregister_display; 917 918 for_each_gt(gt, xe, id) 919 xe_gt_sanitize_freq(gt); 920 921 xe_vsec_init(xe); 922 923 return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); 924 925 err_unregister_display: 926 xe_display_unregister(xe); 927 928 return err; 929 } 930 931 void xe_device_remove(struct xe_device *xe) 932 { 933 xe_display_unregister(xe); 934 935 xe_nvm_fini(xe); 936 937 drm_dev_unplug(&xe->drm); 938 939 xe_bo_pci_dev_remove_all(xe); 940 } 941 942 void xe_device_shutdown(struct xe_device *xe) 943 { 944 struct xe_gt *gt; 945 u8 id; 946 947 drm_dbg(&xe->drm, "Shutting down device\n"); 948 949 if (xe_driver_flr_disabled(xe)) { 950 xe_display_pm_shutdown(xe); 951 952 xe_irq_suspend(xe); 953 954 for_each_gt(gt, xe, id) 955 xe_gt_shutdown(gt); 956 957 xe_display_pm_shutdown_late(xe); 958 } else { 959 /* BOOM! */ 960 __xe_driver_flr(xe); 961 } 962 } 963 964 /** 965 * xe_device_wmb() - Device specific write memory barrier 966 * @xe: the &xe_device 967 * 968 * While wmb() is sufficient for a barrier if we use system memory, on discrete 969 * platforms with device memory we additionally need to issue a register write. 970 * Since it doesn't matter which register we write to, use the read-only VF_CAP 971 * register that is also marked as accessible by the VFs. 972 */ 973 void xe_device_wmb(struct xe_device *xe) 974 { 975 wmb(); 976 if (IS_DGFX(xe)) 977 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0); 978 } 979 980 /* 981 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt. 982 */ 983 static void tdf_request_sync(struct xe_device *xe) 984 { 985 unsigned int fw_ref; 986 struct xe_gt *gt; 987 u8 id; 988 989 for_each_gt(gt, xe, id) { 990 if (xe_gt_is_media_type(gt)) 991 continue; 992 993 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 994 if (!fw_ref) 995 return; 996 997 xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); 998 999 /* 1000 * FIXME: We can likely do better here with our choice of 1001 * timeout. Currently we just assume the worst case, i.e. 150us, 1002 * which is believed to be sufficient to cover the worst case 1003 * scenario on current platforms if all cache entries are 1004 * transient and need to be flushed.. 1005 */ 1006 if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 1007 150, NULL, false)) 1008 xe_gt_err_once(gt, "TD flush timeout\n"); 1009 1010 xe_force_wake_put(gt_to_fw(gt), fw_ref); 1011 } 1012 } 1013 1014 void xe_device_l2_flush(struct xe_device *xe) 1015 { 1016 struct xe_gt *gt; 1017 unsigned int fw_ref; 1018 1019 gt = xe_root_mmio_gt(xe); 1020 1021 if (!XE_WA(gt, 16023588340)) 1022 return; 1023 1024 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 1025 if (!fw_ref) 1026 return; 1027 1028 spin_lock(>->global_invl_lock); 1029 1030 xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1); 1031 if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 500, NULL, true)) 1032 xe_gt_err_once(gt, "Global invalidation timeout\n"); 1033 1034 spin_unlock(>->global_invl_lock); 1035 1036 xe_force_wake_put(gt_to_fw(gt), fw_ref); 1037 } 1038 1039 /** 1040 * xe_device_td_flush() - Flush transient L3 cache entries 1041 * @xe: The device 1042 * 1043 * Display engine has direct access to memory and is never coherent with L3/L4 1044 * caches (or CPU caches), however KMD is responsible for specifically flushing 1045 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout 1046 * can happen from such a surface without seeing corruption. 1047 * 1048 * Display surfaces can be tagged as transient by mapping it using one of the 1049 * various L3:XD PAT index modes on Xe2. 1050 * 1051 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed 1052 * at the end of each submission via PIPE_CONTROL for compute/render, since SA 1053 * Media is not coherent with L3 and we want to support render-vs-media 1054 * usescases. For other engines like copy/blt the HW internally forces uncached 1055 * behaviour, hence why we can skip the TDF on such platforms. 1056 */ 1057 void xe_device_td_flush(struct xe_device *xe) 1058 { 1059 struct xe_gt *root_gt; 1060 1061 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20) 1062 return; 1063 1064 root_gt = xe_root_mmio_gt(xe); 1065 if (XE_WA(root_gt, 16023588340)) { 1066 /* A transient flush is not sufficient: flush the L2 */ 1067 xe_device_l2_flush(xe); 1068 } else { 1069 xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc); 1070 tdf_request_sync(xe); 1071 xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc); 1072 } 1073 } 1074 1075 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) 1076 { 1077 return xe_device_has_flat_ccs(xe) ? 1078 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0; 1079 } 1080 1081 /** 1082 * xe_device_assert_mem_access - Inspect the current runtime_pm state. 1083 * @xe: xe device instance 1084 * 1085 * To be used before any kind of memory access. It will splat a debug warning 1086 * if the device is currently sleeping. But it doesn't guarantee in any way 1087 * that the device is going to remain awake. Xe PM runtime get and put 1088 * functions might be added to the outer bound of the memory access, while 1089 * this check is intended for inner usage to splat some warning if the worst 1090 * case has just happened. 1091 */ 1092 void xe_device_assert_mem_access(struct xe_device *xe) 1093 { 1094 xe_assert(xe, !xe_pm_runtime_suspended(xe)); 1095 } 1096 1097 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p) 1098 { 1099 struct xe_gt *gt; 1100 u8 id; 1101 1102 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid); 1103 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid); 1104 1105 for_each_gt(gt, xe, id) { 1106 drm_printf(p, "GT id: %u\n", id); 1107 drm_printf(p, "\tTile: %u\n", gt->tile->id); 1108 drm_printf(p, "\tType: %s\n", 1109 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media"); 1110 drm_printf(p, "\tIP ver: %u.%u.%u\n", 1111 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid), 1112 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid), 1113 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid)); 1114 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock); 1115 } 1116 } 1117 1118 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address) 1119 { 1120 return sign_extend64(address, xe->info.va_bits - 1); 1121 } 1122 1123 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) 1124 { 1125 return address & GENMASK_ULL(xe->info.va_bits - 1, 0); 1126 } 1127 1128 static void xe_device_wedged_fini(struct drm_device *drm, void *arg) 1129 { 1130 struct xe_device *xe = arg; 1131 1132 xe_pm_runtime_put(xe); 1133 } 1134 1135 /** 1136 * xe_device_declare_wedged - Declare device wedged 1137 * @xe: xe device instance 1138 * 1139 * This is a final state that can only be cleared with a module 1140 * re-probe (unbind + bind). 1141 * In this state every IOCTL will be blocked so the GT cannot be used. 1142 * In general it will be called upon any critical error such as gt reset 1143 * failure or guc loading failure. Userspace will be notified of this state 1144 * through device wedged uevent. 1145 * If xe.wedged module parameter is set to 2, this function will be called 1146 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump 1147 * snapshot capture. In this mode, GT reset won't be attempted so the state of 1148 * the issue is preserved for further debugging. 1149 */ 1150 void xe_device_declare_wedged(struct xe_device *xe) 1151 { 1152 struct xe_gt *gt; 1153 u8 id; 1154 1155 if (xe->wedged.mode == 0) { 1156 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); 1157 return; 1158 } 1159 1160 xe_pm_runtime_get_noresume(xe); 1161 1162 if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) { 1163 drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n"); 1164 return; 1165 } 1166 1167 if (!atomic_xchg(&xe->wedged.flag, 1)) { 1168 xe->needs_flr_on_fini = true; 1169 drm_err(&xe->drm, 1170 "CRITICAL: Xe has declared device %s as wedged.\n" 1171 "IOCTLs and executions are blocked. Only a rebind may clear the failure\n" 1172 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n", 1173 dev_name(xe->drm.dev)); 1174 1175 /* Notify userspace of wedged device */ 1176 drm_dev_wedged_event(&xe->drm, 1177 DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET, 1178 NULL); 1179 } 1180 1181 for_each_gt(gt, xe, id) 1182 xe_gt_declare_wedged(gt); 1183 } 1184