1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_device.h" 7 8 #include <linux/aperture.h> 9 #include <linux/delay.h> 10 #include <linux/fault-inject.h> 11 #include <linux/units.h> 12 13 #include <drm/drm_atomic_helper.h> 14 #include <drm/drm_client.h> 15 #include <drm/drm_gem_ttm_helper.h> 16 #include <drm/drm_ioctl.h> 17 #include <drm/drm_managed.h> 18 #include <drm/drm_print.h> 19 #include <uapi/drm/xe_drm.h> 20 21 #include "display/xe_display.h" 22 #include "instructions/xe_gpu_commands.h" 23 #include "regs/xe_gt_regs.h" 24 #include "regs/xe_regs.h" 25 #include "xe_bo.h" 26 #include "xe_bo_evict.h" 27 #include "xe_debugfs.h" 28 #include "xe_devcoredump.h" 29 #include "xe_dma_buf.h" 30 #include "xe_drm_client.h" 31 #include "xe_drv.h" 32 #include "xe_exec.h" 33 #include "xe_exec_queue.h" 34 #include "xe_force_wake.h" 35 #include "xe_ggtt.h" 36 #include "xe_gsc_proxy.h" 37 #include "xe_gt.h" 38 #include "xe_gt_mcr.h" 39 #include "xe_gt_printk.h" 40 #include "xe_gt_sriov_vf.h" 41 #include "xe_guc.h" 42 #include "xe_hw_engine_group.h" 43 #include "xe_hwmon.h" 44 #include "xe_irq.h" 45 #include "xe_memirq.h" 46 #include "xe_mmio.h" 47 #include "xe_module.h" 48 #include "xe_oa.h" 49 #include "xe_observation.h" 50 #include "xe_pat.h" 51 #include "xe_pcode.h" 52 #include "xe_pm.h" 53 #include "xe_pmu.h" 54 #include "xe_pxp.h" 55 #include "xe_query.h" 56 #include "xe_shrinker.h" 57 #include "xe_survivability_mode.h" 58 #include "xe_sriov.h" 59 #include "xe_tile.h" 60 #include "xe_ttm_stolen_mgr.h" 61 #include "xe_ttm_sys_mgr.h" 62 #include "xe_vm.h" 63 #include "xe_vram.h" 64 #include "xe_vsec.h" 65 #include "xe_wait_user_fence.h" 66 #include "xe_wa.h" 67 68 #include <generated/xe_wa_oob.h> 69 70 static int xe_file_open(struct drm_device *dev, struct drm_file *file) 71 { 72 struct xe_device *xe = to_xe_device(dev); 73 struct xe_drm_client *client; 74 struct xe_file *xef; 75 int ret = -ENOMEM; 76 struct task_struct *task = NULL; 77 78 xef = kzalloc(sizeof(*xef), GFP_KERNEL); 79 if (!xef) 80 return ret; 81 82 client = xe_drm_client_alloc(); 83 if (!client) { 84 kfree(xef); 85 return ret; 86 } 87 88 xef->drm = file; 89 xef->client = client; 90 xef->xe = xe; 91 92 mutex_init(&xef->vm.lock); 93 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); 94 95 mutex_init(&xef->exec_queue.lock); 96 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); 97 98 file->driver_priv = xef; 99 kref_init(&xef->refcount); 100 101 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID); 102 if (task) { 103 xef->process_name = kstrdup(task->comm, GFP_KERNEL); 104 xef->pid = task->pid; 105 put_task_struct(task); 106 } 107 108 return 0; 109 } 110 111 static void xe_file_destroy(struct kref *ref) 112 { 113 struct xe_file *xef = container_of(ref, struct xe_file, refcount); 114 115 xa_destroy(&xef->exec_queue.xa); 116 mutex_destroy(&xef->exec_queue.lock); 117 xa_destroy(&xef->vm.xa); 118 mutex_destroy(&xef->vm.lock); 119 120 xe_drm_client_put(xef->client); 121 kfree(xef->process_name); 122 kfree(xef); 123 } 124 125 /** 126 * xe_file_get() - Take a reference to the xe file object 127 * @xef: Pointer to the xe file 128 * 129 * Anyone with a pointer to xef must take a reference to the xe file 130 * object using this call. 131 * 132 * Return: xe file pointer 133 */ 134 struct xe_file *xe_file_get(struct xe_file *xef) 135 { 136 kref_get(&xef->refcount); 137 return xef; 138 } 139 140 /** 141 * xe_file_put() - Drop a reference to the xe file object 142 * @xef: Pointer to the xe file 143 * 144 * Used to drop reference to the xef object 145 */ 146 void xe_file_put(struct xe_file *xef) 147 { 148 kref_put(&xef->refcount, xe_file_destroy); 149 } 150 151 static void xe_file_close(struct drm_device *dev, struct drm_file *file) 152 { 153 struct xe_device *xe = to_xe_device(dev); 154 struct xe_file *xef = file->driver_priv; 155 struct xe_vm *vm; 156 struct xe_exec_queue *q; 157 unsigned long idx; 158 159 xe_pm_runtime_get(xe); 160 161 /* 162 * No need for exec_queue.lock here as there is no contention for it 163 * when FD is closing as IOCTLs presumably can't be modifying the 164 * xarray. Taking exec_queue.lock here causes undue dependency on 165 * vm->lock taken during xe_exec_queue_kill(). 166 */ 167 xa_for_each(&xef->exec_queue.xa, idx, q) { 168 if (q->vm && q->hwe->hw_engine_group) 169 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 170 xe_exec_queue_kill(q); 171 xe_exec_queue_put(q); 172 } 173 xa_for_each(&xef->vm.xa, idx, vm) 174 xe_vm_close_and_put(vm); 175 176 xe_file_put(xef); 177 178 xe_pm_runtime_put(xe); 179 } 180 181 static const struct drm_ioctl_desc xe_ioctls[] = { 182 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 183 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), 184 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, 185 DRM_RENDER_ALLOW), 186 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), 187 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), 188 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), 189 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), 190 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, 191 DRM_RENDER_ALLOW), 192 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, 193 DRM_RENDER_ALLOW), 194 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, 195 DRM_RENDER_ALLOW), 196 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, 197 DRM_RENDER_ALLOW), 198 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 199 }; 200 201 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 202 { 203 struct drm_file *file_priv = file->private_data; 204 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 205 long ret; 206 207 if (xe_device_wedged(xe)) 208 return -ECANCELED; 209 210 ret = xe_pm_runtime_get_ioctl(xe); 211 if (ret >= 0) 212 ret = drm_ioctl(file, cmd, arg); 213 xe_pm_runtime_put(xe); 214 215 return ret; 216 } 217 218 #ifdef CONFIG_COMPAT 219 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 220 { 221 struct drm_file *file_priv = file->private_data; 222 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 223 long ret; 224 225 if (xe_device_wedged(xe)) 226 return -ECANCELED; 227 228 ret = xe_pm_runtime_get_ioctl(xe); 229 if (ret >= 0) 230 ret = drm_compat_ioctl(file, cmd, arg); 231 xe_pm_runtime_put(xe); 232 233 return ret; 234 } 235 #else 236 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */ 237 #define xe_drm_compat_ioctl NULL 238 #endif 239 240 static void barrier_open(struct vm_area_struct *vma) 241 { 242 drm_dev_get(vma->vm_private_data); 243 } 244 245 static void barrier_close(struct vm_area_struct *vma) 246 { 247 drm_dev_put(vma->vm_private_data); 248 } 249 250 static void barrier_release_dummy_page(struct drm_device *dev, void *res) 251 { 252 struct page *dummy_page = (struct page *)res; 253 254 __free_page(dummy_page); 255 } 256 257 static vm_fault_t barrier_fault(struct vm_fault *vmf) 258 { 259 struct drm_device *dev = vmf->vma->vm_private_data; 260 struct vm_area_struct *vma = vmf->vma; 261 vm_fault_t ret = VM_FAULT_NOPAGE; 262 pgprot_t prot; 263 int idx; 264 265 prot = vm_get_page_prot(vma->vm_flags); 266 267 if (drm_dev_enter(dev, &idx)) { 268 unsigned long pfn; 269 270 #define LAST_DB_PAGE_OFFSET 0x7ff001 271 pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) + 272 LAST_DB_PAGE_OFFSET); 273 ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn, 274 pgprot_noncached(prot)); 275 drm_dev_exit(idx); 276 } else { 277 struct page *page; 278 279 /* Allocate new dummy page to map all the VA range in this VMA to it*/ 280 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 281 if (!page) 282 return VM_FAULT_OOM; 283 284 /* Set the page to be freed using drmm release action */ 285 if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page)) 286 return VM_FAULT_OOM; 287 288 ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page), 289 prot); 290 } 291 292 return ret; 293 } 294 295 static const struct vm_operations_struct vm_ops_barrier = { 296 .open = barrier_open, 297 .close = barrier_close, 298 .fault = barrier_fault, 299 }; 300 301 static int xe_pci_barrier_mmap(struct file *filp, 302 struct vm_area_struct *vma) 303 { 304 struct drm_file *priv = filp->private_data; 305 struct drm_device *dev = priv->minor->dev; 306 struct xe_device *xe = to_xe_device(dev); 307 308 if (!IS_DGFX(xe)) 309 return -EINVAL; 310 311 if (vma->vm_end - vma->vm_start > SZ_4K) 312 return -EINVAL; 313 314 if (is_cow_mapping(vma->vm_flags)) 315 return -EINVAL; 316 317 if (vma->vm_flags & (VM_READ | VM_EXEC)) 318 return -EINVAL; 319 320 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC); 321 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); 322 vma->vm_ops = &vm_ops_barrier; 323 vma->vm_private_data = dev; 324 drm_dev_get(vma->vm_private_data); 325 326 return 0; 327 } 328 329 static int xe_mmap(struct file *filp, struct vm_area_struct *vma) 330 { 331 struct drm_file *priv = filp->private_data; 332 struct drm_device *dev = priv->minor->dev; 333 334 if (drm_dev_is_unplugged(dev)) 335 return -ENODEV; 336 337 switch (vma->vm_pgoff) { 338 case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT: 339 return xe_pci_barrier_mmap(filp, vma); 340 } 341 342 return drm_gem_mmap(filp, vma); 343 } 344 345 static const struct file_operations xe_driver_fops = { 346 .owner = THIS_MODULE, 347 .open = drm_open, 348 .release = drm_release_noglobal, 349 .unlocked_ioctl = xe_drm_ioctl, 350 .mmap = xe_mmap, 351 .poll = drm_poll, 352 .read = drm_read, 353 .compat_ioctl = xe_drm_compat_ioctl, 354 .llseek = noop_llseek, 355 #ifdef CONFIG_PROC_FS 356 .show_fdinfo = drm_show_fdinfo, 357 #endif 358 .fop_flags = FOP_UNSIGNED_OFFSET, 359 }; 360 361 static struct drm_driver driver = { 362 /* Don't use MTRRs here; the Xserver or userspace app should 363 * deal with them for Intel hardware. 364 */ 365 .driver_features = 366 DRIVER_GEM | 367 DRIVER_RENDER | DRIVER_SYNCOBJ | 368 DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, 369 .open = xe_file_open, 370 .postclose = xe_file_close, 371 372 .gem_prime_import = xe_gem_prime_import, 373 374 .dumb_create = xe_bo_dumb_create, 375 .dumb_map_offset = drm_gem_ttm_dumb_map_offset, 376 #ifdef CONFIG_PROC_FS 377 .show_fdinfo = xe_drm_client_fdinfo, 378 #endif 379 .ioctls = xe_ioctls, 380 .num_ioctls = ARRAY_SIZE(xe_ioctls), 381 .fops = &xe_driver_fops, 382 .name = DRIVER_NAME, 383 .desc = DRIVER_DESC, 384 .major = DRIVER_MAJOR, 385 .minor = DRIVER_MINOR, 386 .patchlevel = DRIVER_PATCHLEVEL, 387 }; 388 389 static void xe_device_destroy(struct drm_device *dev, void *dummy) 390 { 391 struct xe_device *xe = to_xe_device(dev); 392 393 xe_bo_dev_fini(&xe->bo_device); 394 395 if (xe->preempt_fence_wq) 396 destroy_workqueue(xe->preempt_fence_wq); 397 398 if (xe->ordered_wq) 399 destroy_workqueue(xe->ordered_wq); 400 401 if (xe->unordered_wq) 402 destroy_workqueue(xe->unordered_wq); 403 404 if (!IS_ERR_OR_NULL(xe->mem.shrinker)) 405 xe_shrinker_destroy(xe->mem.shrinker); 406 407 if (xe->destroy_wq) 408 destroy_workqueue(xe->destroy_wq); 409 410 ttm_device_fini(&xe->ttm); 411 } 412 413 struct xe_device *xe_device_create(struct pci_dev *pdev, 414 const struct pci_device_id *ent) 415 { 416 struct xe_device *xe; 417 int err; 418 419 xe_display_driver_set_hooks(&driver); 420 421 err = aperture_remove_conflicting_pci_devices(pdev, driver.name); 422 if (err) 423 return ERR_PTR(err); 424 425 xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm); 426 if (IS_ERR(xe)) 427 return xe; 428 429 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, 430 xe->drm.anon_inode->i_mapping, 431 xe->drm.vma_offset_manager, false, false); 432 if (WARN_ON(err)) 433 goto err; 434 435 xe_bo_dev_init(&xe->bo_device); 436 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); 437 if (err) 438 goto err; 439 440 xe->mem.shrinker = xe_shrinker_create(xe); 441 if (IS_ERR(xe->mem.shrinker)) 442 return ERR_CAST(xe->mem.shrinker); 443 444 xe->info.devid = pdev->device; 445 xe->info.revid = pdev->revision; 446 xe->info.force_execlist = xe_modparam.force_execlist; 447 448 err = xe_irq_init(xe); 449 if (err) 450 goto err; 451 452 init_waitqueue_head(&xe->ufence_wq); 453 454 init_rwsem(&xe->usm.lock); 455 456 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); 457 458 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 459 /* Trigger a large asid and an early asid wrap. */ 460 u32 asid; 461 462 BUILD_BUG_ON(XE_MAX_ASID < 2); 463 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL, 464 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1), 465 &xe->usm.next_asid, GFP_KERNEL); 466 drm_WARN_ON(&xe->drm, err); 467 if (err >= 0) 468 xa_erase(&xe->usm.asid_to_vm, asid); 469 } 470 471 err = xe_bo_pinned_init(xe); 472 if (err) 473 goto err; 474 475 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", 476 WQ_MEM_RECLAIM); 477 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); 478 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0); 479 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0); 480 if (!xe->ordered_wq || !xe->unordered_wq || 481 !xe->preempt_fence_wq || !xe->destroy_wq) { 482 /* 483 * Cleanup done in xe_device_destroy via 484 * drmm_add_action_or_reset register above 485 */ 486 drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); 487 err = -ENOMEM; 488 goto err; 489 } 490 491 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); 492 if (err) 493 goto err; 494 495 err = xe_display_create(xe); 496 if (WARN_ON(err)) 497 goto err; 498 499 return xe; 500 501 err: 502 return ERR_PTR(err); 503 } 504 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ 505 506 static bool xe_driver_flr_disabled(struct xe_device *xe) 507 { 508 if (IS_SRIOV_VF(xe)) 509 return true; 510 511 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) { 512 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n"); 513 return true; 514 } 515 516 return false; 517 } 518 519 /* 520 * The driver-initiated FLR is the highest level of reset that we can trigger 521 * from within the driver. It is different from the PCI FLR in that it doesn't 522 * fully reset the SGUnit and doesn't modify the PCI config space and therefore 523 * it doesn't require a re-enumeration of the PCI BARs. However, the 524 * driver-initiated FLR does still cause a reset of both GT and display and a 525 * memory wipe of local and stolen memory, so recovery would require a full HW 526 * re-init and saving/restoring (or re-populating) the wiped memory. Since we 527 * perform the FLR as the very last action before releasing access to the HW 528 * during the driver release flow, we don't attempt recovery at all, because 529 * if/when a new instance of i915 is bound to the device it will do a full 530 * re-init anyway. 531 */ 532 static void __xe_driver_flr(struct xe_device *xe) 533 { 534 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */ 535 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 536 int ret; 537 538 drm_dbg(&xe->drm, "Triggering Driver-FLR\n"); 539 540 /* 541 * Make sure any pending FLR requests have cleared by waiting for the 542 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS 543 * to make sure it's not still set from a prior attempt (it's a write to 544 * clear bit). 545 * Note that we should never be in a situation where a previous attempt 546 * is still pending (unless the HW is totally dead), but better to be 547 * safe in case something unexpected happens 548 */ 549 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 550 if (ret) { 551 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret); 552 return; 553 } 554 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 555 556 /* Trigger the actual Driver-FLR */ 557 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR); 558 559 /* Wait for hardware teardown to complete */ 560 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 561 if (ret) { 562 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); 563 return; 564 } 565 566 /* Wait for hardware/firmware re-init to complete */ 567 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS, 568 flr_timeout, NULL, false); 569 if (ret) { 570 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); 571 return; 572 } 573 574 /* Clear sticky completion status */ 575 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 576 } 577 578 static void xe_driver_flr(struct xe_device *xe) 579 { 580 if (xe_driver_flr_disabled(xe)) 581 return; 582 583 __xe_driver_flr(xe); 584 } 585 586 static void xe_driver_flr_fini(void *arg) 587 { 588 struct xe_device *xe = arg; 589 590 if (xe->needs_flr_on_fini) 591 xe_driver_flr(xe); 592 } 593 594 static void xe_device_sanitize(void *arg) 595 { 596 struct xe_device *xe = arg; 597 struct xe_gt *gt; 598 u8 id; 599 600 for_each_gt(gt, xe, id) 601 xe_gt_sanitize(gt); 602 } 603 604 static int xe_set_dma_info(struct xe_device *xe) 605 { 606 unsigned int mask_size = xe->info.dma_mask_size; 607 int err; 608 609 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev)); 610 611 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 612 if (err) 613 goto mask_err; 614 615 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 616 if (err) 617 goto mask_err; 618 619 return 0; 620 621 mask_err: 622 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); 623 return err; 624 } 625 626 static bool verify_lmem_ready(struct xe_device *xe) 627 { 628 u32 val = xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT; 629 630 return !!val; 631 } 632 633 static int wait_for_lmem_ready(struct xe_device *xe) 634 { 635 unsigned long timeout, start; 636 637 if (!IS_DGFX(xe)) 638 return 0; 639 640 if (IS_SRIOV_VF(xe)) 641 return 0; 642 643 if (verify_lmem_ready(xe)) 644 return 0; 645 646 drm_dbg(&xe->drm, "Waiting for lmem initialization\n"); 647 648 start = jiffies; 649 timeout = start + secs_to_jiffies(60); /* 60 sec! */ 650 651 do { 652 if (signal_pending(current)) 653 return -EINTR; 654 655 /* 656 * The boot firmware initializes local memory and 657 * assesses its health. If memory training fails, 658 * the punit will have been instructed to keep the GT powered 659 * down.we won't be able to communicate with it 660 * 661 * If the status check is done before punit updates the register, 662 * it can lead to the system being unusable. 663 * use a timeout and defer the probe to prevent this. 664 */ 665 if (time_after(jiffies, timeout)) { 666 drm_dbg(&xe->drm, "lmem not initialized by firmware\n"); 667 return -EPROBE_DEFER; 668 } 669 670 msleep(20); 671 672 } while (!verify_lmem_ready(xe)); 673 674 drm_dbg(&xe->drm, "lmem ready after %ums", 675 jiffies_to_msecs(jiffies - start)); 676 677 return 0; 678 } 679 ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */ 680 681 static void sriov_update_device_info(struct xe_device *xe) 682 { 683 /* disable features that are not available/applicable to VFs */ 684 if (IS_SRIOV_VF(xe)) { 685 xe->info.probe_display = 0; 686 xe->info.has_heci_gscfi = 0; 687 xe->info.skip_guc_pc = 1; 688 xe->info.skip_pcode = 1; 689 } 690 } 691 692 /** 693 * xe_device_probe_early: Device early probe 694 * @xe: xe device instance 695 * 696 * Initialize MMIO resources that don't require any 697 * knowledge about tile count. Also initialize pcode and 698 * check vram initialization on root tile. 699 * 700 * Return: 0 on success, error code on failure 701 */ 702 int xe_device_probe_early(struct xe_device *xe) 703 { 704 int err; 705 706 err = xe_mmio_probe_early(xe); 707 if (err) 708 return err; 709 710 xe_sriov_probe_early(xe); 711 712 sriov_update_device_info(xe); 713 714 err = xe_pcode_probe_early(xe); 715 if (err || xe_survivability_mode_is_requested(xe)) { 716 int save_err = err; 717 718 /* 719 * Try to leave device in survivability mode if device is 720 * possible, but still return the previous error for error 721 * propagation 722 */ 723 err = xe_survivability_mode_enable(xe); 724 if (err) 725 return err; 726 727 return save_err; 728 } 729 730 err = wait_for_lmem_ready(xe); 731 if (err) 732 return err; 733 734 xe->wedged.mode = xe_modparam.wedged_mode; 735 736 return 0; 737 } 738 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ 739 740 static int probe_has_flat_ccs(struct xe_device *xe) 741 { 742 struct xe_gt *gt; 743 unsigned int fw_ref; 744 u32 reg; 745 746 /* Always enabled/disabled, no runtime check to do */ 747 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe)) 748 return 0; 749 750 gt = xe_root_mmio_gt(xe); 751 752 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 753 if (!fw_ref) 754 return -ETIMEDOUT; 755 756 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 757 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE); 758 759 if (!xe->info.has_flat_ccs) 760 drm_dbg(&xe->drm, 761 "Flat CCS has been disabled in bios, May lead to performance impact"); 762 763 xe_force_wake_put(gt_to_fw(gt), fw_ref); 764 765 return 0; 766 } 767 768 int xe_device_probe(struct xe_device *xe) 769 { 770 struct xe_tile *tile; 771 struct xe_gt *gt; 772 int err; 773 u8 id; 774 775 xe_pat_init_early(xe); 776 777 err = xe_sriov_init(xe); 778 if (err) 779 return err; 780 781 xe->info.mem_region_mask = 1; 782 783 err = xe_set_dma_info(xe); 784 if (err) 785 return err; 786 787 err = xe_mmio_probe_tiles(xe); 788 if (err) 789 return err; 790 791 err = xe_ttm_sys_mgr_init(xe); 792 if (err) 793 return err; 794 795 for_each_gt(gt, xe, id) { 796 err = xe_gt_init_early(gt); 797 if (err) 798 return err; 799 800 /* 801 * Only after this point can GT-specific MMIO operations 802 * (including things like communication with the GuC) 803 * be performed. 804 */ 805 xe_gt_mmio_init(gt); 806 } 807 808 for_each_tile(tile, xe, id) { 809 if (IS_SRIOV_VF(xe)) { 810 xe_guc_comm_init_early(&tile->primary_gt->uc.guc); 811 err = xe_gt_sriov_vf_bootstrap(tile->primary_gt); 812 if (err) 813 return err; 814 err = xe_gt_sriov_vf_query_config(tile->primary_gt); 815 if (err) 816 return err; 817 } 818 err = xe_ggtt_init_early(tile->mem.ggtt); 819 if (err) 820 return err; 821 err = xe_memirq_init(&tile->memirq); 822 if (err) 823 return err; 824 } 825 826 for_each_gt(gt, xe, id) { 827 err = xe_gt_init_hwconfig(gt); 828 if (err) 829 return err; 830 } 831 832 err = xe_devcoredump_init(xe); 833 if (err) 834 return err; 835 836 /* 837 * From here on, if a step fails, make sure a Driver-FLR is triggereed 838 */ 839 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe); 840 if (err) 841 return err; 842 843 err = probe_has_flat_ccs(xe); 844 if (err) 845 return err; 846 847 err = xe_vram_probe(xe); 848 if (err) 849 return err; 850 851 for_each_tile(tile, xe, id) { 852 err = xe_tile_init_noalloc(tile); 853 if (err) 854 return err; 855 } 856 857 /* Allocate and map stolen after potential VRAM resize */ 858 err = xe_ttm_stolen_mgr_init(xe); 859 if (err) 860 return err; 861 862 /* 863 * Now that GT is initialized (TTM in particular), 864 * we can try to init display, and inherit the initial fb. 865 * This is the reason the first allocation needs to be done 866 * inside display. 867 */ 868 err = xe_display_init_early(xe); 869 if (err) 870 return err; 871 872 for_each_tile(tile, xe, id) { 873 err = xe_tile_init(tile); 874 if (err) 875 return err; 876 } 877 878 err = xe_irq_install(xe); 879 if (err) 880 return err; 881 882 for_each_gt(gt, xe, id) { 883 err = xe_gt_init(gt); 884 if (err) 885 return err; 886 } 887 888 err = xe_heci_gsc_init(xe); 889 if (err) 890 return err; 891 892 err = xe_oa_init(xe); 893 if (err) 894 return err; 895 896 err = xe_display_init(xe); 897 if (err) 898 return err; 899 900 err = xe_pxp_init(xe); 901 if (err) 902 return err; 903 904 err = drm_dev_register(&xe->drm, 0); 905 if (err) 906 return err; 907 908 xe_display_register(xe); 909 910 err = xe_oa_register(xe); 911 if (err) 912 goto err_unregister_display; 913 914 err = xe_pmu_register(&xe->pmu); 915 if (err) 916 goto err_unregister_display; 917 918 xe_debugfs_register(xe); 919 920 err = xe_hwmon_register(xe); 921 if (err) 922 goto err_unregister_display; 923 924 for_each_gt(gt, xe, id) 925 xe_gt_sanitize_freq(gt); 926 927 xe_vsec_init(xe); 928 929 return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); 930 931 err_unregister_display: 932 xe_display_unregister(xe); 933 934 return err; 935 } 936 937 void xe_device_remove(struct xe_device *xe) 938 { 939 xe_display_unregister(xe); 940 941 drm_dev_unplug(&xe->drm); 942 943 xe_bo_pci_dev_remove_all(xe); 944 } 945 946 void xe_device_shutdown(struct xe_device *xe) 947 { 948 struct xe_gt *gt; 949 u8 id; 950 951 drm_dbg(&xe->drm, "Shutting down device\n"); 952 953 if (xe_driver_flr_disabled(xe)) { 954 xe_display_pm_shutdown(xe); 955 956 xe_irq_suspend(xe); 957 958 for_each_gt(gt, xe, id) 959 xe_gt_shutdown(gt); 960 961 xe_display_pm_shutdown_late(xe); 962 } else { 963 /* BOOM! */ 964 __xe_driver_flr(xe); 965 } 966 } 967 968 /** 969 * xe_device_wmb() - Device specific write memory barrier 970 * @xe: the &xe_device 971 * 972 * While wmb() is sufficient for a barrier if we use system memory, on discrete 973 * platforms with device memory we additionally need to issue a register write. 974 * Since it doesn't matter which register we write to, use the read-only VF_CAP 975 * register that is also marked as accessible by the VFs. 976 */ 977 void xe_device_wmb(struct xe_device *xe) 978 { 979 wmb(); 980 if (IS_DGFX(xe)) 981 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0); 982 } 983 984 /** 985 * xe_device_td_flush() - Flush transient L3 cache entries 986 * @xe: The device 987 * 988 * Display engine has direct access to memory and is never coherent with L3/L4 989 * caches (or CPU caches), however KMD is responsible for specifically flushing 990 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout 991 * can happen from such a surface without seeing corruption. 992 * 993 * Display surfaces can be tagged as transient by mapping it using one of the 994 * various L3:XD PAT index modes on Xe2. 995 * 996 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed 997 * at the end of each submission via PIPE_CONTROL for compute/render, since SA 998 * Media is not coherent with L3 and we want to support render-vs-media 999 * usescases. For other engines like copy/blt the HW internally forces uncached 1000 * behaviour, hence why we can skip the TDF on such platforms. 1001 */ 1002 void xe_device_td_flush(struct xe_device *xe) 1003 { 1004 struct xe_gt *gt; 1005 unsigned int fw_ref; 1006 u8 id; 1007 1008 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20) 1009 return; 1010 1011 if (XE_WA(xe_root_mmio_gt(xe), 16023588340)) { 1012 xe_device_l2_flush(xe); 1013 return; 1014 } 1015 1016 for_each_gt(gt, xe, id) { 1017 if (xe_gt_is_media_type(gt)) 1018 continue; 1019 1020 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 1021 if (!fw_ref) 1022 return; 1023 1024 xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); 1025 /* 1026 * FIXME: We can likely do better here with our choice of 1027 * timeout. Currently we just assume the worst case, i.e. 150us, 1028 * which is believed to be sufficient to cover the worst case 1029 * scenario on current platforms if all cache entries are 1030 * transient and need to be flushed.. 1031 */ 1032 if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 1033 150, NULL, false)) 1034 xe_gt_err_once(gt, "TD flush timeout\n"); 1035 1036 xe_force_wake_put(gt_to_fw(gt), fw_ref); 1037 } 1038 } 1039 1040 void xe_device_l2_flush(struct xe_device *xe) 1041 { 1042 struct xe_gt *gt; 1043 unsigned int fw_ref; 1044 1045 gt = xe_root_mmio_gt(xe); 1046 1047 if (!XE_WA(gt, 16023588340)) 1048 return; 1049 1050 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 1051 if (!fw_ref) 1052 return; 1053 1054 spin_lock(>->global_invl_lock); 1055 xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1); 1056 1057 if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 500, NULL, true)) 1058 xe_gt_err_once(gt, "Global invalidation timeout\n"); 1059 spin_unlock(>->global_invl_lock); 1060 1061 xe_force_wake_put(gt_to_fw(gt), fw_ref); 1062 } 1063 1064 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) 1065 { 1066 return xe_device_has_flat_ccs(xe) ? 1067 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0; 1068 } 1069 1070 /** 1071 * xe_device_assert_mem_access - Inspect the current runtime_pm state. 1072 * @xe: xe device instance 1073 * 1074 * To be used before any kind of memory access. It will splat a debug warning 1075 * if the device is currently sleeping. But it doesn't guarantee in any way 1076 * that the device is going to remain awake. Xe PM runtime get and put 1077 * functions might be added to the outer bound of the memory access, while 1078 * this check is intended for inner usage to splat some warning if the worst 1079 * case has just happened. 1080 */ 1081 void xe_device_assert_mem_access(struct xe_device *xe) 1082 { 1083 xe_assert(xe, !xe_pm_runtime_suspended(xe)); 1084 } 1085 1086 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p) 1087 { 1088 struct xe_gt *gt; 1089 u8 id; 1090 1091 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid); 1092 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid); 1093 1094 for_each_gt(gt, xe, id) { 1095 drm_printf(p, "GT id: %u\n", id); 1096 drm_printf(p, "\tTile: %u\n", gt->tile->id); 1097 drm_printf(p, "\tType: %s\n", 1098 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media"); 1099 drm_printf(p, "\tIP ver: %u.%u.%u\n", 1100 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid), 1101 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid), 1102 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid)); 1103 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock); 1104 } 1105 } 1106 1107 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address) 1108 { 1109 return sign_extend64(address, xe->info.va_bits - 1); 1110 } 1111 1112 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) 1113 { 1114 return address & GENMASK_ULL(xe->info.va_bits - 1, 0); 1115 } 1116 1117 static void xe_device_wedged_fini(struct drm_device *drm, void *arg) 1118 { 1119 struct xe_device *xe = arg; 1120 1121 xe_pm_runtime_put(xe); 1122 } 1123 1124 /** 1125 * xe_device_declare_wedged - Declare device wedged 1126 * @xe: xe device instance 1127 * 1128 * This is a final state that can only be cleared with a module 1129 * re-probe (unbind + bind). 1130 * In this state every IOCTL will be blocked so the GT cannot be used. 1131 * In general it will be called upon any critical error such as gt reset 1132 * failure or guc loading failure. Userspace will be notified of this state 1133 * through device wedged uevent. 1134 * If xe.wedged module parameter is set to 2, this function will be called 1135 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump 1136 * snapshot capture. In this mode, GT reset won't be attempted so the state of 1137 * the issue is preserved for further debugging. 1138 */ 1139 void xe_device_declare_wedged(struct xe_device *xe) 1140 { 1141 struct xe_gt *gt; 1142 u8 id; 1143 1144 if (xe->wedged.mode == 0) { 1145 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); 1146 return; 1147 } 1148 1149 xe_pm_runtime_get_noresume(xe); 1150 1151 if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) { 1152 drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n"); 1153 return; 1154 } 1155 1156 if (!atomic_xchg(&xe->wedged.flag, 1)) { 1157 xe->needs_flr_on_fini = true; 1158 drm_err(&xe->drm, 1159 "CRITICAL: Xe has declared device %s as wedged.\n" 1160 "IOCTLs and executions are blocked. Only a rebind may clear the failure\n" 1161 "Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n", 1162 dev_name(xe->drm.dev)); 1163 1164 /* Notify userspace of wedged device */ 1165 drm_dev_wedged_event(&xe->drm, 1166 DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); 1167 } 1168 1169 for_each_gt(gt, xe, id) 1170 xe_gt_declare_wedged(gt); 1171 } 1172