1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_device.h" 7 8 #include <linux/aperture.h> 9 #include <linux/delay.h> 10 #include <linux/error-injection.h> 11 #include <linux/units.h> 12 13 #include <drm/drm_client.h> 14 #include <drm/drm_gem_ttm_helper.h> 15 #include <drm/drm_ioctl.h> 16 #include <drm/drm_managed.h> 17 #include <drm/drm_pagemap_util.h> 18 #include <drm/drm_print.h> 19 #include <kunit/static_stub.h> 20 #include <uapi/drm/xe_drm.h> 21 22 #include "display/xe_display.h" 23 #include "instructions/xe_gpu_commands.h" 24 #include "regs/xe_gt_regs.h" 25 #include "regs/xe_regs.h" 26 #include "xe_bo.h" 27 #include "xe_bo_evict.h" 28 #include "xe_configfs.h" 29 #include "xe_debugfs.h" 30 #include "xe_cpu_bind.h" 31 #include "xe_defaults.h" 32 #include "xe_devcoredump.h" 33 #include "xe_device_sysfs.h" 34 #include "xe_dma_buf.h" 35 #include "xe_drm_client.h" 36 #include "xe_drv.h" 37 #include "xe_exec.h" 38 #include "xe_exec_queue.h" 39 #include "xe_force_wake.h" 40 #include "xe_ggtt.h" 41 #include "xe_gt.h" 42 #include "xe_gt_mcr.h" 43 #include "xe_gt_printk.h" 44 #include "xe_gt_sriov_vf.h" 45 #include "xe_guc.h" 46 #include "xe_guc_pc.h" 47 #include "xe_hw_engine_group.h" 48 #include "xe_hwmon.h" 49 #include "xe_i2c.h" 50 #include "xe_irq.h" 51 #include "xe_late_bind_fw.h" 52 #include "xe_log.h" 53 #include "xe_mmio.h" 54 #include "xe_mmio_gem.h" 55 #include "xe_module.h" 56 #include "xe_nvm.h" 57 #include "xe_oa.h" 58 #include "xe_observation.h" 59 #include "xe_pagefault.h" 60 #include "xe_pat.h" 61 #include "xe_pcode.h" 62 #include "xe_pm.h" 63 #include "xe_pmu.h" 64 #include "xe_psmi.h" 65 #include "xe_pxp.h" 66 #include "xe_query.h" 67 #include "xe_ras.h" 68 #include "xe_shrinker.h" 69 #include "xe_soc_remapper.h" 70 #include "xe_survivability_mode.h" 71 #include "xe_sriov.h" 72 #include "xe_svm.h" 73 #include "xe_sysctrl.h" 74 #include "xe_tile.h" 75 #include "xe_ttm_stolen_mgr.h" 76 #include "xe_ttm_sys_mgr.h" 77 #include "xe_vm.h" 78 #include "xe_vm_madvise.h" 79 #include "xe_vram.h" 80 #include "xe_vram_types.h" 81 #include "xe_vsec.h" 82 #include "xe_wait_user_fence.h" 83 #include "xe_wa.h" 84 85 #include <generated/xe_device_wa_oob.h> 86 #include <generated/xe_wa_oob.h> 87 88 static int xe_file_open(struct drm_device *dev, struct drm_file *file) 89 { 90 struct xe_device *xe = to_xe_device(dev); 91 struct xe_drm_client *client; 92 struct xe_file *xef; 93 int ret = -ENOMEM; 94 struct task_struct *task = NULL; 95 96 xef = kzalloc_obj(*xef); 97 if (!xef) 98 return ret; 99 100 client = xe_drm_client_alloc(); 101 if (!client) { 102 kfree(xef); 103 return ret; 104 } 105 106 xef->drm = file; 107 xef->client = client; 108 xef->xe = xe; 109 110 mutex_init(&xef->vm.lock); 111 xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); 112 113 mutex_init(&xef->exec_queue.lock); 114 xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); 115 116 mutex_init(&xef->mmio_gem.lock); 117 118 file->driver_priv = xef; 119 kref_init(&xef->refcount); 120 121 task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID); 122 if (task) { 123 xef->process_name = kstrdup(task->comm, GFP_KERNEL); 124 xef->pid = task->pid; 125 put_task_struct(task); 126 } 127 128 return 0; 129 } 130 131 static void xe_file_destroy(struct kref *ref) 132 { 133 struct xe_file *xef = container_of(ref, struct xe_file, refcount); 134 135 xa_destroy(&xef->exec_queue.xa); 136 mutex_destroy(&xef->exec_queue.lock); 137 xa_destroy(&xef->vm.xa); 138 mutex_destroy(&xef->vm.lock); 139 140 mutex_destroy(&xef->mmio_gem.lock); 141 142 xe_drm_client_put(xef->client); 143 kfree(xef->process_name); 144 kfree(xef); 145 } 146 147 /** 148 * xe_file_get() - Take a reference to the xe file object 149 * @xef: Pointer to the xe file 150 * 151 * Anyone with a pointer to xef must take a reference to the xe file 152 * object using this call. 153 * 154 * Return: xe file pointer 155 */ 156 struct xe_file *xe_file_get(struct xe_file *xef) 157 { 158 kref_get(&xef->refcount); 159 return xef; 160 } 161 162 /** 163 * xe_file_put() - Drop a reference to the xe file object 164 * @xef: Pointer to the xe file 165 * 166 * Used to drop reference to the xef object 167 */ 168 void xe_file_put(struct xe_file *xef) 169 { 170 kref_put(&xef->refcount, xe_file_destroy); 171 } 172 173 static void xe_file_close(struct drm_device *dev, struct drm_file *file) 174 { 175 struct xe_device *xe = to_xe_device(dev); 176 struct xe_file *xef = file->driver_priv; 177 struct xe_vm *vm; 178 struct xe_exec_queue *q; 179 unsigned long idx; 180 181 guard(xe_pm_runtime)(xe); 182 183 /* 184 * No need for exec_queue.lock here as there is no contention for it 185 * when FD is closing as IOCTLs presumably can't be modifying the 186 * xarray. Taking exec_queue.lock here causes undue dependency on 187 * vm->lock taken during xe_exec_queue_kill(). 188 */ 189 xa_for_each(&xef->exec_queue.xa, idx, q) { 190 if (q->vm && q->hwe->hw_engine_group) 191 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); 192 xe_exec_queue_kill(q); 193 xe_exec_queue_put(q); 194 } 195 xa_for_each(&xef->vm.xa, idx, vm) 196 xe_vm_close_and_put(vm); 197 198 scoped_guard(mutex, &xef->mmio_gem.lock) { 199 if (xef->mmio_gem.pci_barrier) { 200 xe_mmio_gem_destroy(xef->mmio_gem.pci_barrier, file); 201 xef->mmio_gem.pci_barrier = NULL; 202 } 203 } 204 205 xe_file_put(xef); 206 } 207 208 static const struct drm_ioctl_desc xe_ioctls[] = { 209 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 210 DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), 211 DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, 212 DRM_RENDER_ALLOW), 213 DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), 214 DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), 215 DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), 216 DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), 217 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, 218 DRM_RENDER_ALLOW), 219 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, 220 DRM_RENDER_ALLOW), 221 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, 222 DRM_RENDER_ALLOW), 223 DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, 224 DRM_RENDER_ALLOW), 225 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 226 DRM_IOCTL_DEF_DRV(XE_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), 227 DRM_IOCTL_DEF_DRV(XE_VM_QUERY_MEM_RANGE_ATTRS, xe_vm_query_vmas_attrs_ioctl, 228 DRM_RENDER_ALLOW), 229 DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, 230 DRM_RENDER_ALLOW), 231 DRM_IOCTL_DEF_DRV(XE_VM_GET_PROPERTY, xe_vm_get_property_ioctl, 232 DRM_RENDER_ALLOW), 233 }; 234 235 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 236 { 237 struct drm_file *file_priv = file->private_data; 238 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 239 long ret; 240 241 if (xe_device_wedged(xe)) 242 return -ECANCELED; 243 244 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); 245 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); 246 if (ret >= 0) 247 ret = drm_ioctl(file, cmd, arg); 248 249 return ret; 250 } 251 252 #ifdef CONFIG_COMPAT 253 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 254 { 255 struct drm_file *file_priv = file->private_data; 256 struct xe_device *xe = to_xe_device(file_priv->minor->dev); 257 long ret; 258 259 if (xe_device_wedged(xe)) 260 return -ECANCELED; 261 262 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); 263 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); 264 if (ret >= 0) 265 ret = drm_compat_ioctl(file, cmd, arg); 266 267 return ret; 268 } 269 #else 270 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */ 271 #define xe_drm_compat_ioctl NULL 272 #endif 273 274 static int xe_mmap(struct file *filp, struct vm_area_struct *vma) 275 { 276 struct drm_file *priv = filp->private_data; 277 struct drm_device *dev = priv->minor->dev; 278 279 if (drm_dev_is_unplugged(dev)) 280 return -ENODEV; 281 282 return drm_gem_mmap(filp, vma); 283 } 284 285 static const struct file_operations xe_driver_fops = { 286 .owner = THIS_MODULE, 287 .open = drm_open, 288 .release = drm_release_noglobal, 289 .unlocked_ioctl = xe_drm_ioctl, 290 .mmap = xe_mmap, 291 .poll = drm_poll, 292 .read = drm_read, 293 .compat_ioctl = xe_drm_compat_ioctl, 294 .llseek = noop_llseek, 295 #ifdef CONFIG_PROC_FS 296 .show_fdinfo = drm_show_fdinfo, 297 #endif 298 .fop_flags = FOP_UNSIGNED_OFFSET, 299 }; 300 301 /** 302 * xe_is_xe_file() - Is the file an xe device file? 303 * @file: The file. 304 * 305 * Checks whether the file is opened against 306 * an xe device. 307 * 308 * Return: %true if an xe file, %false if not. 309 */ 310 bool xe_is_xe_file(const struct file *file) 311 { 312 return file->f_op == &xe_driver_fops; 313 } 314 315 static const struct drm_driver regular_driver = { 316 .driver_features = 317 XE_DISPLAY_DRIVER_FEATURES | 318 DRIVER_GEM | 319 DRIVER_RENDER | DRIVER_SYNCOBJ | 320 DRIVER_SYNCOBJ_TIMELINE, 321 .open = xe_file_open, 322 .postclose = xe_file_close, 323 324 .gem_prime_import = xe_gem_prime_import, 325 326 .dumb_create = xe_bo_dumb_create, 327 .dumb_map_offset = drm_gem_ttm_dumb_map_offset, 328 #ifdef CONFIG_PROC_FS 329 .show_fdinfo = xe_drm_client_fdinfo, 330 #endif 331 .ioctls = xe_ioctls, 332 .num_ioctls = ARRAY_SIZE(xe_ioctls), 333 .fops = &xe_driver_fops, 334 .name = DRIVER_NAME, 335 .desc = DRIVER_DESC, 336 .major = DRIVER_MAJOR, 337 .minor = DRIVER_MINOR, 338 .patchlevel = DRIVER_PATCHLEVEL, 339 XE_DISPLAY_DRIVER_OPS, 340 }; 341 342 #ifdef CONFIG_PCI_IOV 343 static const struct drm_ioctl_desc xe_ioctls_admin_only[] = { 344 DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), 345 DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW), 346 }; 347 348 static const struct drm_driver admin_only_driver = { 349 .driver_features = 350 DRIVER_GEM | DRIVER_RENDER, 351 .open = xe_file_open, 352 .postclose = xe_file_close, 353 .ioctls = xe_ioctls_admin_only, 354 .num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only), 355 .fops = &xe_driver_fops, 356 .name = DRIVER_NAME, 357 .desc = DRIVER_DESC, 358 .major = DRIVER_MAJOR, 359 .minor = DRIVER_MINOR, 360 .patchlevel = DRIVER_PATCHLEVEL, 361 }; 362 363 /** 364 * xe_device_is_admin_only() - Check whether device is admin only or not. 365 * @xe: the &xe_device to check 366 * 367 * Return: true if the device is admin only, false otherwise. 368 */ 369 bool xe_device_is_admin_only(const struct xe_device *xe) 370 { 371 KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe); 372 return xe->drm.driver == &admin_only_driver; 373 } 374 #endif 375 376 static void xe_device_destroy(struct drm_device *dev, void *dummy) 377 { 378 struct xe_device *xe = to_xe_device(dev); 379 380 xe_bo_dev_fini(&xe->bo_device); 381 382 if (xe->preempt_fence_wq) 383 destroy_workqueue(xe->preempt_fence_wq); 384 385 if (xe->ordered_wq) 386 destroy_workqueue(xe->ordered_wq); 387 388 if (xe->unordered_wq) 389 destroy_workqueue(xe->unordered_wq); 390 391 if (xe->destroy_wq) 392 destroy_workqueue(xe->destroy_wq); 393 394 ttm_device_fini(&xe->ttm); 395 } 396 397 /** 398 * xe_device_create() - Create a new &xe_device instance 399 * @pdev: the parent &pci_dev 400 * 401 * Allocate and initialize a device managed Xe device structure. 402 * 403 * Return: pointer to new &xe_device on success, or ERR_PTR on failure. 404 */ 405 struct xe_device *xe_device_create(struct pci_dev *pdev) 406 { 407 const struct drm_driver *driver = ®ular_driver; 408 struct xe_device *xe; 409 int err; 410 411 #ifdef CONFIG_PCI_IOV 412 /* 413 * Since XE device is not initialized yet, read from configfs 414 * directly to decide whether we are in admin-only PF mode or not. 415 */ 416 if (xe_configfs_admin_only_pf(pdev)) 417 driver = &admin_only_driver; 418 #endif 419 420 err = aperture_remove_conflicting_pci_devices(pdev, driver->name); 421 if (err) 422 return ERR_PTR(err); 423 424 xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm); 425 if (IS_ERR(xe)) 426 return xe; 427 428 err = xe_device_init_early(xe); 429 if (err) 430 return ERR_PTR(err); 431 432 return xe; 433 } 434 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */ 435 436 static void xe_device_parse_modparam(struct xe_device *xe) 437 { 438 xe->atomic_svm_timeslice_ms = 5; 439 xe->min_run_period_lr_ms = 5; 440 xe->info.num_pf_work = xe_modparam.num_pf_work; 441 if (xe->info.num_pf_work < 1) 442 xe->info.num_pf_work = 1; 443 else if (xe->info.num_pf_work > XE_PAGEFAULT_WORK_MAX) 444 xe->info.num_pf_work = XE_PAGEFAULT_WORK_MAX; 445 } 446 447 /** 448 * xe_device_init_early() - Initialize a new &xe_device instance 449 * @xe: the &xe_device to initialize 450 * 451 * Return: 0 on success or a negative error code on failure. 452 */ 453 int xe_device_init_early(struct xe_device *xe) 454 { 455 int err; 456 457 err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, 458 xe->drm.anon_inode->i_mapping, 459 xe->drm.vma_offset_manager, 460 TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M))); 461 if (err) 462 return err; 463 464 xe_bo_dev_init(&xe->bo_device); 465 err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); 466 if (err) 467 return err; 468 469 err = xe_shrinker_create(xe); 470 if (err) 471 return err; 472 473 xe_device_parse_modparam(xe); 474 475 err = xe_irq_init(xe); 476 if (err) 477 return err; 478 479 xe_validation_device_init(&xe->val); 480 481 init_waitqueue_head(&xe->ufence_wq); 482 483 init_rwsem(&xe->usm.lock); 484 485 err = xe_pagemap_shrinker_create(xe); 486 if (err) 487 return err; 488 489 xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC); 490 491 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 492 /* Trigger a large asid and an early asid wrap. */ 493 u32 asid; 494 495 BUILD_BUG_ON(XE_MAX_ASID < 2); 496 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL, 497 XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1), 498 &xe->usm.next_asid, GFP_KERNEL); 499 drm_WARN_ON(&xe->drm, err); 500 if (err >= 0) 501 xa_erase(&xe->usm.asid_to_vm, asid); 502 } 503 504 err = xe_bo_pinned_init(xe); 505 if (err) 506 return err; 507 508 xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", 509 WQ_MEM_RECLAIM); 510 xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); 511 xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0); 512 xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0); 513 if (!xe->ordered_wq || !xe->unordered_wq || 514 !xe->preempt_fence_wq || !xe->destroy_wq) { 515 /* 516 * Cleanup done in xe_device_destroy via 517 * drmm_add_action_or_reset register above 518 */ 519 drm_err(&xe->drm, "Failed to allocate xe workqueues\n"); 520 return -ENOMEM; 521 } 522 523 err = drmm_mutex_init(&xe->drm, &xe->pmt.lock); 524 if (err) 525 return err; 526 527 err = xe_pm_init_early(xe); 528 if (err) 529 return err; 530 531 return 0; 532 } 533 534 static bool xe_driver_flr_disabled(struct xe_device *xe) 535 { 536 if (IS_SRIOV_VF(xe)) 537 return true; 538 539 if (xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS) { 540 drm_info(&xe->drm, "Driver-FLR disabled by BIOS\n"); 541 return true; 542 } 543 544 return false; 545 } 546 547 /* 548 * The driver-initiated FLR is the highest level of reset that we can trigger 549 * from within the driver. It is different from the PCI FLR in that it doesn't 550 * fully reset the SGUnit and doesn't modify the PCI config space and therefore 551 * it doesn't require a re-enumeration of the PCI BARs. However, the 552 * driver-initiated FLR does still cause a reset of both GT and display and a 553 * memory wipe of local and stolen memory, so recovery would require a full HW 554 * re-init and saving/restoring (or re-populating) the wiped memory. Since we 555 * perform the FLR as the very last action before releasing access to the HW 556 * during the driver release flow, we don't attempt recovery at all, because 557 * if/when a new instance of Xe is bound to the device it will do a full 558 * re-init anyway. 559 */ 560 static void __xe_driver_flr(struct xe_device *xe) 561 { 562 const unsigned int flr_timeout = 3 * USEC_PER_SEC; /* specs recommend a 3s wait */ 563 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 564 int ret; 565 566 drm_dbg(&xe->drm, "Triggering Driver-FLR\n"); 567 568 /* 569 * Make sure any pending FLR requests have cleared by waiting for the 570 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS 571 * to make sure it's not still set from a prior attempt (it's a write to 572 * clear bit). 573 * Note that we should never be in a situation where a previous attempt 574 * is still pending (unless the HW is totally dead), but better to be 575 * safe in case something unexpected happens 576 */ 577 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 578 if (ret) { 579 drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret); 580 return; 581 } 582 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 583 584 /* Trigger the actual Driver-FLR */ 585 xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR); 586 587 /* Wait for hardware teardown to complete */ 588 ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false); 589 if (ret) { 590 drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret); 591 return; 592 } 593 594 /* Wait for hardware/firmware re-init to complete */ 595 ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS, 596 flr_timeout, NULL, false); 597 if (ret) { 598 drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret); 599 return; 600 } 601 602 /* Clear sticky completion status */ 603 xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS); 604 } 605 606 static void xe_driver_flr(struct xe_device *xe) 607 { 608 if (xe_driver_flr_disabled(xe)) 609 return; 610 611 __xe_driver_flr(xe); 612 } 613 614 static void xe_driver_flr_fini(void *arg) 615 { 616 struct xe_device *xe = arg; 617 618 if (xe->needs_flr_on_fini) 619 xe_driver_flr(xe); 620 } 621 622 static void xe_device_sanitize(void *arg) 623 { 624 struct xe_device *xe = arg; 625 struct xe_gt *gt; 626 u8 id; 627 628 for_each_gt(gt, xe, id) 629 xe_gt_sanitize(gt); 630 } 631 632 static int xe_set_dma_info(struct xe_device *xe) 633 { 634 unsigned int mask_size = xe->info.dma_mask_size; 635 int err; 636 637 dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev)); 638 639 err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 640 if (err) 641 goto mask_err; 642 643 err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); 644 if (err) 645 goto mask_err; 646 647 return 0; 648 649 mask_err: 650 drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); 651 return err; 652 } 653 654 static void assert_lmem_ready(struct xe_device *xe) 655 { 656 if (!IS_DGFX(xe) || IS_SRIOV_VF(xe)) 657 return; 658 659 xe_assert(xe, xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & 660 LMEM_INIT); 661 } 662 663 static void vf_update_device_info(struct xe_device *xe) 664 { 665 xe_assert(xe, IS_SRIOV_VF(xe)); 666 /* disable features that are not available/applicable to VFs */ 667 xe->info.probe_display = 0; 668 xe->info.has_heci_cscfi = 0; 669 xe->info.has_heci_gscfi = 0; 670 xe->info.has_i2c = 0; 671 xe->info.has_late_bind = 0; 672 xe->info.skip_guc_pc = 1; 673 xe->info.skip_pcode = 1; 674 xe->info.has_drm_ras = false; 675 xe->info.has_device_uid = false; 676 } 677 678 static int xe_device_vram_alloc(struct xe_device *xe) 679 { 680 struct xe_vram_region *vram; 681 682 if (!IS_DGFX(xe)) 683 return 0; 684 685 vram = drmm_kzalloc(&xe->drm, sizeof(*vram), GFP_KERNEL); 686 if (!vram) 687 return -ENOMEM; 688 689 xe->mem.vram = vram; 690 return 0; 691 } 692 693 /** 694 * xe_device_probe_early: Device early probe 695 * @xe: xe device instance 696 * 697 * Initialize MMIO resources that don't require any 698 * knowledge about tile count. Also initialize pcode and 699 * check vram initialization on root tile. 700 * 701 * Return: 0 on success, error code on failure 702 */ 703 int xe_device_probe_early(struct xe_device *xe) 704 { 705 int err; 706 707 xe_wa_device_init(xe); 708 xe_wa_process_device_oob(xe); 709 710 err = xe_mmio_probe_early(xe); 711 if (err) 712 return err; 713 714 xe_sriov_probe_early(xe); 715 716 if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) { 717 xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n"); 718 return -ENODEV; 719 } 720 721 if (IS_SRIOV_VF(xe)) 722 vf_update_device_info(xe); 723 724 /* 725 * Check for pcode uncore_init status to confirm if the SoC 726 * initialization is complete. Until done, any MMIO or lmem access from 727 * the driver will be blocked 728 */ 729 err = xe_pcode_probe_early(xe); 730 if (err || xe_survivability_mode_is_requested(xe)) { 731 int save_err = err; 732 733 /* 734 * Try to leave device in survivability mode if device is 735 * possible, but still return the previous error for error 736 * propagation 737 */ 738 err = xe_survivability_mode_boot_enable(xe); 739 if (err) 740 return err; 741 742 return save_err; 743 } 744 745 /* 746 * Make sure the lmem is initialized and ready to use. xe_pcode_ready() 747 * is flagged after full initialization is complete. Assert if lmem is 748 * not initialized. 749 */ 750 assert_lmem_ready(xe); 751 752 xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ? 753 XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode; 754 drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n", 755 xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode)); 756 757 err = xe_device_vram_alloc(xe); 758 if (err) 759 return err; 760 761 return 0; 762 } 763 ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ 764 765 static int probe_has_flat_ccs(struct xe_device *xe) 766 { 767 struct xe_gt *gt; 768 u32 reg; 769 770 /* Always enabled/disabled, no runtime check to do */ 771 if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe)) 772 return 0; 773 774 gt = xe_root_mmio_gt(xe); 775 if (!gt) 776 return 0; 777 778 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 779 if (!fw_ref.domains) 780 return -ETIMEDOUT; 781 782 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 783 xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE); 784 785 if (!xe->info.has_flat_ccs) 786 drm_dbg(&xe->drm, 787 "Flat CCS has been disabled in bios, May lead to performance impact"); 788 789 return 0; 790 } 791 792 /* 793 * Detect if the driver is being run on pre-production hardware. We don't 794 * keep workarounds for pre-production hardware long term, so print an 795 * error and add taint if we're being loaded on a pre-production platform 796 * for which the pre-prod workarounds have already been removed. 797 * 798 * The general policy is that we'll remove any workarounds that only apply to 799 * pre-production hardware around the time force_probe restrictions are lifted 800 * for a platform of the next major IP generation (for example, Xe2 pre-prod 801 * workarounds should be removed around the time the first Xe3 platforms have 802 * force_probe lifted). 803 */ 804 static void detect_preproduction_hw(struct xe_device *xe) 805 { 806 struct xe_gt *gt; 807 int id; 808 809 /* 810 * SR-IOV VFs don't have access to the FUSE2 register, so we can't 811 * check pre-production status there. But the host OS will notice 812 * and report the pre-production status, which should be enough to 813 * help us catch mistaken use of pre-production hardware. 814 */ 815 if (IS_SRIOV_VF(xe)) 816 return; 817 818 /* 819 * The "SW_CAP" fuse contains a bit indicating whether the device is a 820 * production or pre-production device. This fuse is reflected through 821 * the GT "FUSE2" register, even though the contents of the fuse are 822 * not GT-specific. Every GT's reflection of this fuse should show the 823 * same value, so we'll just use the first available GT for lookup. 824 */ 825 for_each_gt(gt, xe, id) 826 break; 827 828 if (!gt) 829 return; 830 831 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 832 if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FW_GT)) { 833 xe_gt_err(gt, "Forcewake failure; cannot determine production/pre-production hw status.\n"); 834 return; 835 } 836 837 if (xe_mmio_read32(>->mmio, FUSE2) & PRODUCTION_HW) 838 return; 839 840 xe_info(xe, "Pre-production hardware detected.\n"); 841 if (!xe->info.has_pre_prod_wa) { 842 xe_err(xe, "Pre-production workarounds for this platform have already been removed.\n"); 843 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_STILL_OK); 844 } 845 } 846 847 static void xe_device_wedged_fini(struct drm_device *drm, void *arg) 848 { 849 struct xe_device *xe = arg; 850 851 if (atomic_read(&xe->wedged.flag)) 852 xe_pm_runtime_put(xe); 853 } 854 855 #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE 856 static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe) 857 { 858 int err; 859 860 err = drmm_mutex_init(&xe->drm, &xe->page_size_alloc_ctrl.lock); 861 if (err) 862 return err; 863 864 xe->page_size_alloc_ctrl.mode = XE_PAGE_SIZE_ALLOC_CTRL_MODE_NONE; 865 xe->page_size_alloc_ctrl.cur_index = 0; 866 867 return 0; 868 } 869 #else 870 static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe) 871 { 872 return 0; 873 } 874 #endif 875 876 static void xe_uid_probe(struct xe_device *xe) 877 { 878 if (xe->info.has_device_uid) 879 xe->device_uid = xe_mmio_read64_2x32(xe_root_tile_mmio(xe), CRI_DEVICE_UID); 880 } 881 882 int xe_device_probe(struct xe_device *xe) 883 { 884 struct xe_tile *tile; 885 struct xe_gt *gt; 886 int err; 887 u8 id; 888 889 xe_uid_probe(xe); 890 891 xe_pat_init_early(xe); 892 893 err = xe_sriov_init(xe); 894 if (err) 895 return err; 896 897 xe->info.mem_region_mask = 1; 898 899 err = xe_set_dma_info(xe); 900 if (err) 901 return err; 902 903 err = xe_mmio_probe_tiles(xe); 904 if (err) 905 return err; 906 907 for_each_gt(gt, xe, id) { 908 err = xe_gt_init_early(gt); 909 if (err) 910 return err; 911 } 912 913 /* 914 * Wa_16029380221: The affected GT will always use non-coherent 915 * access to page tables, so we must do uncached writes from the 916 * CPU. 917 */ 918 for_each_gt(gt, xe, id) 919 if (XE_GT_WA(gt, 16029380221)) 920 xe->info.has_cached_pt = false; 921 922 for_each_tile(tile, xe, id) { 923 err = xe_ggtt_init_early(tile->mem.ggtt); 924 if (err) 925 return err; 926 } 927 928 /* 929 * From here on, if a step fails, make sure a Driver-FLR is triggereed 930 */ 931 err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe); 932 if (err) 933 return err; 934 935 err = probe_has_flat_ccs(xe); 936 if (err) 937 return err; 938 939 err = xe_vram_probe(xe); 940 if (err) 941 return err; 942 943 for_each_tile(tile, xe, id) { 944 err = xe_tile_init_noalloc(tile); 945 if (err) 946 return err; 947 } 948 949 /* 950 * Allow allocations only now to ensure xe_display_init_early() 951 * is the first to allocate, always. 952 */ 953 err = xe_ttm_sys_mgr_init(xe); 954 if (err) 955 return err; 956 957 /* Allocate and map stolen after potential VRAM resize */ 958 err = xe_ttm_stolen_mgr_init(xe); 959 if (err) 960 return err; 961 962 err = xe_soc_remapper_init(xe); 963 if (err) 964 return err; 965 966 err = xe_sysctrl_init(xe); 967 if (err) 968 return err; 969 970 xe_ras_init(xe); 971 972 /* 973 * Now that GT is initialized (TTM in particular), 974 * we can try to init display, and inherit the initial fb. 975 * This is the reason the first allocation needs to be done 976 * inside display. 977 */ 978 err = xe_display_init_early(xe); 979 if (err) 980 return err; 981 982 err = xe_vram_reserve_memtest_bo(xe); 983 if (err) 984 return err; 985 986 for_each_tile(tile, xe, id) { 987 err = xe_tile_init(tile); 988 if (err) 989 return err; 990 } 991 992 err = xe_irq_install(xe); 993 if (err) 994 return err; 995 996 for_each_gt(gt, xe, id) { 997 err = xe_gt_init(gt); 998 if (err) 999 return err; 1000 } 1001 1002 err = xe_vram_memtest(xe); 1003 if (err) 1004 return err; 1005 1006 err = xe_cpu_bind_init(xe); 1007 if (err) 1008 return err; 1009 1010 err = xe_pagefault_init(xe); 1011 if (err) 1012 return err; 1013 1014 if (xe->tiles->media_gt && 1015 XE_GT_WA(xe->tiles->media_gt, 15015404425_disable)) 1016 XE_DEVICE_WA_DISABLE(xe, 15015404425); 1017 1018 err = xe_devcoredump_init(xe); 1019 if (err) 1020 return err; 1021 1022 xe_nvm_init(xe); 1023 1024 err = xe_heci_gsc_init(xe); 1025 if (err) 1026 return err; 1027 1028 err = xe_late_bind_init(&xe->late_bind); 1029 if (err) 1030 return err; 1031 1032 err = xe_oa_init(xe); 1033 if (err) 1034 return err; 1035 1036 err = xe_display_init(xe); 1037 if (err) 1038 return err; 1039 1040 err = xe_pxp_init(xe); 1041 if (err) 1042 return err; 1043 1044 err = xe_psmi_init(xe); 1045 if (err) 1046 return err; 1047 1048 err = xe_debug_page_size_alloc_ctrl_init(xe); 1049 if (err) 1050 return err; 1051 1052 err = drm_dev_register(&xe->drm, 0); 1053 if (err) 1054 return err; 1055 1056 xe_display_register(xe); 1057 1058 err = xe_oa_register(xe); 1059 if (err) 1060 goto err_unregister_display; 1061 1062 err = xe_pmu_register(&xe->pmu); 1063 if (err) 1064 goto err_unregister_display; 1065 1066 err = xe_device_sysfs_init(xe); 1067 if (err) 1068 goto err_unregister_display; 1069 1070 xe_debugfs_register(xe); 1071 1072 err = xe_hwmon_register(xe); 1073 if (err) 1074 goto err_unregister_display; 1075 1076 err = xe_i2c_probe(xe); 1077 if (err) 1078 goto err_unregister_display; 1079 1080 for_each_gt(gt, xe, id) 1081 xe_gt_sanitize_freq(gt); 1082 1083 xe_vsec_init(xe); 1084 1085 err = xe_sriov_init_late(xe); 1086 if (err) 1087 goto err_unregister_display; 1088 1089 detect_preproduction_hw(xe); 1090 1091 err = drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe); 1092 if (err) 1093 goto err_unregister_display; 1094 1095 /* 1096 * Process and log any errors detected by hardware. Possible results can 1097 * include declaring the device as wedged, which must be done only after 1098 * xe_device_wedged_fini() is registered. 1099 */ 1100 xe_ras_process_errors(xe); 1101 1102 err = devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); 1103 if (err) 1104 goto err_unregister_display; 1105 1106 return 0; 1107 1108 err_unregister_display: 1109 xe_display_unregister(xe); 1110 drm_dev_unregister(&xe->drm); 1111 1112 return err; 1113 } 1114 1115 void xe_device_remove(struct xe_device *xe) 1116 { 1117 xe_display_unregister(xe); 1118 1119 drm_dev_unplug(&xe->drm); 1120 1121 xe_bo_pci_dev_remove_all(xe); 1122 } 1123 1124 void xe_device_shutdown(struct xe_device *xe) 1125 { 1126 struct xe_gt *gt; 1127 u8 id; 1128 1129 drm_dbg(&xe->drm, "Shutting down device\n"); 1130 1131 xe_display_shutdown(xe); 1132 1133 xe_irq_suspend(xe); 1134 1135 for_each_gt(gt, xe, id) 1136 xe_gt_shutdown(gt); 1137 1138 xe_display_shutdown_late(xe); 1139 1140 if (!xe_driver_flr_disabled(xe)) { 1141 /* BOOM! */ 1142 __xe_driver_flr(xe); 1143 } 1144 } 1145 1146 /** 1147 * xe_device_wmb() - Device specific write memory barrier 1148 * @xe: the &xe_device 1149 * 1150 * While wmb() is sufficient for a barrier if we use system memory, on discrete 1151 * platforms with device memory we additionally need to issue a register write. 1152 * Since it doesn't matter which register we write to, use the read-only VF_CAP 1153 * register that is also marked as accessible by the VFs. 1154 */ 1155 void xe_device_wmb(struct xe_device *xe) 1156 { 1157 wmb(); 1158 if (IS_DGFX(xe)) 1159 xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0); 1160 } 1161 1162 /* 1163 * Issue a TRANSIENT_FLUSH_REQUEST and wait for completion on each gt. 1164 */ 1165 static void tdf_request_sync(struct xe_device *xe) 1166 { 1167 struct xe_gt *gt; 1168 u8 id; 1169 1170 for_each_gt_with_type(gt, xe, id, BIT(XE_GT_TYPE_MAIN)) { 1171 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 1172 if (!fw_ref.domains) 1173 return; 1174 1175 xe_mmio_write32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST); 1176 1177 /* 1178 * FIXME: We can likely do better here with our choice of 1179 * timeout. Currently we just assume the worst case, i.e. 150us, 1180 * which is believed to be sufficient to cover the worst case 1181 * scenario on current platforms if all cache entries are 1182 * transient and need to be flushed.. 1183 */ 1184 if (xe_mmio_wait32(>->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0, 1185 300, NULL, false)) 1186 xe_gt_err_once(gt, "TD flush timeout\n"); 1187 } 1188 } 1189 1190 /** 1191 * xe_device_is_l2_flush_optimized - if L2 flush is optimized by HW 1192 * @xe: The device to check. 1193 * 1194 * Return: true if the HW device optimizing L2 flush, false otherwise. 1195 */ 1196 bool xe_device_is_l2_flush_optimized(struct xe_device *xe) 1197 { 1198 /* XA is *always* flushed, like at the end-of-submssion (and maybe other 1199 * places), just that internally as an optimisation hw doesn't need to make 1200 * that a full flush (which will also include XA) when Media is 1201 * off/powergated, since it doesn't need to worry about GT caches vs Media 1202 * coherency, and only CPU vs GPU coherency, so can make that flush a 1203 * targeted XA flush, since stuff tagged with XA now means it's shared with 1204 * the CPU. The main implication is that we now need to somehow flush non-XA before 1205 * freeing system memory pages, otherwise dirty cachelines could be flushed after the free 1206 * (like if Media suddenly turns on and does a full flush) 1207 */ 1208 if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe)) 1209 return true; 1210 return false; 1211 } 1212 1213 void xe_device_l2_flush(struct xe_device *xe, bool force) 1214 { 1215 struct xe_gt *gt; 1216 1217 gt = xe_root_mmio_gt(xe); 1218 if (!gt) 1219 return; 1220 1221 if (!force && !XE_GT_WA(gt, 16023588340)) 1222 return; 1223 1224 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 1225 if (!fw_ref.domains) 1226 return; 1227 1228 spin_lock(>->global_invl_lock); 1229 1230 xe_mmio_write32(>->mmio, XE2_GLOBAL_INVAL, 0x1); 1231 if (xe_mmio_wait32(>->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 1000, NULL, true)) 1232 xe_gt_err_once(gt, "Global invalidation timeout\n"); 1233 1234 spin_unlock(>->global_invl_lock); 1235 } 1236 1237 /** 1238 * xe_device_td_flush() - Flush transient L3 cache entries 1239 * @xe: The device 1240 * 1241 * Display engine has direct access to memory and is never coherent with L3/L4 1242 * caches (or CPU caches), however KMD is responsible for specifically flushing 1243 * transient L3 GPU cache entries prior to the flip sequence to ensure scanout 1244 * can happen from such a surface without seeing corruption. 1245 * 1246 * Display surfaces can be tagged as transient by mapping it using one of the 1247 * various L3:XD PAT index modes on Xe2. 1248 * 1249 * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed 1250 * at the end of each submission via PIPE_CONTROL for compute/render, since SA 1251 * Media is not coherent with L3 and we want to support render-vs-media 1252 * usescases. For other engines like copy/blt the HW internally forces uncached 1253 * behaviour, hence why we can skip the TDF on such platforms. 1254 */ 1255 void xe_device_td_flush(struct xe_device *xe) 1256 { 1257 struct xe_gt *root_gt; 1258 1259 /* 1260 * From Xe3p onward the HW takes care of flush of TD entries also along 1261 * with flushing XA entries, which will be at the usual sync points, 1262 * like at the end of submission, so no manual flush is needed here. 1263 */ 1264 if (GRAPHICS_VER(xe) >= 35) 1265 return; 1266 1267 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20) 1268 return; 1269 1270 root_gt = xe_root_mmio_gt(xe); 1271 if (!root_gt) 1272 return; 1273 1274 if (XE_GT_WA(root_gt, 16023588340)) { 1275 /* A transient flush is not sufficient: flush the L2 */ 1276 xe_device_l2_flush(xe, false); 1277 } else { 1278 xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc); 1279 tdf_request_sync(xe); 1280 xe_guc_pc_remove_flush_freq_limit(&root_gt->uc.guc.pc); 1281 } 1282 } 1283 1284 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) 1285 { 1286 return xe_device_has_flat_ccs(xe) ? 1287 DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0; 1288 } 1289 1290 /** 1291 * xe_device_assert_mem_access - Inspect the current runtime_pm state. 1292 * @xe: xe device instance 1293 * 1294 * To be used before any kind of memory access. It will splat a debug warning 1295 * if the device is currently sleeping. But it doesn't guarantee in any way 1296 * that the device is going to remain awake. Xe PM runtime get and put 1297 * functions might be added to the outer bound of the memory access, while 1298 * this check is intended for inner usage to splat some warning if the worst 1299 * case has just happened. 1300 */ 1301 void xe_device_assert_mem_access(struct xe_device *xe) 1302 { 1303 xe_assert(xe, !xe_pm_runtime_suspended(xe)); 1304 } 1305 1306 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p) 1307 { 1308 struct xe_gt *gt; 1309 u8 id; 1310 1311 drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid); 1312 drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid); 1313 1314 for_each_gt(gt, xe, id) { 1315 drm_printf(p, "GT id: %u\n", id); 1316 drm_printf(p, "\tTile: %u\n", gt->tile->id); 1317 drm_printf(p, "\tType: %s\n", 1318 gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media"); 1319 drm_printf(p, "\tIP ver: %u.%u.%u\n", 1320 REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid), 1321 REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid), 1322 REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid)); 1323 drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock); 1324 } 1325 } 1326 1327 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address) 1328 { 1329 return sign_extend64(address, xe->info.va_bits - 1); 1330 } 1331 1332 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) 1333 { 1334 return address & GENMASK_ULL(xe->info.va_bits - 1, 0); 1335 } 1336 1337 /** 1338 * DOC: Xe Device Wedging 1339 * 1340 * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. 1341 * When device is in wedged state, every IOCTL will be blocked and GT cannot 1342 * be used. The conditions under which the driver declares the device wedged 1343 * depend on the wedged mode configuration (see &enum xe_wedged_mode). The 1344 * default recovery method for a wedged state is rebind/bus-reset. 1345 * 1346 * Another recovery method is vendor-specific. Below are the cases that send 1347 * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. 1348 * 1349 * Case: Firmware Flash 1350 * -------------------- 1351 * 1352 * Identification Hint 1353 * +++++++++++++++++++ 1354 * 1355 * ``WEDGED=vendor-specific`` drm device wedged uevent with 1356 * :ref:`Runtime Survivability mode <xe-survivability-mode>` is used to notify 1357 * admin/userspace consumer about the need for a firmware flash. 1358 * 1359 * Recovery Procedure 1360 * ++++++++++++++++++ 1361 * 1362 * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow 1363 * the below steps 1364 * 1365 * - Check Runtime Survivability mode sysfs. 1366 * If enabled, firmware flash is required to recover the device. 1367 * 1368 * /sys/bus/pci/devices/<device>/survivability_mode 1369 * 1370 * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash 1371 * firmware and restore device to normal operation. 1372 */ 1373 1374 /** 1375 * xe_device_set_wedged_method - Set wedged recovery method 1376 * @xe: xe device instance 1377 * @method: recovery method to set 1378 * 1379 * Set wedged recovery method to be sent in drm wedged uevent. 1380 */ 1381 void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method) 1382 { 1383 xe->wedged.method = method; 1384 } 1385 1386 #define WEDGED_URL "https://docs.kernel.org/gpu/drm-uapi.html#device-wedging" 1387 #define XE_BUG_URL "https://gitlab.freedesktop.org/drm/xe/kernel/issues/new" 1388 1389 /** 1390 * xe_device_declare_wedged - Declare device wedged 1391 * @xe: xe device instance 1392 * 1393 * This is a final state that can only be cleared with the recovery method 1394 * specified in the drm wedged uevent. The method can be set using 1395 * xe_device_set_wedged_method before declaring the device as wedged. If no method 1396 * is set, reprobe (unbind/re-bind) will be sent by default. 1397 * 1398 * In this state every IOCTL will be blocked so the GT cannot be used. 1399 * In general it will be called upon any critical error such as gt reset 1400 * failure or guc loading failure. Userspace will be notified of this state 1401 * through device wedged uevent. 1402 * If xe.wedged module parameter is set to 2, this function will be called 1403 * on every single execution timeout (a.k.a. GPU hang) right after devcoredump 1404 * snapshot capture. In this mode, GT reset won't be attempted so the state of 1405 * the issue is preserved for further debugging. 1406 */ 1407 void xe_device_declare_wedged(struct xe_device *xe) 1408 { 1409 struct xe_gt *gt; 1410 u8 id; 1411 1412 if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) { 1413 drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); 1414 return; 1415 } 1416 1417 if (!atomic_xchg(&xe->wedged.flag, 1)) { 1418 xe->needs_flr_on_fini = true; 1419 xe_pm_runtime_get_noresume(xe); 1420 1421 xe_log_err_fatal(xe, WEDGED, -EIO, "Device declared wedged!\n"); 1422 xe_err_once(xe, "IOCTLs and executions are now blocked!\n" 1423 "For recovery procedure, refer to %s\n" 1424 "Please file a _new_ bug report at %s\n", 1425 WEDGED_URL, XE_BUG_URL); 1426 } 1427 1428 for_each_gt(gt, xe, id) 1429 xe_gt_declare_wedged(gt); 1430 1431 if (xe_device_wedged(xe)) { 1432 /* 1433 * XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging 1434 * hangs, so wedge the device with 'none' recovery method and have 1435 * it available to the user for debugging. 1436 */ 1437 if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) 1438 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_NONE); 1439 /* If no wedge recovery method is set, use default */ 1440 else if (!xe->wedged.method) 1441 xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_REBIND | 1442 DRM_WEDGE_RECOVERY_BUS_RESET); 1443 1444 /* Notify userspace of wedged device */ 1445 drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); 1446 } 1447 } 1448 1449 /** 1450 * xe_device_validate_wedged_mode - Check if given mode is supported 1451 * @xe: the &xe_device 1452 * @mode: requested mode to validate 1453 * 1454 * Check whether the provided wedged mode is supported. 1455 * 1456 * Return: 0 if mode is supported, error code otherwise. 1457 */ 1458 int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode) 1459 { 1460 if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { 1461 drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode); 1462 return -EINVAL; 1463 } else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) || 1464 (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) { 1465 drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n", 1466 mode, xe_wedged_mode_to_string(mode), 1467 xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); 1468 return -EPERM; 1469 } 1470 1471 return 0; 1472 } 1473 1474 /** 1475 * xe_wedged_mode_to_string - Convert enum value to string. 1476 * @mode: the &xe_wedged_mode to convert 1477 * 1478 * Returns: wedged mode as a user friendly string. 1479 */ 1480 const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode) 1481 { 1482 switch (mode) { 1483 case XE_WEDGED_MODE_NEVER: 1484 return "never"; 1485 case XE_WEDGED_MODE_UPON_CRITICAL_ERROR: 1486 return "upon-critical-error"; 1487 case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: 1488 return "upon-any-hang-no-reset"; 1489 default: 1490 return "<invalid>"; 1491 } 1492 } 1493 1494 /** 1495 * xe_device_asid_to_vm() - Find VM from ASID 1496 * @xe: the &xe_device 1497 * @asid: Address space ID 1498 * 1499 * Find a VM from ASID and take a reference to VM which caller must drop. 1500 * Reclaim safe. 1501 * 1502 * Return: VM on success, ERR_PTR on failure 1503 */ 1504 struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid) 1505 { 1506 struct xe_vm *vm; 1507 1508 down_read(&xe->usm.lock); 1509 vm = xa_load(&xe->usm.asid_to_vm, asid); 1510 if (vm) 1511 xe_vm_get(vm); 1512 else 1513 vm = ERR_PTR(-EINVAL); 1514 up_read(&xe->usm.lock); 1515 1516 return vm; 1517 } 1518