67 	struct xe_device *xe = to_xe_device(dev);  in xe_file_open()  local
85 	xef->xe = xe;  in xe_file_open()
148 	struct xe_device *xe = to_xe_device(dev);  in xe_file_close()  local
154 	xe_pm_runtime_get(xe);  in xe_file_close()
173 	xe_pm_runtime_put(xe);  in xe_file_close()
199 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);  in xe_drm_ioctl()  local
202 	if (xe_device_wedged(xe))  in xe_drm_ioctl()
205 	ret = xe_pm_runtime_get_ioctl(xe);  in xe_drm_ioctl()
208 	xe_pm_runtime_put(xe);  in xe_drm_ioctl()
217 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);  in xe_drm_compat_ioctl()  local
220 	if (xe_device_wedged(xe))  in xe_drm_compat_ioctl()
223 	ret = xe_pm_runtime_get_ioctl(xe);  in xe_drm_compat_ioctl()
226 	xe_pm_runtime_put(xe);  in xe_drm_compat_ioctl()
281 	struct xe_device *xe = to_xe_device(dev);  in xe_device_destroy()  local
283 	if (xe->preempt_fence_wq)  in xe_device_destroy()
284 		destroy_workqueue(xe->preempt_fence_wq);  in xe_device_destroy()
286 	if (xe->ordered_wq)  in xe_device_destroy()
287 		destroy_workqueue(xe->ordered_wq);  in xe_device_destroy()
289 	if (xe->unordered_wq)  in xe_device_destroy()
290 		destroy_workqueue(xe->unordered_wq);  in xe_device_destroy()
292 	if (xe->destroy_wq)  in xe_device_destroy()
293 		destroy_workqueue(xe->destroy_wq);  in xe_device_destroy()
295 	ttm_device_fini(&xe->ttm);  in xe_device_destroy()
301 	struct xe_device *xe;  in xe_device_create()  local
310 	xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);  in xe_device_create()
311 	if (IS_ERR(xe))  in xe_device_create()
312 		return xe;  in xe_device_create()
314 	err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,  in xe_device_create()
315 			      xe->drm.anon_inode->i_mapping,  in xe_device_create()
316 			      xe->drm.vma_offset_manager, false, false);  in xe_device_create()
320 	err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);  in xe_device_create()
324 	xe->info.devid = pdev->device;  in xe_device_create()
325 	xe->info.revid = pdev->revision;  in xe_device_create()
326 	xe->info.force_execlist = xe_modparam.force_execlist;  in xe_device_create()
328 	err = xe_irq_init(xe);  in xe_device_create()
332 	init_waitqueue_head(&xe->ufence_wq);  in xe_device_create()
334 	init_rwsem(&xe->usm.lock);  in xe_device_create()
336 	xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);  in xe_device_create()
343 		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,  in xe_device_create()
345 				      &xe->usm.next_asid, GFP_KERNEL);  in xe_device_create()
346 		drm_WARN_ON(&xe->drm, err);  in xe_device_create()
348 			xa_erase(&xe->usm.asid_to_vm, asid);  in xe_device_create()
351 	spin_lock_init(&xe->pinned.lock);  in xe_device_create()
352 	INIT_LIST_HEAD(&xe->pinned.kernel_bo_present);  in xe_device_create()
353 	INIT_LIST_HEAD(&xe->pinned.external_vram);  in xe_device_create()
354 	INIT_LIST_HEAD(&xe->pinned.evicted);  in xe_device_create()
356 	xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",  in xe_device_create()
358 	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);  in xe_device_create()
359 	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);  in xe_device_create()
360 	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0);  in xe_device_create()
361 	if (!xe->ordered_wq || !xe->unordered_wq ||  in xe_device_create()
362 	    !xe->preempt_fence_wq || !xe->destroy_wq) {  in xe_device_create()
367 		drm_err(&xe->drm, "Failed to allocate xe workqueues\n");  in xe_device_create()
372 	err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);  in xe_device_create()
376 	err = xe_display_create(xe);  in xe_device_create()
380 	return xe;  in xe_device_create()
387 static bool xe_driver_flr_disabled(struct xe_device *xe)  in xe_driver_flr_disabled()  argument
389 	return xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS;  in xe_driver_flr_disabled()
405 static void __xe_driver_flr(struct xe_device *xe)  in __xe_driver_flr()  argument
408 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);  in __xe_driver_flr()
411 	drm_dbg(&xe->drm, "Triggering Driver-FLR\n");  in __xe_driver_flr()
424 		drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);  in __xe_driver_flr()
435 		drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);  in __xe_driver_flr()
443 		drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);  in __xe_driver_flr()
451 static void xe_driver_flr(struct xe_device *xe)  in xe_driver_flr()  argument
453 	if (xe_driver_flr_disabled(xe)) {  in xe_driver_flr()
454 		drm_info_once(&xe->drm, "BIOS Disabled Driver-FLR\n");  in xe_driver_flr()
458 	__xe_driver_flr(xe);  in xe_driver_flr()
463 	struct xe_device *xe = arg;  in xe_driver_flr_fini()  local
465 	if (xe->needs_flr_on_fini)  in xe_driver_flr_fini()
466 		xe_driver_flr(xe);  in xe_driver_flr_fini()
471 	struct xe_device *xe = arg;  in xe_device_sanitize()  local
475 	for_each_gt(gt, xe, id)  in xe_device_sanitize()
479 static int xe_set_dma_info(struct xe_device *xe)  in xe_set_dma_info()  argument
481 	unsigned int mask_size = xe->info.dma_mask_size;  in xe_set_dma_info()
484 	dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));  in xe_set_dma_info()
486 	err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));  in xe_set_dma_info()
490 	err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));  in xe_set_dma_info()
497 	drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);  in xe_set_dma_info()
501 static bool verify_lmem_ready(struct xe_device *xe)  in verify_lmem_ready()  argument
503 	u32 val = xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT;  in verify_lmem_ready()
508 static int wait_for_lmem_ready(struct xe_device *xe)  in wait_for_lmem_ready()  argument
512 	if (!IS_DGFX(xe))  in wait_for_lmem_ready()
515 	if (IS_SRIOV_VF(xe))  in wait_for_lmem_ready()
518 	if (verify_lmem_ready(xe))  in wait_for_lmem_ready()
521 	drm_dbg(&xe->drm, "Waiting for lmem initialization\n");  in wait_for_lmem_ready()
541 			drm_dbg(&xe->drm, "lmem not initialized by firmware\n");  in wait_for_lmem_ready()
547 	} while (!verify_lmem_ready(xe));  in wait_for_lmem_ready()
549 	drm_dbg(&xe->drm, "lmem ready after %ums",  in wait_for_lmem_ready()
556 static void update_device_info(struct xe_device *xe)  in update_device_info()  argument
559 	if (IS_SRIOV_VF(xe)) {  in update_device_info()
560 		xe->info.probe_display = 0;  in update_device_info()
561 		xe->info.has_heci_gscfi = 0;  in update_device_info()
562 		xe->info.skip_guc_pc = 1;  in update_device_info()
563 		xe->info.skip_pcode = 1;  in update_device_info()
577 int xe_device_probe_early(struct xe_device *xe)  in xe_device_probe_early()  argument
581 	err = xe_mmio_init(xe);  in xe_device_probe_early()
585 	xe_sriov_probe_early(xe);  in xe_device_probe_early()
587 	update_device_info(xe);  in xe_device_probe_early()
589 	err = xe_pcode_probe_early(xe);  in xe_device_probe_early()
593 	err = wait_for_lmem_ready(xe);  in xe_device_probe_early()
597 	xe->wedged.mode = xe_modparam.wedged_mode;  in xe_device_probe_early()
602 static int probe_has_flat_ccs(struct xe_device *xe)  in probe_has_flat_ccs()  argument
609 	if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))  in probe_has_flat_ccs()
612 	gt = xe_root_mmio_gt(xe);  in probe_has_flat_ccs()
619 	xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);  in probe_has_flat_ccs()
621 	if (!xe->info.has_flat_ccs)  in probe_has_flat_ccs()
622 		drm_dbg(&xe->drm,  in probe_has_flat_ccs()
629 int xe_device_probe(struct xe_device *xe)  in xe_device_probe()  argument
637 	xe_pat_init_early(xe);  in xe_device_probe()
639 	err = xe_sriov_init(xe);  in xe_device_probe()
643 	xe->info.mem_region_mask = 1;  in xe_device_probe()
644 	err = xe_display_init_nommio(xe);  in xe_device_probe()
648 	err = xe_set_dma_info(xe);  in xe_device_probe()
652 	err = xe_mmio_probe_tiles(xe);  in xe_device_probe()
656 	xe_ttm_sys_mgr_init(xe);  in xe_device_probe()
658 	for_each_gt(gt, xe, id) {  in xe_device_probe()
671 	for_each_tile(tile, xe, id) {  in xe_device_probe()
672 		if (IS_SRIOV_VF(xe)) {  in xe_device_probe()
689 	for_each_gt(gt, xe, id) {  in xe_device_probe()
695 	err = xe_devcoredump_init(xe);  in xe_device_probe()
698 	err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);  in xe_device_probe()
702 	err = xe_display_init_noirq(xe);  in xe_device_probe()
706 	err = xe_irq_install(xe);  in xe_device_probe()
710 	err = probe_has_flat_ccs(xe);  in xe_device_probe()
714 	err = xe_vram_probe(xe);  in xe_device_probe()
718 	for_each_tile(tile, xe, id) {  in xe_device_probe()
725 	xe_ttm_stolen_mgr_init(xe);  in xe_device_probe()
733 	err = xe_display_init_noaccel(xe);  in xe_device_probe()
737 	for_each_gt(gt, xe, id) {  in xe_device_probe()
745 	xe_heci_gsc_init(xe);  in xe_device_probe()
747 	err = xe_oa_init(xe);  in xe_device_probe()
751 	err = xe_display_init(xe);  in xe_device_probe()
755 	err = drm_dev_register(&xe->drm, 0);  in xe_device_probe()
759 	xe_display_register(xe);  in xe_device_probe()
761 	xe_oa_register(xe);  in xe_device_probe()
763 	xe_debugfs_register(xe);  in xe_device_probe()
765 	xe_hwmon_register(xe);  in xe_device_probe()
767 	for_each_gt(gt, xe, id)  in xe_device_probe()
770 	xe_vsec_init(xe);  in xe_device_probe()
772 	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);  in xe_device_probe()
775 	xe_display_driver_remove(xe);  in xe_device_probe()
778 	xe_oa_fini(xe);  in xe_device_probe()
781 	for_each_gt(gt, xe, id) {  in xe_device_probe()
789 	xe_display_fini(xe);  in xe_device_probe()
793 static void xe_device_remove_display(struct xe_device *xe)  in xe_device_remove_display()  argument
795 	xe_display_unregister(xe);  in xe_device_remove_display()
797 	drm_dev_unplug(&xe->drm);  in xe_device_remove_display()
798 	xe_display_driver_remove(xe);  in xe_device_remove_display()
801 void xe_device_remove(struct xe_device *xe)  in xe_device_remove()  argument
806 	xe_oa_unregister(xe);  in xe_device_remove()
808 	xe_device_remove_display(xe);  in xe_device_remove()
810 	xe_display_fini(xe);  in xe_device_remove()
812 	xe_oa_fini(xe);  in xe_device_remove()
814 	xe_heci_gsc_fini(xe);  in xe_device_remove()
816 	for_each_gt(gt, xe, id)  in xe_device_remove()
820 void xe_device_shutdown(struct xe_device *xe)  in xe_device_shutdown()  argument
825 	drm_dbg(&xe->drm, "Shutting down device\n");  in xe_device_shutdown()
827 	if (xe_driver_flr_disabled(xe)) {  in xe_device_shutdown()
828 		xe_display_pm_shutdown(xe);  in xe_device_shutdown()
830 		xe_irq_suspend(xe);  in xe_device_shutdown()
832 		for_each_gt(gt, xe, id)  in xe_device_shutdown()
835 		xe_display_pm_shutdown_late(xe);  in xe_device_shutdown()
838 		__xe_driver_flr(xe);  in xe_device_shutdown()
851 void xe_device_wmb(struct xe_device *xe)  in xe_device_wmb()  argument
854 	if (IS_DGFX(xe))  in xe_device_wmb()
855 		xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);  in xe_device_wmb()
876 void xe_device_td_flush(struct xe_device *xe)  in xe_device_td_flush()  argument
882 	if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)  in xe_device_td_flush()
885 	if (XE_WA(xe_root_mmio_gt(xe), 16023588340)) {  in xe_device_td_flush()
886 		xe_device_l2_flush(xe);  in xe_device_td_flush()
890 	for_each_gt(gt, xe, id) {  in xe_device_td_flush()
914 void xe_device_l2_flush(struct xe_device *xe)  in xe_device_l2_flush()  argument
919 	gt = xe_root_mmio_gt(xe);  in xe_device_l2_flush()
938 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)  in xe_device_ccs_bytes()  argument
940 	return xe_device_has_flat_ccs(xe) ?  in xe_device_ccs_bytes()
941 		DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;  in xe_device_ccs_bytes()
955 void xe_device_assert_mem_access(struct xe_device *xe)  in xe_device_assert_mem_access()  argument
957 	xe_assert(xe, !xe_pm_runtime_suspended(xe));  in xe_device_assert_mem_access()
960 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)  in xe_device_snapshot_print()  argument
965 	drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);  in xe_device_snapshot_print()
966 	drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);  in xe_device_snapshot_print()
968 	for_each_gt(gt, xe, id) {  in xe_device_snapshot_print()
981 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)  in xe_device_canonicalize_addr()  argument
983 	return sign_extend64(address, xe->info.va_bits - 1);  in xe_device_canonicalize_addr()
986 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)  in xe_device_uncanonicalize_addr()  argument
988 	return address & GENMASK_ULL(xe->info.va_bits - 1, 0);  in xe_device_uncanonicalize_addr()
993 	struct xe_device *xe = arg;  in xe_device_wedged_fini()  local
995 	xe_pm_runtime_put(xe);  in xe_device_wedged_fini()
1012 void xe_device_declare_wedged(struct xe_device *xe)  in xe_device_declare_wedged()  argument
1017 	if (xe->wedged.mode == 0) {  in xe_device_declare_wedged()
1018 		drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");  in xe_device_declare_wedged()
1022 	xe_pm_runtime_get_noresume(xe);  in xe_device_declare_wedged()
1024 	if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) {  in xe_device_declare_wedged()
1025 …drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n…  in xe_device_declare_wedged()
1029 	if (!atomic_xchg(&xe->wedged.flag, 1)) {  in xe_device_declare_wedged()
1030 		xe->needs_flr_on_fini = true;  in xe_device_declare_wedged()
1031 		drm_err(&xe->drm,  in xe_device_declare_wedged()
1035 			dev_name(xe->drm.dev));  in xe_device_declare_wedged()
1038 	for_each_gt(gt, xe, id)  in xe_device_declare_wedged()