Lines Matching full:gt
40 void intel_gt_common_init_early(struct intel_gt *gt) in intel_gt_common_init_early() argument
42 spin_lock_init(gt->irq_lock); in intel_gt_common_init_early()
44 INIT_LIST_HEAD(>->closed_vma); in intel_gt_common_init_early()
45 spin_lock_init(>->closed_lock); in intel_gt_common_init_early()
47 init_llist_head(>->watchdog.list); in intel_gt_common_init_early()
48 INIT_WORK(>->watchdog.work, intel_gt_watchdog_work); in intel_gt_common_init_early()
50 intel_gt_init_buffer_pool(gt); in intel_gt_common_init_early()
51 intel_gt_init_reset(gt); in intel_gt_common_init_early()
52 intel_gt_init_requests(gt); in intel_gt_common_init_early()
53 intel_gt_init_timelines(gt); in intel_gt_common_init_early()
54 intel_gt_init_tlb(gt); in intel_gt_common_init_early()
55 intel_gt_pm_init_early(gt); in intel_gt_common_init_early()
57 intel_wopcm_init_early(>->wopcm); in intel_gt_common_init_early()
58 intel_uc_init_early(>->uc); in intel_gt_common_init_early()
59 intel_rps_init_early(>->rps); in intel_gt_common_init_early()
65 struct intel_gt *gt; in intel_root_gt_init_early() local
67 gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL); in intel_root_gt_init_early()
68 if (!gt) in intel_root_gt_init_early()
71 i915->gt[0] = gt; in intel_root_gt_init_early()
73 gt->i915 = i915; in intel_root_gt_init_early()
74 gt->uncore = &i915->uncore; in intel_root_gt_init_early()
75 gt->irq_lock = drmm_kzalloc(&i915->drm, sizeof(*gt->irq_lock), GFP_KERNEL); in intel_root_gt_init_early()
76 if (!gt->irq_lock) in intel_root_gt_init_early()
79 intel_gt_common_init_early(gt); in intel_root_gt_init_early()
84 static int intel_gt_probe_lmem(struct intel_gt *gt) in intel_gt_probe_lmem() argument
86 struct drm_i915_private *i915 = gt->i915; in intel_gt_probe_lmem()
87 unsigned int instance = gt->info.id; in intel_gt_probe_lmem()
92 mem = intel_gt_setup_lmem(gt); in intel_gt_probe_lmem()
98 gt_err(gt, "Failed to setup region(%d) type=%d\n", in intel_gt_probe_lmem()
115 int intel_gt_assign_ggtt(struct intel_gt *gt) in intel_gt_assign_ggtt() argument
117 /* Media GT shares primary GT's GGTT */ in intel_gt_assign_ggtt()
118 if (gt->type == GT_MEDIA) { in intel_gt_assign_ggtt()
119 gt->ggtt = to_gt(gt->i915)->ggtt; in intel_gt_assign_ggtt()
121 gt->ggtt = i915_ggtt_create(gt->i915); in intel_gt_assign_ggtt()
122 if (IS_ERR(gt->ggtt)) in intel_gt_assign_ggtt()
123 return PTR_ERR(gt->ggtt); in intel_gt_assign_ggtt()
126 list_add_tail(>->ggtt_link, >->ggtt->gt_list); in intel_gt_assign_ggtt()
131 int intel_gt_init_mmio(struct intel_gt *gt) in intel_gt_init_mmio() argument
133 intel_gt_init_clock_frequency(gt); in intel_gt_init_mmio()
135 intel_uc_init_mmio(>->uc); in intel_gt_init_mmio()
136 intel_sseu_info_init(gt); in intel_gt_init_mmio()
137 intel_gt_mcr_init(gt); in intel_gt_init_mmio()
139 return intel_engines_init_mmio(gt); in intel_gt_init_mmio()
142 static void init_unused_ring(struct intel_gt *gt, u32 base) in init_unused_ring() argument
144 struct intel_uncore *uncore = gt->uncore; in init_unused_ring()
152 static void init_unused_rings(struct intel_gt *gt) in init_unused_rings() argument
154 struct drm_i915_private *i915 = gt->i915; in init_unused_rings()
157 init_unused_ring(gt, PRB1_BASE); in init_unused_rings()
158 init_unused_ring(gt, SRB0_BASE); in init_unused_rings()
159 init_unused_ring(gt, SRB1_BASE); in init_unused_rings()
160 init_unused_ring(gt, SRB2_BASE); in init_unused_rings()
161 init_unused_ring(gt, SRB3_BASE); in init_unused_rings()
163 init_unused_ring(gt, SRB0_BASE); in init_unused_rings()
164 init_unused_ring(gt, SRB1_BASE); in init_unused_rings()
166 init_unused_ring(gt, PRB1_BASE); in init_unused_rings()
167 init_unused_ring(gt, PRB2_BASE); in init_unused_rings()
171 int intel_gt_init_hw(struct intel_gt *gt) in intel_gt_init_hw() argument
173 struct drm_i915_private *i915 = gt->i915; in intel_gt_init_hw()
174 struct intel_uncore *uncore = gt->uncore; in intel_gt_init_hw()
177 gt->last_init_time = ktime_get(); in intel_gt_init_hw()
191 /* Apply the GT workarounds... */ in intel_gt_init_hw()
192 intel_gt_apply_workarounds(gt); in intel_gt_init_hw()
194 intel_gt_verify_workarounds(gt, "init"); in intel_gt_init_hw()
196 intel_gt_init_swizzling(gt); in intel_gt_init_hw()
204 init_unused_rings(gt); in intel_gt_init_hw()
206 ret = i915_ppgtt_init_hw(gt); in intel_gt_init_hw()
208 gt_err(gt, "Enabling PPGTT failed (%d)\n", ret); in intel_gt_init_hw()
213 ret = intel_uc_init_hw(>->uc); in intel_gt_init_hw()
215 gt_probe_error(gt, "Enabling uc failed (%d)\n", ret); in intel_gt_init_hw()
219 intel_mocs_init(gt); in intel_gt_init_hw()
232 i915_reg_t intel_gt_perf_limit_reasons_reg(struct intel_gt *gt) in intel_gt_perf_limit_reasons_reg() argument
235 if (GRAPHICS_VER(gt->i915) < 11) in intel_gt_perf_limit_reasons_reg()
238 return gt->type == GT_MEDIA ? in intel_gt_perf_limit_reasons_reg()
243 intel_gt_clear_error_registers(struct intel_gt *gt, in intel_gt_clear_error_registers() argument
246 struct drm_i915_private *i915 = gt->i915; in intel_gt_clear_error_registers()
247 struct intel_uncore *uncore = gt->uncore; in intel_gt_clear_error_registers()
265 gt_dbg(gt, "EIR stuck: 0x%08x, masking\n", eir); in intel_gt_clear_error_registers()
272 * For the media GT, this ring fault register is not replicated, in intel_gt_clear_error_registers()
275 if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) { in intel_gt_clear_error_registers()
282 intel_gt_mcr_multicast_rmw(gt, XEHP_RING_FAULT_REG, in intel_gt_clear_error_registers()
284 intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG); in intel_gt_clear_error_registers()
296 for_each_engine_masked(engine, gt, engine_mask, id) in intel_gt_clear_error_registers()
301 static void gen6_check_faults(struct intel_gt *gt) in gen6_check_faults() argument
307 for_each_engine(engine, gt, id) { in gen6_check_faults()
310 gt_dbg(gt, "Unexpected fault\n" in gen6_check_faults()
324 static void xehp_check_faults(struct intel_gt *gt) in xehp_check_faults() argument
335 fault = intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG); in xehp_check_faults()
340 fault_data0 = intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA0); in xehp_check_faults()
341 fault_data1 = intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA1); in xehp_check_faults()
346 gt_dbg(gt, "Unexpected fault\n" in xehp_check_faults()
360 static void gen8_check_faults(struct intel_gt *gt) in gen8_check_faults() argument
362 struct intel_uncore *uncore = gt->uncore; in gen8_check_faults()
366 if (GRAPHICS_VER(gt->i915) >= 12) { in gen8_check_faults()
387 gt_dbg(gt, "Unexpected fault\n" in gen8_check_faults()
401 void intel_gt_check_and_clear_faults(struct intel_gt *gt) in intel_gt_check_and_clear_faults() argument
403 struct drm_i915_private *i915 = gt->i915; in intel_gt_check_and_clear_faults()
407 xehp_check_faults(gt); in intel_gt_check_and_clear_faults()
409 gen8_check_faults(gt); in intel_gt_check_and_clear_faults()
411 gen6_check_faults(gt); in intel_gt_check_and_clear_faults()
415 intel_gt_clear_error_registers(gt, ALL_ENGINES); in intel_gt_check_and_clear_faults()
418 void intel_gt_flush_ggtt_writes(struct intel_gt *gt) in intel_gt_flush_ggtt_writes() argument
420 struct intel_uncore *uncore = gt->uncore; in intel_gt_flush_ggtt_writes()
444 if (INTEL_INFO(gt->i915)->has_coherent_ggtt) in intel_gt_flush_ggtt_writes()
447 intel_gt_chipset_flush(gt); in intel_gt_flush_ggtt_writes()
459 void intel_gt_chipset_flush(struct intel_gt *gt) in intel_gt_chipset_flush() argument
462 if (GRAPHICS_VER(gt->i915) < 6) in intel_gt_chipset_flush()
466 void intel_gt_driver_register(struct intel_gt *gt) in intel_gt_driver_register() argument
468 intel_gsc_init(>->gsc, gt->i915); in intel_gt_driver_register()
470 intel_rps_driver_register(>->rps); in intel_gt_driver_register()
472 intel_gt_debugfs_register(gt); in intel_gt_driver_register()
473 intel_gt_sysfs_register(gt); in intel_gt_driver_register()
476 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) in intel_gt_init_scratch() argument
478 struct drm_i915_private *i915 = gt->i915; in intel_gt_init_scratch()
491 gt_err(gt, "Failed to allocate scratch page\n"); in intel_gt_init_scratch()
495 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); in intel_gt_init_scratch()
505 gt->scratch = i915_vma_make_unshrinkable(vma); in intel_gt_init_scratch()
514 static void intel_gt_fini_scratch(struct intel_gt *gt) in intel_gt_fini_scratch() argument
516 i915_vma_unpin_and_release(>->scratch, 0); in intel_gt_fini_scratch()
519 static struct i915_address_space *kernel_vm(struct intel_gt *gt) in kernel_vm() argument
521 if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING) in kernel_vm()
522 return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm; in kernel_vm()
524 return i915_vm_get(>->ggtt->vm); in kernel_vm()
527 static int __engines_record_defaults(struct intel_gt *gt) in __engines_record_defaults() argument
543 for_each_engine(engine, gt, id) { in __engines_record_defaults()
588 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { in __engines_record_defaults()
626 intel_gt_set_wedged(gt); in __engines_record_defaults()
643 static int __engines_verify_workarounds(struct intel_gt *gt) in __engines_verify_workarounds() argument
652 for_each_engine(engine, gt, id) { in __engines_verify_workarounds()
658 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) in __engines_verify_workarounds()
664 static void __intel_gt_disable(struct intel_gt *gt) in __intel_gt_disable() argument
666 intel_gt_set_wedged_on_fini(gt); in __intel_gt_disable()
668 intel_gt_suspend_prepare(gt); in __intel_gt_disable()
669 intel_gt_suspend_late(gt); in __intel_gt_disable()
671 GEM_BUG_ON(intel_gt_pm_is_awake(gt)); in __intel_gt_disable()
674 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout) in intel_gt_wait_for_idle() argument
679 if (!intel_gt_pm_is_awake(gt)) in intel_gt_wait_for_idle()
682 while ((timeout = intel_gt_retire_requests_timeout(gt, timeout, in intel_gt_wait_for_idle()
695 return intel_uc_wait_for_idle(>->uc, remaining_timeout); in intel_gt_wait_for_idle()
698 int intel_gt_init(struct intel_gt *gt) in intel_gt_init() argument
702 err = i915_inject_probe_error(gt->i915, -ENODEV); in intel_gt_init()
706 intel_gt_init_workarounds(gt); in intel_gt_init()
715 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); in intel_gt_init()
717 err = intel_gt_init_scratch(gt, in intel_gt_init()
718 GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K); in intel_gt_init()
722 intel_gt_pm_init(gt); in intel_gt_init()
724 gt->vm = kernel_vm(gt); in intel_gt_init()
725 if (!gt->vm) { in intel_gt_init()
730 intel_set_mocs_index(gt); in intel_gt_init()
732 err = intel_engines_init(gt); in intel_gt_init()
736 err = intel_uc_init(>->uc); in intel_gt_init()
740 err = intel_gt_resume(gt); in intel_gt_init()
744 err = intel_gt_init_hwconfig(gt); in intel_gt_init()
746 gt_err(gt, "Failed to retrieve hwconfig table: %pe\n", ERR_PTR(err)); in intel_gt_init()
748 err = __engines_record_defaults(gt); in intel_gt_init()
752 err = __engines_verify_workarounds(gt); in intel_gt_init()
756 err = i915_inject_probe_error(gt->i915, -EIO); in intel_gt_init()
760 intel_uc_init_late(>->uc); in intel_gt_init()
762 intel_migrate_init(>->migrate, gt); in intel_gt_init()
766 __intel_gt_disable(gt); in intel_gt_init()
767 intel_uc_fini_hw(>->uc); in intel_gt_init()
769 intel_uc_fini(>->uc); in intel_gt_init()
771 intel_engines_release(gt); in intel_gt_init()
772 i915_vm_put(fetch_and_zero(>->vm)); in intel_gt_init()
774 intel_gt_pm_fini(gt); in intel_gt_init()
775 intel_gt_fini_scratch(gt); in intel_gt_init()
778 intel_gt_set_wedged_on_init(gt); in intel_gt_init()
779 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); in intel_gt_init()
783 void intel_gt_driver_remove(struct intel_gt *gt) in intel_gt_driver_remove() argument
785 __intel_gt_disable(gt); in intel_gt_driver_remove()
787 intel_migrate_fini(>->migrate); in intel_gt_driver_remove()
788 intel_uc_driver_remove(>->uc); in intel_gt_driver_remove()
790 intel_engines_release(gt); in intel_gt_driver_remove()
792 intel_gt_flush_buffer_pool(gt); in intel_gt_driver_remove()
795 void intel_gt_driver_unregister(struct intel_gt *gt) in intel_gt_driver_unregister() argument
799 intel_gt_sysfs_unregister(gt); in intel_gt_driver_unregister()
800 intel_rps_driver_unregister(>->rps); in intel_gt_driver_unregister()
801 intel_gsc_fini(>->gsc); in intel_gt_driver_unregister()
814 * implications for subsystems outside of GT (i.e., PXP, HDCP), it's in intel_gt_driver_unregister()
824 intel_gsc_uc_flush_work(>->uc.gsc); in intel_gt_driver_unregister()
831 intel_gt_set_wedged_on_fini(gt); in intel_gt_driver_unregister()
834 with_intel_runtime_pm(gt->uncore->rpm, wakeref) in intel_gt_driver_unregister()
835 intel_gt_reset_all_engines(gt); in intel_gt_driver_unregister()
838 void intel_gt_driver_release(struct intel_gt *gt) in intel_gt_driver_release() argument
842 vm = fetch_and_zero(>->vm); in intel_gt_driver_release()
846 intel_wa_list_free(>->wa_list); in intel_gt_driver_release()
847 intel_gt_pm_fini(gt); in intel_gt_driver_release()
848 intel_gt_fini_scratch(gt); in intel_gt_driver_release()
849 intel_gt_fini_buffer_pool(gt); in intel_gt_driver_release()
850 intel_gt_fini_hwconfig(gt); in intel_gt_driver_release()
855 struct intel_gt *gt; in intel_gt_driver_late_release_all() local
861 for_each_gt(gt, i915, id) { in intel_gt_driver_late_release_all()
862 intel_uc_driver_late_release(>->uc); in intel_gt_driver_late_release_all()
863 intel_gt_fini_requests(gt); in intel_gt_driver_late_release_all()
864 intel_gt_fini_reset(gt); in intel_gt_driver_late_release_all()
865 intel_gt_fini_timelines(gt); in intel_gt_driver_late_release_all()
866 intel_gt_fini_tlb(gt); in intel_gt_driver_late_release_all()
867 intel_engines_free(gt); in intel_gt_driver_late_release_all()
871 static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr) in intel_gt_tile_setup() argument
875 if (!gt_is_root(gt)) { in intel_gt_tile_setup()
879 uncore = drmm_kzalloc(>->i915->drm, sizeof(*uncore), GFP_KERNEL); in intel_gt_tile_setup()
883 irq_lock = drmm_kzalloc(>->i915->drm, sizeof(*irq_lock), GFP_KERNEL); in intel_gt_tile_setup()
887 gt->uncore = uncore; in intel_gt_tile_setup()
888 gt->irq_lock = irq_lock; in intel_gt_tile_setup()
890 intel_gt_common_init_early(gt); in intel_gt_tile_setup()
893 intel_uncore_init_early(gt->uncore, gt); in intel_gt_tile_setup()
895 ret = intel_uncore_setup_mmio(gt->uncore, phys_addr); in intel_gt_tile_setup()
899 gt->phys_addr = phys_addr; in intel_gt_tile_setup()
907 struct intel_gt *gt = to_gt(i915); in intel_gt_probe_all() local
918 * We always have at least one primary GT on any device in intel_gt_probe_all()
922 gt->i915 = i915; in intel_gt_probe_all()
923 gt->name = "Primary GT"; in intel_gt_probe_all()
924 gt->info.engine_mask = INTEL_INFO(i915)->platform_engine_mask; in intel_gt_probe_all()
926 gt_dbg(gt, "Setting up %s\n", gt->name); in intel_gt_probe_all()
927 ret = intel_gt_tile_setup(gt, phys_addr); in intel_gt_probe_all()
937 gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL); in intel_gt_probe_all()
938 if (!gt) { in intel_gt_probe_all()
943 gt->i915 = i915; in intel_gt_probe_all()
944 gt->name = gtdef->name; in intel_gt_probe_all()
945 gt->type = gtdef->type; in intel_gt_probe_all()
946 gt->info.engine_mask = gtdef->engine_mask; in intel_gt_probe_all()
947 gt->info.id = i; in intel_gt_probe_all()
949 gt_dbg(gt, "Setting up %s\n", gt->name); in intel_gt_probe_all()
960 ret = intel_gt_tile_setup(gt, phys_addr + gtdef->mapping_base); in intel_gt_probe_all()
964 ret = intel_sa_mediagt_setup(gt, phys_addr + gtdef->mapping_base, in intel_gt_probe_all()
969 /* Primary GT should not appear in extra GT list */ in intel_gt_probe_all()
978 i915->gt[i] = gt; in intel_gt_probe_all()
990 struct intel_gt *gt; in intel_gt_tiles_init() local
994 for_each_gt(gt, i915, id) { in intel_gt_tiles_init()
995 ret = intel_gt_probe_lmem(gt); in intel_gt_tiles_init()
1011 enum i915_map_type intel_gt_coherent_map_type(struct intel_gt *gt, in intel_gt_coherent_map_type() argument
1017 * version 13.0 when the object is on the Media GT in intel_gt_coherent_map_type()
1019 if (i915_gem_object_is_lmem(obj) || intel_gt_needs_wa_22016122933(gt)) in intel_gt_coherent_map_type()
1021 if (HAS_LLC(gt->i915) || always_coherent) in intel_gt_coherent_map_type()
1027 bool intel_gt_needs_wa_16018031267(struct intel_gt *gt) in intel_gt_needs_wa_16018031267() argument
1030 return IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 55), IP_VER(12, 71)); in intel_gt_needs_wa_16018031267()
1033 bool intel_gt_needs_wa_22016122933(struct intel_gt *gt) in intel_gt_needs_wa_22016122933() argument
1035 return MEDIA_VER_FULL(gt->i915) == IP_VER(13, 0) && gt->type == GT_MEDIA; in intel_gt_needs_wa_22016122933()
1038 static void __intel_gt_bind_context_set_ready(struct intel_gt *gt, bool ready) in __intel_gt_bind_context_set_ready() argument
1040 struct intel_engine_cs *engine = gt->engine[BCS0]; in __intel_gt_bind_context_set_ready()
1049 * @gt: GT structure
1053 void intel_gt_bind_context_set_ready(struct intel_gt *gt) in intel_gt_bind_context_set_ready() argument
1055 __intel_gt_bind_context_set_ready(gt, true); in intel_gt_bind_context_set_ready()
1060 * @gt: GT structure
1065 void intel_gt_bind_context_set_unready(struct intel_gt *gt) in intel_gt_bind_context_set_unready() argument
1067 __intel_gt_bind_context_set_ready(gt, false); in intel_gt_bind_context_set_unready()
1073 * @gt: GT structure
1077 bool intel_gt_is_bind_context_ready(struct intel_gt *gt) in intel_gt_is_bind_context_ready() argument
1079 struct intel_engine_cs *engine = gt->engine[BCS0]; in intel_gt_is_bind_context_ready()