33 		return intel_gt_mcr_wait_for_reg(engine->gt,  in wait_for_invalidate()
40 		return __intel_wait_for_register_fw(engine->gt->uncore,  in wait_for_invalidate()
49 static void mmio_invalidate_full(struct intel_gt *gt)  in mmio_invalidate_full()  argument
51 	struct drm_i915_private *i915 = gt->i915;  in mmio_invalidate_full()
52 	struct intel_uncore *uncore = gt->uncore;  in mmio_invalidate_full()
63 	intel_gt_mcr_lock(gt, &flags);  in mmio_invalidate_full()
64 	spin_lock(&uncore->lock); /* serialise invalidate with GT reset */  in mmio_invalidate_full()
67 	for_each_engine(engine, gt, id) {  in mmio_invalidate_full()
72 			intel_gt_mcr_multicast_write_fw(gt,  in mmio_invalidate_full()
83 	GT_TRACE(gt, "invalidated engines %08x\n", awake);  in mmio_invalidate_full()
95 	intel_gt_mcr_unlock(gt, flags);  in mmio_invalidate_full()
97 	for_each_engine_masked(engine, gt, awake, tmp) {  in mmio_invalidate_full()
99 			gt_err_ratelimited(gt,  in mmio_invalidate_full()
113 static bool tlb_seqno_passed(const struct intel_gt *gt, u32 seqno)  in tlb_seqno_passed()  argument
115 	u32 cur = intel_gt_tlb_seqno(gt);  in tlb_seqno_passed()
121 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)  in intel_gt_invalidate_tlb_full()  argument
125 	if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))  in intel_gt_invalidate_tlb_full()
128 	if (intel_gt_is_wedged(gt))  in intel_gt_invalidate_tlb_full()
131 	if (tlb_seqno_passed(gt, seqno))  in intel_gt_invalidate_tlb_full()
134 	with_intel_gt_pm_if_awake(gt, wakeref) {  in intel_gt_invalidate_tlb_full()
135 		struct intel_guc *guc = gt_to_guc(gt);  in intel_gt_invalidate_tlb_full()
137 		mutex_lock(&gt->tlb.invalidate_lock);  in intel_gt_invalidate_tlb_full()
138 		if (tlb_seqno_passed(gt, seqno))  in intel_gt_invalidate_tlb_full()
141 		if (HAS_GUC_TLB_INVALIDATION(gt->i915)) {  in intel_gt_invalidate_tlb_full()
144 			 * The only time GuC could not be ready is on GT reset,  in intel_gt_invalidate_tlb_full()
151 			mmio_invalidate_full(gt);  in intel_gt_invalidate_tlb_full()
154 		write_seqcount_invalidate(&gt->tlb.seqno);  in intel_gt_invalidate_tlb_full()
156 		mutex_unlock(&gt->tlb.invalidate_lock);  in intel_gt_invalidate_tlb_full()
160 void intel_gt_init_tlb(struct intel_gt *gt)  in intel_gt_init_tlb()  argument
162 	mutex_init(&gt->tlb.invalidate_lock);  in intel_gt_init_tlb()
163 	seqcount_mutex_init(&gt->tlb.seqno, &gt->tlb.invalidate_lock);  in intel_gt_init_tlb()
166 void intel_gt_fini_tlb(struct intel_gt *gt)  in intel_gt_fini_tlb()  argument
168 	mutex_destroy(&gt->tlb.invalidate_lock);  in intel_gt_fini_tlb()