28 static void user_forcewake(struct intel_gt *gt, bool suspend)  in user_forcewake()  argument
30 	int count = atomic_read(&gt->user_wakeref);  in user_forcewake()
37 	wakeref = intel_gt_pm_get(gt);  in user_forcewake()
39 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));  in user_forcewake()
40 		atomic_sub(count, &gt->wakeref.count);  in user_forcewake()
42 		atomic_add(count, &gt->wakeref.count);  in user_forcewake()
44 	intel_gt_pm_put(gt, wakeref);  in user_forcewake()
47 static void runtime_begin(struct intel_gt *gt)  in runtime_begin()  argument
50 	write_seqcount_begin(&gt->stats.lock);  in runtime_begin()
51 	gt->stats.start = ktime_get();  in runtime_begin()
52 	gt->stats.active = true;  in runtime_begin()
53 	write_seqcount_end(&gt->stats.lock);  in runtime_begin()
57 static void runtime_end(struct intel_gt *gt)  in runtime_end()  argument
60 	write_seqcount_begin(&gt->stats.lock);  in runtime_end()
61 	gt->stats.active = false;  in runtime_end()
62 	gt->stats.total =  in runtime_end()
63 		ktime_add(gt->stats.total,  in runtime_end()
64 			  ktime_sub(ktime_get(), gt->stats.start));  in runtime_end()
65 	write_seqcount_end(&gt->stats.lock);  in runtime_end()
71 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);  in __gt_unpark()  local
72 	struct drm_i915_private *i915 = gt->i915;  in __gt_unpark()
74 	GT_TRACE(gt, "\n");  in __gt_unpark()
84 	 * Work around it by grabbing a GT IRQ power domain whilst there is any  in __gt_unpark()
85 	 * GT activity, preventing any DC state transitions.  in __gt_unpark()
87 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);  in __gt_unpark()
88 	GEM_BUG_ON(!gt->awake);  in __gt_unpark()
90 	intel_rc6_unpark(&gt->rc6);  in __gt_unpark()
91 	intel_rps_unpark(&gt->rps);  in __gt_unpark()
92 	i915_pmu_gt_unparked(gt);  in __gt_unpark()
93 	intel_guc_busyness_unpark(gt);  in __gt_unpark()
95 	intel_gt_unpark_requests(gt);  in __gt_unpark()
96 	runtime_begin(gt);  in __gt_unpark()
103 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);  in __gt_park()  local
104 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);  in __gt_park()
105 	struct drm_i915_private *i915 = gt->i915;  in __gt_park()
107 	GT_TRACE(gt, "\n");  in __gt_park()
109 	runtime_end(gt);  in __gt_park()
110 	intel_gt_park_requests(gt);  in __gt_park()
112 	intel_guc_busyness_park(gt);  in __gt_park()
113 	i915_vma_parked(gt);  in __gt_park()
114 	i915_pmu_gt_parked(gt);  in __gt_park()
115 	intel_rps_park(&gt->rps);  in __gt_park()
116 	intel_rc6_park(&gt->rc6);  in __gt_park()
133 void intel_gt_pm_init_early(struct intel_gt *gt)  in intel_gt_pm_init_early()  argument
136 	 * We access the runtime_pm structure via gt->i915 here rather than  in intel_gt_pm_init_early()
137 	 * gt->uncore as we do elsewhere in the file because gt->uncore is not  in intel_gt_pm_init_early()
142 	intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops, "GT");  in intel_gt_pm_init_early()
143 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);  in intel_gt_pm_init_early()
146 void intel_gt_pm_init(struct intel_gt *gt)  in intel_gt_pm_init()  argument
153 	intel_rc6_init(&gt->rc6);  in intel_gt_pm_init()
154 	intel_rps_init(&gt->rps);  in intel_gt_pm_init()
157 static bool reset_engines(struct intel_gt *gt)  in reset_engines()  argument
159 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)  in reset_engines()
162 	return intel_gt_reset_all_engines(gt) == 0;  in reset_engines()
165 static void gt_sanitize(struct intel_gt *gt, bool force)  in gt_sanitize()  argument
171 	GT_TRACE(gt, "force:%s\n", str_yes_no(force));  in gt_sanitize()
174 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);  in gt_sanitize()
175 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
177 	intel_gt_check_clock_frequency(gt);  in gt_sanitize()
185 	if (intel_gt_is_wedged(gt))  in gt_sanitize()
186 		intel_gt_unset_wedged(gt);  in gt_sanitize()
189 	intel_uc_reset_prepare(&gt->uc);  in gt_sanitize()
191 	for_each_engine(engine, gt, id) {  in gt_sanitize()
199 	if (reset_engines(gt) || force) {  in gt_sanitize()
200 		for_each_engine(engine, gt, id)  in gt_sanitize()
204 	intel_uc_reset(&gt->uc, false);  in gt_sanitize()
206 	for_each_engine(engine, gt, id)  in gt_sanitize()
210 	intel_rps_sanitize(&gt->rps);  in gt_sanitize()
212 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
213 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);  in gt_sanitize()
216 void intel_gt_pm_fini(struct intel_gt *gt)  in intel_gt_pm_fini()  argument
218 	intel_rc6_fini(&gt->rc6);  in intel_gt_pm_fini()
221 void intel_gt_resume_early(struct intel_gt *gt)  in intel_gt_resume_early()  argument
230 	intel_gt_mcr_lock_sanitize(gt);  in intel_gt_resume_early()
232 	intel_uncore_resume_early(gt->uncore);  in intel_gt_resume_early()
233 	intel_gt_check_and_clear_faults(gt);  in intel_gt_resume_early()
236 int intel_gt_resume(struct intel_gt *gt)  in intel_gt_resume()  argument
243 	err = intel_gt_has_unrecoverable_error(gt);  in intel_gt_resume()
247 	GT_TRACE(gt, "\n");  in intel_gt_resume()
255 	gt_sanitize(gt, true);  in intel_gt_resume()
257 	wakeref = intel_gt_pm_get(gt);  in intel_gt_resume()
259 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
260 	intel_rc6_sanitize(&gt->rc6);  in intel_gt_resume()
261 	if (intel_gt_is_wedged(gt)) {  in intel_gt_resume()
267 	err = intel_gt_init_hw(gt);  in intel_gt_resume()
269 		gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");  in intel_gt_resume()
273 	intel_uc_reset_finish(&gt->uc);  in intel_gt_resume()
275 	intel_rps_enable(&gt->rps);  in intel_gt_resume()
276 	intel_llc_enable(&gt->llc);  in intel_gt_resume()
278 	for_each_engine(engine, gt, id) {  in intel_gt_resume()
286 			gt_err(gt, "Failed to restart %s (%d)\n",  in intel_gt_resume()
292 	intel_rc6_enable(&gt->rc6);  in intel_gt_resume()
294 	intel_uc_resume(&gt->uc);  in intel_gt_resume()
296 	user_forcewake(gt, false);  in intel_gt_resume()
299 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
300 	intel_gt_pm_put(gt, wakeref);  in intel_gt_resume()
301 	intel_gt_bind_context_set_ready(gt);  in intel_gt_resume()
305 	intel_gt_set_wedged(gt);  in intel_gt_resume()
309 static void wait_for_suspend(struct intel_gt *gt)  in wait_for_suspend()  argument
311 	if (!intel_gt_pm_is_awake(gt))  in wait_for_suspend()
314 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {  in wait_for_suspend()
319 		intel_gt_set_wedged(gt);  in wait_for_suspend()
320 		intel_gt_retire_requests(gt);  in wait_for_suspend()
323 	intel_gt_pm_wait_for_idle(gt);  in wait_for_suspend()
326 void intel_gt_suspend_prepare(struct intel_gt *gt)  in intel_gt_suspend_prepare()  argument
328 	intel_gt_bind_context_set_unready(gt);  in intel_gt_suspend_prepare()
329 	user_forcewake(gt, true);  in intel_gt_suspend_prepare()
330 	wait_for_suspend(gt);  in intel_gt_suspend_prepare()
342 void intel_gt_suspend_late(struct intel_gt *gt)  in intel_gt_suspend_late()  argument
347 	wait_for_suspend(gt);  in intel_gt_suspend_late()
349 	if (is_mock_gt(gt))  in intel_gt_suspend_late()
352 	GEM_BUG_ON(gt->awake);  in intel_gt_suspend_late()
354 	intel_uc_suspend(&gt->uc);  in intel_gt_suspend_late()
369 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {  in intel_gt_suspend_late()
370 		intel_rps_disable(&gt->rps);  in intel_gt_suspend_late()
371 		intel_rc6_disable(&gt->rc6);  in intel_gt_suspend_late()
372 		intel_llc_disable(&gt->llc);  in intel_gt_suspend_late()
375 	gt_sanitize(gt, false);  in intel_gt_suspend_late()
377 	GT_TRACE(gt, "\n");  in intel_gt_suspend_late()
380 void intel_gt_runtime_suspend(struct intel_gt *gt)  in intel_gt_runtime_suspend()  argument
382 	intel_gt_bind_context_set_unready(gt);  in intel_gt_runtime_suspend()
383 	intel_uc_runtime_suspend(&gt->uc);  in intel_gt_runtime_suspend()
385 	GT_TRACE(gt, "\n");  in intel_gt_runtime_suspend()
388 int intel_gt_runtime_resume(struct intel_gt *gt)  in intel_gt_runtime_resume()  argument
392 	GT_TRACE(gt, "\n");  in intel_gt_runtime_resume()
393 	intel_gt_init_swizzling(gt);  in intel_gt_runtime_resume()
394 	intel_ggtt_restore_fences(gt->ggtt);  in intel_gt_runtime_resume()
396 	ret = intel_uc_runtime_resume(&gt->uc);  in intel_gt_runtime_resume()
400 	intel_gt_bind_context_set_ready(gt);  in intel_gt_runtime_resume()
404 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)  in __intel_gt_get_awake_time()  argument
406 	ktime_t total = gt->stats.total;  in __intel_gt_get_awake_time()
408 	if (gt->stats.active)  in __intel_gt_get_awake_time()
410 				  ktime_sub(ktime_get(), gt->stats.start));  in __intel_gt_get_awake_time()
415 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)  in intel_gt_get_awake_time()  argument
421 		seq = read_seqcount_begin(&gt->stats.lock);  in intel_gt_get_awake_time()
422 		total = __intel_gt_get_awake_time(gt);  in intel_gt_get_awake_time()
423 	} while (read_seqcount_retry(&gt->stats.lock, seq));  in intel_gt_get_awake_time()