xref: /linux/drivers/gpu/drm/i915/gt/intel_gt_pm.c (revision fd7d598270724cc787982ea48bbe17ad383a8b7f)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 #include <linux/suspend.h>
8 
9 #include "i915_drv.h"
10 #include "i915_irq.h"
11 #include "i915_params.h"
12 #include "intel_context.h"
13 #include "intel_engine_pm.h"
14 #include "intel_gt.h"
15 #include "intel_gt_clock_utils.h"
16 #include "intel_gt_mcr.h"
17 #include "intel_gt_pm.h"
18 #include "intel_gt_print.h"
19 #include "intel_gt_requests.h"
20 #include "intel_llc.h"
21 #include "intel_rc6.h"
22 #include "intel_rps.h"
23 #include "intel_wakeref.h"
24 #include "pxp/intel_pxp_pm.h"
25 
26 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
27 
28 static void user_forcewake(struct intel_gt *gt, bool suspend)
29 {
30 	int count = atomic_read(&gt->user_wakeref);
31 
32 	/* Inside suspend/resume so single threaded, no races to worry about. */
33 	if (likely(!count))
34 		return;
35 
36 	intel_gt_pm_get(gt);
37 	if (suspend) {
38 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
39 		atomic_sub(count, &gt->wakeref.count);
40 	} else {
41 		atomic_add(count, &gt->wakeref.count);
42 	}
43 	intel_gt_pm_put(gt);
44 }
45 
46 static void runtime_begin(struct intel_gt *gt)
47 {
48 	local_irq_disable();
49 	write_seqcount_begin(&gt->stats.lock);
50 	gt->stats.start = ktime_get();
51 	gt->stats.active = true;
52 	write_seqcount_end(&gt->stats.lock);
53 	local_irq_enable();
54 }
55 
56 static void runtime_end(struct intel_gt *gt)
57 {
58 	local_irq_disable();
59 	write_seqcount_begin(&gt->stats.lock);
60 	gt->stats.active = false;
61 	gt->stats.total =
62 		ktime_add(gt->stats.total,
63 			  ktime_sub(ktime_get(), gt->stats.start));
64 	write_seqcount_end(&gt->stats.lock);
65 	local_irq_enable();
66 }
67 
68 static int __gt_unpark(struct intel_wakeref *wf)
69 {
70 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
71 	struct drm_i915_private *i915 = gt->i915;
72 
73 	GT_TRACE(gt, "\n");
74 
75 	/*
76 	 * It seems that the DMC likes to transition between the DC states a lot
77 	 * when there are no connected displays (no active power domains) during
78 	 * command submission.
79 	 *
80 	 * This activity has negative impact on the performance of the chip with
81 	 * huge latencies observed in the interrupt handler and elsewhere.
82 	 *
83 	 * Work around it by grabbing a GT IRQ power domain whilst there is any
84 	 * GT activity, preventing any DC state transitions.
85 	 */
86 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
87 	GEM_BUG_ON(!gt->awake);
88 
89 	intel_rc6_unpark(&gt->rc6);
90 	intel_rps_unpark(&gt->rps);
91 	i915_pmu_gt_unparked(gt);
92 	intel_guc_busyness_unpark(gt);
93 
94 	intel_gt_unpark_requests(gt);
95 	runtime_begin(gt);
96 
97 	return 0;
98 }
99 
100 static int __gt_park(struct intel_wakeref *wf)
101 {
102 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
103 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
104 	struct drm_i915_private *i915 = gt->i915;
105 
106 	GT_TRACE(gt, "\n");
107 
108 	runtime_end(gt);
109 	intel_gt_park_requests(gt);
110 
111 	intel_guc_busyness_park(gt);
112 	i915_vma_parked(gt);
113 	i915_pmu_gt_parked(gt);
114 	intel_rps_park(&gt->rps);
115 	intel_rc6_park(&gt->rc6);
116 
117 	/* Everything switched off, flush any residual interrupt just in case */
118 	intel_synchronize_irq(i915);
119 
120 	/* Defer dropping the display power well for 100ms, it's slow! */
121 	GEM_BUG_ON(!wakeref);
122 	intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
123 
124 	return 0;
125 }
126 
127 static const struct intel_wakeref_ops wf_ops = {
128 	.get = __gt_unpark,
129 	.put = __gt_park,
130 };
131 
132 void intel_gt_pm_init_early(struct intel_gt *gt)
133 {
134 	/*
135 	 * We access the runtime_pm structure via gt->i915 here rather than
136 	 * gt->uncore as we do elsewhere in the file because gt->uncore is not
137 	 * yet initialized for all tiles at this point in the driver startup.
138 	 * runtime_pm is per-device rather than per-tile, so this is still the
139 	 * correct structure.
140 	 */
141 	intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops);
142 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
143 }
144 
145 void intel_gt_pm_init(struct intel_gt *gt)
146 {
147 	/*
148 	 * Enabling power-management should be "self-healing". If we cannot
149 	 * enable a feature, simply leave it disabled with a notice to the
150 	 * user.
151 	 */
152 	intel_rc6_init(&gt->rc6);
153 	intel_rps_init(&gt->rps);
154 }
155 
156 static bool reset_engines(struct intel_gt *gt)
157 {
158 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
159 		return false;
160 
161 	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
162 }
163 
164 static void gt_sanitize(struct intel_gt *gt, bool force)
165 {
166 	struct intel_engine_cs *engine;
167 	enum intel_engine_id id;
168 	intel_wakeref_t wakeref;
169 
170 	GT_TRACE(gt, "force:%s", str_yes_no(force));
171 
172 	/* Use a raw wakeref to avoid calling intel_display_power_get early */
173 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
174 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
175 
176 	intel_gt_check_clock_frequency(gt);
177 
178 	/*
179 	 * As we have just resumed the machine and woken the device up from
180 	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
181 	 * back to defaults, recovering from whatever wedged state we left it
182 	 * in and so worth trying to use the device once more.
183 	 */
184 	if (intel_gt_is_wedged(gt))
185 		intel_gt_unset_wedged(gt);
186 
187 	/* For GuC mode, ensure submission is disabled before stopping ring */
188 	intel_uc_reset_prepare(&gt->uc);
189 
190 	for_each_engine(engine, gt, id) {
191 		if (engine->reset.prepare)
192 			engine->reset.prepare(engine);
193 
194 		if (engine->sanitize)
195 			engine->sanitize(engine);
196 	}
197 
198 	if (reset_engines(gt) || force) {
199 		for_each_engine(engine, gt, id)
200 			__intel_engine_reset(engine, false);
201 	}
202 
203 	intel_uc_reset(&gt->uc, false);
204 
205 	for_each_engine(engine, gt, id)
206 		if (engine->reset.finish)
207 			engine->reset.finish(engine);
208 
209 	intel_rps_sanitize(&gt->rps);
210 
211 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
212 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
213 }
214 
215 void intel_gt_pm_fini(struct intel_gt *gt)
216 {
217 	intel_rc6_fini(&gt->rc6);
218 }
219 
220 void intel_gt_resume_early(struct intel_gt *gt)
221 {
222 	/*
223 	 * Sanitize steer semaphores during driver resume. This is necessary
224 	 * to address observed cases of steer semaphores being
225 	 * held after a suspend operation. Confirmation from the hardware team
226 	 * assures the safety of this operation, as no lock acquisitions
227 	 * by other agents occur during driver load/resume process.
228 	 */
229 	intel_gt_mcr_lock_sanitize(gt);
230 
231 	intel_uncore_resume_early(gt->uncore);
232 	intel_gt_check_and_clear_faults(gt);
233 }
234 
235 int intel_gt_resume(struct intel_gt *gt)
236 {
237 	struct intel_engine_cs *engine;
238 	enum intel_engine_id id;
239 	int err;
240 
241 	err = intel_gt_has_unrecoverable_error(gt);
242 	if (err)
243 		return err;
244 
245 	GT_TRACE(gt, "\n");
246 
247 	/*
248 	 * After resume, we may need to poke into the pinned kernel
249 	 * contexts to paper over any damage caused by the sudden suspend.
250 	 * Only the kernel contexts should remain pinned over suspend,
251 	 * allowing us to fixup the user contexts on their first pin.
252 	 */
253 	gt_sanitize(gt, true);
254 
255 	intel_gt_pm_get(gt);
256 
257 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
258 	intel_rc6_sanitize(&gt->rc6);
259 	if (intel_gt_is_wedged(gt)) {
260 		err = -EIO;
261 		goto out_fw;
262 	}
263 
264 	/* Only when the HW is re-initialised, can we replay the requests */
265 	err = intel_gt_init_hw(gt);
266 	if (err) {
267 		gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");
268 		goto err_wedged;
269 	}
270 
271 	intel_uc_reset_finish(&gt->uc);
272 
273 	intel_rps_enable(&gt->rps);
274 	intel_llc_enable(&gt->llc);
275 
276 	for_each_engine(engine, gt, id) {
277 		intel_engine_pm_get(engine);
278 
279 		engine->serial++; /* kernel context lost */
280 		err = intel_engine_resume(engine);
281 
282 		intel_engine_pm_put(engine);
283 		if (err) {
284 			gt_err(gt, "Failed to restart %s (%d)\n",
285 			       engine->name, err);
286 			goto err_wedged;
287 		}
288 	}
289 
290 	intel_rc6_enable(&gt->rc6);
291 
292 	intel_uc_resume(&gt->uc);
293 
294 	user_forcewake(gt, false);
295 
296 out_fw:
297 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
298 	intel_gt_pm_put(gt);
299 	intel_gt_bind_context_set_ready(gt);
300 	return err;
301 
302 err_wedged:
303 	intel_gt_set_wedged(gt);
304 	goto out_fw;
305 }
306 
307 static void wait_for_suspend(struct intel_gt *gt)
308 {
309 	if (!intel_gt_pm_is_awake(gt))
310 		return;
311 
312 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
313 		/*
314 		 * Forcibly cancel outstanding work and leave
315 		 * the gpu quiet.
316 		 */
317 		intel_gt_set_wedged(gt);
318 		intel_gt_retire_requests(gt);
319 	}
320 
321 	intel_gt_pm_wait_for_idle(gt);
322 }
323 
324 void intel_gt_suspend_prepare(struct intel_gt *gt)
325 {
326 	intel_gt_bind_context_set_unready(gt);
327 	user_forcewake(gt, true);
328 	wait_for_suspend(gt);
329 }
330 
331 static suspend_state_t pm_suspend_target(void)
332 {
333 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
334 	return pm_suspend_target_state;
335 #else
336 	return PM_SUSPEND_TO_IDLE;
337 #endif
338 }
339 
340 void intel_gt_suspend_late(struct intel_gt *gt)
341 {
342 	intel_wakeref_t wakeref;
343 
344 	/* We expect to be idle already; but also want to be independent */
345 	wait_for_suspend(gt);
346 
347 	if (is_mock_gt(gt))
348 		return;
349 
350 	GEM_BUG_ON(gt->awake);
351 
352 	intel_uc_suspend(&gt->uc);
353 
354 	/*
355 	 * On disabling the device, we want to turn off HW access to memory
356 	 * that we no longer own.
357 	 *
358 	 * However, not all suspend-states disable the device. S0 (s2idle)
359 	 * is effectively runtime-suspend, the device is left powered on
360 	 * but needs to be put into a low power state. We need to keep
361 	 * powermanagement enabled, but we also retain system state and so
362 	 * it remains safe to keep on using our allocated memory.
363 	 */
364 	if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
365 		return;
366 
367 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
368 		intel_rps_disable(&gt->rps);
369 		intel_rc6_disable(&gt->rc6);
370 		intel_llc_disable(&gt->llc);
371 	}
372 
373 	gt_sanitize(gt, false);
374 
375 	GT_TRACE(gt, "\n");
376 }
377 
378 void intel_gt_runtime_suspend(struct intel_gt *gt)
379 {
380 	intel_gt_bind_context_set_unready(gt);
381 	intel_uc_runtime_suspend(&gt->uc);
382 
383 	GT_TRACE(gt, "\n");
384 }
385 
386 int intel_gt_runtime_resume(struct intel_gt *gt)
387 {
388 	int ret;
389 
390 	GT_TRACE(gt, "\n");
391 	intel_gt_init_swizzling(gt);
392 	intel_ggtt_restore_fences(gt->ggtt);
393 
394 	ret = intel_uc_runtime_resume(&gt->uc);
395 	if (ret)
396 		return ret;
397 
398 	intel_gt_bind_context_set_ready(gt);
399 	return 0;
400 }
401 
402 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
403 {
404 	ktime_t total = gt->stats.total;
405 
406 	if (gt->stats.active)
407 		total = ktime_add(total,
408 				  ktime_sub(ktime_get(), gt->stats.start));
409 
410 	return total;
411 }
412 
413 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
414 {
415 	unsigned int seq;
416 	ktime_t total;
417 
418 	do {
419 		seq = read_seqcount_begin(&gt->stats.lock);
420 		total = __intel_gt_get_awake_time(gt);
421 	} while (read_seqcount_retry(&gt->stats.lock, seq));
422 
423 	return total;
424 }
425 
426 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
427 #include "selftest_gt_pm.c"
428 #endif
429