xref: /linux/drivers/gpu/drm/i915/display/intel_display_driver.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022-2023 Intel Corporation
4  *
5  * High level display driver entry points. This is a layer between top level
6  * driver code and low level display functionality; no low level display code or
7  * details here.
8  */
9 
10 #include <linux/error-injection.h>
11 #include <linux/vga_switcheroo.h>
12 #include <acpi/video.h>
13 #include <drm/display/drm_dp_mst_helper.h>
14 #include <drm/drm_atomic_helper.h>
15 #include <drm/drm_client_event.h>
16 #include <drm/drm_mode_config.h>
17 #include <drm/drm_privacy_screen_consumer.h>
18 #include <drm/drm_print.h>
19 #include <drm/drm_probe_helper.h>
20 #include <drm/drm_vblank.h>
21 
22 #include "i9xx_wm.h"
23 #include "intel_acpi.h"
24 #include "intel_atomic.h"
25 #include "intel_audio.h"
26 #include "intel_bios.h"
27 #include "intel_bw.h"
28 #include "intel_cdclk.h"
29 #include "intel_color.h"
30 #include "intel_crtc.h"
31 #include "intel_cursor.h"
32 #include "intel_dbuf_bw.h"
33 #include "intel_display_core.h"
34 #include "intel_display_debugfs.h"
35 #include "intel_display_driver.h"
36 #include "intel_display_irq.h"
37 #include "intel_display_power.h"
38 #include "intel_display_types.h"
39 #include "intel_display_utils.h"
40 #include "intel_display_wa.h"
41 #include "intel_dkl_phy.h"
42 #include "intel_dmc.h"
43 #include "intel_dp.h"
44 #include "intel_dp_tunnel.h"
45 #include "intel_dpll.h"
46 #include "intel_dpll_mgr.h"
47 #include "intel_dram.h"
48 #include "intel_encoder.h"
49 #include "intel_fb.h"
50 #include "intel_fbc.h"
51 #include "intel_fbdev.h"
52 #include "intel_fdi.h"
53 #include "intel_flipq.h"
54 #include "intel_gmbus.h"
55 #include "intel_hdcp.h"
56 #include "intel_hotplug.h"
57 #include "intel_hti.h"
58 #include "intel_initial_plane.h"
59 #include "intel_modeset_lock.h"
60 #include "intel_modeset_setup.h"
61 #include "intel_opregion.h"
62 #include "intel_overlay.h"
63 #include "intel_pmdemand.h"
64 #include "intel_pps.h"
65 #include "intel_psr.h"
66 #include "intel_quirks.h"
67 #include "intel_vga.h"
68 #include "intel_wm.h"
69 #include "skl_watermark.h"
70 
71 static int __intel_display_driver_pm_suspend(struct intel_display *display, bool shutdown);
72 
73 bool intel_display_driver_probe_defer(struct pci_dev *pdev)
74 {
75 	struct drm_privacy_screen *privacy_screen;
76 
77 	/*
78 	 * apple-gmux is needed on dual GPU MacBook Pro
79 	 * to probe the panel if we're the inactive GPU.
80 	 */
81 	if (vga_switcheroo_client_probe_defer(pdev))
82 		return true;
83 
84 	/* If the LCD panel has a privacy-screen, wait for it */
85 	privacy_screen = drm_privacy_screen_get(&pdev->dev, NULL);
86 	if (IS_ERR(privacy_screen) && PTR_ERR(privacy_screen) == -EPROBE_DEFER)
87 		return true;
88 
89 	drm_privacy_screen_put(privacy_screen);
90 
91 	return false;
92 }
93 
94 void intel_display_driver_init_hw(struct intel_display *display)
95 {
96 	if (!HAS_DISPLAY(display))
97 		return;
98 
99 	intel_cdclk_read_hw(display);
100 
101 	intel_display_wa_apply(display);
102 }
103 
104 static const struct drm_mode_config_funcs intel_mode_funcs = {
105 	.fb_create = intel_user_framebuffer_create,
106 	.get_format_info = intel_fb_get_format_info,
107 	.mode_valid = intel_mode_valid,
108 	.atomic_check = intel_atomic_check,
109 	.atomic_commit = intel_atomic_commit,
110 	.atomic_state_alloc = intel_atomic_state_alloc,
111 	.atomic_state_clear = intel_atomic_state_clear,
112 	.atomic_state_free = intel_atomic_state_free,
113 };
114 
115 static const struct drm_mode_config_helper_funcs intel_mode_config_funcs = {
116 	.atomic_commit_setup = drm_dp_mst_atomic_setup_commit,
117 };
118 
119 static void intel_mode_config_init(struct intel_display *display)
120 {
121 	struct drm_mode_config *mode_config = &display->drm->mode_config;
122 
123 	drm_mode_config_init(display->drm);
124 	INIT_LIST_HEAD(&display->global.obj_list);
125 	INIT_LIST_HEAD(&display->pipe_list);
126 
127 	mode_config->min_width = 0;
128 	mode_config->min_height = 0;
129 
130 	mode_config->preferred_depth = 24;
131 	mode_config->prefer_shadow = 1;
132 
133 	mode_config->funcs = &intel_mode_funcs;
134 	mode_config->helper_private = &intel_mode_config_funcs;
135 
136 	mode_config->async_page_flip = HAS_ASYNC_FLIPS(display);
137 
138 	/*
139 	 * Maximum framebuffer dimensions, chosen to match
140 	 * the maximum render engine surface size on gen4+.
141 	 */
142 	if (DISPLAY_VER(display) >= 7) {
143 		mode_config->max_width = 16384;
144 		mode_config->max_height = 16384;
145 	} else if (DISPLAY_VER(display) >= 4) {
146 		mode_config->max_width = 8192;
147 		mode_config->max_height = 8192;
148 	} else if (DISPLAY_VER(display) == 3) {
149 		mode_config->max_width = 4096;
150 		mode_config->max_height = 4096;
151 	} else {
152 		mode_config->max_width = 2048;
153 		mode_config->max_height = 2048;
154 	}
155 
156 	intel_cursor_mode_config_init(display);
157 }
158 
159 static void intel_mode_config_cleanup(struct intel_display *display)
160 {
161 	intel_atomic_global_obj_cleanup(display);
162 	drm_mode_config_cleanup(display->drm);
163 }
164 
165 static void intel_plane_possible_crtcs_init(struct intel_display *display)
166 {
167 	struct intel_plane *plane;
168 
169 	for_each_intel_plane(display->drm, plane) {
170 		struct intel_crtc *crtc = intel_crtc_for_pipe(display,
171 							      plane->pipe);
172 
173 		plane->base.possible_crtcs = drm_crtc_mask(&crtc->base);
174 	}
175 }
176 
177 void intel_display_driver_early_probe(struct intel_display *display)
178 {
179 	/* This must be called before any calls to HAS_PCH_* */
180 	intel_pch_detect(display);
181 
182 	if (!HAS_DISPLAY(display))
183 		return;
184 
185 	spin_lock_init(&display->fb_tracking.lock);
186 	mutex_init(&display->backlight.lock);
187 	mutex_init(&display->audio.mutex);
188 	mutex_init(&display->wm.wm_mutex);
189 	mutex_init(&display->pps.mutex);
190 	mutex_init(&display->hdcp.hdcp_mutex);
191 
192 	intel_display_irq_init(display);
193 	intel_dkl_phy_init(display);
194 	intel_color_init_hooks(display);
195 	intel_init_cdclk_hooks(display);
196 	intel_audio_hooks_init(display);
197 	intel_dpll_init_clock_hook(display);
198 	intel_init_display_hooks(display);
199 	intel_fdi_init_hook(display);
200 	intel_dmc_wl_init(display);
201 }
202 
203 /* part #1: call before irq install */
204 int intel_display_driver_probe_noirq(struct intel_display *display)
205 {
206 	int ret;
207 
208 	intel_opregion_setup(display);
209 
210 	/*
211 	 * Fill the dram structure to get the system dram info. This will be
212 	 * used for memory latency calculation.
213 	 */
214 	ret = intel_dram_detect(display);
215 	if (ret)
216 		goto cleanup_opregion;
217 
218 	intel_bw_init_hw(display);
219 
220 	if (HAS_DISPLAY(display)) {
221 		ret = drm_vblank_init(display->drm,
222 				      INTEL_NUM_PIPES(display));
223 		if (ret)
224 			goto cleanup_opregion;
225 	}
226 
227 	intel_bios_init(display);
228 
229 	intel_psr_dc5_dc6_wa_init(display);
230 
231 	/* FIXME: completely on the wrong abstraction layer */
232 	ret = intel_display_power_init(display);
233 	if (ret < 0)
234 		goto cleanup_bios;
235 
236 	intel_pmdemand_init_early(display);
237 
238 	intel_display_power_init_hw(display);
239 
240 	if (!HAS_DISPLAY(display))
241 		return 0;
242 
243 	display->hotplug.dp_wq = alloc_ordered_workqueue("intel-dp", 0);
244 	if (!display->hotplug.dp_wq) {
245 		ret = -ENOMEM;
246 		goto cleanup_pw_domain_dmc;
247 	}
248 
249 	display->wq.modeset = alloc_ordered_workqueue("i915_modeset", 0);
250 	if (!display->wq.modeset) {
251 		ret = -ENOMEM;
252 		goto cleanup_wq_dp;
253 	}
254 
255 	display->wq.flip = alloc_workqueue("i915_flip", WQ_HIGHPRI |
256 						WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
257 	if (!display->wq.flip) {
258 		ret = -ENOMEM;
259 		goto cleanup_wq_modeset;
260 	}
261 
262 	display->wq.cleanup = alloc_workqueue("i915_cleanup", WQ_HIGHPRI | WQ_PERCPU, 0);
263 	if (!display->wq.cleanup) {
264 		ret = -ENOMEM;
265 		goto cleanup_wq_flip;
266 	}
267 
268 	display->wq.unordered = alloc_workqueue("display_unordered", WQ_PERCPU, 0);
269 	if (!display->wq.unordered) {
270 		ret = -ENOMEM;
271 		goto cleanup_wq_cleanup;
272 	}
273 
274 	intel_dmc_init(display);
275 
276 	intel_mode_config_init(display);
277 
278 	ret = intel_cdclk_init(display);
279 	if (ret)
280 		goto cleanup_wq_unordered;
281 
282 	ret = intel_color_init(display);
283 	if (ret)
284 		goto cleanup_wq_unordered;
285 
286 	ret = intel_dbuf_init(display);
287 	if (ret)
288 		goto cleanup_wq_unordered;
289 
290 	ret = intel_dbuf_bw_init(display);
291 	if (ret)
292 		goto cleanup_wq_unordered;
293 
294 	ret = intel_bw_init(display);
295 	if (ret)
296 		goto cleanup_wq_unordered;
297 
298 	ret = intel_pmdemand_init(display);
299 	if (ret)
300 		goto cleanup_wq_unordered;
301 
302 	intel_init_quirks(display);
303 
304 	intel_fbc_init(display);
305 
306 	return 0;
307 
308 cleanup_wq_unordered:
309 	destroy_workqueue(display->wq.unordered);
310 cleanup_wq_cleanup:
311 	destroy_workqueue(display->wq.cleanup);
312 cleanup_wq_flip:
313 	destroy_workqueue(display->wq.flip);
314 cleanup_wq_modeset:
315 	destroy_workqueue(display->wq.modeset);
316 cleanup_wq_dp:
317 	destroy_workqueue(display->hotplug.dp_wq);
318 cleanup_pw_domain_dmc:
319 	intel_dmc_fini(display);
320 	intel_display_power_driver_remove(display);
321 cleanup_bios:
322 	intel_bios_driver_remove(display);
323 cleanup_opregion:
324 	intel_opregion_cleanup(display);
325 
326 	return ret;
327 }
328 ALLOW_ERROR_INJECTION(intel_display_driver_probe_noirq, ERRNO);
329 
330 static void set_display_access(struct intel_display *display,
331 			       bool any_task_allowed,
332 			       struct task_struct *allowed_task)
333 {
334 	struct drm_modeset_acquire_ctx ctx;
335 	int err;
336 
337 	intel_modeset_lock_ctx_retry(&ctx, NULL, 0, err) {
338 		err = drm_modeset_lock_all_ctx(display->drm, &ctx);
339 		if (err)
340 			continue;
341 
342 		display->access.any_task_allowed = any_task_allowed;
343 		display->access.allowed_task = allowed_task;
344 	}
345 
346 	drm_WARN_ON(display->drm, err);
347 }
348 
349 /**
350  * intel_display_driver_enable_user_access - Enable display HW access for all threads
351  * @display: display device instance
352  *
353  * Enable the display HW access for all threads. Examples for such accesses
354  * are modeset commits and connector probing.
355  *
356  * This function should be called during driver loading and system resume once
357  * all the HW initialization steps are done.
358  */
359 void intel_display_driver_enable_user_access(struct intel_display *display)
360 {
361 	set_display_access(display, true, NULL);
362 
363 	intel_hpd_enable_detection_work(display);
364 }
365 
366 /**
367  * intel_display_driver_disable_user_access - Disable display HW access for user threads
368  * @display: display device instance
369  *
370  * Disable the display HW access for user threads. Examples for such accesses
371  * are modeset commits and connector probing. For the current thread the
372  * access is still enabled, which should only perform HW init/deinit
373  * programming (as the initial modeset during driver loading or the disabling
374  * modeset during driver unloading and system suspend/shutdown). This function
375  * should be followed by calling either intel_display_driver_enable_user_access()
376  * after completing the HW init programming or
377  * intel_display_driver_suspend_access() after completing the HW deinit
378  * programming.
379  *
380  * This function should be called during driver loading/unloading and system
381  * suspend/shutdown before starting the HW init/deinit programming.
382  */
383 void intel_display_driver_disable_user_access(struct intel_display *display)
384 {
385 	intel_hpd_disable_detection_work(display);
386 
387 	set_display_access(display, false, current);
388 }
389 
390 /**
391  * intel_display_driver_suspend_access - Suspend display HW access for all threads
392  * @display: display device instance
393  *
394  * Disable the display HW access for all threads. Examples for such accesses
395  * are modeset commits and connector probing. This call should be either
396  * followed by calling intel_display_driver_resume_access(), or the driver
397  * should be unloaded/shutdown.
398  *
399  * This function should be called during driver unloading and system
400  * suspend/shutdown after completing the HW deinit programming.
401  */
402 void intel_display_driver_suspend_access(struct intel_display *display)
403 {
404 	set_display_access(display, false, NULL);
405 }
406 
407 /**
408  * intel_display_driver_resume_access - Resume display HW access for the resume thread
409  * @display: display device instance
410  *
411  * Enable the display HW access for the current resume thread, keeping the
412  * access disabled for all other (user) threads. Examples for such accesses
413  * are modeset commits and connector probing. The resume thread should only
414  * perform HW init programming (as the restoring modeset). This function
415  * should be followed by calling intel_display_driver_enable_user_access(),
416  * after completing the HW init programming steps.
417  *
418  * This function should be called during system resume before starting the HW
419  * init steps.
420  */
421 void intel_display_driver_resume_access(struct intel_display *display)
422 {
423 	set_display_access(display, false, current);
424 }
425 
426 /**
427  * intel_display_driver_check_access - Check if the current thread has disaplay HW access
428  * @display: display device instance
429  *
430  * Check whether the current thread has display HW access, print a debug
431  * message if it doesn't. Such accesses are modeset commits and connector
432  * probing. If the function returns %false any HW access should be prevented.
433  *
434  * Returns %true if the current thread has display HW access, %false
435  * otherwise.
436  */
437 bool intel_display_driver_check_access(struct intel_display *display)
438 {
439 	char current_task[TASK_COMM_LEN + 16];
440 	char allowed_task[TASK_COMM_LEN + 16] = "none";
441 
442 	if (display->access.any_task_allowed ||
443 	    display->access.allowed_task == current)
444 		return true;
445 
446 	snprintf(current_task, sizeof(current_task), "%s[%d]",
447 		 current->comm, task_pid_vnr(current));
448 
449 	if (display->access.allowed_task)
450 		snprintf(allowed_task, sizeof(allowed_task), "%s[%d]",
451 			 display->access.allowed_task->comm,
452 			 task_pid_vnr(display->access.allowed_task));
453 
454 	drm_dbg_kms(display->drm,
455 		    "Reject display access from task %s (allowed to %s)\n",
456 		    current_task, allowed_task);
457 
458 	return false;
459 }
460 
461 /* part #2: call after irq install, but before gem init */
462 int intel_display_driver_probe_nogem(struct intel_display *display)
463 {
464 	int ret;
465 
466 	if (!HAS_DISPLAY(display))
467 		return 0;
468 
469 	intel_wm_init(display);
470 
471 	intel_panel_sanitize_ssc(display);
472 
473 	intel_pps_setup(display);
474 
475 	intel_gmbus_setup(display);
476 
477 	ret = intel_crtc_init(display);
478 	if (ret)
479 		goto err_mode_config;
480 
481 	intel_plane_possible_crtcs_init(display);
482 	intel_dpll_init(display);
483 	intel_fdi_pll_freq_update(display);
484 
485 	intel_display_driver_init_hw(display);
486 	intel_dpll_update_ref_clks(display);
487 
488 	if (display->cdclk.max_cdclk_freq == 0)
489 		intel_update_max_cdclk(display);
490 
491 	intel_hti_init(display);
492 
493 	intel_setup_outputs(display);
494 
495 	ret = intel_dp_tunnel_mgr_init(display);
496 	if (ret)
497 		goto err_hdcp;
498 
499 	intel_display_driver_disable_user_access(display);
500 
501 	drm_modeset_lock_all(display->drm);
502 	intel_modeset_setup_hw_state(display, display->drm->mode_config.acquire_ctx);
503 	intel_acpi_assign_connector_fwnodes(display);
504 	drm_modeset_unlock_all(display->drm);
505 
506 	intel_initial_plane_config(display);
507 
508 	/*
509 	 * Make sure hardware watermarks really match the state we read out.
510 	 * Note that we need to do this after reconstructing the BIOS fb's
511 	 * since the watermark calculation done here will use pstate->fb.
512 	 */
513 	if (!HAS_GMCH(display))
514 		ilk_wm_sanitize(display);
515 
516 	return 0;
517 
518 err_hdcp:
519 	intel_hdcp_component_fini(display);
520 err_mode_config:
521 	intel_mode_config_cleanup(display);
522 
523 	return ret;
524 }
525 
526 /* part #3: call after gem init */
527 int intel_display_driver_probe(struct intel_display *display)
528 {
529 	int ret;
530 
531 	if (!HAS_DISPLAY(display))
532 		return 0;
533 
534 	/*
535 	 * This will bind stuff into ggtt, so it needs to be done after
536 	 * the BIOS fb takeover and whatever else magic ggtt reservations
537 	 * happen during gem/ggtt init.
538 	 */
539 	intel_hdcp_component_init(display);
540 
541 	intel_flipq_init(display);
542 
543 	/*
544 	 * Force all active planes to recompute their states. So that on
545 	 * mode_setcrtc after probe, all the intel_plane_state variables
546 	 * are already calculated and there is no assert_plane warnings
547 	 * during bootup.
548 	 */
549 	ret = intel_initial_commit(display);
550 	if (ret)
551 		drm_dbg_kms(display->drm, "Initial modeset failed, %d\n", ret);
552 
553 	intel_overlay_setup(display);
554 
555 	/* Only enable hotplug handling once the fbdev is fully set up. */
556 	intel_hpd_init(display);
557 
558 	skl_watermark_ipc_init(display);
559 
560 	return 0;
561 }
562 
563 void intel_display_driver_register(struct intel_display *display)
564 {
565 	struct drm_printer p = drm_dbg_printer(display->drm, DRM_UT_KMS,
566 					       "i915 display info:");
567 
568 	if (!HAS_DISPLAY(display))
569 		return;
570 
571 	intel_vga_register(display);
572 
573 	/* Must be done after probing outputs */
574 	intel_opregion_register(display);
575 	intel_acpi_video_register(display);
576 
577 	intel_audio_init(display);
578 
579 	intel_display_driver_enable_user_access(display);
580 
581 	intel_audio_register(display);
582 
583 	intel_display_debugfs_register(display);
584 
585 	/*
586 	 * We need to coordinate the hotplugs with the asynchronous
587 	 * fbdev configuration, for which we use the
588 	 * fbdev->async_cookie.
589 	 */
590 	drm_kms_helper_poll_init(display->drm);
591 	intel_hpd_poll_disable(display);
592 
593 	intel_fbdev_setup(display);
594 
595 	intel_display_device_info_print(DISPLAY_INFO(display),
596 					DISPLAY_RUNTIME_INFO(display), &p);
597 
598 	intel_register_dsm_handler();
599 }
600 
601 /* part #1: call before irq uninstall */
602 void intel_display_driver_remove(struct intel_display *display)
603 {
604 	if (!HAS_DISPLAY(display))
605 		return;
606 
607 	flush_workqueue(display->wq.flip);
608 	flush_workqueue(display->wq.modeset);
609 	flush_workqueue(display->wq.cleanup);
610 	flush_workqueue(display->wq.unordered);
611 
612 	/*
613 	 * MST topology needs to be suspended so we don't have any calls to
614 	 * fbdev after it's finalized. MST will be destroyed later as part of
615 	 * drm_mode_config_cleanup()
616 	 */
617 	intel_dp_mst_suspend(display);
618 }
619 
620 /* part #2: call after irq uninstall */
621 void intel_display_driver_remove_noirq(struct intel_display *display)
622 {
623 	if (!HAS_DISPLAY(display))
624 		return;
625 
626 	intel_hpd_cancel_work(display);
627 
628 	intel_display_driver_suspend_access(display);
629 
630 	/*
631 	 * Due to the hpd irq storm handling the hotplug work can re-arm the
632 	 * poll handlers. Hence disable polling after hpd handling is shut down.
633 	 */
634 	intel_hpd_poll_fini(display);
635 
636 	intel_unregister_dsm_handler();
637 
638 	/* flush any delayed tasks or pending work */
639 	flush_workqueue(display->wq.unordered);
640 
641 	intel_hdcp_component_fini(display);
642 
643 	intel_mode_config_cleanup(display);
644 
645 	intel_dp_tunnel_mgr_cleanup(display);
646 
647 	intel_overlay_cleanup(display);
648 
649 	intel_gmbus_teardown(display);
650 
651 	destroy_workqueue(display->hotplug.dp_wq);
652 	destroy_workqueue(display->wq.flip);
653 	destroy_workqueue(display->wq.modeset);
654 	destroy_workqueue(display->wq.cleanup);
655 	destroy_workqueue(display->wq.unordered);
656 
657 	intel_fbc_cleanup(display);
658 }
659 
660 /* part #3: call after gem init */
661 void intel_display_driver_remove_nogem(struct intel_display *display)
662 {
663 	intel_dmc_fini(display);
664 
665 	intel_display_power_driver_remove(display);
666 
667 	intel_bios_driver_remove(display);
668 
669 	intel_opregion_cleanup(display);
670 }
671 
672 void intel_display_driver_unregister(struct intel_display *display)
673 {
674 	if (!HAS_DISPLAY(display))
675 		return;
676 
677 	intel_unregister_dsm_handler();
678 
679 	drm_client_dev_unregister(display->drm);
680 
681 	/*
682 	 * After flushing the fbdev (incl. a late async config which
683 	 * will have delayed queuing of a hotplug event), then flush
684 	 * the hotplug events.
685 	 */
686 	drm_kms_helper_poll_fini(display->drm);
687 
688 	intel_display_driver_disable_user_access(display);
689 
690 	intel_audio_deinit(display);
691 
692 	drm_atomic_helper_shutdown(display->drm);
693 
694 	acpi_video_unregister();
695 	intel_opregion_unregister(display);
696 
697 	intel_vga_unregister(display);
698 }
699 
700 void intel_display_driver_shutdown(struct intel_display *display)
701 {
702 	if (!HAS_DISPLAY(display))
703 		return;
704 
705 	__intel_display_driver_pm_suspend(display, true);
706 
707 	intel_encoder_shutdown_all(display);
708 }
709 
710 void intel_display_driver_shutdown_late(struct intel_display *display)
711 {
712 	if (!HAS_DISPLAY(display))
713 		return;
714 
715 	/*
716 	 * The only requirement is to reboot with display DC states disabled,
717 	 * for now leaving all display power wells in the INIT power domain
718 	 * enabled.
719 	 */
720 	intel_display_power_driver_remove(display);
721 }
722 
723 /*
724  * turn all crtc's off, but do not adjust state
725  * This has to be paired with a call to intel_modeset_setup_hw_state.
726  */
727 static int __intel_display_driver_pm_suspend(struct intel_display *display, bool shutdown)
728 {
729 	int ret = 0;
730 
731 	if (!HAS_DISPLAY(display))
732 		return 0;
733 
734 	/*
735 	 * We do a lot of poking in a lot of registers, make sure they work
736 	 * properly.
737 	 */
738 	intel_display_power_disable(display);
739 
740 	drm_client_dev_suspend(display->drm);
741 
742 	drm_kms_helper_poll_disable(display->drm);
743 	intel_display_driver_disable_user_access(display);
744 
745 	if (shutdown) {
746 		drm_atomic_helper_shutdown(display->drm);
747 	} else {
748 		struct drm_atomic_commit *state;
749 
750 		state = drm_atomic_helper_suspend(display->drm);
751 		ret = PTR_ERR_OR_ZERO(state);
752 		if (ret)
753 			drm_err(display->drm, "Suspending crtc's failed with %i\n",
754 				ret);
755 		else
756 			display->restore.modeset_state = state;
757 	}
758 
759 	/* ensure all DPT VMAs have been unpinned for intel_dpt_suspend() */
760 	flush_workqueue(display->wq.cleanup);
761 
762 	intel_dp_mst_suspend(display);
763 
764 	intel_encoder_block_all_hpds(display);
765 
766 	intel_hpd_cancel_work(display);
767 
768 	intel_display_driver_suspend_access(display);
769 
770 	intel_encoder_suspend_all(display);
771 
772 	return ret;
773 }
774 
775 int intel_display_driver_pm_suspend(struct intel_display *display)
776 {
777 	return __intel_display_driver_pm_suspend(display, false);
778 }
779 
780 void intel_display_driver_pm_suspend_late(struct intel_display *display, bool s2idle)
781 {
782 	if (!HAS_DISPLAY(display))
783 		return;
784 
785 	intel_display_power_suspend_late(display, s2idle);
786 }
787 
788 void intel_display_driver_pm_resume_early(struct intel_display *display)
789 {
790 	if (!HAS_DISPLAY(display))
791 		return;
792 
793 	intel_display_power_resume_early(display);
794 }
795 
796 int
797 __intel_display_driver_resume(struct intel_display *display,
798 			      struct drm_atomic_commit *state,
799 			      struct drm_modeset_acquire_ctx *ctx)
800 {
801 	struct drm_crtc_state *crtc_state;
802 	struct drm_crtc *crtc;
803 	int ret, i;
804 
805 	intel_modeset_setup_hw_state(display, ctx);
806 
807 	if (!state)
808 		return 0;
809 
810 	/*
811 	 * We've duplicated the state, pointers to the old state are invalid.
812 	 *
813 	 * Don't attempt to use the old state until we commit the duplicated state.
814 	 */
815 	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
816 		/*
817 		 * Force recalculation even if we restore
818 		 * current state. With fast modeset this may not result
819 		 * in a modeset when the state is compatible.
820 		 */
821 		crtc_state->mode_changed = true;
822 	}
823 
824 	/* ignore any reset values/BIOS leftovers in the WM registers */
825 	if (!HAS_GMCH(display))
826 		to_intel_atomic_state(state)->skip_intermediate_wm = true;
827 
828 	ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
829 
830 	drm_WARN_ON(display->drm, ret == -EDEADLK);
831 
832 	return ret;
833 }
834 
835 void intel_display_driver_pm_resume(struct intel_display *display)
836 {
837 	struct drm_atomic_commit *state = display->restore.modeset_state;
838 	struct drm_modeset_acquire_ctx ctx;
839 	int ret;
840 
841 	if (!HAS_DISPLAY(display))
842 		return;
843 
844 	intel_display_driver_resume_access(display);
845 
846 	intel_hpd_init(display);
847 
848 	intel_encoder_unblock_all_hpds(display);
849 
850 	/* MST sideband requires HPD interrupts enabled */
851 	intel_dp_mst_resume(display);
852 
853 	display->restore.modeset_state = NULL;
854 	if (state)
855 		state->acquire_ctx = &ctx;
856 
857 	drm_modeset_acquire_init(&ctx, 0);
858 
859 	while (1) {
860 		ret = drm_modeset_lock_all_ctx(display->drm, &ctx);
861 		if (ret != -EDEADLK)
862 			break;
863 
864 		drm_modeset_backoff(&ctx);
865 	}
866 
867 	if (!ret)
868 		ret = __intel_display_driver_resume(display, state, &ctx);
869 
870 	skl_watermark_ipc_update(display);
871 	drm_modeset_drop_locks(&ctx);
872 	drm_modeset_acquire_fini(&ctx);
873 
874 	if (ret)
875 		drm_err(display->drm,
876 			"Restoring old state failed with %i\n", ret);
877 	if (state)
878 		drm_atomic_commit_put(state);
879 
880 	intel_display_driver_enable_user_access(display);
881 	drm_kms_helper_poll_enable(display->drm);
882 
883 	intel_hpd_poll_disable(display);
884 
885 	intel_opregion_resume(display);
886 
887 	drm_client_dev_resume(display->drm);
888 
889 	intel_display_power_enable(display);
890 }
891 
892 void intel_display_driver_runtime_pm_enable(struct intel_display *display)
893 {
894 	intel_display_power_enable(display);
895 }
896 
897 void intel_display_driver_runtime_pm_disable(struct intel_display *display)
898 {
899 	intel_display_power_disable(display);
900 }
901 
902 /* before irq suspend */
903 void intel_display_driver_pm_runtime_suspend(struct intel_display *display)
904 {
905 }
906 
907 /* after irq suspend */
908 void intel_display_driver_pm_runtime_suspend_late(struct intel_display *display)
909 {
910 	intel_display_power_runtime_suspend(display);
911 
912 	/*
913 	 * FIXME: We really should find a document that references the arguments
914 	 * used below!
915 	 */
916 	if (display->platform.broadwell) {
917 		/*
918 		 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
919 		 * being detected, and the call we do at i915_pm_runtime_resume()
920 		 * won't be able to restore them. Since PCI_D3hot matches the
921 		 * actual specification and appears to be working, use it.
922 		 */
923 		intel_opregion_notify_adapter(display, PCI_D3hot);
924 	} else {
925 		/*
926 		 * current versions of firmware which depend on this opregion
927 		 * notification have repurposed the D1 definition to mean
928 		 * "runtime suspended" vs. what you would normally expect (D3)
929 		 * to distinguish it from notifications that might be sent via
930 		 * the suspend path.
931 		 */
932 		intel_opregion_notify_adapter(display, PCI_D1);
933 	}
934 
935 	if (!display->platform.valleyview && !display->platform.cherryview)
936 		intel_hpd_poll_enable(display);
937 }
938 
939 /* before irq resume */
940 void intel_display_driver_pm_runtime_resume_early(struct intel_display *display)
941 {
942 	intel_opregion_notify_adapter(display, PCI_D0);
943 
944 	intel_display_power_runtime_resume(display);
945 }
946 
947 /* after irq resume */
948 void intel_display_driver_pm_runtime_resume(struct intel_display *display)
949 {
950 	/*
951 	 * On VLV/CHV display interrupts are part of the display
952 	 * power well, so hpd is reinitialized from there. For
953 	 * everyone else do it here.
954 	 */
955 	if (!display->platform.valleyview && !display->platform.cherryview) {
956 		intel_hpd_init(display);
957 		intel_hpd_poll_disable(display);
958 	}
959 
960 	skl_watermark_ipc_update(display);
961 }
962