xref: /linux/drivers/gpu/drm/i915/display/intel_psr.c (revision ef9226cd56b718c79184a3466d32984a51cb449c)
1 /*
2  * Copyright © 2014 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <drm/drm_atomic_helper.h>
25 #include <drm/drm_damage_helper.h>
26 #include <drm/drm_debugfs.h>
27 
28 #include "i915_drv.h"
29 #include "i915_reg.h"
30 #include "intel_atomic.h"
31 #include "intel_crtc.h"
32 #include "intel_ddi.h"
33 #include "intel_de.h"
34 #include "intel_display_types.h"
35 #include "intel_dp.h"
36 #include "intel_dp_aux.h"
37 #include "intel_frontbuffer.h"
38 #include "intel_hdmi.h"
39 #include "intel_psr.h"
40 #include "intel_psr_regs.h"
41 #include "intel_snps_phy.h"
42 #include "skl_universal_plane.h"
43 
44 /**
45  * DOC: Panel Self Refresh (PSR/SRD)
46  *
47  * Since Haswell Display controller supports Panel Self-Refresh on display
48  * panels witch have a remote frame buffer (RFB) implemented according to PSR
49  * spec in eDP1.3. PSR feature allows the display to go to lower standby states
50  * when system is idle but display is on as it eliminates display refresh
51  * request to DDR memory completely as long as the frame buffer for that
52  * display is unchanged.
53  *
54  * Panel Self Refresh must be supported by both Hardware (source) and
55  * Panel (sink).
56  *
57  * PSR saves power by caching the framebuffer in the panel RFB, which allows us
58  * to power down the link and memory controller. For DSI panels the same idea
59  * is called "manual mode".
60  *
61  * The implementation uses the hardware-based PSR support which automatically
62  * enters/exits self-refresh mode. The hardware takes care of sending the
63  * required DP aux message and could even retrain the link (that part isn't
64  * enabled yet though). The hardware also keeps track of any frontbuffer
65  * changes to know when to exit self-refresh mode again. Unfortunately that
66  * part doesn't work too well, hence why the i915 PSR support uses the
67  * software frontbuffer tracking to make sure it doesn't miss a screen
68  * update. For this integration intel_psr_invalidate() and intel_psr_flush()
69  * get called by the frontbuffer tracking code. Note that because of locking
70  * issues the self-refresh re-enable code is done from a work queue, which
71  * must be correctly synchronized/cancelled when shutting down the pipe."
72  *
73  * DC3CO (DC3 clock off)
74  *
75  * On top of PSR2, GEN12 adds a intermediate power savings state that turns
76  * clock off automatically during PSR2 idle state.
77  * The smaller overhead of DC3co entry/exit vs. the overhead of PSR2 deep sleep
78  * entry/exit allows the HW to enter a low-power state even when page flipping
79  * periodically (for instance a 30fps video playback scenario).
80  *
81  * Every time a flips occurs PSR2 will get out of deep sleep state(if it was),
82  * so DC3CO is enabled and tgl_dc3co_disable_work is schedule to run after 6
83  * frames, if no other flip occurs and the function above is executed, DC3CO is
84  * disabled and PSR2 is configured to enter deep sleep, resetting again in case
85  * of another flip.
86  * Front buffer modifications do not trigger DC3CO activation on purpose as it
87  * would bring a lot of complexity and most of the moderns systems will only
88  * use page flips.
89  */
90 
91 /*
92  * Description of PSR mask bits:
93  *
94  * EDP_PSR_DEBUG[16]/EDP_PSR_DEBUG_MASK_DISP_REG_WRITE (hsw-skl):
95  *
96  *  When unmasked (nearly) all display register writes (eg. even
97  *  SWF) trigger a PSR exit. Some registers are excluded from this
98  *  and they have a more specific mask (described below). On icl+
99  *  this bit no longer exists and is effectively always set.
100  *
101  * PIPE_MISC[21]/PIPE_MISC_PSR_MASK_PIPE_REG_WRITE (skl+):
102  *
103  *  When unmasked (nearly) all pipe/plane register writes
104  *  trigger a PSR exit. Some plane registers are excluded from this
105  *  and they have a more specific mask (described below).
106  *
107  * CHICKEN_PIPESL_1[11]/SKL_PSR_MASK_PLANE_FLIP (skl+):
108  * PIPE_MISC[23]/PIPE_MISC_PSR_MASK_PRIMARY_FLIP (bdw):
109  * EDP_PSR_DEBUG[23]/EDP_PSR_DEBUG_MASK_PRIMARY_FLIP (hsw):
110  *
111  *  When unmasked PRI_SURF/PLANE_SURF writes trigger a PSR exit.
112  *  SPR_SURF/CURBASE are not included in this and instead are
113  *  controlled by PIPE_MISC_PSR_MASK_PIPE_REG_WRITE (skl+) or
114  *  EDP_PSR_DEBUG_MASK_DISP_REG_WRITE (hsw/bdw).
115  *
116  * PIPE_MISC[22]/PIPE_MISC_PSR_MASK_SPRITE_ENABLE (bdw):
117  * EDP_PSR_DEBUG[21]/EDP_PSR_DEBUG_MASK_SPRITE_ENABLE (hsw):
118  *
119  *  When unmasked PSR is blocked as long as the sprite
120  *  plane is enabled. skl+ with their universal planes no
121  *  longer have a mask bit like this, and no plane being
122  *  enabledb blocks PSR.
123  *
124  * PIPE_MISC[21]/PIPE_MISC_PSR_MASK_CURSOR_MOVE (bdw):
125  * EDP_PSR_DEBUG[20]/EDP_PSR_DEBUG_MASK_CURSOR_MOVE (hsw):
126  *
127  *  When umasked CURPOS writes trigger a PSR exit. On skl+
128  *  this doesn't exit but CURPOS is included in the
129  *  PIPE_MISC_PSR_MASK_PIPE_REG_WRITE mask.
130  *
131  * PIPE_MISC[20]/PIPE_MISC_PSR_MASK_VBLANK_VSYNC_INT (bdw+):
132  * EDP_PSR_DEBUG[19]/EDP_PSR_DEBUG_MASK_VBLANK_VSYNC_INT (hsw):
133  *
134  *  When unmasked PSR is blocked as long as vblank and/or vsync
135  *  interrupt is unmasked in IMR *and* enabled in IER.
136  *
137  * CHICKEN_TRANS[30]/SKL_UNMASK_VBL_TO_PIPE_IN_SRD (skl+):
138  * CHICKEN_PAR1_1[15]/HSW_MASK_VBL_TO_PIPE_IN_SRD (hsw/bdw):
139  *
140  *  Selectcs whether PSR exit generates an extra vblank before
141  *  the first frame is transmitted. Also note the opposite polarity
142  *  if the bit on hsw/bdw vs. skl+ (masked==generate the extra vblank,
143  *  unmasked==do not generate the extra vblank).
144  *
145  *  With DC states enabled the extra vblank happens after link training,
146  *  with DC states disabled it happens immediately upuon PSR exit trigger.
147  *  No idea as of now why there is a difference. HSW/BDW (which don't
148  *  even have DMC) always generate it after link training. Go figure.
149  *
150  *  Unfortunately CHICKEN_TRANS itself seems to be double buffered
151  *  and thus won't latch until the first vblank. So with DC states
152  *  enabled the register effctively uses the reset value during DC5
153  *  exit+PSR exit sequence, and thus the bit does nothing until
154  *  latched by the vblank that it was trying to prevent from being
155  *  generated in the first place. So we should probably call this
156  *  one a chicken/egg bit instead on skl+.
157  *
158  *  In standby mode (as opposed to link-off) this makes no difference
159  *  as the timing generator keeps running the whole time generating
160  *  normal periodic vblanks.
161  *
162  *  WaPsrDPAMaskVBlankInSRD asks us to set the bit on hsw/bdw,
163  *  and doing so makes the behaviour match the skl+ reset value.
164  *
165  * CHICKEN_PIPESL_1[0]/BDW_UNMASK_VBL_TO_REGS_IN_SRD (bdw):
166  * CHICKEN_PIPESL_1[15]/HSW_UNMASK_VBL_TO_REGS_IN_SRD (hsw):
167  *
168  *  On BDW without this bit is no vblanks whatsoever are
169  *  generated after PSR exit. On HSW this has no apparant effect.
170  *  WaPsrDPRSUnmaskVBlankInSRD says to set this.
171  *
172  * The rest of the bits are more self-explanatory and/or
173  * irrelevant for normal operation.
174  */
175 
176 #define CAN_PSR(intel_dp) ((intel_dp)->psr.sink_support && \
177 			   (intel_dp)->psr.source_support)
178 
179 #define CAN_PANEL_REPLAY(intel_dp) ((intel_dp)->psr.sink_panel_replay_support && \
180 				    (intel_dp)->psr.source_panel_replay_support)
181 
182 bool intel_encoder_can_psr(struct intel_encoder *encoder)
183 {
184 	if (intel_encoder_is_dp(encoder) || encoder->type == INTEL_OUTPUT_DP_MST)
185 		return CAN_PSR(enc_to_intel_dp(encoder)) ||
186 		       CAN_PANEL_REPLAY(enc_to_intel_dp(encoder));
187 	else
188 		return false;
189 }
190 
191 static bool psr_global_enabled(struct intel_dp *intel_dp)
192 {
193 	struct intel_connector *connector = intel_dp->attached_connector;
194 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
195 
196 	switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
197 	case I915_PSR_DEBUG_DEFAULT:
198 		if (i915->display.params.enable_psr == -1)
199 			return connector->panel.vbt.psr.enable;
200 		return i915->display.params.enable_psr;
201 	case I915_PSR_DEBUG_DISABLE:
202 		return false;
203 	default:
204 		return true;
205 	}
206 }
207 
208 static bool psr2_global_enabled(struct intel_dp *intel_dp)
209 {
210 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
211 
212 	switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
213 	case I915_PSR_DEBUG_DISABLE:
214 	case I915_PSR_DEBUG_FORCE_PSR1:
215 		return false;
216 	default:
217 		if (i915->display.params.enable_psr == 1)
218 			return false;
219 		return true;
220 	}
221 }
222 
223 static u32 psr_irq_psr_error_bit_get(struct intel_dp *intel_dp)
224 {
225 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
226 
227 	return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_ERROR :
228 		EDP_PSR_ERROR(intel_dp->psr.transcoder);
229 }
230 
231 static u32 psr_irq_post_exit_bit_get(struct intel_dp *intel_dp)
232 {
233 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
234 
235 	return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_POST_EXIT :
236 		EDP_PSR_POST_EXIT(intel_dp->psr.transcoder);
237 }
238 
239 static u32 psr_irq_pre_entry_bit_get(struct intel_dp *intel_dp)
240 {
241 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
242 
243 	return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_PRE_ENTRY :
244 		EDP_PSR_PRE_ENTRY(intel_dp->psr.transcoder);
245 }
246 
247 static u32 psr_irq_mask_get(struct intel_dp *intel_dp)
248 {
249 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
250 
251 	return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_MASK :
252 		EDP_PSR_MASK(intel_dp->psr.transcoder);
253 }
254 
255 static i915_reg_t psr_ctl_reg(struct drm_i915_private *dev_priv,
256 			      enum transcoder cpu_transcoder)
257 {
258 	if (DISPLAY_VER(dev_priv) >= 8)
259 		return EDP_PSR_CTL(cpu_transcoder);
260 	else
261 		return HSW_SRD_CTL;
262 }
263 
264 static i915_reg_t psr_debug_reg(struct drm_i915_private *dev_priv,
265 				enum transcoder cpu_transcoder)
266 {
267 	if (DISPLAY_VER(dev_priv) >= 8)
268 		return EDP_PSR_DEBUG(cpu_transcoder);
269 	else
270 		return HSW_SRD_DEBUG;
271 }
272 
273 static i915_reg_t psr_perf_cnt_reg(struct drm_i915_private *dev_priv,
274 				   enum transcoder cpu_transcoder)
275 {
276 	if (DISPLAY_VER(dev_priv) >= 8)
277 		return EDP_PSR_PERF_CNT(cpu_transcoder);
278 	else
279 		return HSW_SRD_PERF_CNT;
280 }
281 
282 static i915_reg_t psr_status_reg(struct drm_i915_private *dev_priv,
283 				 enum transcoder cpu_transcoder)
284 {
285 	if (DISPLAY_VER(dev_priv) >= 8)
286 		return EDP_PSR_STATUS(cpu_transcoder);
287 	else
288 		return HSW_SRD_STATUS;
289 }
290 
291 static i915_reg_t psr_imr_reg(struct drm_i915_private *dev_priv,
292 			      enum transcoder cpu_transcoder)
293 {
294 	if (DISPLAY_VER(dev_priv) >= 12)
295 		return TRANS_PSR_IMR(cpu_transcoder);
296 	else
297 		return EDP_PSR_IMR;
298 }
299 
300 static i915_reg_t psr_iir_reg(struct drm_i915_private *dev_priv,
301 			      enum transcoder cpu_transcoder)
302 {
303 	if (DISPLAY_VER(dev_priv) >= 12)
304 		return TRANS_PSR_IIR(cpu_transcoder);
305 	else
306 		return EDP_PSR_IIR;
307 }
308 
309 static i915_reg_t psr_aux_ctl_reg(struct drm_i915_private *dev_priv,
310 				  enum transcoder cpu_transcoder)
311 {
312 	if (DISPLAY_VER(dev_priv) >= 8)
313 		return EDP_PSR_AUX_CTL(cpu_transcoder);
314 	else
315 		return HSW_SRD_AUX_CTL;
316 }
317 
318 static i915_reg_t psr_aux_data_reg(struct drm_i915_private *dev_priv,
319 				   enum transcoder cpu_transcoder, int i)
320 {
321 	if (DISPLAY_VER(dev_priv) >= 8)
322 		return EDP_PSR_AUX_DATA(cpu_transcoder, i);
323 	else
324 		return HSW_SRD_AUX_DATA(i);
325 }
326 
327 static void psr_irq_control(struct intel_dp *intel_dp)
328 {
329 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
330 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
331 	u32 mask;
332 
333 	mask = psr_irq_psr_error_bit_get(intel_dp);
334 	if (intel_dp->psr.debug & I915_PSR_DEBUG_IRQ)
335 		mask |= psr_irq_post_exit_bit_get(intel_dp) |
336 			psr_irq_pre_entry_bit_get(intel_dp);
337 
338 	intel_de_rmw(dev_priv, psr_imr_reg(dev_priv, cpu_transcoder),
339 		     psr_irq_mask_get(intel_dp), ~mask);
340 }
341 
342 static void psr_event_print(struct drm_i915_private *i915,
343 			    u32 val, bool psr2_enabled)
344 {
345 	drm_dbg_kms(&i915->drm, "PSR exit events: 0x%x\n", val);
346 	if (val & PSR_EVENT_PSR2_WD_TIMER_EXPIRE)
347 		drm_dbg_kms(&i915->drm, "\tPSR2 watchdog timer expired\n");
348 	if ((val & PSR_EVENT_PSR2_DISABLED) && psr2_enabled)
349 		drm_dbg_kms(&i915->drm, "\tPSR2 disabled\n");
350 	if (val & PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN)
351 		drm_dbg_kms(&i915->drm, "\tSU dirty FIFO underrun\n");
352 	if (val & PSR_EVENT_SU_CRC_FIFO_UNDERRUN)
353 		drm_dbg_kms(&i915->drm, "\tSU CRC FIFO underrun\n");
354 	if (val & PSR_EVENT_GRAPHICS_RESET)
355 		drm_dbg_kms(&i915->drm, "\tGraphics reset\n");
356 	if (val & PSR_EVENT_PCH_INTERRUPT)
357 		drm_dbg_kms(&i915->drm, "\tPCH interrupt\n");
358 	if (val & PSR_EVENT_MEMORY_UP)
359 		drm_dbg_kms(&i915->drm, "\tMemory up\n");
360 	if (val & PSR_EVENT_FRONT_BUFFER_MODIFY)
361 		drm_dbg_kms(&i915->drm, "\tFront buffer modification\n");
362 	if (val & PSR_EVENT_WD_TIMER_EXPIRE)
363 		drm_dbg_kms(&i915->drm, "\tPSR watchdog timer expired\n");
364 	if (val & PSR_EVENT_PIPE_REGISTERS_UPDATE)
365 		drm_dbg_kms(&i915->drm, "\tPIPE registers updated\n");
366 	if (val & PSR_EVENT_REGISTER_UPDATE)
367 		drm_dbg_kms(&i915->drm, "\tRegister updated\n");
368 	if (val & PSR_EVENT_HDCP_ENABLE)
369 		drm_dbg_kms(&i915->drm, "\tHDCP enabled\n");
370 	if (val & PSR_EVENT_KVMR_SESSION_ENABLE)
371 		drm_dbg_kms(&i915->drm, "\tKVMR session enabled\n");
372 	if (val & PSR_EVENT_VBI_ENABLE)
373 		drm_dbg_kms(&i915->drm, "\tVBI enabled\n");
374 	if (val & PSR_EVENT_LPSP_MODE_EXIT)
375 		drm_dbg_kms(&i915->drm, "\tLPSP mode exited\n");
376 	if ((val & PSR_EVENT_PSR_DISABLE) && !psr2_enabled)
377 		drm_dbg_kms(&i915->drm, "\tPSR disabled\n");
378 }
379 
380 void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir)
381 {
382 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
383 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
384 	ktime_t time_ns =  ktime_get();
385 
386 	if (psr_iir & psr_irq_pre_entry_bit_get(intel_dp)) {
387 		intel_dp->psr.last_entry_attempt = time_ns;
388 		drm_dbg_kms(&dev_priv->drm,
389 			    "[transcoder %s] PSR entry attempt in 2 vblanks\n",
390 			    transcoder_name(cpu_transcoder));
391 	}
392 
393 	if (psr_iir & psr_irq_post_exit_bit_get(intel_dp)) {
394 		intel_dp->psr.last_exit = time_ns;
395 		drm_dbg_kms(&dev_priv->drm,
396 			    "[transcoder %s] PSR exit completed\n",
397 			    transcoder_name(cpu_transcoder));
398 
399 		if (DISPLAY_VER(dev_priv) >= 9) {
400 			u32 val;
401 
402 			val = intel_de_rmw(dev_priv, PSR_EVENT(cpu_transcoder), 0, 0);
403 
404 			psr_event_print(dev_priv, val, intel_dp->psr.psr2_enabled);
405 		}
406 	}
407 
408 	if (psr_iir & psr_irq_psr_error_bit_get(intel_dp)) {
409 		drm_warn(&dev_priv->drm, "[transcoder %s] PSR aux error\n",
410 			 transcoder_name(cpu_transcoder));
411 
412 		intel_dp->psr.irq_aux_error = true;
413 
414 		/*
415 		 * If this interruption is not masked it will keep
416 		 * interrupting so fast that it prevents the scheduled
417 		 * work to run.
418 		 * Also after a PSR error, we don't want to arm PSR
419 		 * again so we don't care about unmask the interruption
420 		 * or unset irq_aux_error.
421 		 */
422 		intel_de_rmw(dev_priv, psr_imr_reg(dev_priv, cpu_transcoder),
423 			     0, psr_irq_psr_error_bit_get(intel_dp));
424 
425 		queue_work(dev_priv->unordered_wq, &intel_dp->psr.work);
426 	}
427 }
428 
429 static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
430 {
431 	u8 alpm_caps = 0;
432 
433 	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,
434 			      &alpm_caps) != 1)
435 		return false;
436 	return alpm_caps & DP_ALPM_CAP;
437 }
438 
439 static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)
440 {
441 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
442 	u8 val = 8; /* assume the worst if we can't read the value */
443 
444 	if (drm_dp_dpcd_readb(&intel_dp->aux,
445 			      DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val) == 1)
446 		val &= DP_MAX_RESYNC_FRAME_COUNT_MASK;
447 	else
448 		drm_dbg_kms(&i915->drm,
449 			    "Unable to get sink synchronization latency, assuming 8 frames\n");
450 	return val;
451 }
452 
453 static void intel_dp_get_su_granularity(struct intel_dp *intel_dp)
454 {
455 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
456 	ssize_t r;
457 	u16 w;
458 	u8 y;
459 
460 	/* If sink don't have specific granularity requirements set legacy ones */
461 	if (!(intel_dp->psr_dpcd[1] & DP_PSR2_SU_GRANULARITY_REQUIRED)) {
462 		/* As PSR2 HW sends full lines, we do not care about x granularity */
463 		w = 4;
464 		y = 4;
465 		goto exit;
466 	}
467 
468 	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &w, 2);
469 	if (r != 2)
470 		drm_dbg_kms(&i915->drm,
471 			    "Unable to read DP_PSR2_SU_X_GRANULARITY\n");
472 	/*
473 	 * Spec says that if the value read is 0 the default granularity should
474 	 * be used instead.
475 	 */
476 	if (r != 2 || w == 0)
477 		w = 4;
478 
479 	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_Y_GRANULARITY, &y, 1);
480 	if (r != 1) {
481 		drm_dbg_kms(&i915->drm,
482 			    "Unable to read DP_PSR2_SU_Y_GRANULARITY\n");
483 		y = 4;
484 	}
485 	if (y == 0)
486 		y = 1;
487 
488 exit:
489 	intel_dp->psr.su_w_granularity = w;
490 	intel_dp->psr.su_y_granularity = y;
491 }
492 
493 static void _panel_replay_init_dpcd(struct intel_dp *intel_dp)
494 {
495 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
496 	u8 pr_dpcd = 0;
497 
498 	intel_dp->psr.sink_panel_replay_support = false;
499 	drm_dp_dpcd_readb(&intel_dp->aux, DP_PANEL_REPLAY_CAP, &pr_dpcd);
500 
501 	if (!(pr_dpcd & DP_PANEL_REPLAY_SUPPORT)) {
502 		drm_dbg_kms(&i915->drm,
503 			    "Panel replay is not supported by panel\n");
504 		return;
505 	}
506 
507 	drm_dbg_kms(&i915->drm,
508 		    "Panel replay is supported by panel\n");
509 	intel_dp->psr.sink_panel_replay_support = true;
510 }
511 
512 static void _psr_init_dpcd(struct intel_dp *intel_dp)
513 {
514 	struct drm_i915_private *i915 =
515 		to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
516 
517 	drm_dbg_kms(&i915->drm, "eDP panel supports PSR version %x\n",
518 		    intel_dp->psr_dpcd[0]);
519 
520 	if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) {
521 		drm_dbg_kms(&i915->drm,
522 			    "PSR support not currently available for this panel\n");
523 		return;
524 	}
525 
526 	if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) {
527 		drm_dbg_kms(&i915->drm,
528 			    "Panel lacks power state control, PSR cannot be enabled\n");
529 		return;
530 	}
531 
532 	intel_dp->psr.sink_support = true;
533 	intel_dp->psr.sink_sync_latency =
534 		intel_dp_get_sink_sync_latency(intel_dp);
535 
536 	if (DISPLAY_VER(i915) >= 9 &&
537 	    intel_dp->psr_dpcd[0] >= DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) {
538 		bool y_req = intel_dp->psr_dpcd[1] &
539 			     DP_PSR2_SU_Y_COORDINATE_REQUIRED;
540 		bool alpm = intel_dp_get_alpm_status(intel_dp);
541 
542 		/*
543 		 * All panels that supports PSR version 03h (PSR2 +
544 		 * Y-coordinate) can handle Y-coordinates in VSC but we are
545 		 * only sure that it is going to be used when required by the
546 		 * panel. This way panel is capable to do selective update
547 		 * without a aux frame sync.
548 		 *
549 		 * To support PSR version 02h and PSR version 03h without
550 		 * Y-coordinate requirement panels we would need to enable
551 		 * GTC first.
552 		 */
553 		intel_dp->psr.sink_psr2_support = y_req && alpm;
554 		drm_dbg_kms(&i915->drm, "PSR2 %ssupported\n",
555 			    intel_dp->psr.sink_psr2_support ? "" : "not ");
556 	}
557 }
558 
559 void intel_psr_init_dpcd(struct intel_dp *intel_dp)
560 {
561 	_panel_replay_init_dpcd(intel_dp);
562 
563 	drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
564 			 sizeof(intel_dp->psr_dpcd));
565 
566 	if (intel_dp->psr_dpcd[0])
567 		_psr_init_dpcd(intel_dp);
568 
569 	if (intel_dp->psr.sink_psr2_support)
570 		intel_dp_get_su_granularity(intel_dp);
571 }
572 
573 static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
574 {
575 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
576 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
577 	u32 aux_clock_divider, aux_ctl;
578 	/* write DP_SET_POWER=D0 */
579 	static const u8 aux_msg[] = {
580 		[0] = (DP_AUX_NATIVE_WRITE << 4) | ((DP_SET_POWER >> 16) & 0xf),
581 		[1] = (DP_SET_POWER >> 8) & 0xff,
582 		[2] = DP_SET_POWER & 0xff,
583 		[3] = 1 - 1,
584 		[4] = DP_SET_POWER_D0,
585 	};
586 	int i;
587 
588 	BUILD_BUG_ON(sizeof(aux_msg) > 20);
589 	for (i = 0; i < sizeof(aux_msg); i += 4)
590 		intel_de_write(dev_priv,
591 			       psr_aux_data_reg(dev_priv, cpu_transcoder, i >> 2),
592 			       intel_dp_aux_pack(&aux_msg[i], sizeof(aux_msg) - i));
593 
594 	aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
595 
596 	/* Start with bits set for DDI_AUX_CTL register */
597 	aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, sizeof(aux_msg),
598 					     aux_clock_divider);
599 
600 	/* Select only valid bits for SRD_AUX_CTL */
601 	aux_ctl &= EDP_PSR_AUX_CTL_TIME_OUT_MASK |
602 		EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK |
603 		EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK |
604 		EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK;
605 
606 	intel_de_write(dev_priv, psr_aux_ctl_reg(dev_priv, cpu_transcoder),
607 		       aux_ctl);
608 }
609 
610 static bool psr2_su_region_et_valid(struct intel_dp *intel_dp)
611 {
612 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
613 
614 	if (DISPLAY_VER(i915) >= 20 &&
615 	    intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_ET_SUPPORTED &&
616 	    !(intel_dp->psr.debug & I915_PSR_DEBUG_SU_REGION_ET_DISABLE))
617 		return true;
618 
619 	return false;
620 }
621 
622 static void intel_psr_enable_sink(struct intel_dp *intel_dp)
623 {
624 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
625 	u8 dpcd_val = DP_PSR_ENABLE;
626 
627 	if (intel_dp->psr.panel_replay_enabled)
628 		return;
629 
630 	if (intel_dp->psr.psr2_enabled) {
631 		/* Enable ALPM at sink for psr2 */
632 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG,
633 				   DP_ALPM_ENABLE |
634 				   DP_ALPM_LOCK_ERROR_IRQ_HPD_ENABLE);
635 
636 		dpcd_val |= DP_PSR_ENABLE_PSR2 | DP_PSR_IRQ_HPD_WITH_CRC_ERRORS;
637 		if (psr2_su_region_et_valid(intel_dp))
638 			dpcd_val |= DP_PSR_ENABLE_SU_REGION_ET;
639 	} else {
640 		if (intel_dp->psr.link_standby)
641 			dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
642 
643 		if (DISPLAY_VER(dev_priv) >= 8)
644 			dpcd_val |= DP_PSR_CRC_VERIFICATION;
645 	}
646 
647 	if (intel_dp->psr.req_psr2_sdp_prior_scanline)
648 		dpcd_val |= DP_PSR_SU_REGION_SCANLINE_CAPTURE;
649 
650 	if (intel_dp->psr.entry_setup_frames > 0)
651 		dpcd_val |= DP_PSR_FRAME_CAPTURE;
652 
653 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
654 
655 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
656 }
657 
658 static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
659 {
660 	struct intel_connector *connector = intel_dp->attached_connector;
661 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
662 	u32 val = 0;
663 
664 	if (DISPLAY_VER(dev_priv) >= 11)
665 		val |= EDP_PSR_TP4_TIME_0us;
666 
667 	if (dev_priv->display.params.psr_safest_params) {
668 		val |= EDP_PSR_TP1_TIME_2500us;
669 		val |= EDP_PSR_TP2_TP3_TIME_2500us;
670 		goto check_tp3_sel;
671 	}
672 
673 	if (connector->panel.vbt.psr.tp1_wakeup_time_us == 0)
674 		val |= EDP_PSR_TP1_TIME_0us;
675 	else if (connector->panel.vbt.psr.tp1_wakeup_time_us <= 100)
676 		val |= EDP_PSR_TP1_TIME_100us;
677 	else if (connector->panel.vbt.psr.tp1_wakeup_time_us <= 500)
678 		val |= EDP_PSR_TP1_TIME_500us;
679 	else
680 		val |= EDP_PSR_TP1_TIME_2500us;
681 
682 	if (connector->panel.vbt.psr.tp2_tp3_wakeup_time_us == 0)
683 		val |= EDP_PSR_TP2_TP3_TIME_0us;
684 	else if (connector->panel.vbt.psr.tp2_tp3_wakeup_time_us <= 100)
685 		val |= EDP_PSR_TP2_TP3_TIME_100us;
686 	else if (connector->panel.vbt.psr.tp2_tp3_wakeup_time_us <= 500)
687 		val |= EDP_PSR_TP2_TP3_TIME_500us;
688 	else
689 		val |= EDP_PSR_TP2_TP3_TIME_2500us;
690 
691 	/*
692 	 * WA 0479: hsw,bdw
693 	 * "Do not skip both TP1 and TP2/TP3"
694 	 */
695 	if (DISPLAY_VER(dev_priv) < 9 &&
696 	    connector->panel.vbt.psr.tp1_wakeup_time_us == 0 &&
697 	    connector->panel.vbt.psr.tp2_tp3_wakeup_time_us == 0)
698 		val |= EDP_PSR_TP2_TP3_TIME_100us;
699 
700 check_tp3_sel:
701 	if (intel_dp_source_supports_tps3(dev_priv) &&
702 	    drm_dp_tps3_supported(intel_dp->dpcd))
703 		val |= EDP_PSR_TP_TP1_TP3;
704 	else
705 		val |= EDP_PSR_TP_TP1_TP2;
706 
707 	return val;
708 }
709 
710 static u8 psr_compute_idle_frames(struct intel_dp *intel_dp)
711 {
712 	struct intel_connector *connector = intel_dp->attached_connector;
713 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
714 	int idle_frames;
715 
716 	/* Let's use 6 as the minimum to cover all known cases including the
717 	 * off-by-one issue that HW has in some cases.
718 	 */
719 	idle_frames = max(6, connector->panel.vbt.psr.idle_frames);
720 	idle_frames = max(idle_frames, intel_dp->psr.sink_sync_latency + 1);
721 
722 	if (drm_WARN_ON(&dev_priv->drm, idle_frames > 0xf))
723 		idle_frames = 0xf;
724 
725 	return idle_frames;
726 }
727 
728 static void hsw_activate_psr1(struct intel_dp *intel_dp)
729 {
730 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
731 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
732 	u32 max_sleep_time = 0x1f;
733 	u32 val = EDP_PSR_ENABLE;
734 
735 	val |= EDP_PSR_IDLE_FRAMES(psr_compute_idle_frames(intel_dp));
736 
737 	if (DISPLAY_VER(dev_priv) < 20)
738 		val |= EDP_PSR_MAX_SLEEP_TIME(max_sleep_time);
739 
740 	if (IS_HASWELL(dev_priv))
741 		val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
742 
743 	if (intel_dp->psr.link_standby)
744 		val |= EDP_PSR_LINK_STANDBY;
745 
746 	val |= intel_psr1_get_tp_time(intel_dp);
747 
748 	if (DISPLAY_VER(dev_priv) >= 8)
749 		val |= EDP_PSR_CRC_ENABLE;
750 
751 	if (DISPLAY_VER(dev_priv) >= 20)
752 		val |= LNL_EDP_PSR_ENTRY_SETUP_FRAMES(intel_dp->psr.entry_setup_frames);
753 
754 	intel_de_rmw(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder),
755 		     ~EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK, val);
756 }
757 
758 static u32 intel_psr2_get_tp_time(struct intel_dp *intel_dp)
759 {
760 	struct intel_connector *connector = intel_dp->attached_connector;
761 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
762 	u32 val = 0;
763 
764 	if (dev_priv->display.params.psr_safest_params)
765 		return EDP_PSR2_TP2_TIME_2500us;
766 
767 	if (connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us >= 0 &&
768 	    connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 50)
769 		val |= EDP_PSR2_TP2_TIME_50us;
770 	else if (connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 100)
771 		val |= EDP_PSR2_TP2_TIME_100us;
772 	else if (connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 500)
773 		val |= EDP_PSR2_TP2_TIME_500us;
774 	else
775 		val |= EDP_PSR2_TP2_TIME_2500us;
776 
777 	return val;
778 }
779 
780 static int psr2_block_count_lines(struct intel_dp *intel_dp)
781 {
782 	return intel_dp->psr.alpm_parameters.io_wake_lines < 9 &&
783 		intel_dp->psr.alpm_parameters.fast_wake_lines < 9 ? 8 : 12;
784 }
785 
786 static int psr2_block_count(struct intel_dp *intel_dp)
787 {
788 	return psr2_block_count_lines(intel_dp) / 4;
789 }
790 
791 static u8 frames_before_su_entry(struct intel_dp *intel_dp)
792 {
793 	u8 frames_before_su_entry;
794 
795 	frames_before_su_entry = max_t(u8,
796 				       intel_dp->psr.sink_sync_latency + 1,
797 				       2);
798 
799 	/* Entry setup frames must be at least 1 less than frames before SU entry */
800 	if (intel_dp->psr.entry_setup_frames >= frames_before_su_entry)
801 		frames_before_su_entry = intel_dp->psr.entry_setup_frames + 1;
802 
803 	return frames_before_su_entry;
804 }
805 
806 static void dg2_activate_panel_replay(struct intel_dp *intel_dp)
807 {
808 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
809 
810 	intel_de_rmw(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder),
811 		     0, ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME);
812 
813 	intel_de_rmw(dev_priv, TRANS_DP2_CTL(intel_dp->psr.transcoder), 0,
814 		     TRANS_DP2_PANEL_REPLAY_ENABLE);
815 }
816 
817 static void hsw_activate_psr2(struct intel_dp *intel_dp)
818 {
819 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
820 	struct intel_psr *psr = &intel_dp->psr;
821 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
822 	u32 val = EDP_PSR2_ENABLE;
823 	u32 psr_val = 0;
824 
825 	val |= EDP_PSR2_IDLE_FRAMES(psr_compute_idle_frames(intel_dp));
826 
827 	if (DISPLAY_VER(dev_priv) < 14 && !IS_ALDERLAKE_P(dev_priv))
828 		val |= EDP_SU_TRACK_ENABLE;
829 
830 	if (DISPLAY_VER(dev_priv) >= 10 && DISPLAY_VER(dev_priv) < 13)
831 		val |= EDP_Y_COORDINATE_ENABLE;
832 
833 	val |= EDP_PSR2_FRAME_BEFORE_SU(frames_before_su_entry(intel_dp));
834 
835 	val |= intel_psr2_get_tp_time(intel_dp);
836 
837 	if (DISPLAY_VER(dev_priv) >= 12) {
838 		if (psr2_block_count(intel_dp) > 2)
839 			val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_3;
840 		else
841 			val |= TGL_EDP_PSR2_BLOCK_COUNT_NUM_2;
842 	}
843 
844 	/* Wa_22012278275:adl-p */
845 	if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_E0)) {
846 		static const u8 map[] = {
847 			2, /* 5 lines */
848 			1, /* 6 lines */
849 			0, /* 7 lines */
850 			3, /* 8 lines */
851 			6, /* 9 lines */
852 			5, /* 10 lines */
853 			4, /* 11 lines */
854 			7, /* 12 lines */
855 		};
856 		/*
857 		 * Still using the default IO_BUFFER_WAKE and FAST_WAKE, see
858 		 * comments bellow for more information
859 		 */
860 		int tmp;
861 
862 		tmp = map[psr->alpm_parameters.io_wake_lines -
863 			  TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES];
864 		val |= TGL_EDP_PSR2_IO_BUFFER_WAKE(tmp + TGL_EDP_PSR2_IO_BUFFER_WAKE_MIN_LINES);
865 
866 		tmp = map[psr->alpm_parameters.fast_wake_lines - TGL_EDP_PSR2_FAST_WAKE_MIN_LINES];
867 		val |= TGL_EDP_PSR2_FAST_WAKE(tmp + TGL_EDP_PSR2_FAST_WAKE_MIN_LINES);
868 	} else if (DISPLAY_VER(dev_priv) >= 12) {
869 		val |= TGL_EDP_PSR2_IO_BUFFER_WAKE(psr->alpm_parameters.io_wake_lines);
870 		val |= TGL_EDP_PSR2_FAST_WAKE(psr->alpm_parameters.fast_wake_lines);
871 	} else if (DISPLAY_VER(dev_priv) >= 9) {
872 		val |= EDP_PSR2_IO_BUFFER_WAKE(psr->alpm_parameters.io_wake_lines);
873 		val |= EDP_PSR2_FAST_WAKE(psr->alpm_parameters.fast_wake_lines);
874 	}
875 
876 	if (intel_dp->psr.req_psr2_sdp_prior_scanline)
877 		val |= EDP_PSR2_SU_SDP_SCANLINE;
878 
879 	if (DISPLAY_VER(dev_priv) >= 20)
880 		psr_val |= LNL_EDP_PSR_ENTRY_SETUP_FRAMES(intel_dp->psr.entry_setup_frames);
881 
882 	if (intel_dp->psr.psr2_sel_fetch_enabled) {
883 		u32 tmp;
884 
885 		tmp = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder));
886 		drm_WARN_ON(&dev_priv->drm, !(tmp & PSR2_MAN_TRK_CTL_ENABLE));
887 	} else if (HAS_PSR2_SEL_FETCH(dev_priv)) {
888 		intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder), 0);
889 	}
890 
891 	if (psr2_su_region_et_valid(intel_dp))
892 		val |= LNL_EDP_PSR2_SU_REGION_ET_ENABLE;
893 
894 	/*
895 	 * PSR2 HW is incorrectly using EDP_PSR_TP1_TP3_SEL and BSpec is
896 	 * recommending keep this bit unset while PSR2 is enabled.
897 	 */
898 	intel_de_write(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder), psr_val);
899 
900 	intel_de_write(dev_priv, EDP_PSR2_CTL(cpu_transcoder), val);
901 }
902 
903 static bool
904 transcoder_has_psr2(struct drm_i915_private *dev_priv, enum transcoder cpu_transcoder)
905 {
906 	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
907 		return cpu_transcoder == TRANSCODER_A || cpu_transcoder == TRANSCODER_B;
908 	else if (DISPLAY_VER(dev_priv) >= 12)
909 		return cpu_transcoder == TRANSCODER_A;
910 	else if (DISPLAY_VER(dev_priv) >= 9)
911 		return cpu_transcoder == TRANSCODER_EDP;
912 	else
913 		return false;
914 }
915 
916 static u32 intel_get_frame_time_us(const struct intel_crtc_state *crtc_state)
917 {
918 	if (!crtc_state->hw.active)
919 		return 0;
920 
921 	return DIV_ROUND_UP(1000 * 1000,
922 			    drm_mode_vrefresh(&crtc_state->hw.adjusted_mode));
923 }
924 
925 static void psr2_program_idle_frames(struct intel_dp *intel_dp,
926 				     u32 idle_frames)
927 {
928 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
929 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
930 
931 	intel_de_rmw(dev_priv, EDP_PSR2_CTL(cpu_transcoder),
932 		     EDP_PSR2_IDLE_FRAMES_MASK,
933 		     EDP_PSR2_IDLE_FRAMES(idle_frames));
934 }
935 
936 static void tgl_psr2_enable_dc3co(struct intel_dp *intel_dp)
937 {
938 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
939 
940 	psr2_program_idle_frames(intel_dp, 0);
941 	intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_DC3CO);
942 }
943 
944 static void tgl_psr2_disable_dc3co(struct intel_dp *intel_dp)
945 {
946 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
947 
948 	intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
949 	psr2_program_idle_frames(intel_dp, psr_compute_idle_frames(intel_dp));
950 }
951 
952 static void tgl_dc3co_disable_work(struct work_struct *work)
953 {
954 	struct intel_dp *intel_dp =
955 		container_of(work, typeof(*intel_dp), psr.dc3co_work.work);
956 
957 	mutex_lock(&intel_dp->psr.lock);
958 	/* If delayed work is pending, it is not idle */
959 	if (delayed_work_pending(&intel_dp->psr.dc3co_work))
960 		goto unlock;
961 
962 	tgl_psr2_disable_dc3co(intel_dp);
963 unlock:
964 	mutex_unlock(&intel_dp->psr.lock);
965 }
966 
967 static void tgl_disallow_dc3co_on_psr2_exit(struct intel_dp *intel_dp)
968 {
969 	if (!intel_dp->psr.dc3co_exitline)
970 		return;
971 
972 	cancel_delayed_work(&intel_dp->psr.dc3co_work);
973 	/* Before PSR2 exit disallow dc3co*/
974 	tgl_psr2_disable_dc3co(intel_dp);
975 }
976 
977 static bool
978 dc3co_is_pipe_port_compatible(struct intel_dp *intel_dp,
979 			      struct intel_crtc_state *crtc_state)
980 {
981 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
982 	enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
983 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
984 	enum port port = dig_port->base.port;
985 
986 	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
987 		return pipe <= PIPE_B && port <= PORT_B;
988 	else
989 		return pipe == PIPE_A && port == PORT_A;
990 }
991 
992 static void
993 tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
994 				  struct intel_crtc_state *crtc_state)
995 {
996 	const u32 crtc_vdisplay = crtc_state->uapi.adjusted_mode.crtc_vdisplay;
997 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
998 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
999 	u32 exit_scanlines;
1000 
1001 	/*
1002 	 * FIXME: Due to the changed sequence of activating/deactivating DC3CO,
1003 	 * disable DC3CO until the changed dc3co activating/deactivating sequence
1004 	 * is applied. B.Specs:49196
1005 	 */
1006 	return;
1007 
1008 	/*
1009 	 * DMC's DC3CO exit mechanism has an issue with Selective Fecth
1010 	 * TODO: when the issue is addressed, this restriction should be removed.
1011 	 */
1012 	if (crtc_state->enable_psr2_sel_fetch)
1013 		return;
1014 
1015 	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC3CO))
1016 		return;
1017 
1018 	if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state))
1019 		return;
1020 
1021 	/* Wa_16011303918:adl-p */
1022 	if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
1023 		return;
1024 
1025 	/*
1026 	 * DC3CO Exit time 200us B.Spec 49196
1027 	 * PSR2 transcoder Early Exit scanlines = ROUNDUP(200 / line time) + 1
1028 	 */
1029 	exit_scanlines =
1030 		intel_usecs_to_scanlines(&crtc_state->uapi.adjusted_mode, 200) + 1;
1031 
1032 	if (drm_WARN_ON(&dev_priv->drm, exit_scanlines > crtc_vdisplay))
1033 		return;
1034 
1035 	crtc_state->dc3co_exitline = crtc_vdisplay - exit_scanlines;
1036 }
1037 
1038 static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,
1039 					      struct intel_crtc_state *crtc_state)
1040 {
1041 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1042 
1043 	if (!dev_priv->display.params.enable_psr2_sel_fetch &&
1044 	    intel_dp->psr.debug != I915_PSR_DEBUG_ENABLE_SEL_FETCH) {
1045 		drm_dbg_kms(&dev_priv->drm,
1046 			    "PSR2 sel fetch not enabled, disabled by parameter\n");
1047 		return false;
1048 	}
1049 
1050 	if (crtc_state->uapi.async_flip) {
1051 		drm_dbg_kms(&dev_priv->drm,
1052 			    "PSR2 sel fetch not enabled, async flip enabled\n");
1053 		return false;
1054 	}
1055 
1056 	if (psr2_su_region_et_valid(intel_dp))
1057 		crtc_state->enable_psr2_su_region_et = true;
1058 
1059 	return crtc_state->enable_psr2_sel_fetch = true;
1060 }
1061 
1062 static bool psr2_granularity_check(struct intel_dp *intel_dp,
1063 				   struct intel_crtc_state *crtc_state)
1064 {
1065 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1066 	const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config;
1067 	const int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
1068 	const int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
1069 	u16 y_granularity = 0;
1070 
1071 	/* PSR2 HW only send full lines so we only need to validate the width */
1072 	if (crtc_hdisplay % intel_dp->psr.su_w_granularity)
1073 		return false;
1074 
1075 	if (crtc_vdisplay % intel_dp->psr.su_y_granularity)
1076 		return false;
1077 
1078 	/* HW tracking is only aligned to 4 lines */
1079 	if (!crtc_state->enable_psr2_sel_fetch)
1080 		return intel_dp->psr.su_y_granularity == 4;
1081 
1082 	/*
1083 	 * adl_p and mtl platforms have 1 line granularity.
1084 	 * For other platforms with SW tracking we can adjust the y coordinates
1085 	 * to match sink requirement if multiple of 4.
1086 	 */
1087 	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
1088 		y_granularity = intel_dp->psr.su_y_granularity;
1089 	else if (intel_dp->psr.su_y_granularity <= 2)
1090 		y_granularity = 4;
1091 	else if ((intel_dp->psr.su_y_granularity % 4) == 0)
1092 		y_granularity = intel_dp->psr.su_y_granularity;
1093 
1094 	if (y_granularity == 0 || crtc_vdisplay % y_granularity)
1095 		return false;
1096 
1097 	if (crtc_state->dsc.compression_enable &&
1098 	    vdsc_cfg->slice_height % y_granularity)
1099 		return false;
1100 
1101 	crtc_state->su_y_granularity = y_granularity;
1102 	return true;
1103 }
1104 
1105 static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_dp,
1106 							struct intel_crtc_state *crtc_state)
1107 {
1108 	const struct drm_display_mode *adjusted_mode = &crtc_state->uapi.adjusted_mode;
1109 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1110 	u32 hblank_total, hblank_ns, req_ns;
1111 
1112 	hblank_total = adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start;
1113 	hblank_ns = div_u64(1000000ULL * hblank_total, adjusted_mode->crtc_clock);
1114 
1115 	/* From spec: ((60 / number of lanes) + 11) * 1000 / symbol clock frequency MHz */
1116 	req_ns = ((60 / crtc_state->lane_count) + 11) * 1000 / (crtc_state->port_clock / 1000);
1117 
1118 	if ((hblank_ns - req_ns) > 100)
1119 		return true;
1120 
1121 	/* Not supported <13 / Wa_22012279113:adl-p */
1122 	if (DISPLAY_VER(dev_priv) < 14 || intel_dp->edp_dpcd[0] < DP_EDP_14b)
1123 		return false;
1124 
1125 	crtc_state->req_psr2_sdp_prior_scanline = true;
1126 	return true;
1127 }
1128 
1129 static bool _lnl_compute_alpm_params(struct intel_dp *intel_dp,
1130 				     struct intel_crtc_state *crtc_state)
1131 {
1132 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
1133 	int check_entry_lines;
1134 
1135 	if (DISPLAY_VER(i915) < 20)
1136 		return true;
1137 
1138 	/* ALPM Entry Check = 2 + CEILING( 5us /tline ) */
1139 	check_entry_lines = 2 +
1140 		intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, 5);
1141 
1142 	if (check_entry_lines > 15)
1143 		return false;
1144 
1145 	if (i915->display.params.psr_safest_params)
1146 		check_entry_lines = 15;
1147 
1148 	intel_dp->psr.alpm_parameters.check_entry_lines = check_entry_lines;
1149 
1150 	return true;
1151 }
1152 
1153 static bool _compute_alpm_params(struct intel_dp *intel_dp,
1154 				 struct intel_crtc_state *crtc_state)
1155 {
1156 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
1157 	int io_wake_lines, io_wake_time, fast_wake_lines, fast_wake_time;
1158 	u8 max_wake_lines;
1159 
1160 	if (DISPLAY_VER(i915) >= 12) {
1161 		io_wake_time = 42;
1162 		/*
1163 		 * According to Bspec it's 42us, but based on testing
1164 		 * it is not enough -> use 45 us.
1165 		 */
1166 		fast_wake_time = 45;
1167 
1168 		/* TODO: Check how we can use ALPM_CTL fast wake extended field */
1169 		max_wake_lines = 12;
1170 	} else {
1171 		io_wake_time = 50;
1172 		fast_wake_time = 32;
1173 		max_wake_lines = 8;
1174 	}
1175 
1176 	io_wake_lines = intel_usecs_to_scanlines(
1177 		&crtc_state->hw.adjusted_mode, io_wake_time);
1178 	fast_wake_lines = intel_usecs_to_scanlines(
1179 		&crtc_state->hw.adjusted_mode, fast_wake_time);
1180 
1181 	if (io_wake_lines > max_wake_lines ||
1182 	    fast_wake_lines > max_wake_lines)
1183 		return false;
1184 
1185 	if (!_lnl_compute_alpm_params(intel_dp, crtc_state))
1186 		return false;
1187 
1188 	if (i915->display.params.psr_safest_params)
1189 		io_wake_lines = fast_wake_lines = max_wake_lines;
1190 
1191 	/* According to Bspec lower limit should be set as 7 lines. */
1192 	intel_dp->psr.alpm_parameters.io_wake_lines = max(io_wake_lines, 7);
1193 	intel_dp->psr.alpm_parameters.fast_wake_lines = max(fast_wake_lines, 7);
1194 
1195 	return true;
1196 }
1197 
1198 static int intel_psr_entry_setup_frames(struct intel_dp *intel_dp,
1199 					const struct drm_display_mode *adjusted_mode)
1200 {
1201 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
1202 	int psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);
1203 	int entry_setup_frames = 0;
1204 
1205 	if (psr_setup_time < 0) {
1206 		drm_dbg_kms(&i915->drm,
1207 			    "PSR condition failed: Invalid PSR setup time (0x%02x)\n",
1208 			    intel_dp->psr_dpcd[1]);
1209 		return -ETIME;
1210 	}
1211 
1212 	if (intel_usecs_to_scanlines(adjusted_mode, psr_setup_time) >
1213 	    adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay - 1) {
1214 		if (DISPLAY_VER(i915) >= 20) {
1215 			/* setup entry frames can be up to 3 frames */
1216 			entry_setup_frames = 1;
1217 			drm_dbg_kms(&i915->drm,
1218 				    "PSR setup entry frames %d\n",
1219 				    entry_setup_frames);
1220 		} else {
1221 			drm_dbg_kms(&i915->drm,
1222 				    "PSR condition failed: PSR setup time (%d us) too long\n",
1223 				    psr_setup_time);
1224 			return -ETIME;
1225 		}
1226 	}
1227 
1228 	return entry_setup_frames;
1229 }
1230 
1231 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
1232 				    struct intel_crtc_state *crtc_state)
1233 {
1234 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1235 	int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
1236 	int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
1237 	int psr_max_h = 0, psr_max_v = 0, max_bpp = 0;
1238 
1239 	if (!intel_dp->psr.sink_psr2_support)
1240 		return false;
1241 
1242 	/* JSL and EHL only supports eDP 1.3 */
1243 	if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
1244 		drm_dbg_kms(&dev_priv->drm, "PSR2 not supported by phy\n");
1245 		return false;
1246 	}
1247 
1248 	/* Wa_16011181250 */
1249 	if (IS_ROCKETLAKE(dev_priv) || IS_ALDERLAKE_S(dev_priv) ||
1250 	    IS_DG2(dev_priv)) {
1251 		drm_dbg_kms(&dev_priv->drm, "PSR2 is defeatured for this platform\n");
1252 		return false;
1253 	}
1254 
1255 	if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
1256 		drm_dbg_kms(&dev_priv->drm, "PSR2 not completely functional in this stepping\n");
1257 		return false;
1258 	}
1259 
1260 	if (!transcoder_has_psr2(dev_priv, crtc_state->cpu_transcoder)) {
1261 		drm_dbg_kms(&dev_priv->drm,
1262 			    "PSR2 not supported in transcoder %s\n",
1263 			    transcoder_name(crtc_state->cpu_transcoder));
1264 		return false;
1265 	}
1266 
1267 	if (!psr2_global_enabled(intel_dp)) {
1268 		drm_dbg_kms(&dev_priv->drm, "PSR2 disabled by flag\n");
1269 		return false;
1270 	}
1271 
1272 	/*
1273 	 * DSC and PSR2 cannot be enabled simultaneously. If a requested
1274 	 * resolution requires DSC to be enabled, priority is given to DSC
1275 	 * over PSR2.
1276 	 */
1277 	if (crtc_state->dsc.compression_enable &&
1278 	    (DISPLAY_VER(dev_priv) < 14 && !IS_ALDERLAKE_P(dev_priv))) {
1279 		drm_dbg_kms(&dev_priv->drm,
1280 			    "PSR2 cannot be enabled since DSC is enabled\n");
1281 		return false;
1282 	}
1283 
1284 	if (crtc_state->crc_enabled) {
1285 		drm_dbg_kms(&dev_priv->drm,
1286 			    "PSR2 not enabled because it would inhibit pipe CRC calculation\n");
1287 		return false;
1288 	}
1289 
1290 	if (DISPLAY_VER(dev_priv) >= 12) {
1291 		psr_max_h = 5120;
1292 		psr_max_v = 3200;
1293 		max_bpp = 30;
1294 	} else if (DISPLAY_VER(dev_priv) >= 10) {
1295 		psr_max_h = 4096;
1296 		psr_max_v = 2304;
1297 		max_bpp = 24;
1298 	} else if (DISPLAY_VER(dev_priv) == 9) {
1299 		psr_max_h = 3640;
1300 		psr_max_v = 2304;
1301 		max_bpp = 24;
1302 	}
1303 
1304 	if (crtc_state->pipe_bpp > max_bpp) {
1305 		drm_dbg_kms(&dev_priv->drm,
1306 			    "PSR2 not enabled, pipe bpp %d > max supported %d\n",
1307 			    crtc_state->pipe_bpp, max_bpp);
1308 		return false;
1309 	}
1310 
1311 	/* Wa_16011303918:adl-p */
1312 	if (crtc_state->vrr.enable &&
1313 	    IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
1314 		drm_dbg_kms(&dev_priv->drm,
1315 			    "PSR2 not enabled, not compatible with HW stepping + VRR\n");
1316 		return false;
1317 	}
1318 
1319 	if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
1320 		drm_dbg_kms(&dev_priv->drm,
1321 			    "PSR2 not enabled, PSR2 SDP indication do not fit in hblank\n");
1322 		return false;
1323 	}
1324 
1325 	if (!_compute_alpm_params(intel_dp, crtc_state)) {
1326 		drm_dbg_kms(&dev_priv->drm,
1327 			    "PSR2 not enabled, Unable to use long enough wake times\n");
1328 		return false;
1329 	}
1330 
1331 	/* Vblank >= PSR2_CTL Block Count Number maximum line count */
1332 	if (crtc_state->hw.adjusted_mode.crtc_vblank_end -
1333 	    crtc_state->hw.adjusted_mode.crtc_vblank_start <
1334 	    psr2_block_count_lines(intel_dp)) {
1335 		drm_dbg_kms(&dev_priv->drm,
1336 			    "PSR2 not enabled, too short vblank time\n");
1337 		return false;
1338 	}
1339 
1340 	if (HAS_PSR2_SEL_FETCH(dev_priv)) {
1341 		if (!intel_psr2_sel_fetch_config_valid(intel_dp, crtc_state) &&
1342 		    !HAS_PSR_HW_TRACKING(dev_priv)) {
1343 			drm_dbg_kms(&dev_priv->drm,
1344 				    "PSR2 not enabled, selective fetch not valid and no HW tracking available\n");
1345 			return false;
1346 		}
1347 	}
1348 
1349 	if (!psr2_granularity_check(intel_dp, crtc_state)) {
1350 		drm_dbg_kms(&dev_priv->drm, "PSR2 not enabled, SU granularity not compatible\n");
1351 		goto unsupported;
1352 	}
1353 
1354 	if (!crtc_state->enable_psr2_sel_fetch &&
1355 	    (crtc_hdisplay > psr_max_h || crtc_vdisplay > psr_max_v)) {
1356 		drm_dbg_kms(&dev_priv->drm,
1357 			    "PSR2 not enabled, resolution %dx%d > max supported %dx%d\n",
1358 			    crtc_hdisplay, crtc_vdisplay,
1359 			    psr_max_h, psr_max_v);
1360 		goto unsupported;
1361 	}
1362 
1363 	tgl_dc3co_exitline_compute_config(intel_dp, crtc_state);
1364 	return true;
1365 
1366 unsupported:
1367 	crtc_state->enable_psr2_sel_fetch = false;
1368 	return false;
1369 }
1370 
1371 static bool _psr_compute_config(struct intel_dp *intel_dp,
1372 				struct intel_crtc_state *crtc_state)
1373 {
1374 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1375 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
1376 	int entry_setup_frames;
1377 
1378 	/*
1379 	 * Current PSR panels don't work reliably with VRR enabled
1380 	 * So if VRR is enabled, do not enable PSR.
1381 	 */
1382 	if (crtc_state->vrr.enable)
1383 		return false;
1384 
1385 	if (!CAN_PSR(intel_dp))
1386 		return false;
1387 
1388 	entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, adjusted_mode);
1389 
1390 	if (entry_setup_frames >= 0) {
1391 		intel_dp->psr.entry_setup_frames = entry_setup_frames;
1392 	} else {
1393 		drm_dbg_kms(&dev_priv->drm,
1394 			    "PSR condition failed: PSR setup timing not met\n");
1395 		return false;
1396 	}
1397 
1398 	return true;
1399 }
1400 
1401 void intel_psr_compute_config(struct intel_dp *intel_dp,
1402 			      struct intel_crtc_state *crtc_state,
1403 			      struct drm_connector_state *conn_state)
1404 {
1405 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1406 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
1407 
1408 	if (!psr_global_enabled(intel_dp)) {
1409 		drm_dbg_kms(&dev_priv->drm, "PSR disabled by flag\n");
1410 		return;
1411 	}
1412 
1413 	if (intel_dp->psr.sink_not_reliable) {
1414 		drm_dbg_kms(&dev_priv->drm,
1415 			    "PSR sink implementation is not reliable\n");
1416 		return;
1417 	}
1418 
1419 	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
1420 		drm_dbg_kms(&dev_priv->drm,
1421 			    "PSR condition failed: Interlaced mode enabled\n");
1422 		return;
1423 	}
1424 
1425 	/*
1426 	 * FIXME figure out what is wrong with PSR+bigjoiner and
1427 	 * fix it. Presumably something related to the fact that
1428 	 * PSR is a transcoder level feature.
1429 	 */
1430 	if (crtc_state->bigjoiner_pipes) {
1431 		drm_dbg_kms(&dev_priv->drm,
1432 			    "PSR disabled due to bigjoiner\n");
1433 		return;
1434 	}
1435 
1436 	if (CAN_PANEL_REPLAY(intel_dp))
1437 		crtc_state->has_panel_replay = true;
1438 	else
1439 		crtc_state->has_psr = _psr_compute_config(intel_dp, crtc_state);
1440 
1441 	if (!(crtc_state->has_panel_replay || crtc_state->has_psr))
1442 		return;
1443 
1444 	crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);
1445 }
1446 
1447 void intel_psr_get_config(struct intel_encoder *encoder,
1448 			  struct intel_crtc_state *pipe_config)
1449 {
1450 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1451 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
1452 	enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
1453 	struct intel_dp *intel_dp;
1454 	u32 val;
1455 
1456 	if (!dig_port)
1457 		return;
1458 
1459 	intel_dp = &dig_port->dp;
1460 	if (!(CAN_PSR(intel_dp) || CAN_PANEL_REPLAY(intel_dp)))
1461 		return;
1462 
1463 	mutex_lock(&intel_dp->psr.lock);
1464 	if (!intel_dp->psr.enabled)
1465 		goto unlock;
1466 
1467 	if (intel_dp->psr.panel_replay_enabled) {
1468 		pipe_config->has_panel_replay = true;
1469 	} else {
1470 		/*
1471 		 * Not possible to read EDP_PSR/PSR2_CTL registers as it is
1472 		 * enabled/disabled because of frontbuffer tracking and others.
1473 		 */
1474 		pipe_config->has_psr = true;
1475 	}
1476 
1477 	pipe_config->has_psr2 = intel_dp->psr.psr2_enabled;
1478 	pipe_config->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC);
1479 
1480 	if (!intel_dp->psr.psr2_enabled)
1481 		goto unlock;
1482 
1483 	if (HAS_PSR2_SEL_FETCH(dev_priv)) {
1484 		val = intel_de_read(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder));
1485 		if (val & PSR2_MAN_TRK_CTL_ENABLE)
1486 			pipe_config->enable_psr2_sel_fetch = true;
1487 	}
1488 
1489 	if (DISPLAY_VER(dev_priv) >= 12) {
1490 		val = intel_de_read(dev_priv, TRANS_EXITLINE(cpu_transcoder));
1491 		pipe_config->dc3co_exitline = REG_FIELD_GET(EXITLINE_MASK, val);
1492 	}
1493 unlock:
1494 	mutex_unlock(&intel_dp->psr.lock);
1495 }
1496 
1497 static void intel_psr_activate(struct intel_dp *intel_dp)
1498 {
1499 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1500 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1501 
1502 	drm_WARN_ON(&dev_priv->drm,
1503 		    transcoder_has_psr2(dev_priv, cpu_transcoder) &&
1504 		    intel_de_read(dev_priv, EDP_PSR2_CTL(cpu_transcoder)) & EDP_PSR2_ENABLE);
1505 
1506 	drm_WARN_ON(&dev_priv->drm,
1507 		    intel_de_read(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder)) & EDP_PSR_ENABLE);
1508 
1509 	drm_WARN_ON(&dev_priv->drm, intel_dp->psr.active);
1510 
1511 	lockdep_assert_held(&intel_dp->psr.lock);
1512 
1513 	/* psr1, psr2 and panel-replay are mutually exclusive.*/
1514 	if (intel_dp->psr.panel_replay_enabled)
1515 		dg2_activate_panel_replay(intel_dp);
1516 	else if (intel_dp->psr.psr2_enabled)
1517 		hsw_activate_psr2(intel_dp);
1518 	else
1519 		hsw_activate_psr1(intel_dp);
1520 
1521 	intel_dp->psr.active = true;
1522 }
1523 
1524 static u32 wa_16013835468_bit_get(struct intel_dp *intel_dp)
1525 {
1526 	switch (intel_dp->psr.pipe) {
1527 	case PIPE_A:
1528 		return LATENCY_REPORTING_REMOVED_PIPE_A;
1529 	case PIPE_B:
1530 		return LATENCY_REPORTING_REMOVED_PIPE_B;
1531 	case PIPE_C:
1532 		return LATENCY_REPORTING_REMOVED_PIPE_C;
1533 	case PIPE_D:
1534 		return LATENCY_REPORTING_REMOVED_PIPE_D;
1535 	default:
1536 		MISSING_CASE(intel_dp->psr.pipe);
1537 		return 0;
1538 	}
1539 }
1540 
1541 /*
1542  * Wa_16013835468
1543  * Wa_14015648006
1544  */
1545 static void wm_optimization_wa(struct intel_dp *intel_dp,
1546 			       const struct intel_crtc_state *crtc_state)
1547 {
1548 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1549 	bool set_wa_bit = false;
1550 
1551 	/* Wa_14015648006 */
1552 	if (IS_DISPLAY_VER(dev_priv, 11, 14))
1553 		set_wa_bit |= crtc_state->wm_level_disabled;
1554 
1555 	/* Wa_16013835468 */
1556 	if (DISPLAY_VER(dev_priv) == 12)
1557 		set_wa_bit |= crtc_state->hw.adjusted_mode.crtc_vblank_start !=
1558 			crtc_state->hw.adjusted_mode.crtc_vdisplay;
1559 
1560 	if (set_wa_bit)
1561 		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
1562 			     0, wa_16013835468_bit_get(intel_dp));
1563 	else
1564 		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
1565 			     wa_16013835468_bit_get(intel_dp), 0);
1566 }
1567 
1568 static void lnl_alpm_configure(struct intel_dp *intel_dp)
1569 {
1570 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1571 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1572 	struct intel_psr *psr = &intel_dp->psr;
1573 
1574 	if (DISPLAY_VER(dev_priv) < 20)
1575 		return;
1576 
1577 	intel_de_write(dev_priv, ALPM_CTL(cpu_transcoder),
1578 		       ALPM_CTL_EXTENDED_FAST_WAKE_ENABLE |
1579 		       ALPM_CTL_ALPM_ENTRY_CHECK(psr->alpm_parameters.check_entry_lines) |
1580 		       ALPM_CTL_EXTENDED_FAST_WAKE_TIME(psr->alpm_parameters.fast_wake_lines));
1581 }
1582 
1583 static void intel_psr_enable_source(struct intel_dp *intel_dp,
1584 				    const struct intel_crtc_state *crtc_state)
1585 {
1586 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1587 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1588 	u32 mask;
1589 
1590 	/*
1591 	 * Only HSW and BDW have PSR AUX registers that need to be setup.
1592 	 * SKL+ use hardcoded values PSR AUX transactions
1593 	 */
1594 	if (DISPLAY_VER(dev_priv) < 9)
1595 		hsw_psr_setup_aux(intel_dp);
1596 
1597 	/*
1598 	 * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD also
1599 	 * mask LPSP to avoid dependency on other drivers that might block
1600 	 * runtime_pm besides preventing  other hw tracking issues now we
1601 	 * can rely on frontbuffer tracking.
1602 	 */
1603 	mask = EDP_PSR_DEBUG_MASK_MEMUP |
1604 	       EDP_PSR_DEBUG_MASK_HPD;
1605 
1606 	/*
1607 	 * For some unknown reason on HSW non-ULT (or at least on
1608 	 * Dell Latitude E6540) external displays start to flicker
1609 	 * when PSR is enabled on the eDP. SR/PC6 residency is much
1610 	 * higher than should be possible with an external display.
1611 	 * As a workaround leave LPSP unmasked to prevent PSR entry
1612 	 * when external displays are active.
1613 	 */
1614 	if (DISPLAY_VER(dev_priv) >= 8 || IS_HASWELL_ULT(dev_priv))
1615 		mask |= EDP_PSR_DEBUG_MASK_LPSP;
1616 
1617 	if (DISPLAY_VER(dev_priv) < 20)
1618 		mask |= EDP_PSR_DEBUG_MASK_MAX_SLEEP;
1619 
1620 	/*
1621 	 * No separate pipe reg write mask on hsw/bdw, so have to unmask all
1622 	 * registers in order to keep the CURSURFLIVE tricks working :(
1623 	 */
1624 	if (IS_DISPLAY_VER(dev_priv, 9, 10))
1625 		mask |= EDP_PSR_DEBUG_MASK_DISP_REG_WRITE;
1626 
1627 	/* allow PSR with sprite enabled */
1628 	if (IS_HASWELL(dev_priv))
1629 		mask |= EDP_PSR_DEBUG_MASK_SPRITE_ENABLE;
1630 
1631 	intel_de_write(dev_priv, psr_debug_reg(dev_priv, cpu_transcoder), mask);
1632 
1633 	psr_irq_control(intel_dp);
1634 
1635 	/*
1636 	 * TODO: if future platforms supports DC3CO in more than one
1637 	 * transcoder, EXITLINE will need to be unset when disabling PSR
1638 	 */
1639 	if (intel_dp->psr.dc3co_exitline)
1640 		intel_de_rmw(dev_priv, TRANS_EXITLINE(cpu_transcoder), EXITLINE_MASK,
1641 			     intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT | EXITLINE_ENABLE);
1642 
1643 	if (HAS_PSR_HW_TRACKING(dev_priv) && HAS_PSR2_SEL_FETCH(dev_priv))
1644 		intel_de_rmw(dev_priv, CHICKEN_PAR1_1, IGNORE_PSR2_HW_TRACKING,
1645 			     intel_dp->psr.psr2_sel_fetch_enabled ?
1646 			     IGNORE_PSR2_HW_TRACKING : 0);
1647 
1648 	lnl_alpm_configure(intel_dp);
1649 
1650 	/*
1651 	 * Wa_16013835468
1652 	 * Wa_14015648006
1653 	 */
1654 	wm_optimization_wa(intel_dp, crtc_state);
1655 
1656 	if (intel_dp->psr.psr2_enabled) {
1657 		if (DISPLAY_VER(dev_priv) == 9)
1658 			intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0,
1659 				     PSR2_VSC_ENABLE_PROG_HEADER |
1660 				     PSR2_ADD_VERTICAL_LINE_COUNT);
1661 
1662 		/*
1663 		 * Wa_16014451276:adlp,mtl[a0,b0]
1664 		 * All supported adlp panels have 1-based X granularity, this may
1665 		 * cause issues if non-supported panels are used.
1666 		 */
1667 		if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0) ||
1668 		    IS_ALDERLAKE_P(dev_priv))
1669 			intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, cpu_transcoder),
1670 				     0, ADLP_1_BASED_X_GRANULARITY);
1671 
1672 		/* Wa_16012604467:adlp,mtl[a0,b0] */
1673 		if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0))
1674 			intel_de_rmw(dev_priv,
1675 				     MTL_CLKGATE_DIS_TRANS(cpu_transcoder), 0,
1676 				     MTL_CLKGATE_DIS_TRANS_DMASC_GATING_DIS);
1677 		else if (IS_ALDERLAKE_P(dev_priv))
1678 			intel_de_rmw(dev_priv, CLKGATE_DIS_MISC, 0,
1679 				     CLKGATE_DIS_MISC_DMASC_GATING_DIS);
1680 	}
1681 }
1682 
1683 static bool psr_interrupt_error_check(struct intel_dp *intel_dp)
1684 {
1685 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1686 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1687 	u32 val;
1688 
1689 	/*
1690 	 * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR
1691 	 * will still keep the error set even after the reset done in the
1692 	 * irq_preinstall and irq_uninstall hooks.
1693 	 * And enabling in this situation cause the screen to freeze in the
1694 	 * first time that PSR HW tries to activate so lets keep PSR disabled
1695 	 * to avoid any rendering problems.
1696 	 */
1697 	val = intel_de_read(dev_priv, psr_iir_reg(dev_priv, cpu_transcoder));
1698 	val &= psr_irq_psr_error_bit_get(intel_dp);
1699 	if (val) {
1700 		intel_dp->psr.sink_not_reliable = true;
1701 		drm_dbg_kms(&dev_priv->drm,
1702 			    "PSR interruption error set, not enabling PSR\n");
1703 		return false;
1704 	}
1705 
1706 	return true;
1707 }
1708 
1709 static void intel_psr_enable_locked(struct intel_dp *intel_dp,
1710 				    const struct intel_crtc_state *crtc_state)
1711 {
1712 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
1713 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1714 	enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port);
1715 	u32 val;
1716 
1717 	drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled);
1718 
1719 	intel_dp->psr.psr2_enabled = crtc_state->has_psr2;
1720 	intel_dp->psr.panel_replay_enabled = crtc_state->has_panel_replay;
1721 	intel_dp->psr.busy_frontbuffer_bits = 0;
1722 	intel_dp->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
1723 	intel_dp->psr.transcoder = crtc_state->cpu_transcoder;
1724 	/* DC5/DC6 requires at least 6 idle frames */
1725 	val = usecs_to_jiffies(intel_get_frame_time_us(crtc_state) * 6);
1726 	intel_dp->psr.dc3co_exit_delay = val;
1727 	intel_dp->psr.dc3co_exitline = crtc_state->dc3co_exitline;
1728 	intel_dp->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch;
1729 	intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
1730 	intel_dp->psr.req_psr2_sdp_prior_scanline =
1731 		crtc_state->req_psr2_sdp_prior_scanline;
1732 
1733 	if (!psr_interrupt_error_check(intel_dp))
1734 		return;
1735 
1736 	if (intel_dp->psr.panel_replay_enabled)
1737 		drm_dbg_kms(&dev_priv->drm, "Enabling Panel Replay\n");
1738 	else
1739 		drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n",
1740 			    intel_dp->psr.psr2_enabled ? "2" : "1");
1741 
1742 	intel_snps_phy_update_psr_power_state(dev_priv, phy, true);
1743 	intel_psr_enable_sink(intel_dp);
1744 	intel_psr_enable_source(intel_dp, crtc_state);
1745 	intel_dp->psr.enabled = true;
1746 	intel_dp->psr.paused = false;
1747 
1748 	intel_psr_activate(intel_dp);
1749 }
1750 
1751 static void intel_psr_exit(struct intel_dp *intel_dp)
1752 {
1753 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1754 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1755 	u32 val;
1756 
1757 	if (!intel_dp->psr.active) {
1758 		if (transcoder_has_psr2(dev_priv, cpu_transcoder)) {
1759 			val = intel_de_read(dev_priv, EDP_PSR2_CTL(cpu_transcoder));
1760 			drm_WARN_ON(&dev_priv->drm, val & EDP_PSR2_ENABLE);
1761 		}
1762 
1763 		val = intel_de_read(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder));
1764 		drm_WARN_ON(&dev_priv->drm, val & EDP_PSR_ENABLE);
1765 
1766 		return;
1767 	}
1768 
1769 	if (intel_dp->psr.panel_replay_enabled) {
1770 		intel_de_rmw(dev_priv, TRANS_DP2_CTL(intel_dp->psr.transcoder),
1771 			     TRANS_DP2_PANEL_REPLAY_ENABLE, 0);
1772 	} else if (intel_dp->psr.psr2_enabled) {
1773 		tgl_disallow_dc3co_on_psr2_exit(intel_dp);
1774 
1775 		val = intel_de_rmw(dev_priv, EDP_PSR2_CTL(cpu_transcoder),
1776 				   EDP_PSR2_ENABLE, 0);
1777 
1778 		drm_WARN_ON(&dev_priv->drm, !(val & EDP_PSR2_ENABLE));
1779 	} else {
1780 		val = intel_de_rmw(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder),
1781 				   EDP_PSR_ENABLE, 0);
1782 
1783 		drm_WARN_ON(&dev_priv->drm, !(val & EDP_PSR_ENABLE));
1784 	}
1785 	intel_dp->psr.active = false;
1786 }
1787 
1788 static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp)
1789 {
1790 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1791 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1792 	i915_reg_t psr_status;
1793 	u32 psr_status_mask;
1794 
1795 	if (intel_dp->psr.psr2_enabled) {
1796 		psr_status = EDP_PSR2_STATUS(cpu_transcoder);
1797 		psr_status_mask = EDP_PSR2_STATUS_STATE_MASK;
1798 	} else {
1799 		psr_status = psr_status_reg(dev_priv, cpu_transcoder);
1800 		psr_status_mask = EDP_PSR_STATUS_STATE_MASK;
1801 	}
1802 
1803 	/* Wait till PSR is idle */
1804 	if (intel_de_wait_for_clear(dev_priv, psr_status,
1805 				    psr_status_mask, 2000))
1806 		drm_err(&dev_priv->drm, "Timed out waiting PSR idle state\n");
1807 }
1808 
1809 static void intel_psr_disable_locked(struct intel_dp *intel_dp)
1810 {
1811 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1812 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1813 	enum phy phy = intel_port_to_phy(dev_priv,
1814 					 dp_to_dig_port(intel_dp)->base.port);
1815 
1816 	lockdep_assert_held(&intel_dp->psr.lock);
1817 
1818 	if (!intel_dp->psr.enabled)
1819 		return;
1820 
1821 	if (intel_dp->psr.panel_replay_enabled)
1822 		drm_dbg_kms(&dev_priv->drm, "Disabling Panel Replay\n");
1823 	else
1824 		drm_dbg_kms(&dev_priv->drm, "Disabling PSR%s\n",
1825 			    intel_dp->psr.psr2_enabled ? "2" : "1");
1826 
1827 	intel_psr_exit(intel_dp);
1828 	intel_psr_wait_exit_locked(intel_dp);
1829 
1830 	/*
1831 	 * Wa_16013835468
1832 	 * Wa_14015648006
1833 	 */
1834 	if (DISPLAY_VER(dev_priv) >= 11)
1835 		intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1,
1836 			     wa_16013835468_bit_get(intel_dp), 0);
1837 
1838 	if (intel_dp->psr.psr2_enabled) {
1839 		/* Wa_16012604467:adlp,mtl[a0,b0] */
1840 		if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0))
1841 			intel_de_rmw(dev_priv,
1842 				     MTL_CLKGATE_DIS_TRANS(cpu_transcoder),
1843 				     MTL_CLKGATE_DIS_TRANS_DMASC_GATING_DIS, 0);
1844 		else if (IS_ALDERLAKE_P(dev_priv))
1845 			intel_de_rmw(dev_priv, CLKGATE_DIS_MISC,
1846 				     CLKGATE_DIS_MISC_DMASC_GATING_DIS, 0);
1847 	}
1848 
1849 	intel_snps_phy_update_psr_power_state(dev_priv, phy, false);
1850 
1851 	/* Disable PSR on Sink */
1852 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
1853 
1854 	if (intel_dp->psr.psr2_enabled)
1855 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, 0);
1856 
1857 	intel_dp->psr.enabled = false;
1858 	intel_dp->psr.panel_replay_enabled = false;
1859 	intel_dp->psr.psr2_enabled = false;
1860 	intel_dp->psr.psr2_sel_fetch_enabled = false;
1861 	intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
1862 }
1863 
1864 /**
1865  * intel_psr_disable - Disable PSR
1866  * @intel_dp: Intel DP
1867  * @old_crtc_state: old CRTC state
1868  *
1869  * This function needs to be called before disabling pipe.
1870  */
1871 void intel_psr_disable(struct intel_dp *intel_dp,
1872 		       const struct intel_crtc_state *old_crtc_state)
1873 {
1874 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1875 
1876 	if (!old_crtc_state->has_psr)
1877 		return;
1878 
1879 	if (drm_WARN_ON(&dev_priv->drm, !CAN_PSR(intel_dp)))
1880 		return;
1881 
1882 	mutex_lock(&intel_dp->psr.lock);
1883 
1884 	intel_psr_disable_locked(intel_dp);
1885 
1886 	mutex_unlock(&intel_dp->psr.lock);
1887 	cancel_work_sync(&intel_dp->psr.work);
1888 	cancel_delayed_work_sync(&intel_dp->psr.dc3co_work);
1889 }
1890 
1891 /**
1892  * intel_psr_pause - Pause PSR
1893  * @intel_dp: Intel DP
1894  *
1895  * This function need to be called after enabling psr.
1896  */
1897 void intel_psr_pause(struct intel_dp *intel_dp)
1898 {
1899 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1900 	struct intel_psr *psr = &intel_dp->psr;
1901 
1902 	if (!CAN_PSR(intel_dp))
1903 		return;
1904 
1905 	mutex_lock(&psr->lock);
1906 
1907 	if (!psr->enabled) {
1908 		mutex_unlock(&psr->lock);
1909 		return;
1910 	}
1911 
1912 	/* If we ever hit this, we will need to add refcount to pause/resume */
1913 	drm_WARN_ON(&dev_priv->drm, psr->paused);
1914 
1915 	intel_psr_exit(intel_dp);
1916 	intel_psr_wait_exit_locked(intel_dp);
1917 	psr->paused = true;
1918 
1919 	mutex_unlock(&psr->lock);
1920 
1921 	cancel_work_sync(&psr->work);
1922 	cancel_delayed_work_sync(&psr->dc3co_work);
1923 }
1924 
1925 /**
1926  * intel_psr_resume - Resume PSR
1927  * @intel_dp: Intel DP
1928  *
1929  * This function need to be called after pausing psr.
1930  */
1931 void intel_psr_resume(struct intel_dp *intel_dp)
1932 {
1933 	struct intel_psr *psr = &intel_dp->psr;
1934 
1935 	if (!CAN_PSR(intel_dp))
1936 		return;
1937 
1938 	mutex_lock(&psr->lock);
1939 
1940 	if (!psr->paused)
1941 		goto unlock;
1942 
1943 	psr->paused = false;
1944 	intel_psr_activate(intel_dp);
1945 
1946 unlock:
1947 	mutex_unlock(&psr->lock);
1948 }
1949 
1950 static u32 man_trk_ctl_enable_bit_get(struct drm_i915_private *dev_priv)
1951 {
1952 	return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ? 0 :
1953 		PSR2_MAN_TRK_CTL_ENABLE;
1954 }
1955 
1956 static u32 man_trk_ctl_single_full_frame_bit_get(struct drm_i915_private *dev_priv)
1957 {
1958 	return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ?
1959 	       ADLP_PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME :
1960 	       PSR2_MAN_TRK_CTL_SF_SINGLE_FULL_FRAME;
1961 }
1962 
1963 static u32 man_trk_ctl_partial_frame_bit_get(struct drm_i915_private *dev_priv)
1964 {
1965 	return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ?
1966 	       ADLP_PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE :
1967 	       PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE;
1968 }
1969 
1970 static u32 man_trk_ctl_continuos_full_frame(struct drm_i915_private *dev_priv)
1971 {
1972 	return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ?
1973 	       ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME :
1974 	       PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME;
1975 }
1976 
1977 static void psr_force_hw_tracking_exit(struct intel_dp *intel_dp)
1978 {
1979 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1980 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
1981 
1982 	if (intel_dp->psr.psr2_sel_fetch_enabled)
1983 		intel_de_write(dev_priv,
1984 			       PSR2_MAN_TRK_CTL(cpu_transcoder),
1985 			       man_trk_ctl_enable_bit_get(dev_priv) |
1986 			       man_trk_ctl_partial_frame_bit_get(dev_priv) |
1987 			       man_trk_ctl_single_full_frame_bit_get(dev_priv) |
1988 			       man_trk_ctl_continuos_full_frame(dev_priv));
1989 
1990 	/*
1991 	 * Display WA #0884: skl+
1992 	 * This documented WA for bxt can be safely applied
1993 	 * broadly so we can force HW tracking to exit PSR
1994 	 * instead of disabling and re-enabling.
1995 	 * Workaround tells us to write 0 to CUR_SURFLIVE_A,
1996 	 * but it makes more sense write to the current active
1997 	 * pipe.
1998 	 *
1999 	 * This workaround do not exist for platforms with display 10 or newer
2000 	 * but testing proved that it works for up display 13, for newer
2001 	 * than that testing will be needed.
2002 	 */
2003 	intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
2004 }
2005 
2006 void intel_psr2_program_trans_man_trk_ctl(const struct intel_crtc_state *crtc_state)
2007 {
2008 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2009 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2010 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2011 	struct intel_encoder *encoder;
2012 
2013 	if (!crtc_state->enable_psr2_sel_fetch)
2014 		return;
2015 
2016 	for_each_intel_encoder_mask_with_psr(&dev_priv->drm, encoder,
2017 					     crtc_state->uapi.encoder_mask) {
2018 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2019 
2020 		lockdep_assert_held(&intel_dp->psr.lock);
2021 		if (intel_dp->psr.psr2_sel_fetch_cff_enabled)
2022 			return;
2023 		break;
2024 	}
2025 
2026 	intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder),
2027 		       crtc_state->psr2_man_track_ctl);
2028 
2029 	if (!crtc_state->enable_psr2_su_region_et)
2030 		return;
2031 
2032 	intel_de_write(dev_priv, PIPE_SRCSZ_ERLY_TPT(crtc->pipe),
2033 		       crtc_state->pipe_srcsz_early_tpt);
2034 }
2035 
2036 static void psr2_man_trk_ctl_calc(struct intel_crtc_state *crtc_state,
2037 				  bool full_update)
2038 {
2039 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2040 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2041 	u32 val = man_trk_ctl_enable_bit_get(dev_priv);
2042 
2043 	/* SF partial frame enable has to be set even on full update */
2044 	val |= man_trk_ctl_partial_frame_bit_get(dev_priv);
2045 
2046 	if (full_update) {
2047 		val |= man_trk_ctl_single_full_frame_bit_get(dev_priv);
2048 		val |= man_trk_ctl_continuos_full_frame(dev_priv);
2049 		goto exit;
2050 	}
2051 
2052 	if (crtc_state->psr2_su_area.y1 == -1)
2053 		goto exit;
2054 
2055 	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14) {
2056 		val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(crtc_state->psr2_su_area.y1);
2057 		val |= ADLP_PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(crtc_state->psr2_su_area.y2 - 1);
2058 	} else {
2059 		drm_WARN_ON(crtc_state->uapi.crtc->dev,
2060 			    crtc_state->psr2_su_area.y1 % 4 ||
2061 			    crtc_state->psr2_su_area.y2 % 4);
2062 
2063 		val |= PSR2_MAN_TRK_CTL_SU_REGION_START_ADDR(
2064 			crtc_state->psr2_su_area.y1 / 4 + 1);
2065 		val |= PSR2_MAN_TRK_CTL_SU_REGION_END_ADDR(
2066 			crtc_state->psr2_su_area.y2 / 4 + 1);
2067 	}
2068 exit:
2069 	crtc_state->psr2_man_track_ctl = val;
2070 }
2071 
2072 static u32 psr2_pipe_srcsz_early_tpt_calc(struct intel_crtc_state *crtc_state,
2073 					  bool full_update)
2074 {
2075 	int width, height;
2076 
2077 	if (!crtc_state->enable_psr2_su_region_et || full_update)
2078 		return 0;
2079 
2080 	width = drm_rect_width(&crtc_state->psr2_su_area);
2081 	height = drm_rect_height(&crtc_state->psr2_su_area);
2082 
2083 	return PIPESRC_WIDTH(width - 1) | PIPESRC_HEIGHT(height - 1);
2084 }
2085 
2086 static void clip_area_update(struct drm_rect *overlap_damage_area,
2087 			     struct drm_rect *damage_area,
2088 			     struct drm_rect *pipe_src)
2089 {
2090 	if (!drm_rect_intersect(damage_area, pipe_src))
2091 		return;
2092 
2093 	if (overlap_damage_area->y1 == -1) {
2094 		overlap_damage_area->y1 = damage_area->y1;
2095 		overlap_damage_area->y2 = damage_area->y2;
2096 		return;
2097 	}
2098 
2099 	if (damage_area->y1 < overlap_damage_area->y1)
2100 		overlap_damage_area->y1 = damage_area->y1;
2101 
2102 	if (damage_area->y2 > overlap_damage_area->y2)
2103 		overlap_damage_area->y2 = damage_area->y2;
2104 }
2105 
2106 static void intel_psr2_sel_fetch_pipe_alignment(struct intel_crtc_state *crtc_state)
2107 {
2108 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2109 	const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config;
2110 	u16 y_alignment;
2111 
2112 	/* ADLP aligns the SU region to vdsc slice height in case dsc is enabled */
2113 	if (crtc_state->dsc.compression_enable &&
2114 	    (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14))
2115 		y_alignment = vdsc_cfg->slice_height;
2116 	else
2117 		y_alignment = crtc_state->su_y_granularity;
2118 
2119 	crtc_state->psr2_su_area.y1 -= crtc_state->psr2_su_area.y1 % y_alignment;
2120 	if (crtc_state->psr2_su_area.y2 % y_alignment)
2121 		crtc_state->psr2_su_area.y2 = ((crtc_state->psr2_su_area.y2 /
2122 						y_alignment) + 1) * y_alignment;
2123 }
2124 
2125 /*
2126  * When early transport is in use we need to extend SU area to cover
2127  * cursor fully when cursor is in SU area.
2128  */
2129 static void
2130 intel_psr2_sel_fetch_et_alignment(struct intel_atomic_state *state,
2131 				  struct intel_crtc *crtc)
2132 {
2133 	struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
2134 	struct intel_plane_state *new_plane_state;
2135 	struct intel_plane *plane;
2136 	int i;
2137 
2138 	if (!crtc_state->enable_psr2_su_region_et)
2139 		return;
2140 
2141 	for_each_new_intel_plane_in_state(state, plane, new_plane_state, i) {
2142 		struct drm_rect inter;
2143 
2144 		if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc)
2145 			continue;
2146 
2147 		if (plane->id != PLANE_CURSOR)
2148 			continue;
2149 
2150 		if (!new_plane_state->uapi.visible)
2151 			continue;
2152 
2153 		inter = crtc_state->psr2_su_area;
2154 		if (!drm_rect_intersect(&inter, &new_plane_state->uapi.dst))
2155 			continue;
2156 
2157 		clip_area_update(&crtc_state->psr2_su_area, &new_plane_state->uapi.dst,
2158 				 &crtc_state->pipe_src);
2159 	}
2160 }
2161 
2162 /*
2163  * TODO: Not clear how to handle planes with negative position,
2164  * also planes are not updated if they have a negative X
2165  * position so for now doing a full update in this cases
2166  *
2167  * Plane scaling and rotation is not supported by selective fetch and both
2168  * properties can change without a modeset, so need to be check at every
2169  * atomic commit.
2170  */
2171 static bool psr2_sel_fetch_plane_state_supported(const struct intel_plane_state *plane_state)
2172 {
2173 	if (plane_state->uapi.dst.y1 < 0 ||
2174 	    plane_state->uapi.dst.x1 < 0 ||
2175 	    plane_state->scaler_id >= 0 ||
2176 	    plane_state->uapi.rotation != DRM_MODE_ROTATE_0)
2177 		return false;
2178 
2179 	return true;
2180 }
2181 
2182 /*
2183  * Check for pipe properties that is not supported by selective fetch.
2184  *
2185  * TODO: pipe scaling causes a modeset but skl_update_scaler_crtc() is executed
2186  * after intel_psr_compute_config(), so for now keeping PSR2 selective fetch
2187  * enabled and going to the full update path.
2188  */
2189 static bool psr2_sel_fetch_pipe_state_supported(const struct intel_crtc_state *crtc_state)
2190 {
2191 	if (crtc_state->scaler_state.scaler_id >= 0)
2192 		return false;
2193 
2194 	return true;
2195 }
2196 
2197 int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
2198 				struct intel_crtc *crtc)
2199 {
2200 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2201 	struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
2202 	struct intel_plane_state *new_plane_state, *old_plane_state;
2203 	struct intel_plane *plane;
2204 	bool full_update = false;
2205 	int i, ret;
2206 
2207 	if (!crtc_state->enable_psr2_sel_fetch)
2208 		return 0;
2209 
2210 	if (!psr2_sel_fetch_pipe_state_supported(crtc_state)) {
2211 		full_update = true;
2212 		goto skip_sel_fetch_set_loop;
2213 	}
2214 
2215 	crtc_state->psr2_su_area.x1 = 0;
2216 	crtc_state->psr2_su_area.y1 = -1;
2217 	crtc_state->psr2_su_area.x2 = INT_MAX;
2218 	crtc_state->psr2_su_area.y2 = -1;
2219 
2220 	/*
2221 	 * Calculate minimal selective fetch area of each plane and calculate
2222 	 * the pipe damaged area.
2223 	 * In the next loop the plane selective fetch area will actually be set
2224 	 * using whole pipe damaged area.
2225 	 */
2226 	for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
2227 					     new_plane_state, i) {
2228 		struct drm_rect src, damaged_area = { .x1 = 0, .y1 = -1,
2229 						      .x2 = INT_MAX };
2230 
2231 		if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc)
2232 			continue;
2233 
2234 		if (!new_plane_state->uapi.visible &&
2235 		    !old_plane_state->uapi.visible)
2236 			continue;
2237 
2238 		if (!psr2_sel_fetch_plane_state_supported(new_plane_state)) {
2239 			full_update = true;
2240 			break;
2241 		}
2242 
2243 		/*
2244 		 * If visibility or plane moved, mark the whole plane area as
2245 		 * damaged as it needs to be complete redraw in the new and old
2246 		 * position.
2247 		 */
2248 		if (new_plane_state->uapi.visible != old_plane_state->uapi.visible ||
2249 		    !drm_rect_equals(&new_plane_state->uapi.dst,
2250 				     &old_plane_state->uapi.dst)) {
2251 			if (old_plane_state->uapi.visible) {
2252 				damaged_area.y1 = old_plane_state->uapi.dst.y1;
2253 				damaged_area.y2 = old_plane_state->uapi.dst.y2;
2254 				clip_area_update(&crtc_state->psr2_su_area, &damaged_area,
2255 						 &crtc_state->pipe_src);
2256 			}
2257 
2258 			if (new_plane_state->uapi.visible) {
2259 				damaged_area.y1 = new_plane_state->uapi.dst.y1;
2260 				damaged_area.y2 = new_plane_state->uapi.dst.y2;
2261 				clip_area_update(&crtc_state->psr2_su_area, &damaged_area,
2262 						 &crtc_state->pipe_src);
2263 			}
2264 			continue;
2265 		} else if (new_plane_state->uapi.alpha != old_plane_state->uapi.alpha) {
2266 			/* If alpha changed mark the whole plane area as damaged */
2267 			damaged_area.y1 = new_plane_state->uapi.dst.y1;
2268 			damaged_area.y2 = new_plane_state->uapi.dst.y2;
2269 			clip_area_update(&crtc_state->psr2_su_area, &damaged_area,
2270 					 &crtc_state->pipe_src);
2271 			continue;
2272 		}
2273 
2274 		src = drm_plane_state_src(&new_plane_state->uapi);
2275 		drm_rect_fp_to_int(&src, &src);
2276 
2277 		if (!drm_atomic_helper_damage_merged(&old_plane_state->uapi,
2278 						     &new_plane_state->uapi, &damaged_area))
2279 			continue;
2280 
2281 		damaged_area.y1 += new_plane_state->uapi.dst.y1 - src.y1;
2282 		damaged_area.y2 += new_plane_state->uapi.dst.y1 - src.y1;
2283 		damaged_area.x1 += new_plane_state->uapi.dst.x1 - src.x1;
2284 		damaged_area.x2 += new_plane_state->uapi.dst.x1 - src.x1;
2285 
2286 		clip_area_update(&crtc_state->psr2_su_area, &damaged_area, &crtc_state->pipe_src);
2287 	}
2288 
2289 	/*
2290 	 * TODO: For now we are just using full update in case
2291 	 * selective fetch area calculation fails. To optimize this we
2292 	 * should identify cases where this happens and fix the area
2293 	 * calculation for those.
2294 	 */
2295 	if (crtc_state->psr2_su_area.y1 == -1) {
2296 		drm_info_once(&dev_priv->drm,
2297 			      "Selective fetch area calculation failed in pipe %c\n",
2298 			      pipe_name(crtc->pipe));
2299 		full_update = true;
2300 	}
2301 
2302 	if (full_update)
2303 		goto skip_sel_fetch_set_loop;
2304 
2305 	/* Wa_14014971492 */
2306 	if ((IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0) ||
2307 	     IS_ALDERLAKE_P(dev_priv) || IS_TIGERLAKE(dev_priv)) &&
2308 	    crtc_state->splitter.enable)
2309 		crtc_state->psr2_su_area.y1 = 0;
2310 
2311 	ret = drm_atomic_add_affected_planes(&state->base, &crtc->base);
2312 	if (ret)
2313 		return ret;
2314 
2315 	/*
2316 	 * Adjust su area to cover cursor fully as necessary (early
2317 	 * transport). This needs to be done after
2318 	 * drm_atomic_add_affected_planes to ensure visible cursor is added into
2319 	 * affected planes even when cursor is not updated by itself.
2320 	 */
2321 	intel_psr2_sel_fetch_et_alignment(state, crtc);
2322 
2323 	intel_psr2_sel_fetch_pipe_alignment(crtc_state);
2324 
2325 	/*
2326 	 * Now that we have the pipe damaged area check if it intersect with
2327 	 * every plane, if it does set the plane selective fetch area.
2328 	 */
2329 	for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
2330 					     new_plane_state, i) {
2331 		struct drm_rect *sel_fetch_area, inter;
2332 		struct intel_plane *linked = new_plane_state->planar_linked_plane;
2333 
2334 		if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc ||
2335 		    !new_plane_state->uapi.visible)
2336 			continue;
2337 
2338 		inter = crtc_state->psr2_su_area;
2339 		sel_fetch_area = &new_plane_state->psr2_sel_fetch_area;
2340 		if (!drm_rect_intersect(&inter, &new_plane_state->uapi.dst)) {
2341 			sel_fetch_area->y1 = -1;
2342 			sel_fetch_area->y2 = -1;
2343 			/*
2344 			 * if plane sel fetch was previously enabled ->
2345 			 * disable it
2346 			 */
2347 			if (drm_rect_height(&old_plane_state->psr2_sel_fetch_area) > 0)
2348 				crtc_state->update_planes |= BIT(plane->id);
2349 
2350 			continue;
2351 		}
2352 
2353 		if (!psr2_sel_fetch_plane_state_supported(new_plane_state)) {
2354 			full_update = true;
2355 			break;
2356 		}
2357 
2358 		sel_fetch_area = &new_plane_state->psr2_sel_fetch_area;
2359 		sel_fetch_area->y1 = inter.y1 - new_plane_state->uapi.dst.y1;
2360 		sel_fetch_area->y2 = inter.y2 - new_plane_state->uapi.dst.y1;
2361 		crtc_state->update_planes |= BIT(plane->id);
2362 
2363 		/*
2364 		 * Sel_fetch_area is calculated for UV plane. Use
2365 		 * same area for Y plane as well.
2366 		 */
2367 		if (linked) {
2368 			struct intel_plane_state *linked_new_plane_state;
2369 			struct drm_rect *linked_sel_fetch_area;
2370 
2371 			linked_new_plane_state = intel_atomic_get_plane_state(state, linked);
2372 			if (IS_ERR(linked_new_plane_state))
2373 				return PTR_ERR(linked_new_plane_state);
2374 
2375 			linked_sel_fetch_area = &linked_new_plane_state->psr2_sel_fetch_area;
2376 			linked_sel_fetch_area->y1 = sel_fetch_area->y1;
2377 			linked_sel_fetch_area->y2 = sel_fetch_area->y2;
2378 			crtc_state->update_planes |= BIT(linked->id);
2379 		}
2380 	}
2381 
2382 skip_sel_fetch_set_loop:
2383 	psr2_man_trk_ctl_calc(crtc_state, full_update);
2384 	crtc_state->pipe_srcsz_early_tpt =
2385 		psr2_pipe_srcsz_early_tpt_calc(crtc_state, full_update);
2386 	return 0;
2387 }
2388 
2389 void intel_psr_pre_plane_update(struct intel_atomic_state *state,
2390 				struct intel_crtc *crtc)
2391 {
2392 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2393 	const struct intel_crtc_state *old_crtc_state =
2394 		intel_atomic_get_old_crtc_state(state, crtc);
2395 	const struct intel_crtc_state *new_crtc_state =
2396 		intel_atomic_get_new_crtc_state(state, crtc);
2397 	struct intel_encoder *encoder;
2398 
2399 	if (!HAS_PSR(i915))
2400 		return;
2401 
2402 	for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
2403 					     old_crtc_state->uapi.encoder_mask) {
2404 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2405 		struct intel_psr *psr = &intel_dp->psr;
2406 		bool needs_to_disable = false;
2407 
2408 		mutex_lock(&psr->lock);
2409 
2410 		/*
2411 		 * Reasons to disable:
2412 		 * - PSR disabled in new state
2413 		 * - All planes will go inactive
2414 		 * - Changing between PSR versions
2415 		 * - Display WA #1136: skl, bxt
2416 		 */
2417 		needs_to_disable |= intel_crtc_needs_modeset(new_crtc_state);
2418 		needs_to_disable |= !new_crtc_state->has_psr;
2419 		needs_to_disable |= !new_crtc_state->active_planes;
2420 		needs_to_disable |= new_crtc_state->has_psr2 != psr->psr2_enabled;
2421 		needs_to_disable |= DISPLAY_VER(i915) < 11 &&
2422 			new_crtc_state->wm_level_disabled;
2423 
2424 		if (psr->enabled && needs_to_disable)
2425 			intel_psr_disable_locked(intel_dp);
2426 		else if (psr->enabled && new_crtc_state->wm_level_disabled)
2427 			/* Wa_14015648006 */
2428 			wm_optimization_wa(intel_dp, new_crtc_state);
2429 
2430 		mutex_unlock(&psr->lock);
2431 	}
2432 }
2433 
2434 void intel_psr_post_plane_update(struct intel_atomic_state *state,
2435 				 struct intel_crtc *crtc)
2436 {
2437 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2438 	const struct intel_crtc_state *crtc_state =
2439 		intel_atomic_get_new_crtc_state(state, crtc);
2440 	struct intel_encoder *encoder;
2441 
2442 	if (!(crtc_state->has_psr || crtc_state->has_panel_replay))
2443 		return;
2444 
2445 	for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
2446 					     crtc_state->uapi.encoder_mask) {
2447 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2448 		struct intel_psr *psr = &intel_dp->psr;
2449 		bool keep_disabled = false;
2450 
2451 		mutex_lock(&psr->lock);
2452 
2453 		drm_WARN_ON(&dev_priv->drm, psr->enabled && !crtc_state->active_planes);
2454 
2455 		keep_disabled |= psr->sink_not_reliable;
2456 		keep_disabled |= !crtc_state->active_planes;
2457 
2458 		/* Display WA #1136: skl, bxt */
2459 		keep_disabled |= DISPLAY_VER(dev_priv) < 11 &&
2460 			crtc_state->wm_level_disabled;
2461 
2462 		if (!psr->enabled && !keep_disabled)
2463 			intel_psr_enable_locked(intel_dp, crtc_state);
2464 		else if (psr->enabled && !crtc_state->wm_level_disabled)
2465 			/* Wa_14015648006 */
2466 			wm_optimization_wa(intel_dp, crtc_state);
2467 
2468 		/* Force a PSR exit when enabling CRC to avoid CRC timeouts */
2469 		if (crtc_state->crc_enabled && psr->enabled)
2470 			psr_force_hw_tracking_exit(intel_dp);
2471 
2472 		/*
2473 		 * Clear possible busy bits in case we have
2474 		 * invalidate -> flip -> flush sequence.
2475 		 */
2476 		intel_dp->psr.busy_frontbuffer_bits = 0;
2477 
2478 		mutex_unlock(&psr->lock);
2479 	}
2480 }
2481 
2482 static int _psr2_ready_for_pipe_update_locked(struct intel_dp *intel_dp)
2483 {
2484 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2485 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2486 
2487 	/*
2488 	 * Any state lower than EDP_PSR2_STATUS_STATE_DEEP_SLEEP is enough.
2489 	 * As all higher states has bit 4 of PSR2 state set we can just wait for
2490 	 * EDP_PSR2_STATUS_STATE_DEEP_SLEEP to be cleared.
2491 	 */
2492 	return intel_de_wait_for_clear(dev_priv,
2493 				       EDP_PSR2_STATUS(cpu_transcoder),
2494 				       EDP_PSR2_STATUS_STATE_DEEP_SLEEP, 50);
2495 }
2496 
2497 static int _psr1_ready_for_pipe_update_locked(struct intel_dp *intel_dp)
2498 {
2499 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2500 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2501 
2502 	/*
2503 	 * From bspec: Panel Self Refresh (BDW+)
2504 	 * Max. time for PSR to idle = Inverse of the refresh rate + 6 ms of
2505 	 * exit training time + 1.5 ms of aux channel handshake. 50 ms is
2506 	 * defensive enough to cover everything.
2507 	 */
2508 	return intel_de_wait_for_clear(dev_priv,
2509 				       psr_status_reg(dev_priv, cpu_transcoder),
2510 				       EDP_PSR_STATUS_STATE_MASK, 50);
2511 }
2512 
2513 /**
2514  * intel_psr_wait_for_idle_locked - wait for PSR be ready for a pipe update
2515  * @new_crtc_state: new CRTC state
2516  *
2517  * This function is expected to be called from pipe_update_start() where it is
2518  * not expected to race with PSR enable or disable.
2519  */
2520 void intel_psr_wait_for_idle_locked(const struct intel_crtc_state *new_crtc_state)
2521 {
2522 	struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev);
2523 	struct intel_encoder *encoder;
2524 
2525 	if (!new_crtc_state->has_psr)
2526 		return;
2527 
2528 	for_each_intel_encoder_mask_with_psr(&dev_priv->drm, encoder,
2529 					     new_crtc_state->uapi.encoder_mask) {
2530 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2531 		int ret;
2532 
2533 		lockdep_assert_held(&intel_dp->psr.lock);
2534 
2535 		if (!intel_dp->psr.enabled)
2536 			continue;
2537 
2538 		if (intel_dp->psr.psr2_enabled)
2539 			ret = _psr2_ready_for_pipe_update_locked(intel_dp);
2540 		else
2541 			ret = _psr1_ready_for_pipe_update_locked(intel_dp);
2542 
2543 		if (ret)
2544 			drm_err(&dev_priv->drm, "PSR wait timed out, atomic update may fail\n");
2545 	}
2546 }
2547 
2548 static bool __psr_wait_for_idle_locked(struct intel_dp *intel_dp)
2549 {
2550 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2551 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2552 	i915_reg_t reg;
2553 	u32 mask;
2554 	int err;
2555 
2556 	if (!intel_dp->psr.enabled)
2557 		return false;
2558 
2559 	if (intel_dp->psr.psr2_enabled) {
2560 		reg = EDP_PSR2_STATUS(cpu_transcoder);
2561 		mask = EDP_PSR2_STATUS_STATE_MASK;
2562 	} else {
2563 		reg = psr_status_reg(dev_priv, cpu_transcoder);
2564 		mask = EDP_PSR_STATUS_STATE_MASK;
2565 	}
2566 
2567 	mutex_unlock(&intel_dp->psr.lock);
2568 
2569 	err = intel_de_wait_for_clear(dev_priv, reg, mask, 50);
2570 	if (err)
2571 		drm_err(&dev_priv->drm,
2572 			"Timed out waiting for PSR Idle for re-enable\n");
2573 
2574 	/* After the unlocked wait, verify that PSR is still wanted! */
2575 	mutex_lock(&intel_dp->psr.lock);
2576 	return err == 0 && intel_dp->psr.enabled;
2577 }
2578 
2579 static int intel_psr_fastset_force(struct drm_i915_private *dev_priv)
2580 {
2581 	struct drm_connector_list_iter conn_iter;
2582 	struct drm_modeset_acquire_ctx ctx;
2583 	struct drm_atomic_state *state;
2584 	struct drm_connector *conn;
2585 	int err = 0;
2586 
2587 	state = drm_atomic_state_alloc(&dev_priv->drm);
2588 	if (!state)
2589 		return -ENOMEM;
2590 
2591 	drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
2592 
2593 	state->acquire_ctx = &ctx;
2594 	to_intel_atomic_state(state)->internal = true;
2595 
2596 retry:
2597 	drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter);
2598 	drm_for_each_connector_iter(conn, &conn_iter) {
2599 		struct drm_connector_state *conn_state;
2600 		struct drm_crtc_state *crtc_state;
2601 
2602 		if (conn->connector_type != DRM_MODE_CONNECTOR_eDP)
2603 			continue;
2604 
2605 		conn_state = drm_atomic_get_connector_state(state, conn);
2606 		if (IS_ERR(conn_state)) {
2607 			err = PTR_ERR(conn_state);
2608 			break;
2609 		}
2610 
2611 		if (!conn_state->crtc)
2612 			continue;
2613 
2614 		crtc_state = drm_atomic_get_crtc_state(state, conn_state->crtc);
2615 		if (IS_ERR(crtc_state)) {
2616 			err = PTR_ERR(crtc_state);
2617 			break;
2618 		}
2619 
2620 		/* Mark mode as changed to trigger a pipe->update() */
2621 		crtc_state->mode_changed = true;
2622 	}
2623 	drm_connector_list_iter_end(&conn_iter);
2624 
2625 	if (err == 0)
2626 		err = drm_atomic_commit(state);
2627 
2628 	if (err == -EDEADLK) {
2629 		drm_atomic_state_clear(state);
2630 		err = drm_modeset_backoff(&ctx);
2631 		if (!err)
2632 			goto retry;
2633 	}
2634 
2635 	drm_modeset_drop_locks(&ctx);
2636 	drm_modeset_acquire_fini(&ctx);
2637 	drm_atomic_state_put(state);
2638 
2639 	return err;
2640 }
2641 
2642 int intel_psr_debug_set(struct intel_dp *intel_dp, u64 val)
2643 {
2644 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2645 	const u32 mode = val & I915_PSR_DEBUG_MODE_MASK;
2646 	u32 old_mode;
2647 	int ret;
2648 
2649 	if (val & ~(I915_PSR_DEBUG_IRQ | I915_PSR_DEBUG_MODE_MASK) ||
2650 	    mode > I915_PSR_DEBUG_ENABLE_SEL_FETCH) {
2651 		drm_dbg_kms(&dev_priv->drm, "Invalid debug mask %llx\n", val);
2652 		return -EINVAL;
2653 	}
2654 
2655 	ret = mutex_lock_interruptible(&intel_dp->psr.lock);
2656 	if (ret)
2657 		return ret;
2658 
2659 	old_mode = intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK;
2660 	intel_dp->psr.debug = val;
2661 
2662 	/*
2663 	 * Do it right away if it's already enabled, otherwise it will be done
2664 	 * when enabling the source.
2665 	 */
2666 	if (intel_dp->psr.enabled)
2667 		psr_irq_control(intel_dp);
2668 
2669 	mutex_unlock(&intel_dp->psr.lock);
2670 
2671 	if (old_mode != mode)
2672 		ret = intel_psr_fastset_force(dev_priv);
2673 
2674 	return ret;
2675 }
2676 
2677 static void intel_psr_handle_irq(struct intel_dp *intel_dp)
2678 {
2679 	struct intel_psr *psr = &intel_dp->psr;
2680 
2681 	intel_psr_disable_locked(intel_dp);
2682 	psr->sink_not_reliable = true;
2683 	/* let's make sure that sink is awaken */
2684 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
2685 }
2686 
2687 static void intel_psr_work(struct work_struct *work)
2688 {
2689 	struct intel_dp *intel_dp =
2690 		container_of(work, typeof(*intel_dp), psr.work);
2691 
2692 	mutex_lock(&intel_dp->psr.lock);
2693 
2694 	if (!intel_dp->psr.enabled)
2695 		goto unlock;
2696 
2697 	if (READ_ONCE(intel_dp->psr.irq_aux_error))
2698 		intel_psr_handle_irq(intel_dp);
2699 
2700 	/*
2701 	 * We have to make sure PSR is ready for re-enable
2702 	 * otherwise it keeps disabled until next full enable/disable cycle.
2703 	 * PSR might take some time to get fully disabled
2704 	 * and be ready for re-enable.
2705 	 */
2706 	if (!__psr_wait_for_idle_locked(intel_dp))
2707 		goto unlock;
2708 
2709 	/*
2710 	 * The delayed work can race with an invalidate hence we need to
2711 	 * recheck. Since psr_flush first clears this and then reschedules we
2712 	 * won't ever miss a flush when bailing out here.
2713 	 */
2714 	if (intel_dp->psr.busy_frontbuffer_bits || intel_dp->psr.active)
2715 		goto unlock;
2716 
2717 	intel_psr_activate(intel_dp);
2718 unlock:
2719 	mutex_unlock(&intel_dp->psr.lock);
2720 }
2721 
2722 static void _psr_invalidate_handle(struct intel_dp *intel_dp)
2723 {
2724 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2725 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2726 
2727 	if (intel_dp->psr.psr2_sel_fetch_enabled) {
2728 		u32 val;
2729 
2730 		if (intel_dp->psr.psr2_sel_fetch_cff_enabled) {
2731 			/* Send one update otherwise lag is observed in screen */
2732 			intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
2733 			return;
2734 		}
2735 
2736 		val = man_trk_ctl_enable_bit_get(dev_priv) |
2737 		      man_trk_ctl_partial_frame_bit_get(dev_priv) |
2738 		      man_trk_ctl_continuos_full_frame(dev_priv);
2739 		intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder), val);
2740 		intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
2741 		intel_dp->psr.psr2_sel_fetch_cff_enabled = true;
2742 	} else {
2743 		intel_psr_exit(intel_dp);
2744 	}
2745 }
2746 
2747 /**
2748  * intel_psr_invalidate - Invalidate PSR
2749  * @dev_priv: i915 device
2750  * @frontbuffer_bits: frontbuffer plane tracking bits
2751  * @origin: which operation caused the invalidate
2752  *
2753  * Since the hardware frontbuffer tracking has gaps we need to integrate
2754  * with the software frontbuffer tracking. This function gets called every
2755  * time frontbuffer rendering starts and a buffer gets dirtied. PSR must be
2756  * disabled if the frontbuffer mask contains a buffer relevant to PSR.
2757  *
2758  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits."
2759  */
2760 void intel_psr_invalidate(struct drm_i915_private *dev_priv,
2761 			  unsigned frontbuffer_bits, enum fb_op_origin origin)
2762 {
2763 	struct intel_encoder *encoder;
2764 
2765 	if (origin == ORIGIN_FLIP)
2766 		return;
2767 
2768 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2769 		unsigned int pipe_frontbuffer_bits = frontbuffer_bits;
2770 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2771 
2772 		mutex_lock(&intel_dp->psr.lock);
2773 		if (!intel_dp->psr.enabled) {
2774 			mutex_unlock(&intel_dp->psr.lock);
2775 			continue;
2776 		}
2777 
2778 		pipe_frontbuffer_bits &=
2779 			INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);
2780 		intel_dp->psr.busy_frontbuffer_bits |= pipe_frontbuffer_bits;
2781 
2782 		if (pipe_frontbuffer_bits)
2783 			_psr_invalidate_handle(intel_dp);
2784 
2785 		mutex_unlock(&intel_dp->psr.lock);
2786 	}
2787 }
2788 /*
2789  * When we will be completely rely on PSR2 S/W tracking in future,
2790  * intel_psr_flush() will invalidate and flush the PSR for ORIGIN_FLIP
2791  * event also therefore tgl_dc3co_flush_locked() require to be changed
2792  * accordingly in future.
2793  */
2794 static void
2795 tgl_dc3co_flush_locked(struct intel_dp *intel_dp, unsigned int frontbuffer_bits,
2796 		       enum fb_op_origin origin)
2797 {
2798 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
2799 
2800 	if (!intel_dp->psr.dc3co_exitline || !intel_dp->psr.psr2_enabled ||
2801 	    !intel_dp->psr.active)
2802 		return;
2803 
2804 	/*
2805 	 * At every frontbuffer flush flip event modified delay of delayed work,
2806 	 * when delayed work schedules that means display has been idle.
2807 	 */
2808 	if (!(frontbuffer_bits &
2809 	    INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe)))
2810 		return;
2811 
2812 	tgl_psr2_enable_dc3co(intel_dp);
2813 	mod_delayed_work(i915->unordered_wq, &intel_dp->psr.dc3co_work,
2814 			 intel_dp->psr.dc3co_exit_delay);
2815 }
2816 
2817 static void _psr_flush_handle(struct intel_dp *intel_dp)
2818 {
2819 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2820 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
2821 
2822 	if (intel_dp->psr.psr2_sel_fetch_enabled) {
2823 		if (intel_dp->psr.psr2_sel_fetch_cff_enabled) {
2824 			/* can we turn CFF off? */
2825 			if (intel_dp->psr.busy_frontbuffer_bits == 0) {
2826 				u32 val = man_trk_ctl_enable_bit_get(dev_priv) |
2827 					man_trk_ctl_partial_frame_bit_get(dev_priv) |
2828 					man_trk_ctl_single_full_frame_bit_get(dev_priv) |
2829 					man_trk_ctl_continuos_full_frame(dev_priv);
2830 
2831 				/*
2832 				 * Set psr2_sel_fetch_cff_enabled as false to allow selective
2833 				 * updates. Still keep cff bit enabled as we don't have proper
2834 				 * SU configuration in case update is sent for any reason after
2835 				 * sff bit gets cleared by the HW on next vblank.
2836 				 */
2837 				intel_de_write(dev_priv, PSR2_MAN_TRK_CTL(cpu_transcoder),
2838 					       val);
2839 				intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0);
2840 				intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
2841 			}
2842 		} else {
2843 			/*
2844 			 * continuous full frame is disabled, only a single full
2845 			 * frame is required
2846 			 */
2847 			psr_force_hw_tracking_exit(intel_dp);
2848 		}
2849 	} else {
2850 		psr_force_hw_tracking_exit(intel_dp);
2851 
2852 		if (!intel_dp->psr.active && !intel_dp->psr.busy_frontbuffer_bits)
2853 			queue_work(dev_priv->unordered_wq, &intel_dp->psr.work);
2854 	}
2855 }
2856 
2857 /**
2858  * intel_psr_flush - Flush PSR
2859  * @dev_priv: i915 device
2860  * @frontbuffer_bits: frontbuffer plane tracking bits
2861  * @origin: which operation caused the flush
2862  *
2863  * Since the hardware frontbuffer tracking has gaps we need to integrate
2864  * with the software frontbuffer tracking. This function gets called every
2865  * time frontbuffer rendering has completed and flushed out to memory. PSR
2866  * can be enabled again if no other frontbuffer relevant to PSR is dirty.
2867  *
2868  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits.
2869  */
2870 void intel_psr_flush(struct drm_i915_private *dev_priv,
2871 		     unsigned frontbuffer_bits, enum fb_op_origin origin)
2872 {
2873 	struct intel_encoder *encoder;
2874 
2875 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2876 		unsigned int pipe_frontbuffer_bits = frontbuffer_bits;
2877 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2878 
2879 		mutex_lock(&intel_dp->psr.lock);
2880 		if (!intel_dp->psr.enabled) {
2881 			mutex_unlock(&intel_dp->psr.lock);
2882 			continue;
2883 		}
2884 
2885 		pipe_frontbuffer_bits &=
2886 			INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe);
2887 		intel_dp->psr.busy_frontbuffer_bits &= ~pipe_frontbuffer_bits;
2888 
2889 		/*
2890 		 * If the PSR is paused by an explicit intel_psr_paused() call,
2891 		 * we have to ensure that the PSR is not activated until
2892 		 * intel_psr_resume() is called.
2893 		 */
2894 		if (intel_dp->psr.paused)
2895 			goto unlock;
2896 
2897 		if (origin == ORIGIN_FLIP ||
2898 		    (origin == ORIGIN_CURSOR_UPDATE &&
2899 		     !intel_dp->psr.psr2_sel_fetch_enabled)) {
2900 			tgl_dc3co_flush_locked(intel_dp, frontbuffer_bits, origin);
2901 			goto unlock;
2902 		}
2903 
2904 		if (pipe_frontbuffer_bits == 0)
2905 			goto unlock;
2906 
2907 		/* By definition flush = invalidate + flush */
2908 		_psr_flush_handle(intel_dp);
2909 unlock:
2910 		mutex_unlock(&intel_dp->psr.lock);
2911 	}
2912 }
2913 
2914 /**
2915  * intel_psr_init - Init basic PSR work and mutex.
2916  * @intel_dp: Intel DP
2917  *
2918  * This function is called after the initializing connector.
2919  * (the initializing of connector treats the handling of connector capabilities)
2920  * And it initializes basic PSR stuff for each DP Encoder.
2921  */
2922 void intel_psr_init(struct intel_dp *intel_dp)
2923 {
2924 	struct intel_connector *connector = intel_dp->attached_connector;
2925 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
2926 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2927 
2928 	if (!(HAS_PSR(dev_priv) || HAS_DP20(dev_priv)))
2929 		return;
2930 
2931 	/*
2932 	 * HSW spec explicitly says PSR is tied to port A.
2933 	 * BDW+ platforms have a instance of PSR registers per transcoder but
2934 	 * BDW, GEN9 and GEN11 are not validated by HW team in other transcoder
2935 	 * than eDP one.
2936 	 * For now it only supports one instance of PSR for BDW, GEN9 and GEN11.
2937 	 * So lets keep it hardcoded to PORT_A for BDW, GEN9 and GEN11.
2938 	 * But GEN12 supports a instance of PSR registers per transcoder.
2939 	 */
2940 	if (DISPLAY_VER(dev_priv) < 12 && dig_port->base.port != PORT_A) {
2941 		drm_dbg_kms(&dev_priv->drm,
2942 			    "PSR condition failed: Port not supported\n");
2943 		return;
2944 	}
2945 
2946 	if (HAS_DP20(dev_priv) && !intel_dp_is_edp(intel_dp))
2947 		intel_dp->psr.source_panel_replay_support = true;
2948 	else
2949 		intel_dp->psr.source_support = true;
2950 
2951 	/* Disable early transport for now */
2952 	intel_dp->psr.debug |= I915_PSR_DEBUG_SU_REGION_ET_DISABLE;
2953 
2954 	/* Set link_standby x link_off defaults */
2955 	if (DISPLAY_VER(dev_priv) < 12)
2956 		/* For new platforms up to TGL let's respect VBT back again */
2957 		intel_dp->psr.link_standby = connector->panel.vbt.psr.full_link;
2958 
2959 	INIT_WORK(&intel_dp->psr.work, intel_psr_work);
2960 	INIT_DELAYED_WORK(&intel_dp->psr.dc3co_work, tgl_dc3co_disable_work);
2961 	mutex_init(&intel_dp->psr.lock);
2962 }
2963 
2964 static int psr_get_status_and_error_status(struct intel_dp *intel_dp,
2965 					   u8 *status, u8 *error_status)
2966 {
2967 	struct drm_dp_aux *aux = &intel_dp->aux;
2968 	int ret;
2969 	unsigned int offset;
2970 
2971 	offset = intel_dp->psr.panel_replay_enabled ?
2972 		 DP_SINK_DEVICE_PR_AND_FRAME_LOCK_STATUS : DP_PSR_STATUS;
2973 
2974 	ret = drm_dp_dpcd_readb(aux, offset, status);
2975 	if (ret != 1)
2976 		return ret;
2977 
2978 	offset = intel_dp->psr.panel_replay_enabled ?
2979 		 DP_PANEL_REPLAY_ERROR_STATUS : DP_PSR_ERROR_STATUS;
2980 
2981 	ret = drm_dp_dpcd_readb(aux, offset, error_status);
2982 	if (ret != 1)
2983 		return ret;
2984 
2985 	*status = *status & DP_PSR_SINK_STATE_MASK;
2986 
2987 	return 0;
2988 }
2989 
2990 static void psr_alpm_check(struct intel_dp *intel_dp)
2991 {
2992 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
2993 	struct drm_dp_aux *aux = &intel_dp->aux;
2994 	struct intel_psr *psr = &intel_dp->psr;
2995 	u8 val;
2996 	int r;
2997 
2998 	if (!psr->psr2_enabled)
2999 		return;
3000 
3001 	r = drm_dp_dpcd_readb(aux, DP_RECEIVER_ALPM_STATUS, &val);
3002 	if (r != 1) {
3003 		drm_err(&dev_priv->drm, "Error reading ALPM status\n");
3004 		return;
3005 	}
3006 
3007 	if (val & DP_ALPM_LOCK_TIMEOUT_ERROR) {
3008 		intel_psr_disable_locked(intel_dp);
3009 		psr->sink_not_reliable = true;
3010 		drm_dbg_kms(&dev_priv->drm,
3011 			    "ALPM lock timeout error, disabling PSR\n");
3012 
3013 		/* Clearing error */
3014 		drm_dp_dpcd_writeb(aux, DP_RECEIVER_ALPM_STATUS, val);
3015 	}
3016 }
3017 
3018 static void psr_capability_changed_check(struct intel_dp *intel_dp)
3019 {
3020 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
3021 	struct intel_psr *psr = &intel_dp->psr;
3022 	u8 val;
3023 	int r;
3024 
3025 	r = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ESI, &val);
3026 	if (r != 1) {
3027 		drm_err(&dev_priv->drm, "Error reading DP_PSR_ESI\n");
3028 		return;
3029 	}
3030 
3031 	if (val & DP_PSR_CAPS_CHANGE) {
3032 		intel_psr_disable_locked(intel_dp);
3033 		psr->sink_not_reliable = true;
3034 		drm_dbg_kms(&dev_priv->drm,
3035 			    "Sink PSR capability changed, disabling PSR\n");
3036 
3037 		/* Clearing it */
3038 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ESI, val);
3039 	}
3040 }
3041 
3042 void intel_psr_short_pulse(struct intel_dp *intel_dp)
3043 {
3044 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
3045 	struct intel_psr *psr = &intel_dp->psr;
3046 	u8 status, error_status;
3047 	const u8 errors = DP_PSR_RFB_STORAGE_ERROR |
3048 			  DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
3049 			  DP_PSR_LINK_CRC_ERROR;
3050 
3051 	if (!CAN_PSR(intel_dp))
3052 		return;
3053 
3054 	mutex_lock(&psr->lock);
3055 
3056 	if (!psr->enabled)
3057 		goto exit;
3058 
3059 	if (psr_get_status_and_error_status(intel_dp, &status, &error_status)) {
3060 		drm_err(&dev_priv->drm,
3061 			"Error reading PSR status or error status\n");
3062 		goto exit;
3063 	}
3064 
3065 	if (status == DP_PSR_SINK_INTERNAL_ERROR || (error_status & errors)) {
3066 		intel_psr_disable_locked(intel_dp);
3067 		psr->sink_not_reliable = true;
3068 	}
3069 
3070 	if (status == DP_PSR_SINK_INTERNAL_ERROR && !error_status)
3071 		drm_dbg_kms(&dev_priv->drm,
3072 			    "PSR sink internal error, disabling PSR\n");
3073 	if (error_status & DP_PSR_RFB_STORAGE_ERROR)
3074 		drm_dbg_kms(&dev_priv->drm,
3075 			    "PSR RFB storage error, disabling PSR\n");
3076 	if (error_status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
3077 		drm_dbg_kms(&dev_priv->drm,
3078 			    "PSR VSC SDP uncorrectable error, disabling PSR\n");
3079 	if (error_status & DP_PSR_LINK_CRC_ERROR)
3080 		drm_dbg_kms(&dev_priv->drm,
3081 			    "PSR Link CRC error, disabling PSR\n");
3082 
3083 	if (error_status & ~errors)
3084 		drm_err(&dev_priv->drm,
3085 			"PSR_ERROR_STATUS unhandled errors %x\n",
3086 			error_status & ~errors);
3087 	/* clear status register */
3088 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, error_status);
3089 
3090 	psr_alpm_check(intel_dp);
3091 	psr_capability_changed_check(intel_dp);
3092 
3093 exit:
3094 	mutex_unlock(&psr->lock);
3095 }
3096 
3097 bool intel_psr_enabled(struct intel_dp *intel_dp)
3098 {
3099 	bool ret;
3100 
3101 	if (!CAN_PSR(intel_dp))
3102 		return false;
3103 
3104 	mutex_lock(&intel_dp->psr.lock);
3105 	ret = intel_dp->psr.enabled;
3106 	mutex_unlock(&intel_dp->psr.lock);
3107 
3108 	return ret;
3109 }
3110 
3111 /**
3112  * intel_psr_lock - grab PSR lock
3113  * @crtc_state: the crtc state
3114  *
3115  * This is initially meant to be used by around CRTC update, when
3116  * vblank sensitive registers are updated and we need grab the lock
3117  * before it to avoid vblank evasion.
3118  */
3119 void intel_psr_lock(const struct intel_crtc_state *crtc_state)
3120 {
3121 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
3122 	struct intel_encoder *encoder;
3123 
3124 	if (!crtc_state->has_psr)
3125 		return;
3126 
3127 	for_each_intel_encoder_mask_with_psr(&i915->drm, encoder,
3128 					     crtc_state->uapi.encoder_mask) {
3129 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
3130 
3131 		mutex_lock(&intel_dp->psr.lock);
3132 		break;
3133 	}
3134 }
3135 
3136 /**
3137  * intel_psr_unlock - release PSR lock
3138  * @crtc_state: the crtc state
3139  *
3140  * Release the PSR lock that was held during pipe update.
3141  */
3142 void intel_psr_unlock(const struct intel_crtc_state *crtc_state)
3143 {
3144 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
3145 	struct intel_encoder *encoder;
3146 
3147 	if (!crtc_state->has_psr)
3148 		return;
3149 
3150 	for_each_intel_encoder_mask_with_psr(&i915->drm, encoder,
3151 					     crtc_state->uapi.encoder_mask) {
3152 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
3153 
3154 		mutex_unlock(&intel_dp->psr.lock);
3155 		break;
3156 	}
3157 }
3158 
3159 static void
3160 psr_source_status(struct intel_dp *intel_dp, struct seq_file *m)
3161 {
3162 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
3163 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
3164 	const char *status = "unknown";
3165 	u32 val, status_val;
3166 
3167 	if (intel_dp->psr.psr2_enabled) {
3168 		static const char * const live_status[] = {
3169 			"IDLE",
3170 			"CAPTURE",
3171 			"CAPTURE_FS",
3172 			"SLEEP",
3173 			"BUFON_FW",
3174 			"ML_UP",
3175 			"SU_STANDBY",
3176 			"FAST_SLEEP",
3177 			"DEEP_SLEEP",
3178 			"BUF_ON",
3179 			"TG_ON"
3180 		};
3181 		val = intel_de_read(dev_priv, EDP_PSR2_STATUS(cpu_transcoder));
3182 		status_val = REG_FIELD_GET(EDP_PSR2_STATUS_STATE_MASK, val);
3183 		if (status_val < ARRAY_SIZE(live_status))
3184 			status = live_status[status_val];
3185 	} else {
3186 		static const char * const live_status[] = {
3187 			"IDLE",
3188 			"SRDONACK",
3189 			"SRDENT",
3190 			"BUFOFF",
3191 			"BUFON",
3192 			"AUXACK",
3193 			"SRDOFFACK",
3194 			"SRDENT_ON",
3195 		};
3196 		val = intel_de_read(dev_priv, psr_status_reg(dev_priv, cpu_transcoder));
3197 		status_val = REG_FIELD_GET(EDP_PSR_STATUS_STATE_MASK, val);
3198 		if (status_val < ARRAY_SIZE(live_status))
3199 			status = live_status[status_val];
3200 	}
3201 
3202 	seq_printf(m, "Source PSR/PanelReplay status: %s [0x%08x]\n", status, val);
3203 }
3204 
3205 static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
3206 {
3207 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
3208 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
3209 	struct intel_psr *psr = &intel_dp->psr;
3210 	intel_wakeref_t wakeref;
3211 	const char *status;
3212 	bool enabled;
3213 	u32 val;
3214 
3215 	seq_printf(m, "Sink support: PSR = %s",
3216 		   str_yes_no(psr->sink_support));
3217 
3218 	if (psr->sink_support)
3219 		seq_printf(m, " [0x%02x]", intel_dp->psr_dpcd[0]);
3220 	seq_printf(m, ", Panel Replay = %s\n", str_yes_no(psr->sink_panel_replay_support));
3221 
3222 	if (!(psr->sink_support || psr->sink_panel_replay_support))
3223 		return 0;
3224 
3225 	wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
3226 	mutex_lock(&psr->lock);
3227 
3228 	if (psr->panel_replay_enabled)
3229 		status = "Panel Replay Enabled";
3230 	else if (psr->enabled)
3231 		status = psr->psr2_enabled ? "PSR2 enabled" : "PSR1 enabled";
3232 	else
3233 		status = "disabled";
3234 	seq_printf(m, "PSR mode: %s\n", status);
3235 
3236 	if (!psr->enabled) {
3237 		seq_printf(m, "PSR sink not reliable: %s\n",
3238 			   str_yes_no(psr->sink_not_reliable));
3239 
3240 		goto unlock;
3241 	}
3242 
3243 	if (psr->panel_replay_enabled) {
3244 		val = intel_de_read(dev_priv, TRANS_DP2_CTL(cpu_transcoder));
3245 		enabled = val & TRANS_DP2_PANEL_REPLAY_ENABLE;
3246 	} else if (psr->psr2_enabled) {
3247 		val = intel_de_read(dev_priv, EDP_PSR2_CTL(cpu_transcoder));
3248 		enabled = val & EDP_PSR2_ENABLE;
3249 	} else {
3250 		val = intel_de_read(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder));
3251 		enabled = val & EDP_PSR_ENABLE;
3252 	}
3253 	seq_printf(m, "Source PSR/PanelReplay ctl: %s [0x%08x]\n",
3254 		   str_enabled_disabled(enabled), val);
3255 	psr_source_status(intel_dp, m);
3256 	seq_printf(m, "Busy frontbuffer bits: 0x%08x\n",
3257 		   psr->busy_frontbuffer_bits);
3258 
3259 	/*
3260 	 * SKL+ Perf counter is reset to 0 everytime DC state is entered
3261 	 */
3262 	val = intel_de_read(dev_priv, psr_perf_cnt_reg(dev_priv, cpu_transcoder));
3263 	seq_printf(m, "Performance counter: %u\n",
3264 		   REG_FIELD_GET(EDP_PSR_PERF_CNT_MASK, val));
3265 
3266 	if (psr->debug & I915_PSR_DEBUG_IRQ) {
3267 		seq_printf(m, "Last attempted entry at: %lld\n",
3268 			   psr->last_entry_attempt);
3269 		seq_printf(m, "Last exit at: %lld\n", psr->last_exit);
3270 	}
3271 
3272 	if (psr->psr2_enabled) {
3273 		u32 su_frames_val[3];
3274 		int frame;
3275 
3276 		/*
3277 		 * Reading all 3 registers before hand to minimize crossing a
3278 		 * frame boundary between register reads
3279 		 */
3280 		for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame += 3) {
3281 			val = intel_de_read(dev_priv, PSR2_SU_STATUS(cpu_transcoder, frame));
3282 			su_frames_val[frame / 3] = val;
3283 		}
3284 
3285 		seq_puts(m, "Frame:\tPSR2 SU blocks:\n");
3286 
3287 		for (frame = 0; frame < PSR2_SU_STATUS_FRAMES; frame++) {
3288 			u32 su_blocks;
3289 
3290 			su_blocks = su_frames_val[frame / 3] &
3291 				    PSR2_SU_STATUS_MASK(frame);
3292 			su_blocks = su_blocks >> PSR2_SU_STATUS_SHIFT(frame);
3293 			seq_printf(m, "%d\t%d\n", frame, su_blocks);
3294 		}
3295 
3296 		seq_printf(m, "PSR2 selective fetch: %s\n",
3297 			   str_enabled_disabled(psr->psr2_sel_fetch_enabled));
3298 	}
3299 
3300 unlock:
3301 	mutex_unlock(&psr->lock);
3302 	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
3303 
3304 	return 0;
3305 }
3306 
3307 static int i915_edp_psr_status_show(struct seq_file *m, void *data)
3308 {
3309 	struct drm_i915_private *dev_priv = m->private;
3310 	struct intel_dp *intel_dp = NULL;
3311 	struct intel_encoder *encoder;
3312 
3313 	if (!HAS_PSR(dev_priv))
3314 		return -ENODEV;
3315 
3316 	/* Find the first EDP which supports PSR */
3317 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
3318 		intel_dp = enc_to_intel_dp(encoder);
3319 		break;
3320 	}
3321 
3322 	if (!intel_dp)
3323 		return -ENODEV;
3324 
3325 	return intel_psr_status(m, intel_dp);
3326 }
3327 DEFINE_SHOW_ATTRIBUTE(i915_edp_psr_status);
3328 
3329 static int
3330 i915_edp_psr_debug_set(void *data, u64 val)
3331 {
3332 	struct drm_i915_private *dev_priv = data;
3333 	struct intel_encoder *encoder;
3334 	intel_wakeref_t wakeref;
3335 	int ret = -ENODEV;
3336 
3337 	if (!HAS_PSR(dev_priv))
3338 		return ret;
3339 
3340 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
3341 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
3342 
3343 		drm_dbg_kms(&dev_priv->drm, "Setting PSR debug to %llx\n", val);
3344 
3345 		wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
3346 
3347 		// TODO: split to each transcoder's PSR debug state
3348 		ret = intel_psr_debug_set(intel_dp, val);
3349 
3350 		intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
3351 	}
3352 
3353 	return ret;
3354 }
3355 
3356 static int
3357 i915_edp_psr_debug_get(void *data, u64 *val)
3358 {
3359 	struct drm_i915_private *dev_priv = data;
3360 	struct intel_encoder *encoder;
3361 
3362 	if (!HAS_PSR(dev_priv))
3363 		return -ENODEV;
3364 
3365 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
3366 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
3367 
3368 		// TODO: split to each transcoder's PSR debug state
3369 		*val = READ_ONCE(intel_dp->psr.debug);
3370 		return 0;
3371 	}
3372 
3373 	return -ENODEV;
3374 }
3375 
3376 DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops,
3377 			i915_edp_psr_debug_get, i915_edp_psr_debug_set,
3378 			"%llu\n");
3379 
3380 void intel_psr_debugfs_register(struct drm_i915_private *i915)
3381 {
3382 	struct drm_minor *minor = i915->drm.primary;
3383 
3384 	debugfs_create_file("i915_edp_psr_debug", 0644, minor->debugfs_root,
3385 			    i915, &i915_edp_psr_debug_fops);
3386 
3387 	debugfs_create_file("i915_edp_psr_status", 0444, minor->debugfs_root,
3388 			    i915, &i915_edp_psr_status_fops);
3389 }
3390 
3391 static const char *psr_mode_str(struct intel_dp *intel_dp)
3392 {
3393 	if (intel_dp->psr.panel_replay_enabled)
3394 		return "PANEL-REPLAY";
3395 	else if (intel_dp->psr.enabled)
3396 		return "PSR";
3397 
3398 	return "unknown";
3399 }
3400 
3401 static int i915_psr_sink_status_show(struct seq_file *m, void *data)
3402 {
3403 	struct intel_connector *connector = m->private;
3404 	struct intel_dp *intel_dp = intel_attached_dp(connector);
3405 	static const char * const sink_status[] = {
3406 		"inactive",
3407 		"transition to active, capture and display",
3408 		"active, display from RFB",
3409 		"active, capture and display on sink device timings",
3410 		"transition to inactive, capture and display, timing re-sync",
3411 		"reserved",
3412 		"reserved",
3413 		"sink internal error",
3414 	};
3415 	static const char * const panel_replay_status[] = {
3416 		"Sink device frame is locked to the Source device",
3417 		"Sink device is coasting, using the VTotal target",
3418 		"Sink device is governing the frame rate (frame rate unlock is granted)",
3419 		"Sink device in the process of re-locking with the Source device",
3420 	};
3421 	const char *str;
3422 	int ret;
3423 	u8 status, error_status;
3424 	u32 idx;
3425 
3426 	if (!(CAN_PSR(intel_dp) || CAN_PANEL_REPLAY(intel_dp))) {
3427 		seq_puts(m, "PSR/Panel-Replay Unsupported\n");
3428 		return -ENODEV;
3429 	}
3430 
3431 	if (connector->base.status != connector_status_connected)
3432 		return -ENODEV;
3433 
3434 	ret = psr_get_status_and_error_status(intel_dp, &status, &error_status);
3435 	if (ret)
3436 		return ret;
3437 
3438 	str = "unknown";
3439 	if (intel_dp->psr.panel_replay_enabled) {
3440 		idx = (status & DP_SINK_FRAME_LOCKED_MASK) >> DP_SINK_FRAME_LOCKED_SHIFT;
3441 		if (idx < ARRAY_SIZE(panel_replay_status))
3442 			str = panel_replay_status[idx];
3443 	} else if (intel_dp->psr.enabled) {
3444 		idx = status & DP_PSR_SINK_STATE_MASK;
3445 		if (idx < ARRAY_SIZE(sink_status))
3446 			str = sink_status[idx];
3447 	}
3448 
3449 	seq_printf(m, "Sink %s status: 0x%x [%s]\n", psr_mode_str(intel_dp), status, str);
3450 
3451 	seq_printf(m, "Sink %s error status: 0x%x", psr_mode_str(intel_dp), error_status);
3452 
3453 	if (error_status & (DP_PSR_RFB_STORAGE_ERROR |
3454 			    DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
3455 			    DP_PSR_LINK_CRC_ERROR))
3456 		seq_puts(m, ":\n");
3457 	else
3458 		seq_puts(m, "\n");
3459 	if (error_status & DP_PSR_RFB_STORAGE_ERROR)
3460 		seq_printf(m, "\t%s RFB storage error\n", psr_mode_str(intel_dp));
3461 	if (error_status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
3462 		seq_printf(m, "\t%s VSC SDP uncorrectable error\n", psr_mode_str(intel_dp));
3463 	if (error_status & DP_PSR_LINK_CRC_ERROR)
3464 		seq_printf(m, "\t%s Link CRC error\n", psr_mode_str(intel_dp));
3465 
3466 	return ret;
3467 }
3468 DEFINE_SHOW_ATTRIBUTE(i915_psr_sink_status);
3469 
3470 static int i915_psr_status_show(struct seq_file *m, void *data)
3471 {
3472 	struct intel_connector *connector = m->private;
3473 	struct intel_dp *intel_dp = intel_attached_dp(connector);
3474 
3475 	return intel_psr_status(m, intel_dp);
3476 }
3477 DEFINE_SHOW_ATTRIBUTE(i915_psr_status);
3478 
3479 void intel_psr_connector_debugfs_add(struct intel_connector *connector)
3480 {
3481 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
3482 	struct dentry *root = connector->base.debugfs_entry;
3483 
3484 	/* TODO: Add support for MST connectors as well. */
3485 	if ((connector->base.connector_type != DRM_MODE_CONNECTOR_eDP &&
3486 	     connector->base.connector_type != DRM_MODE_CONNECTOR_DisplayPort) ||
3487 	    connector->mst_port)
3488 		return;
3489 
3490 	debugfs_create_file("i915_psr_sink_status", 0444, root,
3491 			    connector, &i915_psr_sink_status_fops);
3492 
3493 	if (HAS_PSR(i915) || HAS_DP20(i915))
3494 		debugfs_create_file("i915_psr_status", 0444, root,
3495 				    connector, &i915_psr_status_fops);
3496 }
3497