xref: /linux/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c (revision fb7399cf2d0b33825b8039f95c45395c7deba25c)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
4  * Copyright (c) 2014-2021 The Linux Foundation. All rights reserved.
5  * Copyright (C) 2013 Red Hat
6  * Author: Rob Clark <robdclark@gmail.com>
7  */
8 
9 #define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
10 #include <linux/sort.h>
11 #include <linux/debugfs.h>
12 #include <linux/ktime.h>
13 #include <linux/bits.h>
14 
15 #include <drm/drm_atomic.h>
16 #include <drm/drm_blend.h>
17 #include <drm/drm_crtc.h>
18 #include <drm/drm_flip_work.h>
19 #include <drm/drm_framebuffer.h>
20 #include <drm/drm_mode.h>
21 #include <drm/drm_probe_helper.h>
22 #include <drm/drm_rect.h>
23 #include <drm/drm_vblank.h>
24 #include <drm/drm_self_refresh_helper.h>
25 
26 #include "dpu_kms.h"
27 #include "dpu_hw_lm.h"
28 #include "dpu_hw_ctl.h"
29 #include "dpu_hw_dspp.h"
30 #include "dpu_crtc.h"
31 #include "dpu_plane.h"
32 #include "dpu_encoder.h"
33 #include "dpu_vbif.h"
34 #include "dpu_core_perf.h"
35 #include "dpu_trace.h"
36 
37 /* layer mixer index on dpu_crtc */
38 #define LEFT_MIXER 0
39 #define RIGHT_MIXER 1
40 
41 /* timeout in ms waiting for frame done */
42 #define DPU_CRTC_FRAME_DONE_TIMEOUT_MS	60
43 
44 #define	CONVERT_S3_15(val) \
45 	(((((u64)val) & ~BIT_ULL(63)) >> 17) & GENMASK_ULL(17, 0))
46 
47 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
48 {
49 	struct msm_drm_private *priv = crtc->dev->dev_private;
50 
51 	return to_dpu_kms(priv->kms);
52 }
53 
54 static struct drm_encoder *get_encoder_from_crtc(struct drm_crtc *crtc)
55 {
56 	struct drm_device *dev = crtc->dev;
57 	struct drm_encoder *encoder;
58 
59 	drm_for_each_encoder(encoder, dev)
60 		if (encoder->crtc == crtc)
61 			return encoder;
62 
63 	return NULL;
64 }
65 
66 static enum dpu_crtc_crc_source dpu_crtc_parse_crc_source(const char *src_name)
67 {
68 	if (!src_name ||
69 	    !strcmp(src_name, "none"))
70 		return DPU_CRTC_CRC_SOURCE_NONE;
71 	if (!strcmp(src_name, "auto") ||
72 	    !strcmp(src_name, "lm"))
73 		return DPU_CRTC_CRC_SOURCE_LAYER_MIXER;
74 	if (!strcmp(src_name, "encoder"))
75 		return DPU_CRTC_CRC_SOURCE_ENCODER;
76 
77 	return DPU_CRTC_CRC_SOURCE_INVALID;
78 }
79 
80 static int dpu_crtc_verify_crc_source(struct drm_crtc *crtc,
81 		const char *src_name, size_t *values_cnt)
82 {
83 	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
84 	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
85 
86 	if (source < 0) {
87 		DRM_DEBUG_DRIVER("Invalid source %s for CRTC%d\n", src_name, crtc->index);
88 		return -EINVAL;
89 	}
90 
91 	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER) {
92 		*values_cnt = crtc_state->num_mixers;
93 	} else if (source == DPU_CRTC_CRC_SOURCE_ENCODER) {
94 		struct drm_encoder *drm_enc;
95 
96 		*values_cnt = 0;
97 
98 		drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
99 			*values_cnt += dpu_encoder_get_crc_values_cnt(drm_enc);
100 	}
101 
102 	return 0;
103 }
104 
105 static void dpu_crtc_setup_lm_misr(struct dpu_crtc_state *crtc_state)
106 {
107 	struct dpu_crtc_mixer *m;
108 	int i;
109 
110 	for (i = 0; i < crtc_state->num_mixers; ++i) {
111 		m = &crtc_state->mixers[i];
112 
113 		if (!m->hw_lm || !m->hw_lm->ops.setup_misr)
114 			continue;
115 
116 		/* Calculate MISR over 1 frame */
117 		m->hw_lm->ops.setup_misr(m->hw_lm);
118 	}
119 }
120 
121 static void dpu_crtc_setup_encoder_misr(struct drm_crtc *crtc)
122 {
123 	struct drm_encoder *drm_enc;
124 
125 	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask)
126 		dpu_encoder_setup_misr(drm_enc);
127 }
128 
129 static int dpu_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
130 {
131 	enum dpu_crtc_crc_source source = dpu_crtc_parse_crc_source(src_name);
132 	enum dpu_crtc_crc_source current_source;
133 	struct dpu_crtc_state *crtc_state;
134 	struct drm_device *drm_dev = crtc->dev;
135 
136 	bool was_enabled;
137 	bool enable = false;
138 	int ret = 0;
139 
140 	if (source < 0) {
141 		DRM_DEBUG_DRIVER("Invalid CRC source %s for CRTC%d\n", src_name, crtc->index);
142 		return -EINVAL;
143 	}
144 
145 	ret = drm_modeset_lock(&crtc->mutex, NULL);
146 
147 	if (ret)
148 		return ret;
149 
150 	enable = (source != DPU_CRTC_CRC_SOURCE_NONE);
151 	crtc_state = to_dpu_crtc_state(crtc->state);
152 
153 	spin_lock_irq(&drm_dev->event_lock);
154 	current_source = crtc_state->crc_source;
155 	spin_unlock_irq(&drm_dev->event_lock);
156 
157 	was_enabled = (current_source != DPU_CRTC_CRC_SOURCE_NONE);
158 
159 	if (!was_enabled && enable) {
160 		ret = drm_crtc_vblank_get(crtc);
161 
162 		if (ret)
163 			goto cleanup;
164 
165 	} else if (was_enabled && !enable) {
166 		drm_crtc_vblank_put(crtc);
167 	}
168 
169 	spin_lock_irq(&drm_dev->event_lock);
170 	crtc_state->crc_source = source;
171 	spin_unlock_irq(&drm_dev->event_lock);
172 
173 	crtc_state->crc_frame_skip_count = 0;
174 
175 	if (source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
176 		dpu_crtc_setup_lm_misr(crtc_state);
177 	else if (source == DPU_CRTC_CRC_SOURCE_ENCODER)
178 		dpu_crtc_setup_encoder_misr(crtc);
179 	else
180 		ret = -EINVAL;
181 
182 cleanup:
183 	drm_modeset_unlock(&crtc->mutex);
184 
185 	return ret;
186 }
187 
188 static u32 dpu_crtc_get_vblank_counter(struct drm_crtc *crtc)
189 {
190 	struct drm_encoder *encoder = get_encoder_from_crtc(crtc);
191 	if (!encoder) {
192 		DRM_ERROR("no encoder found for crtc %d\n", crtc->index);
193 		return 0;
194 	}
195 
196 	return dpu_encoder_get_vsync_count(encoder);
197 }
198 
199 static int dpu_crtc_get_lm_crc(struct drm_crtc *crtc,
200 		struct dpu_crtc_state *crtc_state)
201 {
202 	struct dpu_crtc_mixer *m;
203 	u32 crcs[CRTC_DUAL_MIXERS];
204 
205 	int rc = 0;
206 	int i;
207 
208 	BUILD_BUG_ON(ARRAY_SIZE(crcs) != ARRAY_SIZE(crtc_state->mixers));
209 
210 	for (i = 0; i < crtc_state->num_mixers; ++i) {
211 
212 		m = &crtc_state->mixers[i];
213 
214 		if (!m->hw_lm || !m->hw_lm->ops.collect_misr)
215 			continue;
216 
217 		rc = m->hw_lm->ops.collect_misr(m->hw_lm, &crcs[i]);
218 
219 		if (rc) {
220 			if (rc != -ENODATA)
221 				DRM_DEBUG_DRIVER("MISR read failed\n");
222 			return rc;
223 		}
224 	}
225 
226 	return drm_crtc_add_crc_entry(crtc, true,
227 			drm_crtc_accurate_vblank_count(crtc), crcs);
228 }
229 
230 static int dpu_crtc_get_encoder_crc(struct drm_crtc *crtc)
231 {
232 	struct drm_encoder *drm_enc;
233 	int rc, pos = 0;
234 	u32 crcs[INTF_MAX];
235 
236 	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc->state->encoder_mask) {
237 		rc = dpu_encoder_get_crc(drm_enc, crcs, pos);
238 		if (rc < 0) {
239 			if (rc != -ENODATA)
240 				DRM_DEBUG_DRIVER("MISR read failed\n");
241 
242 			return rc;
243 		}
244 
245 		pos += rc;
246 	}
247 
248 	return drm_crtc_add_crc_entry(crtc, true,
249 			drm_crtc_accurate_vblank_count(crtc), crcs);
250 }
251 
252 static int dpu_crtc_get_crc(struct drm_crtc *crtc)
253 {
254 	struct dpu_crtc_state *crtc_state = to_dpu_crtc_state(crtc->state);
255 
256 	/* Skip first 2 frames in case of "uncooked" CRCs */
257 	if (crtc_state->crc_frame_skip_count < 2) {
258 		crtc_state->crc_frame_skip_count++;
259 		return 0;
260 	}
261 
262 	if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_LAYER_MIXER)
263 		return dpu_crtc_get_lm_crc(crtc, crtc_state);
264 	else if (crtc_state->crc_source == DPU_CRTC_CRC_SOURCE_ENCODER)
265 		return dpu_crtc_get_encoder_crc(crtc);
266 
267 	return -EINVAL;
268 }
269 
270 static bool dpu_crtc_get_scanout_position(struct drm_crtc *crtc,
271 					   bool in_vblank_irq,
272 					   int *vpos, int *hpos,
273 					   ktime_t *stime, ktime_t *etime,
274 					   const struct drm_display_mode *mode)
275 {
276 	unsigned int pipe = crtc->index;
277 	struct drm_encoder *encoder;
278 	int line, vsw, vbp, vactive_start, vactive_end, vfp_end;
279 
280 	encoder = get_encoder_from_crtc(crtc);
281 	if (!encoder) {
282 		DRM_ERROR("no encoder found for crtc %d\n", pipe);
283 		return false;
284 	}
285 
286 	vsw = mode->crtc_vsync_end - mode->crtc_vsync_start;
287 	vbp = mode->crtc_vtotal - mode->crtc_vsync_end;
288 
289 	/*
290 	 * the line counter is 1 at the start of the VSYNC pulse and VTOTAL at
291 	 * the end of VFP. Translate the porch values relative to the line
292 	 * counter positions.
293 	 */
294 
295 	vactive_start = vsw + vbp + 1;
296 	vactive_end = vactive_start + mode->crtc_vdisplay;
297 
298 	/* last scan line before VSYNC */
299 	vfp_end = mode->crtc_vtotal;
300 
301 	if (stime)
302 		*stime = ktime_get();
303 
304 	line = dpu_encoder_get_linecount(encoder);
305 
306 	if (line < vactive_start)
307 		line -= vactive_start;
308 	else if (line > vactive_end)
309 		line = line - vfp_end - vactive_start;
310 	else
311 		line -= vactive_start;
312 
313 	*vpos = line;
314 	*hpos = 0;
315 
316 	if (etime)
317 		*etime = ktime_get();
318 
319 	return true;
320 }
321 
322 static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
323 		struct dpu_plane_state *pstate, const struct msm_format *format)
324 {
325 	struct dpu_hw_mixer *lm = mixer->hw_lm;
326 	uint32_t blend_op;
327 	uint32_t fg_alpha, bg_alpha;
328 
329 	fg_alpha = pstate->base.alpha >> 8;
330 	bg_alpha = 0xff - fg_alpha;
331 
332 	/* default to opaque blending */
333 	if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE ||
334 	    !format->alpha_enable) {
335 		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
336 			DPU_BLEND_BG_ALPHA_BG_CONST;
337 	} else if (pstate->base.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) {
338 		blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
339 			DPU_BLEND_BG_ALPHA_FG_PIXEL;
340 		if (fg_alpha != 0xff) {
341 			bg_alpha = fg_alpha;
342 			blend_op |= DPU_BLEND_BG_MOD_ALPHA |
343 				    DPU_BLEND_BG_INV_MOD_ALPHA;
344 		} else {
345 			blend_op |= DPU_BLEND_BG_INV_ALPHA;
346 		}
347 	} else {
348 		/* coverage blending */
349 		blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL |
350 			DPU_BLEND_BG_ALPHA_FG_PIXEL;
351 		if (fg_alpha != 0xff) {
352 			bg_alpha = fg_alpha;
353 			blend_op |= DPU_BLEND_FG_MOD_ALPHA |
354 				    DPU_BLEND_FG_INV_MOD_ALPHA |
355 				    DPU_BLEND_BG_MOD_ALPHA |
356 				    DPU_BLEND_BG_INV_MOD_ALPHA;
357 		} else {
358 			blend_op |= DPU_BLEND_BG_INV_ALPHA;
359 		}
360 	}
361 
362 	lm->ops.setup_blend_config(lm, pstate->stage,
363 				fg_alpha, bg_alpha, blend_op);
364 
365 	DRM_DEBUG_ATOMIC("format:%p4cc, alpha_en:%u blend_op:0x%x\n",
366 		  &format->pixel_format, format->alpha_enable, blend_op);
367 }
368 
369 static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
370 {
371 	struct dpu_crtc_state *crtc_state;
372 	int lm_idx, lm_horiz_position;
373 
374 	crtc_state = to_dpu_crtc_state(crtc->state);
375 
376 	lm_horiz_position = 0;
377 	for (lm_idx = 0; lm_idx < crtc_state->num_mixers; lm_idx++) {
378 		const struct drm_rect *lm_roi = &crtc_state->lm_bounds[lm_idx];
379 		struct dpu_hw_mixer *hw_lm = crtc_state->mixers[lm_idx].hw_lm;
380 		struct dpu_hw_mixer_cfg cfg;
381 
382 		if (!lm_roi || !drm_rect_visible(lm_roi))
383 			continue;
384 
385 		cfg.out_width = drm_rect_width(lm_roi);
386 		cfg.out_height = drm_rect_height(lm_roi);
387 		cfg.right_mixer = lm_horiz_position++;
388 		cfg.flags = 0;
389 		hw_lm->ops.setup_mixer_out(hw_lm, &cfg);
390 	}
391 }
392 
393 static void _dpu_crtc_blend_setup_pipe(struct drm_crtc *crtc,
394 				       struct drm_plane *plane,
395 				       struct dpu_crtc_mixer *mixer,
396 				       u32 num_mixers,
397 				       enum dpu_stage stage,
398 				       const struct msm_format *format,
399 				       uint64_t modifier,
400 				       struct dpu_sw_pipe *pipe,
401 				       unsigned int stage_idx,
402 				       struct dpu_hw_stage_cfg *stage_cfg
403 				      )
404 {
405 	uint32_t lm_idx;
406 	enum dpu_sspp sspp_idx;
407 	struct drm_plane_state *state;
408 
409 	sspp_idx = pipe->sspp->idx;
410 
411 	state = plane->state;
412 
413 	trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
414 				   state, to_dpu_plane_state(state), stage_idx,
415 				   format->pixel_format,
416 				   modifier);
417 
418 	DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d multirect_idx %d\n",
419 			 crtc->base.id,
420 			 stage,
421 			 plane->base.id,
422 			 sspp_idx - SSPP_NONE,
423 			 state->fb ? state->fb->base.id : -1,
424 			 pipe->multirect_index);
425 
426 	stage_cfg->stage[stage][stage_idx] = sspp_idx;
427 	stage_cfg->multirect_index[stage][stage_idx] = pipe->multirect_index;
428 
429 	/* blend config update */
430 	for (lm_idx = 0; lm_idx < num_mixers; lm_idx++)
431 		mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl, sspp_idx);
432 }
433 
434 static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
435 	struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer,
436 	struct dpu_hw_stage_cfg *stage_cfg)
437 {
438 	struct drm_plane *plane;
439 	struct drm_framebuffer *fb;
440 	struct drm_plane_state *state;
441 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
442 	struct dpu_plane_state *pstate = NULL;
443 	const struct msm_format *format;
444 	struct dpu_hw_ctl *ctl = mixer->lm_ctl;
445 
446 	uint32_t lm_idx;
447 	bool bg_alpha_enable = false;
448 	DECLARE_BITMAP(active_fetch, SSPP_MAX);
449 
450 	memset(active_fetch, 0, sizeof(active_fetch));
451 	drm_atomic_crtc_for_each_plane(plane, crtc) {
452 		state = plane->state;
453 		if (!state)
454 			continue;
455 
456 		if (!state->visible)
457 			continue;
458 
459 		pstate = to_dpu_plane_state(state);
460 		fb = state->fb;
461 
462 		format = msm_framebuffer_format(pstate->base.fb);
463 
464 		if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
465 			bg_alpha_enable = true;
466 
467 		set_bit(pstate->pipe.sspp->idx, active_fetch);
468 		_dpu_crtc_blend_setup_pipe(crtc, plane,
469 					   mixer, cstate->num_mixers,
470 					   pstate->stage,
471 					   format, fb ? fb->modifier : 0,
472 					   &pstate->pipe, 0, stage_cfg);
473 
474 		if (pstate->r_pipe.sspp) {
475 			set_bit(pstate->r_pipe.sspp->idx, active_fetch);
476 			_dpu_crtc_blend_setup_pipe(crtc, plane,
477 						   mixer, cstate->num_mixers,
478 						   pstate->stage,
479 						   format, fb ? fb->modifier : 0,
480 						   &pstate->r_pipe, 1, stage_cfg);
481 		}
482 
483 		/* blend config update */
484 		for (lm_idx = 0; lm_idx < cstate->num_mixers; lm_idx++) {
485 			_dpu_crtc_setup_blend_cfg(mixer + lm_idx, pstate, format);
486 
487 			if (bg_alpha_enable && !format->alpha_enable)
488 				mixer[lm_idx].mixer_op_mode = 0;
489 			else
490 				mixer[lm_idx].mixer_op_mode |=
491 						1 << pstate->stage;
492 		}
493 	}
494 
495 	if (ctl->ops.set_active_fetch_pipes)
496 		ctl->ops.set_active_fetch_pipes(ctl, active_fetch);
497 
498 	_dpu_crtc_program_lm_output_roi(crtc);
499 }
500 
501 /**
502  * _dpu_crtc_blend_setup - configure crtc mixers
503  * @crtc: Pointer to drm crtc structure
504  */
505 static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
506 {
507 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
508 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
509 	struct dpu_crtc_mixer *mixer = cstate->mixers;
510 	struct dpu_hw_ctl *ctl;
511 	struct dpu_hw_mixer *lm;
512 	struct dpu_hw_stage_cfg stage_cfg;
513 	int i;
514 
515 	DRM_DEBUG_ATOMIC("%s\n", dpu_crtc->name);
516 
517 	for (i = 0; i < cstate->num_mixers; i++) {
518 		mixer[i].mixer_op_mode = 0;
519 		if (mixer[i].lm_ctl->ops.clear_all_blendstages)
520 			mixer[i].lm_ctl->ops.clear_all_blendstages(
521 					mixer[i].lm_ctl);
522 		if (mixer[i].lm_ctl->ops.set_active_fetch_pipes)
523 			mixer[i].lm_ctl->ops.set_active_fetch_pipes(mixer[i].lm_ctl, NULL);
524 	}
525 
526 	/* initialize stage cfg */
527 	memset(&stage_cfg, 0, sizeof(struct dpu_hw_stage_cfg));
528 
529 	_dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer, &stage_cfg);
530 
531 	for (i = 0; i < cstate->num_mixers; i++) {
532 		ctl = mixer[i].lm_ctl;
533 		lm = mixer[i].hw_lm;
534 
535 		lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
536 
537 		/* stage config flush mask */
538 		ctl->ops.update_pending_flush_mixer(ctl,
539 			mixer[i].hw_lm->idx);
540 
541 		DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d\n",
542 			mixer[i].hw_lm->idx - LM_0,
543 			mixer[i].mixer_op_mode,
544 			ctl->idx - CTL_0);
545 
546 		ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
547 			&stage_cfg);
548 	}
549 }
550 
551 /**
552  *  _dpu_crtc_complete_flip - signal pending page_flip events
553  * Any pending vblank events are added to the vblank_event_list
554  * so that the next vblank interrupt shall signal them.
555  * However PAGE_FLIP events are not handled through the vblank_event_list.
556  * This API signals any pending PAGE_FLIP events requested through
557  * DRM_IOCTL_MODE_PAGE_FLIP and are cached in the dpu_crtc->event.
558  * @crtc: Pointer to drm crtc structure
559  */
560 static void _dpu_crtc_complete_flip(struct drm_crtc *crtc)
561 {
562 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
563 	struct drm_device *dev = crtc->dev;
564 	unsigned long flags;
565 
566 	spin_lock_irqsave(&dev->event_lock, flags);
567 	if (dpu_crtc->event) {
568 		DRM_DEBUG_VBL("%s: send event: %pK\n", dpu_crtc->name,
569 			      dpu_crtc->event);
570 		trace_dpu_crtc_complete_flip(DRMID(crtc));
571 		drm_crtc_send_vblank_event(crtc, dpu_crtc->event);
572 		dpu_crtc->event = NULL;
573 	}
574 	spin_unlock_irqrestore(&dev->event_lock, flags);
575 }
576 
577 /**
578  * dpu_crtc_get_intf_mode - get interface mode of the given crtc
579  * @crtc: Pointert to crtc
580  */
581 enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc)
582 {
583 	struct drm_encoder *encoder;
584 
585 	/*
586 	 * TODO: This function is called from dpu debugfs and as part of atomic
587 	 * check. When called from debugfs, the crtc->mutex must be held to
588 	 * read crtc->state. However reading crtc->state from atomic check isn't
589 	 * allowed (unless you have a good reason, a big comment, and a deep
590 	 * understanding of how the atomic/modeset locks work (<- and this is
591 	 * probably not possible)). So we'll keep the WARN_ON here for now, but
592 	 * really we need to figure out a better way to track our operating mode
593 	 */
594 	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
595 
596 	/* TODO: Returns the first INTF_MODE, could there be multiple values? */
597 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
598 		return dpu_encoder_get_intf_mode(encoder);
599 
600 	return INTF_MODE_NONE;
601 }
602 
603 /**
604  * dpu_crtc_vblank_callback - called on vblank irq, issues completion events
605  * @crtc: Pointer to drm crtc object
606  */
607 void dpu_crtc_vblank_callback(struct drm_crtc *crtc)
608 {
609 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
610 
611 	/* keep statistics on vblank callback - with auto reset via debugfs */
612 	if (ktime_compare(dpu_crtc->vblank_cb_time, ktime_set(0, 0)) == 0)
613 		dpu_crtc->vblank_cb_time = ktime_get();
614 	else
615 		dpu_crtc->vblank_cb_count++;
616 
617 	dpu_crtc_get_crc(crtc);
618 
619 	drm_crtc_handle_vblank(crtc);
620 	trace_dpu_crtc_vblank_cb(DRMID(crtc));
621 }
622 
623 static void dpu_crtc_frame_event_work(struct kthread_work *work)
624 {
625 	struct dpu_crtc_frame_event *fevent = container_of(work,
626 			struct dpu_crtc_frame_event, work);
627 	struct drm_crtc *crtc = fevent->crtc;
628 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
629 	unsigned long flags;
630 	bool frame_done = false;
631 
632 	DPU_ATRACE_BEGIN("crtc_frame_event");
633 
634 	DRM_DEBUG_ATOMIC("crtc%d event:%u ts:%lld\n", crtc->base.id, fevent->event,
635 			ktime_to_ns(fevent->ts));
636 
637 	if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
638 				| DPU_ENCODER_FRAME_EVENT_ERROR
639 				| DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
640 
641 		if (atomic_read(&dpu_crtc->frame_pending) < 1) {
642 			/* ignore vblank when not pending */
643 		} else if (atomic_dec_return(&dpu_crtc->frame_pending) == 0) {
644 			/* release bandwidth and other resources */
645 			trace_dpu_crtc_frame_event_done(DRMID(crtc),
646 							fevent->event);
647 			dpu_core_perf_crtc_release_bw(crtc);
648 		} else {
649 			trace_dpu_crtc_frame_event_more_pending(DRMID(crtc),
650 								fevent->event);
651 		}
652 
653 		if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
654 					| DPU_ENCODER_FRAME_EVENT_ERROR))
655 			frame_done = true;
656 	}
657 
658 	if (fevent->event & DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)
659 		DPU_ERROR("crtc%d ts:%lld received panel dead event\n",
660 				crtc->base.id, ktime_to_ns(fevent->ts));
661 
662 	if (frame_done)
663 		complete_all(&dpu_crtc->frame_done_comp);
664 
665 	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
666 	list_add_tail(&fevent->list, &dpu_crtc->frame_event_list);
667 	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
668 	DPU_ATRACE_END("crtc_frame_event");
669 }
670 
671 /**
672  * dpu_crtc_frame_event_cb - crtc frame event callback API
673  * @crtc: Pointer to crtc
674  * @event: Event to process
675  *
676  * Encoder may call this for different events from different context - IRQ,
677  * user thread, commit_thread, etc. Each event should be carefully reviewed and
678  * should be processed in proper task context to avoid schedulin delay or
679  * properly manage the irq context's bottom half processing.
680  */
681 void dpu_crtc_frame_event_cb(struct drm_crtc *crtc, u32 event)
682 {
683 	struct dpu_crtc *dpu_crtc;
684 	struct msm_drm_private *priv;
685 	struct dpu_crtc_frame_event *fevent;
686 	unsigned long flags;
687 	u32 crtc_id;
688 
689 	/* Nothing to do on idle event */
690 	if (event & DPU_ENCODER_FRAME_EVENT_IDLE)
691 		return;
692 
693 	dpu_crtc = to_dpu_crtc(crtc);
694 	priv = crtc->dev->dev_private;
695 	crtc_id = drm_crtc_index(crtc);
696 
697 	trace_dpu_crtc_frame_event_cb(DRMID(crtc), event);
698 
699 	spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
700 	fevent = list_first_entry_or_null(&dpu_crtc->frame_event_list,
701 			struct dpu_crtc_frame_event, list);
702 	if (fevent)
703 		list_del_init(&fevent->list);
704 	spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
705 
706 	if (!fevent) {
707 		DRM_ERROR_RATELIMITED("crtc%d event %d overflow\n", crtc->base.id, event);
708 		return;
709 	}
710 
711 	fevent->event = event;
712 	fevent->crtc = crtc;
713 	fevent->ts = ktime_get();
714 	kthread_queue_work(priv->event_thread[crtc_id].worker, &fevent->work);
715 }
716 
717 /**
718  * dpu_crtc_complete_commit - callback signalling completion of current commit
719  * @crtc: Pointer to drm crtc object
720  */
721 void dpu_crtc_complete_commit(struct drm_crtc *crtc)
722 {
723 	trace_dpu_crtc_complete_commit(DRMID(crtc));
724 	dpu_core_perf_crtc_update(crtc, 0);
725 	_dpu_crtc_complete_flip(crtc);
726 }
727 
728 static int _dpu_crtc_check_and_setup_lm_bounds(struct drm_crtc *crtc,
729 		struct drm_crtc_state *state)
730 {
731 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
732 	struct drm_display_mode *adj_mode = &state->adjusted_mode;
733 	u32 crtc_split_width = adj_mode->hdisplay / cstate->num_mixers;
734 	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
735 	int i;
736 
737 	/* if we cannot merge 2 LMs (no 3d mux) better to fail earlier
738 	 * before even checking the width after the split
739 	 */
740 	if (!dpu_kms->catalog->caps->has_3d_merge &&
741 	    adj_mode->hdisplay > dpu_kms->catalog->caps->max_mixer_width)
742 		return -E2BIG;
743 
744 	for (i = 0; i < cstate->num_mixers; i++) {
745 		struct drm_rect *r = &cstate->lm_bounds[i];
746 		r->x1 = crtc_split_width * i;
747 		r->y1 = 0;
748 		r->x2 = r->x1 + crtc_split_width;
749 		r->y2 = adj_mode->vdisplay;
750 
751 		trace_dpu_crtc_setup_lm_bounds(DRMID(crtc), i, r);
752 
753 		if (drm_rect_width(r) > dpu_kms->catalog->caps->max_mixer_width)
754 			return -E2BIG;
755 	}
756 
757 	return 0;
758 }
759 
760 static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state,
761 		struct dpu_hw_pcc_cfg *cfg)
762 {
763 	struct drm_color_ctm *ctm;
764 
765 	memset(cfg, 0, sizeof(struct dpu_hw_pcc_cfg));
766 
767 	ctm = (struct drm_color_ctm *)state->ctm->data;
768 
769 	if (!ctm)
770 		return;
771 
772 	cfg->r.r = CONVERT_S3_15(ctm->matrix[0]);
773 	cfg->g.r = CONVERT_S3_15(ctm->matrix[1]);
774 	cfg->b.r = CONVERT_S3_15(ctm->matrix[2]);
775 
776 	cfg->r.g = CONVERT_S3_15(ctm->matrix[3]);
777 	cfg->g.g = CONVERT_S3_15(ctm->matrix[4]);
778 	cfg->b.g = CONVERT_S3_15(ctm->matrix[5]);
779 
780 	cfg->r.b = CONVERT_S3_15(ctm->matrix[6]);
781 	cfg->g.b = CONVERT_S3_15(ctm->matrix[7]);
782 	cfg->b.b = CONVERT_S3_15(ctm->matrix[8]);
783 }
784 
785 static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
786 {
787 	struct drm_crtc_state *state = crtc->state;
788 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
789 	struct dpu_crtc_mixer *mixer = cstate->mixers;
790 	struct dpu_hw_pcc_cfg cfg;
791 	struct dpu_hw_ctl *ctl;
792 	struct dpu_hw_dspp *dspp;
793 	int i;
794 
795 
796 	if (!state->color_mgmt_changed && !drm_atomic_crtc_needs_modeset(state))
797 		return;
798 
799 	for (i = 0; i < cstate->num_mixers; i++) {
800 		ctl = mixer[i].lm_ctl;
801 		dspp = mixer[i].hw_dspp;
802 
803 		if (!dspp || !dspp->ops.setup_pcc)
804 			continue;
805 
806 		if (!state->ctm) {
807 			dspp->ops.setup_pcc(dspp, NULL);
808 		} else {
809 			_dpu_crtc_get_pcc_coeff(state, &cfg);
810 			dspp->ops.setup_pcc(dspp, &cfg);
811 		}
812 
813 		/* stage config flush mask */
814 		ctl->ops.update_pending_flush_dspp(ctl,
815 			mixer[i].hw_dspp->idx, DPU_DSPP_PCC);
816 	}
817 }
818 
819 static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
820 		struct drm_atomic_state *state)
821 {
822 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
823 	struct drm_encoder *encoder;
824 
825 	if (!crtc->state->enable) {
826 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_begin\n",
827 				crtc->base.id, crtc->state->enable);
828 		return;
829 	}
830 
831 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
832 
833 	_dpu_crtc_check_and_setup_lm_bounds(crtc, crtc->state);
834 
835 	/* encoder will trigger pending mask now */
836 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
837 		dpu_encoder_trigger_kickoff_pending(encoder);
838 
839 	/*
840 	 * If no mixers have been allocated in dpu_crtc_atomic_check(),
841 	 * it means we are trying to flush a CRTC whose state is disabled:
842 	 * nothing else needs to be done.
843 	 */
844 	if (unlikely(!cstate->num_mixers))
845 		return;
846 
847 	_dpu_crtc_blend_setup(crtc);
848 
849 	_dpu_crtc_setup_cp_blocks(crtc);
850 
851 	/*
852 	 * PP_DONE irq is only used by command mode for now.
853 	 * It is better to request pending before FLUSH and START trigger
854 	 * to make sure no pp_done irq missed.
855 	 * This is safe because no pp_done will happen before SW trigger
856 	 * in command mode.
857 	 */
858 }
859 
860 static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
861 		struct drm_atomic_state *state)
862 {
863 	struct dpu_crtc *dpu_crtc;
864 	struct drm_device *dev;
865 	struct drm_plane *plane;
866 	struct msm_drm_private *priv;
867 	unsigned long flags;
868 	struct dpu_crtc_state *cstate;
869 
870 	if (!crtc->state->enable) {
871 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, skip atomic_flush\n",
872 				crtc->base.id, crtc->state->enable);
873 		return;
874 	}
875 
876 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
877 
878 	dpu_crtc = to_dpu_crtc(crtc);
879 	cstate = to_dpu_crtc_state(crtc->state);
880 	dev = crtc->dev;
881 	priv = dev->dev_private;
882 
883 	if (crtc->index >= ARRAY_SIZE(priv->event_thread)) {
884 		DPU_ERROR("invalid crtc index[%d]\n", crtc->index);
885 		return;
886 	}
887 
888 	WARN_ON(dpu_crtc->event);
889 	spin_lock_irqsave(&dev->event_lock, flags);
890 	dpu_crtc->event = crtc->state->event;
891 	crtc->state->event = NULL;
892 	spin_unlock_irqrestore(&dev->event_lock, flags);
893 
894 	/*
895 	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
896 	 * it means we are trying to flush a CRTC whose state is disabled:
897 	 * nothing else needs to be done.
898 	 */
899 	if (unlikely(!cstate->num_mixers))
900 		return;
901 
902 	/* update performance setting before crtc kickoff */
903 	dpu_core_perf_crtc_update(crtc, 1);
904 
905 	/*
906 	 * Final plane updates: Give each plane a chance to complete all
907 	 *                      required writes/flushing before crtc's "flush
908 	 *                      everything" call below.
909 	 */
910 	drm_atomic_crtc_for_each_plane(plane, crtc) {
911 		if (dpu_crtc->smmu_state.transition_error)
912 			dpu_plane_set_error(plane, true);
913 		dpu_plane_flush(plane);
914 	}
915 
916 	/* Kickoff will be scheduled by outer layer */
917 }
918 
919 /**
920  * dpu_crtc_destroy_state - state destroy hook
921  * @crtc: drm CRTC
922  * @state: CRTC state object to release
923  */
924 static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
925 		struct drm_crtc_state *state)
926 {
927 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
928 
929 	DRM_DEBUG_ATOMIC("crtc%d\n", crtc->base.id);
930 
931 	__drm_atomic_helper_crtc_destroy_state(state);
932 
933 	kfree(cstate);
934 }
935 
936 static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
937 {
938 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
939 	int ret, rc = 0;
940 
941 	if (!atomic_read(&dpu_crtc->frame_pending)) {
942 		DRM_DEBUG_ATOMIC("no frames pending\n");
943 		return 0;
944 	}
945 
946 	DPU_ATRACE_BEGIN("frame done completion wait");
947 	ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
948 			msecs_to_jiffies(DPU_CRTC_FRAME_DONE_TIMEOUT_MS));
949 	if (!ret) {
950 		DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
951 		rc = -ETIMEDOUT;
952 	}
953 	DPU_ATRACE_END("frame done completion wait");
954 
955 	return rc;
956 }
957 
958 static int dpu_crtc_kickoff_clone_mode(struct drm_crtc *crtc)
959 {
960 	struct drm_encoder *encoder;
961 	struct drm_encoder *rt_encoder = NULL, *wb_encoder = NULL;
962 	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
963 
964 	/* Find encoder for real time display */
965 	drm_for_each_encoder_mask(encoder, crtc->dev,
966 				  crtc->state->encoder_mask) {
967 		if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL)
968 			wb_encoder = encoder;
969 		else
970 			rt_encoder = encoder;
971 	}
972 
973 	if (!rt_encoder || !wb_encoder) {
974 		DRM_DEBUG_ATOMIC("real time or wb encoder not found\n");
975 		return -EINVAL;
976 	}
977 
978 	dpu_encoder_prepare_for_kickoff(wb_encoder);
979 	dpu_encoder_prepare_for_kickoff(rt_encoder);
980 
981 	dpu_vbif_clear_errors(dpu_kms);
982 
983 	/*
984 	 * Kickoff real time encoder last as it's the encoder that
985 	 * will do the flush
986 	 */
987 	dpu_encoder_kickoff(wb_encoder);
988 	dpu_encoder_kickoff(rt_encoder);
989 
990 	/* Don't start frame done timers until the kickoffs have finished */
991 	dpu_encoder_start_frame_done_timer(wb_encoder);
992 	dpu_encoder_start_frame_done_timer(rt_encoder);
993 
994 	return 0;
995 }
996 
997 /**
998  * dpu_crtc_commit_kickoff - trigger kickoff of the commit for this crtc
999  * @crtc: Pointer to drm crtc object
1000  */
1001 void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
1002 {
1003 	struct drm_encoder *encoder;
1004 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1005 	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
1006 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
1007 
1008 	/*
1009 	 * If no mixers has been allocated in dpu_crtc_atomic_check(),
1010 	 * it means we are trying to start a CRTC whose state is disabled:
1011 	 * nothing else needs to be done.
1012 	 */
1013 	if (unlikely(!cstate->num_mixers))
1014 		return;
1015 
1016 	DPU_ATRACE_BEGIN("crtc_commit");
1017 
1018 	drm_for_each_encoder_mask(encoder, crtc->dev,
1019 			crtc->state->encoder_mask) {
1020 		if (!dpu_encoder_is_valid_for_commit(encoder)) {
1021 			DRM_DEBUG_ATOMIC("invalid FB not kicking off crtc\n");
1022 			goto end;
1023 		}
1024 	}
1025 
1026 	if (drm_crtc_in_clone_mode(crtc->state)) {
1027 		if (dpu_crtc_kickoff_clone_mode(crtc))
1028 			goto end;
1029 	} else {
1030 		/*
1031 		 * Encoder will flush/start now, unless it has a tx pending.
1032 		 * If so, it may delay and flush at an irq event (e.g. ppdone)
1033 		 */
1034 		drm_for_each_encoder_mask(encoder, crtc->dev,
1035 				crtc->state->encoder_mask)
1036 			dpu_encoder_prepare_for_kickoff(encoder);
1037 
1038 		dpu_vbif_clear_errors(dpu_kms);
1039 
1040 		drm_for_each_encoder_mask(encoder, crtc->dev,
1041 				crtc->state->encoder_mask) {
1042 			dpu_encoder_kickoff(encoder);
1043 			dpu_encoder_start_frame_done_timer(encoder);
1044 		}
1045 	}
1046 
1047 	if (atomic_inc_return(&dpu_crtc->frame_pending) == 1) {
1048 		/* acquire bandwidth and other resources */
1049 		DRM_DEBUG_ATOMIC("crtc%d first commit\n", crtc->base.id);
1050 	} else
1051 		DRM_DEBUG_ATOMIC("crtc%d commit\n", crtc->base.id);
1052 
1053 	dpu_crtc->play_count++;
1054 
1055 	reinit_completion(&dpu_crtc->frame_done_comp);
1056 
1057 end:
1058 	DPU_ATRACE_END("crtc_commit");
1059 }
1060 
1061 static void dpu_crtc_reset(struct drm_crtc *crtc)
1062 {
1063 	struct dpu_crtc_state *cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
1064 
1065 	if (crtc->state)
1066 		dpu_crtc_destroy_state(crtc, crtc->state);
1067 
1068 	if (cstate)
1069 		__drm_atomic_helper_crtc_reset(crtc, &cstate->base);
1070 	else
1071 		__drm_atomic_helper_crtc_reset(crtc, NULL);
1072 }
1073 
1074 /**
1075  * dpu_crtc_duplicate_state - state duplicate hook
1076  * @crtc: Pointer to drm crtc structure
1077  */
1078 static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
1079 {
1080 	struct dpu_crtc_state *cstate, *old_cstate = to_dpu_crtc_state(crtc->state);
1081 
1082 	cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
1083 	if (!cstate) {
1084 		DPU_ERROR("failed to allocate state\n");
1085 		return NULL;
1086 	}
1087 
1088 	/* duplicate base helper */
1089 	__drm_atomic_helper_crtc_duplicate_state(crtc, &cstate->base);
1090 
1091 	return &cstate->base;
1092 }
1093 
1094 static void dpu_crtc_atomic_print_state(struct drm_printer *p,
1095 					const struct drm_crtc_state *state)
1096 {
1097 	const struct dpu_crtc_state *cstate = to_dpu_crtc_state(state);
1098 	int i;
1099 
1100 	for (i = 0; i < cstate->num_mixers; i++) {
1101 		drm_printf(p, "\tlm[%d]=%d\n", i, cstate->mixers[i].hw_lm->idx - LM_0);
1102 		drm_printf(p, "\tctl[%d]=%d\n", i, cstate->mixers[i].lm_ctl->idx - CTL_0);
1103 		if (cstate->mixers[i].hw_dspp)
1104 			drm_printf(p, "\tdspp[%d]=%d\n", i, cstate->mixers[i].hw_dspp->idx - DSPP_0);
1105 	}
1106 }
1107 
1108 static void dpu_crtc_disable(struct drm_crtc *crtc,
1109 			     struct drm_atomic_state *state)
1110 {
1111 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
1112 									      crtc);
1113 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1114 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
1115 	struct drm_encoder *encoder;
1116 	unsigned long flags;
1117 	bool release_bandwidth = false;
1118 
1119 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1120 
1121 	/* If disable is triggered while in self refresh mode,
1122 	 * reset the encoder software state so that in enable
1123 	 * it won't trigger a warn while assigning crtc.
1124 	 */
1125 	if (old_crtc_state->self_refresh_active) {
1126 		drm_for_each_encoder_mask(encoder, crtc->dev,
1127 					old_crtc_state->encoder_mask) {
1128 			dpu_encoder_assign_crtc(encoder, NULL);
1129 		}
1130 		return;
1131 	}
1132 
1133 	/* Disable/save vblank irq handling */
1134 	drm_crtc_vblank_off(crtc);
1135 
1136 	drm_for_each_encoder_mask(encoder, crtc->dev,
1137 				  old_crtc_state->encoder_mask) {
1138 		/* in video mode, we hold an extra bandwidth reference
1139 		 * as we cannot drop bandwidth at frame-done if any
1140 		 * crtc is being used in video mode.
1141 		 */
1142 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1143 			release_bandwidth = true;
1144 
1145 		/*
1146 		 * If disable is triggered during psr active(e.g: screen dim in PSR),
1147 		 * we will need encoder->crtc connection to process the device sleep &
1148 		 * preserve it during psr sequence.
1149 		 */
1150 		if (!crtc->state->self_refresh_active)
1151 			dpu_encoder_assign_crtc(encoder, NULL);
1152 	}
1153 
1154 	/* wait for frame_event_done completion */
1155 	if (_dpu_crtc_wait_for_frame_done(crtc))
1156 		DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
1157 				crtc->base.id,
1158 				atomic_read(&dpu_crtc->frame_pending));
1159 
1160 	trace_dpu_crtc_disable(DRMID(crtc), false, dpu_crtc);
1161 	dpu_crtc->enabled = false;
1162 
1163 	if (atomic_read(&dpu_crtc->frame_pending)) {
1164 		trace_dpu_crtc_disable_frame_pending(DRMID(crtc),
1165 				     atomic_read(&dpu_crtc->frame_pending));
1166 		if (release_bandwidth)
1167 			dpu_core_perf_crtc_release_bw(crtc);
1168 		atomic_set(&dpu_crtc->frame_pending, 0);
1169 	}
1170 
1171 	dpu_core_perf_crtc_update(crtc, 0);
1172 
1173 	/* disable clk & bw control until clk & bw properties are set */
1174 	cstate->bw_control = false;
1175 	cstate->bw_split_vote = false;
1176 
1177 	if (crtc->state->event && !crtc->state->active) {
1178 		spin_lock_irqsave(&crtc->dev->event_lock, flags);
1179 		drm_crtc_send_vblank_event(crtc, crtc->state->event);
1180 		crtc->state->event = NULL;
1181 		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
1182 	}
1183 
1184 	pm_runtime_put_sync(crtc->dev->dev);
1185 }
1186 
1187 static void dpu_crtc_enable(struct drm_crtc *crtc,
1188 		struct drm_atomic_state *state)
1189 {
1190 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1191 	struct drm_encoder *encoder;
1192 	bool request_bandwidth = false;
1193 	struct drm_crtc_state *old_crtc_state;
1194 
1195 	old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
1196 
1197 	pm_runtime_get_sync(crtc->dev->dev);
1198 
1199 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1200 
1201 	drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask) {
1202 		/* in video mode, we hold an extra bandwidth reference
1203 		 * as we cannot drop bandwidth at frame-done if any
1204 		 * crtc is being used in video mode.
1205 		 */
1206 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO)
1207 			request_bandwidth = true;
1208 	}
1209 
1210 	if (request_bandwidth)
1211 		atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1212 
1213 	trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
1214 	dpu_crtc->enabled = true;
1215 
1216 	if (!old_crtc_state->self_refresh_active) {
1217 		drm_for_each_encoder_mask(encoder, crtc->dev, crtc->state->encoder_mask)
1218 			dpu_encoder_assign_crtc(encoder, crtc);
1219 	}
1220 
1221 	/* Enable/restore vblank irq handling */
1222 	drm_crtc_vblank_on(crtc);
1223 }
1224 
1225 static bool dpu_crtc_needs_dirtyfb(struct drm_crtc_state *cstate)
1226 {
1227 	struct drm_crtc *crtc = cstate->crtc;
1228 	struct drm_encoder *encoder;
1229 
1230 	if (cstate->self_refresh_active)
1231 		return true;
1232 
1233 	drm_for_each_encoder_mask (encoder, crtc->dev, cstate->encoder_mask) {
1234 		if (dpu_encoder_get_intf_mode(encoder) == INTF_MODE_CMD) {
1235 			return true;
1236 		}
1237 	}
1238 
1239 	return false;
1240 }
1241 
1242 static int dpu_crtc_reassign_planes(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state)
1243 {
1244 	int total_planes = crtc->dev->mode_config.num_total_plane;
1245 	struct drm_atomic_state *state = crtc_state->state;
1246 	struct dpu_global_state *global_state;
1247 	struct drm_plane_state **states;
1248 	struct drm_plane *plane;
1249 	int ret;
1250 
1251 	global_state = dpu_kms_get_global_state(crtc_state->state);
1252 	if (IS_ERR(global_state))
1253 		return PTR_ERR(global_state);
1254 
1255 	dpu_rm_release_all_sspp(global_state, crtc);
1256 
1257 	if (!crtc_state->enable)
1258 		return 0;
1259 
1260 	states = kcalloc(total_planes, sizeof(*states), GFP_KERNEL);
1261 	if (!states)
1262 		return -ENOMEM;
1263 
1264 	drm_atomic_crtc_state_for_each_plane(plane, crtc_state) {
1265 		struct drm_plane_state *plane_state =
1266 			drm_atomic_get_plane_state(state, plane);
1267 
1268 		if (IS_ERR(plane_state)) {
1269 			ret = PTR_ERR(plane_state);
1270 			goto done;
1271 		}
1272 
1273 		states[plane_state->normalized_zpos] = plane_state;
1274 	}
1275 
1276 	ret = dpu_assign_plane_resources(global_state, state, crtc, states, total_planes);
1277 
1278 done:
1279 	kfree(states);
1280 	return ret;
1281 }
1282 
1283 #define MAX_CHANNELS_PER_CRTC 2
1284 #define MAX_HDISPLAY_SPLIT 1080
1285 
1286 static struct msm_display_topology dpu_crtc_get_topology(
1287 		struct drm_crtc *crtc,
1288 		struct dpu_kms *dpu_kms,
1289 		struct drm_crtc_state *crtc_state)
1290 {
1291 	struct drm_display_mode *mode = &crtc_state->adjusted_mode;
1292 	struct msm_display_topology topology = {0};
1293 	struct drm_encoder *drm_enc;
1294 
1295 	drm_for_each_encoder_mask(drm_enc, crtc->dev, crtc_state->encoder_mask)
1296 		dpu_encoder_update_topology(drm_enc, &topology, crtc_state->state,
1297 					    &crtc_state->adjusted_mode);
1298 
1299 	topology.cwb_enabled = drm_crtc_in_clone_mode(crtc_state);
1300 
1301 	/*
1302 	 * Datapath topology selection
1303 	 *
1304 	 * Dual display
1305 	 * 2 LM, 2 INTF ( Split display using 2 interfaces)
1306 	 *
1307 	 * Single display
1308 	 * 1 LM, 1 INTF
1309 	 * 2 LM, 1 INTF (stream merge to support high resolution interfaces)
1310 	 *
1311 	 * If DSC is enabled, use 2 LMs for 2:2:1 topology
1312 	 *
1313 	 * Add dspps to the reservation requirements if ctm is requested
1314 	 *
1315 	 * Only hardcode num_lm to 2 for cases where num_intf == 2 and CWB is not
1316 	 * enabled. This is because in cases where CWB is enabled, num_intf will
1317 	 * count both the WB and real-time phys encoders.
1318 	 *
1319 	 * For non-DSC CWB usecases, have the num_lm be decided by the
1320 	 * (mode->hdisplay > MAX_HDISPLAY_SPLIT) check.
1321 	 */
1322 
1323 	if (topology.num_intf == 2 && !topology.cwb_enabled)
1324 		topology.num_lm = 2;
1325 	else if (topology.num_dsc == 2)
1326 		topology.num_lm = 2;
1327 	else if (dpu_kms->catalog->caps->has_3d_merge)
1328 		topology.num_lm = (mode->hdisplay > MAX_HDISPLAY_SPLIT) ? 2 : 1;
1329 	else
1330 		topology.num_lm = 1;
1331 
1332 	if (crtc_state->ctm)
1333 		topology.num_dspp = topology.num_lm;
1334 
1335 	return topology;
1336 }
1337 
1338 static int dpu_crtc_assign_resources(struct drm_crtc *crtc,
1339 				     struct drm_crtc_state *crtc_state)
1340 {
1341 	struct dpu_hw_blk *hw_ctl[MAX_CHANNELS_PER_CRTC];
1342 	struct dpu_hw_blk *hw_lm[MAX_CHANNELS_PER_CRTC];
1343 	struct dpu_hw_blk *hw_dspp[MAX_CHANNELS_PER_CRTC];
1344 	int i, num_lm, num_ctl, num_dspp;
1345 	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
1346 	struct dpu_global_state *global_state;
1347 	struct dpu_crtc_state *cstate;
1348 	struct msm_display_topology topology;
1349 	int ret;
1350 
1351 	/*
1352 	 * Release and Allocate resources on every modeset
1353 	 */
1354 	global_state = dpu_kms_get_global_state(crtc_state->state);
1355 	if (IS_ERR(global_state))
1356 		return PTR_ERR(global_state);
1357 
1358 	dpu_rm_release(global_state, crtc);
1359 
1360 	if (!crtc_state->enable)
1361 		return 0;
1362 
1363 	topology = dpu_crtc_get_topology(crtc, dpu_kms, crtc_state);
1364 	ret = dpu_rm_reserve(&dpu_kms->rm, global_state,
1365 			     crtc_state->crtc, &topology);
1366 	if (ret)
1367 		return ret;
1368 
1369 	cstate = to_dpu_crtc_state(crtc_state);
1370 
1371 	num_ctl = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1372 						crtc_state->crtc,
1373 						DPU_HW_BLK_CTL, hw_ctl,
1374 						ARRAY_SIZE(hw_ctl));
1375 	num_lm = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1376 					       crtc_state->crtc,
1377 					       DPU_HW_BLK_LM, hw_lm,
1378 					       ARRAY_SIZE(hw_lm));
1379 	num_dspp = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
1380 						 crtc_state->crtc,
1381 						 DPU_HW_BLK_DSPP, hw_dspp,
1382 						 ARRAY_SIZE(hw_dspp));
1383 
1384 	for (i = 0; i < num_lm; i++) {
1385 		int ctl_idx = (i < num_ctl) ? i : (num_ctl-1);
1386 
1387 		cstate->mixers[i].hw_lm = to_dpu_hw_mixer(hw_lm[i]);
1388 		cstate->mixers[i].lm_ctl = to_dpu_hw_ctl(hw_ctl[ctl_idx]);
1389 		if (i < num_dspp)
1390 			cstate->mixers[i].hw_dspp = to_dpu_hw_dspp(hw_dspp[i]);
1391 	}
1392 
1393 	cstate->num_mixers = num_lm;
1394 
1395 	return 0;
1396 }
1397 
1398 /**
1399  * dpu_crtc_check_mode_changed: check if full modeset is required
1400  * @old_crtc_state:	Previous CRTC state
1401  * @new_crtc_state:	Corresponding CRTC state to be checked
1402  *
1403  * Check if the changes in the object properties demand full mode set.
1404  */
1405 int dpu_crtc_check_mode_changed(struct drm_crtc_state *old_crtc_state,
1406 				struct drm_crtc_state *new_crtc_state)
1407 {
1408 	struct drm_encoder *drm_enc;
1409 	struct drm_crtc *crtc = new_crtc_state->crtc;
1410 	bool clone_mode_enabled = drm_crtc_in_clone_mode(old_crtc_state);
1411 	bool clone_mode_requested = drm_crtc_in_clone_mode(new_crtc_state);
1412 
1413 	DRM_DEBUG_ATOMIC("%d\n", crtc->base.id);
1414 
1415 	/* there might be cases where encoder needs a modeset too */
1416 	drm_for_each_encoder_mask(drm_enc, crtc->dev, new_crtc_state->encoder_mask) {
1417 		if (dpu_encoder_needs_modeset(drm_enc, new_crtc_state->state))
1418 			new_crtc_state->mode_changed = true;
1419 	}
1420 
1421 	if ((clone_mode_requested && !clone_mode_enabled) ||
1422 	    (!clone_mode_requested && clone_mode_enabled))
1423 		new_crtc_state->mode_changed = true;
1424 
1425 	return 0;
1426 }
1427 
1428 static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
1429 		struct drm_atomic_state *state)
1430 {
1431 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
1432 									  crtc);
1433 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1434 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc_state);
1435 
1436 	const struct drm_plane_state *pstate;
1437 	struct drm_plane *plane;
1438 
1439 	int rc = 0;
1440 
1441 	bool needs_dirtyfb = dpu_crtc_needs_dirtyfb(crtc_state);
1442 
1443 	/* don't reallocate resources if only ACTIVE has beeen changed */
1444 	if (crtc_state->mode_changed || crtc_state->connectors_changed) {
1445 		rc = dpu_crtc_assign_resources(crtc, crtc_state);
1446 		if (rc < 0)
1447 			return rc;
1448 	}
1449 
1450 	if (dpu_use_virtual_planes &&
1451 	    (crtc_state->planes_changed || crtc_state->zpos_changed)) {
1452 		rc = dpu_crtc_reassign_planes(crtc, crtc_state);
1453 		if (rc < 0)
1454 			return rc;
1455 	}
1456 
1457 	if (!crtc_state->enable || !drm_atomic_crtc_effectively_active(crtc_state)) {
1458 		DRM_DEBUG_ATOMIC("crtc%d -> enable %d, active %d, skip atomic_check\n",
1459 				crtc->base.id, crtc_state->enable,
1460 				crtc_state->active);
1461 		memset(&cstate->new_perf, 0, sizeof(cstate->new_perf));
1462 		return 0;
1463 	}
1464 
1465 	DRM_DEBUG_ATOMIC("%s: check\n", dpu_crtc->name);
1466 
1467 	if (cstate->num_mixers) {
1468 		rc = _dpu_crtc_check_and_setup_lm_bounds(crtc, crtc_state);
1469 		if (rc)
1470 			return rc;
1471 	}
1472 
1473 	/* FIXME: move this to dpu_plane_atomic_check? */
1474 	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
1475 		struct dpu_plane_state *dpu_pstate = to_dpu_plane_state(pstate);
1476 
1477 		if (IS_ERR_OR_NULL(pstate)) {
1478 			rc = PTR_ERR(pstate);
1479 			DPU_ERROR("%s: failed to get plane%d state, %d\n",
1480 					dpu_crtc->name, plane->base.id, rc);
1481 			return rc;
1482 		}
1483 
1484 		if (!pstate->visible)
1485 			continue;
1486 
1487 		dpu_pstate->needs_dirtyfb = needs_dirtyfb;
1488 	}
1489 
1490 	atomic_inc(&_dpu_crtc_get_kms(crtc)->bandwidth_ref);
1491 
1492 	rc = dpu_core_perf_crtc_check(crtc, crtc_state);
1493 	if (rc) {
1494 		DPU_ERROR("crtc%d failed performance check %d\n",
1495 				crtc->base.id, rc);
1496 		return rc;
1497 	}
1498 
1499 	return 0;
1500 }
1501 
1502 static enum drm_mode_status dpu_crtc_mode_valid(struct drm_crtc *crtc,
1503 						const struct drm_display_mode *mode)
1504 {
1505 	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
1506 
1507 	/* if there is no 3d_mux block we cannot merge LMs so we cannot
1508 	 * split the large layer into 2 LMs, filter out such modes
1509 	 */
1510 	if (!dpu_kms->catalog->caps->has_3d_merge &&
1511 	    mode->hdisplay > dpu_kms->catalog->caps->max_mixer_width)
1512 		return MODE_BAD_HVALUE;
1513 	/*
1514 	 * max crtc width is equal to the max mixer width * 2 and max height is 4K
1515 	 */
1516 	return drm_mode_validate_size(mode,
1517 				      2 * dpu_kms->catalog->caps->max_mixer_width,
1518 				      4096);
1519 }
1520 
1521 /**
1522  * dpu_crtc_vblank - enable or disable vblanks for this crtc
1523  * @crtc: Pointer to drm crtc object
1524  * @en: true to enable vblanks, false to disable
1525  */
1526 int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
1527 {
1528 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1529 	struct drm_encoder *enc;
1530 
1531 	trace_dpu_crtc_vblank(DRMID(&dpu_crtc->base), en, dpu_crtc);
1532 
1533 	/*
1534 	 * Normally we would iterate through encoder_mask in crtc state to find
1535 	 * attached encoders. In this case, we might be disabling vblank _after_
1536 	 * encoder_mask has been cleared.
1537 	 *
1538 	 * Instead, we "assign" a crtc to the encoder in enable and clear it in
1539 	 * disable (which is also after encoder_mask is cleared). So instead of
1540 	 * using encoder mask, we'll ask the encoder to toggle itself iff it's
1541 	 * currently assigned to our crtc.
1542 	 *
1543 	 * Note also that this function cannot be called while crtc is disabled
1544 	 * since we use drm_crtc_vblank_on/off. So we don't need to worry
1545 	 * about the assigned crtcs being inconsistent with the current state
1546 	 * (which means no need to worry about modeset locks).
1547 	 */
1548 	list_for_each_entry(enc, &crtc->dev->mode_config.encoder_list, head) {
1549 		trace_dpu_crtc_vblank_enable(DRMID(crtc), DRMID(enc), en,
1550 					     dpu_crtc);
1551 
1552 		dpu_encoder_toggle_vblank_for_crtc(enc, crtc, en);
1553 	}
1554 
1555 	return 0;
1556 }
1557 
1558 #ifdef CONFIG_DEBUG_FS
1559 static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
1560 {
1561 	struct dpu_crtc *dpu_crtc;
1562 	struct dpu_plane_state *pstate = NULL;
1563 	struct dpu_crtc_mixer *m;
1564 
1565 	struct drm_crtc *crtc;
1566 	struct drm_plane *plane;
1567 	struct drm_display_mode *mode;
1568 	struct drm_framebuffer *fb;
1569 	struct drm_plane_state *state;
1570 	struct dpu_crtc_state *cstate;
1571 
1572 	int i, out_width;
1573 
1574 	dpu_crtc = s->private;
1575 	crtc = &dpu_crtc->base;
1576 
1577 	drm_modeset_lock_all(crtc->dev);
1578 	cstate = to_dpu_crtc_state(crtc->state);
1579 
1580 	mode = &crtc->state->adjusted_mode;
1581 	out_width = mode->hdisplay / cstate->num_mixers;
1582 
1583 	seq_printf(s, "crtc:%d width:%d height:%d\n", crtc->base.id,
1584 				mode->hdisplay, mode->vdisplay);
1585 
1586 	seq_puts(s, "\n");
1587 
1588 	for (i = 0; i < cstate->num_mixers; ++i) {
1589 		m = &cstate->mixers[i];
1590 		seq_printf(s, "\tmixer:%d ctl:%d width:%d height:%d\n",
1591 			m->hw_lm->idx - LM_0, m->lm_ctl->idx - CTL_0,
1592 			out_width, mode->vdisplay);
1593 	}
1594 
1595 	seq_puts(s, "\n");
1596 
1597 	drm_atomic_crtc_for_each_plane(plane, crtc) {
1598 		pstate = to_dpu_plane_state(plane->state);
1599 		state = plane->state;
1600 
1601 		if (!pstate || !state)
1602 			continue;
1603 
1604 		seq_printf(s, "\tplane:%u stage:%d\n", plane->base.id,
1605 			pstate->stage);
1606 
1607 		if (plane->state->fb) {
1608 			fb = plane->state->fb;
1609 
1610 			seq_printf(s, "\tfb:%d image format:%4.4s wxh:%ux%u ",
1611 				fb->base.id, (char *) &fb->format->format,
1612 				fb->width, fb->height);
1613 			for (i = 0; i < ARRAY_SIZE(fb->format->cpp); ++i)
1614 				seq_printf(s, "cpp[%d]:%u ",
1615 						i, fb->format->cpp[i]);
1616 			seq_puts(s, "\n\t");
1617 
1618 			seq_printf(s, "modifier:%8llu ", fb->modifier);
1619 			seq_puts(s, "\n");
1620 
1621 			seq_puts(s, "\t");
1622 			for (i = 0; i < ARRAY_SIZE(fb->pitches); i++)
1623 				seq_printf(s, "pitches[%d]:%8u ", i,
1624 							fb->pitches[i]);
1625 			seq_puts(s, "\n");
1626 
1627 			seq_puts(s, "\t");
1628 			for (i = 0; i < ARRAY_SIZE(fb->offsets); i++)
1629 				seq_printf(s, "offsets[%d]:%8u ", i,
1630 							fb->offsets[i]);
1631 			seq_puts(s, "\n");
1632 		}
1633 
1634 		seq_printf(s, "\tsrc_x:%4d src_y:%4d src_w:%4d src_h:%4d\n",
1635 			state->src_x, state->src_y, state->src_w, state->src_h);
1636 
1637 		seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
1638 			state->crtc_x, state->crtc_y, state->crtc_w,
1639 			state->crtc_h);
1640 		seq_printf(s, "\tsspp[0]:%s\n",
1641 			   pstate->pipe.sspp->cap->name);
1642 		seq_printf(s, "\tmultirect[0]: mode: %d index: %d\n",
1643 			pstate->pipe.multirect_mode, pstate->pipe.multirect_index);
1644 		if (pstate->r_pipe.sspp) {
1645 			seq_printf(s, "\tsspp[1]:%s\n",
1646 				   pstate->r_pipe.sspp->cap->name);
1647 			seq_printf(s, "\tmultirect[1]: mode: %d index: %d\n",
1648 				   pstate->r_pipe.multirect_mode, pstate->r_pipe.multirect_index);
1649 		}
1650 
1651 		seq_puts(s, "\n");
1652 	}
1653 	if (dpu_crtc->vblank_cb_count) {
1654 		ktime_t diff = ktime_sub(ktime_get(), dpu_crtc->vblank_cb_time);
1655 		s64 diff_ms = ktime_to_ms(diff);
1656 		s64 fps = diff_ms ? div_s64(
1657 				dpu_crtc->vblank_cb_count * 1000, diff_ms) : 0;
1658 
1659 		seq_printf(s,
1660 			"vblank fps:%lld count:%u total:%llums total_framecount:%llu\n",
1661 				fps, dpu_crtc->vblank_cb_count,
1662 				ktime_to_ms(diff), dpu_crtc->play_count);
1663 
1664 		/* reset time & count for next measurement */
1665 		dpu_crtc->vblank_cb_count = 0;
1666 		dpu_crtc->vblank_cb_time = ktime_set(0, 0);
1667 	}
1668 
1669 	drm_modeset_unlock_all(crtc->dev);
1670 
1671 	return 0;
1672 }
1673 
1674 DEFINE_SHOW_ATTRIBUTE(_dpu_debugfs_status);
1675 
1676 static int dpu_crtc_debugfs_state_show(struct seq_file *s, void *v)
1677 {
1678 	struct drm_crtc *crtc = s->private;
1679 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1680 
1681 	seq_printf(s, "client type: %d\n", dpu_crtc_get_client_type(crtc));
1682 	seq_printf(s, "intf_mode: %d\n", dpu_crtc_get_intf_mode(crtc));
1683 	seq_printf(s, "core_clk_rate: %llu\n",
1684 			dpu_crtc->cur_perf.core_clk_rate);
1685 	seq_printf(s, "bw_ctl: %uk\n",
1686 		   (u32)DIV_ROUND_UP_ULL(dpu_crtc->cur_perf.bw_ctl, 1000));
1687 	seq_printf(s, "max_per_pipe_ib: %u\n",
1688 				dpu_crtc->cur_perf.max_per_pipe_ib);
1689 
1690 	return 0;
1691 }
1692 DEFINE_SHOW_ATTRIBUTE(dpu_crtc_debugfs_state);
1693 
1694 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1695 {
1696 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1697 
1698 	debugfs_create_file("status", 0400,
1699 			crtc->debugfs_entry,
1700 			dpu_crtc, &_dpu_debugfs_status_fops);
1701 	debugfs_create_file("state", 0600,
1702 			crtc->debugfs_entry,
1703 			&dpu_crtc->base,
1704 			&dpu_crtc_debugfs_state_fops);
1705 
1706 	return 0;
1707 }
1708 #else
1709 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1710 {
1711 	return 0;
1712 }
1713 #endif /* CONFIG_DEBUG_FS */
1714 
1715 static int dpu_crtc_late_register(struct drm_crtc *crtc)
1716 {
1717 	return _dpu_crtc_init_debugfs(crtc);
1718 }
1719 
1720 static const struct drm_crtc_funcs dpu_crtc_funcs = {
1721 	.set_config = drm_atomic_helper_set_config,
1722 	.page_flip = drm_atomic_helper_page_flip,
1723 	.reset = dpu_crtc_reset,
1724 	.atomic_duplicate_state = dpu_crtc_duplicate_state,
1725 	.atomic_destroy_state = dpu_crtc_destroy_state,
1726 	.atomic_print_state = dpu_crtc_atomic_print_state,
1727 	.late_register = dpu_crtc_late_register,
1728 	.verify_crc_source = dpu_crtc_verify_crc_source,
1729 	.set_crc_source = dpu_crtc_set_crc_source,
1730 	.enable_vblank  = msm_crtc_enable_vblank,
1731 	.disable_vblank = msm_crtc_disable_vblank,
1732 	.get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
1733 	.get_vblank_counter = dpu_crtc_get_vblank_counter,
1734 };
1735 
1736 static const struct drm_crtc_helper_funcs dpu_crtc_helper_funcs = {
1737 	.atomic_disable = dpu_crtc_disable,
1738 	.atomic_enable = dpu_crtc_enable,
1739 	.atomic_check = dpu_crtc_atomic_check,
1740 	.atomic_begin = dpu_crtc_atomic_begin,
1741 	.atomic_flush = dpu_crtc_atomic_flush,
1742 	.mode_valid = dpu_crtc_mode_valid,
1743 	.get_scanout_position = dpu_crtc_get_scanout_position,
1744 };
1745 
1746 /**
1747  * dpu_crtc_init - create a new crtc object
1748  * @dev: dpu device
1749  * @plane: base plane
1750  * @cursor: cursor plane
1751  * @return: new crtc object or error
1752  *
1753  * initialize CRTC
1754  */
1755 struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
1756 				struct drm_plane *cursor)
1757 {
1758 	struct msm_drm_private *priv = dev->dev_private;
1759 	struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
1760 	struct drm_crtc *crtc = NULL;
1761 	struct dpu_crtc *dpu_crtc;
1762 	int i, ret;
1763 
1764 	dpu_crtc = drmm_crtc_alloc_with_planes(dev, struct dpu_crtc, base,
1765 					       plane, cursor,
1766 					       &dpu_crtc_funcs,
1767 					       NULL);
1768 
1769 	if (IS_ERR(dpu_crtc))
1770 		return ERR_CAST(dpu_crtc);
1771 
1772 	crtc = &dpu_crtc->base;
1773 	crtc->dev = dev;
1774 
1775 	spin_lock_init(&dpu_crtc->spin_lock);
1776 	atomic_set(&dpu_crtc->frame_pending, 0);
1777 
1778 	init_completion(&dpu_crtc->frame_done_comp);
1779 
1780 	INIT_LIST_HEAD(&dpu_crtc->frame_event_list);
1781 
1782 	for (i = 0; i < ARRAY_SIZE(dpu_crtc->frame_events); i++) {
1783 		INIT_LIST_HEAD(&dpu_crtc->frame_events[i].list);
1784 		list_add(&dpu_crtc->frame_events[i].list,
1785 				&dpu_crtc->frame_event_list);
1786 		kthread_init_work(&dpu_crtc->frame_events[i].work,
1787 				dpu_crtc_frame_event_work);
1788 	}
1789 
1790 	drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
1791 
1792 	if (dpu_kms->catalog->dspp_count)
1793 		drm_crtc_enable_color_mgmt(crtc, 0, true, 0);
1794 
1795 	/* save user friendly CRTC name for later */
1796 	snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
1797 
1798 	/* initialize event handling */
1799 	spin_lock_init(&dpu_crtc->event_lock);
1800 
1801 	ret = drm_self_refresh_helper_init(crtc);
1802 	if (ret) {
1803 		DPU_ERROR("Failed to initialize %s with self-refresh helpers %d\n",
1804 			crtc->name, ret);
1805 		return ERR_PTR(ret);
1806 	}
1807 
1808 	DRM_DEBUG_KMS("%s: successfully initialized crtc\n", dpu_crtc->name);
1809 	return crtc;
1810 }
1811