166 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
179 drm_vblank_crtc(struct drm_device *dev, unsigned int pipe)  in drm_vblank_crtc()  argument
181 	return &dev->vblank[pipe];  in drm_vblank_crtc()
191 static void store_vblank(struct drm_device *dev, unsigned int pipe,  in store_vblank()  argument
195 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in store_vblank()
207 static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)  in drm_max_vblank_count()  argument
209 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_max_vblank_count()
218 static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)  in drm_vblank_no_hw_counter()  argument
220 	drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);  in drm_vblank_no_hw_counter()
224 static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)  in __get_vblank_counter()  argument
227 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in __get_vblank_counter()
236 	return drm_vblank_no_hw_counter(dev, pipe);  in __get_vblank_counter()
248 static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)  in drm_reset_vblank_timestamp()  argument
262 		cur_vblank = __get_vblank_counter(dev, pipe);  in drm_reset_vblank_timestamp()
263 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);  in drm_reset_vblank_timestamp()
264 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);  in drm_reset_vblank_timestamp()
278 	store_vblank(dev, pipe, 1, t_vblank, cur_vblank);  in drm_reset_vblank_timestamp()
295 static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,  in drm_update_vblank_count()  argument
298 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_update_vblank_count()
304 	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);  in drm_update_vblank_count()
319 		cur_vblank = __get_vblank_counter(dev, pipe);  in drm_update_vblank_count()
320 		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);  in drm_update_vblank_count()
321 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);  in drm_update_vblank_count()
337 			    pipe, (long long)diff_ns, framedur_ns);  in drm_update_vblank_count()
343 				    pipe);  in drm_update_vblank_count()
361 			    " due to pre-modeset.\n", pipe, diff);  in drm_update_vblank_count()
367 		    pipe, (unsigned long long)atomic64_read(&vblank->count),  in drm_update_vblank_count()
384 	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);  in drm_update_vblank_count()
387 u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)  in drm_vblank_count()  argument
389 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_vblank_count()
392 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_count()
423 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_accurate_vblank_count()  local
433 	drm_update_vblank_count(dev, pipe, false);  in drm_crtc_accurate_vblank_count()
434 	vblank = drm_vblank_count(dev, pipe);  in drm_crtc_accurate_vblank_count()
442 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)  in __disable_vblank()  argument
445 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in __disable_vblank()
461 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)  in drm_vblank_disable_and_save()  argument
463 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_vblank_disable_and_save()
489 	drm_update_vblank_count(dev, pipe, false);  in drm_vblank_disable_and_save()
490 	__disable_vblank(dev, pipe);  in drm_vblank_disable_and_save()
502 	unsigned int pipe = vblank->pipe;  in vblank_disable_fn()  local
507 		drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);  in vblank_disable_fn()
508 		drm_vblank_disable_and_save(dev, pipe);  in vblank_disable_fn()
557 		vblank->pipe = i;  in drm_vblank_init()
630 	unsigned int pipe = drm_crtc_index(crtc);  in drm_calc_timestamping_constants()  local
638 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_calc_timestamping_constants()
714 	unsigned int pipe = crtc->index;  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()  local
722 	if (pipe >= dev->num_crtcs) {  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
723 		drm_err(dev, "Invalid crtc %u\n", pipe);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
746 			     pipe);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
772 				     pipe);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
788 			     pipe, duration_ns / 1000, *max_error / 1000, i);  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
814 		    pipe, hpos, vpos, &ts_etime, &ts_vblank_time,  in drm_crtc_vblank_helper_get_vblank_timestamp_internal()
907 drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,  in drm_get_last_vbltimestamp()  argument
910 	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in drm_get_last_vbltimestamp()
956 static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,  in drm_vblank_count_and_time()  argument
959 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_vblank_count_and_time()
963 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {  in drm_vblank_count_and_time()
1064 	trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);  in send_vblank_event()
1117 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_arm_vblank_event()  local
1121 	e->pipe = pipe;  in drm_crtc_arm_vblank_event()
1143 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_send_vblank_event()  local
1147 		seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_send_vblank_event()
1153 	e->pipe = pipe;  in drm_crtc_send_vblank_event()
1158 static int __enable_vblank(struct drm_device *dev, unsigned int pipe)  in __enable_vblank()  argument
1161 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in __enable_vblank()
1173 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)  in drm_vblank_enable()  argument
1175 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_vblank_enable()
1190 		ret = __enable_vblank(dev, pipe);  in drm_vblank_enable()
1192 			     pipe, ret);  in drm_vblank_enable()
1196 			drm_update_vblank_count(dev, pipe, 0);  in drm_vblank_enable()
1211 int drm_vblank_get(struct drm_device *dev, unsigned int pipe)  in drm_vblank_get()  argument
1213 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_vblank_get()
1220 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_get()
1226 		ret = drm_vblank_enable(dev, pipe);  in drm_vblank_get()
1254 void drm_vblank_put(struct drm_device *dev, unsigned int pipe)  in drm_vblank_put()  argument
1256 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_vblank_put()
1259 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_put()
1304 	int pipe = drm_crtc_index(crtc);  in drm_crtc_wait_one_vblank()  local
1309 	ret = drm_vblank_get(dev, pipe);  in drm_crtc_wait_one_vblank()
1311 		     pipe, ret))  in drm_crtc_wait_one_vblank()
1314 	last = drm_vblank_count(dev, pipe);  in drm_crtc_wait_one_vblank()
1317 				 last != drm_vblank_count(dev, pipe),  in drm_crtc_wait_one_vblank()
1320 	drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);  in drm_crtc_wait_one_vblank()
1322 	drm_vblank_put(dev, pipe);  in drm_crtc_wait_one_vblank()
1342 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_vblank_off()  local
1348 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_crtc_vblank_off()
1359 		    pipe, vblank->enabled, vblank->inmodeset);  in drm_crtc_vblank_off()
1364 		drm_vblank_disable_and_save(dev, pipe);  in drm_crtc_vblank_off()
1379 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_vblank_off()
1382 		if (e->pipe != pipe)  in drm_crtc_vblank_off()
1388 		drm_vblank_put(dev, pipe);  in drm_crtc_vblank_off()
1486 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_vblank_on_config()  local
1489 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_crtc_vblank_on_config()
1494 		    pipe, vblank->enabled, vblank->inmodeset);  in drm_crtc_vblank_on_config()
1504 	drm_reset_vblank_timestamp(dev, pipe);  in drm_crtc_vblank_on_config()
1511 		drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));  in drm_crtc_vblank_on_config()
1539 static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)  in drm_vblank_restore()  argument
1547 	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);  in drm_vblank_restore()
1549 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_vblank_restore()
1555 	vblank = drm_vblank_crtc(dev, pipe);  in drm_vblank_restore()
1562 		cur_vblank = __get_vblank_counter(dev, pipe);  in drm_vblank_restore()
1563 		drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);  in drm_vblank_restore()
1564 	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);  in drm_vblank_restore()
1596 	unsigned int pipe = drm_crtc_index(crtc);  in drm_crtc_vblank_restore()  local
1597 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_crtc_vblank_restore()
1603 	drm_vblank_restore(dev, pipe);  in drm_crtc_vblank_restore()
1607 static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,  in drm_queue_vblank_event()  argument
1612 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_queue_vblank_event()
1624 	e->pipe = pipe;  in drm_queue_vblank_event()
1630 		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in drm_queue_vblank_event()
1655 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_queue_vblank_event()
1658 		     req_seq, seq, pipe);  in drm_queue_vblank_event()
1660 	trace_drm_vblank_event_queued(file_priv, pipe, req_seq);  in drm_queue_vblank_event()
1664 		drm_vblank_put(dev, pipe);  in drm_queue_vblank_event()
1681 	drm_vblank_put(dev, pipe);  in drm_queue_vblank_event()
1712 static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,  in drm_wait_vblank_reply()  argument
1723 	reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);  in drm_wait_vblank_reply()
1743 	unsigned int flags, pipe, high_pipe;  in drm_wait_vblank_ioctl()  local
1771 		pipe = 0;  in drm_wait_vblank_ioctl()
1778 			pipe++;  in drm_wait_vblank_ioctl()
1781 		pipe = pipe_index;  in drm_wait_vblank_ioctl()
1784 	if (pipe >= dev->num_crtcs)  in drm_wait_vblank_ioctl()
1787 	vblank = drm_vblank_crtc(dev, pipe);  in drm_wait_vblank_ioctl()
1795 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);  in drm_wait_vblank_ioctl()
1799 	ret = drm_vblank_get(dev, pipe);  in drm_wait_vblank_ioctl()
1803 			     pipe, ret);  in drm_wait_vblank_ioctl()
1806 	seq = drm_vblank_count(dev, pipe);  in drm_wait_vblank_ioctl()
1833 		return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);  in drm_wait_vblank_ioctl()
1840 			     req_seq, pipe);  in drm_wait_vblank_ioctl()
1842 			drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||  in drm_wait_vblank_ioctl()
1862 		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);  in drm_wait_vblank_ioctl()
1865 			     pipe, vblwait->reply.sequence);  in drm_wait_vblank_ioctl()
1868 			     pipe);  in drm_wait_vblank_ioctl()
1872 	drm_vblank_put(dev, pipe);  in drm_wait_vblank_ioctl()
1876 static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)  in drm_handle_vblank_events()  argument
1878 	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);  in drm_handle_vblank_events()
1886 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_handle_vblank_events()
1889 		if (e->pipe != pipe)  in drm_handle_vblank_events()
1898 		drm_vblank_put(dev, pipe);  in drm_handle_vblank_events()
1905 	trace_drm_vblank_event(pipe, seq, now, high_prec);  in drm_handle_vblank_events()
1918 bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)  in drm_handle_vblank()  argument
1920 	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);  in drm_handle_vblank()
1927 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))  in drm_handle_vblank()
1945 	drm_update_vblank_count(dev, pipe, true);  in drm_handle_vblank()
1960 	drm_handle_vblank_events(dev, pipe);  in drm_handle_vblank()
2011 	int pipe;  in drm_crtc_get_sequence_ioctl()  local
2027 	pipe = drm_crtc_index(crtc);  in drm_crtc_get_sequence_ioctl()
2038 				     pipe, ret);  in drm_crtc_get_sequence_ioctl()
2048 	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_get_sequence_ioctl()
2068 	int pipe;  in drm_crtc_queue_sequence_ioctl()  local
2093 	pipe = drm_crtc_index(crtc);  in drm_crtc_queue_sequence_ioctl()
2105 			     pipe, ret);  in drm_crtc_queue_sequence_ioctl()
2109 	seq = drm_vblank_count_and_time(dev, pipe, &now);  in drm_crtc_queue_sequence_ioctl()
2118 	e->pipe = pipe;  in drm_crtc_queue_sequence_ioctl()