56 	struct amdgpu_ring		*ring;  member
99 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)  in amdgpu_fence_write()  argument
101 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
115 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)  in amdgpu_fence_read()  argument
117 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
139 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,  in amdgpu_fence_emit()  argument
142 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_emit()
155 		am_fence->ring = ring;  in amdgpu_fence_emit()
161 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
170 				       &ring->fence_drv.lock,  in amdgpu_fence_emit()
171 				       adev->fence_context + ring->idx, seq);  in amdgpu_fence_emit()
176 				       &ring->fence_drv.lock,  in amdgpu_fence_emit()
177 				       adev->fence_context + ring->idx, seq);  in amdgpu_fence_emit()
181 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
184 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
223 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,  in amdgpu_fence_emit_polling()  argument
232 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit_polling()
233 	r = amdgpu_fence_wait_polling(ring,  in amdgpu_fence_emit_polling()
234 				      seq - ring->fence_drv.num_fences_mask,  in amdgpu_fence_emit_polling()
239 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit_polling()
254 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)  in amdgpu_fence_schedule_fallback()  argument
256 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
271 bool amdgpu_fence_process(struct amdgpu_ring *ring)  in amdgpu_fence_process()  argument
273 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
274 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_process()
278 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
279 		seq = amdgpu_fence_read(ring);  in amdgpu_fence_process()
283 	if (del_timer(&ring->fence_drv.fallback_timer) &&  in amdgpu_fence_process()
284 	    seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
285 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_process()
325 	struct amdgpu_ring *ring = from_timer(ring, t,  in amdgpu_fence_fallback()  local
328 	if (amdgpu_fence_process(ring))  in amdgpu_fence_fallback()
329 		DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name);  in amdgpu_fence_fallback()
340 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)  in amdgpu_fence_wait_empty()  argument
342 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
349 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
373 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,  in amdgpu_fence_wait_polling()  argument
378 	while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {  in amdgpu_fence_wait_polling()
393 unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)  in amdgpu_fence_count_emitted()  argument
401 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
402 	emitted += READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
413 u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)  in amdgpu_fence_last_unsignaled_time_us()  argument
415 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_last_unsignaled_time_us()
419 	last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_last_unsignaled_time_us()
420 	sync_seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_last_unsignaled_time_us()
444 void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)  in amdgpu_fence_update_start_timestamp()  argument
446 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_update_start_timestamp()
470 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_start_ring()  argument
474 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_start_ring()
477 	if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {  in amdgpu_fence_driver_start_ring()
478 		ring->fence_drv.cpu_addr = ring->fence_cpu_addr;  in amdgpu_fence_driver_start_ring()
479 		ring->fence_drv.gpu_addr = ring->fence_gpu_addr;  in amdgpu_fence_driver_start_ring()
483 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;  in amdgpu_fence_driver_start_ring()
484 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;  in amdgpu_fence_driver_start_ring()
486 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
488 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
489 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
490 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
493 		      ring->name, ring->fence_drv.gpu_addr);  in amdgpu_fence_driver_start_ring()
506 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)  in amdgpu_fence_driver_init_ring()  argument
508 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_init_ring()
513 	if (!is_power_of_2(ring->num_hw_submission))  in amdgpu_fence_driver_init_ring()
516 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
517 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
518 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
519 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
520 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
522 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);  in amdgpu_fence_driver_init_ring()
524 	ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
525 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
526 	ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
529 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
563 static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring)  in amdgpu_fence_need_ring_interrupt_restore()  argument
565 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_need_ring_interrupt_restore()
568 	switch (ring->funcs->type) {  in amdgpu_fence_need_ring_interrupt_restore()
601 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_hw_fini()  local
603 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_fini()
608 			r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_hw_fini()
613 			amdgpu_fence_driver_force_completion(ring);  in amdgpu_fence_driver_hw_fini()
616 		    ring->fence_drv.irq_src &&  in amdgpu_fence_driver_hw_fini()
617 		    amdgpu_fence_need_ring_interrupt_restore(ring))  in amdgpu_fence_driver_hw_fini()
618 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_fini()
619 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_fini()
621 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_hw_fini()
631 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_isr_toggle()  local
633 		if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)  in amdgpu_fence_driver_isr_toggle()
648 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_sw_fini()  local
650 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_sw_fini()
659 		if (ring->sched.ops)  in amdgpu_fence_driver_sw_fini()
660 			drm_sched_fini(&ring->sched);  in amdgpu_fence_driver_sw_fini()
662 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_sw_fini()
663 			dma_fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_sw_fini()
664 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_sw_fini()
665 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_sw_fini()
666 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_sw_fini()
687 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_hw_init()  local
689 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_init()
693 		if (ring->fence_drv.irq_src &&  in amdgpu_fence_driver_hw_init()
694 		    amdgpu_fence_need_ring_interrupt_restore(ring))  in amdgpu_fence_driver_hw_init()
695 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_init()
696 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_init()
706 void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring)  in amdgpu_fence_driver_clear_job_fences()  argument
711 	for (i = 0; i <= ring->fence_drv.num_fences_mask; i++) {  in amdgpu_fence_driver_clear_job_fences()
712 		ptr = &ring->fence_drv.fences[i];  in amdgpu_fence_driver_clear_job_fences()
737 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)  in amdgpu_fence_driver_set_error()  argument
739 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_driver_set_error()
760 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)  in amdgpu_fence_driver_force_completion()  argument
762 	amdgpu_fence_driver_set_error(ring, -ECANCELED);  in amdgpu_fence_driver_force_completion()
763 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
764 	amdgpu_fence_process(ring);  in amdgpu_fence_driver_force_completion()
778 	return (const char *)to_amdgpu_fence(f)->ring->name;  in amdgpu_fence_get_timeline_name()
798 	if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
799 		amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring);  in amdgpu_fence_enable_signaling()
901 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_debugfs_fence_info_show()  local
903 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info_show()
906 		amdgpu_fence_process(ring);  in amdgpu_debugfs_fence_info_show()
908 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);  in amdgpu_debugfs_fence_info_show()
910 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info_show()
912 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info_show()
914 		if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||  in amdgpu_debugfs_fence_info_show()
915 		    ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {  in amdgpu_debugfs_fence_info_show()
917 				   le32_to_cpu(*ring->trail_fence_cpu_addr));  in amdgpu_debugfs_fence_info_show()
919 				   ring->trail_seq);  in amdgpu_debugfs_fence_info_show()
922 		if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)  in amdgpu_debugfs_fence_info_show()
927 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));  in amdgpu_debugfs_fence_info_show()
930 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));  in amdgpu_debugfs_fence_info_show()
933 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));  in amdgpu_debugfs_fence_info_show()