178023016SChristian König /* 278023016SChristian König * Copyright 2016 Advanced Micro Devices, Inc. 378023016SChristian König * 478023016SChristian König * Permission is hereby granted, free of charge, to any person obtaining a 578023016SChristian König * copy of this software and associated documentation files (the "Software"), 678023016SChristian König * to deal in the Software without restriction, including without limitation 778023016SChristian König * the rights to use, copy, modify, merge, publish, distribute, sublicense, 878023016SChristian König * and/or sell copies of the Software, and to permit persons to whom the 978023016SChristian König * Software is furnished to do so, subject to the following conditions: 1078023016SChristian König * 1178023016SChristian König * The above copyright notice and this permission notice shall be included in 1278023016SChristian König * all copies or substantial portions of the Software. 1378023016SChristian König * 1478023016SChristian König * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1578023016SChristian König * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1678023016SChristian König * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1778023016SChristian König * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 1878023016SChristian König * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 1978023016SChristian König * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 2078023016SChristian König * OTHER DEALINGS IN THE SOFTWARE. 2178023016SChristian König * 2278023016SChristian König * Authors: Christian König 2378023016SChristian König */ 2478023016SChristian König #ifndef __AMDGPU_RING_H__ 2578023016SChristian König #define __AMDGPU_RING_H__ 2678023016SChristian König 27b2ff0e8aSAndres Rodriguez #include <drm/amdgpu_drm.h> 281b1f42d8SLucas Stach #include <drm/gpu_scheduler.h> 2961b100e9SFelix Kuehling #include <drm/drm_print.h> 30c103a23fSMaarten Lankhorst #include <drm/drm_suballoc.h> 3178023016SChristian König 32a190f8dcSChristian König struct amdgpu_device; 33a190f8dcSChristian König struct amdgpu_ring; 34a190f8dcSChristian König struct amdgpu_ib; 35a190f8dcSChristian König struct amdgpu_cs_parser; 36a190f8dcSChristian König struct amdgpu_job; 37a190f8dcSChristian König struct amdgpu_vm; 38a190f8dcSChristian König 3978023016SChristian König /* max number of rings */ 40bb0ed57bSLe Ma #define AMDGPU_MAX_RINGS 124 411bd99ca2SJames Zhu #define AMDGPU_MAX_HWIP_RINGS 64 42a644d85aSHawking Zhang #define AMDGPU_MAX_GFX_RINGS 2 430c97a19aSJiadong.Zhu #define AMDGPU_MAX_SW_GFX_RINGS 2 4478023016SChristian König #define AMDGPU_MAX_COMPUTE_RINGS 8 4578023016SChristian König #define AMDGPU_MAX_VCE_RINGS 3 46f7243053SLeo Liu #define AMDGPU_MAX_UVD_ENC_RINGS 2 472d6ea3b0SHuang Rui #define AMDGPU_MAX_VPE_RINGS 2 4878023016SChristian König 4934eaf30fSNirmoy Das enum amdgpu_ring_priority_level { 5034eaf30fSNirmoy Das AMDGPU_RING_PRIO_0, 5134eaf30fSNirmoy Das AMDGPU_RING_PRIO_1, 5234eaf30fSNirmoy Das AMDGPU_RING_PRIO_DEFAULT = 1, 5334eaf30fSNirmoy Das AMDGPU_RING_PRIO_2, 5434eaf30fSNirmoy Das AMDGPU_RING_PRIO_MAX 5534eaf30fSNirmoy Das }; 561c6d567bSNirmoy Das 5778023016SChristian König /* some special values for the owner field */ 5878023016SChristian König #define AMDGPU_FENCE_OWNER_UNDEFINED ((void *)0ul) 5978023016SChristian König #define AMDGPU_FENCE_OWNER_VM ((void *)1ul) 60d8d019ccSFelix Kuehling #define AMDGPU_FENCE_OWNER_KFD ((void *)2ul) 6178023016SChristian König 6278023016SChristian König #define AMDGPU_FENCE_FLAG_64BIT (1 << 0) 6378023016SChristian König #define AMDGPU_FENCE_FLAG_INT (1 << 1) 64d240cd9eSMarek Olšák #define AMDGPU_FENCE_FLAG_TC_WB_ONLY (1 << 2) 65be254550SJiadong.Zhu #define AMDGPU_FENCE_FLAG_EXEC (1 << 3) 6678023016SChristian König 670e28b10fSChristian König #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched) 680e28b10fSChristian König 699ecefb19SChristian König #define AMDGPU_IB_POOL_SIZE (1024 * 1024) 709ecefb19SChristian König 7178023016SChristian König enum amdgpu_ring_type { 7207e14845SNirmoy Das AMDGPU_RING_TYPE_GFX = AMDGPU_HW_IP_GFX, 7307e14845SNirmoy Das AMDGPU_RING_TYPE_COMPUTE = AMDGPU_HW_IP_COMPUTE, 7407e14845SNirmoy Das AMDGPU_RING_TYPE_SDMA = AMDGPU_HW_IP_DMA, 7507e14845SNirmoy Das AMDGPU_RING_TYPE_UVD = AMDGPU_HW_IP_UVD, 7607e14845SNirmoy Das AMDGPU_RING_TYPE_VCE = AMDGPU_HW_IP_VCE, 7707e14845SNirmoy Das AMDGPU_RING_TYPE_UVD_ENC = AMDGPU_HW_IP_UVD_ENC, 7807e14845SNirmoy Das AMDGPU_RING_TYPE_VCN_DEC = AMDGPU_HW_IP_VCN_DEC, 7907e14845SNirmoy Das AMDGPU_RING_TYPE_VCN_ENC = AMDGPU_HW_IP_VCN_ENC, 8007e14845SNirmoy Das AMDGPU_RING_TYPE_VCN_JPEG = AMDGPU_HW_IP_VCN_JPEG, 812d6ea3b0SHuang Rui AMDGPU_RING_TYPE_VPE = AMDGPU_HW_IP_VPE, 82cdca7979SJack Xiao AMDGPU_RING_TYPE_KIQ, 831a29f367SLang Yu AMDGPU_RING_TYPE_MES, 841a29f367SLang Yu AMDGPU_RING_TYPE_UMSCH_MM, 8578023016SChristian König }; 8678023016SChristian König 879ecefb19SChristian König enum amdgpu_ib_pool_type { 889ecefb19SChristian König /* Normal submissions to the top of the pipeline. */ 899ecefb19SChristian König AMDGPU_IB_POOL_DELAYED, 909ecefb19SChristian König /* Immediate submissions to the bottom of the pipeline. */ 919ecefb19SChristian König AMDGPU_IB_POOL_IMMEDIATE, 929ecefb19SChristian König /* Direct submission to the ring buffer during init and reset. */ 939ecefb19SChristian König AMDGPU_IB_POOL_DIRECT, 949ecefb19SChristian König 959ecefb19SChristian König AMDGPU_IB_POOL_MAX 969ecefb19SChristian König }; 979ecefb19SChristian König 98a190f8dcSChristian König struct amdgpu_ib { 99c103a23fSMaarten Lankhorst struct drm_suballoc *sa_bo; 100a190f8dcSChristian König uint32_t length_dw; 101a190f8dcSChristian König uint64_t gpu_addr; 102a190f8dcSChristian König uint32_t *ptr; 103a190f8dcSChristian König uint32_t flags; 104a190f8dcSChristian König }; 10578023016SChristian König 1061c6d567bSNirmoy Das struct amdgpu_sched { 1071c6d567bSNirmoy Das u32 num_scheds; 1081c6d567bSNirmoy Das struct drm_gpu_scheduler *sched[AMDGPU_MAX_HWIP_RINGS]; 1091c6d567bSNirmoy Das }; 1101c6d567bSNirmoy Das 11178023016SChristian König /* 11278023016SChristian König * Fences. 11378023016SChristian König */ 11478023016SChristian König struct amdgpu_fence_driver { 11578023016SChristian König uint64_t gpu_addr; 11678023016SChristian König volatile uint32_t *cpu_addr; 11778023016SChristian König /* sync_seq is protected by ring emission lock */ 11878023016SChristian König uint32_t sync_seq; 11978023016SChristian König atomic_t last_seq; 12078023016SChristian König bool initialized; 12178023016SChristian König struct amdgpu_irq_src *irq_src; 12278023016SChristian König unsigned irq_type; 1238c5e13ecSAndrey Grodzovsky struct timer_list fallback_timer; 12478023016SChristian König unsigned num_fences_mask; 12578023016SChristian König spinlock_t lock; 126220196b3SDave Airlie struct dma_fence **fences; 12778023016SChristian König }; 12878023016SChristian König 129a190f8dcSChristian König extern const struct drm_sched_backend_ops amdgpu_sched_ops; 130a190f8dcSChristian König 131bf67014dSHuang Rui void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring); 132b13eb02bSChristian König void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error); 1332f9d4084SMonk Liu void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring); 13478023016SChristian König 1355fd8518dSAndrey Grodzovsky int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring); 13678023016SChristian König int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, 13778023016SChristian König struct amdgpu_irq_src *irq_src, 13878023016SChristian König unsigned irq_type); 1398d35a259SLikun Gao void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev); 140067f44c8SGuchun Chen void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev); 141067f44c8SGuchun Chen int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev); 142067f44c8SGuchun Chen void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev); 143c530b02fSJack Zhang int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence, struct amdgpu_job *job, 144d240cd9eSMarek Olšák unsigned flags); 14504e4e2e9SYintian Tao int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s, 14604e4e2e9SYintian Tao uint32_t timeout); 14795d7fc4aSAndrey Grodzovsky bool amdgpu_fence_process(struct amdgpu_ring *ring); 14878023016SChristian König int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); 14943ca8efaSpding signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring, 15043ca8efaSpding uint32_t wait_seq, 15143ca8efaSpding signed long timeout); 15278023016SChristian König unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring); 1533f4c175dSJiadong.Zhu 1549e225fb9SAndrey Grodzovsky void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop); 15578023016SChristian König 1563f4c175dSJiadong.Zhu u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring); 1573f4c175dSJiadong.Zhu void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, 1583f4c175dSJiadong.Zhu ktime_t timestamp); 1593f4c175dSJiadong.Zhu 16078023016SChristian König /* 16178023016SChristian König * Rings. 16278023016SChristian König */ 16378023016SChristian König 16478023016SChristian König /* provided by hw blocks that expose a ring buffer for commands */ 16578023016SChristian König struct amdgpu_ring_funcs { 16621cd942eSChristian König enum amdgpu_ring_type type; 16779887142SChristian König uint32_t align_mask; 16879887142SChristian König u32 nop; 169536fbf94SKen Wang bool support_64bit_ptrs; 170120c2125SLeo Liu bool no_user_fence; 1718c0f11ffSLang Yu bool secure_submission_supported; 172c8c1a1d2SBoyuan Zhang unsigned extra_dw; 17321cd942eSChristian König 17478023016SChristian König /* ring read/write ptr handling */ 175536fbf94SKen Wang u64 (*get_rptr)(struct amdgpu_ring *ring); 176536fbf94SKen Wang u64 (*get_wptr)(struct amdgpu_ring *ring); 17778023016SChristian König void (*set_wptr)(struct amdgpu_ring *ring); 17878023016SChristian König /* validating and patching of IBs */ 179cdc7893fSChristian König int (*parse_cs)(struct amdgpu_cs_parser *p, 180cdc7893fSChristian König struct amdgpu_job *job, 181cdc7893fSChristian König struct amdgpu_ib *ib); 182cdc7893fSChristian König int (*patch_cs_in_place)(struct amdgpu_cs_parser *p, 183cdc7893fSChristian König struct amdgpu_job *job, 184cdc7893fSChristian König struct amdgpu_ib *ib); 185e12f3d7aSChristian König /* constants to calculate how many DW are needed for an emit */ 186e12f3d7aSChristian König unsigned emit_frame_size; 187e12f3d7aSChristian König unsigned emit_ib_size; 18878023016SChristian König /* command emit functions */ 18978023016SChristian König void (*emit_ib)(struct amdgpu_ring *ring, 19034955e03SRex Zhu struct amdgpu_job *job, 19178023016SChristian König struct amdgpu_ib *ib, 192c4c905ecSJack Xiao uint32_t flags); 19378023016SChristian König void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr, 19478023016SChristian König uint64_t seq, unsigned flags); 19578023016SChristian König void (*emit_pipeline_sync)(struct amdgpu_ring *ring); 196c4f46f22SChristian König void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vmid, 197c633c00bSChristian König uint64_t pd_addr); 19878023016SChristian König void (*emit_hdp_flush)(struct amdgpu_ring *ring); 19978023016SChristian König void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid, 20078023016SChristian König uint32_t gds_base, uint32_t gds_size, 20178023016SChristian König uint32_t gws_base, uint32_t gws_size, 20278023016SChristian König uint32_t oa_base, uint32_t oa_size); 20378023016SChristian König /* testing functions */ 20478023016SChristian König int (*test_ring)(struct amdgpu_ring *ring); 20578023016SChristian König int (*test_ib)(struct amdgpu_ring *ring, long timeout); 20678023016SChristian König /* insert NOP packets */ 20778023016SChristian König void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count); 208ef44f854SLeo Liu void (*insert_start)(struct amdgpu_ring *ring); 209135d4735SLeo Liu void (*insert_end)(struct amdgpu_ring *ring); 21078023016SChristian König /* pad the indirect buffer to the necessary number of dw */ 21178023016SChristian König void (*pad_ib)(struct amdgpu_ring *ring, struct amdgpu_ib *ib); 212c68cbbfdSChristian König unsigned (*init_cond_exec)(struct amdgpu_ring *ring, uint64_t addr); 21378023016SChristian König /* note usage for clock and power gating */ 21478023016SChristian König void (*begin_use)(struct amdgpu_ring *ring); 21578023016SChristian König void (*end_use)(struct amdgpu_ring *ring); 21678023016SChristian König void (*emit_switch_buffer) (struct amdgpu_ring *ring); 2170bb5d5b0SLuben Tuikov void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags); 218ac928705SChristian König void (*emit_gfx_shadow)(struct amdgpu_ring *ring, u64 shadow_va, u64 csa_va, 219ac928705SChristian König u64 gds_va, bool init_shadow, int vmid); 22054208194SYintian Tao void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg, 22154208194SYintian Tao uint32_t reg_val_offs); 222b6091c12SXiangliang Yu void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val); 223c1e877daSChristian König void (*emit_reg_wait)(struct amdgpu_ring *ring, uint32_t reg, 224c1e877daSChristian König uint32_t val, uint32_t mask); 22582853638SAlex Deucher void (*emit_reg_write_reg_wait)(struct amdgpu_ring *ring, 22682853638SAlex Deucher uint32_t reg0, uint32_t reg1, 22782853638SAlex Deucher uint32_t ref, uint32_t mask); 228f77c9affSHuang Rui void (*emit_frame_cntl)(struct amdgpu_ring *ring, bool start, 229f77c9affSHuang Rui bool secure); 2307876fa4fSChristian König /* Try to soft recover the ring to make the fence signal */ 2317876fa4fSChristian König void (*soft_recovery)(struct amdgpu_ring *ring, unsigned vmid); 232692441f2SRex Zhu int (*preempt_ib)(struct amdgpu_ring *ring); 23322301177SAndrey Grodzovsky void (*emit_mem_sync)(struct amdgpu_ring *ring); 2340a52a6caSNirmoy Das void (*emit_wave_limit)(struct amdgpu_ring *ring, bool enable); 2358ff865beSJiadong Zhu void (*patch_cntl)(struct amdgpu_ring *ring, unsigned offset); 2368ff865beSJiadong Zhu void (*patch_ce)(struct amdgpu_ring *ring, unsigned offset); 2378ff865beSJiadong Zhu void (*patch_de)(struct amdgpu_ring *ring, unsigned offset); 23857a372f6SAlex Deucher int (*reset)(struct amdgpu_ring *ring, unsigned int vmid); 239*ee7a846eSAlex Deucher void (*emit_cleaner_shader)(struct amdgpu_ring *ring); 24078023016SChristian König }; 24178023016SChristian König 24278023016SChristian König struct amdgpu_ring { 24378023016SChristian König struct amdgpu_device *adev; 24478023016SChristian König const struct amdgpu_ring_funcs *funcs; 24578023016SChristian König struct amdgpu_fence_driver fence_drv; 2461b1f42d8SLucas Stach struct drm_gpu_scheduler sched; 24778023016SChristian König 24878023016SChristian König struct amdgpu_bo *ring_obj; 24978023016SChristian König volatile uint32_t *ring; 25078023016SChristian König unsigned rptr_offs; 251d74c5b06SJack Xiao u64 rptr_gpu_addr; 252d74c5b06SJack Xiao volatile u32 *rptr_cpu_addr; 253536fbf94SKen Wang u64 wptr; 254536fbf94SKen Wang u64 wptr_old; 25578023016SChristian König unsigned ring_size; 25678023016SChristian König unsigned max_dw; 25778023016SChristian König int count_dw; 25878023016SChristian König uint64_t gpu_addr; 259536fbf94SKen Wang uint64_t ptr_mask; 260536fbf94SKen Wang uint32_t buf_mask; 26178023016SChristian König u32 idx; 262541372bbSLe Ma u32 xcc_id; 263797a0a14SJames Zhu u32 xcp_id; 26478023016SChristian König u32 me; 26578023016SChristian König u32 pipe; 26678023016SChristian König u32 queue; 26778023016SChristian König struct amdgpu_bo *mqd_obj; 268f3972b53SMonk Liu uint64_t mqd_gpu_addr; 26959a82d7dSXiangliang Yu void *mqd_ptr; 270b185c318SAlex Deucher unsigned mqd_size; 27134534610SAlex Deucher uint64_t eop_gpu_addr; 27278023016SChristian König u32 doorbell_index; 27378023016SChristian König bool use_doorbell; 2742ffe31deSPixel Ding bool use_pollmem; 27578023016SChristian König unsigned wptr_offs; 276d74c5b06SJack Xiao u64 wptr_gpu_addr; 277d74c5b06SJack Xiao volatile u32 *wptr_cpu_addr; 27878023016SChristian König unsigned fence_offs; 279d74c5b06SJack Xiao u64 fence_gpu_addr; 280d74c5b06SJack Xiao volatile u32 *fence_cpu_addr; 28178023016SChristian König uint64_t current_ctx; 28278023016SChristian König char name[16]; 283ef3e1323SJack Xiao u32 trail_seq; 284ef3e1323SJack Xiao unsigned trail_fence_offs; 285ef3e1323SJack Xiao u64 trail_fence_gpu_addr; 286ef3e1323SJack Xiao volatile u32 *trail_fence_cpu_addr; 28778023016SChristian König unsigned cond_exe_offs; 28878023016SChristian König u64 cond_exe_gpu_addr; 28978023016SChristian König volatile u32 *cond_exe_cpu_addr; 2908bc75586SChristian König unsigned int set_q_mode_offs; 2918bc75586SChristian König volatile u32 *set_q_mode_ptr; 2928bc75586SChristian König u64 set_q_mode_token; 2930530553bSLe Ma unsigned vm_hub; 2944789c463SChristian König unsigned vm_inv_eng; 2953af81440SChristian König struct dma_fence *vmid_wait; 296dd684d31SAlex Xie bool has_compute_vm_bug; 297cb3d1085SAlex Deucher bool no_scheduler; 298ebdd2e9dSNirmoy Das int hw_prio; 2995fd8518dSAndrey Grodzovsky unsigned num_hw_submission; 3005fd8518dSAndrey Grodzovsky atomic_t *sched_score; 301c6abbcbcSJack Xiao 302c6abbcbcSJack Xiao /* used for mes */ 303c6abbcbcSJack Xiao bool is_mes_queue; 304c6abbcbcSJack Xiao uint32_t hw_queue_id; 305c6abbcbcSJack Xiao struct amdgpu_mes_ctx_data *mes_ctx; 306ded946f3SJiadong.Zhu 307ded946f3SJiadong.Zhu bool is_sw_ring; 308ded946f3SJiadong.Zhu unsigned int entry_index; 309ded946f3SJiadong.Zhu 31078023016SChristian König }; 31178023016SChristian König 312cdc7893fSChristian König #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib))) 313cdc7893fSChristian König #define amdgpu_ring_patch_cs_in_place(r, p, job, ib) ((r)->funcs->patch_cs_in_place((p), (job), (ib))) 3140a7845dbSHuang Rui #define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r)) 3156c1a6d0bSJesseZhang #define amdgpu_ring_test_ib(r, t) ((r)->funcs->test_ib ? (r)->funcs->test_ib((r), (t)) : 0) 3160a7845dbSHuang Rui #define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r)) 3170a7845dbSHuang Rui #define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r)) 3180a7845dbSHuang Rui #define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r)) 319c4c905ecSJack Xiao #define amdgpu_ring_emit_ib(r, job, ib, flags) ((r)->funcs->emit_ib((r), (job), (ib), (flags))) 3200a7845dbSHuang Rui #define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r)) 3210a7845dbSHuang Rui #define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr)) 3220a7845dbSHuang Rui #define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags)) 3230a7845dbSHuang Rui #define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as)) 3240a7845dbSHuang Rui #define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r)) 3250a7845dbSHuang Rui #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r)) 3260bb5d5b0SLuben Tuikov #define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d)) 327ac928705SChristian König #define amdgpu_ring_emit_gfx_shadow(r, s, c, g, i, v) ((r)->funcs->emit_gfx_shadow((r), (s), (c), (g), (i), (v))) 32854208194SYintian Tao #define amdgpu_ring_emit_rreg(r, d, o) (r)->funcs->emit_rreg((r), (d), (o)) 3290a7845dbSHuang Rui #define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v)) 3300a7845dbSHuang Rui #define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m)) 3310a7845dbSHuang Rui #define amdgpu_ring_emit_reg_write_reg_wait(r, d0, d1, v, m) (r)->funcs->emit_reg_write_reg_wait((r), (d0), (d1), (v), (m)) 332f77c9affSHuang Rui #define amdgpu_ring_emit_frame_cntl(r, b, s) (r)->funcs->emit_frame_cntl((r), (b), (s)) 3330a7845dbSHuang Rui #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib))) 334c68cbbfdSChristian König #define amdgpu_ring_init_cond_exec(r, a) (r)->funcs->init_cond_exec((r), (a)) 335692441f2SRex Zhu #define amdgpu_ring_preempt_ib(r) (r)->funcs->preempt_ib(r) 3368ff865beSJiadong Zhu #define amdgpu_ring_patch_cntl(r, o) ((r)->funcs->patch_cntl((r), (o))) 3378ff865beSJiadong Zhu #define amdgpu_ring_patch_ce(r, o) ((r)->funcs->patch_ce((r), (o))) 3388ff865beSJiadong Zhu #define amdgpu_ring_patch_de(r, o) ((r)->funcs->patch_de((r), (o))) 33957a372f6SAlex Deucher #define amdgpu_ring_reset(r, v) (r)->funcs->reset((r), (v)) 3400a7845dbSHuang Rui 341c30ddcecSBas Nieuwenhuizen unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type); 34278023016SChristian König int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw); 3433f4c175dSJiadong.Zhu void amdgpu_ring_ib_begin(struct amdgpu_ring *ring); 3443f4c175dSJiadong.Zhu void amdgpu_ring_ib_end(struct amdgpu_ring *ring); 3458ff865beSJiadong Zhu void amdgpu_ring_ib_on_emit_cntl(struct amdgpu_ring *ring); 3468ff865beSJiadong Zhu void amdgpu_ring_ib_on_emit_ce(struct amdgpu_ring *ring); 3478ff865beSJiadong Zhu void amdgpu_ring_ib_on_emit_de(struct amdgpu_ring *ring); 3483f4c175dSJiadong.Zhu 34978023016SChristian König void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count); 35078023016SChristian König void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib); 35178023016SChristian König void amdgpu_ring_commit(struct amdgpu_ring *ring); 35278023016SChristian König void amdgpu_ring_undo(struct amdgpu_ring *ring); 35378023016SChristian König int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, 354cf8cc382SMa Jun unsigned int max_dw, struct amdgpu_irq_src *irq_src, 355cf8cc382SMa Jun unsigned int irq_type, unsigned int hw_prio, 356c107171bSChristian König atomic_t *sched_score); 35778023016SChristian König void amdgpu_ring_fini(struct amdgpu_ring *ring); 35882853638SAlex Deucher void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring, 35982853638SAlex Deucher uint32_t reg0, uint32_t val0, 36082853638SAlex Deucher uint32_t reg1, uint32_t val1); 3617876fa4fSChristian König bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, 3627876fa4fSChristian König struct dma_fence *fence); 36382853638SAlex Deucher 364dfc98479SRex Zhu static inline void amdgpu_ring_set_preempt_cond_exec(struct amdgpu_ring *ring, 365dfc98479SRex Zhu bool cond_exec) 366dfc98479SRex Zhu { 367dfc98479SRex Zhu *ring->cond_exe_cpu_addr = cond_exec; 368dfc98479SRex Zhu } 369dfc98479SRex Zhu 370c79ecfbfSMonk Liu static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring) 371c79ecfbfSMonk Liu { 372c79ecfbfSMonk Liu int i = 0; 373e09706f4SMonk Liu while (i <= ring->buf_mask) 374c79ecfbfSMonk Liu ring->ring[i++] = ring->funcs->nop; 375c79ecfbfSMonk Liu 376c79ecfbfSMonk Liu } 37778023016SChristian König 378e8110b1cSChristian König static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v) 379e8110b1cSChristian König { 380e8110b1cSChristian König if (ring->count_dw <= 0) 381e8110b1cSChristian König DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n"); 382e8110b1cSChristian König ring->ring[ring->wptr++ & ring->buf_mask] = v; 383e8110b1cSChristian König ring->wptr &= ring->ptr_mask; 384e8110b1cSChristian König ring->count_dw--; 385e8110b1cSChristian König } 386e8110b1cSChristian König 387e8110b1cSChristian König static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, 388e8110b1cSChristian König void *src, int count_dw) 389e8110b1cSChristian König { 390e8110b1cSChristian König unsigned occupied, chunk1, chunk2; 391e8110b1cSChristian König void *dst; 392e8110b1cSChristian König 393369421cbSChristian König if (unlikely(ring->count_dw < count_dw)) 394e8110b1cSChristian König DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n"); 395e8110b1cSChristian König 396e8110b1cSChristian König occupied = ring->wptr & ring->buf_mask; 397e8110b1cSChristian König dst = (void *)&ring->ring[occupied]; 398e8110b1cSChristian König chunk1 = ring->buf_mask + 1 - occupied; 399e8110b1cSChristian König chunk1 = (chunk1 >= count_dw) ? count_dw : chunk1; 400e8110b1cSChristian König chunk2 = count_dw - chunk1; 401e8110b1cSChristian König chunk1 <<= 2; 402e8110b1cSChristian König chunk2 <<= 2; 403e8110b1cSChristian König 404e8110b1cSChristian König if (chunk1) 405e8110b1cSChristian König memcpy(dst, src, chunk1); 406e8110b1cSChristian König 407e8110b1cSChristian König if (chunk2) { 408e8110b1cSChristian König src += chunk1; 409e8110b1cSChristian König dst = (void *)ring->ring; 410e8110b1cSChristian König memcpy(dst, src, chunk2); 411e8110b1cSChristian König } 412e8110b1cSChristian König 413e8110b1cSChristian König ring->wptr += count_dw; 414e8110b1cSChristian König ring->wptr &= ring->ptr_mask; 415e8110b1cSChristian König ring->count_dw -= count_dw; 416e8110b1cSChristian König } 417e8110b1cSChristian König 418c68cbbfdSChristian König /** 419c68cbbfdSChristian König * amdgpu_ring_patch_cond_exec - patch dw count of conditional execute 420c68cbbfdSChristian König * @ring: amdgpu_ring structure 421c68cbbfdSChristian König * @offset: offset returned by amdgpu_ring_init_cond_exec 422c68cbbfdSChristian König * 423c68cbbfdSChristian König * Calculate the dw count and patch it into a cond_exec command. 424c68cbbfdSChristian König */ 425c68cbbfdSChristian König static inline void amdgpu_ring_patch_cond_exec(struct amdgpu_ring *ring, 426c68cbbfdSChristian König unsigned int offset) 427c68cbbfdSChristian König { 428c68cbbfdSChristian König unsigned cur; 429c68cbbfdSChristian König 430c68cbbfdSChristian König if (!ring->funcs->init_cond_exec) 431c68cbbfdSChristian König return; 432c68cbbfdSChristian König 433c68cbbfdSChristian König WARN_ON(offset > ring->buf_mask); 434c68cbbfdSChristian König WARN_ON(ring->ring[offset] != 0); 435c68cbbfdSChristian König 436c68cbbfdSChristian König cur = (ring->wptr - 1) & ring->buf_mask; 437c68cbbfdSChristian König if (cur < offset) 438c68cbbfdSChristian König cur += ring->ring_size >> 2; 439c68cbbfdSChristian König ring->ring[offset] = cur - offset; 440c68cbbfdSChristian König } 441c68cbbfdSChristian König 4422bc956efSJack Xiao #define amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset) \ 4432bc956efSJack Xiao (ring->is_mes_queue && ring->mes_ctx ? \ 4442bc956efSJack Xiao (ring->mes_ctx->meta_data_gpu_addr + offset) : 0) 4452bc956efSJack Xiao 4462bc956efSJack Xiao #define amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset) \ 4472bc956efSJack Xiao (ring->is_mes_queue && ring->mes_ctx ? \ 4482bc956efSJack Xiao (void *)((uint8_t *)(ring->mes_ctx->meta_data_ptr) + offset) : \ 4492bc956efSJack Xiao NULL) 4502bc956efSJack Xiao 451c66ed765SAndrey Grodzovsky int amdgpu_ring_test_helper(struct amdgpu_ring *ring); 452c66ed765SAndrey Grodzovsky 45362d266b2SNirmoy Das void amdgpu_debugfs_ring_init(struct amdgpu_device *adev, 454fd23cfccSAlex Deucher struct amdgpu_ring *ring); 455a190f8dcSChristian König 45680af9daaSJack Xiao int amdgpu_ring_init_mqd(struct amdgpu_ring *ring); 45780af9daaSJack Xiao 458cdc7893fSChristian König static inline u32 amdgpu_ib_get_value(struct amdgpu_ib *ib, int idx) 459cdc7893fSChristian König { 460cdc7893fSChristian König return ib->ptr[idx]; 461cdc7893fSChristian König } 462cdc7893fSChristian König 463cdc7893fSChristian König static inline void amdgpu_ib_set_value(struct amdgpu_ib *ib, int idx, 464cdc7893fSChristian König uint32_t value) 465cdc7893fSChristian König { 466cdc7893fSChristian König ib->ptr[idx] = value; 467cdc7893fSChristian König } 468cdc7893fSChristian König 469a190f8dcSChristian König int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, 470a190f8dcSChristian König unsigned size, 471a190f8dcSChristian König enum amdgpu_ib_pool_type pool, 472a190f8dcSChristian König struct amdgpu_ib *ib); 473a190f8dcSChristian König void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib, 474a190f8dcSChristian König struct dma_fence *f); 475a190f8dcSChristian König int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, 476a190f8dcSChristian König struct amdgpu_ib *ibs, struct amdgpu_job *job, 477a190f8dcSChristian König struct dma_fence **f); 478a190f8dcSChristian König int amdgpu_ib_pool_init(struct amdgpu_device *adev); 479a190f8dcSChristian König void amdgpu_ib_pool_fini(struct amdgpu_device *adev); 480a190f8dcSChristian König int amdgpu_ib_ring_tests(struct amdgpu_device *adev); 4819749c868SMa Jun bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring); 48278023016SChristian König #endif 483