Lines Matching +full:0 +full:x007fffff
57 #define SDMA1_REG_OFFSET 0x600
58 #define SDMA0_HYP_DEC_REG_START 0x5880
59 #define SDMA0_HYP_DEC_REG_END 0x5893
60 #define SDMA1_HYP_DEC_REG_OFFSET 0x20
63 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS_REG),
64 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS1_REG),
65 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS2_REG),
66 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS3_REG),
67 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UCODE_CHECKSUM),
68 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH_HI),
69 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH),
70 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_STATUS),
71 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_STATUS),
72 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK0),
73 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK1),
74 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK0),
75 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK1),
76 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_CNTL),
77 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR),
78 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR_HI),
79 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR),
80 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR_HI),
81 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_OFFSET),
82 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_LO),
83 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_HI),
84 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_CNTL),
85 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_RPTR),
86 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_SUB_REMAIN),
87 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_DUMMY_REG),
88 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_CNTL),
89 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR),
90 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR_HI),
91 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR),
92 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR_HI),
93 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_OFFSET),
94 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_LO),
95 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_HI),
96 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_DUMMY_REG),
97 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_CNTL),
98 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR),
99 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR_HI),
100 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR),
101 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR_HI),
102 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_OFFSET),
103 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_LO),
104 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_HI),
105 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_DUMMY_REG),
106 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_INT_STATUS),
107 SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_VM_CNTL),
108 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2)
117 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_CHICKEN_BITS, 0xffbf1f0f, 0x03ab0107),
118 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
119 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
120 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
121 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
122 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
123 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
124 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
125 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
126 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
127 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
128 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_UTCL1_PAGE, 0x00ffffff, 0x000c5c00),
129 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_CHICKEN_BITS, 0xffbf1f0f, 0x03ab0107),
130 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
131 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
132 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
133 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
134 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
135 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
136 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
137 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
138 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
139 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
140 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_UTCL1_PAGE, 0x00ffffff, 0x000c5c00)
144 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
145 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
146 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
147 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
148 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
149 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
150 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
151 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
152 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
153 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
154 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
155 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
156 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
157 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
158 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
159 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
160 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
161 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
162 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
163 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
167 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
168 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
172 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
173 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
177 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
178 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GB_ADDR_CONFIG, 0x001877ff, 0x00000044),
179 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x001877ff, 0x00000044),
180 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GB_ADDR_CONFIG, 0x001877ff, 0x00000044),
181 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x001877ff, 0x00000044),
182 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
186 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_CHICKEN_BITS, 0xffbf1f0f, 0x03ab0107),
187 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GB_ADDR_CONFIG, 0x001877ff, 0x00000044),
188 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x001877ff, 0x00000044),
189 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
190 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
191 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
192 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
193 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
194 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
195 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
196 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
197 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
198 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
199 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA0_UTCL1_PAGE, 0x007fffff, 0x004c5c00),
200 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_CHICKEN_BITS, 0xffbf1f0f, 0x03ab0107),
201 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GB_ADDR_CONFIG, 0x001877ff, 0x00000044),
202 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x001877ff, 0x00000044),
203 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
204 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
205 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
206 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
207 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
208 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
209 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
210 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
211 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
212 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
213 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSDMA1_UTCL1_PAGE, 0x007fffff, 0x004c5c00)
222 base = adev->reg_offset[GC_HWIP][0][1]; in sdma_v5_0_get_reg_offset()
226 base = adev->reg_offset[GC_HWIP][0][0]; in sdma_v5_0_get_reg_offset()
236 switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { in sdma_v5_0_init_golden_registers()
237 case IP_VERSION(5, 0, 0): in sdma_v5_0_init_golden_registers()
245 case IP_VERSION(5, 0, 2): in sdma_v5_0_init_golden_registers()
253 case IP_VERSION(5, 0, 5): in sdma_v5_0_init_golden_registers()
266 case IP_VERSION(5, 0, 1): in sdma_v5_0_init_golden_registers()
283 * Returns 0 on success, error on failure.
292 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_init_microcode()
313 amdgpu_ring_write(ring, 0); in sdma_v5_0_ring_init_cond_exec()
332 DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr); in sdma_v5_0_ring_get_rptr()
351 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr); in sdma_v5_0_ring_get_wptr()
356 DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n", ring->me, wptr); in sdma_v5_0_ring_get_wptr()
391 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n", in sdma_v5_0_ring_set_wptr()
395 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n", in sdma_v5_0_ring_set_wptr()
406 "wptr_offs == 0x%08x " in sdma_v5_0_ring_set_wptr()
407 "lower_32_bits(ring->wptr) << 2 == 0x%08x " in sdma_v5_0_ring_set_wptr()
408 "upper_32_bits(ring->wptr) << 2 == 0x%08x\n", in sdma_v5_0_ring_set_wptr()
415 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n", in sdma_v5_0_ring_set_wptr()
420 "mmSDMA%i_GFX_RB_WPTR == 0x%08x " in sdma_v5_0_ring_set_wptr()
421 "mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n", in sdma_v5_0_ring_set_wptr()
441 for (i = 0; i < count; i++) in sdma_v5_0_ring_insert_nop()
442 if (sdma && sdma->burst_nop && (i == 0)) in sdma_v5_0_ring_insert_nop()
471 * wptr + 6 + x = 8k, k >= 0, which in C is, in sdma_v5_0_ring_emit_ib()
472 * (wptr + 6 + x) % 8 = 0. in sdma_v5_0_ring_emit_ib()
478 SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf)); in sdma_v5_0_ring_emit_ib()
480 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0); in sdma_v5_0_ring_emit_ib()
502 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0)); in sdma_v5_0_ring_emit_mem_sync()
504 SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0)); in sdma_v5_0_ring_emit_mem_sync()
505 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) | in sdma_v5_0_ring_emit_mem_sync()
507 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) | in sdma_v5_0_ring_emit_mem_sync()
508 SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0)); in sdma_v5_0_ring_emit_mem_sync()
521 u32 ref_and_mask = 0; in sdma_v5_0_ring_emit_hdp_flush()
524 if (ring->me == 0) in sdma_v5_0_ring_emit_hdp_flush()
536 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | in sdma_v5_0_ring_emit_hdp_flush()
558 SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ in sdma_v5_0_ring_emit_fence()
560 BUG_ON(addr & 0x3); in sdma_v5_0_ring_emit_fence()
569 SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); in sdma_v5_0_ring_emit_fence()
571 BUG_ON(addr & 0x3); in sdma_v5_0_ring_emit_fence()
579 (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0; in sdma_v5_0_ring_emit_fence()
599 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_gfx_stop()
601 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0); in sdma_v5_0_gfx_stop()
604 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0); in sdma_v5_0_gfx_stop()
631 u32 f32_cntl = 0, phase_quantum = 0; in sdma_v5_0_ctx_switch_enable()
636 unsigned unit = 0; in sdma_v5_0_ctx_switch_enable()
658 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_ctx_switch_enable()
662 AUTO_CTXSW_ENABLE, enable ? 1 : 0); in sdma_v5_0_ctx_switch_enable()
700 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_enable()
702 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1); in sdma_v5_0_enable()
715 * Return 0 for success.
731 WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0); in sdma_v5_0_gfx_resume_instance()
751 WREG32_SOC15_IP(GC, sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0); in sdma_v5_0_gfx_resume_instance()
752 WREG32_SOC15_IP(GC, sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0); in sdma_v5_0_gfx_resume_instance()
753 WREG32_SOC15_IP(GC, sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0); in sdma_v5_0_gfx_resume_instance()
754 WREG32_SOC15_IP(GC, sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0); in sdma_v5_0_gfx_resume_instance()
772 upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF); in sdma_v5_0_gfx_resume_instance()
774 lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC); in sdma_v5_0_gfx_resume_instance()
784 ring->wptr = 0; in sdma_v5_0_gfx_resume_instance()
805 doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0); in sdma_v5_0_gfx_resume_instance()
817 /* set minor_ptr_update to 0 after wptr programed */ in sdma_v5_0_gfx_resume_instance()
818 WREG32_SOC15_IP(GC, sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0); in sdma_v5_0_gfx_resume_instance()
838 temp &= 0xFF0FFF; in sdma_v5_0_gfx_resume_instance()
846 temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0); in sdma_v5_0_gfx_resume_instance()
876 * Returns 0 for success, error for failure.
882 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_gfx_resume()
888 return 0; in sdma_v5_0_gfx_resume()
897 * Returns 0 for success, error for failure.
901 return 0; in sdma_v5_0_rlc_resume()
910 * Returns 0 for success, -EINVAL if the ucode is not available.
922 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_load_microcode()
934 WREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0); in sdma_v5_0_load_microcode()
936 for (j = 0; j < fw_size; j++) { in sdma_v5_0_load_microcode()
937 if (amdgpu_emu_mode == 1 && j % 500 == 0) in sdma_v5_0_load_microcode()
945 return 0; in sdma_v5_0_load_microcode()
954 * Returns 0 for success, error for failure.
958 int r = 0; in sdma_v5_0_start()
1004 m->sdmax_rlcx_rb_wptr_poll_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, 0, in sdma_v5_0_mqd_init()
1015 m->sdmax_rlcx_ib_cntl = RREG32(sdma_v5_0_get_reg_offset(adev, 0, in sdma_v5_0_mqd_init()
1021 m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_RLC0_DOORBELL, ENABLE, 1); in sdma_v5_0_mqd_init()
1023 return 0; in sdma_v5_0_mqd_init()
1039 * Returns 0 for success, error for failure.
1051 tmp = 0xCAFEDEAD; in sdma_v5_0_ring_test_ring()
1054 uint32_t offset = 0; in sdma_v5_0_ring_test_ring()
1083 amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0)); in sdma_v5_0_ring_test_ring()
1084 amdgpu_ring_write(ring, 0xDEADBEEF); in sdma_v5_0_ring_test_ring()
1087 for (i = 0; i < adev->usec_timeout; i++) { in sdma_v5_0_ring_test_ring()
1092 if (tmp == 0xDEADBEEF) in sdma_v5_0_ring_test_ring()
1116 * Returns 0 on success, error on failure.
1125 u32 tmp = 0; in sdma_v5_0_ring_test_ib()
1129 tmp = 0xCAFEDEAD; in sdma_v5_0_ring_test_ib()
1130 memset(&ib, 0, sizeof(ib)); in sdma_v5_0_ring_test_ib()
1133 uint32_t offset = 0; in sdma_v5_0_ring_test_ib()
1161 ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | in sdma_v5_0_ring_test_ib()
1165 ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0); in sdma_v5_0_ring_test_ib()
1166 ib.ptr[4] = 0xDEADBEEF; in sdma_v5_0_ring_test_ib()
1177 if (r == 0) { in sdma_v5_0_ring_test_ib()
1181 } else if (r < 0) { in sdma_v5_0_ring_test_ib()
1191 if (tmp == 0xDEADBEEF) in sdma_v5_0_ring_test_ib()
1192 r = 0; in sdma_v5_0_ring_test_ib()
1225 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ in sdma_v5_0_vm_copy_pte()
1255 for (; ndw > 0; ndw -= 2) { in sdma_v5_0_vm_write_pte()
1288 ib->ptr[ib->length_dw++] = 0; in sdma_v5_0_vm_set_pte_pde()
1305 pad_count = (-ib->length_dw) & 0x7; in sdma_v5_0_ring_pad_ib()
1306 for (i = 0; i < pad_count; i++) in sdma_v5_0_ring_pad_ib()
1307 if (sdma && sdma->burst_nop && (i == 0)) in sdma_v5_0_ring_pad_ib()
1331 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | in sdma_v5_0_ring_emit_pipeline_sync()
1334 amdgpu_ring_write(ring, addr & 0xfffffffc); in sdma_v5_0_ring_emit_pipeline_sync()
1335 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); in sdma_v5_0_ring_emit_pipeline_sync()
1337 amdgpu_ring_write(ring, 0xffffffff); /* mask */ in sdma_v5_0_ring_emit_pipeline_sync()
1338 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | in sdma_v5_0_ring_emit_pipeline_sync()
1363 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf)); in sdma_v5_0_ring_emit_wreg()
1372 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) | in sdma_v5_0_ring_emit_reg_wait()
1375 amdgpu_ring_write(ring, 0); in sdma_v5_0_ring_emit_reg_wait()
1378 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) | in sdma_v5_0_ring_emit_reg_wait()
1388 amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0); in sdma_v5_0_ring_emit_reg_write_reg_wait()
1407 return 0; in sdma_v5_0_early_init()
1433 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_sw_init()
1441 ring->doorbell_index = (i == 0) ? in sdma_v5_0_sw_init()
1442 (adev->doorbell_index.sdma_engine[0] << 1) //get DWORD offset in sdma_v5_0_sw_init()
1445 ring->vm_hub = AMDGPU_GFXHUB(0); in sdma_v5_0_sw_init()
1448 (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 : in sdma_v5_0_sw_init()
1456 amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring); in sdma_v5_0_sw_init()
1457 switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { in sdma_v5_0_sw_init()
1458 case IP_VERSION(5, 0, 0): in sdma_v5_0_sw_init()
1459 case IP_VERSION(5, 0, 2): in sdma_v5_0_sw_init()
1460 case IP_VERSION(5, 0, 5): in sdma_v5_0_sw_init()
1461 if (adev->sdma.instance[0].fw_version >= 35) in sdma_v5_0_sw_init()
1487 for (i = 0; i < adev->sdma.num_instances; i++) in sdma_v5_0_sw_fini()
1495 return 0; in sdma_v5_0_sw_fini()
1515 return 0; in sdma_v5_0_hw_fini()
1520 return 0; in sdma_v5_0_hw_fini()
1538 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_is_idle()
1554 for (i = 0; i < adev->usec_timeout; i++) { in sdma_v5_0_wait_for_idle()
1555 sdma0 = RREG32(sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG)); in sdma_v5_0_wait_for_idle()
1559 return 0; in sdma_v5_0_wait_for_idle()
1569 return 0; in sdma_v5_0_soft_reset()
1581 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_reset_queue()
1591 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); in sdma_v5_0_reset_queue()
1595 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0); in sdma_v5_0_reset_queue()
1599 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0); in sdma_v5_0_reset_queue()
1607 for (j = 0; j < adev->usec_timeout; j++) { in sdma_v5_0_reset_queue()
1617 if ((stat1_reg & 0x3FF) != 0x3FF) { in sdma_v5_0_reset_queue()
1629 cntl = REG_SET_FIELD(cntl, SDMA0_CNTL, UTC_L1_ENABLE, 0); in sdma_v5_0_reset_queue()
1632 /* soft reset SDMA_GFX_PREEMPT.IB_PREEMPT = 0 mmGRBM_SOFT_RESET.SOFT_RESET_SDMA0/1 = 1 */ in sdma_v5_0_reset_queue()
1634 preempt = REG_SET_FIELD(preempt, SDMA0_GFX_PREEMPT, IB_PREEMPT, 0); in sdma_v5_0_reset_queue()
1637 soft_reset = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); in sdma_v5_0_reset_queue()
1640 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, soft_reset); in sdma_v5_0_reset_queue()
1645 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, soft_reset); in sdma_v5_0_reset_queue()
1649 freeze = REG_SET_FIELD(freeze, SDMA0_FREEZE, FREEZE, 0); in sdma_v5_0_reset_queue()
1655 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); in sdma_v5_0_reset_queue()
1661 int i, r = 0; in sdma_v5_0_ring_preempt_ib()
1663 u32 index = 0; in sdma_v5_0_ring_preempt_ib()
1667 if (index == 0) in sdma_v5_0_ring_preempt_ib()
1679 ring->trail_seq, 0); in sdma_v5_0_ring_preempt_ib()
1686 for (i = 0; i < adev->usec_timeout; i++) { in sdma_v5_0_ring_preempt_ib()
1699 WREG32(sdma_gfx_preempt, 0); in sdma_v5_0_ring_preempt_ib()
1715 sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_CNTL) : in sdma_v5_0_set_trap_irq_state()
1720 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); in sdma_v5_0_set_trap_irq_state()
1724 return 0; in sdma_v5_0_set_trap_irq_state()
1731 uint32_t mes_queue_id = entry->src_data[0]; in sdma_v5_0_process_trap_irq()
1747 return 0; in sdma_v5_0_process_trap_irq()
1753 case 0: in sdma_v5_0_process_trap_irq()
1754 amdgpu_fence_process(&adev->sdma.instance[0].ring); in sdma_v5_0_process_trap_irq()
1769 case 0: in sdma_v5_0_process_trap_irq()
1784 return 0; in sdma_v5_0_process_trap_irq()
1791 return 0; in sdma_v5_0_process_illegal_inst_irq()
1800 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_update_medium_grain_clock_gating()
1837 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_update_medium_grain_light_sleep()
1862 return 0; in sdma_v5_0_set_clockgating_state()
1864 switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { in sdma_v5_0_set_clockgating_state()
1865 case IP_VERSION(5, 0, 0): in sdma_v5_0_set_clockgating_state()
1866 case IP_VERSION(5, 0, 2): in sdma_v5_0_set_clockgating_state()
1867 case IP_VERSION(5, 0, 5): in sdma_v5_0_set_clockgating_state()
1877 return 0; in sdma_v5_0_set_clockgating_state()
1883 return 0; in sdma_v5_0_set_powergating_state()
1892 *flags = 0; in sdma_v5_0_get_clockgating_state()
1895 data = RREG32(sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_CLK_CTRL)); in sdma_v5_0_get_clockgating_state()
1900 data = RREG32(sdma_v5_0_get_reg_offset(adev, 0, mmSDMA0_POWER_CNTL)); in sdma_v5_0_get_clockgating_state()
1916 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_print_ip_state()
1920 for (j = 0; j < reg_count; j++) in sdma_v5_0_print_ip_state()
1921 drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_5_0[j].reg_name, in sdma_v5_0_print_ip_state()
1937 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_dump_ip_state()
1939 for (j = 0; j < reg_count; j++) in sdma_v5_0_dump_ip_state()
1968 .align_mask = 0xf,
2007 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_set_ring_funcs()
2051 SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0); in sdma_v5_0_emit_copy_buffer()
2053 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ in sdma_v5_0_emit_copy_buffer()
2083 .copy_max_bytes = 0x400000,
2087 .fill_max_bytes = 0x400000,
2096 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring; in sdma_v5_0_set_buffer_funcs()
2113 for (i = 0; i < adev->sdma.num_instances; i++) { in sdma_v5_0_set_vm_pte_funcs()
2124 .minor = 0,
2125 .rev = 0,