xref: /linux/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29 
30 #include "amdgpu.h"
31 #include "amdgpu_gfx.h"
32 #include "soc15.h"
33 #include "soc15d.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_pm.h"
36 
37 #include "gc/gc_9_0_offset.h"
38 #include "gc/gc_9_0_sh_mask.h"
39 
40 #include "vega10_enum.h"
41 
42 #include "soc15_common.h"
43 #include "clearstate_gfx9.h"
44 #include "v9_structs.h"
45 
46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47 
48 #include "amdgpu_ras.h"
49 
50 #include "amdgpu_ring_mux.h"
51 #include "gfx_v9_4.h"
52 #include "gfx_v9_0.h"
53 #include "gfx_v9_0_cleaner_shader.h"
54 #include "gfx_v9_4_2.h"
55 
56 #include "asic_reg/pwr/pwr_10_0_offset.h"
57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
58 #include "asic_reg/gc/gc_9_0_default.h"
59 
60 #define GFX9_NUM_GFX_RINGS     1
61 #define GFX9_NUM_SW_GFX_RINGS  2
62 #define GFX9_MEC_HPD_SIZE 4096
63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
65 
66 #define mmGCEA_PROBE_MAP                        0x070c
67 #define mmGCEA_PROBE_MAP_BASE_IDX               0
68 
69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
71 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
75 
76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
78 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
82 
83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
85 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
89 
90 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
92 MODULE_FIRMWARE("amdgpu/raven_me.bin");
93 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
96 
97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
104 
105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
107 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
112 
113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
115 
116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
118 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
121 
122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
128 
129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
134 
135 #define mmTCP_CHAN_STEER_0_ARCT								0x0b03
136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX							0
137 #define mmTCP_CHAN_STEER_1_ARCT								0x0b04
138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX							0
139 #define mmTCP_CHAN_STEER_2_ARCT								0x0b09
140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX							0
141 #define mmTCP_CHAN_STEER_3_ARCT								0x0b0a
142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX							0
143 #define mmTCP_CHAN_STEER_4_ARCT								0x0b0b
144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX							0
145 #define mmTCP_CHAN_STEER_5_ARCT								0x0b0c
146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX							0
147 
148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir                0x0025
149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX       1
150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir                0x0026
151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX       1
152 
153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = {
154 	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),
155 	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),
156 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),
157 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),
158 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),
159 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),
160 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),
161 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),
162 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),
163 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),
164 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),
165 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),
166 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),
167 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),
168 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),
169 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),
170 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),
171 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),
172 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),
173 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),
174 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),
175 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
176 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
177 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),
178 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),
179 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),
180 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),
181 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),
182 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),
183 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),
184 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),
185 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),
186 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),
187 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),
188 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),
189 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),
190 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),
191 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),
192 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),
193 	SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),
194 	SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),
195 	SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),
196 	SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),
197 	SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),
198 	SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),
199 	SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL),
200 	SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),
201 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),
202 	SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),
203 	SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS),
204 	SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS),
205 	SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS),
206 	SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS),
207 	SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),
208 	SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL),
209 	SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS),
210 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),
211 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),
212 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),
213 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),
214 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),
215 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),
216 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),
217 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),
218 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),
219 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),
220 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),
221 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),
222 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),
223 	SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),
224 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),
225 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),
226 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),
227 	SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),
228 	/* SE status registers */
229 	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0),
230 	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1),
231 	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2),
232 	SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3),
233 	/* packet headers */
234 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
235 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
236 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
237 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
238 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
239 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
240 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
241 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
242 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
243 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
244 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
245 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
246 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
247 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
248 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
249 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
250 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
251 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
252 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
253 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
254 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
255 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
256 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
257 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP)
258 };
259 
260 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = {
261 	/* compute queue registers */
262 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID),
263 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE),
264 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE),
265 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY),
266 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY),
267 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM),
268 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE),
269 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI),
270 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR),
271 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
272 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
273 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
274 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL),
275 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR),
276 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI),
277 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR),
278 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL),
279 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),
280 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
281 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),
282 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL),
283 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR),
284 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR),
285 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS),
286 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO),
287 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI),
288 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL),
289 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET),
290 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE),
291 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET),
292 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE),
293 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE),
294 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR),
295 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM),
296 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO),
297 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI),
298 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS),
299 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
300 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
301 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
302 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
303 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
304 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
305 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
306 	SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP)
307 };
308 
309 enum ta_ras_gfx_subblock {
310 	/*CPC*/
311 	TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
312 	TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
313 	TA_RAS_BLOCK__GFX_CPC_UCODE,
314 	TA_RAS_BLOCK__GFX_DC_STATE_ME1,
315 	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
316 	TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
317 	TA_RAS_BLOCK__GFX_DC_STATE_ME2,
318 	TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
319 	TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
320 	TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
321 	/* CPF*/
322 	TA_RAS_BLOCK__GFX_CPF_INDEX_START,
323 	TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
324 	TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
325 	TA_RAS_BLOCK__GFX_CPF_TAG,
326 	TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
327 	/* CPG*/
328 	TA_RAS_BLOCK__GFX_CPG_INDEX_START,
329 	TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
330 	TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
331 	TA_RAS_BLOCK__GFX_CPG_TAG,
332 	TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
333 	/* GDS*/
334 	TA_RAS_BLOCK__GFX_GDS_INDEX_START,
335 	TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
336 	TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
337 	TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
338 	TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
339 	TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
340 	TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
341 	/* SPI*/
342 	TA_RAS_BLOCK__GFX_SPI_SR_MEM,
343 	/* SQ*/
344 	TA_RAS_BLOCK__GFX_SQ_INDEX_START,
345 	TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
346 	TA_RAS_BLOCK__GFX_SQ_LDS_D,
347 	TA_RAS_BLOCK__GFX_SQ_LDS_I,
348 	TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
349 	TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
350 	/* SQC (3 ranges)*/
351 	TA_RAS_BLOCK__GFX_SQC_INDEX_START,
352 	/* SQC range 0*/
353 	TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
354 	TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
355 		TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
356 	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
357 	TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
358 	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
359 	TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
360 	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
361 	TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
362 	TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
363 		TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
364 	/* SQC range 1*/
365 	TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
366 	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
367 		TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
368 	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
369 	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
370 	TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
371 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
372 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
373 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
374 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
375 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
376 	TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
377 		TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
378 	/* SQC range 2*/
379 	TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
380 	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
381 		TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
382 	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
383 	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
384 	TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
385 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
386 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
387 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
388 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
389 	TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
390 	TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
391 		TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
392 	TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
393 	/* TA*/
394 	TA_RAS_BLOCK__GFX_TA_INDEX_START,
395 	TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
396 	TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
397 	TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
398 	TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
399 	TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
400 	TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
401 	/* TCA*/
402 	TA_RAS_BLOCK__GFX_TCA_INDEX_START,
403 	TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
404 	TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
405 	TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
406 	/* TCC (5 sub-ranges)*/
407 	TA_RAS_BLOCK__GFX_TCC_INDEX_START,
408 	/* TCC range 0*/
409 	TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
410 	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
411 	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
412 	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
413 	TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
414 	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
415 	TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
416 	TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
417 	TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
418 	TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
419 	/* TCC range 1*/
420 	TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
421 	TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
422 	TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
423 	TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
424 		TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
425 	/* TCC range 2*/
426 	TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
427 	TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
428 	TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
429 	TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
430 	TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
431 	TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
432 	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
433 	TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
434 	TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
435 	TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
436 		TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
437 	/* TCC range 3*/
438 	TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
439 	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
440 	TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
441 	TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
442 		TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
443 	/* TCC range 4*/
444 	TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
445 	TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
446 		TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
447 	TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
448 	TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
449 		TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
450 	TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
451 	/* TCI*/
452 	TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
453 	/* TCP*/
454 	TA_RAS_BLOCK__GFX_TCP_INDEX_START,
455 	TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
456 	TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
457 	TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
458 	TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
459 	TA_RAS_BLOCK__GFX_TCP_DB_RAM,
460 	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
461 	TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
462 	TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
463 	/* TD*/
464 	TA_RAS_BLOCK__GFX_TD_INDEX_START,
465 	TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
466 	TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
467 	TA_RAS_BLOCK__GFX_TD_CS_FIFO,
468 	TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
469 	/* EA (3 sub-ranges)*/
470 	TA_RAS_BLOCK__GFX_EA_INDEX_START,
471 	/* EA range 0*/
472 	TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
473 	TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
474 	TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
475 	TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
476 	TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
477 	TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
478 	TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
479 	TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
480 	TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
481 	TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
482 	/* EA range 1*/
483 	TA_RAS_BLOCK__GFX_EA_INDEX1_START,
484 	TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
485 	TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
486 	TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
487 	TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
488 	TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
489 	TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
490 	TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
491 	TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
492 	/* EA range 2*/
493 	TA_RAS_BLOCK__GFX_EA_INDEX2_START,
494 	TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
495 	TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
496 	TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
497 	TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
498 	TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
499 	TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
500 	/* UTC VM L2 bank*/
501 	TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
502 	/* UTC VM walker*/
503 	TA_RAS_BLOCK__UTC_VML2_WALKER,
504 	/* UTC ATC L2 2MB cache*/
505 	TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
506 	/* UTC ATC L2 4KB cache*/
507 	TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
508 	TA_RAS_BLOCK__GFX_MAX
509 };
510 
511 struct ras_gfx_subblock {
512 	unsigned char *name;
513 	int ta_subblock;
514 	int hw_supported_error_type;
515 	int sw_supported_error_type;
516 };
517 
518 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h)                             \
519 	[AMDGPU_RAS_BLOCK__##subblock] = {                                     \
520 		#subblock,                                                     \
521 		TA_RAS_BLOCK__##subblock,                                      \
522 		((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)),                  \
523 		(((e) << 1) | ((f) << 3) | (g) | ((h) << 2)),                  \
524 	}
525 
526 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
527 	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
528 	AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
529 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
530 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
531 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
532 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
533 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
534 	AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
535 	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
536 	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
537 	AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
538 	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
539 	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
540 	AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
541 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
542 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
543 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
544 			     0),
545 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
546 			     0),
547 	AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
548 	AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
549 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
550 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
551 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
552 	AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
553 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
554 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
555 			     0, 0),
556 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
557 			     0),
558 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
559 			     0, 0),
560 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
561 			     0),
562 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
563 			     0, 0),
564 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
565 			     0),
566 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
567 			     1),
568 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
569 			     0, 0, 0),
570 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
571 			     0),
572 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
573 			     0),
574 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
575 			     0),
576 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
577 			     0),
578 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
579 			     0),
580 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
581 			     0, 0),
582 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
583 			     0),
584 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
585 			     0),
586 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
587 			     0, 0, 0),
588 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
589 			     0),
590 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
591 			     0),
592 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
593 			     0),
594 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
595 			     0),
596 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
597 			     0),
598 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
599 			     0, 0),
600 	AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
601 			     0),
602 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
603 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
604 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
605 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
606 	AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
607 	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
608 	AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
609 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
610 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
611 			     1),
612 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
613 			     1),
614 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
615 			     1),
616 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
617 			     0),
618 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
619 			     0),
620 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
621 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
622 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
623 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
624 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
625 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
626 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
627 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
628 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
629 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
630 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
631 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
632 			     0),
633 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
634 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
635 			     0),
636 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
637 			     0, 0),
638 	AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
639 			     0),
640 	AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
641 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
642 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
643 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
644 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
645 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
646 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
647 	AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
648 	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
649 	AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
650 	AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
651 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
652 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
653 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
654 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
655 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
656 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
657 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
658 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
659 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
660 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
661 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
662 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
663 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
664 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
665 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
666 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
667 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
668 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
669 	AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
670 	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
671 	AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
672 	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
673 	AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
674 };
675 
676 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
677 {
678 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
679 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
680 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
681 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
682 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
683 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
684 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
685 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
686 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
687 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
688 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
689 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
690 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
691 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
692 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
693 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
694 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
695 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
696 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
697 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
698 };
699 
700 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
701 {
702 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
703 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
704 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
705 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
706 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
707 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
708 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
709 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
710 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
711 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
712 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
713 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
714 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
715 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
716 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
717 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
718 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
719 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
720 };
721 
722 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
723 {
724 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
725 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
726 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
727 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
728 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
729 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
730 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
731 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
732 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
733 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
734 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
735 };
736 
737 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
738 {
739 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
740 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
741 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
742 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
743 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
744 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
745 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
746 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
747 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
748 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
749 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
750 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
751 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
752 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
753 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
754 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
755 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
756 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
757 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
758 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
759 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
760 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
761 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
762 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
763 };
764 
765 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
766 {
767 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
768 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
769 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
770 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
771 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
772 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
773 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
774 };
775 
776 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
777 {
778 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
779 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
780 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
781 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
782 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
783 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
784 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
785 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
786 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
787 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
788 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
789 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
790 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
791 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
792 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
793 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
794 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
795 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
796 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
797 };
798 
799 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
800 {
801 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
802 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
803 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
804 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
805 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
806 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
807 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
808 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
809 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
810 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
811 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
812 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
813 };
814 
815 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
816 {
817 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
818 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
819 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
820 };
821 
822 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
823 {
824 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
825 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
826 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
827 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
828 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
829 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
830 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
831 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
832 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
833 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
834 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
835 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
836 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
837 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
838 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
839 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
840 };
841 
842 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
843 {
844 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
845 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
846 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
847 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
848 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
849 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
850 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
851 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
852 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
853 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
854 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
855 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
856 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
857 };
858 
859 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
860 {
861 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
862 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
863 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
864 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
865 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
866 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
867 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
868 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
869 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
870 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
871 	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
872 };
873 
874 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
875 	{SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
876 	{SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
877 };
878 
879 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
880 {
881 	mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
882 	mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
883 	mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
884 	mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
885 	mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
886 	mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
887 	mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
888 	mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
889 };
890 
891 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
892 {
893 	mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
894 	mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
895 	mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
896 	mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
897 	mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
898 	mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
899 	mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
900 	mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
901 };
902 
903 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
904 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
905 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
906 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
907 
908 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
909 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
910 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
911 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
912 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
913 				struct amdgpu_cu_info *cu_info);
914 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
915 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
916 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
917 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
918 					  void *ras_error_status);
919 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
920 				     void *inject_if, uint32_t instance_mask);
921 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
922 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
923 					      unsigned int vmid);
924 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
925 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
926 
927 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
928 				uint64_t queue_mask)
929 {
930 	struct amdgpu_device *adev = kiq_ring->adev;
931 	u64 shader_mc_addr;
932 
933 	/* Cleaner shader MC address */
934 	shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
935 
936 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
937 	amdgpu_ring_write(kiq_ring,
938 		PACKET3_SET_RESOURCES_VMID_MASK(0) |
939 		/* vmid_mask:0* queue_type:0 (KIQ) */
940 		PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
941 	amdgpu_ring_write(kiq_ring,
942 			lower_32_bits(queue_mask));	/* queue mask lo */
943 	amdgpu_ring_write(kiq_ring,
944 			upper_32_bits(queue_mask));	/* queue mask hi */
945 	amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
946 	amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
947 	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */
948 	amdgpu_ring_write(kiq_ring, 0);	/* gds heap base:0, gds heap size:0 */
949 }
950 
951 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
952 				 struct amdgpu_ring *ring)
953 {
954 	uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
955 	uint64_t wptr_addr = ring->wptr_gpu_addr;
956 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
957 
958 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
959 	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
960 	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
961 			 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
962 			 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
963 			 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
964 			 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
965 			 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
966 			 /*queue_type: normal compute queue */
967 			 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
968 			 /* alloc format: all_on_one_pipe */
969 			 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
970 			 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
971 			 /* num_queues: must be 1 */
972 			 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
973 	amdgpu_ring_write(kiq_ring,
974 			PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
975 	amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
976 	amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
977 	amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
978 	amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
979 }
980 
981 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
982 				   struct amdgpu_ring *ring,
983 				   enum amdgpu_unmap_queues_action action,
984 				   u64 gpu_addr, u64 seq)
985 {
986 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
987 
988 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
989 	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
990 			  PACKET3_UNMAP_QUEUES_ACTION(action) |
991 			  PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
992 			  PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
993 			  PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
994 	amdgpu_ring_write(kiq_ring,
995 			PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
996 
997 	if (action == PREEMPT_QUEUES_NO_UNMAP) {
998 		amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
999 		amdgpu_ring_write(kiq_ring, 0);
1000 		amdgpu_ring_write(kiq_ring, 0);
1001 
1002 	} else {
1003 		amdgpu_ring_write(kiq_ring, 0);
1004 		amdgpu_ring_write(kiq_ring, 0);
1005 		amdgpu_ring_write(kiq_ring, 0);
1006 	}
1007 }
1008 
1009 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
1010 				   struct amdgpu_ring *ring,
1011 				   u64 addr,
1012 				   u64 seq)
1013 {
1014 	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
1015 
1016 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
1017 	amdgpu_ring_write(kiq_ring,
1018 			  PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
1019 			  PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
1020 			  PACKET3_QUERY_STATUS_COMMAND(2));
1021 	/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
1022 	amdgpu_ring_write(kiq_ring,
1023 			PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
1024 			PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
1025 	amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
1026 	amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
1027 	amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
1028 	amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
1029 }
1030 
1031 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
1032 				uint16_t pasid, uint32_t flush_type,
1033 				bool all_hub)
1034 {
1035 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
1036 	amdgpu_ring_write(kiq_ring,
1037 			PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
1038 			PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
1039 			PACKET3_INVALIDATE_TLBS_PASID(pasid) |
1040 			PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
1041 }
1042 
1043 
1044 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
1045 					uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
1046 					uint32_t xcc_id, uint32_t vmid)
1047 {
1048 	struct amdgpu_device *adev = kiq_ring->adev;
1049 	unsigned i;
1050 
1051 	/* enter save mode */
1052 	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1053 	mutex_lock(&adev->srbm_mutex);
1054 	soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0);
1055 
1056 	if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1057 		WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2);
1058 		WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1);
1059 		/* wait till dequeue take effects */
1060 		for (i = 0; i < adev->usec_timeout; i++) {
1061 			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
1062 				break;
1063 			udelay(1);
1064 		}
1065 		if (i >= adev->usec_timeout)
1066 			dev_err(adev->dev, "fail to wait on hqd deactive\n");
1067 	} else {
1068 		dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);
1069 	}
1070 
1071 	soc15_grbm_select(adev, 0, 0, 0, 0, 0);
1072 	mutex_unlock(&adev->srbm_mutex);
1073 	/* exit safe mode */
1074 	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1075 }
1076 
1077 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
1078 	.kiq_set_resources = gfx_v9_0_kiq_set_resources,
1079 	.kiq_map_queues = gfx_v9_0_kiq_map_queues,
1080 	.kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
1081 	.kiq_query_status = gfx_v9_0_kiq_query_status,
1082 	.kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
1083 	.kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue,
1084 	.set_resources_size = 8,
1085 	.map_queues_size = 7,
1086 	.unmap_queues_size = 6,
1087 	.query_status_size = 7,
1088 	.invalidate_tlbs_size = 2,
1089 };
1090 
1091 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
1092 {
1093 	adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
1094 }
1095 
1096 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
1097 {
1098 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1099 	case IP_VERSION(9, 0, 1):
1100 		soc15_program_register_sequence(adev,
1101 						golden_settings_gc_9_0,
1102 						ARRAY_SIZE(golden_settings_gc_9_0));
1103 		soc15_program_register_sequence(adev,
1104 						golden_settings_gc_9_0_vg10,
1105 						ARRAY_SIZE(golden_settings_gc_9_0_vg10));
1106 		break;
1107 	case IP_VERSION(9, 2, 1):
1108 		soc15_program_register_sequence(adev,
1109 						golden_settings_gc_9_2_1,
1110 						ARRAY_SIZE(golden_settings_gc_9_2_1));
1111 		soc15_program_register_sequence(adev,
1112 						golden_settings_gc_9_2_1_vg12,
1113 						ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
1114 		break;
1115 	case IP_VERSION(9, 4, 0):
1116 		soc15_program_register_sequence(adev,
1117 						golden_settings_gc_9_0,
1118 						ARRAY_SIZE(golden_settings_gc_9_0));
1119 		soc15_program_register_sequence(adev,
1120 						golden_settings_gc_9_0_vg20,
1121 						ARRAY_SIZE(golden_settings_gc_9_0_vg20));
1122 		break;
1123 	case IP_VERSION(9, 4, 1):
1124 		soc15_program_register_sequence(adev,
1125 						golden_settings_gc_9_4_1_arct,
1126 						ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
1127 		break;
1128 	case IP_VERSION(9, 2, 2):
1129 	case IP_VERSION(9, 1, 0):
1130 		soc15_program_register_sequence(adev, golden_settings_gc_9_1,
1131 						ARRAY_SIZE(golden_settings_gc_9_1));
1132 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1133 			soc15_program_register_sequence(adev,
1134 							golden_settings_gc_9_1_rv2,
1135 							ARRAY_SIZE(golden_settings_gc_9_1_rv2));
1136 		else
1137 			soc15_program_register_sequence(adev,
1138 							golden_settings_gc_9_1_rv1,
1139 							ARRAY_SIZE(golden_settings_gc_9_1_rv1));
1140 		break;
1141 	 case IP_VERSION(9, 3, 0):
1142 		soc15_program_register_sequence(adev,
1143 						golden_settings_gc_9_1_rn,
1144 						ARRAY_SIZE(golden_settings_gc_9_1_rn));
1145 		return; /* for renoir, don't need common goldensetting */
1146 	case IP_VERSION(9, 4, 2):
1147 		gfx_v9_4_2_init_golden_registers(adev,
1148 						 adev->smuio.funcs->get_die_id(adev));
1149 		break;
1150 	default:
1151 		break;
1152 	}
1153 
1154 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1155 	    (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))
1156 		soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1157 						(const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1158 }
1159 
1160 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1161 				       bool wc, uint32_t reg, uint32_t val)
1162 {
1163 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1164 	amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1165 				WRITE_DATA_DST_SEL(0) |
1166 				(wc ? WR_CONFIRM : 0));
1167 	amdgpu_ring_write(ring, reg);
1168 	amdgpu_ring_write(ring, 0);
1169 	amdgpu_ring_write(ring, val);
1170 }
1171 
1172 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1173 				  int mem_space, int opt, uint32_t addr0,
1174 				  uint32_t addr1, uint32_t ref, uint32_t mask,
1175 				  uint32_t inv)
1176 {
1177 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1178 	amdgpu_ring_write(ring,
1179 				 /* memory (1) or register (0) */
1180 				 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1181 				 WAIT_REG_MEM_OPERATION(opt) | /* wait */
1182 				 WAIT_REG_MEM_FUNCTION(3) |  /* equal */
1183 				 WAIT_REG_MEM_ENGINE(eng_sel)));
1184 
1185 	if (mem_space)
1186 		WARN_ON(addr0 & 0x3); /* Dword align */
1187 	amdgpu_ring_write(ring, addr0);
1188 	amdgpu_ring_write(ring, addr1);
1189 	amdgpu_ring_write(ring, ref);
1190 	amdgpu_ring_write(ring, mask);
1191 	amdgpu_ring_write(ring, inv); /* poll interval */
1192 }
1193 
1194 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1195 {
1196 	struct amdgpu_device *adev = ring->adev;
1197 	uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1198 	uint32_t tmp = 0;
1199 	unsigned i;
1200 	int r;
1201 
1202 	WREG32(scratch, 0xCAFEDEAD);
1203 	r = amdgpu_ring_alloc(ring, 3);
1204 	if (r)
1205 		return r;
1206 
1207 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1208 	amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);
1209 	amdgpu_ring_write(ring, 0xDEADBEEF);
1210 	amdgpu_ring_commit(ring);
1211 
1212 	for (i = 0; i < adev->usec_timeout; i++) {
1213 		tmp = RREG32(scratch);
1214 		if (tmp == 0xDEADBEEF)
1215 			break;
1216 		udelay(1);
1217 	}
1218 
1219 	if (i >= adev->usec_timeout)
1220 		r = -ETIMEDOUT;
1221 	return r;
1222 }
1223 
1224 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1225 {
1226 	struct amdgpu_device *adev = ring->adev;
1227 	struct amdgpu_job *job;
1228 	struct amdgpu_ib *ib;
1229 	struct dma_fence *f = NULL;
1230 	unsigned index;
1231 	uint64_t gpu_addr;
1232 	uint32_t tmp;
1233 	long r;
1234 
1235 	r = amdgpu_wb_get(adev, &index);
1236 	if (r)
1237 		return r;
1238 
1239 	gpu_addr = adev->wb.gpu_addr + (index * 4);
1240 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1241 
1242 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 20,
1243 				     AMDGPU_IB_POOL_DIRECT,
1244 				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
1245 				     &job);
1246 	if (r)
1247 		goto err1;
1248 
1249 	ib = &job->ibs[0];
1250 	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1251 	ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1252 	ib->ptr[2] = lower_32_bits(gpu_addr);
1253 	ib->ptr[3] = upper_32_bits(gpu_addr);
1254 	ib->ptr[4] = 0xDEADBEEF;
1255 	ib->length_dw = 5;
1256 
1257 	r = amdgpu_job_submit_direct(job, ring, &f);
1258 	if (r) {
1259 		amdgpu_job_free(job);
1260 		goto err2;
1261 	}
1262 
1263 	r = dma_fence_wait_timeout(f, false, timeout);
1264 	if (r == 0) {
1265 		r = -ETIMEDOUT;
1266 		goto err2;
1267 	} else if (r < 0) {
1268 		goto err2;
1269 	}
1270 
1271 	tmp = adev->wb.wb[index];
1272 	if (tmp == 0xDEADBEEF)
1273 		r = 0;
1274 	else
1275 		r = -EINVAL;
1276 
1277 err2:
1278 	dma_fence_put(f);
1279 err1:
1280 	amdgpu_wb_free(adev, index);
1281 	return r;
1282 }
1283 
1284 
1285 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1286 {
1287 	amdgpu_ucode_release(&adev->gfx.pfp_fw);
1288 	amdgpu_ucode_release(&adev->gfx.me_fw);
1289 	amdgpu_ucode_release(&adev->gfx.ce_fw);
1290 	amdgpu_ucode_release(&adev->gfx.rlc_fw);
1291 	amdgpu_ucode_release(&adev->gfx.mec_fw);
1292 	amdgpu_ucode_release(&adev->gfx.mec2_fw);
1293 
1294 	kfree(adev->gfx.rlc.register_list_format);
1295 }
1296 
1297 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1298 {
1299 	adev->gfx.me_fw_write_wait = false;
1300 	adev->gfx.mec_fw_write_wait = false;
1301 
1302 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1303 	    (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) &&
1304 	    ((adev->gfx.mec_fw_version < 0x000001a5) ||
1305 	     (adev->gfx.mec_feature_version < 46) ||
1306 	     (adev->gfx.pfp_fw_version < 0x000000b7) ||
1307 	     (adev->gfx.pfp_feature_version < 46)))
1308 		drm_warn_once(adev_to_drm(adev),
1309 			      "CP firmware version too old, please update!");
1310 
1311 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1312 	case IP_VERSION(9, 0, 1):
1313 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1314 		    (adev->gfx.me_feature_version >= 42) &&
1315 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1316 		    (adev->gfx.pfp_feature_version >= 42))
1317 			adev->gfx.me_fw_write_wait = true;
1318 
1319 		if ((adev->gfx.mec_fw_version >=  0x00000193) &&
1320 		    (adev->gfx.mec_feature_version >= 42))
1321 			adev->gfx.mec_fw_write_wait = true;
1322 		break;
1323 	case IP_VERSION(9, 2, 1):
1324 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1325 		    (adev->gfx.me_feature_version >= 44) &&
1326 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1327 		    (adev->gfx.pfp_feature_version >= 44))
1328 			adev->gfx.me_fw_write_wait = true;
1329 
1330 		if ((adev->gfx.mec_fw_version >=  0x00000196) &&
1331 		    (adev->gfx.mec_feature_version >= 44))
1332 			adev->gfx.mec_fw_write_wait = true;
1333 		break;
1334 	case IP_VERSION(9, 4, 0):
1335 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1336 		    (adev->gfx.me_feature_version >= 44) &&
1337 		    (adev->gfx.pfp_fw_version >=  0x000000b2) &&
1338 		    (adev->gfx.pfp_feature_version >= 44))
1339 			adev->gfx.me_fw_write_wait = true;
1340 
1341 		if ((adev->gfx.mec_fw_version >=  0x00000197) &&
1342 		    (adev->gfx.mec_feature_version >= 44))
1343 			adev->gfx.mec_fw_write_wait = true;
1344 		break;
1345 	case IP_VERSION(9, 1, 0):
1346 	case IP_VERSION(9, 2, 2):
1347 		if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1348 		    (adev->gfx.me_feature_version >= 42) &&
1349 		    (adev->gfx.pfp_fw_version >=  0x000000b1) &&
1350 		    (adev->gfx.pfp_feature_version >= 42))
1351 			adev->gfx.me_fw_write_wait = true;
1352 
1353 		if ((adev->gfx.mec_fw_version >=  0x00000192) &&
1354 		    (adev->gfx.mec_feature_version >= 42))
1355 			adev->gfx.mec_fw_write_wait = true;
1356 		break;
1357 	default:
1358 		adev->gfx.me_fw_write_wait = true;
1359 		adev->gfx.mec_fw_write_wait = true;
1360 		break;
1361 	}
1362 }
1363 
1364 struct amdgpu_gfxoff_quirk {
1365 	u16 chip_vendor;
1366 	u16 chip_device;
1367 	u16 subsys_vendor;
1368 	u16 subsys_device;
1369 	u8 revision;
1370 };
1371 
1372 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1373 	/* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1374 	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1375 	/* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1376 	{ 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1377 	/* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1378 	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1379 	/* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1380 	{ 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1381 	/* https://bbs.openkylin.top/t/topic/171497 */
1382 	{ 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 },
1383 	/* HP 705G4 DM with R5 2400G */
1384 	{ 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 },
1385 	{ 0, 0, 0, 0, 0 },
1386 };
1387 
1388 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1389 {
1390 	const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1391 
1392 	while (p && p->chip_device != 0) {
1393 		if (pdev->vendor == p->chip_vendor &&
1394 		    pdev->device == p->chip_device &&
1395 		    pdev->subsystem_vendor == p->subsys_vendor &&
1396 		    pdev->subsystem_device == p->subsys_device &&
1397 		    pdev->revision == p->revision) {
1398 			return true;
1399 		}
1400 		++p;
1401 	}
1402 	return false;
1403 }
1404 
1405 static bool is_raven_kicker(struct amdgpu_device *adev)
1406 {
1407 	if (adev->pm.fw_version >= 0x41e2b)
1408 		return true;
1409 	else
1410 		return false;
1411 }
1412 
1413 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1414 {
1415 	if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) &&
1416 	    (adev->gfx.me_fw_version >= 0x000000a5) &&
1417 	    (adev->gfx.me_feature_version >= 52))
1418 		return true;
1419 	else
1420 		return false;
1421 }
1422 
1423 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1424 {
1425 	if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1426 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1427 
1428 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1429 	case IP_VERSION(9, 0, 1):
1430 	case IP_VERSION(9, 2, 1):
1431 	case IP_VERSION(9, 4, 0):
1432 		break;
1433 	case IP_VERSION(9, 2, 2):
1434 	case IP_VERSION(9, 1, 0):
1435 		if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1436 		      (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1437 		    ((!is_raven_kicker(adev) &&
1438 		      adev->gfx.rlc_fw_version < 531) ||
1439 		     (adev->gfx.rlc_feature_version < 1) ||
1440 		     !adev->gfx.rlc.is_rlc_v2_1))
1441 			adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1442 
1443 		if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1444 			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1445 				AMD_PG_SUPPORT_CP |
1446 				AMD_PG_SUPPORT_RLC_SMU_HS;
1447 		break;
1448 	case IP_VERSION(9, 3, 0):
1449 		if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1450 			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1451 				AMD_PG_SUPPORT_CP |
1452 				AMD_PG_SUPPORT_RLC_SMU_HS;
1453 		break;
1454 	default:
1455 		break;
1456 	}
1457 }
1458 
1459 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1460 					  char *chip_name)
1461 {
1462 	int err;
1463 
1464 	err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
1465 				   AMDGPU_UCODE_REQUIRED,
1466 				   "amdgpu/%s_pfp.bin", chip_name);
1467 	if (err)
1468 		goto out;
1469 	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
1470 
1471 	err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1472 				   AMDGPU_UCODE_REQUIRED,
1473 				   "amdgpu/%s_me.bin", chip_name);
1474 	if (err)
1475 		goto out;
1476 	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
1477 
1478 	err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1479 				   AMDGPU_UCODE_REQUIRED,
1480 				   "amdgpu/%s_ce.bin", chip_name);
1481 	if (err)
1482 		goto out;
1483 	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
1484 
1485 out:
1486 	if (err) {
1487 		amdgpu_ucode_release(&adev->gfx.pfp_fw);
1488 		amdgpu_ucode_release(&adev->gfx.me_fw);
1489 		amdgpu_ucode_release(&adev->gfx.ce_fw);
1490 	}
1491 	return err;
1492 }
1493 
1494 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1495 				       char *chip_name)
1496 {
1497 	int err;
1498 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
1499 	uint16_t version_major;
1500 	uint16_t version_minor;
1501 	uint32_t smu_version;
1502 
1503 	/*
1504 	 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1505 	 * instead of picasso_rlc.bin.
1506 	 * Judgment method:
1507 	 * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1508 	 *          or revision >= 0xD8 && revision <= 0xDF
1509 	 * otherwise is PCO FP5
1510 	 */
1511 	if (!strcmp(chip_name, "picasso") &&
1512 		(((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1513 		((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1514 		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1515 					   AMDGPU_UCODE_REQUIRED,
1516 					   "amdgpu/%s_rlc_am4.bin", chip_name);
1517 	else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1518 		(smu_version >= 0x41e2b))
1519 		/**
1520 		*SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1521 		*/
1522 		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1523 					   AMDGPU_UCODE_REQUIRED,
1524 					   "amdgpu/%s_kicker_rlc.bin", chip_name);
1525 	else
1526 		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1527 					   AMDGPU_UCODE_REQUIRED,
1528 					   "amdgpu/%s_rlc.bin", chip_name);
1529 	if (err)
1530 		goto out;
1531 
1532 	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1533 	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1534 	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1535 	err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1536 out:
1537 	if (err)
1538 		amdgpu_ucode_release(&adev->gfx.rlc_fw);
1539 
1540 	return err;
1541 }
1542 
1543 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1544 {
1545 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
1546 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1547 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0))
1548 		return false;
1549 
1550 	return true;
1551 }
1552 
1553 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1554 					      char *chip_name)
1555 {
1556 	int err;
1557 
1558 	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1559 		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1560 				   AMDGPU_UCODE_REQUIRED,
1561 				   "amdgpu/%s_sjt_mec.bin", chip_name);
1562 	else
1563 		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1564 					   AMDGPU_UCODE_REQUIRED,
1565 					   "amdgpu/%s_mec.bin", chip_name);
1566 	if (err)
1567 		goto out;
1568 
1569 	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
1570 	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
1571 
1572 	if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1573 		if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1574 			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1575 						   AMDGPU_UCODE_REQUIRED,
1576 						   "amdgpu/%s_sjt_mec2.bin", chip_name);
1577 		else
1578 			err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1579 						   AMDGPU_UCODE_REQUIRED,
1580 						   "amdgpu/%s_mec2.bin", chip_name);
1581 		if (!err) {
1582 			amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
1583 			amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
1584 		} else {
1585 			err = 0;
1586 			amdgpu_ucode_release(&adev->gfx.mec2_fw);
1587 		}
1588 	} else {
1589 		adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1590 		adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1591 	}
1592 
1593 	gfx_v9_0_check_if_need_gfxoff(adev);
1594 	gfx_v9_0_check_fw_write_wait(adev);
1595 
1596 out:
1597 	if (err)
1598 		amdgpu_ucode_release(&adev->gfx.mec_fw);
1599 	return err;
1600 }
1601 
1602 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1603 {
1604 	char ucode_prefix[30];
1605 	int r;
1606 
1607 	DRM_DEBUG("\n");
1608 	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
1609 
1610 	/* No CPG in Arcturus */
1611 	if (adev->gfx.num_gfx_rings) {
1612 		r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);
1613 		if (r)
1614 			return r;
1615 	}
1616 
1617 	r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);
1618 	if (r)
1619 		return r;
1620 
1621 	r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);
1622 	if (r)
1623 		return r;
1624 
1625 	return r;
1626 }
1627 
1628 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1629 {
1630 	u32 count = 0;
1631 	const struct cs_section_def *sect = NULL;
1632 	const struct cs_extent_def *ext = NULL;
1633 
1634 	/* begin clear state */
1635 	count += 2;
1636 	/* context control state */
1637 	count += 3;
1638 
1639 	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1640 		for (ext = sect->section; ext->extent != NULL; ++ext) {
1641 			if (sect->id == SECT_CONTEXT)
1642 				count += 2 + ext->reg_count;
1643 			else
1644 				return 0;
1645 		}
1646 	}
1647 
1648 	/* end clear state */
1649 	count += 2;
1650 	/* clear state */
1651 	count += 2;
1652 
1653 	return count;
1654 }
1655 
1656 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
1657 {
1658 	u32 count = 0;
1659 
1660 	if (adev->gfx.rlc.cs_data == NULL)
1661 		return;
1662 	if (buffer == NULL)
1663 		return;
1664 
1665 	count = amdgpu_gfx_csb_preamble_start(buffer);
1666 	count = amdgpu_gfx_csb_data_parser(adev, buffer, count);
1667 	amdgpu_gfx_csb_preamble_end(buffer, count);
1668 }
1669 
1670 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1671 {
1672 	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1673 	uint32_t pg_always_on_cu_num = 2;
1674 	uint32_t always_on_cu_num;
1675 	uint32_t i, j, k;
1676 	uint32_t mask, cu_bitmap, counter;
1677 
1678 	if (adev->flags & AMD_IS_APU)
1679 		always_on_cu_num = 4;
1680 	else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1))
1681 		always_on_cu_num = 8;
1682 	else
1683 		always_on_cu_num = 12;
1684 
1685 	mutex_lock(&adev->grbm_idx_mutex);
1686 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1687 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1688 			mask = 1;
1689 			cu_bitmap = 0;
1690 			counter = 0;
1691 			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1692 
1693 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1694 				if (cu_info->bitmap[0][i][j] & mask) {
1695 					if (counter == pg_always_on_cu_num)
1696 						WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1697 					if (counter < always_on_cu_num)
1698 						cu_bitmap |= mask;
1699 					else
1700 						break;
1701 					counter++;
1702 				}
1703 				mask <<= 1;
1704 			}
1705 
1706 			WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1707 			cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1708 		}
1709 	}
1710 	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1711 	mutex_unlock(&adev->grbm_idx_mutex);
1712 }
1713 
1714 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1715 {
1716 	uint32_t data;
1717 
1718 	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1719 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1720 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1721 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1722 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1723 
1724 	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1725 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1726 
1727 	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1728 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1729 
1730 	mutex_lock(&adev->grbm_idx_mutex);
1731 	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1732 	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1733 	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1734 
1735 	/* set mmRLC_LB_PARAMS = 0x003F_1006 */
1736 	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1737 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1738 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1739 	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1740 
1741 	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1742 	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1743 	data &= 0x0000FFFF;
1744 	data |= 0x00C00000;
1745 	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1746 
1747 	/*
1748 	 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1749 	 * programmed in gfx_v9_0_init_always_on_cu_mask()
1750 	 */
1751 
1752 	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1753 	 * but used for RLC_LB_CNTL configuration */
1754 	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1755 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1756 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1757 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1758 	mutex_unlock(&adev->grbm_idx_mutex);
1759 
1760 	gfx_v9_0_init_always_on_cu_mask(adev);
1761 }
1762 
1763 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1764 {
1765 	uint32_t data;
1766 
1767 	/* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1768 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1769 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1770 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1771 	WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1772 
1773 	/* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1774 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1775 
1776 	/* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1777 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1778 
1779 	mutex_lock(&adev->grbm_idx_mutex);
1780 	/* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1781 	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1782 	WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1783 
1784 	/* set mmRLC_LB_PARAMS = 0x003F_1006 */
1785 	data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1786 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1787 	data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1788 	WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1789 
1790 	/* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1791 	data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1792 	data &= 0x0000FFFF;
1793 	data |= 0x00C00000;
1794 	WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1795 
1796 	/*
1797 	 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1798 	 * programmed in gfx_v9_0_init_always_on_cu_mask()
1799 	 */
1800 
1801 	/* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1802 	 * but used for RLC_LB_CNTL configuration */
1803 	data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1804 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1805 	data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1806 	WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1807 	mutex_unlock(&adev->grbm_idx_mutex);
1808 
1809 	gfx_v9_0_init_always_on_cu_mask(adev);
1810 }
1811 
1812 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1813 {
1814 	WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1815 }
1816 
1817 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1818 {
1819 	if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1820 		return 5;
1821 	else
1822 		return 4;
1823 }
1824 
1825 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1826 {
1827 	struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1828 
1829 	reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1830 	reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1831 	reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1832 	reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1833 	reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1834 	reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1835 	reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1836 	reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1837 	adev->gfx.rlc.rlcg_reg_access_supported = true;
1838 }
1839 
1840 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1841 {
1842 	const struct cs_section_def *cs_data;
1843 	int r;
1844 
1845 	adev->gfx.rlc.cs_data = gfx9_cs_data;
1846 
1847 	cs_data = adev->gfx.rlc.cs_data;
1848 
1849 	if (cs_data) {
1850 		/* init clear state block */
1851 		r = amdgpu_gfx_rlc_init_csb(adev);
1852 		if (r)
1853 			return r;
1854 	}
1855 
1856 	if (adev->flags & AMD_IS_APU) {
1857 		/* TODO: double check the cp_table_size for RV */
1858 		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1859 		r = amdgpu_gfx_rlc_init_cpt(adev);
1860 		if (r)
1861 			return r;
1862 	}
1863 
1864 	return 0;
1865 }
1866 
1867 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1868 {
1869 	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1870 	amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1871 }
1872 
1873 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1874 {
1875 	int r;
1876 	u32 *hpd;
1877 	const __le32 *fw_data;
1878 	unsigned fw_size;
1879 	u32 *fw;
1880 	size_t mec_hpd_size;
1881 
1882 	const struct gfx_firmware_header_v1_0 *mec_hdr;
1883 
1884 	bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1885 
1886 	/* take ownership of the relevant compute queues */
1887 	amdgpu_gfx_compute_queue_acquire(adev);
1888 	mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1889 	if (mec_hpd_size) {
1890 		r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1891 					      AMDGPU_GEM_DOMAIN_VRAM |
1892 					      AMDGPU_GEM_DOMAIN_GTT,
1893 					      &adev->gfx.mec.hpd_eop_obj,
1894 					      &adev->gfx.mec.hpd_eop_gpu_addr,
1895 					      (void **)&hpd);
1896 		if (r) {
1897 			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1898 			gfx_v9_0_mec_fini(adev);
1899 			return r;
1900 		}
1901 
1902 		memset(hpd, 0, mec_hpd_size);
1903 
1904 		amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1905 		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1906 	}
1907 
1908 	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1909 
1910 	fw_data = (const __le32 *)
1911 		(adev->gfx.mec_fw->data +
1912 		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1913 	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1914 
1915 	r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1916 				      PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1917 				      &adev->gfx.mec.mec_fw_obj,
1918 				      &adev->gfx.mec.mec_fw_gpu_addr,
1919 				      (void **)&fw);
1920 	if (r) {
1921 		dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1922 		gfx_v9_0_mec_fini(adev);
1923 		return r;
1924 	}
1925 
1926 	memcpy(fw, fw_data, fw_size);
1927 
1928 	amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1929 	amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1930 
1931 	return 0;
1932 }
1933 
1934 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1935 {
1936 	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1937 		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1938 		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1939 		(address << SQ_IND_INDEX__INDEX__SHIFT) |
1940 		(SQ_IND_INDEX__FORCE_READ_MASK));
1941 	return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1942 }
1943 
1944 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1945 			   uint32_t wave, uint32_t thread,
1946 			   uint32_t regno, uint32_t num, uint32_t *out)
1947 {
1948 	WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1949 		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1950 		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1951 		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
1952 		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1953 		(SQ_IND_INDEX__FORCE_READ_MASK) |
1954 		(SQ_IND_INDEX__AUTO_INCR_MASK));
1955 	while (num--)
1956 		*(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1957 }
1958 
1959 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1960 {
1961 	/* type 1 wave data */
1962 	dst[(*no_fields)++] = 1;
1963 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1964 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1965 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1966 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1967 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1968 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1969 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1970 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1971 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1972 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1973 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1974 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1975 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1976 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1977 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1978 }
1979 
1980 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1981 				     uint32_t wave, uint32_t start,
1982 				     uint32_t size, uint32_t *dst)
1983 {
1984 	wave_read_regs(
1985 		adev, simd, wave, 0,
1986 		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1987 }
1988 
1989 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1990 				     uint32_t wave, uint32_t thread,
1991 				     uint32_t start, uint32_t size,
1992 				     uint32_t *dst)
1993 {
1994 	wave_read_regs(
1995 		adev, simd, wave, thread,
1996 		start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1997 }
1998 
1999 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2000 				  u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
2001 {
2002 	soc15_grbm_select(adev, me, pipe, q, vm, 0);
2003 }
2004 
2005 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2006         .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2007         .select_se_sh = &gfx_v9_0_select_se_sh,
2008         .read_wave_data = &gfx_v9_0_read_wave_data,
2009         .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2010         .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2011         .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2012 	.get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
2013 };
2014 
2015 const struct amdgpu_ras_block_hw_ops  gfx_v9_0_ras_ops = {
2016 		.ras_error_inject = &gfx_v9_0_ras_error_inject,
2017 		.query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2018 		.reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2019 };
2020 
2021 static struct amdgpu_gfx_ras gfx_v9_0_ras = {
2022 	.ras_block = {
2023 		.hw_ops = &gfx_v9_0_ras_ops,
2024 	},
2025 };
2026 
2027 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2028 {
2029 	u32 gb_addr_config;
2030 	int err;
2031 
2032 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2033 	case IP_VERSION(9, 0, 1):
2034 		adev->gfx.config.max_hw_contexts = 8;
2035 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2036 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2037 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2038 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2039 		gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2040 		break;
2041 	case IP_VERSION(9, 2, 1):
2042 		adev->gfx.config.max_hw_contexts = 8;
2043 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2044 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2045 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2046 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2047 		gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2048 		drm_info(adev_to_drm(adev), "fix gfx.config for vega12\n");
2049 		break;
2050 	case IP_VERSION(9, 4, 0):
2051 		adev->gfx.ras = &gfx_v9_0_ras;
2052 		adev->gfx.config.max_hw_contexts = 8;
2053 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2054 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2055 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2056 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2057 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2058 		gb_addr_config &= ~0xf3e777ff;
2059 		gb_addr_config |= 0x22014042;
2060 		/* check vbios table if gpu info is not available */
2061 		err = amdgpu_atomfirmware_get_gfx_info(adev);
2062 		if (err)
2063 			return err;
2064 		break;
2065 	case IP_VERSION(9, 2, 2):
2066 	case IP_VERSION(9, 1, 0):
2067 		adev->gfx.config.max_hw_contexts = 8;
2068 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2069 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2070 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2071 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2072 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2073 			gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2074 		else
2075 			gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2076 		break;
2077 	case IP_VERSION(9, 4, 1):
2078 		adev->gfx.ras = &gfx_v9_4_ras;
2079 		adev->gfx.config.max_hw_contexts = 8;
2080 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2081 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2082 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2083 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2084 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2085 		gb_addr_config &= ~0xf3e777ff;
2086 		gb_addr_config |= 0x22014042;
2087 		break;
2088 	case IP_VERSION(9, 3, 0):
2089 		adev->gfx.config.max_hw_contexts = 8;
2090 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2091 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2092 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2093 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2094 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2095 		gb_addr_config &= ~0xf3e777ff;
2096 		gb_addr_config |= 0x22010042;
2097 		break;
2098 	case IP_VERSION(9, 4, 2):
2099 		adev->gfx.ras = &gfx_v9_4_2_ras;
2100 		adev->gfx.config.max_hw_contexts = 8;
2101 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2102 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2103 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2104 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2105 		gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2106 		gb_addr_config &= ~0xf3e777ff;
2107 		gb_addr_config |= 0x22014042;
2108 		/* check vbios table if gpu info is not available */
2109 		err = amdgpu_atomfirmware_get_gfx_info(adev);
2110 		if (err)
2111 			return err;
2112 		break;
2113 	default:
2114 		dev_warn(adev->dev, "Unsupported GC version 0x%08x\n",
2115 			 amdgpu_ip_version(adev, GC_HWIP, 0));
2116 		return -EINVAL;
2117 	}
2118 
2119 	adev->gfx.config.gb_addr_config = gb_addr_config;
2120 
2121 	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2122 			REG_GET_FIELD(
2123 					adev->gfx.config.gb_addr_config,
2124 					GB_ADDR_CONFIG,
2125 					NUM_PIPES);
2126 
2127 	adev->gfx.config.max_tile_pipes =
2128 		adev->gfx.config.gb_addr_config_fields.num_pipes;
2129 
2130 	adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2131 			REG_GET_FIELD(
2132 					adev->gfx.config.gb_addr_config,
2133 					GB_ADDR_CONFIG,
2134 					NUM_BANKS);
2135 	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2136 			REG_GET_FIELD(
2137 					adev->gfx.config.gb_addr_config,
2138 					GB_ADDR_CONFIG,
2139 					MAX_COMPRESSED_FRAGS);
2140 	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2141 			REG_GET_FIELD(
2142 					adev->gfx.config.gb_addr_config,
2143 					GB_ADDR_CONFIG,
2144 					NUM_RB_PER_SE);
2145 	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2146 			REG_GET_FIELD(
2147 					adev->gfx.config.gb_addr_config,
2148 					GB_ADDR_CONFIG,
2149 					NUM_SHADER_ENGINES);
2150 	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2151 			REG_GET_FIELD(
2152 					adev->gfx.config.gb_addr_config,
2153 					GB_ADDR_CONFIG,
2154 					PIPE_INTERLEAVE_SIZE));
2155 
2156 	return 0;
2157 }
2158 
2159 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2160 				      int mec, int pipe, int queue)
2161 {
2162 	unsigned irq_type;
2163 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2164 	unsigned int hw_prio;
2165 
2166 	ring = &adev->gfx.compute_ring[ring_id];
2167 
2168 	/* mec0 is me1 */
2169 	ring->me = mec + 1;
2170 	ring->pipe = pipe;
2171 	ring->queue = queue;
2172 
2173 	ring->ring_obj = NULL;
2174 	ring->use_doorbell = true;
2175 	ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2176 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2177 				+ (ring_id * GFX9_MEC_HPD_SIZE);
2178 	ring->vm_hub = AMDGPU_GFXHUB(0);
2179 	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2180 
2181 	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2182 		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2183 		+ ring->pipe;
2184 	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2185 			AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
2186 	/* type-2 packets are deprecated on MEC, use type-3 instead */
2187 	return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2188 				hw_prio, NULL);
2189 }
2190 
2191 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)
2192 {
2193 	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
2194 	uint32_t *ptr;
2195 	uint32_t inst;
2196 
2197 	ptr = kzalloc_objs(*ptr, reg_count);
2198 	if (!ptr) {
2199 		DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
2200 		adev->gfx.ip_dump_core = NULL;
2201 	} else {
2202 		adev->gfx.ip_dump_core = ptr;
2203 	}
2204 
2205 	/* Allocate memory for compute queue registers for all the instances */
2206 	reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
2207 	inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
2208 		adev->gfx.mec.num_queue_per_pipe;
2209 
2210 	ptr = kzalloc_objs(*ptr, reg_count * inst);
2211 	if (!ptr) {
2212 		DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
2213 		adev->gfx.ip_dump_compute_queues = NULL;
2214 	} else {
2215 		adev->gfx.ip_dump_compute_queues = ptr;
2216 	}
2217 }
2218 
2219 static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
2220 {
2221 	int i, j, k, r, ring_id;
2222 	int xcc_id = 0;
2223 	struct amdgpu_ring *ring;
2224 	struct amdgpu_device *adev = ip_block->adev;
2225 	unsigned int hw_prio;
2226 
2227 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2228 	case IP_VERSION(9, 0, 1):
2229 	case IP_VERSION(9, 2, 1):
2230 	case IP_VERSION(9, 4, 0):
2231 	case IP_VERSION(9, 2, 2):
2232 	case IP_VERSION(9, 1, 0):
2233 	case IP_VERSION(9, 4, 1):
2234 	case IP_VERSION(9, 3, 0):
2235 	case IP_VERSION(9, 4, 2):
2236 		adev->gfx.mec.num_mec = 2;
2237 		break;
2238 	default:
2239 		adev->gfx.mec.num_mec = 1;
2240 		break;
2241 	}
2242 
2243 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2244 	case IP_VERSION(9, 0, 1):
2245 	case IP_VERSION(9, 2, 1):
2246 	case IP_VERSION(9, 4, 0):
2247 	case IP_VERSION(9, 2, 2):
2248 	case IP_VERSION(9, 1, 0):
2249 	case IP_VERSION(9, 3, 0):
2250 		adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2251 		adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2252 		if (adev->gfx.me_fw_version  >= 167 &&
2253 		    adev->gfx.pfp_fw_version >= 196 &&
2254 		    adev->gfx.mec_fw_version >= 474) {
2255 			adev->gfx.enable_cleaner_shader = true;
2256 			r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2257 			if (r) {
2258 				adev->gfx.enable_cleaner_shader = false;
2259 				dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2260 			}
2261 		}
2262 		break;
2263 	case IP_VERSION(9, 4, 2):
2264 		adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2265 		adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2266 		if (adev->gfx.mec_fw_version >= 88) {
2267 			adev->gfx.enable_cleaner_shader = true;
2268 			r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2269 			if (r) {
2270 				adev->gfx.enable_cleaner_shader = false;
2271 				dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2272 			}
2273 		}
2274 		break;
2275 	default:
2276 		adev->gfx.enable_cleaner_shader = false;
2277 		break;
2278 	}
2279 
2280 	adev->gfx.mec.num_pipe_per_mec = 4;
2281 	adev->gfx.mec.num_queue_per_pipe = 8;
2282 
2283 	/* EOP Event */
2284 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2285 	if (r)
2286 		return r;
2287 
2288 	/* Bad opcode Event */
2289 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
2290 			      GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
2291 			      &adev->gfx.bad_op_irq);
2292 	if (r)
2293 		return r;
2294 
2295 	/* Privileged reg */
2296 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2297 			      &adev->gfx.priv_reg_irq);
2298 	if (r)
2299 		return r;
2300 
2301 	/* Privileged inst */
2302 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2303 			      &adev->gfx.priv_inst_irq);
2304 	if (r)
2305 		return r;
2306 
2307 	/* ECC error */
2308 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2309 			      &adev->gfx.cp_ecc_error_irq);
2310 	if (r)
2311 		return r;
2312 
2313 	/* FUE error */
2314 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2315 			      &adev->gfx.cp_ecc_error_irq);
2316 	if (r)
2317 		return r;
2318 
2319 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2320 
2321 	if (adev->gfx.rlc.funcs) {
2322 		if (adev->gfx.rlc.funcs->init) {
2323 			r = adev->gfx.rlc.funcs->init(adev);
2324 			if (r) {
2325 				dev_err(adev->dev, "Failed to init rlc BOs!\n");
2326 				return r;
2327 			}
2328 		}
2329 	}
2330 
2331 	r = gfx_v9_0_mec_init(adev);
2332 	if (r) {
2333 		DRM_ERROR("Failed to init MEC BOs!\n");
2334 		return r;
2335 	}
2336 
2337 	/* set up the gfx ring */
2338 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2339 		ring = &adev->gfx.gfx_ring[i];
2340 		ring->ring_obj = NULL;
2341 		if (!i)
2342 			sprintf(ring->name, "gfx");
2343 		else
2344 			sprintf(ring->name, "gfx_%d", i);
2345 		ring->use_doorbell = true;
2346 		ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2347 
2348 		/* disable scheduler on the real ring */
2349 		ring->no_scheduler = adev->gfx.mcbp;
2350 		ring->vm_hub = AMDGPU_GFXHUB(0);
2351 		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2352 				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2353 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
2354 		if (r)
2355 			return r;
2356 	}
2357 
2358 	/* set up the software rings */
2359 	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2360 		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2361 			ring = &adev->gfx.sw_gfx_ring[i];
2362 			ring->ring_obj = NULL;
2363 			strscpy(ring->name, amdgpu_sw_ring_name(i), sizeof(ring->name));
2364 			ring->use_doorbell = true;
2365 			ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2366 			ring->is_sw_ring = true;
2367 			hw_prio = amdgpu_sw_ring_priority(i);
2368 			ring->vm_hub = AMDGPU_GFXHUB(0);
2369 			r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2370 					     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2371 					     NULL);
2372 			if (r)
2373 				return r;
2374 			ring->wptr = 0;
2375 		}
2376 
2377 		/* init the muxer and add software rings */
2378 		r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],
2379 					 GFX9_NUM_SW_GFX_RINGS);
2380 		if (r) {
2381 			DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2382 			return r;
2383 		}
2384 		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2385 			r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,
2386 							&adev->gfx.sw_gfx_ring[i]);
2387 			if (r) {
2388 				DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2389 				return r;
2390 			}
2391 		}
2392 	}
2393 
2394 	/* set up the compute queues - allocate horizontally across pipes */
2395 	ring_id = 0;
2396 	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2397 		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2398 			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2399 				if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
2400 								     k, j))
2401 					continue;
2402 
2403 				r = gfx_v9_0_compute_ring_init(adev,
2404 							       ring_id,
2405 							       i, k, j);
2406 				if (r)
2407 					return r;
2408 
2409 				ring_id++;
2410 			}
2411 		}
2412 	}
2413 
2414 	/* TODO: Add queue reset mask when FW fully supports it */
2415 	adev->gfx.gfx_supported_reset =
2416 		amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
2417 	adev->gfx.compute_supported_reset =
2418 		amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
2419 	if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_gpu_ring_reset) {
2420 		adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
2421 		adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
2422 	}
2423 
2424 	r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0);
2425 	if (r) {
2426 		DRM_ERROR("Failed to init KIQ BOs!\n");
2427 		return r;
2428 	}
2429 
2430 	r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2431 	if (r)
2432 		return r;
2433 
2434 	/* create MQD for all compute queues as wel as KIQ for SRIOV case */
2435 	r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0);
2436 	if (r)
2437 		return r;
2438 
2439 	adev->gfx.ce_ram_size = 0x8000;
2440 
2441 	r = gfx_v9_0_gpu_early_init(adev);
2442 	if (r)
2443 		return r;
2444 
2445 	if (amdgpu_gfx_ras_sw_init(adev)) {
2446 		dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2447 		return -EINVAL;
2448 	}
2449 
2450 	r = amdgpu_gfx_sysfs_init(adev);
2451 	if (r)
2452 		return r;
2453 
2454 	gfx_v9_0_alloc_ip_dump(adev);
2455 
2456 	return 0;
2457 }
2458 
2459 
2460 static int gfx_v9_0_sw_fini(struct amdgpu_ip_block *ip_block)
2461 {
2462 	int i;
2463 	struct amdgpu_device *adev = ip_block->adev;
2464 
2465 	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2466 		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2467 			amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);
2468 		amdgpu_ring_mux_fini(&adev->gfx.muxer);
2469 	}
2470 
2471 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2472 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2473 	for (i = 0; i < adev->gfx.num_compute_rings; i++)
2474 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2475 
2476 	amdgpu_gfx_mqd_sw_fini(adev, 0);
2477 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
2478 	amdgpu_gfx_kiq_fini(adev, 0);
2479 
2480 	amdgpu_gfx_cleaner_shader_sw_fini(adev);
2481 
2482 	gfx_v9_0_mec_fini(adev);
2483 	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2484 				&adev->gfx.rlc.clear_state_gpu_addr,
2485 				(void **)&adev->gfx.rlc.cs_ptr);
2486 	if (adev->flags & AMD_IS_APU) {
2487 		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2488 				&adev->gfx.rlc.cp_table_gpu_addr,
2489 				(void **)&adev->gfx.rlc.cp_table_ptr);
2490 	}
2491 	gfx_v9_0_free_microcode(adev);
2492 
2493 	amdgpu_gfx_sysfs_fini(adev);
2494 
2495 	kfree(adev->gfx.ip_dump_core);
2496 	kfree(adev->gfx.ip_dump_compute_queues);
2497 
2498 	return 0;
2499 }
2500 
2501 
2502 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2503 {
2504 	/* TODO */
2505 }
2506 
2507 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2508 			   u32 instance, int xcc_id)
2509 {
2510 	u32 data;
2511 
2512 	if (instance == 0xffffffff)
2513 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2514 	else
2515 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2516 
2517 	if (se_num == 0xffffffff)
2518 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2519 	else
2520 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2521 
2522 	if (sh_num == 0xffffffff)
2523 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2524 	else
2525 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2526 
2527 	WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2528 }
2529 
2530 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2531 {
2532 	u32 data, mask;
2533 
2534 	data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2535 	data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2536 
2537 	data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2538 	data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2539 
2540 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2541 					 adev->gfx.config.max_sh_per_se);
2542 
2543 	return (~data) & mask;
2544 }
2545 
2546 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2547 {
2548 	int i, j;
2549 	u32 data;
2550 	u32 active_rbs = 0;
2551 	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2552 					adev->gfx.config.max_sh_per_se;
2553 
2554 	mutex_lock(&adev->grbm_idx_mutex);
2555 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2556 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2557 			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2558 			data = gfx_v9_0_get_rb_active_bitmap(adev);
2559 			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2560 					       rb_bitmap_width_per_sh);
2561 		}
2562 	}
2563 	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2564 	mutex_unlock(&adev->grbm_idx_mutex);
2565 
2566 	adev->gfx.config.backend_enable_mask = active_rbs;
2567 	adev->gfx.config.num_rbs = hweight32(active_rbs);
2568 }
2569 
2570 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2571 				uint32_t first_vmid,
2572 				uint32_t last_vmid)
2573 {
2574 	uint32_t data;
2575 	uint32_t trap_config_vmid_mask = 0;
2576 	int i;
2577 
2578 	/* Calculate trap config vmid mask */
2579 	for (i = first_vmid; i < last_vmid; i++)
2580 		trap_config_vmid_mask |= (1 << i);
2581 
2582 	data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2583 			VMID_SEL, trap_config_vmid_mask);
2584 	data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2585 			TRAP_EN, 1);
2586 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2587 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2588 
2589 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2590 	WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2591 }
2592 
2593 #define DEFAULT_SH_MEM_BASES	(0x6000)
2594 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2595 {
2596 	int i;
2597 	uint32_t sh_mem_config;
2598 	uint32_t sh_mem_bases;
2599 
2600 	/*
2601 	 * Configure apertures:
2602 	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
2603 	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
2604 	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
2605 	 */
2606 	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2607 
2608 	sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2609 			SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2610 			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2611 
2612 	mutex_lock(&adev->srbm_mutex);
2613 	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2614 		soc15_grbm_select(adev, 0, 0, 0, i, 0);
2615 		/* CP and shaders */
2616 		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2617 		WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2618 	}
2619 	soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2620 	mutex_unlock(&adev->srbm_mutex);
2621 
2622 	/* Initialize all compute VMIDs to have no GDS, GWS, or OA
2623 	   access. These should be enabled by FW for target VMIDs. */
2624 	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2625 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2626 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2627 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2628 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2629 	}
2630 }
2631 
2632 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2633 {
2634 	int vmid;
2635 
2636 	/*
2637 	 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2638 	 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2639 	 * the driver can enable them for graphics. VMID0 should maintain
2640 	 * access so that HWS firmware can save/restore entries.
2641 	 */
2642 	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2643 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2644 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2645 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2646 		WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2647 	}
2648 }
2649 
2650 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2651 {
2652 	uint32_t tmp;
2653 
2654 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2655 	case IP_VERSION(9, 4, 1):
2656 		tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2657 		tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2658 				!READ_ONCE(adev->barrier_has_auto_waitcnt));
2659 		WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2660 		break;
2661 	case IP_VERSION(9, 4, 2):
2662 		gfx_v9_4_2_init_sq(adev);
2663 		break;
2664 	default:
2665 		break;
2666 	}
2667 }
2668 
2669 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2670 {
2671 	u32 tmp;
2672 	int i;
2673 
2674 	if (!amdgpu_sriov_vf(adev) ||
2675 	    amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) {
2676 		WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2677 	}
2678 
2679 	gfx_v9_0_tiling_mode_table_init(adev);
2680 
2681 	if (adev->gfx.num_gfx_rings)
2682 		gfx_v9_0_setup_rb(adev);
2683 	gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2684 	adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2685 
2686 	/* XXX SH_MEM regs */
2687 	/* where to put LDS, scratch, GPUVM in FSA64 space */
2688 	mutex_lock(&adev->srbm_mutex);
2689 	for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) {
2690 		soc15_grbm_select(adev, 0, 0, 0, i, 0);
2691 		/* CP and shaders */
2692 		if (i == 0) {
2693 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2694 					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2695 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2696 					    !!adev->gmc.noretry);
2697 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2698 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2699 		} else {
2700 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2701 					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2702 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2703 					    !!adev->gmc.noretry);
2704 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2705 			tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2706 				(adev->gmc.private_aperture_start >> 48));
2707 			tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2708 				(adev->gmc.shared_aperture_start >> 48));
2709 			WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2710 		}
2711 	}
2712 	soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2713 
2714 	mutex_unlock(&adev->srbm_mutex);
2715 
2716 	gfx_v9_0_init_compute_vmid(adev);
2717 	gfx_v9_0_init_gds_vmid(adev);
2718 	gfx_v9_0_init_sq_config(adev);
2719 }
2720 
2721 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2722 {
2723 	u32 i, j, k;
2724 	u32 mask;
2725 
2726 	mutex_lock(&adev->grbm_idx_mutex);
2727 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2728 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2729 			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2730 			for (k = 0; k < adev->usec_timeout; k++) {
2731 				if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2732 					break;
2733 				udelay(1);
2734 			}
2735 			if (k == adev->usec_timeout) {
2736 				amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2737 						      0xffffffff, 0xffffffff, 0);
2738 				mutex_unlock(&adev->grbm_idx_mutex);
2739 				drm_info(adev_to_drm(adev), "Timeout wait for RLC serdes %u,%u\n",
2740 					 i, j);
2741 				return;
2742 			}
2743 		}
2744 	}
2745 	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2746 	mutex_unlock(&adev->grbm_idx_mutex);
2747 
2748 	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2749 		RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2750 		RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2751 		RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2752 	for (k = 0; k < adev->usec_timeout; k++) {
2753 		if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2754 			break;
2755 		udelay(1);
2756 	}
2757 }
2758 
2759 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2760 					       bool enable)
2761 {
2762 	u32 tmp;
2763 
2764 	/* These interrupts should be enabled to drive DS clock */
2765 
2766 	tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2767 
2768 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2769 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2770 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2771 	if (adev->gfx.num_gfx_rings)
2772 		tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2773 
2774 	WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2775 }
2776 
2777 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2778 {
2779 	adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2780 	/* csib */
2781 	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2782 			adev->gfx.rlc.clear_state_gpu_addr >> 32);
2783 	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2784 			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2785 	WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2786 			adev->gfx.rlc.clear_state_size);
2787 }
2788 
2789 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2790 				int indirect_offset,
2791 				int list_size,
2792 				int *unique_indirect_regs,
2793 				int unique_indirect_reg_count,
2794 				int *indirect_start_offsets,
2795 				int *indirect_start_offsets_count,
2796 				int max_start_offsets_count)
2797 {
2798 	int idx;
2799 
2800 	for (; indirect_offset < list_size; indirect_offset++) {
2801 		WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2802 		indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2803 		*indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2804 
2805 		while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2806 			indirect_offset += 2;
2807 
2808 			/* look for the matching indice */
2809 			for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2810 				if (unique_indirect_regs[idx] ==
2811 					register_list_format[indirect_offset] ||
2812 					!unique_indirect_regs[idx])
2813 					break;
2814 			}
2815 
2816 			BUG_ON(idx >= unique_indirect_reg_count);
2817 
2818 			if (!unique_indirect_regs[idx])
2819 				unique_indirect_regs[idx] = register_list_format[indirect_offset];
2820 
2821 			indirect_offset++;
2822 		}
2823 	}
2824 }
2825 
2826 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2827 {
2828 	int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2829 	int unique_indirect_reg_count = 0;
2830 
2831 	int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2832 	int indirect_start_offsets_count = 0;
2833 
2834 	int list_size = 0;
2835 	int i = 0, j = 0;
2836 	u32 tmp = 0;
2837 
2838 	u32 *register_list_format =
2839 		kmemdup(adev->gfx.rlc.register_list_format,
2840 			adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2841 	if (!register_list_format)
2842 		return -ENOMEM;
2843 
2844 	/* setup unique_indirect_regs array and indirect_start_offsets array */
2845 	unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2846 	gfx_v9_1_parse_ind_reg_list(register_list_format,
2847 				    adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2848 				    adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2849 				    unique_indirect_regs,
2850 				    unique_indirect_reg_count,
2851 				    indirect_start_offsets,
2852 				    &indirect_start_offsets_count,
2853 				    ARRAY_SIZE(indirect_start_offsets));
2854 
2855 	/* enable auto inc in case it is disabled */
2856 	tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2857 	tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2858 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2859 
2860 	/* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2861 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2862 		RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2863 	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2864 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2865 			adev->gfx.rlc.register_restore[i]);
2866 
2867 	/* load indirect register */
2868 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2869 		adev->gfx.rlc.reg_list_format_start);
2870 
2871 	/* direct register portion */
2872 	for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2873 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2874 			register_list_format[i]);
2875 
2876 	/* indirect register portion */
2877 	while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2878 		if (register_list_format[i] == 0xFFFFFFFF) {
2879 			WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2880 			continue;
2881 		}
2882 
2883 		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2884 		WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2885 
2886 		for (j = 0; j < unique_indirect_reg_count; j++) {
2887 			if (register_list_format[i] == unique_indirect_regs[j]) {
2888 				WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2889 				break;
2890 			}
2891 		}
2892 
2893 		BUG_ON(j >= unique_indirect_reg_count);
2894 
2895 		i++;
2896 	}
2897 
2898 	/* set save/restore list size */
2899 	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2900 	list_size = list_size >> 1;
2901 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2902 		adev->gfx.rlc.reg_restore_list_size);
2903 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2904 
2905 	/* write the starting offsets to RLC scratch ram */
2906 	WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2907 		adev->gfx.rlc.starting_offsets_start);
2908 	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2909 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2910 		       indirect_start_offsets[i]);
2911 
2912 	/* load unique indirect regs*/
2913 	for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2914 		if (unique_indirect_regs[i] != 0) {
2915 			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2916 			       + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2917 			       unique_indirect_regs[i] & 0x3FFFF);
2918 
2919 			WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2920 			       + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2921 			       unique_indirect_regs[i] >> 20);
2922 		}
2923 	}
2924 
2925 	kfree(register_list_format);
2926 	return 0;
2927 }
2928 
2929 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2930 {
2931 	WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2932 }
2933 
2934 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2935 					     bool enable)
2936 {
2937 	uint32_t data = 0;
2938 	uint32_t default_data = 0;
2939 
2940 	default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2941 	if (enable) {
2942 		/* enable GFXIP control over CGPG */
2943 		data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2944 		if(default_data != data)
2945 			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2946 
2947 		/* update status */
2948 		data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2949 		data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2950 		if(default_data != data)
2951 			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2952 	} else {
2953 		/* restore GFXIP control over GCPG */
2954 		data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2955 		if(default_data != data)
2956 			WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2957 	}
2958 }
2959 
2960 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2961 {
2962 	uint32_t data = 0;
2963 
2964 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2965 			      AMD_PG_SUPPORT_GFX_SMG |
2966 			      AMD_PG_SUPPORT_GFX_DMG)) {
2967 		/* init IDLE_POLL_COUNT = 60 */
2968 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2969 		data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2970 		data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2971 		WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2972 
2973 		/* init RLC PG Delay */
2974 		data = 0;
2975 		data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2976 		data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2977 		data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2978 		data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2979 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2980 
2981 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2982 		data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2983 		data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2984 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2985 
2986 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2987 		data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2988 		data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2989 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2990 
2991 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2992 		data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2993 
2994 		/* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2995 		data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2996 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2997 		if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0))
2998 			pwr_10_0_gfxip_control_over_cgpg(adev, true);
2999 	}
3000 }
3001 
3002 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
3003 						bool enable)
3004 {
3005 	uint32_t data = 0;
3006 	uint32_t default_data = 0;
3007 
3008 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3009 	data = REG_SET_FIELD(data, RLC_PG_CNTL,
3010 			     SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
3011 			     enable ? 1 : 0);
3012 	if (default_data != data)
3013 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3014 }
3015 
3016 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
3017 						bool enable)
3018 {
3019 	uint32_t data = 0;
3020 	uint32_t default_data = 0;
3021 
3022 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3023 	data = REG_SET_FIELD(data, RLC_PG_CNTL,
3024 			     SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
3025 			     enable ? 1 : 0);
3026 	if(default_data != data)
3027 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3028 }
3029 
3030 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
3031 					bool enable)
3032 {
3033 	uint32_t data = 0;
3034 	uint32_t default_data = 0;
3035 
3036 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3037 	data = REG_SET_FIELD(data, RLC_PG_CNTL,
3038 			     CP_PG_DISABLE,
3039 			     enable ? 0 : 1);
3040 	if(default_data != data)
3041 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3042 }
3043 
3044 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
3045 						bool enable)
3046 {
3047 	uint32_t data, default_data;
3048 
3049 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3050 	data = REG_SET_FIELD(data, RLC_PG_CNTL,
3051 			     GFX_POWER_GATING_ENABLE,
3052 			     enable ? 1 : 0);
3053 	if(default_data != data)
3054 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3055 }
3056 
3057 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
3058 						bool enable)
3059 {
3060 	uint32_t data, default_data;
3061 
3062 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3063 	data = REG_SET_FIELD(data, RLC_PG_CNTL,
3064 			     GFX_PIPELINE_PG_ENABLE,
3065 			     enable ? 1 : 0);
3066 	if(default_data != data)
3067 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3068 
3069 	if (!enable)
3070 		/* read any GFX register to wake up GFX */
3071 		data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3072 }
3073 
3074 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3075 						       bool enable)
3076 {
3077 	uint32_t data, default_data;
3078 
3079 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3080 	data = REG_SET_FIELD(data, RLC_PG_CNTL,
3081 			     STATIC_PER_CU_PG_ENABLE,
3082 			     enable ? 1 : 0);
3083 	if(default_data != data)
3084 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3085 }
3086 
3087 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3088 						bool enable)
3089 {
3090 	uint32_t data, default_data;
3091 
3092 	default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3093 	data = REG_SET_FIELD(data, RLC_PG_CNTL,
3094 			     DYN_PER_CU_PG_ENABLE,
3095 			     enable ? 1 : 0);
3096 	if(default_data != data)
3097 		WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3098 }
3099 
3100 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3101 {
3102 	gfx_v9_0_init_csb(adev);
3103 
3104 	/*
3105 	 * Rlc save restore list is workable since v2_1.
3106 	 * And it's needed by gfxoff feature.
3107 	 */
3108 	if (adev->gfx.rlc.is_rlc_v2_1) {
3109 		if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3110 			    IP_VERSION(9, 2, 1) ||
3111 		    (adev->apu_flags & AMD_APU_IS_RAVEN2))
3112 			gfx_v9_1_init_rlc_save_restore_list(adev);
3113 		gfx_v9_0_enable_save_restore_machine(adev);
3114 	}
3115 
3116 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3117 			      AMD_PG_SUPPORT_GFX_SMG |
3118 			      AMD_PG_SUPPORT_GFX_DMG |
3119 			      AMD_PG_SUPPORT_CP |
3120 			      AMD_PG_SUPPORT_GDS |
3121 			      AMD_PG_SUPPORT_RLC_SMU_HS)) {
3122 		WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
3123 			     adev->gfx.rlc.cp_table_gpu_addr >> 8);
3124 		gfx_v9_0_init_gfx_power_gating(adev);
3125 	}
3126 }
3127 
3128 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3129 {
3130 	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3131 	gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3132 	gfx_v9_0_wait_for_rlc_serdes(adev);
3133 }
3134 
3135 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3136 {
3137 	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3138 	udelay(50);
3139 	WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3140 	udelay(50);
3141 }
3142 
3143 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3144 {
3145 #ifdef AMDGPU_RLC_DEBUG_RETRY
3146 	u32 rlc_ucode_ver;
3147 #endif
3148 
3149 	WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3150 	udelay(50);
3151 
3152 	/* carrizo do enable cp interrupt after cp inited */
3153 	if (!(adev->flags & AMD_IS_APU)) {
3154 		gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3155 		udelay(50);
3156 	}
3157 
3158 #ifdef AMDGPU_RLC_DEBUG_RETRY
3159 	/* RLC_GPM_GENERAL_6 : RLC Ucode version */
3160 	rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3161 	if(rlc_ucode_ver == 0x108) {
3162 		drm_info(adev_to_drm(adev), "Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i\n",
3163 				rlc_ucode_ver, adev->gfx.rlc_fw_version);
3164 		/* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3165 		 * default is 0x9C4 to create a 100us interval */
3166 		WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3167 		/* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3168 		 * to disable the page fault retry interrupts, default is
3169 		 * 0x100 (256) */
3170 		WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3171 	}
3172 #endif
3173 }
3174 
3175 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3176 {
3177 	const struct rlc_firmware_header_v2_0 *hdr;
3178 	const __le32 *fw_data;
3179 	unsigned i, fw_size;
3180 
3181 	if (!adev->gfx.rlc_fw)
3182 		return -EINVAL;
3183 
3184 	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3185 	amdgpu_ucode_print_rlc_hdr(&hdr->header);
3186 
3187 	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3188 			   le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3189 	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3190 
3191 	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3192 			RLCG_UCODE_LOADING_START_ADDRESS);
3193 	for (i = 0; i < fw_size; i++)
3194 		WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3195 	WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3196 
3197 	return 0;
3198 }
3199 
3200 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3201 {
3202 	int r;
3203 
3204 	if (amdgpu_sriov_vf(adev)) {
3205 		gfx_v9_0_init_csb(adev);
3206 		return 0;
3207 	}
3208 
3209 	adev->gfx.rlc.funcs->stop(adev);
3210 
3211 	/* disable CG */
3212 	WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3213 
3214 	gfx_v9_0_init_pg(adev);
3215 
3216 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3217 		/* legacy rlc firmware loading */
3218 		r = gfx_v9_0_rlc_load_microcode(adev);
3219 		if (r)
3220 			return r;
3221 	}
3222 
3223 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3224 	case IP_VERSION(9, 2, 2):
3225 	case IP_VERSION(9, 1, 0):
3226 		gfx_v9_0_init_lbpw(adev);
3227 		if (amdgpu_lbpw == 0)
3228 			gfx_v9_0_enable_lbpw(adev, false);
3229 		else
3230 			gfx_v9_0_enable_lbpw(adev, true);
3231 		break;
3232 	case IP_VERSION(9, 4, 0):
3233 		gfx_v9_4_init_lbpw(adev);
3234 		if (amdgpu_lbpw > 0)
3235 			gfx_v9_0_enable_lbpw(adev, true);
3236 		else
3237 			gfx_v9_0_enable_lbpw(adev, false);
3238 		break;
3239 	default:
3240 		break;
3241 	}
3242 
3243 	gfx_v9_0_update_spm_vmid_internal(adev, 0xf);
3244 
3245 	adev->gfx.rlc.funcs->start(adev);
3246 
3247 	return 0;
3248 }
3249 
3250 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3251 {
3252 	u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3253 
3254 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_INVALIDATE_ICACHE, enable ? 0 : 1);
3255 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_INVALIDATE_ICACHE, enable ? 0 : 1);
3256 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_INVALIDATE_ICACHE, enable ? 0 : 1);
3257 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE0_RESET, enable ? 0 : 1);
3258 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE1_RESET, enable ? 0 : 1);
3259 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, enable ? 0 : 1);
3260 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, enable ? 0 : 1);
3261 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, enable ? 0 : 1);
3262 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, enable ? 0 : 1);
3263 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3264 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3265 	tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3266 	WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3267 	udelay(50);
3268 }
3269 
3270 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3271 {
3272 	const struct gfx_firmware_header_v1_0 *pfp_hdr;
3273 	const struct gfx_firmware_header_v1_0 *ce_hdr;
3274 	const struct gfx_firmware_header_v1_0 *me_hdr;
3275 	const __le32 *fw_data;
3276 	unsigned i, fw_size;
3277 
3278 	if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3279 		return -EINVAL;
3280 
3281 	pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3282 		adev->gfx.pfp_fw->data;
3283 	ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3284 		adev->gfx.ce_fw->data;
3285 	me_hdr = (const struct gfx_firmware_header_v1_0 *)
3286 		adev->gfx.me_fw->data;
3287 
3288 	amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3289 	amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3290 	amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3291 
3292 	gfx_v9_0_cp_gfx_enable(adev, false);
3293 
3294 	/* PFP */
3295 	fw_data = (const __le32 *)
3296 		(adev->gfx.pfp_fw->data +
3297 		 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3298 	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3299 	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3300 	for (i = 0; i < fw_size; i++)
3301 		WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3302 	WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3303 
3304 	/* CE */
3305 	fw_data = (const __le32 *)
3306 		(adev->gfx.ce_fw->data +
3307 		 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3308 	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3309 	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3310 	for (i = 0; i < fw_size; i++)
3311 		WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3312 	WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3313 
3314 	/* ME */
3315 	fw_data = (const __le32 *)
3316 		(adev->gfx.me_fw->data +
3317 		 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3318 	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3319 	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3320 	for (i = 0; i < fw_size; i++)
3321 		WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3322 	WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3323 
3324 	return 0;
3325 }
3326 
3327 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3328 {
3329 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3330 	const struct cs_section_def *sect = NULL;
3331 	const struct cs_extent_def *ext = NULL;
3332 	int r, i, tmp;
3333 
3334 	/* init the CP */
3335 	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3336 	WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3337 
3338 	gfx_v9_0_cp_gfx_enable(adev, true);
3339 
3340 	/* Now only limit the quirk on the APU gfx9 series and already
3341 	 * confirmed that the APU gfx10/gfx11 needn't such update.
3342 	 */
3343 	if (adev->flags & AMD_IS_APU &&
3344 			adev->in_s3 && !pm_resume_via_firmware()) {
3345 		drm_info(adev_to_drm(adev), "Will skip the CSB packet resubmit\n");
3346 		return 0;
3347 	}
3348 	r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3349 	if (r) {
3350 		drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r);
3351 		return r;
3352 	}
3353 
3354 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3355 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3356 
3357 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3358 	amdgpu_ring_write(ring, 0x80000000);
3359 	amdgpu_ring_write(ring, 0x80000000);
3360 
3361 	for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3362 		for (ext = sect->section; ext->extent != NULL; ++ext) {
3363 			if (sect->id == SECT_CONTEXT) {
3364 				amdgpu_ring_write(ring,
3365 				       PACKET3(PACKET3_SET_CONTEXT_REG,
3366 					       ext->reg_count));
3367 				amdgpu_ring_write(ring,
3368 				       ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3369 				for (i = 0; i < ext->reg_count; i++)
3370 					amdgpu_ring_write(ring, ext->extent[i]);
3371 			}
3372 		}
3373 	}
3374 
3375 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3376 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3377 
3378 	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3379 	amdgpu_ring_write(ring, 0);
3380 
3381 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3382 	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3383 	amdgpu_ring_write(ring, 0x8000);
3384 	amdgpu_ring_write(ring, 0x8000);
3385 
3386 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3387 	tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3388 		(SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3389 	amdgpu_ring_write(ring, tmp);
3390 	amdgpu_ring_write(ring, 0);
3391 
3392 	amdgpu_ring_commit(ring);
3393 
3394 	return 0;
3395 }
3396 
3397 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3398 {
3399 	struct amdgpu_ring *ring;
3400 	u32 tmp;
3401 	u32 rb_bufsz;
3402 	u64 rb_addr, rptr_addr, wptr_gpu_addr;
3403 
3404 	/* Set the write pointer delay */
3405 	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3406 
3407 	/* set the RB to use vmid 0 */
3408 	WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3409 
3410 	/* Set ring buffer size */
3411 	ring = &adev->gfx.gfx_ring[0];
3412 	rb_bufsz = order_base_2(ring->ring_size / 8);
3413 	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3414 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3415 #ifdef __BIG_ENDIAN
3416 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3417 #endif
3418 	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3419 
3420 	/* Initialize the ring buffer's write pointers */
3421 	ring->wptr = 0;
3422 	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3423 	WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3424 
3425 	/* set the wb address whether it's enabled or not */
3426 	rptr_addr = ring->rptr_gpu_addr;
3427 	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3428 	WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3429 
3430 	wptr_gpu_addr = ring->wptr_gpu_addr;
3431 	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3432 	WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3433 
3434 	mdelay(1);
3435 	WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3436 
3437 	rb_addr = ring->gpu_addr >> 8;
3438 	WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3439 	WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3440 
3441 	tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3442 	if (ring->use_doorbell) {
3443 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3444 				    DOORBELL_OFFSET, ring->doorbell_index);
3445 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3446 				    DOORBELL_EN, 1);
3447 	} else {
3448 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3449 	}
3450 	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3451 
3452 	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3453 			DOORBELL_RANGE_LOWER, ring->doorbell_index);
3454 	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3455 
3456 	WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3457 		       CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3458 
3459 
3460 	/* start the ring */
3461 	gfx_v9_0_cp_gfx_start(adev);
3462 
3463 	return 0;
3464 }
3465 
3466 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3467 {
3468 	if (enable) {
3469 		WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3470 	} else {
3471 		WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3472 				 (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
3473 				  CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
3474 				  CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
3475 				  CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
3476 				  CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
3477 				  CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
3478 				  CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
3479 				  CP_MEC_CNTL__MEC_ME1_HALT_MASK |
3480 				  CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3481 		adev->gfx.kiq[0].ring.sched.ready = false;
3482 	}
3483 	udelay(50);
3484 }
3485 
3486 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3487 {
3488 	const struct gfx_firmware_header_v1_0 *mec_hdr;
3489 	const __le32 *fw_data;
3490 	unsigned i;
3491 	u32 tmp;
3492 
3493 	if (!adev->gfx.mec_fw)
3494 		return -EINVAL;
3495 
3496 	gfx_v9_0_cp_compute_enable(adev, false);
3497 
3498 	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3499 	amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3500 
3501 	fw_data = (const __le32 *)
3502 		(adev->gfx.mec_fw->data +
3503 		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3504 	tmp = 0;
3505 	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3506 	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3507 	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3508 
3509 	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3510 		adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3511 	WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3512 		upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3513 
3514 	/* MEC1 */
3515 	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3516 			 mec_hdr->jt_offset);
3517 	for (i = 0; i < mec_hdr->jt_size; i++)
3518 		WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3519 			le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3520 
3521 	WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3522 			adev->gfx.mec_fw_version);
3523 	/* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3524 
3525 	return 0;
3526 }
3527 
3528 /* KIQ functions */
3529 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3530 {
3531 	uint32_t tmp;
3532 	struct amdgpu_device *adev = ring->adev;
3533 
3534 	/* tell RLC which is KIQ queue */
3535 	tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3536 	tmp &= 0xffffff00;
3537 	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3538 	WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp | 0x80);
3539 }
3540 
3541 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3542 {
3543 	struct amdgpu_device *adev = ring->adev;
3544 
3545 	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3546 		if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3547 			mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3548 			mqd->cp_hqd_queue_priority =
3549 				AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3550 		}
3551 	}
3552 }
3553 
3554 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3555 {
3556 	struct amdgpu_device *adev = ring->adev;
3557 	struct v9_mqd *mqd = ring->mqd_ptr;
3558 	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3559 	uint32_t tmp;
3560 
3561 	mqd->header = 0xC0310800;
3562 	mqd->compute_pipelinestat_enable = 0x00000001;
3563 	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3564 	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3565 	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3566 	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3567 	mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3568 	mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3569 	mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3570 	mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3571 	mqd->compute_misc_reserved = 0x00000003;
3572 
3573 	mqd->dynamic_cu_mask_addr_lo =
3574 		lower_32_bits(ring->mqd_gpu_addr
3575 			      + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3576 	mqd->dynamic_cu_mask_addr_hi =
3577 		upper_32_bits(ring->mqd_gpu_addr
3578 			      + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3579 
3580 	eop_base_addr = ring->eop_gpu_addr >> 8;
3581 	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3582 	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3583 
3584 	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3585 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3586 	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3587 			(order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3588 
3589 	mqd->cp_hqd_eop_control = tmp;
3590 
3591 	/* enable doorbell? */
3592 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3593 
3594 	if (ring->use_doorbell) {
3595 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3596 				    DOORBELL_OFFSET, ring->doorbell_index);
3597 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3598 				    DOORBELL_EN, 1);
3599 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3600 				    DOORBELL_SOURCE, 0);
3601 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3602 				    DOORBELL_HIT, 0);
3603 	} else {
3604 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3605 					 DOORBELL_EN, 0);
3606 	}
3607 
3608 	mqd->cp_hqd_pq_doorbell_control = tmp;
3609 
3610 	/* disable the queue if it's active */
3611 	ring->wptr = 0;
3612 	mqd->cp_hqd_dequeue_request = 0;
3613 	mqd->cp_hqd_pq_rptr = 0;
3614 	mqd->cp_hqd_pq_wptr_lo = 0;
3615 	mqd->cp_hqd_pq_wptr_hi = 0;
3616 
3617 	/* set the pointer to the MQD */
3618 	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3619 	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3620 
3621 	/* set MQD vmid to 0 */
3622 	tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3623 	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3624 	mqd->cp_mqd_control = tmp;
3625 
3626 	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3627 	hqd_gpu_addr = ring->gpu_addr >> 8;
3628 	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3629 	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3630 
3631 	/* set up the HQD, this is similar to CP_RB0_CNTL */
3632 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3633 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3634 			    (order_base_2(ring->ring_size / 4) - 1));
3635 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3636 			(order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3637 #ifdef __BIG_ENDIAN
3638 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3639 #endif
3640 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3641 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3642 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3643 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3644 	mqd->cp_hqd_pq_control = tmp;
3645 
3646 	/* set the wb address whether it's enabled or not */
3647 	wb_gpu_addr = ring->rptr_gpu_addr;
3648 	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3649 	mqd->cp_hqd_pq_rptr_report_addr_hi =
3650 		upper_32_bits(wb_gpu_addr) & 0xffff;
3651 
3652 	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3653 	wb_gpu_addr = ring->wptr_gpu_addr;
3654 	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3655 	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3656 
3657 	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3658 	ring->wptr = 0;
3659 	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3660 
3661 	/* set the vmid for the queue */
3662 	mqd->cp_hqd_vmid = 0;
3663 
3664 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3665 	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3666 	mqd->cp_hqd_persistent_state = tmp;
3667 
3668 	/* set MIN_IB_AVAIL_SIZE */
3669 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3670 	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3671 	mqd->cp_hqd_ib_control = tmp;
3672 
3673 	/* set static priority for a queue/ring */
3674 	gfx_v9_0_mqd_set_priority(ring, mqd);
3675 	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3676 	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
3677 	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
3678 	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
3679 	mqd->cp_hqd_quantum = tmp;
3680 
3681 	/* map_queues packet doesn't need activate the queue,
3682 	 * so only kiq need set this field.
3683 	 */
3684 	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3685 		mqd->cp_hqd_active = 1;
3686 
3687 	return 0;
3688 }
3689 
3690 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3691 {
3692 	struct amdgpu_device *adev = ring->adev;
3693 	struct v9_mqd *mqd = ring->mqd_ptr;
3694 	int j;
3695 
3696 	/* disable wptr polling */
3697 	WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3698 
3699 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3700 	       mqd->cp_hqd_eop_base_addr_lo);
3701 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3702 	       mqd->cp_hqd_eop_base_addr_hi);
3703 
3704 	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3705 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3706 	       mqd->cp_hqd_eop_control);
3707 
3708 	/* enable doorbell? */
3709 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3710 	       mqd->cp_hqd_pq_doorbell_control);
3711 
3712 	/* disable the queue if it's active */
3713 	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3714 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3715 		for (j = 0; j < adev->usec_timeout; j++) {
3716 			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3717 				break;
3718 			udelay(1);
3719 		}
3720 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3721 		       mqd->cp_hqd_dequeue_request);
3722 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3723 		       mqd->cp_hqd_pq_rptr);
3724 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3725 		       mqd->cp_hqd_pq_wptr_lo);
3726 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3727 		       mqd->cp_hqd_pq_wptr_hi);
3728 	}
3729 
3730 	/* set the pointer to the MQD */
3731 	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3732 	       mqd->cp_mqd_base_addr_lo);
3733 	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3734 	       mqd->cp_mqd_base_addr_hi);
3735 
3736 	/* set MQD vmid to 0 */
3737 	WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3738 	       mqd->cp_mqd_control);
3739 
3740 	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3741 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3742 	       mqd->cp_hqd_pq_base_lo);
3743 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3744 	       mqd->cp_hqd_pq_base_hi);
3745 
3746 	/* set up the HQD, this is similar to CP_RB0_CNTL */
3747 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3748 	       mqd->cp_hqd_pq_control);
3749 
3750 	/* set the wb address whether it's enabled or not */
3751 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3752 				mqd->cp_hqd_pq_rptr_report_addr_lo);
3753 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3754 				mqd->cp_hqd_pq_rptr_report_addr_hi);
3755 
3756 	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3757 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3758 	       mqd->cp_hqd_pq_wptr_poll_addr_lo);
3759 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3760 	       mqd->cp_hqd_pq_wptr_poll_addr_hi);
3761 
3762 	/* enable the doorbell if requested */
3763 	if (ring->use_doorbell) {
3764 		WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3765 					(adev->doorbell_index.kiq * 2) << 2);
3766 		/* If GC has entered CGPG, ringing doorbell > first page
3767 		 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3768 		 * workaround this issue. And this change has to align with firmware
3769 		 * update.
3770 		 */
3771 		if (check_if_enlarge_doorbell_range(adev))
3772 			WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3773 					(adev->doorbell.size - 4));
3774 		else
3775 			WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3776 					(adev->doorbell_index.userqueue_end * 2) << 2);
3777 	}
3778 
3779 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3780 	       mqd->cp_hqd_pq_doorbell_control);
3781 
3782 	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3783 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3784 	       mqd->cp_hqd_pq_wptr_lo);
3785 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3786 	       mqd->cp_hqd_pq_wptr_hi);
3787 
3788 	/* set the vmid for the queue */
3789 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3790 
3791 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3792 	       mqd->cp_hqd_persistent_state);
3793 
3794 	/* activate the queue */
3795 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3796 	       mqd->cp_hqd_active);
3797 
3798 	if (ring->use_doorbell)
3799 		WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3800 
3801 	return 0;
3802 }
3803 
3804 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3805 {
3806 	struct amdgpu_device *adev = ring->adev;
3807 	int j;
3808 
3809 	/* disable the queue if it's active */
3810 	if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3811 
3812 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3813 
3814 		for (j = 0; j < adev->usec_timeout; j++) {
3815 			if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3816 				break;
3817 			udelay(1);
3818 		}
3819 
3820 		if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3821 			DRM_DEBUG("KIQ dequeue request failed.\n");
3822 
3823 			/* Manual disable if dequeue request times out */
3824 			WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3825 		}
3826 
3827 		WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3828 		      0);
3829 	}
3830 
3831 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3832 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3833 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3834 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3835 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3836 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3837 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3838 	WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3839 
3840 	return 0;
3841 }
3842 
3843 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3844 {
3845 	struct amdgpu_device *adev = ring->adev;
3846 	struct v9_mqd *mqd = ring->mqd_ptr;
3847 	struct v9_mqd *tmp_mqd;
3848 
3849 	gfx_v9_0_kiq_setting(ring);
3850 
3851 	/* GPU could be in bad state during probe, driver trigger the reset
3852 	 * after load the SMU, in this case , the mqd is not be initialized.
3853 	 * driver need to re-init the mqd.
3854 	 * check mqd->cp_hqd_pq_control since this value should not be 0
3855 	 */
3856 	tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3857 	if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3858 		/* for GPU_RESET case , reset MQD to a clean status */
3859 		if (adev->gfx.kiq[0].mqd_backup)
3860 			memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3861 
3862 		/* reset ring buffer */
3863 		ring->wptr = 0;
3864 		amdgpu_ring_clear_ring(ring);
3865 
3866 		mutex_lock(&adev->srbm_mutex);
3867 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3868 		gfx_v9_0_kiq_init_register(ring);
3869 		soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3870 		mutex_unlock(&adev->srbm_mutex);
3871 	} else {
3872 		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3873 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3874 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3875 		if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3876 			amdgpu_ring_clear_ring(ring);
3877 		mutex_lock(&adev->srbm_mutex);
3878 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3879 		gfx_v9_0_mqd_init(ring);
3880 		gfx_v9_0_kiq_init_register(ring);
3881 		soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3882 		mutex_unlock(&adev->srbm_mutex);
3883 
3884 		if (adev->gfx.kiq[0].mqd_backup)
3885 			memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3886 	}
3887 
3888 	return 0;
3889 }
3890 
3891 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)
3892 {
3893 	struct amdgpu_device *adev = ring->adev;
3894 	struct v9_mqd *mqd = ring->mqd_ptr;
3895 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
3896 	struct v9_mqd *tmp_mqd;
3897 
3898 	/* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3899 	 * is not be initialized before
3900 	 */
3901 	tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3902 
3903 	if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
3904 	    (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
3905 		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3906 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3907 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3908 		mutex_lock(&adev->srbm_mutex);
3909 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3910 		gfx_v9_0_mqd_init(ring);
3911 		soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3912 		mutex_unlock(&adev->srbm_mutex);
3913 
3914 		if (adev->gfx.mec.mqd_backup[mqd_idx])
3915 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3916 	} else {
3917 		/* restore MQD to a clean status */
3918 		if (adev->gfx.mec.mqd_backup[mqd_idx])
3919 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3920 		/* reset ring buffer */
3921 		ring->wptr = 0;
3922 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3923 		amdgpu_ring_clear_ring(ring);
3924 	}
3925 
3926 	return 0;
3927 }
3928 
3929 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3930 {
3931 	gfx_v9_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
3932 	return 0;
3933 }
3934 
3935 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3936 {
3937 	int i, r;
3938 
3939 	gfx_v9_0_cp_compute_enable(adev, true);
3940 
3941 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3942 		r = gfx_v9_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
3943 		if (r)
3944 			return r;
3945 	}
3946 
3947 	return amdgpu_gfx_enable_kcq(adev, 0);
3948 }
3949 
3950 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3951 {
3952 	int r, i;
3953 	struct amdgpu_ring *ring;
3954 
3955 	if (!(adev->flags & AMD_IS_APU))
3956 		gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3957 
3958 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3959 		if (adev->gfx.num_gfx_rings) {
3960 			/* legacy firmware loading */
3961 			r = gfx_v9_0_cp_gfx_load_microcode(adev);
3962 			if (r)
3963 				return r;
3964 		}
3965 
3966 		r = gfx_v9_0_cp_compute_load_microcode(adev);
3967 		if (r)
3968 			return r;
3969 	}
3970 
3971 	if (adev->gfx.num_gfx_rings)
3972 		gfx_v9_0_cp_gfx_enable(adev, false);
3973 	gfx_v9_0_cp_compute_enable(adev, false);
3974 
3975 	r = gfx_v9_0_kiq_resume(adev);
3976 	if (r)
3977 		return r;
3978 
3979 	if (adev->gfx.num_gfx_rings) {
3980 		r = gfx_v9_0_cp_gfx_resume(adev);
3981 		if (r)
3982 			return r;
3983 	}
3984 
3985 	r = gfx_v9_0_kcq_resume(adev);
3986 	if (r)
3987 		return r;
3988 
3989 	if (adev->gfx.num_gfx_rings) {
3990 		ring = &adev->gfx.gfx_ring[0];
3991 		r = amdgpu_ring_test_helper(ring);
3992 		if (r)
3993 			return r;
3994 	}
3995 
3996 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3997 		ring = &adev->gfx.compute_ring[i];
3998 		amdgpu_ring_test_helper(ring);
3999 	}
4000 
4001 	gfx_v9_0_enable_gui_idle_interrupt(adev, true);
4002 
4003 	return 0;
4004 }
4005 
4006 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
4007 {
4008 	u32 tmp;
4009 
4010 	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&
4011 	    amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
4012 		return;
4013 
4014 	tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
4015 	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
4016 				adev->df.hash_status.hash_64k);
4017 	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
4018 				adev->df.hash_status.hash_2m);
4019 	tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
4020 				adev->df.hash_status.hash_1g);
4021 	WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
4022 }
4023 
4024 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
4025 {
4026 	if (adev->gfx.num_gfx_rings)
4027 		gfx_v9_0_cp_gfx_enable(adev, enable);
4028 	gfx_v9_0_cp_compute_enable(adev, enable);
4029 }
4030 
4031 static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
4032 {
4033 	int r;
4034 	struct amdgpu_device *adev = ip_block->adev;
4035 
4036 	amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4037 				       adev->gfx.cleaner_shader_ptr);
4038 
4039 	if (!amdgpu_sriov_vf(adev))
4040 		gfx_v9_0_init_golden_registers(adev);
4041 
4042 	gfx_v9_0_constants_init(adev);
4043 
4044 	gfx_v9_0_init_tcp_config(adev);
4045 
4046 	r = adev->gfx.rlc.funcs->resume(adev);
4047 	if (r)
4048 		return r;
4049 
4050 	r = gfx_v9_0_cp_resume(adev);
4051 	if (r)
4052 		return r;
4053 
4054 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) &&
4055 	    !amdgpu_sriov_vf(adev))
4056 		gfx_v9_4_2_set_power_brake_sequence(adev);
4057 
4058 	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4059 	if (r)
4060 		return r;
4061 
4062 	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4063 	if (r)
4064 		goto err_priv_inst;
4065 
4066 	r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
4067 	if (r)
4068 		goto err_bad_op;
4069 
4070 	return 0;
4071 
4072 err_bad_op:
4073 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4074 err_priv_inst:
4075 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4076 	return r;
4077 }
4078 
4079 static void gfx_v9_0_deactivate_kcq_hqd(struct amdgpu_device *adev)
4080 {
4081 	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4082 	for (int i = 0; i < adev->gfx.num_compute_rings; i++) {
4083 		u32 tmp;
4084 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4085 
4086 		mutex_lock(&adev->srbm_mutex);
4087 		soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
4088 		tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE);
4089 		/* disable the queue if it's active */
4090 		if (tmp & CP_HQD_ACTIVE__ACTIVE_MASK) {
4091 			int j;
4092 
4093 			WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
4094 			for (j = 0; j < adev->usec_timeout; j++) {
4095 				tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE);
4096 				if (!(tmp & CP_HQD_ACTIVE__ACTIVE_MASK))
4097 					break;
4098 				udelay(1);
4099 			}
4100 			if (j == AMDGPU_MAX_USEC_TIMEOUT) {
4101 				DRM_DEBUG("comp_%u_%u_%u dequeue request failed.\n",
4102 							ring->me, ring->pipe, ring->queue);
4103 				/* Manual disable if dequeue request times out */
4104 				WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0);
4105 			}
4106 			WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0);
4107 		}
4108 		soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4109 		mutex_unlock(&adev->srbm_mutex);
4110 	}
4111 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4112 }
4113 
4114 static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
4115 {
4116 	struct amdgpu_device *adev = ip_block->adev;
4117 
4118 	amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4119 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4120 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4121 
4122 	/* DF freeze and kcq disable will fail */
4123 	if (!amdgpu_ras_intr_triggered())
4124 		/* disable KCQ to avoid CPC touch memory not valid anymore */
4125 		amdgpu_gfx_disable_kcq(adev, 0);
4126 
4127 	if (amdgpu_sriov_vf(adev)) {
4128 		gfx_v9_0_cp_gfx_enable(adev, false);
4129 		/* must disable polling for SRIOV when hw finished, otherwise
4130 		 * CPC engine may still keep fetching WB address which is already
4131 		 * invalid after sw finished and trigger DMAR reading error in
4132 		 * hypervisor side.
4133 		 */
4134 		WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4135 		return 0;
4136 	}
4137 
4138 	if ((adev->flags & AMD_IS_APU) && amdgpu_in_reset(adev) &&
4139 		amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_MODE2)
4140 		gfx_v9_0_deactivate_kcq_hqd(adev);
4141 
4142 	/* Use deinitialize sequence from CAIL when unbinding device from driver,
4143 	 * otherwise KIQ is hanging when binding back
4144 	 */
4145 	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4146 		mutex_lock(&adev->srbm_mutex);
4147 		soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,
4148 				adev->gfx.kiq[0].ring.pipe,
4149 				adev->gfx.kiq[0].ring.queue, 0, 0);
4150 		gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);
4151 		soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4152 		mutex_unlock(&adev->srbm_mutex);
4153 	}
4154 
4155 	gfx_v9_0_cp_enable(adev, false);
4156 
4157 	/* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
4158 	if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
4159 	    (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {
4160 		dev_dbg(adev->dev, "Skipping RLC halt\n");
4161 		return 0;
4162 	}
4163 
4164 	adev->gfx.rlc.funcs->stop(adev);
4165 	return 0;
4166 }
4167 
4168 static int gfx_v9_0_suspend(struct amdgpu_ip_block *ip_block)
4169 {
4170 	return gfx_v9_0_hw_fini(ip_block);
4171 }
4172 
4173 static int gfx_v9_0_resume(struct amdgpu_ip_block *ip_block)
4174 {
4175 	return gfx_v9_0_hw_init(ip_block);
4176 }
4177 
4178 static bool gfx_v9_0_is_idle(struct amdgpu_ip_block *ip_block)
4179 {
4180 	struct amdgpu_device *adev = ip_block->adev;
4181 
4182 	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4183 				GRBM_STATUS, GUI_ACTIVE))
4184 		return false;
4185 	else
4186 		return true;
4187 }
4188 
4189 static int gfx_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4190 {
4191 	unsigned i;
4192 	struct amdgpu_device *adev = ip_block->adev;
4193 
4194 	for (i = 0; i < adev->usec_timeout; i++) {
4195 		if (gfx_v9_0_is_idle(ip_block))
4196 			return 0;
4197 		udelay(1);
4198 	}
4199 	return -ETIMEDOUT;
4200 }
4201 
4202 static int gfx_v9_0_soft_reset(struct amdgpu_ip_block *ip_block)
4203 {
4204 	u32 grbm_soft_reset = 0;
4205 	u32 tmp;
4206 	struct amdgpu_device *adev = ip_block->adev;
4207 
4208 	/* GRBM_STATUS */
4209 	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4210 	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4211 		   GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4212 		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4213 		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4214 		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4215 		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4216 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4217 						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4218 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4219 						GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4220 	}
4221 
4222 	if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4223 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4224 						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4225 	}
4226 
4227 	/* GRBM_STATUS2 */
4228 	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4229 	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4230 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4231 						GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4232 
4233 
4234 	if (grbm_soft_reset) {
4235 		/* stop the rlc */
4236 		adev->gfx.rlc.funcs->stop(adev);
4237 
4238 		if (adev->gfx.num_gfx_rings)
4239 			/* Disable GFX parsing/prefetching */
4240 			gfx_v9_0_cp_gfx_enable(adev, false);
4241 
4242 		/* Disable MEC parsing/prefetching */
4243 		gfx_v9_0_cp_compute_enable(adev, false);
4244 
4245 		tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4246 		tmp |= grbm_soft_reset;
4247 		dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4248 		WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4249 		tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4250 
4251 		udelay(50);
4252 
4253 		tmp &= ~grbm_soft_reset;
4254 		WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4255 		tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4256 
4257 		/* Wait a little for things to settle down */
4258 		udelay(50);
4259 	}
4260 	return 0;
4261 }
4262 
4263 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4264 {
4265 	signed long r, cnt = 0;
4266 	unsigned long flags;
4267 	uint32_t seq, reg_val_offs = 0;
4268 	uint64_t value = 0;
4269 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4270 	struct amdgpu_ring *ring = &kiq->ring;
4271 
4272 	BUG_ON(!ring->funcs->emit_rreg);
4273 
4274 	spin_lock_irqsave(&kiq->ring_lock, flags);
4275 	if (amdgpu_wb_get(adev, &reg_val_offs)) {
4276 		pr_err("critical bug! too many kiq readers\n");
4277 		goto failed_unlock;
4278 	}
4279 	amdgpu_ring_alloc(ring, 32);
4280 	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4281 	amdgpu_ring_write(ring, 9 |	/* src: register*/
4282 				(5 << 8) |	/* dst: memory */
4283 				(1 << 16) |	/* count sel */
4284 				(1 << 20));	/* write confirm */
4285 	amdgpu_ring_write(ring, 0);
4286 	amdgpu_ring_write(ring, 0);
4287 	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4288 				reg_val_offs * 4));
4289 	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4290 				reg_val_offs * 4));
4291 	r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4292 	if (r)
4293 		goto failed_undo;
4294 
4295 	amdgpu_ring_commit(ring);
4296 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
4297 
4298 	r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4299 
4300 	/* don't wait anymore for gpu reset case because this way may
4301 	 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4302 	 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4303 	 * never return if we keep waiting in virt_kiq_rreg, which cause
4304 	 * gpu_recover() hang there.
4305 	 *
4306 	 * also don't wait anymore for IRQ context
4307 	 * */
4308 	if (r < 1 && (amdgpu_in_reset(adev)))
4309 		goto failed_kiq_read;
4310 
4311 	might_sleep();
4312 	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4313 		msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4314 		r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4315 	}
4316 
4317 	if (cnt > MAX_KIQ_REG_TRY)
4318 		goto failed_kiq_read;
4319 
4320 	mb();
4321 	value = (uint64_t)adev->wb.wb[reg_val_offs] |
4322 		(uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4323 	amdgpu_wb_free(adev, reg_val_offs);
4324 	return value;
4325 
4326 failed_undo:
4327 	amdgpu_ring_undo(ring);
4328 failed_unlock:
4329 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
4330 failed_kiq_read:
4331 	if (reg_val_offs)
4332 		amdgpu_wb_free(adev, reg_val_offs);
4333 	pr_err("failed to read gpu clock\n");
4334 	return ~0;
4335 }
4336 
4337 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4338 {
4339 	uint64_t clock, clock_lo, clock_hi, hi_check;
4340 
4341 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
4342 	case IP_VERSION(9, 3, 0):
4343 		preempt_disable();
4344 		clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4345 		clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4346 		hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4347 		/* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
4348 		 * roughly every 42 seconds.
4349 		 */
4350 		if (hi_check != clock_hi) {
4351 			clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4352 			clock_hi = hi_check;
4353 		}
4354 		preempt_enable();
4355 		clock = clock_lo | (clock_hi << 32ULL);
4356 		break;
4357 	default:
4358 		amdgpu_gfx_off_ctrl(adev, false);
4359 		mutex_lock(&adev->gfx.gpu_clock_mutex);
4360 		if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
4361 			    IP_VERSION(9, 0, 1) &&
4362 		    amdgpu_sriov_runtime(adev)) {
4363 			clock = gfx_v9_0_kiq_read_clock(adev);
4364 		} else {
4365 			WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4366 			clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4367 				((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4368 		}
4369 		mutex_unlock(&adev->gfx.gpu_clock_mutex);
4370 		amdgpu_gfx_off_ctrl(adev, true);
4371 		break;
4372 	}
4373 	return clock;
4374 }
4375 
4376 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4377 					  uint32_t vmid,
4378 					  uint32_t gds_base, uint32_t gds_size,
4379 					  uint32_t gws_base, uint32_t gws_size,
4380 					  uint32_t oa_base, uint32_t oa_size)
4381 {
4382 	struct amdgpu_device *adev = ring->adev;
4383 
4384 	/* GDS Base */
4385 	gfx_v9_0_write_data_to_reg(ring, 0, false,
4386 				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4387 				   gds_base);
4388 
4389 	/* GDS Size */
4390 	gfx_v9_0_write_data_to_reg(ring, 0, false,
4391 				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4392 				   gds_size);
4393 
4394 	/* GWS */
4395 	gfx_v9_0_write_data_to_reg(ring, 0, false,
4396 				   SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4397 				   gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4398 
4399 	/* OA */
4400 	gfx_v9_0_write_data_to_reg(ring, 0, false,
4401 				   SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4402 				   (1 << (oa_size + oa_base)) - (1 << oa_base));
4403 }
4404 
4405 static const u32 vgpr_init_compute_shader[] =
4406 {
4407 	0xb07c0000, 0xbe8000ff,
4408 	0x000000f8, 0xbf110800,
4409 	0x7e000280, 0x7e020280,
4410 	0x7e040280, 0x7e060280,
4411 	0x7e080280, 0x7e0a0280,
4412 	0x7e0c0280, 0x7e0e0280,
4413 	0x80808800, 0xbe803200,
4414 	0xbf84fff5, 0xbf9c0000,
4415 	0xd28c0001, 0x0001007f,
4416 	0xd28d0001, 0x0002027e,
4417 	0x10020288, 0xb8810904,
4418 	0xb7814000, 0xd1196a01,
4419 	0x00000301, 0xbe800087,
4420 	0xbefc00c1, 0xd89c4000,
4421 	0x00020201, 0xd89cc080,
4422 	0x00040401, 0x320202ff,
4423 	0x00000800, 0x80808100,
4424 	0xbf84fff8, 0x7e020280,
4425 	0xbf810000, 0x00000000,
4426 };
4427 
4428 static const u32 sgpr_init_compute_shader[] =
4429 {
4430 	0xb07c0000, 0xbe8000ff,
4431 	0x0000005f, 0xbee50080,
4432 	0xbe812c65, 0xbe822c65,
4433 	0xbe832c65, 0xbe842c65,
4434 	0xbe852c65, 0xb77c0005,
4435 	0x80808500, 0xbf84fff8,
4436 	0xbe800080, 0xbf810000,
4437 };
4438 
4439 static const u32 vgpr_init_compute_shader_arcturus[] = {
4440 	0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4441 	0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4442 	0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4443 	0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4444 	0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4445 	0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4446 	0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4447 	0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4448 	0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4449 	0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4450 	0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4451 	0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4452 	0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4453 	0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4454 	0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4455 	0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4456 	0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4457 	0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4458 	0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4459 	0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4460 	0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4461 	0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4462 	0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4463 	0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4464 	0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4465 	0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4466 	0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4467 	0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4468 	0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4469 	0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4470 	0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4471 	0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4472 	0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4473 	0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4474 	0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4475 	0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4476 	0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4477 	0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4478 	0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4479 	0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4480 	0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4481 	0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4482 	0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4483 	0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4484 	0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4485 	0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4486 	0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4487 	0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4488 	0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4489 	0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4490 	0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4491 	0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4492 	0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4493 	0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4494 	0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4495 	0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4496 	0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4497 	0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4498 	0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4499 	0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4500 	0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4501 	0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4502 	0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4503 	0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4504 	0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4505 	0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4506 	0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4507 	0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4508 	0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4509 	0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4510 	0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4511 	0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4512 	0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4513 	0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4514 	0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4515 	0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4516 	0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4517 	0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4518 	0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4519 	0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4520 	0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4521 	0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4522 	0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4523 	0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4524 	0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4525 	0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4526 	0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4527 	0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4528 	0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4529 	0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4530 	0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4531 	0xbf84fff8, 0xbf810000,
4532 };
4533 
4534 /* When below register arrays changed, please update gpr_reg_size,
4535   and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4536   to cover all gfx9 ASICs */
4537 static const struct soc15_reg_entry vgpr_init_regs[] = {
4538    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4539    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4540    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4541    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4542    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4543    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4544    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4545    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4546    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4547    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4548    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4549    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4550    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4551    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4552 };
4553 
4554 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4555    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4556    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4557    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4558    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4559    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4560    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 },  /* 64KB LDS */
4561    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4562    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4563    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4564    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4565    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4566    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4567    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4568    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4569 };
4570 
4571 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4572    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4573    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4574    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4575    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4576    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4577    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4578    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4579    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4580    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4581    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4582    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4583    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4584    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4585    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4586 };
4587 
4588 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4589    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4590    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4591    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4592    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4593    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4594    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4595    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4596    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4597    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4598    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4599    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4600    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4601    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4602    { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4603 };
4604 
4605 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4606    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4607    { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4608    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4609    { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4610    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4611    { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4612    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4613    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4614    { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4615    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4616    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4617    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4618    { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4619    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4620    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4621    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4622    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4623    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4624    { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4625    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4626    { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4627    { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4628    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4629    { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4630    { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4631    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4632    { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4633    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4634    { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4635    { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4636    { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4637    { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4638    { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4639 };
4640 
4641 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4642 {
4643 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4644 	int i, r;
4645 
4646 	/* only support when RAS is enabled */
4647 	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4648 		return 0;
4649 
4650 	r = amdgpu_ring_alloc(ring, 7);
4651 	if (r) {
4652 		drm_err(adev_to_drm(adev), "GDS workarounds failed to lock ring %s (%d).\n",
4653 			ring->name, r);
4654 		return r;
4655 	}
4656 
4657 	WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4658 	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4659 
4660 	amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4661 	amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4662 				PACKET3_DMA_DATA_DST_SEL(1) |
4663 				PACKET3_DMA_DATA_SRC_SEL(2) |
4664 				PACKET3_DMA_DATA_ENGINE(0)));
4665 	amdgpu_ring_write(ring, 0);
4666 	amdgpu_ring_write(ring, 0);
4667 	amdgpu_ring_write(ring, 0);
4668 	amdgpu_ring_write(ring, 0);
4669 	amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4670 				adev->gds.gds_size);
4671 
4672 	amdgpu_ring_commit(ring);
4673 
4674 	for (i = 0; i < adev->usec_timeout; i++) {
4675 		if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4676 			break;
4677 		udelay(1);
4678 	}
4679 
4680 	if (i >= adev->usec_timeout)
4681 		r = -ETIMEDOUT;
4682 
4683 	WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4684 
4685 	return r;
4686 }
4687 
4688 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4689 {
4690 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4691 	struct amdgpu_job *job;
4692 	struct amdgpu_ib *ib;
4693 	struct dma_fence *f = NULL;
4694 	int r, i;
4695 	unsigned total_size, vgpr_offset, sgpr_offset;
4696 	u64 gpu_addr;
4697 
4698 	int compute_dim_x = adev->gfx.config.max_shader_engines *
4699 						adev->gfx.config.max_cu_per_sh *
4700 						adev->gfx.config.max_sh_per_se;
4701 	int sgpr_work_group_size = 5;
4702 	int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4703 	int vgpr_init_shader_size;
4704 	const u32 *vgpr_init_shader_ptr;
4705 	const struct soc15_reg_entry *vgpr_init_regs_ptr;
4706 
4707 	/* only support when RAS is enabled */
4708 	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4709 		return 0;
4710 
4711 	/* bail if the compute ring is not ready */
4712 	if (!ring->sched.ready)
4713 		return 0;
4714 
4715 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
4716 		vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4717 		vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4718 		vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4719 	} else {
4720 		vgpr_init_shader_ptr = vgpr_init_compute_shader;
4721 		vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4722 		vgpr_init_regs_ptr = vgpr_init_regs;
4723 	}
4724 
4725 	total_size =
4726 		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4727 	total_size +=
4728 		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4729 	total_size +=
4730 		(gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4731 	total_size = ALIGN(total_size, 256);
4732 	vgpr_offset = total_size;
4733 	total_size += ALIGN(vgpr_init_shader_size, 256);
4734 	sgpr_offset = total_size;
4735 	total_size += sizeof(sgpr_init_compute_shader);
4736 
4737 	/* allocate an indirect buffer to put the commands in */
4738 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, total_size,
4739 				     AMDGPU_IB_POOL_DIRECT,
4740 				     AMDGPU_KERNEL_JOB_ID_RUN_SHADER,
4741 				     &job);
4742 	if (r) {
4743 		drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r);
4744 		return r;
4745 	}
4746 	ib = &job->ibs[0];
4747 
4748 	/* load the compute shaders */
4749 	for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4750 		ib->ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4751 
4752 	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4753 		ib->ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4754 
4755 	/* init the ib length to 0 */
4756 	ib->length_dw = 0;
4757 
4758 	/* VGPR */
4759 	/* write the register state for the compute dispatch */
4760 	for (i = 0; i < gpr_reg_size; i++) {
4761 		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4762 		ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4763 								- PACKET3_SET_SH_REG_START;
4764 		ib->ptr[ib->length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4765 	}
4766 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4767 	gpu_addr = (ib->gpu_addr + (u64)vgpr_offset) >> 8;
4768 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4769 	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4770 							- PACKET3_SET_SH_REG_START;
4771 	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
4772 	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
4773 
4774 	/* write dispatch packet */
4775 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4776 	ib->ptr[ib->length_dw++] = compute_dim_x * 2; /* x */
4777 	ib->ptr[ib->length_dw++] = 1; /* y */
4778 	ib->ptr[ib->length_dw++] = 1; /* z */
4779 	ib->ptr[ib->length_dw++] =
4780 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4781 
4782 	/* write CS partial flush packet */
4783 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4784 	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4785 
4786 	/* SGPR1 */
4787 	/* write the register state for the compute dispatch */
4788 	for (i = 0; i < gpr_reg_size; i++) {
4789 		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4790 		ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4791 								- PACKET3_SET_SH_REG_START;
4792 		ib->ptr[ib->length_dw++] = sgpr1_init_regs[i].reg_value;
4793 	}
4794 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4795 	gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8;
4796 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4797 	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4798 							- PACKET3_SET_SH_REG_START;
4799 	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
4800 	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
4801 
4802 	/* write dispatch packet */
4803 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4804 	ib->ptr[ib->length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4805 	ib->ptr[ib->length_dw++] = 1; /* y */
4806 	ib->ptr[ib->length_dw++] = 1; /* z */
4807 	ib->ptr[ib->length_dw++] =
4808 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4809 
4810 	/* write CS partial flush packet */
4811 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4812 	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4813 
4814 	/* SGPR2 */
4815 	/* write the register state for the compute dispatch */
4816 	for (i = 0; i < gpr_reg_size; i++) {
4817 		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4818 		ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4819 								- PACKET3_SET_SH_REG_START;
4820 		ib->ptr[ib->length_dw++] = sgpr2_init_regs[i].reg_value;
4821 	}
4822 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4823 	gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8;
4824 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4825 	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4826 							- PACKET3_SET_SH_REG_START;
4827 	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
4828 	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
4829 
4830 	/* write dispatch packet */
4831 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4832 	ib->ptr[ib->length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4833 	ib->ptr[ib->length_dw++] = 1; /* y */
4834 	ib->ptr[ib->length_dw++] = 1; /* z */
4835 	ib->ptr[ib->length_dw++] =
4836 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4837 
4838 	/* write CS partial flush packet */
4839 	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4840 	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4841 
4842 	/* schedule the ib on the ring */
4843 	r = amdgpu_job_submit_direct(job, ring, &f);
4844 	if (r) {
4845 		drm_err(adev_to_drm(adev), "ib schedule failed (%d).\n", r);
4846 		amdgpu_job_free(job);
4847 		goto fail;
4848 	}
4849 
4850 	/* wait for the GPU to finish processing the IB */
4851 	r = dma_fence_wait(f, false);
4852 	if (r) {
4853 		drm_err(adev_to_drm(adev), "fence wait failed (%d).\n", r);
4854 		goto fail;
4855 	}
4856 
4857 fail:
4858 	dma_fence_put(f);
4859 
4860 	return r;
4861 }
4862 
4863 static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block)
4864 {
4865 	struct amdgpu_device *adev = ip_block->adev;
4866 
4867 	adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4868 
4869 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
4870 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4871 		adev->gfx.num_gfx_rings = 0;
4872 	else
4873 		adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4874 	adev->gfx.xcc_mask = 1;
4875 	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4876 					  AMDGPU_MAX_COMPUTE_RINGS);
4877 	gfx_v9_0_set_kiq_pm4_funcs(adev);
4878 	gfx_v9_0_set_ring_funcs(adev);
4879 	gfx_v9_0_set_irq_funcs(adev);
4880 	gfx_v9_0_set_gds_init(adev);
4881 	gfx_v9_0_set_rlc_funcs(adev);
4882 
4883 	/* init rlcg reg access ctrl */
4884 	gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4885 
4886 	amdgpu_init_rlc_reg_funcs(adev);
4887 
4888 	return gfx_v9_0_init_microcode(adev);
4889 }
4890 
4891 static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block)
4892 {
4893 	struct amdgpu_device *adev = ip_block->adev;
4894 	int r;
4895 
4896 	/*
4897 	 * Temp workaround to fix the issue that CP firmware fails to
4898 	 * update read pointer when CPDMA is writing clearing operation
4899 	 * to GDS in suspend/resume sequence on several cards. So just
4900 	 * limit this operation in cold boot sequence.
4901 	 */
4902 	if ((!adev->in_suspend) &&
4903 	    (adev->gds.gds_size)) {
4904 		r = gfx_v9_0_do_edc_gds_workarounds(adev);
4905 		if (r)
4906 			return r;
4907 	}
4908 
4909 	/* requires IBs so do in late init after IB pool is initialized */
4910 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4911 		r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4912 	else
4913 		r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4914 
4915 	if (r)
4916 		return r;
4917 
4918 	if (adev->gfx.ras &&
4919 	    adev->gfx.ras->enable_watchdog_timer)
4920 		adev->gfx.ras->enable_watchdog_timer(adev);
4921 
4922 	return 0;
4923 }
4924 
4925 static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block)
4926 {
4927 	struct amdgpu_device *adev = ip_block->adev;
4928 	int r;
4929 
4930 	r = gfx_v9_0_ecc_late_init(ip_block);
4931 	if (r)
4932 		return r;
4933 
4934 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4935 		gfx_v9_4_2_debug_trap_config_init(adev,
4936 			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4937 	else
4938 		gfx_v9_0_debug_trap_config_init(adev,
4939 			adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4940 
4941 	return 0;
4942 }
4943 
4944 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4945 {
4946 	uint32_t rlc_setting;
4947 
4948 	/* if RLC is not enabled, do nothing */
4949 	rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4950 	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4951 		return false;
4952 
4953 	return true;
4954 }
4955 
4956 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4957 {
4958 	uint32_t data;
4959 	unsigned i;
4960 
4961 	data = RLC_SAFE_MODE__CMD_MASK;
4962 	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4963 	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4964 
4965 	/* wait for RLC_SAFE_MODE */
4966 	for (i = 0; i < adev->usec_timeout; i++) {
4967 		if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4968 			break;
4969 		udelay(1);
4970 	}
4971 }
4972 
4973 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4974 {
4975 	uint32_t data;
4976 
4977 	data = RLC_SAFE_MODE__CMD_MASK;
4978 	WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4979 }
4980 
4981 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4982 						bool enable)
4983 {
4984 	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4985 
4986 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4987 		gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4988 		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4989 			gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4990 	} else {
4991 		gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4992 		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4993 			gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4994 	}
4995 
4996 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4997 }
4998 
4999 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
5000 						bool enable)
5001 {
5002 	/* TODO: double check if we need to perform under safe mode */
5003 	/* gfx_v9_0_enter_rlc_safe_mode(adev); */
5004 
5005 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5006 		gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
5007 	else
5008 		gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
5009 
5010 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5011 		gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5012 	else
5013 		gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5014 
5015 	/* gfx_v9_0_exit_rlc_safe_mode(adev); */
5016 }
5017 
5018 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5019 						      bool enable)
5020 {
5021 	uint32_t data, def;
5022 
5023 	/* It is disabled by HW by default */
5024 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
5025 		/* 1 - RLC_CGTT_MGCG_OVERRIDE */
5026 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5027 
5028 		if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
5029 			data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
5030 
5031 		data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5032 			  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
5033 			  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
5034 
5035 		/* only for Vega10 & Raven1 */
5036 		data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
5037 
5038 		if (def != data)
5039 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5040 
5041 		/* MGLS is a global flag to control all MGLS in GFX */
5042 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5043 			/* 2 - RLC memory Light sleep */
5044 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
5045 				def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
5046 				data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5047 				if (def != data)
5048 					WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
5049 			}
5050 			/* 3 - CP memory Light sleep */
5051 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
5052 				def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
5053 				data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5054 				if (def != data)
5055 					WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
5056 			}
5057 		}
5058 	} else {
5059 		/* 1 - MGCG_OVERRIDE */
5060 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5061 
5062 		if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
5063 			data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
5064 
5065 		data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5066 			 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5067 			 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
5068 			 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
5069 
5070 		if (def != data)
5071 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5072 
5073 		/* 2 - disable MGLS in RLC */
5074 		data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
5075 		if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5076 			data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5077 			WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
5078 		}
5079 
5080 		/* 3 - disable MGLS in CP */
5081 		data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
5082 		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5083 			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5084 			WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
5085 		}
5086 	}
5087 }
5088 
5089 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
5090 					   bool enable)
5091 {
5092 	uint32_t data, def;
5093 
5094 	if (!adev->gfx.num_gfx_rings)
5095 		return;
5096 
5097 	/* Enable 3D CGCG/CGLS */
5098 	if (enable) {
5099 		/* write cmd to clear cgcg/cgls ov */
5100 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5101 		/* unset CGCG override */
5102 		data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5103 		/* update CGCG and CGLS override bits */
5104 		if (def != data)
5105 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5106 
5107 		/* enable 3Dcgcg FSM(0x0000363f) */
5108 		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5109 
5110 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5111 			data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5112 				RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5113 		else
5114 			data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
5115 
5116 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5117 			data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5118 				RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5119 		if (def != data)
5120 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5121 
5122 		/* set IDLE_POLL_COUNT(0x00900100) */
5123 		def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5124 		data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5125 			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5126 		if (def != data)
5127 			WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5128 	} else {
5129 		/* Disable CGCG/CGLS */
5130 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5131 		/* disable cgcg, cgls should be disabled */
5132 		data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5133 			  RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5134 		/* disable cgcg and cgls in FSM */
5135 		if (def != data)
5136 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5137 	}
5138 }
5139 
5140 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5141 						      bool enable)
5142 {
5143 	uint32_t def, data;
5144 
5145 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5146 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5147 		/* unset CGCG override */
5148 		data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5149 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5150 			data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5151 		else
5152 			data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5153 		/* update CGCG and CGLS override bits */
5154 		if (def != data)
5155 			WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5156 
5157 		/* enable cgcg FSM(0x0000363F) */
5158 		def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5159 
5160 		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))
5161 			data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5162 				RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5163 		else
5164 			data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5165 				RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5166 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5167 			data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5168 				RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5169 		if (def != data)
5170 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5171 
5172 		/* set IDLE_POLL_COUNT(0x00900100) */
5173 		def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5174 		data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5175 			(0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5176 		if (def != data)
5177 			WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5178 	} else {
5179 		def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5180 		/* reset CGCG/CGLS bits */
5181 		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5182 		/* disable cgcg and cgls in FSM */
5183 		if (def != data)
5184 			WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5185 	}
5186 }
5187 
5188 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5189 					    bool enable)
5190 {
5191 	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5192 	if (enable) {
5193 		/* CGCG/CGLS should be enabled after MGCG/MGLS
5194 		 * ===  MGCG + MGLS ===
5195 		 */
5196 		gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5197 		/* ===  CGCG /CGLS for GFX 3D Only === */
5198 		gfx_v9_0_update_3d_clock_gating(adev, enable);
5199 		/* ===  CGCG + CGLS === */
5200 		gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5201 	} else {
5202 		/* CGCG/CGLS should be disabled before MGCG/MGLS
5203 		 * ===  CGCG + CGLS ===
5204 		 */
5205 		gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5206 		/* ===  CGCG /CGLS for GFX 3D Only === */
5207 		gfx_v9_0_update_3d_clock_gating(adev, enable);
5208 		/* ===  MGCG + MGLS === */
5209 		gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5210 	}
5211 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5212 	return 0;
5213 }
5214 
5215 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
5216 					      unsigned int vmid)
5217 {
5218 	u32 reg, data;
5219 
5220 	reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5221 	if (amdgpu_sriov_is_pp_one_vf(adev))
5222 		data = RREG32_NO_KIQ(reg);
5223 	else
5224 		data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
5225 
5226 	data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5227 	data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5228 
5229 	if (amdgpu_sriov_is_pp_one_vf(adev))
5230 		WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5231 	else
5232 		WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5233 }
5234 
5235 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
5236 		struct amdgpu_ring *ring, unsigned int vmid)
5237 {
5238 	amdgpu_gfx_off_ctrl(adev, false);
5239 
5240 	gfx_v9_0_update_spm_vmid_internal(adev, vmid);
5241 
5242 	amdgpu_gfx_off_ctrl(adev, true);
5243 }
5244 
5245 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5246 					uint32_t offset,
5247 					struct soc15_reg_rlcg *entries, int arr_size)
5248 {
5249 	int i;
5250 	uint32_t reg;
5251 
5252 	if (!entries)
5253 		return false;
5254 
5255 	for (i = 0; i < arr_size; i++) {
5256 		const struct soc15_reg_rlcg *entry;
5257 
5258 		entry = &entries[i];
5259 		reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5260 		if (offset == reg)
5261 			return true;
5262 	}
5263 
5264 	return false;
5265 }
5266 
5267 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5268 {
5269 	return gfx_v9_0_check_rlcg_range(adev, offset,
5270 					(void *)rlcg_access_gc_9_0,
5271 					ARRAY_SIZE(rlcg_access_gc_9_0));
5272 }
5273 
5274 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5275 	.is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5276 	.set_safe_mode = gfx_v9_0_set_safe_mode,
5277 	.unset_safe_mode = gfx_v9_0_unset_safe_mode,
5278 	.init = gfx_v9_0_rlc_init,
5279 	.get_csb_size = gfx_v9_0_get_csb_size,
5280 	.get_csb_buffer = gfx_v9_0_get_csb_buffer,
5281 	.get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5282 	.resume = gfx_v9_0_rlc_resume,
5283 	.stop = gfx_v9_0_rlc_stop,
5284 	.reset = gfx_v9_0_rlc_reset,
5285 	.start = gfx_v9_0_rlc_start,
5286 	.update_spm_vmid = gfx_v9_0_update_spm_vmid,
5287 	.is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5288 };
5289 
5290 static int gfx_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5291 					  enum amd_powergating_state state)
5292 {
5293 	struct amdgpu_device *adev = ip_block->adev;
5294 	bool enable = (state == AMD_PG_STATE_GATE);
5295 
5296 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5297 	case IP_VERSION(9, 2, 2):
5298 	case IP_VERSION(9, 1, 0):
5299 	case IP_VERSION(9, 3, 0):
5300 		if (!enable)
5301 			amdgpu_gfx_off_ctrl_immediate(adev, false);
5302 
5303 		if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5304 			gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5305 			gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5306 		} else {
5307 			gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5308 			gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5309 		}
5310 
5311 		if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5312 			gfx_v9_0_enable_cp_power_gating(adev, true);
5313 		else
5314 			gfx_v9_0_enable_cp_power_gating(adev, false);
5315 
5316 		/* update gfx cgpg state */
5317 		gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5318 
5319 		/* update mgcg state */
5320 		gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5321 
5322 		if (enable)
5323 			amdgpu_gfx_off_ctrl_immediate(adev, true);
5324 		break;
5325 	case IP_VERSION(9, 2, 1):
5326 		amdgpu_gfx_off_ctrl_immediate(adev, enable);
5327 		break;
5328 	default:
5329 		break;
5330 	}
5331 
5332 	return 0;
5333 }
5334 
5335 static int gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5336 					  enum amd_clockgating_state state)
5337 {
5338 	struct amdgpu_device *adev = ip_block->adev;
5339 
5340 	if (amdgpu_sriov_vf(adev))
5341 		return 0;
5342 
5343 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5344 	case IP_VERSION(9, 0, 1):
5345 	case IP_VERSION(9, 2, 1):
5346 	case IP_VERSION(9, 4, 0):
5347 	case IP_VERSION(9, 2, 2):
5348 	case IP_VERSION(9, 1, 0):
5349 	case IP_VERSION(9, 4, 1):
5350 	case IP_VERSION(9, 3, 0):
5351 	case IP_VERSION(9, 4, 2):
5352 		gfx_v9_0_update_gfx_clock_gating(adev,
5353 						 state == AMD_CG_STATE_GATE);
5354 		break;
5355 	default:
5356 		break;
5357 	}
5358 	return 0;
5359 }
5360 
5361 static void gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
5362 {
5363 	struct amdgpu_device *adev = ip_block->adev;
5364 	int data;
5365 
5366 	if (amdgpu_sriov_vf(adev))
5367 		*flags = 0;
5368 
5369 	/* AMD_CG_SUPPORT_GFX_MGCG */
5370 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5371 	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5372 		*flags |= AMD_CG_SUPPORT_GFX_MGCG;
5373 
5374 	/* AMD_CG_SUPPORT_GFX_CGCG */
5375 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5376 	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5377 		*flags |= AMD_CG_SUPPORT_GFX_CGCG;
5378 
5379 	/* AMD_CG_SUPPORT_GFX_CGLS */
5380 	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5381 		*flags |= AMD_CG_SUPPORT_GFX_CGLS;
5382 
5383 	/* AMD_CG_SUPPORT_GFX_RLC_LS */
5384 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5385 	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5386 		*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5387 
5388 	/* AMD_CG_SUPPORT_GFX_CP_LS */
5389 	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5390 	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5391 		*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5392 
5393 	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {
5394 		/* AMD_CG_SUPPORT_GFX_3D_CGCG */
5395 		data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5396 		if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5397 			*flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5398 
5399 		/* AMD_CG_SUPPORT_GFX_3D_CGLS */
5400 		if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5401 			*flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5402 	}
5403 }
5404 
5405 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5406 {
5407 	return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5408 }
5409 
5410 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5411 {
5412 	struct amdgpu_device *adev = ring->adev;
5413 	u64 wptr;
5414 
5415 	/* XXX check if swapping is necessary on BE */
5416 	if (ring->use_doorbell) {
5417 		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5418 	} else {
5419 		wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5420 		wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5421 	}
5422 
5423 	return wptr;
5424 }
5425 
5426 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5427 {
5428 	struct amdgpu_device *adev = ring->adev;
5429 
5430 	if (ring->use_doorbell) {
5431 		/* XXX check if swapping is necessary on BE */
5432 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5433 		WDOORBELL64(ring->doorbell_index, ring->wptr);
5434 	} else {
5435 		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5436 		WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5437 	}
5438 }
5439 
5440 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5441 {
5442 	struct amdgpu_device *adev = ring->adev;
5443 	u32 ref_and_mask, reg_mem_engine;
5444 
5445 	if (!adev->gfx.funcs->get_hdp_flush_mask) {
5446 		dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
5447 		return;
5448 	}
5449 
5450 	adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, &reg_mem_engine);
5451 	gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5452 			      adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5453 			      adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5454 			      ref_and_mask, ref_and_mask, 0x20);
5455 }
5456 
5457 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5458 					struct amdgpu_job *job,
5459 					struct amdgpu_ib *ib,
5460 					uint32_t flags)
5461 {
5462 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5463 	u32 header, control = 0;
5464 
5465 	if (ib->flags & AMDGPU_IB_FLAG_CE)
5466 		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5467 	else
5468 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5469 
5470 	control |= ib->length_dw | (vmid << 24);
5471 
5472 	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5473 		control |= INDIRECT_BUFFER_PRE_ENB(1);
5474 
5475 		if (flags & AMDGPU_IB_PREEMPTED)
5476 			control |= INDIRECT_BUFFER_PRE_RESUME(1);
5477 
5478 		if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5479 			gfx_v9_0_ring_emit_de_meta(ring,
5480 						   (!amdgpu_sriov_vf(ring->adev) &&
5481 						   flags & AMDGPU_IB_PREEMPTED) ?
5482 						   true : false,
5483 						   job->gds_size > 0 && job->gds_base != 0);
5484 	}
5485 
5486 	amdgpu_ring_write(ring, header);
5487 	WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
5488 	amdgpu_ring_write(ring,
5489 #ifdef __BIG_ENDIAN
5490 		(2 << 0) |
5491 #endif
5492 		lower_32_bits(ib->gpu_addr));
5493 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5494 	amdgpu_ring_ib_on_emit_cntl(ring);
5495 	amdgpu_ring_write(ring, control);
5496 }
5497 
5498 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5499 				     unsigned offset)
5500 {
5501 	u32 control = ring->ring[offset];
5502 
5503 	control |= INDIRECT_BUFFER_PRE_RESUME(1);
5504 	ring->ring[offset] = control;
5505 }
5506 
5507 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5508 					unsigned offset)
5509 {
5510 	struct amdgpu_device *adev = ring->adev;
5511 	void *ce_payload_cpu_addr;
5512 	uint64_t payload_offset, payload_size;
5513 
5514 	payload_size = sizeof(struct v9_ce_ib_state);
5515 
5516 	payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5517 	ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5518 
5519 	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5520 		memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5521 	} else {
5522 		memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5523 		       (ring->buf_mask + 1 - offset) << 2);
5524 		payload_size -= (ring->buf_mask + 1 - offset) << 2;
5525 		memcpy((void *)&ring->ring[0],
5526 		       ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5527 		       payload_size);
5528 	}
5529 }
5530 
5531 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5532 					unsigned offset)
5533 {
5534 	struct amdgpu_device *adev = ring->adev;
5535 	void *de_payload_cpu_addr;
5536 	uint64_t payload_offset, payload_size;
5537 
5538 	payload_size = sizeof(struct v9_de_ib_state);
5539 
5540 	payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5541 	de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5542 
5543 	((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5544 		IB_COMPLETION_STATUS_PREEMPTED;
5545 
5546 	if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5547 		memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5548 	} else {
5549 		memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5550 		       (ring->buf_mask + 1 - offset) << 2);
5551 		payload_size -= (ring->buf_mask + 1 - offset) << 2;
5552 		memcpy((void *)&ring->ring[0],
5553 		       de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5554 		       payload_size);
5555 	}
5556 }
5557 
5558 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5559 					  struct amdgpu_job *job,
5560 					  struct amdgpu_ib *ib,
5561 					  uint32_t flags)
5562 {
5563 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5564 	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5565 
5566 	/* Currently, there is a high possibility to get wave ID mismatch
5567 	 * between ME and GDS, leading to a hw deadlock, because ME generates
5568 	 * different wave IDs than the GDS expects. This situation happens
5569 	 * randomly when at least 5 compute pipes use GDS ordered append.
5570 	 * The wave IDs generated by ME are also wrong after suspend/resume.
5571 	 * Those are probably bugs somewhere else in the kernel driver.
5572 	 *
5573 	 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5574 	 * GDS to 0 for this ring (me/pipe).
5575 	 */
5576 	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5577 		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5578 		amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5579 		amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5580 	}
5581 
5582 	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5583 	WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
5584 	amdgpu_ring_write(ring,
5585 #ifdef __BIG_ENDIAN
5586 				(2 << 0) |
5587 #endif
5588 				lower_32_bits(ib->gpu_addr));
5589 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5590 	amdgpu_ring_write(ring, control);
5591 }
5592 
5593 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5594 				     u64 seq, unsigned flags)
5595 {
5596 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5597 	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5598 	bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5599 	bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5600 	uint32_t dw2 = 0;
5601 
5602 	/* RELEASE_MEM - flush caches, send int */
5603 	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5604 
5605 	if (writeback) {
5606 		dw2 = EOP_TC_NC_ACTION_EN;
5607 	} else {
5608 		dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5609 				EOP_TC_MD_ACTION_EN;
5610 	}
5611 	dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5612 				EVENT_INDEX(5);
5613 	if (exec)
5614 		dw2 |= EOP_EXEC;
5615 
5616 	amdgpu_ring_write(ring, dw2);
5617 	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5618 
5619 	/*
5620 	 * the address should be Qword aligned if 64bit write, Dword
5621 	 * aligned if only send 32bit data low (discard data high)
5622 	 */
5623 	if (write64bit)
5624 		WARN_ON(addr & 0x7);
5625 	else
5626 		WARN_ON(addr & 0x3);
5627 	amdgpu_ring_write(ring, lower_32_bits(addr));
5628 	amdgpu_ring_write(ring, upper_32_bits(addr));
5629 	amdgpu_ring_write(ring, lower_32_bits(seq));
5630 	amdgpu_ring_write(ring, upper_32_bits(seq));
5631 	amdgpu_ring_write(ring, 0);
5632 }
5633 
5634 static void gfx_v9_0_ring_emit_event_write(struct amdgpu_ring *ring,
5635 					   uint32_t event_type,
5636 					   uint32_t event_index)
5637 {
5638 	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
5639 	amdgpu_ring_write(ring, EVENT_TYPE(event_type) |
5640 			  EVENT_INDEX(event_index));
5641 }
5642 
5643 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
5644 {
5645 	const unsigned int cp_coher_cntl =
5646 			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
5647 			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
5648 			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
5649 			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
5650 			PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
5651 
5652 	/* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
5653 	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
5654 	amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
5655 	amdgpu_ring_write(ring, 0xffffffff);  /* CP_COHER_SIZE */
5656 	amdgpu_ring_write(ring, 0xffffff);  /* CP_COHER_SIZE_HI */
5657 	amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
5658 	amdgpu_ring_write(ring, 0);  /* CP_COHER_BASE_HI */
5659 	amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
5660 }
5661 
5662 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5663 {
5664 	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5665 		gfx_v9_0_ring_emit_event_write(ring, VS_PARTIAL_FLUSH, 4);
5666 		gfx_v9_0_ring_emit_event_write(ring, PS_PARTIAL_FLUSH, 4);
5667 	}
5668 	gfx_v9_0_ring_emit_event_write(ring, CS_PARTIAL_FLUSH, 4);
5669 	gfx_v9_0_emit_mem_sync(ring);
5670 }
5671 
5672 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5673 					unsigned vmid, uint64_t pd_addr)
5674 {
5675 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5676 
5677 	/* compute doesn't have PFP */
5678 	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5679 		/* sync PFP to ME, otherwise we might get invalid PFP reads */
5680 		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5681 		amdgpu_ring_write(ring, 0x0);
5682 	}
5683 }
5684 
5685 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5686 {
5687 	return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5688 }
5689 
5690 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5691 {
5692 	struct amdgpu_device *adev = ring->adev;
5693 	u64 wptr;
5694 
5695 	/* XXX check if swapping is necessary on BE */
5696 	if (ring->use_doorbell) {
5697 		wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5698 	} else {
5699 		dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
5700 		wptr = 0;
5701 	}
5702 	return wptr;
5703 }
5704 
5705 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5706 {
5707 	struct amdgpu_device *adev = ring->adev;
5708 
5709 	/* XXX check if swapping is necessary on BE */
5710 	if (ring->use_doorbell) {
5711 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5712 		WDOORBELL64(ring->doorbell_index, ring->wptr);
5713 	} else {
5714 		dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
5715 	}
5716 }
5717 
5718 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5719 					 u64 seq, unsigned int flags)
5720 {
5721 	struct amdgpu_device *adev = ring->adev;
5722 
5723 	/* write fence seq to the "addr" */
5724 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5725 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5726 				 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5727 	amdgpu_ring_write(ring, lower_32_bits(addr));
5728 	amdgpu_ring_write(ring, upper_32_bits(addr));
5729 	amdgpu_ring_write(ring, lower_32_bits(seq));
5730 
5731 	if (flags & AMDGPU_FENCE_FLAG_INT) {
5732 		/* set register to trigger INT */
5733 		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5734 		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5735 					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5736 		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5737 		amdgpu_ring_write(ring, 0);
5738 		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5739 	}
5740 }
5741 
5742 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5743 {
5744 	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5745 	amdgpu_ring_write(ring, 0);
5746 }
5747 
5748 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5749 {
5750 	struct amdgpu_device *adev = ring->adev;
5751 	struct v9_ce_ib_state ce_payload = {0};
5752 	uint64_t offset, ce_payload_gpu_addr;
5753 	void *ce_payload_cpu_addr;
5754 	int cnt;
5755 
5756 	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5757 
5758 	offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5759 	ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5760 	ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5761 
5762 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5763 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5764 				 WRITE_DATA_DST_SEL(8) |
5765 				 WR_CONFIRM) |
5766 				 WRITE_DATA_CACHE_POLICY(0));
5767 	amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5768 	amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5769 
5770 	amdgpu_ring_ib_on_emit_ce(ring);
5771 
5772 	if (resume)
5773 		amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,
5774 					   sizeof(ce_payload) >> 2);
5775 	else
5776 		amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
5777 					   sizeof(ce_payload) >> 2);
5778 }
5779 
5780 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5781 {
5782 	int i, r = 0;
5783 	struct amdgpu_device *adev = ring->adev;
5784 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5785 	struct amdgpu_ring *kiq_ring = &kiq->ring;
5786 	unsigned long flags;
5787 
5788 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5789 		return -EINVAL;
5790 
5791 	spin_lock_irqsave(&kiq->ring_lock, flags);
5792 
5793 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5794 		spin_unlock_irqrestore(&kiq->ring_lock, flags);
5795 		return -ENOMEM;
5796 	}
5797 
5798 	/* assert preemption condition */
5799 	amdgpu_ring_set_preempt_cond_exec(ring, false);
5800 
5801 	ring->trail_seq += 1;
5802 	amdgpu_ring_alloc(ring, 13);
5803 	gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
5804 				 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5805 
5806 	/* assert IB preemption, emit the trailing fence */
5807 	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5808 				   ring->trail_fence_gpu_addr,
5809 				   ring->trail_seq);
5810 
5811 	amdgpu_ring_commit(kiq_ring);
5812 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
5813 
5814 	/* poll the trailing fence */
5815 	for (i = 0; i < adev->usec_timeout; i++) {
5816 		if (ring->trail_seq ==
5817 			le32_to_cpu(*ring->trail_fence_cpu_addr))
5818 			break;
5819 		udelay(1);
5820 	}
5821 
5822 	if (i >= adev->usec_timeout) {
5823 		r = -EINVAL;
5824 		drm_warn(adev_to_drm(adev), "ring %d timeout to preempt ib\n", ring->idx);
5825 	}
5826 
5827 	/*reset the CP_VMID_PREEMPT after trailing fence*/
5828 	amdgpu_ring_emit_wreg(ring,
5829 			      SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5830 			      0x0);
5831 	amdgpu_ring_commit(ring);
5832 
5833 	/* deassert preemption condition */
5834 	amdgpu_ring_set_preempt_cond_exec(ring, true);
5835 	return r;
5836 }
5837 
5838 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5839 {
5840 	struct amdgpu_device *adev = ring->adev;
5841 	struct v9_de_ib_state de_payload = {0};
5842 	uint64_t offset, gds_addr, de_payload_gpu_addr;
5843 	void *de_payload_cpu_addr;
5844 	int cnt;
5845 
5846 	offset = offsetof(struct v9_gfx_meta_data, de_payload);
5847 	de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5848 	de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5849 
5850 	gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5851 			 AMDGPU_CSA_SIZE - adev->gds.gds_size,
5852 			 PAGE_SIZE);
5853 
5854 	if (usegds) {
5855 		de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5856 		de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5857 	}
5858 
5859 	cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5860 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5861 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5862 				 WRITE_DATA_DST_SEL(8) |
5863 				 WR_CONFIRM) |
5864 				 WRITE_DATA_CACHE_POLICY(0));
5865 	amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5866 	amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5867 
5868 	amdgpu_ring_ib_on_emit_de(ring);
5869 	if (resume)
5870 		amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5871 					   sizeof(de_payload) >> 2);
5872 	else
5873 		amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5874 					   sizeof(de_payload) >> 2);
5875 }
5876 
5877 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5878 				   bool secure)
5879 {
5880 	uint32_t v = secure ? FRAME_TMZ : 0;
5881 
5882 	amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5883 	amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5884 }
5885 
5886 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5887 {
5888 	uint32_t dw2 = 0;
5889 
5890 	gfx_v9_0_ring_emit_ce_meta(ring,
5891 				   (!amdgpu_sriov_vf(ring->adev) &&
5892 				   flags & AMDGPU_IB_PREEMPTED) ? true : false);
5893 
5894 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5895 	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5896 		/* set load_global_config & load_global_uconfig */
5897 		dw2 |= 0x8001;
5898 		/* set load_cs_sh_regs */
5899 		dw2 |= 0x01000000;
5900 		/* set load_per_context_state & load_gfx_sh_regs for GFX */
5901 		dw2 |= 0x10002;
5902 
5903 		/* set load_ce_ram if preamble presented */
5904 		if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5905 			dw2 |= 0x10000000;
5906 	} else {
5907 		/* still load_ce_ram if this is the first time preamble presented
5908 		 * although there is no context switch happens.
5909 		 */
5910 		if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5911 			dw2 |= 0x10000000;
5912 	}
5913 
5914 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5915 	amdgpu_ring_write(ring, dw2);
5916 	amdgpu_ring_write(ring, 0);
5917 }
5918 
5919 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5920 						  uint64_t addr)
5921 {
5922 	unsigned ret;
5923 	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5924 	amdgpu_ring_write(ring, lower_32_bits(addr));
5925 	amdgpu_ring_write(ring, upper_32_bits(addr));
5926 	/* discard following DWs if *cond_exec_gpu_addr==0 */
5927 	amdgpu_ring_write(ring, 0);
5928 	ret = ring->wptr & ring->buf_mask;
5929 	/* patch dummy value later */
5930 	amdgpu_ring_write(ring, 0);
5931 	return ret;
5932 }
5933 
5934 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5935 				    uint32_t reg_val_offs)
5936 {
5937 	struct amdgpu_device *adev = ring->adev;
5938 
5939 	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5940 	amdgpu_ring_write(ring, 0 |	/* src: register*/
5941 				(5 << 8) |	/* dst: memory */
5942 				(1 << 20));	/* write confirm */
5943 	amdgpu_ring_write(ring, reg);
5944 	amdgpu_ring_write(ring, 0);
5945 	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5946 				reg_val_offs * 4));
5947 	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5948 				reg_val_offs * 4));
5949 }
5950 
5951 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5952 				    uint32_t val)
5953 {
5954 	uint32_t cmd = 0;
5955 
5956 	switch (ring->funcs->type) {
5957 	case AMDGPU_RING_TYPE_GFX:
5958 		cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5959 		break;
5960 	case AMDGPU_RING_TYPE_KIQ:
5961 		cmd = (1 << 16); /* no inc addr */
5962 		break;
5963 	default:
5964 		cmd = WR_CONFIRM;
5965 		break;
5966 	}
5967 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5968 	amdgpu_ring_write(ring, cmd);
5969 	amdgpu_ring_write(ring, reg);
5970 	amdgpu_ring_write(ring, 0);
5971 	amdgpu_ring_write(ring, val);
5972 }
5973 
5974 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5975 					uint32_t val, uint32_t mask)
5976 {
5977 	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5978 }
5979 
5980 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5981 						  uint32_t reg0, uint32_t reg1,
5982 						  uint32_t ref, uint32_t mask)
5983 {
5984 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5985 	struct amdgpu_device *adev = ring->adev;
5986 	bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5987 		adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5988 
5989 	if (fw_version_ok)
5990 		gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5991 				      ref, mask, 0x20);
5992 	else
5993 		amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5994 							   ref, mask);
5995 }
5996 
5997 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5998 {
5999 	struct amdgpu_device *adev = ring->adev;
6000 	uint32_t value = 0;
6001 
6002 	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6003 	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6004 	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6005 	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6006 	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
6007 	WREG32_SOC15(GC, 0, mmSQ_CMD, value);
6008 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
6009 }
6010 
6011 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6012 						 enum amdgpu_interrupt_state state)
6013 {
6014 	switch (state) {
6015 	case AMDGPU_IRQ_STATE_DISABLE:
6016 	case AMDGPU_IRQ_STATE_ENABLE:
6017 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6018 			       TIME_STAMP_INT_ENABLE,
6019 			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6020 		break;
6021 	default:
6022 		break;
6023 	}
6024 }
6025 
6026 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6027 						     int me, int pipe,
6028 						     enum amdgpu_interrupt_state state)
6029 {
6030 	u32 mec_int_cntl, mec_int_cntl_reg;
6031 
6032 	/*
6033 	 * amdgpu controls only the first MEC. That's why this function only
6034 	 * handles the setting of interrupts for this specific MEC. All other
6035 	 * pipes' interrupts are set by amdkfd.
6036 	 */
6037 
6038 	if (me == 1) {
6039 		switch (pipe) {
6040 		case 0:
6041 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6042 			break;
6043 		case 1:
6044 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6045 			break;
6046 		case 2:
6047 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6048 			break;
6049 		case 3:
6050 			mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6051 			break;
6052 		default:
6053 			DRM_DEBUG("invalid pipe %d\n", pipe);
6054 			return;
6055 		}
6056 	} else {
6057 		DRM_DEBUG("invalid me %d\n", me);
6058 		return;
6059 	}
6060 
6061 	switch (state) {
6062 	case AMDGPU_IRQ_STATE_DISABLE:
6063 		mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
6064 		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6065 					     TIME_STAMP_INT_ENABLE, 0);
6066 		WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6067 		break;
6068 	case AMDGPU_IRQ_STATE_ENABLE:
6069 		mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6070 		mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6071 					     TIME_STAMP_INT_ENABLE, 1);
6072 		WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6073 		break;
6074 	default:
6075 		break;
6076 	}
6077 }
6078 
6079 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev,
6080 				     int me, int pipe)
6081 {
6082 	/*
6083 	 * amdgpu controls only the first MEC. That's why this function only
6084 	 * handles the setting of interrupts for this specific MEC. All other
6085 	 * pipes' interrupts are set by amdkfd.
6086 	 */
6087 	if (me != 1)
6088 		return 0;
6089 
6090 	switch (pipe) {
6091 	case 0:
6092 		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6093 	case 1:
6094 		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6095 	case 2:
6096 		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6097 	case 3:
6098 		return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6099 	default:
6100 		return 0;
6101 	}
6102 }
6103 
6104 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6105 					     struct amdgpu_irq_src *source,
6106 					     unsigned type,
6107 					     enum amdgpu_interrupt_state state)
6108 {
6109 	u32 cp_int_cntl_reg, cp_int_cntl;
6110 	int i, j;
6111 
6112 	switch (state) {
6113 	case AMDGPU_IRQ_STATE_DISABLE:
6114 	case AMDGPU_IRQ_STATE_ENABLE:
6115 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6116 			       PRIV_REG_INT_ENABLE,
6117 			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6118 		for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6119 			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6120 				/* MECs start at 1 */
6121 				cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6122 
6123 				if (cp_int_cntl_reg) {
6124 					cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6125 					cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6126 								    PRIV_REG_INT_ENABLE,
6127 								    state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6128 					WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6129 				}
6130 			}
6131 		}
6132 		break;
6133 	default:
6134 		break;
6135 	}
6136 
6137 	return 0;
6138 }
6139 
6140 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6141 					   struct amdgpu_irq_src *source,
6142 					   unsigned type,
6143 					   enum amdgpu_interrupt_state state)
6144 {
6145 	u32 cp_int_cntl_reg, cp_int_cntl;
6146 	int i, j;
6147 
6148 	switch (state) {
6149 	case AMDGPU_IRQ_STATE_DISABLE:
6150 	case AMDGPU_IRQ_STATE_ENABLE:
6151 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6152 			       OPCODE_ERROR_INT_ENABLE,
6153 			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6154 		for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6155 			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6156 				/* MECs start at 1 */
6157 				cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6158 
6159 				if (cp_int_cntl_reg) {
6160 					cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6161 					cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6162 								    OPCODE_ERROR_INT_ENABLE,
6163 								    state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6164 					WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6165 				}
6166 			}
6167 		}
6168 		break;
6169 	default:
6170 		break;
6171 	}
6172 
6173 	return 0;
6174 }
6175 
6176 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6177 					      struct amdgpu_irq_src *source,
6178 					      unsigned type,
6179 					      enum amdgpu_interrupt_state state)
6180 {
6181 	switch (state) {
6182 	case AMDGPU_IRQ_STATE_DISABLE:
6183 	case AMDGPU_IRQ_STATE_ENABLE:
6184 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6185 			       PRIV_INSTR_INT_ENABLE,
6186 			       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6187 		break;
6188 	default:
6189 		break;
6190 	}
6191 
6192 	return 0;
6193 }
6194 
6195 #define ENABLE_ECC_ON_ME_PIPE(me, pipe)				\
6196 	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6197 			CP_ECC_ERROR_INT_ENABLE, 1)
6198 
6199 #define DISABLE_ECC_ON_ME_PIPE(me, pipe)			\
6200 	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6201 			CP_ECC_ERROR_INT_ENABLE, 0)
6202 
6203 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
6204 					      struct amdgpu_irq_src *source,
6205 					      unsigned type,
6206 					      enum amdgpu_interrupt_state state)
6207 {
6208 	switch (state) {
6209 	case AMDGPU_IRQ_STATE_DISABLE:
6210 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6211 				CP_ECC_ERROR_INT_ENABLE, 0);
6212 		DISABLE_ECC_ON_ME_PIPE(1, 0);
6213 		DISABLE_ECC_ON_ME_PIPE(1, 1);
6214 		DISABLE_ECC_ON_ME_PIPE(1, 2);
6215 		DISABLE_ECC_ON_ME_PIPE(1, 3);
6216 		break;
6217 
6218 	case AMDGPU_IRQ_STATE_ENABLE:
6219 		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6220 				CP_ECC_ERROR_INT_ENABLE, 1);
6221 		ENABLE_ECC_ON_ME_PIPE(1, 0);
6222 		ENABLE_ECC_ON_ME_PIPE(1, 1);
6223 		ENABLE_ECC_ON_ME_PIPE(1, 2);
6224 		ENABLE_ECC_ON_ME_PIPE(1, 3);
6225 		break;
6226 	default:
6227 		break;
6228 	}
6229 
6230 	return 0;
6231 }
6232 
6233 
6234 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6235 					    struct amdgpu_irq_src *src,
6236 					    unsigned type,
6237 					    enum amdgpu_interrupt_state state)
6238 {
6239 	switch (type) {
6240 	case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6241 		gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
6242 		break;
6243 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6244 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6245 		break;
6246 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6247 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6248 		break;
6249 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6250 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6251 		break;
6252 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6253 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6254 		break;
6255 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6256 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6257 		break;
6258 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6259 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6260 		break;
6261 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6262 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6263 		break;
6264 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6265 		gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6266 		break;
6267 	default:
6268 		break;
6269 	}
6270 	return 0;
6271 }
6272 
6273 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
6274 			    struct amdgpu_irq_src *source,
6275 			    struct amdgpu_iv_entry *entry)
6276 {
6277 	int i;
6278 	u8 me_id, pipe_id, queue_id;
6279 	struct amdgpu_ring *ring;
6280 
6281 	DRM_DEBUG("IH: CP EOP\n");
6282 	me_id = (entry->ring_id & 0x0c) >> 2;
6283 	pipe_id = (entry->ring_id & 0x03) >> 0;
6284 	queue_id = (entry->ring_id & 0x70) >> 4;
6285 
6286 	switch (me_id) {
6287 	case 0:
6288 		if (adev->gfx.num_gfx_rings) {
6289 			if (!adev->gfx.mcbp) {
6290 				amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6291 			} else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {
6292 				/* Fence signals are handled on the software rings*/
6293 				for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
6294 					amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);
6295 			}
6296 		}
6297 		break;
6298 	case 1:
6299 	case 2:
6300 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6301 			ring = &adev->gfx.compute_ring[i];
6302 			/* Per-queue interrupt is supported for MEC starting from VI.
6303 			  * The interrupt can only be enabled/disabled per pipe instead of per queue.
6304 			  */
6305 			if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6306 				amdgpu_fence_process(ring);
6307 		}
6308 		break;
6309 	}
6310 	return 0;
6311 }
6312 
6313 static void gfx_v9_0_fault(struct amdgpu_device *adev,
6314 			   struct amdgpu_iv_entry *entry)
6315 {
6316 	u8 me_id, pipe_id, queue_id;
6317 	struct amdgpu_ring *ring;
6318 	int i;
6319 
6320 	me_id = (entry->ring_id & 0x0c) >> 2;
6321 	pipe_id = (entry->ring_id & 0x03) >> 0;
6322 	queue_id = (entry->ring_id & 0x70) >> 4;
6323 
6324 	switch (me_id) {
6325 	case 0:
6326 		drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6327 		break;
6328 	case 1:
6329 	case 2:
6330 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6331 			ring = &adev->gfx.compute_ring[i];
6332 			if (ring->me == me_id && ring->pipe == pipe_id &&
6333 			    ring->queue == queue_id)
6334 				drm_sched_fault(&ring->sched);
6335 		}
6336 		break;
6337 	}
6338 }
6339 
6340 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
6341 				 struct amdgpu_irq_src *source,
6342 				 struct amdgpu_iv_entry *entry)
6343 {
6344 	DRM_ERROR("Illegal register access in command stream\n");
6345 	gfx_v9_0_fault(adev, entry);
6346 	return 0;
6347 }
6348 
6349 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev,
6350 			       struct amdgpu_irq_src *source,
6351 			       struct amdgpu_iv_entry *entry)
6352 {
6353 	DRM_ERROR("Illegal opcode in command stream\n");
6354 	gfx_v9_0_fault(adev, entry);
6355 	return 0;
6356 }
6357 
6358 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
6359 				  struct amdgpu_irq_src *source,
6360 				  struct amdgpu_iv_entry *entry)
6361 {
6362 	DRM_ERROR("Illegal instruction in command stream\n");
6363 	gfx_v9_0_fault(adev, entry);
6364 	return 0;
6365 }
6366 
6367 
6368 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
6369 	{ "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
6370 	  SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
6371 	  SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
6372 	},
6373 	{ "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
6374 	  SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
6375 	  SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
6376 	},
6377 	{ "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6378 	  SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
6379 	  0, 0
6380 	},
6381 	{ "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6382 	  SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
6383 	  0, 0
6384 	},
6385 	{ "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
6386 	  SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
6387 	  SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
6388 	},
6389 	{ "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6390 	  SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
6391 	  0, 0
6392 	},
6393 	{ "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6394 	  SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6395 	  SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6396 	},
6397 	{ "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6398 	  SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6399 	  SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6400 	},
6401 	{ "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6402 	  SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6403 	  0, 0
6404 	},
6405 	{ "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6406 	  SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6407 	  0, 0
6408 	},
6409 	{ "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6410 	  SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6411 	  0, 0
6412 	},
6413 	{ "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6414 	  SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6415 	  SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6416 	},
6417 	{ "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6418 	  SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6419 	  0, 0
6420 	},
6421 	{ "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6422 	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6423 	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6424 	},
6425 	{ "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6426 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6427 	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6428 	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6429 	},
6430 	{ "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6431 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6432 	  SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6433 	  0, 0
6434 	},
6435 	{ "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6436 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6437 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6438 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6439 	},
6440 	{ "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6441 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6442 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6443 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6444 	},
6445 	{ "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6446 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6447 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6448 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6449 	},
6450 	{ "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6451 	  SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6452 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6453 	  SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6454 	},
6455 	{ "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6456 	  SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6457 	  0, 0
6458 	},
6459 	{ "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6460 	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6461 	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6462 	},
6463 	{ "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6464 	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6465 	  0, 0
6466 	},
6467 	{ "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6468 	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6469 	  0, 0
6470 	},
6471 	{ "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6472 	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6473 	  0, 0
6474 	},
6475 	{ "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6476 	  SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6477 	  0, 0
6478 	},
6479 	{ "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6480 	  SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6481 	  0, 0
6482 	},
6483 	{ "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6484 	  SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6485 	  0, 0
6486 	},
6487 	{ "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6488 	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6489 	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6490 	},
6491 	{ "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6492 	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6493 	  SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6494 	},
6495 	{ "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6496 	  SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6497 	  SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6498 	},
6499 	{ "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6500 	  SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6501 	  SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6502 	},
6503 	{ "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6504 	  SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6505 	  SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6506 	},
6507 	{ "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6508 	  SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6509 	  0, 0
6510 	},
6511 	{ "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6512 	  SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6513 	  0, 0
6514 	},
6515 	{ "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6516 	  SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6517 	  0, 0
6518 	},
6519 	{ "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6520 	  SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6521 	  0, 0
6522 	},
6523 	{ "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6524 	  SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6525 	  0, 0
6526 	},
6527 	{ "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6528 	  SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6529 	  0, 0
6530 	},
6531 	{ "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6532 	  SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6533 	  0, 0
6534 	},
6535 	{ "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6536 	  SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6537 	  0, 0
6538 	},
6539 	{ "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6540 	  SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6541 	  0, 0
6542 	},
6543 	{ "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6544 	  SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6545 	  0, 0
6546 	},
6547 	{ "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6548 	  SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6549 	  0, 0
6550 	},
6551 	{ "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6552 	  SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6553 	  0, 0
6554 	},
6555 	{ "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6556 	  SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6557 	  0, 0
6558 	},
6559 	{ "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6560 	  SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6561 	  0, 0
6562 	},
6563 	{ "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6564 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6565 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6566 	},
6567 	{ "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6568 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6569 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6570 	},
6571 	{ "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6572 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6573 	  0, 0
6574 	},
6575 	{ "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6576 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6577 	  0, 0
6578 	},
6579 	{ "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6580 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6581 	  0, 0
6582 	},
6583 	{ "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6584 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6585 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6586 	},
6587 	{ "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6588 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6589 	  SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6590 	},
6591 	{ "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6592 	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6593 	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6594 	},
6595 	{ "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6596 	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6597 	  SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6598 	},
6599 	{ "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6600 	  SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6601 	  0, 0
6602 	},
6603 	{ "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6604 	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6605 	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6606 	},
6607 	{ "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6608 	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6609 	  SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6610 	},
6611 	{ "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6612 	  SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6613 	  SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6614 	},
6615 	{ "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6616 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6617 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6618 	},
6619 	{ "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6620 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6621 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6622 	},
6623 	{ "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6624 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6625 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6626 	},
6627 	{ "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6628 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6629 	  SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6630 	},
6631 	{ "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6632 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6633 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6634 	},
6635 	{ "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6636 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6637 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6638 	},
6639 	{ "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6640 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6641 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6642 	},
6643 	{ "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6644 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6645 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6646 	},
6647 	{ "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6648 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6649 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6650 	},
6651 	{ "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6652 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6653 	  SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6654 	},
6655 	{ "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6656 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6657 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6658 	},
6659 	{ "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6660 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6661 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6662 	},
6663 	{ "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6664 	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6665 	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6666 	},
6667 	{ "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6668 	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6669 	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6670 	},
6671 	{ "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6672 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6673 	  0, 0
6674 	},
6675 	{ "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6676 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6677 	  0, 0
6678 	},
6679 	{ "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6680 	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6681 	  0, 0
6682 	},
6683 	{ "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6684 	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6685 	  0, 0
6686 	},
6687 	{ "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6688 	  SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6689 	  0, 0
6690 	},
6691 	{ "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6692 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6693 	  SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6694 	},
6695 	{ "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6696 	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6697 	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6698 	},
6699 	{ "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6700 	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6701 	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6702 	},
6703 	{ "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6704 	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6705 	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6706 	},
6707 	{ "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6708 	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6709 	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6710 	},
6711 	{ "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6712 	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6713 	  0, 0
6714 	},
6715 	{ "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6716 	  SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6717 	  0, 0
6718 	},
6719 	{ "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6720 	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6721 	  0, 0
6722 	},
6723 	{ "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6724 	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6725 	  0, 0
6726 	},
6727 	{ "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6728 	  SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6729 	  0, 0
6730 	},
6731 	{ "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6732 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6733 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6734 	},
6735 	{ "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6736 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6737 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6738 	},
6739 	{ "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6740 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6741 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6742 	},
6743 	{ "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6744 	  SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6745 	  SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6746 	},
6747 	{ "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6748 	  SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6749 	  SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6750 	},
6751 	{ "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6752 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6753 	  0, 0
6754 	},
6755 	{ "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6756 	  SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6757 	  0, 0
6758 	},
6759 	{ "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6760 	  SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6761 	  0, 0
6762 	},
6763 	{ "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6764 	  SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6765 	  0, 0
6766 	},
6767 	{ "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6768 	  SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6769 	  0, 0
6770 	},
6771 	{ "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6772 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6773 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6774 	},
6775 	{ "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6776 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6777 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6778 	},
6779 	{ "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6780 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6781 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6782 	},
6783 	{ "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6784 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6785 	  0, 0
6786 	},
6787 	{ "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6788 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6789 	  0, 0
6790 	},
6791 	{ "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6792 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6793 	  0, 0
6794 	},
6795 	{ "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6796 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6797 	  0, 0
6798 	},
6799 	{ "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6800 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6801 	  0, 0
6802 	},
6803 	{ "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6804 	  SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6805 	  0, 0
6806 	}
6807 };
6808 
6809 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6810 				     void *inject_if, uint32_t instance_mask)
6811 {
6812 	struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6813 	int ret;
6814 	struct ta_ras_trigger_error_input block_info = { 0 };
6815 
6816 	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6817 		return -EINVAL;
6818 
6819 	if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6820 		return -EINVAL;
6821 
6822 	if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6823 		return -EPERM;
6824 
6825 	if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6826 	      info->head.type)) {
6827 		DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6828 			ras_gfx_subblocks[info->head.sub_block_index].name,
6829 			info->head.type);
6830 		return -EPERM;
6831 	}
6832 
6833 	if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6834 	      info->head.type)) {
6835 		DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6836 			ras_gfx_subblocks[info->head.sub_block_index].name,
6837 			info->head.type);
6838 		return -EPERM;
6839 	}
6840 
6841 	block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6842 	block_info.sub_block_index =
6843 		ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6844 	block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6845 	block_info.address = info->address;
6846 	block_info.value = info->value;
6847 
6848 	mutex_lock(&adev->grbm_idx_mutex);
6849 	ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);
6850 	mutex_unlock(&adev->grbm_idx_mutex);
6851 
6852 	return ret;
6853 }
6854 
6855 static const char * const vml2_mems[] = {
6856 	"UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6857 	"UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6858 	"UTC_VML2_BANK_CACHE_0_4K_MEM0",
6859 	"UTC_VML2_BANK_CACHE_0_4K_MEM1",
6860 	"UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6861 	"UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6862 	"UTC_VML2_BANK_CACHE_1_4K_MEM0",
6863 	"UTC_VML2_BANK_CACHE_1_4K_MEM1",
6864 	"UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6865 	"UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6866 	"UTC_VML2_BANK_CACHE_2_4K_MEM0",
6867 	"UTC_VML2_BANK_CACHE_2_4K_MEM1",
6868 	"UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6869 	"UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6870 	"UTC_VML2_BANK_CACHE_3_4K_MEM0",
6871 	"UTC_VML2_BANK_CACHE_3_4K_MEM1",
6872 };
6873 
6874 static const char * const vml2_walker_mems[] = {
6875 	"UTC_VML2_CACHE_PDE0_MEM0",
6876 	"UTC_VML2_CACHE_PDE0_MEM1",
6877 	"UTC_VML2_CACHE_PDE1_MEM0",
6878 	"UTC_VML2_CACHE_PDE1_MEM1",
6879 	"UTC_VML2_CACHE_PDE2_MEM0",
6880 	"UTC_VML2_CACHE_PDE2_MEM1",
6881 	"UTC_VML2_RDIF_LOG_FIFO",
6882 };
6883 
6884 static const char * const atc_l2_cache_2m_mems[] = {
6885 	"UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6886 	"UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6887 	"UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6888 	"UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6889 };
6890 
6891 static const char *atc_l2_cache_4k_mems[] = {
6892 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6893 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6894 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6895 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6896 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6897 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6898 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6899 	"UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6900 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6901 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6902 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6903 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6904 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6905 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6906 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6907 	"UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6908 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6909 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6910 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6911 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6912 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6913 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6914 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6915 	"UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6916 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6917 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6918 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6919 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6920 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6921 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6922 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6923 	"UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6924 };
6925 
6926 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6927 					 struct ras_err_data *err_data)
6928 {
6929 	uint32_t i, data;
6930 	uint32_t sec_count, ded_count;
6931 
6932 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6933 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6934 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6935 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6936 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6937 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6938 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6939 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6940 
6941 	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6942 		WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6943 		data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6944 
6945 		sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6946 		if (sec_count) {
6947 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6948 				"SEC %d\n", i, vml2_mems[i], sec_count);
6949 			err_data->ce_count += sec_count;
6950 		}
6951 
6952 		ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6953 		if (ded_count) {
6954 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6955 				"DED %d\n", i, vml2_mems[i], ded_count);
6956 			err_data->ue_count += ded_count;
6957 		}
6958 	}
6959 
6960 	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6961 		WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6962 		data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6963 
6964 		sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6965 						SEC_COUNT);
6966 		if (sec_count) {
6967 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6968 				"SEC %d\n", i, vml2_walker_mems[i], sec_count);
6969 			err_data->ce_count += sec_count;
6970 		}
6971 
6972 		ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6973 						DED_COUNT);
6974 		if (ded_count) {
6975 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6976 				"DED %d\n", i, vml2_walker_mems[i], ded_count);
6977 			err_data->ue_count += ded_count;
6978 		}
6979 	}
6980 
6981 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6982 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6983 		data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6984 
6985 		sec_count = (data & 0x00006000L) >> 0xd;
6986 		if (sec_count) {
6987 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6988 				"SEC %d\n", i, atc_l2_cache_2m_mems[i],
6989 				sec_count);
6990 			err_data->ce_count += sec_count;
6991 		}
6992 	}
6993 
6994 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6995 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6996 		data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6997 
6998 		sec_count = (data & 0x00006000L) >> 0xd;
6999 		if (sec_count) {
7000 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
7001 				"SEC %d\n", i, atc_l2_cache_4k_mems[i],
7002 				sec_count);
7003 			err_data->ce_count += sec_count;
7004 		}
7005 
7006 		ded_count = (data & 0x00018000L) >> 0xf;
7007 		if (ded_count) {
7008 			dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
7009 				"DED %d\n", i, atc_l2_cache_4k_mems[i],
7010 				ded_count);
7011 			err_data->ue_count += ded_count;
7012 		}
7013 	}
7014 
7015 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7016 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7017 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7018 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7019 
7020 	return 0;
7021 }
7022 
7023 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
7024 	const struct soc15_reg_entry *reg,
7025 	uint32_t se_id, uint32_t inst_id, uint32_t value,
7026 	uint32_t *sec_count, uint32_t *ded_count)
7027 {
7028 	uint32_t i;
7029 	uint32_t sec_cnt, ded_cnt;
7030 
7031 	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
7032 		if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
7033 			gfx_v9_0_ras_fields[i].seg != reg->seg ||
7034 			gfx_v9_0_ras_fields[i].inst != reg->inst)
7035 			continue;
7036 
7037 		sec_cnt = (value &
7038 				gfx_v9_0_ras_fields[i].sec_count_mask) >>
7039 				gfx_v9_0_ras_fields[i].sec_count_shift;
7040 		if (sec_cnt) {
7041 			dev_info(adev->dev, "GFX SubBlock %s, "
7042 				"Instance[%d][%d], SEC %d\n",
7043 				gfx_v9_0_ras_fields[i].name,
7044 				se_id, inst_id,
7045 				sec_cnt);
7046 			*sec_count += sec_cnt;
7047 		}
7048 
7049 		ded_cnt = (value &
7050 				gfx_v9_0_ras_fields[i].ded_count_mask) >>
7051 				gfx_v9_0_ras_fields[i].ded_count_shift;
7052 		if (ded_cnt) {
7053 			dev_info(adev->dev, "GFX SubBlock %s, "
7054 				"Instance[%d][%d], DED %d\n",
7055 				gfx_v9_0_ras_fields[i].name,
7056 				se_id, inst_id,
7057 				ded_cnt);
7058 			*ded_count += ded_cnt;
7059 		}
7060 	}
7061 
7062 	return 0;
7063 }
7064 
7065 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
7066 {
7067 	int i, j, k;
7068 
7069 	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7070 		return;
7071 
7072 	/* read back registers to clear the counters */
7073 	mutex_lock(&adev->grbm_idx_mutex);
7074 	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7075 		for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7076 			for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7077 				amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
7078 				RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7079 			}
7080 		}
7081 	}
7082 	WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
7083 	mutex_unlock(&adev->grbm_idx_mutex);
7084 
7085 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7086 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
7087 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7088 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
7089 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7090 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
7091 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7092 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
7093 
7094 	for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
7095 		WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
7096 		RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
7097 	}
7098 
7099 	for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
7100 		WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
7101 		RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
7102 	}
7103 
7104 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
7105 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
7106 		RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
7107 	}
7108 
7109 	for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
7110 		WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
7111 		RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
7112 	}
7113 
7114 	WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7115 	WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7116 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7117 	WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7118 }
7119 
7120 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
7121 					  void *ras_error_status)
7122 {
7123 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
7124 	uint32_t sec_count = 0, ded_count = 0;
7125 	uint32_t i, j, k;
7126 	uint32_t reg_value;
7127 
7128 	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7129 		return;
7130 
7131 	err_data->ue_count = 0;
7132 	err_data->ce_count = 0;
7133 
7134 	mutex_lock(&adev->grbm_idx_mutex);
7135 
7136 	for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7137 		for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7138 			for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7139 				amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
7140 				reg_value =
7141 					RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7142 				if (reg_value)
7143 					gfx_v9_0_ras_error_count(adev,
7144 						&gfx_v9_0_edc_counter_regs[i],
7145 						j, k, reg_value,
7146 						&sec_count, &ded_count);
7147 			}
7148 		}
7149 	}
7150 
7151 	err_data->ce_count += sec_count;
7152 	err_data->ue_count += ded_count;
7153 
7154 	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7155 	mutex_unlock(&adev->grbm_idx_mutex);
7156 
7157 	gfx_v9_0_query_utc_edc_status(adev, err_data);
7158 }
7159 
7160 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
7161 					uint32_t pipe, bool enable)
7162 {
7163 	struct amdgpu_device *adev = ring->adev;
7164 	uint32_t val;
7165 	uint32_t wcl_cs_reg;
7166 
7167 	/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
7168 	val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
7169 
7170 	switch (pipe) {
7171 	case 0:
7172 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
7173 		break;
7174 	case 1:
7175 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
7176 		break;
7177 	case 2:
7178 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
7179 		break;
7180 	case 3:
7181 		wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
7182 		break;
7183 	default:
7184 		DRM_DEBUG("invalid pipe %d\n", pipe);
7185 		return;
7186 	}
7187 
7188 	amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
7189 
7190 }
7191 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
7192 {
7193 	struct amdgpu_device *adev = ring->adev;
7194 	uint32_t val;
7195 	int i;
7196 
7197 
7198 	/* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
7199 	 * number of gfx waves. Setting 5 bit will make sure gfx only gets
7200 	 * around 25% of gpu resources.
7201 	 */
7202 	val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
7203 	amdgpu_ring_emit_wreg(ring,
7204 			      SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
7205 			      val);
7206 
7207 	/* Restrict waves for normal/low priority compute queues as well
7208 	 * to get best QoS for high priority compute jobs.
7209 	 *
7210 	 * amdgpu controls only 1st ME(0-3 CS pipes).
7211 	 */
7212 	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
7213 		if (i != ring->pipe)
7214 			gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
7215 
7216 	}
7217 }
7218 
7219 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
7220 {
7221 	/* Header itself is a NOP packet */
7222 	if (num_nop == 1) {
7223 		amdgpu_ring_write(ring, ring->funcs->nop);
7224 		return;
7225 	}
7226 
7227 	/* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
7228 	amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
7229 
7230 	/* Header is at index 0, followed by num_nops - 1 NOP packet's */
7231 	amdgpu_ring_insert_nop(ring, num_nop - 1);
7232 }
7233 
7234 static void gfx_v9_0_ring_emit_wreg_me(struct amdgpu_ring *ring,
7235 				       uint32_t reg,
7236 				       uint32_t val)
7237 {
7238 	uint32_t cmd = 0;
7239 
7240 	switch (ring->funcs->type) {
7241 	case AMDGPU_RING_TYPE_KIQ:
7242 		cmd = (1 << 16); /* no inc addr */
7243 		break;
7244 	default:
7245 		cmd = WR_CONFIRM;
7246 		break;
7247 	}
7248 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
7249 	amdgpu_ring_write(ring, cmd);
7250 	amdgpu_ring_write(ring, reg);
7251 	amdgpu_ring_write(ring, 0);
7252 	amdgpu_ring_write(ring, val);
7253 }
7254 
7255 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring,
7256 			      unsigned int vmid,
7257 			      struct amdgpu_fence *timedout_fence)
7258 {
7259 	struct amdgpu_device *adev = ring->adev;
7260 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7261 	struct amdgpu_ring *kiq_ring = &kiq->ring;
7262 	unsigned long flags;
7263 	u32 tmp;
7264 	int r;
7265 
7266 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
7267 
7268 	spin_lock_irqsave(&kiq->ring_lock, flags);
7269 
7270 	if (amdgpu_ring_alloc(kiq_ring, 5)) {
7271 		spin_unlock_irqrestore(&kiq->ring_lock, flags);
7272 		return -ENOMEM;
7273 	}
7274 
7275 	/* send the reset - 5 */
7276 	tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
7277 	gfx_v9_0_ring_emit_wreg(kiq_ring,
7278 				SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp);
7279 	amdgpu_ring_commit(kiq_ring);
7280 	r = amdgpu_ring_test_ring(kiq_ring);
7281 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
7282 	if (r)
7283 		return r;
7284 
7285 	if (amdgpu_ring_alloc(ring, 8 + 7 + 5 + 2 + 8 + 7))
7286 		return -ENOMEM;
7287 	/* emit the fence to finish the reset - 8 */
7288 	ring->trail_seq++;
7289 	gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
7290 				 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC);
7291 	/* wait for the fence - 7 */
7292 	gfx_v9_0_wait_reg_mem(ring, 0, 1, 0,
7293 			      lower_32_bits(ring->trail_fence_gpu_addr),
7294 			      upper_32_bits(ring->trail_fence_gpu_addr),
7295 			      ring->trail_seq, 0xffffffff, 4);
7296 	/* clear mmCP_VMID_RESET - 5 */
7297 	gfx_v9_0_ring_emit_wreg_me(ring,
7298 				   SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0);
7299 	/* event write ENABLE_LEGACY_PIPELINE - 2 */
7300 	gfx_v9_0_ring_emit_event_write(ring, ENABLE_LEGACY_PIPELINE, 0);
7301 	/* emit a regular fence - 8 */
7302 	ring->trail_seq++;
7303 	gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
7304 				 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC);
7305 	/* wait for the fence - 7 */
7306 	gfx_v9_0_wait_reg_mem(ring, 1, 1, 0,
7307 			      lower_32_bits(ring->trail_fence_gpu_addr),
7308 			      upper_32_bits(ring->trail_fence_gpu_addr),
7309 			      ring->trail_seq, 0xffffffff, 4);
7310 	amdgpu_ring_commit(ring);
7311 	/* wait for the commands to complete */
7312 	r = amdgpu_ring_test_ring(ring);
7313 	if (r)
7314 		return r;
7315 
7316 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
7317 }
7318 
7319 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,
7320 			      unsigned int vmid,
7321 			      struct amdgpu_fence *timedout_fence)
7322 {
7323 	struct amdgpu_device *adev = ring->adev;
7324 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7325 	struct amdgpu_ring *kiq_ring = &kiq->ring;
7326 	unsigned long flags;
7327 	int i, r;
7328 
7329 	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7330 		return -EINVAL;
7331 
7332 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
7333 
7334 	spin_lock_irqsave(&kiq->ring_lock, flags);
7335 
7336 	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
7337 		spin_unlock_irqrestore(&kiq->ring_lock, flags);
7338 		return -ENOMEM;
7339 	}
7340 
7341 	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
7342 				   0, 0);
7343 	amdgpu_ring_commit(kiq_ring);
7344 
7345 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
7346 
7347 	r = amdgpu_ring_test_ring(kiq_ring);
7348 	if (r)
7349 		return r;
7350 
7351 	/* make sure dequeue is complete*/
7352 	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
7353 	mutex_lock(&adev->srbm_mutex);
7354 	soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
7355 	for (i = 0; i < adev->usec_timeout; i++) {
7356 		if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
7357 			break;
7358 		udelay(1);
7359 	}
7360 	if (i >= adev->usec_timeout)
7361 		r = -ETIMEDOUT;
7362 	soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7363 	mutex_unlock(&adev->srbm_mutex);
7364 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
7365 	if (r) {
7366 		dev_err(adev->dev, "fail to wait on hqd deactive\n");
7367 		return r;
7368 	}
7369 
7370 	r = gfx_v9_0_kcq_init_queue(ring, true);
7371 	if (r) {
7372 		dev_err(adev->dev, "fail to init kcq\n");
7373 		return r;
7374 	}
7375 	spin_lock_irqsave(&kiq->ring_lock, flags);
7376 	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
7377 	if (r) {
7378 		spin_unlock_irqrestore(&kiq->ring_lock, flags);
7379 		return -ENOMEM;
7380 	}
7381 	kiq->pmf->kiq_map_queues(kiq_ring, ring);
7382 	amdgpu_ring_commit(kiq_ring);
7383 	r = amdgpu_ring_test_ring(kiq_ring);
7384 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
7385 	if (r) {
7386 		DRM_ERROR("fail to remap queue\n");
7387 		return r;
7388 	}
7389 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
7390 }
7391 
7392 static void gfx_v9_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
7393 {
7394 	struct amdgpu_device *adev = ip_block->adev;
7395 	uint32_t i, j, k, reg, index = 0;
7396 	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7397 
7398 	if (!adev->gfx.ip_dump_core)
7399 		return;
7400 
7401 	for (i = 0; i < reg_count; i++)
7402 		drm_printf(p, "%-50s \t 0x%08x\n",
7403 			   gc_reg_list_9[i].reg_name,
7404 			   adev->gfx.ip_dump_core[i]);
7405 
7406 	/* print compute queue registers for all instances */
7407 	if (!adev->gfx.ip_dump_compute_queues)
7408 		return;
7409 
7410 	reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7411 	drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
7412 		   adev->gfx.mec.num_mec,
7413 		   adev->gfx.mec.num_pipe_per_mec,
7414 		   adev->gfx.mec.num_queue_per_pipe);
7415 
7416 	for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7417 		for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7418 			for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7419 				drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
7420 				for (reg = 0; reg < reg_count; reg++) {
7421 					if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP)
7422 						drm_printf(p, "%-50s \t 0x%08x\n",
7423 							   "mmCP_MEC_ME2_HEADER_DUMP",
7424 							   adev->gfx.ip_dump_compute_queues[index + reg]);
7425 					else
7426 						drm_printf(p, "%-50s \t 0x%08x\n",
7427 							   gc_cp_reg_list_9[reg].reg_name,
7428 							   adev->gfx.ip_dump_compute_queues[index + reg]);
7429 				}
7430 				index += reg_count;
7431 			}
7432 		}
7433 	}
7434 
7435 }
7436 
7437 static void gfx_v9_ip_dump(struct amdgpu_ip_block *ip_block)
7438 {
7439 	struct amdgpu_device *adev = ip_block->adev;
7440 	uint32_t i, j, k, reg, index = 0;
7441 	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7442 
7443 	if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)
7444 		return;
7445 
7446 	amdgpu_gfx_off_ctrl(adev, false);
7447 	for (i = 0; i < reg_count; i++)
7448 		adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));
7449 	amdgpu_gfx_off_ctrl(adev, true);
7450 
7451 	/* dump compute queue registers for all instances */
7452 	if (!adev->gfx.ip_dump_compute_queues)
7453 		return;
7454 
7455 	reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7456 	amdgpu_gfx_off_ctrl(adev, false);
7457 	mutex_lock(&adev->srbm_mutex);
7458 	for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7459 		for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7460 			for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7461 				/* ME0 is for GFX so start from 1 for CP */
7462 				soc15_grbm_select(adev, 1 + i, j, k, 0, 0);
7463 
7464 				for (reg = 0; reg < reg_count; reg++) {
7465 					if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP)
7466 						adev->gfx.ip_dump_compute_queues[index + reg] =
7467 							RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME2_HEADER_DUMP));
7468 					else
7469 						adev->gfx.ip_dump_compute_queues[index + reg] =
7470 							RREG32(SOC15_REG_ENTRY_OFFSET(
7471 								       gc_cp_reg_list_9[reg]));
7472 				}
7473 				index += reg_count;
7474 			}
7475 		}
7476 	}
7477 	soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7478 	mutex_unlock(&adev->srbm_mutex);
7479 	amdgpu_gfx_off_ctrl(adev, true);
7480 
7481 }
7482 
7483 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7484 {
7485 	struct amdgpu_device *adev = ring->adev;
7486 
7487 	/* Emit the cleaner shader */
7488 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
7489 		amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7490 	else
7491 		amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER_9_0, 0));
7492 
7493 	amdgpu_ring_write(ring, 0);  /* RESERVED field, programmed to zero */
7494 }
7495 
7496 static void gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring *ring)
7497 {
7498 	struct amdgpu_device *adev = ring->adev;
7499 	struct amdgpu_ip_block *gfx_block =
7500 		amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7501 
7502 	amdgpu_gfx_enforce_isolation_ring_begin_use(ring);
7503 
7504 	/* Raven and PCO APUs seem to have stability issues
7505 	 * with compute and gfxoff and gfx pg.  Disable gfx pg during
7506 	 * submission and allow again afterwards.
7507 	 */
7508 	if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7509 		gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_UNGATE);
7510 }
7511 
7512 static void gfx_v9_0_ring_end_use_compute(struct amdgpu_ring *ring)
7513 {
7514 	struct amdgpu_device *adev = ring->adev;
7515 	struct amdgpu_ip_block *gfx_block =
7516 		amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7517 
7518 	/* Raven and PCO APUs seem to have stability issues
7519 	 * with compute and gfxoff and gfx pg.  Disable gfx pg during
7520 	 * submission and allow again afterwards.
7521 	 */
7522 	if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7523 		gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_GATE);
7524 
7525 	amdgpu_gfx_enforce_isolation_ring_end_use(ring);
7526 }
7527 
7528 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
7529 	.name = "gfx_v9_0",
7530 	.early_init = gfx_v9_0_early_init,
7531 	.late_init = gfx_v9_0_late_init,
7532 	.sw_init = gfx_v9_0_sw_init,
7533 	.sw_fini = gfx_v9_0_sw_fini,
7534 	.hw_init = gfx_v9_0_hw_init,
7535 	.hw_fini = gfx_v9_0_hw_fini,
7536 	.suspend = gfx_v9_0_suspend,
7537 	.resume = gfx_v9_0_resume,
7538 	.wait_for_idle = gfx_v9_0_wait_for_idle,
7539 	.soft_reset = gfx_v9_0_soft_reset,
7540 	.set_clockgating_state = gfx_v9_0_set_clockgating_state,
7541 	.set_powergating_state = gfx_v9_0_set_powergating_state,
7542 	.get_clockgating_state = gfx_v9_0_get_clockgating_state,
7543 	.dump_ip_state = gfx_v9_ip_dump,
7544 	.print_ip_state = gfx_v9_ip_print,
7545 };
7546 
7547 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
7548 	.type = AMDGPU_RING_TYPE_GFX,
7549 	.align_mask = 0xff,
7550 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
7551 	.support_64bit_ptrs = true,
7552 	.secure_submission_supported = true,
7553 	.get_rptr = gfx_v9_0_ring_get_rptr_gfx,
7554 	.get_wptr = gfx_v9_0_ring_get_wptr_gfx,
7555 	.set_wptr = gfx_v9_0_ring_set_wptr_gfx,
7556 	.emit_frame_size = /* totally 242 maximum if 16 IBs */
7557 		5 +  /* COND_EXEC */
7558 		13 +  /* PIPELINE_SYNC */
7559 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7560 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7561 		2 + /* VM_FLUSH */
7562 		8 +  /* FENCE for VM_FLUSH */
7563 		20 + /* GDS switch */
7564 		4 + /* double SWITCH_BUFFER,
7565 		       the first COND_EXEC jump to the place just
7566 			   prior to this double SWITCH_BUFFER  */
7567 		5 + /* COND_EXEC */
7568 		7 +	 /*	HDP_flush */
7569 		4 +	 /*	VGT_flush */
7570 		14 + /*	CE_META */
7571 		31 + /*	DE_META */
7572 		3 + /* CNTX_CTRL */
7573 		5 + /* HDP_INVL */
7574 		8 + 8 + /* FENCE x2 */
7575 		2 + /* SWITCH_BUFFER */
7576 		7 + /* gfx_v9_0_emit_mem_sync */
7577 		2, /* gfx_v9_0_ring_emit_cleaner_shader */
7578 	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_gfx */
7579 	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7580 	.emit_fence = gfx_v9_0_ring_emit_fence,
7581 	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7582 	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7583 	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7584 	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7585 	.test_ring = gfx_v9_0_ring_test_ring,
7586 	.insert_nop = gfx_v9_ring_insert_nop,
7587 	.pad_ib = amdgpu_ring_generic_pad_ib,
7588 	.emit_switch_buffer = gfx_v9_ring_emit_sb,
7589 	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7590 	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7591 	.preempt_ib = gfx_v9_0_ring_preempt_ib,
7592 	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7593 	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7594 	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7595 	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7596 	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
7597 	.emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7598 	.reset = gfx_v9_0_reset_kgq,
7599 	.begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7600 	.end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7601 };
7602 
7603 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
7604 	.type = AMDGPU_RING_TYPE_GFX,
7605 	.align_mask = 0xff,
7606 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
7607 	.support_64bit_ptrs = true,
7608 	.secure_submission_supported = true,
7609 	.get_rptr = amdgpu_sw_ring_get_rptr_gfx,
7610 	.get_wptr = amdgpu_sw_ring_get_wptr_gfx,
7611 	.set_wptr = amdgpu_sw_ring_set_wptr_gfx,
7612 	.emit_frame_size = /* totally 242 maximum if 16 IBs */
7613 		5 +  /* COND_EXEC */
7614 		13 +  /* PIPELINE_SYNC */
7615 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7616 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7617 		2 + /* VM_FLUSH */
7618 		8 +  /* FENCE for VM_FLUSH */
7619 		20 + /* GDS switch */
7620 		4 + /* double SWITCH_BUFFER,
7621 		     * the first COND_EXEC jump to the place just
7622 		     * prior to this double SWITCH_BUFFER
7623 		     */
7624 		5 + /* COND_EXEC */
7625 		7 +	 /*	HDP_flush */
7626 		4 +	 /*	VGT_flush */
7627 		14 + /*	CE_META */
7628 		31 + /*	DE_META */
7629 		3 + /* CNTX_CTRL */
7630 		5 + /* HDP_INVL */
7631 		8 + 8 + /* FENCE x2 */
7632 		2 + /* SWITCH_BUFFER */
7633 		7 + /* gfx_v9_0_emit_mem_sync */
7634 		2, /* gfx_v9_0_ring_emit_cleaner_shader */
7635 	.emit_ib_size =	4, /* gfx_v9_0_ring_emit_ib_gfx */
7636 	.emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7637 	.emit_fence = gfx_v9_0_ring_emit_fence,
7638 	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7639 	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7640 	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7641 	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7642 	.test_ring = gfx_v9_0_ring_test_ring,
7643 	.test_ib = gfx_v9_0_ring_test_ib,
7644 	.insert_nop = gfx_v9_ring_insert_nop,
7645 	.pad_ib = amdgpu_ring_generic_pad_ib,
7646 	.emit_switch_buffer = gfx_v9_ring_emit_sb,
7647 	.emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7648 	.init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7649 	.emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7650 	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7651 	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7652 	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7653 	.soft_recovery = gfx_v9_0_ring_soft_recovery,
7654 	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
7655 	.patch_cntl = gfx_v9_0_ring_patch_cntl,
7656 	.patch_de = gfx_v9_0_ring_patch_de_meta,
7657 	.patch_ce = gfx_v9_0_ring_patch_ce_meta,
7658 	.emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7659 	.begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7660 	.end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7661 };
7662 
7663 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
7664 	.type = AMDGPU_RING_TYPE_COMPUTE,
7665 	.align_mask = 0xff,
7666 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
7667 	.support_64bit_ptrs = true,
7668 	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
7669 	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
7670 	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
7671 	.emit_frame_size =
7672 		20 + /* gfx_v9_0_ring_emit_gds_switch */
7673 		7 + /* gfx_v9_0_ring_emit_hdp_flush */
7674 		5 + /* hdp invalidate */
7675 		9 + /* gfx_v9_0_ring_emit_pipeline_sync */
7676 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7677 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7678 		8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
7679 		7 + /* gfx_v9_0_emit_mem_sync */
7680 		5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
7681 		15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
7682 		2, /* gfx_v9_0_ring_emit_cleaner_shader */
7683 	.emit_ib_size =	7, /* gfx_v9_0_ring_emit_ib_compute */
7684 	.emit_ib = gfx_v9_0_ring_emit_ib_compute,
7685 	.emit_fence = gfx_v9_0_ring_emit_fence,
7686 	.emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7687 	.emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7688 	.emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7689 	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7690 	.test_ring = gfx_v9_0_ring_test_ring,
7691 	.test_ib = gfx_v9_0_ring_test_ib,
7692 	.insert_nop = gfx_v9_ring_insert_nop,
7693 	.pad_ib = amdgpu_ring_generic_pad_ib,
7694 	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7695 	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7696 	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7697 	.emit_mem_sync = gfx_v9_0_emit_mem_sync,
7698 	.emit_wave_limit = gfx_v9_0_emit_wave_limit,
7699 	.reset = gfx_v9_0_reset_kcq,
7700 	.emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7701 	.begin_use = gfx_v9_0_ring_begin_use_compute,
7702 	.end_use = gfx_v9_0_ring_end_use_compute,
7703 };
7704 
7705 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7706 	.type = AMDGPU_RING_TYPE_KIQ,
7707 	.align_mask = 0xff,
7708 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
7709 	.support_64bit_ptrs = true,
7710 	.get_rptr = gfx_v9_0_ring_get_rptr_compute,
7711 	.get_wptr = gfx_v9_0_ring_get_wptr_compute,
7712 	.set_wptr = gfx_v9_0_ring_set_wptr_compute,
7713 	.emit_frame_size =
7714 		20 + /* gfx_v9_0_ring_emit_gds_switch */
7715 		7 + /* gfx_v9_0_ring_emit_hdp_flush */
7716 		5 + /* hdp invalidate */
7717 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7718 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7719 		8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7720 	.emit_ib_size =	7, /* gfx_v9_0_ring_emit_ib_compute */
7721 	.emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7722 	.test_ring = gfx_v9_0_ring_test_ring,
7723 	.insert_nop = amdgpu_ring_insert_nop,
7724 	.pad_ib = amdgpu_ring_generic_pad_ib,
7725 	.emit_rreg = gfx_v9_0_ring_emit_rreg,
7726 	.emit_wreg = gfx_v9_0_ring_emit_wreg,
7727 	.emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7728 	.emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7729 	.emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7730 };
7731 
7732 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7733 {
7734 	int i;
7735 
7736 	adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7737 
7738 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7739 		adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7740 
7741 	if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
7742 		for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7743 			adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7744 	}
7745 
7746 	for (i = 0; i < adev->gfx.num_compute_rings; i++)
7747 		adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7748 }
7749 
7750 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7751 	.set = gfx_v9_0_set_eop_interrupt_state,
7752 	.process = gfx_v9_0_eop_irq,
7753 };
7754 
7755 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7756 	.set = gfx_v9_0_set_priv_reg_fault_state,
7757 	.process = gfx_v9_0_priv_reg_irq,
7758 };
7759 
7760 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = {
7761 	.set = gfx_v9_0_set_bad_op_fault_state,
7762 	.process = gfx_v9_0_bad_op_irq,
7763 };
7764 
7765 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7766 	.set = gfx_v9_0_set_priv_inst_fault_state,
7767 	.process = gfx_v9_0_priv_inst_irq,
7768 };
7769 
7770 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7771 	.set = gfx_v9_0_set_cp_ecc_error_state,
7772 	.process = amdgpu_gfx_cp_ecc_error_irq,
7773 };
7774 
7775 
7776 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7777 {
7778 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7779 	adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7780 
7781 	adev->gfx.priv_reg_irq.num_types = 1;
7782 	adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7783 
7784 	adev->gfx.bad_op_irq.num_types = 1;
7785 	adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;
7786 
7787 	adev->gfx.priv_inst_irq.num_types = 1;
7788 	adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7789 
7790 	adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7791 	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7792 }
7793 
7794 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7795 {
7796 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7797 	case IP_VERSION(9, 0, 1):
7798 	case IP_VERSION(9, 2, 1):
7799 	case IP_VERSION(9, 4, 0):
7800 	case IP_VERSION(9, 2, 2):
7801 	case IP_VERSION(9, 1, 0):
7802 	case IP_VERSION(9, 4, 1):
7803 	case IP_VERSION(9, 3, 0):
7804 	case IP_VERSION(9, 4, 2):
7805 		adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7806 		break;
7807 	default:
7808 		break;
7809 	}
7810 }
7811 
7812 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7813 {
7814 	/* init asci gds info */
7815 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7816 	case IP_VERSION(9, 0, 1):
7817 	case IP_VERSION(9, 2, 1):
7818 	case IP_VERSION(9, 4, 0):
7819 		adev->gds.gds_size = 0x10000;
7820 		break;
7821 	case IP_VERSION(9, 2, 2):
7822 	case IP_VERSION(9, 1, 0):
7823 	case IP_VERSION(9, 4, 1):
7824 		adev->gds.gds_size = 0x1000;
7825 		break;
7826 	case IP_VERSION(9, 4, 2):
7827 		/* aldebaran removed all the GDS internal memory,
7828 		 * only support GWS opcode in kernel, like barrier
7829 		 * semaphore.etc */
7830 		adev->gds.gds_size = 0;
7831 		break;
7832 	default:
7833 		adev->gds.gds_size = 0x10000;
7834 		break;
7835 	}
7836 
7837 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7838 	case IP_VERSION(9, 0, 1):
7839 	case IP_VERSION(9, 4, 0):
7840 		adev->gds.gds_compute_max_wave_id = 0x7ff;
7841 		break;
7842 	case IP_VERSION(9, 2, 1):
7843 		adev->gds.gds_compute_max_wave_id = 0x27f;
7844 		break;
7845 	case IP_VERSION(9, 2, 2):
7846 	case IP_VERSION(9, 1, 0):
7847 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7848 			adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7849 		else
7850 			adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7851 		break;
7852 	case IP_VERSION(9, 4, 1):
7853 		adev->gds.gds_compute_max_wave_id = 0xfff;
7854 		break;
7855 	case IP_VERSION(9, 4, 2):
7856 		/* deprecated for Aldebaran, no usage at all */
7857 		adev->gds.gds_compute_max_wave_id = 0;
7858 		break;
7859 	default:
7860 		/* this really depends on the chip */
7861 		adev->gds.gds_compute_max_wave_id = 0x7ff;
7862 		break;
7863 	}
7864 
7865 	adev->gds.gws_size = 64;
7866 	adev->gds.oa_size = 16;
7867 }
7868 
7869 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7870 						 u32 bitmap)
7871 {
7872 	u32 data;
7873 
7874 	if (!bitmap)
7875 		return;
7876 
7877 	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7878 	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7879 
7880 	WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7881 }
7882 
7883 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7884 {
7885 	u32 data, mask;
7886 
7887 	data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7888 	data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7889 
7890 	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7891 	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7892 
7893 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7894 
7895 	return (~data) & mask;
7896 }
7897 
7898 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7899 				 struct amdgpu_cu_info *cu_info)
7900 {
7901 	int i, j, k, counter, active_cu_number = 0;
7902 	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7903 	unsigned disable_masks[4 * 4];
7904 
7905 	if (!adev || !cu_info)
7906 		return -EINVAL;
7907 
7908 	/*
7909 	 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7910 	 */
7911 	if (adev->gfx.config.max_shader_engines *
7912 		adev->gfx.config.max_sh_per_se > 16)
7913 		return -EINVAL;
7914 
7915 	amdgpu_gfx_parse_disable_cu(adev, disable_masks,
7916 				    adev->gfx.config.max_shader_engines,
7917 				    adev->gfx.config.max_sh_per_se);
7918 
7919 	mutex_lock(&adev->grbm_idx_mutex);
7920 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7921 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7922 			mask = 1;
7923 			ao_bitmap = 0;
7924 			counter = 0;
7925 			amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7926 			gfx_v9_0_set_user_cu_inactive_bitmap(
7927 				adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7928 			bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7929 
7930 			/*
7931 			 * The bitmap(and ao_cu_bitmap) in cu_info structure is
7932 			 * 4x4 size array, and it's usually suitable for Vega
7933 			 * ASICs which has 4*2 SE/SH layout.
7934 			 * But for Arcturus, SE/SH layout is changed to 8*1.
7935 			 * To mostly reduce the impact, we make it compatible
7936 			 * with current bitmap array as below:
7937 			 *    SE4,SH0 --> bitmap[0][1]
7938 			 *    SE5,SH0 --> bitmap[1][1]
7939 			 *    SE6,SH0 --> bitmap[2][1]
7940 			 *    SE7,SH0 --> bitmap[3][1]
7941 			 */
7942 			cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7943 
7944 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7945 				if (bitmap & mask) {
7946 					if (counter < adev->gfx.config.max_cu_per_sh)
7947 						ao_bitmap |= mask;
7948 					counter ++;
7949 				}
7950 				mask <<= 1;
7951 			}
7952 			active_cu_number += counter;
7953 			if (i < 2 && j < 2)
7954 				ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7955 			cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7956 		}
7957 	}
7958 	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7959 	mutex_unlock(&adev->grbm_idx_mutex);
7960 
7961 	cu_info->number = active_cu_number;
7962 	cu_info->ao_cu_mask = ao_cu_mask;
7963 	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7964 
7965 	return 0;
7966 }
7967 
7968 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7969 {
7970 	.type = AMD_IP_BLOCK_TYPE_GFX,
7971 	.major = 9,
7972 	.minor = 0,
7973 	.rev = 0,
7974 	.funcs = &gfx_v9_0_ip_funcs,
7975 };
7976