1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29
30 #include "amdgpu.h"
31 #include "amdgpu_gfx.h"
32 #include "soc15.h"
33 #include "soc15d.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_pm.h"
36
37 #include "gc/gc_9_0_offset.h"
38 #include "gc/gc_9_0_sh_mask.h"
39
40 #include "vega10_enum.h"
41
42 #include "soc15_common.h"
43 #include "clearstate_gfx9.h"
44 #include "v9_structs.h"
45
46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47
48 #include "amdgpu_ras.h"
49
50 #include "amdgpu_ring_mux.h"
51 #include "gfx_v9_4.h"
52 #include "gfx_v9_0.h"
53 #include "gfx_v9_0_cleaner_shader.h"
54 #include "gfx_v9_4_2.h"
55
56 #include "asic_reg/pwr/pwr_10_0_offset.h"
57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
58 #include "asic_reg/gc/gc_9_0_default.h"
59
60 #define GFX9_NUM_GFX_RINGS 1
61 #define GFX9_NUM_SW_GFX_RINGS 2
62 #define GFX9_MEC_HPD_SIZE 4096
63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
65
66 #define mmGCEA_PROBE_MAP 0x070c
67 #define mmGCEA_PROBE_MAP_BASE_IDX 0
68
69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
71 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
75
76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
78 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
82
83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
85 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
89
90 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
92 MODULE_FIRMWARE("amdgpu/raven_me.bin");
93 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
96
97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
104
105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
107 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
112
113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
115
116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
118 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
121
122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
128
129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
134
135 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03
136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
137 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04
138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
139 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09
140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
141 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
143 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
145 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
147
148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025
149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1
150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026
151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1
152
153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = {
154 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),
155 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),
156 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),
157 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),
158 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),
159 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),
160 SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),
161 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),
162 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),
163 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),
164 SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),
165 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),
166 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),
167 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),
168 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),
169 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),
170 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),
171 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),
172 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),
173 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),
174 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),
175 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
176 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
177 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),
178 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),
179 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),
180 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),
181 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),
182 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),
183 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),
184 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),
185 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),
186 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),
187 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),
188 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),
189 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),
190 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),
191 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),
192 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),
193 SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),
194 SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),
195 SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),
196 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),
197 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),
198 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),
199 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL),
200 SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),
201 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),
202 SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),
203 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS),
204 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS),
205 SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS),
206 SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS),
207 SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),
208 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL),
209 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS),
210 SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),
211 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),
212 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),
213 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),
214 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),
215 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),
216 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),
217 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),
218 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),
219 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),
220 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),
221 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),
222 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),
223 SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),
224 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),
225 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),
226 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),
227 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),
228 /* SE status registers */
229 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0),
230 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1),
231 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2),
232 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3),
233 /* packet headers */
234 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
235 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
236 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
237 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
238 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
239 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
240 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
241 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
242 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
243 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
244 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
245 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
246 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
247 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
248 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
249 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
250 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
251 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
252 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
253 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
254 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
255 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
256 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
257 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP)
258 };
259
260 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = {
261 /* compute queue registers */
262 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID),
263 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE),
264 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE),
265 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY),
266 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY),
267 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM),
268 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE),
269 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI),
270 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR),
271 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
272 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
273 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
274 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL),
275 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR),
276 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI),
277 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR),
278 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL),
279 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),
280 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
281 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),
282 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL),
283 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR),
284 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR),
285 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS),
286 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO),
287 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI),
288 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL),
289 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET),
290 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE),
291 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET),
292 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE),
293 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE),
294 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR),
295 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM),
296 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO),
297 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI),
298 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS),
299 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
300 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
301 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
302 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
303 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
304 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
305 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
306 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP)
307 };
308
309 enum ta_ras_gfx_subblock {
310 /*CPC*/
311 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
312 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
313 TA_RAS_BLOCK__GFX_CPC_UCODE,
314 TA_RAS_BLOCK__GFX_DC_STATE_ME1,
315 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
316 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
317 TA_RAS_BLOCK__GFX_DC_STATE_ME2,
318 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
319 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
320 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
321 /* CPF*/
322 TA_RAS_BLOCK__GFX_CPF_INDEX_START,
323 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
324 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
325 TA_RAS_BLOCK__GFX_CPF_TAG,
326 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
327 /* CPG*/
328 TA_RAS_BLOCK__GFX_CPG_INDEX_START,
329 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
330 TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
331 TA_RAS_BLOCK__GFX_CPG_TAG,
332 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
333 /* GDS*/
334 TA_RAS_BLOCK__GFX_GDS_INDEX_START,
335 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
336 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
337 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
338 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
339 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
340 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
341 /* SPI*/
342 TA_RAS_BLOCK__GFX_SPI_SR_MEM,
343 /* SQ*/
344 TA_RAS_BLOCK__GFX_SQ_INDEX_START,
345 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
346 TA_RAS_BLOCK__GFX_SQ_LDS_D,
347 TA_RAS_BLOCK__GFX_SQ_LDS_I,
348 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
349 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
350 /* SQC (3 ranges)*/
351 TA_RAS_BLOCK__GFX_SQC_INDEX_START,
352 /* SQC range 0*/
353 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
354 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
355 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
356 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
357 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
358 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
359 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
360 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
361 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
362 TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
363 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
364 /* SQC range 1*/
365 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
366 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
367 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
368 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
369 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
370 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
371 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
372 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
373 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
374 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
375 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
376 TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
377 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
378 /* SQC range 2*/
379 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
380 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
381 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
382 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
383 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
384 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
385 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
386 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
387 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
388 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
389 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
390 TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
391 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
392 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
393 /* TA*/
394 TA_RAS_BLOCK__GFX_TA_INDEX_START,
395 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
396 TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
397 TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
398 TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
399 TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
400 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
401 /* TCA*/
402 TA_RAS_BLOCK__GFX_TCA_INDEX_START,
403 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
404 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
405 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
406 /* TCC (5 sub-ranges)*/
407 TA_RAS_BLOCK__GFX_TCC_INDEX_START,
408 /* TCC range 0*/
409 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
410 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
411 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
412 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
413 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
414 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
415 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
416 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
417 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
418 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
419 /* TCC range 1*/
420 TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
421 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
422 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
423 TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
424 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
425 /* TCC range 2*/
426 TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
427 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
428 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
429 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
430 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
431 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
432 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
433 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
434 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
435 TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
436 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
437 /* TCC range 3*/
438 TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
439 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
440 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
441 TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
442 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
443 /* TCC range 4*/
444 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
445 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
446 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
447 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
448 TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
449 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
450 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
451 /* TCI*/
452 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
453 /* TCP*/
454 TA_RAS_BLOCK__GFX_TCP_INDEX_START,
455 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
456 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
457 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
458 TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
459 TA_RAS_BLOCK__GFX_TCP_DB_RAM,
460 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
461 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
462 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
463 /* TD*/
464 TA_RAS_BLOCK__GFX_TD_INDEX_START,
465 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
466 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
467 TA_RAS_BLOCK__GFX_TD_CS_FIFO,
468 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
469 /* EA (3 sub-ranges)*/
470 TA_RAS_BLOCK__GFX_EA_INDEX_START,
471 /* EA range 0*/
472 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
473 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
474 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
475 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
476 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
477 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
478 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
479 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
480 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
481 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
482 /* EA range 1*/
483 TA_RAS_BLOCK__GFX_EA_INDEX1_START,
484 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
485 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
486 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
487 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
488 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
489 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
490 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
491 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
492 /* EA range 2*/
493 TA_RAS_BLOCK__GFX_EA_INDEX2_START,
494 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
495 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
496 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
497 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
498 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
499 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
500 /* UTC VM L2 bank*/
501 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
502 /* UTC VM walker*/
503 TA_RAS_BLOCK__UTC_VML2_WALKER,
504 /* UTC ATC L2 2MB cache*/
505 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
506 /* UTC ATC L2 4KB cache*/
507 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
508 TA_RAS_BLOCK__GFX_MAX
509 };
510
511 struct ras_gfx_subblock {
512 unsigned char *name;
513 int ta_subblock;
514 int hw_supported_error_type;
515 int sw_supported_error_type;
516 };
517
518 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
519 [AMDGPU_RAS_BLOCK__##subblock] = { \
520 #subblock, \
521 TA_RAS_BLOCK__##subblock, \
522 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
523 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
524 }
525
526 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
527 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
528 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
529 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
530 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
531 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
532 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
533 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
534 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
535 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
536 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
537 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
538 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
539 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
540 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
541 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
542 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
543 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
544 0),
545 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
546 0),
547 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
548 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
549 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
550 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
551 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
552 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
553 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
554 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
555 0, 0),
556 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
557 0),
558 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
559 0, 0),
560 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
561 0),
562 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
563 0, 0),
564 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
565 0),
566 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
567 1),
568 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
569 0, 0, 0),
570 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
571 0),
572 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
573 0),
574 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
575 0),
576 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
577 0),
578 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
579 0),
580 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
581 0, 0),
582 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
583 0),
584 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
585 0),
586 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
587 0, 0, 0),
588 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
589 0),
590 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
591 0),
592 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
593 0),
594 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
595 0),
596 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
597 0),
598 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
599 0, 0),
600 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
601 0),
602 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
603 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
604 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
605 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
606 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
607 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
608 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
609 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
610 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
611 1),
612 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
613 1),
614 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
615 1),
616 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
617 0),
618 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
619 0),
620 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
621 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
622 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
623 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
624 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
625 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
626 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
627 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
628 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
629 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
630 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
631 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
632 0),
633 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
634 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
635 0),
636 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
637 0, 0),
638 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
639 0),
640 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
641 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
642 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
643 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
644 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
645 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
646 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
647 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
648 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
649 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
650 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
651 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
652 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
653 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
654 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
655 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
656 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
657 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
658 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
659 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
660 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
661 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
662 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
663 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
664 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
665 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
666 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
667 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
668 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
669 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
670 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
671 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
672 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
673 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
674 };
675
676 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
677 {
678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
698 };
699
700 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
701 {
702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
707 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
708 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
709 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
710 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
711 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
712 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
713 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
714 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
715 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
716 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
717 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
718 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
719 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
720 };
721
722 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
723 {
724 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
725 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
726 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
727 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
728 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
729 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
730 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
731 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
732 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
733 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
734 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
735 };
736
737 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
738 {
739 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
740 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
741 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
742 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
743 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
744 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
745 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
746 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
747 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
748 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
749 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
750 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
751 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
752 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
753 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
754 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
755 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
756 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
757 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
758 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
759 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
760 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
761 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
762 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
763 };
764
765 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
766 {
767 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
768 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
769 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
770 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
771 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
772 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
773 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
774 };
775
776 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
777 {
778 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
779 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
780 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
781 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
782 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
783 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
784 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
785 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
786 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
787 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
788 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
789 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
790 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
791 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
792 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
793 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
794 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
795 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
796 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
797 };
798
799 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
800 {
801 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
802 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
803 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
804 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
805 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
806 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
807 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
808 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
809 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
810 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
811 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
812 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
813 };
814
815 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
816 {
817 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
818 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
819 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
820 };
821
822 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
823 {
824 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
825 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
826 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
827 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
828 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
829 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
830 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
831 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
832 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
833 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
834 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
835 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
836 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
837 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
838 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
839 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
840 };
841
842 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
843 {
844 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
845 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
846 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
847 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
848 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
849 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
850 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
851 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
852 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
853 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
854 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
855 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
856 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
857 };
858
859 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
860 {
861 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
862 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
863 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
864 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
865 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
866 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
867 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
868 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
869 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
870 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
871 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
872 };
873
874 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
875 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
876 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
877 };
878
879 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
880 {
881 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
882 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
883 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
884 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
885 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
886 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
887 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
888 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
889 };
890
891 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
892 {
893 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
894 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
895 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
896 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
897 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
898 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
899 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
900 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
901 };
902
903 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
904 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
905 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
906 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
907
908 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
909 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
910 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
911 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
912 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
913 struct amdgpu_cu_info *cu_info);
914 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
915 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
916 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
917 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
918 void *ras_error_status);
919 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
920 void *inject_if, uint32_t instance_mask);
921 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
922 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
923 unsigned int vmid);
924 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
925 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
926
gfx_v9_0_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)927 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
928 uint64_t queue_mask)
929 {
930 struct amdgpu_device *adev = kiq_ring->adev;
931 u64 shader_mc_addr;
932
933 /* Cleaner shader MC address */
934 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
935
936 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
937 amdgpu_ring_write(kiq_ring,
938 PACKET3_SET_RESOURCES_VMID_MASK(0) |
939 /* vmid_mask:0* queue_type:0 (KIQ) */
940 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
941 amdgpu_ring_write(kiq_ring,
942 lower_32_bits(queue_mask)); /* queue mask lo */
943 amdgpu_ring_write(kiq_ring,
944 upper_32_bits(queue_mask)); /* queue mask hi */
945 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
946 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
947 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
948 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
949 }
950
gfx_v9_0_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)951 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
952 struct amdgpu_ring *ring)
953 {
954 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
955 uint64_t wptr_addr = ring->wptr_gpu_addr;
956 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
957
958 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
959 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
960 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
961 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
962 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
963 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
964 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
965 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
966 /*queue_type: normal compute queue */
967 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
968 /* alloc format: all_on_one_pipe */
969 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
970 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
971 /* num_queues: must be 1 */
972 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
973 amdgpu_ring_write(kiq_ring,
974 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
975 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
976 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
977 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
978 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
979 }
980
gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)981 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
982 struct amdgpu_ring *ring,
983 enum amdgpu_unmap_queues_action action,
984 u64 gpu_addr, u64 seq)
985 {
986 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
987
988 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
989 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
990 PACKET3_UNMAP_QUEUES_ACTION(action) |
991 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
992 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
993 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
994 amdgpu_ring_write(kiq_ring,
995 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
996
997 if (action == PREEMPT_QUEUES_NO_UNMAP) {
998 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
999 amdgpu_ring_write(kiq_ring, 0);
1000 amdgpu_ring_write(kiq_ring, 0);
1001
1002 } else {
1003 amdgpu_ring_write(kiq_ring, 0);
1004 amdgpu_ring_write(kiq_ring, 0);
1005 amdgpu_ring_write(kiq_ring, 0);
1006 }
1007 }
1008
gfx_v9_0_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)1009 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
1010 struct amdgpu_ring *ring,
1011 u64 addr,
1012 u64 seq)
1013 {
1014 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
1015
1016 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
1017 amdgpu_ring_write(kiq_ring,
1018 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
1019 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
1020 PACKET3_QUERY_STATUS_COMMAND(2));
1021 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
1022 amdgpu_ring_write(kiq_ring,
1023 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
1024 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
1025 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
1026 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
1027 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
1028 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
1029 }
1030
gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)1031 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
1032 uint16_t pasid, uint32_t flush_type,
1033 bool all_hub)
1034 {
1035 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
1036 amdgpu_ring_write(kiq_ring,
1037 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
1038 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
1039 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
1040 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
1041 }
1042
1043
gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring * kiq_ring,uint32_t queue_type,uint32_t me_id,uint32_t pipe_id,uint32_t queue_id,uint32_t xcc_id,uint32_t vmid)1044 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
1045 uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
1046 uint32_t xcc_id, uint32_t vmid)
1047 {
1048 struct amdgpu_device *adev = kiq_ring->adev;
1049 unsigned i;
1050
1051 /* enter save mode */
1052 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1053 mutex_lock(&adev->srbm_mutex);
1054 soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0);
1055
1056 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1057 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2);
1058 WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1);
1059 /* wait till dequeue take effects */
1060 for (i = 0; i < adev->usec_timeout; i++) {
1061 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
1062 break;
1063 udelay(1);
1064 }
1065 if (i >= adev->usec_timeout)
1066 dev_err(adev->dev, "fail to wait on hqd deactive\n");
1067 } else {
1068 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);
1069 }
1070
1071 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
1072 mutex_unlock(&adev->srbm_mutex);
1073 /* exit safe mode */
1074 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1075 }
1076
1077 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
1078 .kiq_set_resources = gfx_v9_0_kiq_set_resources,
1079 .kiq_map_queues = gfx_v9_0_kiq_map_queues,
1080 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
1081 .kiq_query_status = gfx_v9_0_kiq_query_status,
1082 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
1083 .kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue,
1084 .set_resources_size = 8,
1085 .map_queues_size = 7,
1086 .unmap_queues_size = 6,
1087 .query_status_size = 7,
1088 .invalidate_tlbs_size = 2,
1089 };
1090
gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)1091 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
1092 {
1093 adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
1094 }
1095
gfx_v9_0_init_golden_registers(struct amdgpu_device * adev)1096 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
1097 {
1098 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1099 case IP_VERSION(9, 0, 1):
1100 soc15_program_register_sequence(adev,
1101 golden_settings_gc_9_0,
1102 ARRAY_SIZE(golden_settings_gc_9_0));
1103 soc15_program_register_sequence(adev,
1104 golden_settings_gc_9_0_vg10,
1105 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
1106 break;
1107 case IP_VERSION(9, 2, 1):
1108 soc15_program_register_sequence(adev,
1109 golden_settings_gc_9_2_1,
1110 ARRAY_SIZE(golden_settings_gc_9_2_1));
1111 soc15_program_register_sequence(adev,
1112 golden_settings_gc_9_2_1_vg12,
1113 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
1114 break;
1115 case IP_VERSION(9, 4, 0):
1116 soc15_program_register_sequence(adev,
1117 golden_settings_gc_9_0,
1118 ARRAY_SIZE(golden_settings_gc_9_0));
1119 soc15_program_register_sequence(adev,
1120 golden_settings_gc_9_0_vg20,
1121 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
1122 break;
1123 case IP_VERSION(9, 4, 1):
1124 soc15_program_register_sequence(adev,
1125 golden_settings_gc_9_4_1_arct,
1126 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
1127 break;
1128 case IP_VERSION(9, 2, 2):
1129 case IP_VERSION(9, 1, 0):
1130 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
1131 ARRAY_SIZE(golden_settings_gc_9_1));
1132 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1133 soc15_program_register_sequence(adev,
1134 golden_settings_gc_9_1_rv2,
1135 ARRAY_SIZE(golden_settings_gc_9_1_rv2));
1136 else
1137 soc15_program_register_sequence(adev,
1138 golden_settings_gc_9_1_rv1,
1139 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
1140 break;
1141 case IP_VERSION(9, 3, 0):
1142 soc15_program_register_sequence(adev,
1143 golden_settings_gc_9_1_rn,
1144 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1145 return; /* for renoir, don't need common goldensetting */
1146 case IP_VERSION(9, 4, 2):
1147 gfx_v9_4_2_init_golden_registers(adev,
1148 adev->smuio.funcs->get_die_id(adev));
1149 break;
1150 default:
1151 break;
1152 }
1153
1154 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1155 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))
1156 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1157 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1158 }
1159
gfx_v9_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)1160 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1161 bool wc, uint32_t reg, uint32_t val)
1162 {
1163 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1164 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1165 WRITE_DATA_DST_SEL(0) |
1166 (wc ? WR_CONFIRM : 0));
1167 amdgpu_ring_write(ring, reg);
1168 amdgpu_ring_write(ring, 0);
1169 amdgpu_ring_write(ring, val);
1170 }
1171
gfx_v9_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)1172 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1173 int mem_space, int opt, uint32_t addr0,
1174 uint32_t addr1, uint32_t ref, uint32_t mask,
1175 uint32_t inv)
1176 {
1177 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1178 amdgpu_ring_write(ring,
1179 /* memory (1) or register (0) */
1180 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1181 WAIT_REG_MEM_OPERATION(opt) | /* wait */
1182 WAIT_REG_MEM_FUNCTION(3) | /* equal */
1183 WAIT_REG_MEM_ENGINE(eng_sel)));
1184
1185 if (mem_space)
1186 BUG_ON(addr0 & 0x3); /* Dword align */
1187 amdgpu_ring_write(ring, addr0);
1188 amdgpu_ring_write(ring, addr1);
1189 amdgpu_ring_write(ring, ref);
1190 amdgpu_ring_write(ring, mask);
1191 amdgpu_ring_write(ring, inv); /* poll interval */
1192 }
1193
gfx_v9_0_ring_test_ring(struct amdgpu_ring * ring)1194 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1195 {
1196 struct amdgpu_device *adev = ring->adev;
1197 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1198 uint32_t tmp = 0;
1199 unsigned i;
1200 int r;
1201
1202 WREG32(scratch, 0xCAFEDEAD);
1203 r = amdgpu_ring_alloc(ring, 3);
1204 if (r)
1205 return r;
1206
1207 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1208 amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);
1209 amdgpu_ring_write(ring, 0xDEADBEEF);
1210 amdgpu_ring_commit(ring);
1211
1212 for (i = 0; i < adev->usec_timeout; i++) {
1213 tmp = RREG32(scratch);
1214 if (tmp == 0xDEADBEEF)
1215 break;
1216 udelay(1);
1217 }
1218
1219 if (i >= adev->usec_timeout)
1220 r = -ETIMEDOUT;
1221 return r;
1222 }
1223
gfx_v9_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)1224 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1225 {
1226 struct amdgpu_device *adev = ring->adev;
1227 struct amdgpu_ib ib;
1228 struct dma_fence *f = NULL;
1229
1230 unsigned index;
1231 uint64_t gpu_addr;
1232 uint32_t tmp;
1233 long r;
1234
1235 r = amdgpu_device_wb_get(adev, &index);
1236 if (r)
1237 return r;
1238
1239 gpu_addr = adev->wb.gpu_addr + (index * 4);
1240 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1241 memset(&ib, 0, sizeof(ib));
1242
1243 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
1244 if (r)
1245 goto err1;
1246
1247 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1248 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1249 ib.ptr[2] = lower_32_bits(gpu_addr);
1250 ib.ptr[3] = upper_32_bits(gpu_addr);
1251 ib.ptr[4] = 0xDEADBEEF;
1252 ib.length_dw = 5;
1253
1254 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1255 if (r)
1256 goto err2;
1257
1258 r = dma_fence_wait_timeout(f, false, timeout);
1259 if (r == 0) {
1260 r = -ETIMEDOUT;
1261 goto err2;
1262 } else if (r < 0) {
1263 goto err2;
1264 }
1265
1266 tmp = adev->wb.wb[index];
1267 if (tmp == 0xDEADBEEF)
1268 r = 0;
1269 else
1270 r = -EINVAL;
1271
1272 err2:
1273 amdgpu_ib_free(&ib, NULL);
1274 dma_fence_put(f);
1275 err1:
1276 amdgpu_device_wb_free(adev, index);
1277 return r;
1278 }
1279
1280
gfx_v9_0_free_microcode(struct amdgpu_device * adev)1281 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1282 {
1283 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1284 amdgpu_ucode_release(&adev->gfx.me_fw);
1285 amdgpu_ucode_release(&adev->gfx.ce_fw);
1286 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1287 amdgpu_ucode_release(&adev->gfx.mec_fw);
1288 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1289
1290 kfree(adev->gfx.rlc.register_list_format);
1291 }
1292
gfx_v9_0_check_fw_write_wait(struct amdgpu_device * adev)1293 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1294 {
1295 adev->gfx.me_fw_write_wait = false;
1296 adev->gfx.mec_fw_write_wait = false;
1297
1298 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1299 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) &&
1300 ((adev->gfx.mec_fw_version < 0x000001a5) ||
1301 (adev->gfx.mec_feature_version < 46) ||
1302 (adev->gfx.pfp_fw_version < 0x000000b7) ||
1303 (adev->gfx.pfp_feature_version < 46)))
1304 DRM_WARN_ONCE("CP firmware version too old, please update!");
1305
1306 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1307 case IP_VERSION(9, 0, 1):
1308 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1309 (adev->gfx.me_feature_version >= 42) &&
1310 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1311 (adev->gfx.pfp_feature_version >= 42))
1312 adev->gfx.me_fw_write_wait = true;
1313
1314 if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1315 (adev->gfx.mec_feature_version >= 42))
1316 adev->gfx.mec_fw_write_wait = true;
1317 break;
1318 case IP_VERSION(9, 2, 1):
1319 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1320 (adev->gfx.me_feature_version >= 44) &&
1321 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1322 (adev->gfx.pfp_feature_version >= 44))
1323 adev->gfx.me_fw_write_wait = true;
1324
1325 if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1326 (adev->gfx.mec_feature_version >= 44))
1327 adev->gfx.mec_fw_write_wait = true;
1328 break;
1329 case IP_VERSION(9, 4, 0):
1330 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1331 (adev->gfx.me_feature_version >= 44) &&
1332 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1333 (adev->gfx.pfp_feature_version >= 44))
1334 adev->gfx.me_fw_write_wait = true;
1335
1336 if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1337 (adev->gfx.mec_feature_version >= 44))
1338 adev->gfx.mec_fw_write_wait = true;
1339 break;
1340 case IP_VERSION(9, 1, 0):
1341 case IP_VERSION(9, 2, 2):
1342 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1343 (adev->gfx.me_feature_version >= 42) &&
1344 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1345 (adev->gfx.pfp_feature_version >= 42))
1346 adev->gfx.me_fw_write_wait = true;
1347
1348 if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1349 (adev->gfx.mec_feature_version >= 42))
1350 adev->gfx.mec_fw_write_wait = true;
1351 break;
1352 default:
1353 adev->gfx.me_fw_write_wait = true;
1354 adev->gfx.mec_fw_write_wait = true;
1355 break;
1356 }
1357 }
1358
1359 struct amdgpu_gfxoff_quirk {
1360 u16 chip_vendor;
1361 u16 chip_device;
1362 u16 subsys_vendor;
1363 u16 subsys_device;
1364 u8 revision;
1365 };
1366
1367 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1368 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1369 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1370 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1371 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1372 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1373 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1374 /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1375 { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1376 /* https://bbs.openkylin.top/t/topic/171497 */
1377 { 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 },
1378 /* HP 705G4 DM with R5 2400G */
1379 { 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 },
1380 { 0, 0, 0, 0, 0 },
1381 };
1382
gfx_v9_0_should_disable_gfxoff(struct pci_dev * pdev)1383 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1384 {
1385 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1386
1387 while (p && p->chip_device != 0) {
1388 if (pdev->vendor == p->chip_vendor &&
1389 pdev->device == p->chip_device &&
1390 pdev->subsystem_vendor == p->subsys_vendor &&
1391 pdev->subsystem_device == p->subsys_device &&
1392 pdev->revision == p->revision) {
1393 return true;
1394 }
1395 ++p;
1396 }
1397 return false;
1398 }
1399
is_raven_kicker(struct amdgpu_device * adev)1400 static bool is_raven_kicker(struct amdgpu_device *adev)
1401 {
1402 if (adev->pm.fw_version >= 0x41e2b)
1403 return true;
1404 else
1405 return false;
1406 }
1407
check_if_enlarge_doorbell_range(struct amdgpu_device * adev)1408 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1409 {
1410 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) &&
1411 (adev->gfx.me_fw_version >= 0x000000a5) &&
1412 (adev->gfx.me_feature_version >= 52))
1413 return true;
1414 else
1415 return false;
1416 }
1417
gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device * adev)1418 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1419 {
1420 if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1421 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1422
1423 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1424 case IP_VERSION(9, 0, 1):
1425 case IP_VERSION(9, 2, 1):
1426 case IP_VERSION(9, 4, 0):
1427 break;
1428 case IP_VERSION(9, 2, 2):
1429 case IP_VERSION(9, 1, 0):
1430 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1431 (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1432 ((!is_raven_kicker(adev) &&
1433 adev->gfx.rlc_fw_version < 531) ||
1434 (adev->gfx.rlc_feature_version < 1) ||
1435 !adev->gfx.rlc.is_rlc_v2_1))
1436 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1437
1438 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1439 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1440 AMD_PG_SUPPORT_CP |
1441 AMD_PG_SUPPORT_RLC_SMU_HS;
1442 break;
1443 case IP_VERSION(9, 3, 0):
1444 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1445 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1446 AMD_PG_SUPPORT_CP |
1447 AMD_PG_SUPPORT_RLC_SMU_HS;
1448 break;
1449 default:
1450 break;
1451 }
1452 }
1453
gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device * adev,char * chip_name)1454 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1455 char *chip_name)
1456 {
1457 int err;
1458
1459 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
1460 AMDGPU_UCODE_REQUIRED,
1461 "amdgpu/%s_pfp.bin", chip_name);
1462 if (err)
1463 goto out;
1464 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
1465
1466 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1467 AMDGPU_UCODE_REQUIRED,
1468 "amdgpu/%s_me.bin", chip_name);
1469 if (err)
1470 goto out;
1471 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
1472
1473 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1474 AMDGPU_UCODE_REQUIRED,
1475 "amdgpu/%s_ce.bin", chip_name);
1476 if (err)
1477 goto out;
1478 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
1479
1480 out:
1481 if (err) {
1482 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1483 amdgpu_ucode_release(&adev->gfx.me_fw);
1484 amdgpu_ucode_release(&adev->gfx.ce_fw);
1485 }
1486 return err;
1487 }
1488
gfx_v9_0_init_rlc_microcode(struct amdgpu_device * adev,char * chip_name)1489 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1490 char *chip_name)
1491 {
1492 int err;
1493 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1494 uint16_t version_major;
1495 uint16_t version_minor;
1496 uint32_t smu_version;
1497
1498 /*
1499 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1500 * instead of picasso_rlc.bin.
1501 * Judgment method:
1502 * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1503 * or revision >= 0xD8 && revision <= 0xDF
1504 * otherwise is PCO FP5
1505 */
1506 if (!strcmp(chip_name, "picasso") &&
1507 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1508 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1509 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1510 AMDGPU_UCODE_REQUIRED,
1511 "amdgpu/%s_rlc_am4.bin", chip_name);
1512 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1513 (smu_version >= 0x41e2b))
1514 /**
1515 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1516 */
1517 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1518 AMDGPU_UCODE_REQUIRED,
1519 "amdgpu/%s_kicker_rlc.bin", chip_name);
1520 else
1521 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1522 AMDGPU_UCODE_REQUIRED,
1523 "amdgpu/%s_rlc.bin", chip_name);
1524 if (err)
1525 goto out;
1526
1527 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1528 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1529 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1530 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1531 out:
1532 if (err)
1533 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1534
1535 return err;
1536 }
1537
gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device * adev)1538 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1539 {
1540 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
1541 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1542 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0))
1543 return false;
1544
1545 return true;
1546 }
1547
gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device * adev,char * chip_name)1548 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1549 char *chip_name)
1550 {
1551 int err;
1552
1553 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1554 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1555 AMDGPU_UCODE_REQUIRED,
1556 "amdgpu/%s_sjt_mec.bin", chip_name);
1557 else
1558 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1559 AMDGPU_UCODE_REQUIRED,
1560 "amdgpu/%s_mec.bin", chip_name);
1561 if (err)
1562 goto out;
1563
1564 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
1565 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
1566
1567 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1568 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1569 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1570 AMDGPU_UCODE_REQUIRED,
1571 "amdgpu/%s_sjt_mec2.bin", chip_name);
1572 else
1573 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1574 AMDGPU_UCODE_REQUIRED,
1575 "amdgpu/%s_mec2.bin", chip_name);
1576 if (!err) {
1577 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
1578 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
1579 } else {
1580 err = 0;
1581 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1582 }
1583 } else {
1584 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1585 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1586 }
1587
1588 gfx_v9_0_check_if_need_gfxoff(adev);
1589 gfx_v9_0_check_fw_write_wait(adev);
1590
1591 out:
1592 if (err)
1593 amdgpu_ucode_release(&adev->gfx.mec_fw);
1594 return err;
1595 }
1596
gfx_v9_0_init_microcode(struct amdgpu_device * adev)1597 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1598 {
1599 char ucode_prefix[30];
1600 int r;
1601
1602 DRM_DEBUG("\n");
1603 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
1604
1605 /* No CPG in Arcturus */
1606 if (adev->gfx.num_gfx_rings) {
1607 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);
1608 if (r)
1609 return r;
1610 }
1611
1612 r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);
1613 if (r)
1614 return r;
1615
1616 r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);
1617 if (r)
1618 return r;
1619
1620 return r;
1621 }
1622
gfx_v9_0_get_csb_size(struct amdgpu_device * adev)1623 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1624 {
1625 u32 count = 0;
1626 const struct cs_section_def *sect = NULL;
1627 const struct cs_extent_def *ext = NULL;
1628
1629 /* begin clear state */
1630 count += 2;
1631 /* context control state */
1632 count += 3;
1633
1634 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1635 for (ext = sect->section; ext->extent != NULL; ++ext) {
1636 if (sect->id == SECT_CONTEXT)
1637 count += 2 + ext->reg_count;
1638 else
1639 return 0;
1640 }
1641 }
1642
1643 /* end clear state */
1644 count += 2;
1645 /* clear state */
1646 count += 2;
1647
1648 return count;
1649 }
1650
gfx_v9_0_get_csb_buffer(struct amdgpu_device * adev,volatile u32 * buffer)1651 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1652 volatile u32 *buffer)
1653 {
1654 u32 count = 0;
1655
1656 if (adev->gfx.rlc.cs_data == NULL)
1657 return;
1658 if (buffer == NULL)
1659 return;
1660
1661 count = amdgpu_gfx_csb_preamble_start(buffer);
1662 count = amdgpu_gfx_csb_data_parser(adev, buffer, count);
1663 amdgpu_gfx_csb_preamble_end(buffer, count);
1664 }
1665
gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device * adev)1666 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1667 {
1668 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1669 uint32_t pg_always_on_cu_num = 2;
1670 uint32_t always_on_cu_num;
1671 uint32_t i, j, k;
1672 uint32_t mask, cu_bitmap, counter;
1673
1674 if (adev->flags & AMD_IS_APU)
1675 always_on_cu_num = 4;
1676 else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1))
1677 always_on_cu_num = 8;
1678 else
1679 always_on_cu_num = 12;
1680
1681 mutex_lock(&adev->grbm_idx_mutex);
1682 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1683 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1684 mask = 1;
1685 cu_bitmap = 0;
1686 counter = 0;
1687 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1688
1689 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1690 if (cu_info->bitmap[0][i][j] & mask) {
1691 if (counter == pg_always_on_cu_num)
1692 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1693 if (counter < always_on_cu_num)
1694 cu_bitmap |= mask;
1695 else
1696 break;
1697 counter++;
1698 }
1699 mask <<= 1;
1700 }
1701
1702 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1703 cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1704 }
1705 }
1706 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1707 mutex_unlock(&adev->grbm_idx_mutex);
1708 }
1709
gfx_v9_0_init_lbpw(struct amdgpu_device * adev)1710 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1711 {
1712 uint32_t data;
1713
1714 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1715 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1716 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1717 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1718 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1719
1720 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1721 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1722
1723 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1724 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1725
1726 mutex_lock(&adev->grbm_idx_mutex);
1727 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1728 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1729 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1730
1731 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1732 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1733 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1734 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1735 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1736
1737 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1738 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1739 data &= 0x0000FFFF;
1740 data |= 0x00C00000;
1741 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1742
1743 /*
1744 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1745 * programmed in gfx_v9_0_init_always_on_cu_mask()
1746 */
1747
1748 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1749 * but used for RLC_LB_CNTL configuration */
1750 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1751 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1752 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1753 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1754 mutex_unlock(&adev->grbm_idx_mutex);
1755
1756 gfx_v9_0_init_always_on_cu_mask(adev);
1757 }
1758
gfx_v9_4_init_lbpw(struct amdgpu_device * adev)1759 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1760 {
1761 uint32_t data;
1762
1763 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1764 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1765 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1766 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1767 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1768
1769 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1770 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1771
1772 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1773 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1774
1775 mutex_lock(&adev->grbm_idx_mutex);
1776 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1777 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1778 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1779
1780 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1781 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1782 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1783 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1784 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1785
1786 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1787 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1788 data &= 0x0000FFFF;
1789 data |= 0x00C00000;
1790 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1791
1792 /*
1793 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1794 * programmed in gfx_v9_0_init_always_on_cu_mask()
1795 */
1796
1797 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1798 * but used for RLC_LB_CNTL configuration */
1799 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1800 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1801 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1802 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1803 mutex_unlock(&adev->grbm_idx_mutex);
1804
1805 gfx_v9_0_init_always_on_cu_mask(adev);
1806 }
1807
gfx_v9_0_enable_lbpw(struct amdgpu_device * adev,bool enable)1808 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1809 {
1810 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1811 }
1812
gfx_v9_0_cp_jump_table_num(struct amdgpu_device * adev)1813 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1814 {
1815 if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1816 return 5;
1817 else
1818 return 4;
1819 }
1820
gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)1821 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1822 {
1823 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1824
1825 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1826 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1827 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1828 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1829 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1830 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1831 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1832 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1833 adev->gfx.rlc.rlcg_reg_access_supported = true;
1834 }
1835
gfx_v9_0_rlc_init(struct amdgpu_device * adev)1836 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1837 {
1838 const struct cs_section_def *cs_data;
1839 int r;
1840
1841 adev->gfx.rlc.cs_data = gfx9_cs_data;
1842
1843 cs_data = adev->gfx.rlc.cs_data;
1844
1845 if (cs_data) {
1846 /* init clear state block */
1847 r = amdgpu_gfx_rlc_init_csb(adev);
1848 if (r)
1849 return r;
1850 }
1851
1852 if (adev->flags & AMD_IS_APU) {
1853 /* TODO: double check the cp_table_size for RV */
1854 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1855 r = amdgpu_gfx_rlc_init_cpt(adev);
1856 if (r)
1857 return r;
1858 }
1859
1860 return 0;
1861 }
1862
gfx_v9_0_mec_fini(struct amdgpu_device * adev)1863 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1864 {
1865 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1866 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1867 }
1868
gfx_v9_0_mec_init(struct amdgpu_device * adev)1869 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1870 {
1871 int r;
1872 u32 *hpd;
1873 const __le32 *fw_data;
1874 unsigned fw_size;
1875 u32 *fw;
1876 size_t mec_hpd_size;
1877
1878 const struct gfx_firmware_header_v1_0 *mec_hdr;
1879
1880 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1881
1882 /* take ownership of the relevant compute queues */
1883 amdgpu_gfx_compute_queue_acquire(adev);
1884 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1885 if (mec_hpd_size) {
1886 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1887 AMDGPU_GEM_DOMAIN_VRAM |
1888 AMDGPU_GEM_DOMAIN_GTT,
1889 &adev->gfx.mec.hpd_eop_obj,
1890 &adev->gfx.mec.hpd_eop_gpu_addr,
1891 (void **)&hpd);
1892 if (r) {
1893 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1894 gfx_v9_0_mec_fini(adev);
1895 return r;
1896 }
1897
1898 memset(hpd, 0, mec_hpd_size);
1899
1900 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1901 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1902 }
1903
1904 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1905
1906 fw_data = (const __le32 *)
1907 (adev->gfx.mec_fw->data +
1908 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1909 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1910
1911 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1912 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1913 &adev->gfx.mec.mec_fw_obj,
1914 &adev->gfx.mec.mec_fw_gpu_addr,
1915 (void **)&fw);
1916 if (r) {
1917 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1918 gfx_v9_0_mec_fini(adev);
1919 return r;
1920 }
1921
1922 memcpy(fw, fw_data, fw_size);
1923
1924 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1925 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1926
1927 return 0;
1928 }
1929
wave_read_ind(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t address)1930 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1931 {
1932 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1933 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1934 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1935 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1936 (SQ_IND_INDEX__FORCE_READ_MASK));
1937 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1938 }
1939
wave_read_regs(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)1940 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1941 uint32_t wave, uint32_t thread,
1942 uint32_t regno, uint32_t num, uint32_t *out)
1943 {
1944 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1945 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1946 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1947 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1948 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1949 (SQ_IND_INDEX__FORCE_READ_MASK) |
1950 (SQ_IND_INDEX__AUTO_INCR_MASK));
1951 while (num--)
1952 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1953 }
1954
gfx_v9_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)1955 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1956 {
1957 /* type 1 wave data */
1958 dst[(*no_fields)++] = 1;
1959 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1960 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1961 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1962 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1963 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1964 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1965 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1966 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1967 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1968 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1969 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1970 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1971 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1972 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1973 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1974 }
1975
gfx_v9_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)1976 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1977 uint32_t wave, uint32_t start,
1978 uint32_t size, uint32_t *dst)
1979 {
1980 wave_read_regs(
1981 adev, simd, wave, 0,
1982 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1983 }
1984
gfx_v9_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)1985 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1986 uint32_t wave, uint32_t thread,
1987 uint32_t start, uint32_t size,
1988 uint32_t *dst)
1989 {
1990 wave_read_regs(
1991 adev, simd, wave, thread,
1992 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1993 }
1994
gfx_v9_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)1995 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1996 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1997 {
1998 soc15_grbm_select(adev, me, pipe, q, vm, 0);
1999 }
2000
2001 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2002 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2003 .select_se_sh = &gfx_v9_0_select_se_sh,
2004 .read_wave_data = &gfx_v9_0_read_wave_data,
2005 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2006 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2007 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2008 };
2009
2010 const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = {
2011 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2012 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2013 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2014 };
2015
2016 static struct amdgpu_gfx_ras gfx_v9_0_ras = {
2017 .ras_block = {
2018 .hw_ops = &gfx_v9_0_ras_ops,
2019 },
2020 };
2021
gfx_v9_0_gpu_early_init(struct amdgpu_device * adev)2022 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2023 {
2024 u32 gb_addr_config;
2025 int err;
2026
2027 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2028 case IP_VERSION(9, 0, 1):
2029 adev->gfx.config.max_hw_contexts = 8;
2030 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2031 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2032 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2033 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2034 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2035 break;
2036 case IP_VERSION(9, 2, 1):
2037 adev->gfx.config.max_hw_contexts = 8;
2038 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2039 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2040 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2041 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2042 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2043 DRM_INFO("fix gfx.config for vega12\n");
2044 break;
2045 case IP_VERSION(9, 4, 0):
2046 adev->gfx.ras = &gfx_v9_0_ras;
2047 adev->gfx.config.max_hw_contexts = 8;
2048 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2049 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2050 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2051 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2052 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2053 gb_addr_config &= ~0xf3e777ff;
2054 gb_addr_config |= 0x22014042;
2055 /* check vbios table if gpu info is not available */
2056 err = amdgpu_atomfirmware_get_gfx_info(adev);
2057 if (err)
2058 return err;
2059 break;
2060 case IP_VERSION(9, 2, 2):
2061 case IP_VERSION(9, 1, 0):
2062 adev->gfx.config.max_hw_contexts = 8;
2063 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2064 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2065 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2066 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2067 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2068 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2069 else
2070 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2071 break;
2072 case IP_VERSION(9, 4, 1):
2073 adev->gfx.ras = &gfx_v9_4_ras;
2074 adev->gfx.config.max_hw_contexts = 8;
2075 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2076 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2077 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2078 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2079 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2080 gb_addr_config &= ~0xf3e777ff;
2081 gb_addr_config |= 0x22014042;
2082 break;
2083 case IP_VERSION(9, 3, 0):
2084 adev->gfx.config.max_hw_contexts = 8;
2085 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2086 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2087 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2088 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2089 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2090 gb_addr_config &= ~0xf3e777ff;
2091 gb_addr_config |= 0x22010042;
2092 break;
2093 case IP_VERSION(9, 4, 2):
2094 adev->gfx.ras = &gfx_v9_4_2_ras;
2095 adev->gfx.config.max_hw_contexts = 8;
2096 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2097 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2098 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2099 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2100 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2101 gb_addr_config &= ~0xf3e777ff;
2102 gb_addr_config |= 0x22014042;
2103 /* check vbios table if gpu info is not available */
2104 err = amdgpu_atomfirmware_get_gfx_info(adev);
2105 if (err)
2106 return err;
2107 break;
2108 default:
2109 BUG();
2110 break;
2111 }
2112
2113 adev->gfx.config.gb_addr_config = gb_addr_config;
2114
2115 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2116 REG_GET_FIELD(
2117 adev->gfx.config.gb_addr_config,
2118 GB_ADDR_CONFIG,
2119 NUM_PIPES);
2120
2121 adev->gfx.config.max_tile_pipes =
2122 adev->gfx.config.gb_addr_config_fields.num_pipes;
2123
2124 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2125 REG_GET_FIELD(
2126 adev->gfx.config.gb_addr_config,
2127 GB_ADDR_CONFIG,
2128 NUM_BANKS);
2129 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2130 REG_GET_FIELD(
2131 adev->gfx.config.gb_addr_config,
2132 GB_ADDR_CONFIG,
2133 MAX_COMPRESSED_FRAGS);
2134 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2135 REG_GET_FIELD(
2136 adev->gfx.config.gb_addr_config,
2137 GB_ADDR_CONFIG,
2138 NUM_RB_PER_SE);
2139 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2140 REG_GET_FIELD(
2141 adev->gfx.config.gb_addr_config,
2142 GB_ADDR_CONFIG,
2143 NUM_SHADER_ENGINES);
2144 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2145 REG_GET_FIELD(
2146 adev->gfx.config.gb_addr_config,
2147 GB_ADDR_CONFIG,
2148 PIPE_INTERLEAVE_SIZE));
2149
2150 return 0;
2151 }
2152
gfx_v9_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)2153 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2154 int mec, int pipe, int queue)
2155 {
2156 unsigned irq_type;
2157 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2158 unsigned int hw_prio;
2159
2160 ring = &adev->gfx.compute_ring[ring_id];
2161
2162 /* mec0 is me1 */
2163 ring->me = mec + 1;
2164 ring->pipe = pipe;
2165 ring->queue = queue;
2166
2167 ring->ring_obj = NULL;
2168 ring->use_doorbell = true;
2169 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2170 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2171 + (ring_id * GFX9_MEC_HPD_SIZE);
2172 ring->vm_hub = AMDGPU_GFXHUB(0);
2173 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2174
2175 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2176 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2177 + ring->pipe;
2178 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2179 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
2180 /* type-2 packets are deprecated on MEC, use type-3 instead */
2181 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2182 hw_prio, NULL);
2183 }
2184
gfx_v9_0_alloc_ip_dump(struct amdgpu_device * adev)2185 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)
2186 {
2187 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
2188 uint32_t *ptr;
2189 uint32_t inst;
2190
2191 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
2192 if (!ptr) {
2193 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
2194 adev->gfx.ip_dump_core = NULL;
2195 } else {
2196 adev->gfx.ip_dump_core = ptr;
2197 }
2198
2199 /* Allocate memory for compute queue registers for all the instances */
2200 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
2201 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
2202 adev->gfx.mec.num_queue_per_pipe;
2203
2204 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
2205 if (!ptr) {
2206 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
2207 adev->gfx.ip_dump_compute_queues = NULL;
2208 } else {
2209 adev->gfx.ip_dump_compute_queues = ptr;
2210 }
2211 }
2212
gfx_v9_0_sw_init(struct amdgpu_ip_block * ip_block)2213 static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
2214 {
2215 int i, j, k, r, ring_id;
2216 int xcc_id = 0;
2217 struct amdgpu_ring *ring;
2218 struct amdgpu_device *adev = ip_block->adev;
2219 unsigned int hw_prio;
2220
2221 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2222 case IP_VERSION(9, 0, 1):
2223 case IP_VERSION(9, 2, 1):
2224 case IP_VERSION(9, 4, 0):
2225 case IP_VERSION(9, 2, 2):
2226 case IP_VERSION(9, 1, 0):
2227 case IP_VERSION(9, 4, 1):
2228 case IP_VERSION(9, 3, 0):
2229 case IP_VERSION(9, 4, 2):
2230 adev->gfx.mec.num_mec = 2;
2231 break;
2232 default:
2233 adev->gfx.mec.num_mec = 1;
2234 break;
2235 }
2236
2237 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2238 case IP_VERSION(9, 0, 1):
2239 case IP_VERSION(9, 2, 1):
2240 case IP_VERSION(9, 4, 0):
2241 case IP_VERSION(9, 2, 2):
2242 case IP_VERSION(9, 1, 0):
2243 case IP_VERSION(9, 3, 0):
2244 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2245 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2246 if (adev->gfx.me_fw_version >= 167 &&
2247 adev->gfx.pfp_fw_version >= 196 &&
2248 adev->gfx.mec_fw_version >= 474) {
2249 adev->gfx.enable_cleaner_shader = true;
2250 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2251 if (r) {
2252 adev->gfx.enable_cleaner_shader = false;
2253 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2254 }
2255 }
2256 break;
2257 case IP_VERSION(9, 4, 2):
2258 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2259 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2260 if (adev->gfx.mec_fw_version >= 88) {
2261 adev->gfx.enable_cleaner_shader = true;
2262 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2263 if (r) {
2264 adev->gfx.enable_cleaner_shader = false;
2265 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2266 }
2267 }
2268 break;
2269 default:
2270 adev->gfx.enable_cleaner_shader = false;
2271 break;
2272 }
2273
2274 adev->gfx.mec.num_pipe_per_mec = 4;
2275 adev->gfx.mec.num_queue_per_pipe = 8;
2276
2277 /* EOP Event */
2278 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2279 if (r)
2280 return r;
2281
2282 /* Bad opcode Event */
2283 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
2284 GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
2285 &adev->gfx.bad_op_irq);
2286 if (r)
2287 return r;
2288
2289 /* Privileged reg */
2290 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2291 &adev->gfx.priv_reg_irq);
2292 if (r)
2293 return r;
2294
2295 /* Privileged inst */
2296 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2297 &adev->gfx.priv_inst_irq);
2298 if (r)
2299 return r;
2300
2301 /* ECC error */
2302 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2303 &adev->gfx.cp_ecc_error_irq);
2304 if (r)
2305 return r;
2306
2307 /* FUE error */
2308 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2309 &adev->gfx.cp_ecc_error_irq);
2310 if (r)
2311 return r;
2312
2313 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2314
2315 if (adev->gfx.rlc.funcs) {
2316 if (adev->gfx.rlc.funcs->init) {
2317 r = adev->gfx.rlc.funcs->init(adev);
2318 if (r) {
2319 dev_err(adev->dev, "Failed to init rlc BOs!\n");
2320 return r;
2321 }
2322 }
2323 }
2324
2325 r = gfx_v9_0_mec_init(adev);
2326 if (r) {
2327 DRM_ERROR("Failed to init MEC BOs!\n");
2328 return r;
2329 }
2330
2331 /* set up the gfx ring */
2332 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2333 ring = &adev->gfx.gfx_ring[i];
2334 ring->ring_obj = NULL;
2335 if (!i)
2336 sprintf(ring->name, "gfx");
2337 else
2338 sprintf(ring->name, "gfx_%d", i);
2339 ring->use_doorbell = true;
2340 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2341
2342 /* disable scheduler on the real ring */
2343 ring->no_scheduler = adev->gfx.mcbp;
2344 ring->vm_hub = AMDGPU_GFXHUB(0);
2345 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2346 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2347 AMDGPU_RING_PRIO_DEFAULT, NULL);
2348 if (r)
2349 return r;
2350 }
2351
2352 /* set up the software rings */
2353 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2354 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2355 ring = &adev->gfx.sw_gfx_ring[i];
2356 ring->ring_obj = NULL;
2357 sprintf(ring->name, amdgpu_sw_ring_name(i));
2358 ring->use_doorbell = true;
2359 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2360 ring->is_sw_ring = true;
2361 hw_prio = amdgpu_sw_ring_priority(i);
2362 ring->vm_hub = AMDGPU_GFXHUB(0);
2363 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2364 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2365 NULL);
2366 if (r)
2367 return r;
2368 ring->wptr = 0;
2369 }
2370
2371 /* init the muxer and add software rings */
2372 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],
2373 GFX9_NUM_SW_GFX_RINGS);
2374 if (r) {
2375 DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2376 return r;
2377 }
2378 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2379 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,
2380 &adev->gfx.sw_gfx_ring[i]);
2381 if (r) {
2382 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2383 return r;
2384 }
2385 }
2386 }
2387
2388 /* set up the compute queues - allocate horizontally across pipes */
2389 ring_id = 0;
2390 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2391 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2392 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2393 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
2394 k, j))
2395 continue;
2396
2397 r = gfx_v9_0_compute_ring_init(adev,
2398 ring_id,
2399 i, k, j);
2400 if (r)
2401 return r;
2402
2403 ring_id++;
2404 }
2405 }
2406 }
2407
2408 /* TODO: Add queue reset mask when FW fully supports it */
2409 adev->gfx.gfx_supported_reset =
2410 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
2411 adev->gfx.compute_supported_reset =
2412 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
2413 if (!amdgpu_sriov_vf(adev))
2414 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
2415
2416 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0);
2417 if (r) {
2418 DRM_ERROR("Failed to init KIQ BOs!\n");
2419 return r;
2420 }
2421
2422 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2423 if (r)
2424 return r;
2425
2426 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2427 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0);
2428 if (r)
2429 return r;
2430
2431 adev->gfx.ce_ram_size = 0x8000;
2432
2433 r = gfx_v9_0_gpu_early_init(adev);
2434 if (r)
2435 return r;
2436
2437 if (amdgpu_gfx_ras_sw_init(adev)) {
2438 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2439 return -EINVAL;
2440 }
2441
2442 gfx_v9_0_alloc_ip_dump(adev);
2443
2444 r = amdgpu_gfx_sysfs_init(adev);
2445 if (r)
2446 return r;
2447
2448 return 0;
2449 }
2450
2451
gfx_v9_0_sw_fini(struct amdgpu_ip_block * ip_block)2452 static int gfx_v9_0_sw_fini(struct amdgpu_ip_block *ip_block)
2453 {
2454 int i;
2455 struct amdgpu_device *adev = ip_block->adev;
2456
2457 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2458 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2459 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);
2460 amdgpu_ring_mux_fini(&adev->gfx.muxer);
2461 }
2462
2463 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2464 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2465 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2466 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2467
2468 amdgpu_gfx_mqd_sw_fini(adev, 0);
2469 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
2470 amdgpu_gfx_kiq_fini(adev, 0);
2471
2472 amdgpu_gfx_cleaner_shader_sw_fini(adev);
2473
2474 gfx_v9_0_mec_fini(adev);
2475 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2476 &adev->gfx.rlc.clear_state_gpu_addr,
2477 (void **)&adev->gfx.rlc.cs_ptr);
2478 if (adev->flags & AMD_IS_APU) {
2479 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2480 &adev->gfx.rlc.cp_table_gpu_addr,
2481 (void **)&adev->gfx.rlc.cp_table_ptr);
2482 }
2483 gfx_v9_0_free_microcode(adev);
2484
2485 amdgpu_gfx_sysfs_fini(adev);
2486
2487 kfree(adev->gfx.ip_dump_core);
2488 kfree(adev->gfx.ip_dump_compute_queues);
2489
2490 return 0;
2491 }
2492
2493
gfx_v9_0_tiling_mode_table_init(struct amdgpu_device * adev)2494 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2495 {
2496 /* TODO */
2497 }
2498
gfx_v9_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)2499 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2500 u32 instance, int xcc_id)
2501 {
2502 u32 data;
2503
2504 if (instance == 0xffffffff)
2505 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2506 else
2507 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2508
2509 if (se_num == 0xffffffff)
2510 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2511 else
2512 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2513
2514 if (sh_num == 0xffffffff)
2515 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2516 else
2517 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2518
2519 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2520 }
2521
gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device * adev)2522 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2523 {
2524 u32 data, mask;
2525
2526 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2527 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2528
2529 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2530 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2531
2532 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2533 adev->gfx.config.max_sh_per_se);
2534
2535 return (~data) & mask;
2536 }
2537
gfx_v9_0_setup_rb(struct amdgpu_device * adev)2538 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2539 {
2540 int i, j;
2541 u32 data;
2542 u32 active_rbs = 0;
2543 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2544 adev->gfx.config.max_sh_per_se;
2545
2546 mutex_lock(&adev->grbm_idx_mutex);
2547 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2548 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2549 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2550 data = gfx_v9_0_get_rb_active_bitmap(adev);
2551 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2552 rb_bitmap_width_per_sh);
2553 }
2554 }
2555 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2556 mutex_unlock(&adev->grbm_idx_mutex);
2557
2558 adev->gfx.config.backend_enable_mask = active_rbs;
2559 adev->gfx.config.num_rbs = hweight32(active_rbs);
2560 }
2561
gfx_v9_0_debug_trap_config_init(struct amdgpu_device * adev,uint32_t first_vmid,uint32_t last_vmid)2562 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2563 uint32_t first_vmid,
2564 uint32_t last_vmid)
2565 {
2566 uint32_t data;
2567 uint32_t trap_config_vmid_mask = 0;
2568 int i;
2569
2570 /* Calculate trap config vmid mask */
2571 for (i = first_vmid; i < last_vmid; i++)
2572 trap_config_vmid_mask |= (1 << i);
2573
2574 data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2575 VMID_SEL, trap_config_vmid_mask);
2576 data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2577 TRAP_EN, 1);
2578 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2579 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2580
2581 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2582 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2583 }
2584
2585 #define DEFAULT_SH_MEM_BASES (0x6000)
gfx_v9_0_init_compute_vmid(struct amdgpu_device * adev)2586 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2587 {
2588 int i;
2589 uint32_t sh_mem_config;
2590 uint32_t sh_mem_bases;
2591
2592 /*
2593 * Configure apertures:
2594 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2595 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2596 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2597 */
2598 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2599
2600 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2601 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2602 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2603
2604 mutex_lock(&adev->srbm_mutex);
2605 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2606 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2607 /* CP and shaders */
2608 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2609 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2610 }
2611 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2612 mutex_unlock(&adev->srbm_mutex);
2613
2614 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2615 access. These should be enabled by FW for target VMIDs. */
2616 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2617 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2618 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2619 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2620 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2621 }
2622 }
2623
gfx_v9_0_init_gds_vmid(struct amdgpu_device * adev)2624 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2625 {
2626 int vmid;
2627
2628 /*
2629 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2630 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2631 * the driver can enable them for graphics. VMID0 should maintain
2632 * access so that HWS firmware can save/restore entries.
2633 */
2634 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2635 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2636 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2637 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2638 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2639 }
2640 }
2641
gfx_v9_0_init_sq_config(struct amdgpu_device * adev)2642 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2643 {
2644 uint32_t tmp;
2645
2646 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2647 case IP_VERSION(9, 4, 1):
2648 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2649 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2650 !READ_ONCE(adev->barrier_has_auto_waitcnt));
2651 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2652 break;
2653 default:
2654 break;
2655 }
2656 }
2657
gfx_v9_0_constants_init(struct amdgpu_device * adev)2658 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2659 {
2660 u32 tmp;
2661 int i;
2662
2663 if (!amdgpu_sriov_vf(adev) ||
2664 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) {
2665 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2666 }
2667
2668 gfx_v9_0_tiling_mode_table_init(adev);
2669
2670 if (adev->gfx.num_gfx_rings)
2671 gfx_v9_0_setup_rb(adev);
2672 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2673 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2674
2675 /* XXX SH_MEM regs */
2676 /* where to put LDS, scratch, GPUVM in FSA64 space */
2677 mutex_lock(&adev->srbm_mutex);
2678 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2679 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2680 /* CP and shaders */
2681 if (i == 0) {
2682 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2683 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2684 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2685 !!adev->gmc.noretry);
2686 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2687 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2688 } else {
2689 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2690 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2691 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2692 !!adev->gmc.noretry);
2693 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2694 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2695 (adev->gmc.private_aperture_start >> 48));
2696 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2697 (adev->gmc.shared_aperture_start >> 48));
2698 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2699 }
2700 }
2701 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2702
2703 mutex_unlock(&adev->srbm_mutex);
2704
2705 gfx_v9_0_init_compute_vmid(adev);
2706 gfx_v9_0_init_gds_vmid(adev);
2707 gfx_v9_0_init_sq_config(adev);
2708 }
2709
gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device * adev)2710 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2711 {
2712 u32 i, j, k;
2713 u32 mask;
2714
2715 mutex_lock(&adev->grbm_idx_mutex);
2716 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2717 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2718 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2719 for (k = 0; k < adev->usec_timeout; k++) {
2720 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2721 break;
2722 udelay(1);
2723 }
2724 if (k == adev->usec_timeout) {
2725 amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2726 0xffffffff, 0xffffffff, 0);
2727 mutex_unlock(&adev->grbm_idx_mutex);
2728 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2729 i, j);
2730 return;
2731 }
2732 }
2733 }
2734 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2735 mutex_unlock(&adev->grbm_idx_mutex);
2736
2737 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2738 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2739 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2740 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2741 for (k = 0; k < adev->usec_timeout; k++) {
2742 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2743 break;
2744 udelay(1);
2745 }
2746 }
2747
gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2748 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2749 bool enable)
2750 {
2751 u32 tmp;
2752
2753 /* These interrupts should be enabled to drive DS clock */
2754
2755 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2756
2757 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2758 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2759 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2760 if (adev->gfx.num_gfx_rings)
2761 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2762
2763 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2764 }
2765
gfx_v9_0_init_csb(struct amdgpu_device * adev)2766 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2767 {
2768 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2769 /* csib */
2770 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2771 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2772 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2773 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2774 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2775 adev->gfx.rlc.clear_state_size);
2776 }
2777
gfx_v9_1_parse_ind_reg_list(int * register_list_format,int indirect_offset,int list_size,int * unique_indirect_regs,int unique_indirect_reg_count,int * indirect_start_offsets,int * indirect_start_offsets_count,int max_start_offsets_count)2778 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2779 int indirect_offset,
2780 int list_size,
2781 int *unique_indirect_regs,
2782 int unique_indirect_reg_count,
2783 int *indirect_start_offsets,
2784 int *indirect_start_offsets_count,
2785 int max_start_offsets_count)
2786 {
2787 int idx;
2788
2789 for (; indirect_offset < list_size; indirect_offset++) {
2790 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2791 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2792 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2793
2794 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2795 indirect_offset += 2;
2796
2797 /* look for the matching indice */
2798 for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2799 if (unique_indirect_regs[idx] ==
2800 register_list_format[indirect_offset] ||
2801 !unique_indirect_regs[idx])
2802 break;
2803 }
2804
2805 BUG_ON(idx >= unique_indirect_reg_count);
2806
2807 if (!unique_indirect_regs[idx])
2808 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2809
2810 indirect_offset++;
2811 }
2812 }
2813 }
2814
gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device * adev)2815 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2816 {
2817 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2818 int unique_indirect_reg_count = 0;
2819
2820 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2821 int indirect_start_offsets_count = 0;
2822
2823 int list_size = 0;
2824 int i = 0, j = 0;
2825 u32 tmp = 0;
2826
2827 u32 *register_list_format =
2828 kmemdup(adev->gfx.rlc.register_list_format,
2829 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2830 if (!register_list_format)
2831 return -ENOMEM;
2832
2833 /* setup unique_indirect_regs array and indirect_start_offsets array */
2834 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2835 gfx_v9_1_parse_ind_reg_list(register_list_format,
2836 adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2837 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2838 unique_indirect_regs,
2839 unique_indirect_reg_count,
2840 indirect_start_offsets,
2841 &indirect_start_offsets_count,
2842 ARRAY_SIZE(indirect_start_offsets));
2843
2844 /* enable auto inc in case it is disabled */
2845 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2846 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2847 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2848
2849 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2850 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2851 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2852 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2853 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2854 adev->gfx.rlc.register_restore[i]);
2855
2856 /* load indirect register */
2857 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2858 adev->gfx.rlc.reg_list_format_start);
2859
2860 /* direct register portion */
2861 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2862 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2863 register_list_format[i]);
2864
2865 /* indirect register portion */
2866 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2867 if (register_list_format[i] == 0xFFFFFFFF) {
2868 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2869 continue;
2870 }
2871
2872 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2873 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2874
2875 for (j = 0; j < unique_indirect_reg_count; j++) {
2876 if (register_list_format[i] == unique_indirect_regs[j]) {
2877 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2878 break;
2879 }
2880 }
2881
2882 BUG_ON(j >= unique_indirect_reg_count);
2883
2884 i++;
2885 }
2886
2887 /* set save/restore list size */
2888 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2889 list_size = list_size >> 1;
2890 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2891 adev->gfx.rlc.reg_restore_list_size);
2892 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2893
2894 /* write the starting offsets to RLC scratch ram */
2895 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2896 adev->gfx.rlc.starting_offsets_start);
2897 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2898 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2899 indirect_start_offsets[i]);
2900
2901 /* load unique indirect regs*/
2902 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2903 if (unique_indirect_regs[i] != 0) {
2904 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2905 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2906 unique_indirect_regs[i] & 0x3FFFF);
2907
2908 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2909 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2910 unique_indirect_regs[i] >> 20);
2911 }
2912 }
2913
2914 kfree(register_list_format);
2915 return 0;
2916 }
2917
gfx_v9_0_enable_save_restore_machine(struct amdgpu_device * adev)2918 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2919 {
2920 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2921 }
2922
pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device * adev,bool enable)2923 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2924 bool enable)
2925 {
2926 uint32_t data = 0;
2927 uint32_t default_data = 0;
2928
2929 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2930 if (enable) {
2931 /* enable GFXIP control over CGPG */
2932 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2933 if(default_data != data)
2934 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2935
2936 /* update status */
2937 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2938 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2939 if(default_data != data)
2940 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2941 } else {
2942 /* restore GFXIP control over GCPG */
2943 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2944 if(default_data != data)
2945 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2946 }
2947 }
2948
gfx_v9_0_init_gfx_power_gating(struct amdgpu_device * adev)2949 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2950 {
2951 uint32_t data = 0;
2952
2953 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2954 AMD_PG_SUPPORT_GFX_SMG |
2955 AMD_PG_SUPPORT_GFX_DMG)) {
2956 /* init IDLE_POLL_COUNT = 60 */
2957 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2958 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2959 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2960 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2961
2962 /* init RLC PG Delay */
2963 data = 0;
2964 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2965 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2966 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2967 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2968 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2969
2970 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2971 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2972 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2973 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2974
2975 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2976 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2977 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2978 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2979
2980 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2981 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2982
2983 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2984 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2985 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2986 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0))
2987 pwr_10_0_gfxip_control_over_cgpg(adev, true);
2988 }
2989 }
2990
gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device * adev,bool enable)2991 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2992 bool enable)
2993 {
2994 uint32_t data = 0;
2995 uint32_t default_data = 0;
2996
2997 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2998 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2999 SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
3000 enable ? 1 : 0);
3001 if (default_data != data)
3002 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3003 }
3004
gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device * adev,bool enable)3005 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
3006 bool enable)
3007 {
3008 uint32_t data = 0;
3009 uint32_t default_data = 0;
3010
3011 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3012 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3013 SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
3014 enable ? 1 : 0);
3015 if(default_data != data)
3016 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3017 }
3018
gfx_v9_0_enable_cp_power_gating(struct amdgpu_device * adev,bool enable)3019 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
3020 bool enable)
3021 {
3022 uint32_t data = 0;
3023 uint32_t default_data = 0;
3024
3025 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3026 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3027 CP_PG_DISABLE,
3028 enable ? 0 : 1);
3029 if(default_data != data)
3030 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3031 }
3032
gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)3033 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
3034 bool enable)
3035 {
3036 uint32_t data, default_data;
3037
3038 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3039 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3040 GFX_POWER_GATING_ENABLE,
3041 enable ? 1 : 0);
3042 if(default_data != data)
3043 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3044 }
3045
gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device * adev,bool enable)3046 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
3047 bool enable)
3048 {
3049 uint32_t data, default_data;
3050
3051 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3052 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3053 GFX_PIPELINE_PG_ENABLE,
3054 enable ? 1 : 0);
3055 if(default_data != data)
3056 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3057
3058 if (!enable)
3059 /* read any GFX register to wake up GFX */
3060 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3061 }
3062
gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device * adev,bool enable)3063 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3064 bool enable)
3065 {
3066 uint32_t data, default_data;
3067
3068 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3069 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3070 STATIC_PER_CU_PG_ENABLE,
3071 enable ? 1 : 0);
3072 if(default_data != data)
3073 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3074 }
3075
gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device * adev,bool enable)3076 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3077 bool enable)
3078 {
3079 uint32_t data, default_data;
3080
3081 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3082 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3083 DYN_PER_CU_PG_ENABLE,
3084 enable ? 1 : 0);
3085 if(default_data != data)
3086 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3087 }
3088
gfx_v9_0_init_pg(struct amdgpu_device * adev)3089 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3090 {
3091 gfx_v9_0_init_csb(adev);
3092
3093 /*
3094 * Rlc save restore list is workable since v2_1.
3095 * And it's needed by gfxoff feature.
3096 */
3097 if (adev->gfx.rlc.is_rlc_v2_1) {
3098 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3099 IP_VERSION(9, 2, 1) ||
3100 (adev->apu_flags & AMD_APU_IS_RAVEN2))
3101 gfx_v9_1_init_rlc_save_restore_list(adev);
3102 gfx_v9_0_enable_save_restore_machine(adev);
3103 }
3104
3105 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3106 AMD_PG_SUPPORT_GFX_SMG |
3107 AMD_PG_SUPPORT_GFX_DMG |
3108 AMD_PG_SUPPORT_CP |
3109 AMD_PG_SUPPORT_GDS |
3110 AMD_PG_SUPPORT_RLC_SMU_HS)) {
3111 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
3112 adev->gfx.rlc.cp_table_gpu_addr >> 8);
3113 gfx_v9_0_init_gfx_power_gating(adev);
3114 }
3115 }
3116
gfx_v9_0_rlc_stop(struct amdgpu_device * adev)3117 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3118 {
3119 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3120 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3121 gfx_v9_0_wait_for_rlc_serdes(adev);
3122 }
3123
gfx_v9_0_rlc_reset(struct amdgpu_device * adev)3124 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3125 {
3126 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3127 udelay(50);
3128 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3129 udelay(50);
3130 }
3131
gfx_v9_0_rlc_start(struct amdgpu_device * adev)3132 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3133 {
3134 #ifdef AMDGPU_RLC_DEBUG_RETRY
3135 u32 rlc_ucode_ver;
3136 #endif
3137
3138 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3139 udelay(50);
3140
3141 /* carrizo do enable cp interrupt after cp inited */
3142 if (!(adev->flags & AMD_IS_APU)) {
3143 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3144 udelay(50);
3145 }
3146
3147 #ifdef AMDGPU_RLC_DEBUG_RETRY
3148 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3149 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3150 if(rlc_ucode_ver == 0x108) {
3151 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3152 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3153 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3154 * default is 0x9C4 to create a 100us interval */
3155 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3156 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3157 * to disable the page fault retry interrupts, default is
3158 * 0x100 (256) */
3159 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3160 }
3161 #endif
3162 }
3163
gfx_v9_0_rlc_load_microcode(struct amdgpu_device * adev)3164 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3165 {
3166 const struct rlc_firmware_header_v2_0 *hdr;
3167 const __le32 *fw_data;
3168 unsigned i, fw_size;
3169
3170 if (!adev->gfx.rlc_fw)
3171 return -EINVAL;
3172
3173 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3174 amdgpu_ucode_print_rlc_hdr(&hdr->header);
3175
3176 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3177 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3178 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3179
3180 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3181 RLCG_UCODE_LOADING_START_ADDRESS);
3182 for (i = 0; i < fw_size; i++)
3183 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3184 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3185
3186 return 0;
3187 }
3188
gfx_v9_0_rlc_resume(struct amdgpu_device * adev)3189 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3190 {
3191 int r;
3192
3193 if (amdgpu_sriov_vf(adev)) {
3194 gfx_v9_0_init_csb(adev);
3195 return 0;
3196 }
3197
3198 adev->gfx.rlc.funcs->stop(adev);
3199
3200 /* disable CG */
3201 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3202
3203 gfx_v9_0_init_pg(adev);
3204
3205 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3206 /* legacy rlc firmware loading */
3207 r = gfx_v9_0_rlc_load_microcode(adev);
3208 if (r)
3209 return r;
3210 }
3211
3212 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3213 case IP_VERSION(9, 2, 2):
3214 case IP_VERSION(9, 1, 0):
3215 gfx_v9_0_init_lbpw(adev);
3216 if (amdgpu_lbpw == 0)
3217 gfx_v9_0_enable_lbpw(adev, false);
3218 else
3219 gfx_v9_0_enable_lbpw(adev, true);
3220 break;
3221 case IP_VERSION(9, 4, 0):
3222 gfx_v9_4_init_lbpw(adev);
3223 if (amdgpu_lbpw > 0)
3224 gfx_v9_0_enable_lbpw(adev, true);
3225 else
3226 gfx_v9_0_enable_lbpw(adev, false);
3227 break;
3228 default:
3229 break;
3230 }
3231
3232 gfx_v9_0_update_spm_vmid_internal(adev, 0xf);
3233
3234 adev->gfx.rlc.funcs->start(adev);
3235
3236 return 0;
3237 }
3238
gfx_v9_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)3239 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3240 {
3241 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3242
3243 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_INVALIDATE_ICACHE, enable ? 0 : 1);
3244 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_INVALIDATE_ICACHE, enable ? 0 : 1);
3245 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_INVALIDATE_ICACHE, enable ? 0 : 1);
3246 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE0_RESET, enable ? 0 : 1);
3247 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE1_RESET, enable ? 0 : 1);
3248 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, enable ? 0 : 1);
3249 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, enable ? 0 : 1);
3250 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, enable ? 0 : 1);
3251 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, enable ? 0 : 1);
3252 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3253 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3254 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3255 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3256 udelay(50);
3257 }
3258
gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3259 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3260 {
3261 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3262 const struct gfx_firmware_header_v1_0 *ce_hdr;
3263 const struct gfx_firmware_header_v1_0 *me_hdr;
3264 const __le32 *fw_data;
3265 unsigned i, fw_size;
3266
3267 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3268 return -EINVAL;
3269
3270 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3271 adev->gfx.pfp_fw->data;
3272 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3273 adev->gfx.ce_fw->data;
3274 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3275 adev->gfx.me_fw->data;
3276
3277 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3278 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3279 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3280
3281 gfx_v9_0_cp_gfx_enable(adev, false);
3282
3283 /* PFP */
3284 fw_data = (const __le32 *)
3285 (adev->gfx.pfp_fw->data +
3286 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3287 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3288 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3289 for (i = 0; i < fw_size; i++)
3290 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3291 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3292
3293 /* CE */
3294 fw_data = (const __le32 *)
3295 (adev->gfx.ce_fw->data +
3296 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3297 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3298 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3299 for (i = 0; i < fw_size; i++)
3300 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3301 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3302
3303 /* ME */
3304 fw_data = (const __le32 *)
3305 (adev->gfx.me_fw->data +
3306 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3307 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3308 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3309 for (i = 0; i < fw_size; i++)
3310 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3311 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3312
3313 return 0;
3314 }
3315
gfx_v9_0_cp_gfx_start(struct amdgpu_device * adev)3316 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3317 {
3318 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3319 const struct cs_section_def *sect = NULL;
3320 const struct cs_extent_def *ext = NULL;
3321 int r, i, tmp;
3322
3323 /* init the CP */
3324 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3325 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3326
3327 gfx_v9_0_cp_gfx_enable(adev, true);
3328
3329 /* Now only limit the quirk on the APU gfx9 series and already
3330 * confirmed that the APU gfx10/gfx11 needn't such update.
3331 */
3332 if (adev->flags & AMD_IS_APU &&
3333 adev->in_s3 && !pm_resume_via_firmware()) {
3334 DRM_INFO("Will skip the CSB packet resubmit\n");
3335 return 0;
3336 }
3337 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3338 if (r) {
3339 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3340 return r;
3341 }
3342
3343 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3344 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3345
3346 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3347 amdgpu_ring_write(ring, 0x80000000);
3348 amdgpu_ring_write(ring, 0x80000000);
3349
3350 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3351 for (ext = sect->section; ext->extent != NULL; ++ext) {
3352 if (sect->id == SECT_CONTEXT) {
3353 amdgpu_ring_write(ring,
3354 PACKET3(PACKET3_SET_CONTEXT_REG,
3355 ext->reg_count));
3356 amdgpu_ring_write(ring,
3357 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3358 for (i = 0; i < ext->reg_count; i++)
3359 amdgpu_ring_write(ring, ext->extent[i]);
3360 }
3361 }
3362 }
3363
3364 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3365 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3366
3367 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3368 amdgpu_ring_write(ring, 0);
3369
3370 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3371 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3372 amdgpu_ring_write(ring, 0x8000);
3373 amdgpu_ring_write(ring, 0x8000);
3374
3375 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3376 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3377 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3378 amdgpu_ring_write(ring, tmp);
3379 amdgpu_ring_write(ring, 0);
3380
3381 amdgpu_ring_commit(ring);
3382
3383 return 0;
3384 }
3385
gfx_v9_0_cp_gfx_resume(struct amdgpu_device * adev)3386 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3387 {
3388 struct amdgpu_ring *ring;
3389 u32 tmp;
3390 u32 rb_bufsz;
3391 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3392
3393 /* Set the write pointer delay */
3394 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3395
3396 /* set the RB to use vmid 0 */
3397 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3398
3399 /* Set ring buffer size */
3400 ring = &adev->gfx.gfx_ring[0];
3401 rb_bufsz = order_base_2(ring->ring_size / 8);
3402 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3403 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3404 #ifdef __BIG_ENDIAN
3405 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3406 #endif
3407 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3408
3409 /* Initialize the ring buffer's write pointers */
3410 ring->wptr = 0;
3411 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3412 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3413
3414 /* set the wb address whether it's enabled or not */
3415 rptr_addr = ring->rptr_gpu_addr;
3416 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3417 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3418
3419 wptr_gpu_addr = ring->wptr_gpu_addr;
3420 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3421 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3422
3423 mdelay(1);
3424 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3425
3426 rb_addr = ring->gpu_addr >> 8;
3427 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3428 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3429
3430 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3431 if (ring->use_doorbell) {
3432 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3433 DOORBELL_OFFSET, ring->doorbell_index);
3434 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3435 DOORBELL_EN, 1);
3436 } else {
3437 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3438 }
3439 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3440
3441 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3442 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3443 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3444
3445 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3446 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3447
3448
3449 /* start the ring */
3450 gfx_v9_0_cp_gfx_start(adev);
3451
3452 return 0;
3453 }
3454
gfx_v9_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3455 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3456 {
3457 if (enable) {
3458 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3459 } else {
3460 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3461 (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
3462 CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
3463 CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
3464 CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
3465 CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
3466 CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
3467 CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
3468 CP_MEC_CNTL__MEC_ME1_HALT_MASK |
3469 CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3470 adev->gfx.kiq[0].ring.sched.ready = false;
3471 }
3472 udelay(50);
3473 }
3474
gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device * adev)3475 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3476 {
3477 const struct gfx_firmware_header_v1_0 *mec_hdr;
3478 const __le32 *fw_data;
3479 unsigned i;
3480 u32 tmp;
3481
3482 if (!adev->gfx.mec_fw)
3483 return -EINVAL;
3484
3485 gfx_v9_0_cp_compute_enable(adev, false);
3486
3487 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3488 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3489
3490 fw_data = (const __le32 *)
3491 (adev->gfx.mec_fw->data +
3492 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3493 tmp = 0;
3494 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3495 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3496 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3497
3498 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3499 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3500 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3501 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3502
3503 /* MEC1 */
3504 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3505 mec_hdr->jt_offset);
3506 for (i = 0; i < mec_hdr->jt_size; i++)
3507 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3508 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3509
3510 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3511 adev->gfx.mec_fw_version);
3512 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3513
3514 return 0;
3515 }
3516
3517 /* KIQ functions */
gfx_v9_0_kiq_setting(struct amdgpu_ring * ring)3518 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3519 {
3520 uint32_t tmp;
3521 struct amdgpu_device *adev = ring->adev;
3522
3523 /* tell RLC which is KIQ queue */
3524 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3525 tmp &= 0xffffff00;
3526 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3527 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp | 0x80);
3528 }
3529
gfx_v9_0_mqd_set_priority(struct amdgpu_ring * ring,struct v9_mqd * mqd)3530 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3531 {
3532 struct amdgpu_device *adev = ring->adev;
3533
3534 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3535 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3536 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3537 mqd->cp_hqd_queue_priority =
3538 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3539 }
3540 }
3541 }
3542
gfx_v9_0_mqd_init(struct amdgpu_ring * ring)3543 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3544 {
3545 struct amdgpu_device *adev = ring->adev;
3546 struct v9_mqd *mqd = ring->mqd_ptr;
3547 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3548 uint32_t tmp;
3549
3550 mqd->header = 0xC0310800;
3551 mqd->compute_pipelinestat_enable = 0x00000001;
3552 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3553 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3554 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3555 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3556 mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3557 mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3558 mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3559 mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3560 mqd->compute_misc_reserved = 0x00000003;
3561
3562 mqd->dynamic_cu_mask_addr_lo =
3563 lower_32_bits(ring->mqd_gpu_addr
3564 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3565 mqd->dynamic_cu_mask_addr_hi =
3566 upper_32_bits(ring->mqd_gpu_addr
3567 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3568
3569 eop_base_addr = ring->eop_gpu_addr >> 8;
3570 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3571 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3572
3573 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3574 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3575 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3576 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3577
3578 mqd->cp_hqd_eop_control = tmp;
3579
3580 /* enable doorbell? */
3581 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3582
3583 if (ring->use_doorbell) {
3584 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3585 DOORBELL_OFFSET, ring->doorbell_index);
3586 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3587 DOORBELL_EN, 1);
3588 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3589 DOORBELL_SOURCE, 0);
3590 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3591 DOORBELL_HIT, 0);
3592 } else {
3593 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3594 DOORBELL_EN, 0);
3595 }
3596
3597 mqd->cp_hqd_pq_doorbell_control = tmp;
3598
3599 /* disable the queue if it's active */
3600 ring->wptr = 0;
3601 mqd->cp_hqd_dequeue_request = 0;
3602 mqd->cp_hqd_pq_rptr = 0;
3603 mqd->cp_hqd_pq_wptr_lo = 0;
3604 mqd->cp_hqd_pq_wptr_hi = 0;
3605
3606 /* set the pointer to the MQD */
3607 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3608 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3609
3610 /* set MQD vmid to 0 */
3611 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3612 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3613 mqd->cp_mqd_control = tmp;
3614
3615 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3616 hqd_gpu_addr = ring->gpu_addr >> 8;
3617 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3618 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3619
3620 /* set up the HQD, this is similar to CP_RB0_CNTL */
3621 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3622 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3623 (order_base_2(ring->ring_size / 4) - 1));
3624 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3625 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3626 #ifdef __BIG_ENDIAN
3627 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3628 #endif
3629 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3630 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3631 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3632 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3633 mqd->cp_hqd_pq_control = tmp;
3634
3635 /* set the wb address whether it's enabled or not */
3636 wb_gpu_addr = ring->rptr_gpu_addr;
3637 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3638 mqd->cp_hqd_pq_rptr_report_addr_hi =
3639 upper_32_bits(wb_gpu_addr) & 0xffff;
3640
3641 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3642 wb_gpu_addr = ring->wptr_gpu_addr;
3643 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3644 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3645
3646 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3647 ring->wptr = 0;
3648 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3649
3650 /* set the vmid for the queue */
3651 mqd->cp_hqd_vmid = 0;
3652
3653 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3654 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3655 mqd->cp_hqd_persistent_state = tmp;
3656
3657 /* set MIN_IB_AVAIL_SIZE */
3658 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3659 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3660 mqd->cp_hqd_ib_control = tmp;
3661
3662 /* set static priority for a queue/ring */
3663 gfx_v9_0_mqd_set_priority(ring, mqd);
3664 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3665
3666 /* map_queues packet doesn't need activate the queue,
3667 * so only kiq need set this field.
3668 */
3669 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3670 mqd->cp_hqd_active = 1;
3671
3672 return 0;
3673 }
3674
gfx_v9_0_kiq_init_register(struct amdgpu_ring * ring)3675 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3676 {
3677 struct amdgpu_device *adev = ring->adev;
3678 struct v9_mqd *mqd = ring->mqd_ptr;
3679 int j;
3680
3681 /* disable wptr polling */
3682 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3683
3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3685 mqd->cp_hqd_eop_base_addr_lo);
3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3687 mqd->cp_hqd_eop_base_addr_hi);
3688
3689 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3691 mqd->cp_hqd_eop_control);
3692
3693 /* enable doorbell? */
3694 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3695 mqd->cp_hqd_pq_doorbell_control);
3696
3697 /* disable the queue if it's active */
3698 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3699 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3700 for (j = 0; j < adev->usec_timeout; j++) {
3701 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3702 break;
3703 udelay(1);
3704 }
3705 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3706 mqd->cp_hqd_dequeue_request);
3707 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3708 mqd->cp_hqd_pq_rptr);
3709 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3710 mqd->cp_hqd_pq_wptr_lo);
3711 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3712 mqd->cp_hqd_pq_wptr_hi);
3713 }
3714
3715 /* set the pointer to the MQD */
3716 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3717 mqd->cp_mqd_base_addr_lo);
3718 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3719 mqd->cp_mqd_base_addr_hi);
3720
3721 /* set MQD vmid to 0 */
3722 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3723 mqd->cp_mqd_control);
3724
3725 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3726 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3727 mqd->cp_hqd_pq_base_lo);
3728 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3729 mqd->cp_hqd_pq_base_hi);
3730
3731 /* set up the HQD, this is similar to CP_RB0_CNTL */
3732 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3733 mqd->cp_hqd_pq_control);
3734
3735 /* set the wb address whether it's enabled or not */
3736 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3737 mqd->cp_hqd_pq_rptr_report_addr_lo);
3738 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3739 mqd->cp_hqd_pq_rptr_report_addr_hi);
3740
3741 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3742 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3743 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3744 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3745 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3746
3747 /* enable the doorbell if requested */
3748 if (ring->use_doorbell) {
3749 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3750 (adev->doorbell_index.kiq * 2) << 2);
3751 /* If GC has entered CGPG, ringing doorbell > first page
3752 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3753 * workaround this issue. And this change has to align with firmware
3754 * update.
3755 */
3756 if (check_if_enlarge_doorbell_range(adev))
3757 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3758 (adev->doorbell.size - 4));
3759 else
3760 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3761 (adev->doorbell_index.userqueue_end * 2) << 2);
3762 }
3763
3764 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3765 mqd->cp_hqd_pq_doorbell_control);
3766
3767 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3768 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3769 mqd->cp_hqd_pq_wptr_lo);
3770 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3771 mqd->cp_hqd_pq_wptr_hi);
3772
3773 /* set the vmid for the queue */
3774 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3775
3776 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3777 mqd->cp_hqd_persistent_state);
3778
3779 /* activate the queue */
3780 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3781 mqd->cp_hqd_active);
3782
3783 if (ring->use_doorbell)
3784 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3785
3786 return 0;
3787 }
3788
gfx_v9_0_kiq_fini_register(struct amdgpu_ring * ring)3789 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3790 {
3791 struct amdgpu_device *adev = ring->adev;
3792 int j;
3793
3794 /* disable the queue if it's active */
3795 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3796
3797 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3798
3799 for (j = 0; j < adev->usec_timeout; j++) {
3800 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3801 break;
3802 udelay(1);
3803 }
3804
3805 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3806 DRM_DEBUG("KIQ dequeue request failed.\n");
3807
3808 /* Manual disable if dequeue request times out */
3809 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3810 }
3811
3812 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3813 0);
3814 }
3815
3816 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3817 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3818 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3819 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3820 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3821 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3822 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3823 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3824
3825 return 0;
3826 }
3827
gfx_v9_0_kiq_init_queue(struct amdgpu_ring * ring)3828 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3829 {
3830 struct amdgpu_device *adev = ring->adev;
3831 struct v9_mqd *mqd = ring->mqd_ptr;
3832 struct v9_mqd *tmp_mqd;
3833
3834 gfx_v9_0_kiq_setting(ring);
3835
3836 /* GPU could be in bad state during probe, driver trigger the reset
3837 * after load the SMU, in this case , the mqd is not be initialized.
3838 * driver need to re-init the mqd.
3839 * check mqd->cp_hqd_pq_control since this value should not be 0
3840 */
3841 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3842 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3843 /* for GPU_RESET case , reset MQD to a clean status */
3844 if (adev->gfx.kiq[0].mqd_backup)
3845 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3846
3847 /* reset ring buffer */
3848 ring->wptr = 0;
3849 amdgpu_ring_clear_ring(ring);
3850
3851 mutex_lock(&adev->srbm_mutex);
3852 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3853 gfx_v9_0_kiq_init_register(ring);
3854 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3855 mutex_unlock(&adev->srbm_mutex);
3856 } else {
3857 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3858 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3859 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3860 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3861 amdgpu_ring_clear_ring(ring);
3862 mutex_lock(&adev->srbm_mutex);
3863 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3864 gfx_v9_0_mqd_init(ring);
3865 gfx_v9_0_kiq_init_register(ring);
3866 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3867 mutex_unlock(&adev->srbm_mutex);
3868
3869 if (adev->gfx.kiq[0].mqd_backup)
3870 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3871 }
3872
3873 return 0;
3874 }
3875
gfx_v9_0_kcq_init_queue(struct amdgpu_ring * ring,bool restore)3876 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)
3877 {
3878 struct amdgpu_device *adev = ring->adev;
3879 struct v9_mqd *mqd = ring->mqd_ptr;
3880 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3881 struct v9_mqd *tmp_mqd;
3882
3883 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3884 * is not be initialized before
3885 */
3886 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3887
3888 if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
3889 (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
3890 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3891 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3892 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3893 mutex_lock(&adev->srbm_mutex);
3894 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3895 gfx_v9_0_mqd_init(ring);
3896 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3897 mutex_unlock(&adev->srbm_mutex);
3898
3899 if (adev->gfx.mec.mqd_backup[mqd_idx])
3900 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3901 } else {
3902 /* restore MQD to a clean status */
3903 if (adev->gfx.mec.mqd_backup[mqd_idx])
3904 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3905 /* reset ring buffer */
3906 ring->wptr = 0;
3907 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3908 amdgpu_ring_clear_ring(ring);
3909 }
3910
3911 return 0;
3912 }
3913
gfx_v9_0_kiq_resume(struct amdgpu_device * adev)3914 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3915 {
3916 gfx_v9_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
3917 return 0;
3918 }
3919
gfx_v9_0_kcq_resume(struct amdgpu_device * adev)3920 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3921 {
3922 int i, r;
3923
3924 gfx_v9_0_cp_compute_enable(adev, true);
3925
3926 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3927 r = gfx_v9_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
3928 if (r)
3929 return r;
3930 }
3931
3932 return amdgpu_gfx_enable_kcq(adev, 0);
3933 }
3934
gfx_v9_0_cp_resume(struct amdgpu_device * adev)3935 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3936 {
3937 int r, i;
3938 struct amdgpu_ring *ring;
3939
3940 if (!(adev->flags & AMD_IS_APU))
3941 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3942
3943 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3944 if (adev->gfx.num_gfx_rings) {
3945 /* legacy firmware loading */
3946 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3947 if (r)
3948 return r;
3949 }
3950
3951 r = gfx_v9_0_cp_compute_load_microcode(adev);
3952 if (r)
3953 return r;
3954 }
3955
3956 if (adev->gfx.num_gfx_rings)
3957 gfx_v9_0_cp_gfx_enable(adev, false);
3958 gfx_v9_0_cp_compute_enable(adev, false);
3959
3960 r = gfx_v9_0_kiq_resume(adev);
3961 if (r)
3962 return r;
3963
3964 if (adev->gfx.num_gfx_rings) {
3965 r = gfx_v9_0_cp_gfx_resume(adev);
3966 if (r)
3967 return r;
3968 }
3969
3970 r = gfx_v9_0_kcq_resume(adev);
3971 if (r)
3972 return r;
3973
3974 if (adev->gfx.num_gfx_rings) {
3975 ring = &adev->gfx.gfx_ring[0];
3976 r = amdgpu_ring_test_helper(ring);
3977 if (r)
3978 return r;
3979 }
3980
3981 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3982 ring = &adev->gfx.compute_ring[i];
3983 amdgpu_ring_test_helper(ring);
3984 }
3985
3986 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3987
3988 return 0;
3989 }
3990
gfx_v9_0_init_tcp_config(struct amdgpu_device * adev)3991 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3992 {
3993 u32 tmp;
3994
3995 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&
3996 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
3997 return;
3998
3999 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
4000 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
4001 adev->df.hash_status.hash_64k);
4002 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
4003 adev->df.hash_status.hash_2m);
4004 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
4005 adev->df.hash_status.hash_1g);
4006 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
4007 }
4008
gfx_v9_0_cp_enable(struct amdgpu_device * adev,bool enable)4009 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
4010 {
4011 if (adev->gfx.num_gfx_rings)
4012 gfx_v9_0_cp_gfx_enable(adev, enable);
4013 gfx_v9_0_cp_compute_enable(adev, enable);
4014 }
4015
gfx_v9_0_hw_init(struct amdgpu_ip_block * ip_block)4016 static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
4017 {
4018 int r;
4019 struct amdgpu_device *adev = ip_block->adev;
4020
4021 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4022 adev->gfx.cleaner_shader_ptr);
4023
4024 if (!amdgpu_sriov_vf(adev))
4025 gfx_v9_0_init_golden_registers(adev);
4026
4027 gfx_v9_0_constants_init(adev);
4028
4029 gfx_v9_0_init_tcp_config(adev);
4030
4031 r = adev->gfx.rlc.funcs->resume(adev);
4032 if (r)
4033 return r;
4034
4035 r = gfx_v9_0_cp_resume(adev);
4036 if (r)
4037 return r;
4038
4039 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) &&
4040 !amdgpu_sriov_vf(adev))
4041 gfx_v9_4_2_set_power_brake_sequence(adev);
4042
4043 return r;
4044 }
4045
gfx_v9_0_hw_fini(struct amdgpu_ip_block * ip_block)4046 static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
4047 {
4048 struct amdgpu_device *adev = ip_block->adev;
4049
4050 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4051 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4052 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4053 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4054 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4055
4056 /* DF freeze and kcq disable will fail */
4057 if (!amdgpu_ras_intr_triggered())
4058 /* disable KCQ to avoid CPC touch memory not valid anymore */
4059 amdgpu_gfx_disable_kcq(adev, 0);
4060
4061 if (amdgpu_sriov_vf(adev)) {
4062 gfx_v9_0_cp_gfx_enable(adev, false);
4063 /* must disable polling for SRIOV when hw finished, otherwise
4064 * CPC engine may still keep fetching WB address which is already
4065 * invalid after sw finished and trigger DMAR reading error in
4066 * hypervisor side.
4067 */
4068 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4069 return 0;
4070 }
4071
4072 /* Use deinitialize sequence from CAIL when unbinding device from driver,
4073 * otherwise KIQ is hanging when binding back
4074 */
4075 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4076 mutex_lock(&adev->srbm_mutex);
4077 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,
4078 adev->gfx.kiq[0].ring.pipe,
4079 adev->gfx.kiq[0].ring.queue, 0, 0);
4080 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);
4081 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4082 mutex_unlock(&adev->srbm_mutex);
4083 }
4084
4085 gfx_v9_0_cp_enable(adev, false);
4086
4087 /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
4088 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
4089 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {
4090 dev_dbg(adev->dev, "Skipping RLC halt\n");
4091 return 0;
4092 }
4093
4094 adev->gfx.rlc.funcs->stop(adev);
4095 return 0;
4096 }
4097
gfx_v9_0_suspend(struct amdgpu_ip_block * ip_block)4098 static int gfx_v9_0_suspend(struct amdgpu_ip_block *ip_block)
4099 {
4100 return gfx_v9_0_hw_fini(ip_block);
4101 }
4102
gfx_v9_0_resume(struct amdgpu_ip_block * ip_block)4103 static int gfx_v9_0_resume(struct amdgpu_ip_block *ip_block)
4104 {
4105 return gfx_v9_0_hw_init(ip_block);
4106 }
4107
gfx_v9_0_is_idle(struct amdgpu_ip_block * ip_block)4108 static bool gfx_v9_0_is_idle(struct amdgpu_ip_block *ip_block)
4109 {
4110 struct amdgpu_device *adev = ip_block->adev;
4111
4112 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4113 GRBM_STATUS, GUI_ACTIVE))
4114 return false;
4115 else
4116 return true;
4117 }
4118
gfx_v9_0_wait_for_idle(struct amdgpu_ip_block * ip_block)4119 static int gfx_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4120 {
4121 unsigned i;
4122 struct amdgpu_device *adev = ip_block->adev;
4123
4124 for (i = 0; i < adev->usec_timeout; i++) {
4125 if (gfx_v9_0_is_idle(ip_block))
4126 return 0;
4127 udelay(1);
4128 }
4129 return -ETIMEDOUT;
4130 }
4131
gfx_v9_0_soft_reset(struct amdgpu_ip_block * ip_block)4132 static int gfx_v9_0_soft_reset(struct amdgpu_ip_block *ip_block)
4133 {
4134 u32 grbm_soft_reset = 0;
4135 u32 tmp;
4136 struct amdgpu_device *adev = ip_block->adev;
4137
4138 /* GRBM_STATUS */
4139 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4140 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4141 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4142 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4143 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4144 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4145 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4146 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4147 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4148 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4149 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4150 }
4151
4152 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4153 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4154 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4155 }
4156
4157 /* GRBM_STATUS2 */
4158 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4159 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4160 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4161 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4162
4163
4164 if (grbm_soft_reset) {
4165 /* stop the rlc */
4166 adev->gfx.rlc.funcs->stop(adev);
4167
4168 if (adev->gfx.num_gfx_rings)
4169 /* Disable GFX parsing/prefetching */
4170 gfx_v9_0_cp_gfx_enable(adev, false);
4171
4172 /* Disable MEC parsing/prefetching */
4173 gfx_v9_0_cp_compute_enable(adev, false);
4174
4175 if (grbm_soft_reset) {
4176 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4177 tmp |= grbm_soft_reset;
4178 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4179 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4180 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4181
4182 udelay(50);
4183
4184 tmp &= ~grbm_soft_reset;
4185 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4186 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4187 }
4188
4189 /* Wait a little for things to settle down */
4190 udelay(50);
4191 }
4192 return 0;
4193 }
4194
gfx_v9_0_kiq_read_clock(struct amdgpu_device * adev)4195 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4196 {
4197 signed long r, cnt = 0;
4198 unsigned long flags;
4199 uint32_t seq, reg_val_offs = 0;
4200 uint64_t value = 0;
4201 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4202 struct amdgpu_ring *ring = &kiq->ring;
4203
4204 BUG_ON(!ring->funcs->emit_rreg);
4205
4206 spin_lock_irqsave(&kiq->ring_lock, flags);
4207 if (amdgpu_device_wb_get(adev, ®_val_offs)) {
4208 pr_err("critical bug! too many kiq readers\n");
4209 goto failed_unlock;
4210 }
4211 amdgpu_ring_alloc(ring, 32);
4212 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4213 amdgpu_ring_write(ring, 9 | /* src: register*/
4214 (5 << 8) | /* dst: memory */
4215 (1 << 16) | /* count sel */
4216 (1 << 20)); /* write confirm */
4217 amdgpu_ring_write(ring, 0);
4218 amdgpu_ring_write(ring, 0);
4219 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4220 reg_val_offs * 4));
4221 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4222 reg_val_offs * 4));
4223 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4224 if (r)
4225 goto failed_undo;
4226
4227 amdgpu_ring_commit(ring);
4228 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4229
4230 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4231
4232 /* don't wait anymore for gpu reset case because this way may
4233 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4234 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4235 * never return if we keep waiting in virt_kiq_rreg, which cause
4236 * gpu_recover() hang there.
4237 *
4238 * also don't wait anymore for IRQ context
4239 * */
4240 if (r < 1 && (amdgpu_in_reset(adev)))
4241 goto failed_kiq_read;
4242
4243 might_sleep();
4244 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4245 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4246 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4247 }
4248
4249 if (cnt > MAX_KIQ_REG_TRY)
4250 goto failed_kiq_read;
4251
4252 mb();
4253 value = (uint64_t)adev->wb.wb[reg_val_offs] |
4254 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4255 amdgpu_device_wb_free(adev, reg_val_offs);
4256 return value;
4257
4258 failed_undo:
4259 amdgpu_ring_undo(ring);
4260 failed_unlock:
4261 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4262 failed_kiq_read:
4263 if (reg_val_offs)
4264 amdgpu_device_wb_free(adev, reg_val_offs);
4265 pr_err("failed to read gpu clock\n");
4266 return ~0;
4267 }
4268
gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device * adev)4269 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4270 {
4271 uint64_t clock, clock_lo, clock_hi, hi_check;
4272
4273 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
4274 case IP_VERSION(9, 3, 0):
4275 preempt_disable();
4276 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4277 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4278 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4279 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
4280 * roughly every 42 seconds.
4281 */
4282 if (hi_check != clock_hi) {
4283 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4284 clock_hi = hi_check;
4285 }
4286 preempt_enable();
4287 clock = clock_lo | (clock_hi << 32ULL);
4288 break;
4289 default:
4290 amdgpu_gfx_off_ctrl(adev, false);
4291 mutex_lock(&adev->gfx.gpu_clock_mutex);
4292 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
4293 IP_VERSION(9, 0, 1) &&
4294 amdgpu_sriov_runtime(adev)) {
4295 clock = gfx_v9_0_kiq_read_clock(adev);
4296 } else {
4297 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4298 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4299 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4300 }
4301 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4302 amdgpu_gfx_off_ctrl(adev, true);
4303 break;
4304 }
4305 return clock;
4306 }
4307
gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)4308 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4309 uint32_t vmid,
4310 uint32_t gds_base, uint32_t gds_size,
4311 uint32_t gws_base, uint32_t gws_size,
4312 uint32_t oa_base, uint32_t oa_size)
4313 {
4314 struct amdgpu_device *adev = ring->adev;
4315
4316 /* GDS Base */
4317 gfx_v9_0_write_data_to_reg(ring, 0, false,
4318 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4319 gds_base);
4320
4321 /* GDS Size */
4322 gfx_v9_0_write_data_to_reg(ring, 0, false,
4323 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4324 gds_size);
4325
4326 /* GWS */
4327 gfx_v9_0_write_data_to_reg(ring, 0, false,
4328 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4329 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4330
4331 /* OA */
4332 gfx_v9_0_write_data_to_reg(ring, 0, false,
4333 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4334 (1 << (oa_size + oa_base)) - (1 << oa_base));
4335 }
4336
4337 static const u32 vgpr_init_compute_shader[] =
4338 {
4339 0xb07c0000, 0xbe8000ff,
4340 0x000000f8, 0xbf110800,
4341 0x7e000280, 0x7e020280,
4342 0x7e040280, 0x7e060280,
4343 0x7e080280, 0x7e0a0280,
4344 0x7e0c0280, 0x7e0e0280,
4345 0x80808800, 0xbe803200,
4346 0xbf84fff5, 0xbf9c0000,
4347 0xd28c0001, 0x0001007f,
4348 0xd28d0001, 0x0002027e,
4349 0x10020288, 0xb8810904,
4350 0xb7814000, 0xd1196a01,
4351 0x00000301, 0xbe800087,
4352 0xbefc00c1, 0xd89c4000,
4353 0x00020201, 0xd89cc080,
4354 0x00040401, 0x320202ff,
4355 0x00000800, 0x80808100,
4356 0xbf84fff8, 0x7e020280,
4357 0xbf810000, 0x00000000,
4358 };
4359
4360 static const u32 sgpr_init_compute_shader[] =
4361 {
4362 0xb07c0000, 0xbe8000ff,
4363 0x0000005f, 0xbee50080,
4364 0xbe812c65, 0xbe822c65,
4365 0xbe832c65, 0xbe842c65,
4366 0xbe852c65, 0xb77c0005,
4367 0x80808500, 0xbf84fff8,
4368 0xbe800080, 0xbf810000,
4369 };
4370
4371 static const u32 vgpr_init_compute_shader_arcturus[] = {
4372 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4373 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4374 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4375 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4376 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4377 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4378 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4379 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4380 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4381 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4382 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4383 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4384 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4385 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4386 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4387 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4388 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4389 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4390 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4391 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4392 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4393 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4394 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4395 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4396 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4397 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4398 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4399 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4400 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4401 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4402 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4403 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4404 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4405 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4406 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4407 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4408 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4409 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4410 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4411 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4412 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4413 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4414 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4415 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4416 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4417 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4418 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4419 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4420 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4421 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4422 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4423 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4424 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4425 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4426 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4427 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4428 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4429 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4430 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4431 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4432 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4433 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4434 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4435 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4436 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4437 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4438 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4439 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4440 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4441 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4442 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4443 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4444 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4445 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4446 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4447 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4448 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4449 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4450 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4451 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4452 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4453 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4454 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4455 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4456 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4457 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4458 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4459 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4460 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4461 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4462 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4463 0xbf84fff8, 0xbf810000,
4464 };
4465
4466 /* When below register arrays changed, please update gpr_reg_size,
4467 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4468 to cover all gfx9 ASICs */
4469 static const struct soc15_reg_entry vgpr_init_regs[] = {
4470 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4471 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4472 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4473 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4474 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4475 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4476 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4477 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4478 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4479 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4480 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4481 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4482 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4483 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4484 };
4485
4486 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4487 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4488 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4489 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4490 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4491 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4492 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4493 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4494 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4495 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4496 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4497 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4498 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4499 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4500 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4501 };
4502
4503 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4504 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4505 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4506 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4507 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4508 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4509 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4510 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4511 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4512 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4513 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4514 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4515 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4516 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4517 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4518 };
4519
4520 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4521 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4522 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4523 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4524 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4525 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4526 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4527 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4528 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4529 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4530 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4531 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4532 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4533 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4534 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4535 };
4536
4537 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4538 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4539 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4540 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4541 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4542 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4543 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4544 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4545 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4546 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4547 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4548 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4549 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4550 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4551 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4552 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4553 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4554 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4555 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4556 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4557 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4558 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4559 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4560 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4561 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4562 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4563 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4564 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4565 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4566 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4567 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4568 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4569 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4570 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4571 };
4572
gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device * adev)4573 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4574 {
4575 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4576 int i, r;
4577
4578 /* only support when RAS is enabled */
4579 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4580 return 0;
4581
4582 r = amdgpu_ring_alloc(ring, 7);
4583 if (r) {
4584 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4585 ring->name, r);
4586 return r;
4587 }
4588
4589 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4590 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4591
4592 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4593 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4594 PACKET3_DMA_DATA_DST_SEL(1) |
4595 PACKET3_DMA_DATA_SRC_SEL(2) |
4596 PACKET3_DMA_DATA_ENGINE(0)));
4597 amdgpu_ring_write(ring, 0);
4598 amdgpu_ring_write(ring, 0);
4599 amdgpu_ring_write(ring, 0);
4600 amdgpu_ring_write(ring, 0);
4601 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4602 adev->gds.gds_size);
4603
4604 amdgpu_ring_commit(ring);
4605
4606 for (i = 0; i < adev->usec_timeout; i++) {
4607 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4608 break;
4609 udelay(1);
4610 }
4611
4612 if (i >= adev->usec_timeout)
4613 r = -ETIMEDOUT;
4614
4615 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4616
4617 return r;
4618 }
4619
gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device * adev)4620 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4621 {
4622 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4623 struct amdgpu_ib ib;
4624 struct dma_fence *f = NULL;
4625 int r, i;
4626 unsigned total_size, vgpr_offset, sgpr_offset;
4627 u64 gpu_addr;
4628
4629 int compute_dim_x = adev->gfx.config.max_shader_engines *
4630 adev->gfx.config.max_cu_per_sh *
4631 adev->gfx.config.max_sh_per_se;
4632 int sgpr_work_group_size = 5;
4633 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4634 int vgpr_init_shader_size;
4635 const u32 *vgpr_init_shader_ptr;
4636 const struct soc15_reg_entry *vgpr_init_regs_ptr;
4637
4638 /* only support when RAS is enabled */
4639 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4640 return 0;
4641
4642 /* bail if the compute ring is not ready */
4643 if (!ring->sched.ready)
4644 return 0;
4645
4646 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
4647 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4648 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4649 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4650 } else {
4651 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4652 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4653 vgpr_init_regs_ptr = vgpr_init_regs;
4654 }
4655
4656 total_size =
4657 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4658 total_size +=
4659 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4660 total_size +=
4661 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4662 total_size = ALIGN(total_size, 256);
4663 vgpr_offset = total_size;
4664 total_size += ALIGN(vgpr_init_shader_size, 256);
4665 sgpr_offset = total_size;
4666 total_size += sizeof(sgpr_init_compute_shader);
4667
4668 /* allocate an indirect buffer to put the commands in */
4669 memset(&ib, 0, sizeof(ib));
4670 r = amdgpu_ib_get(adev, NULL, total_size,
4671 AMDGPU_IB_POOL_DIRECT, &ib);
4672 if (r) {
4673 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4674 return r;
4675 }
4676
4677 /* load the compute shaders */
4678 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4679 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4680
4681 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4682 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4683
4684 /* init the ib length to 0 */
4685 ib.length_dw = 0;
4686
4687 /* VGPR */
4688 /* write the register state for the compute dispatch */
4689 for (i = 0; i < gpr_reg_size; i++) {
4690 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4691 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4692 - PACKET3_SET_SH_REG_START;
4693 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4694 }
4695 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4696 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4697 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4698 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4699 - PACKET3_SET_SH_REG_START;
4700 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4701 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4702
4703 /* write dispatch packet */
4704 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4705 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4706 ib.ptr[ib.length_dw++] = 1; /* y */
4707 ib.ptr[ib.length_dw++] = 1; /* z */
4708 ib.ptr[ib.length_dw++] =
4709 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4710
4711 /* write CS partial flush packet */
4712 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4713 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4714
4715 /* SGPR1 */
4716 /* write the register state for the compute dispatch */
4717 for (i = 0; i < gpr_reg_size; i++) {
4718 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4719 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4720 - PACKET3_SET_SH_REG_START;
4721 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4722 }
4723 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4724 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4725 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4726 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4727 - PACKET3_SET_SH_REG_START;
4728 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4729 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4730
4731 /* write dispatch packet */
4732 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4733 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4734 ib.ptr[ib.length_dw++] = 1; /* y */
4735 ib.ptr[ib.length_dw++] = 1; /* z */
4736 ib.ptr[ib.length_dw++] =
4737 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4738
4739 /* write CS partial flush packet */
4740 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4741 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4742
4743 /* SGPR2 */
4744 /* write the register state for the compute dispatch */
4745 for (i = 0; i < gpr_reg_size; i++) {
4746 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4747 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4748 - PACKET3_SET_SH_REG_START;
4749 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4750 }
4751 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4752 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4753 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4754 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4755 - PACKET3_SET_SH_REG_START;
4756 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4757 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4758
4759 /* write dispatch packet */
4760 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4761 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4762 ib.ptr[ib.length_dw++] = 1; /* y */
4763 ib.ptr[ib.length_dw++] = 1; /* z */
4764 ib.ptr[ib.length_dw++] =
4765 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4766
4767 /* write CS partial flush packet */
4768 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4769 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4770
4771 /* shedule the ib on the ring */
4772 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4773 if (r) {
4774 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4775 goto fail;
4776 }
4777
4778 /* wait for the GPU to finish processing the IB */
4779 r = dma_fence_wait(f, false);
4780 if (r) {
4781 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4782 goto fail;
4783 }
4784
4785 fail:
4786 amdgpu_ib_free(&ib, NULL);
4787 dma_fence_put(f);
4788
4789 return r;
4790 }
4791
gfx_v9_0_early_init(struct amdgpu_ip_block * ip_block)4792 static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block)
4793 {
4794 struct amdgpu_device *adev = ip_block->adev;
4795
4796 adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4797
4798 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
4799 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4800 adev->gfx.num_gfx_rings = 0;
4801 else
4802 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4803 adev->gfx.xcc_mask = 1;
4804 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4805 AMDGPU_MAX_COMPUTE_RINGS);
4806 gfx_v9_0_set_kiq_pm4_funcs(adev);
4807 gfx_v9_0_set_ring_funcs(adev);
4808 gfx_v9_0_set_irq_funcs(adev);
4809 gfx_v9_0_set_gds_init(adev);
4810 gfx_v9_0_set_rlc_funcs(adev);
4811
4812 /* init rlcg reg access ctrl */
4813 gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4814
4815 return gfx_v9_0_init_microcode(adev);
4816 }
4817
gfx_v9_0_ecc_late_init(struct amdgpu_ip_block * ip_block)4818 static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block)
4819 {
4820 struct amdgpu_device *adev = ip_block->adev;
4821 int r;
4822
4823 /*
4824 * Temp workaround to fix the issue that CP firmware fails to
4825 * update read pointer when CPDMA is writing clearing operation
4826 * to GDS in suspend/resume sequence on several cards. So just
4827 * limit this operation in cold boot sequence.
4828 */
4829 if ((!adev->in_suspend) &&
4830 (adev->gds.gds_size)) {
4831 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4832 if (r)
4833 return r;
4834 }
4835
4836 /* requires IBs so do in late init after IB pool is initialized */
4837 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4838 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4839 else
4840 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4841
4842 if (r)
4843 return r;
4844
4845 if (adev->gfx.ras &&
4846 adev->gfx.ras->enable_watchdog_timer)
4847 adev->gfx.ras->enable_watchdog_timer(adev);
4848
4849 return 0;
4850 }
4851
gfx_v9_0_late_init(struct amdgpu_ip_block * ip_block)4852 static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block)
4853 {
4854 struct amdgpu_device *adev = ip_block->adev;
4855 int r;
4856
4857 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4858 if (r)
4859 return r;
4860
4861 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4862 if (r)
4863 return r;
4864
4865 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
4866 if (r)
4867 return r;
4868
4869 r = gfx_v9_0_ecc_late_init(ip_block);
4870 if (r)
4871 return r;
4872
4873 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4874 gfx_v9_4_2_debug_trap_config_init(adev,
4875 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4876 else
4877 gfx_v9_0_debug_trap_config_init(adev,
4878 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4879
4880 return 0;
4881 }
4882
gfx_v9_0_is_rlc_enabled(struct amdgpu_device * adev)4883 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4884 {
4885 uint32_t rlc_setting;
4886
4887 /* if RLC is not enabled, do nothing */
4888 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4889 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4890 return false;
4891
4892 return true;
4893 }
4894
gfx_v9_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)4895 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4896 {
4897 uint32_t data;
4898 unsigned i;
4899
4900 data = RLC_SAFE_MODE__CMD_MASK;
4901 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4902 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4903
4904 /* wait for RLC_SAFE_MODE */
4905 for (i = 0; i < adev->usec_timeout; i++) {
4906 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4907 break;
4908 udelay(1);
4909 }
4910 }
4911
gfx_v9_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)4912 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4913 {
4914 uint32_t data;
4915
4916 data = RLC_SAFE_MODE__CMD_MASK;
4917 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4918 }
4919
gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)4920 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4921 bool enable)
4922 {
4923 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4924
4925 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4926 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4927 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4928 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4929 } else {
4930 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4931 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4932 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4933 }
4934
4935 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4936 }
4937
gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device * adev,bool enable)4938 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4939 bool enable)
4940 {
4941 /* TODO: double check if we need to perform under safe mode */
4942 /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4943
4944 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4945 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4946 else
4947 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4948
4949 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4950 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4951 else
4952 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4953
4954 /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4955 }
4956
gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)4957 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4958 bool enable)
4959 {
4960 uint32_t data, def;
4961
4962 /* It is disabled by HW by default */
4963 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4964 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4965 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4966
4967 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
4968 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4969
4970 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4971 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4972 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4973
4974 /* only for Vega10 & Raven1 */
4975 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4976
4977 if (def != data)
4978 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4979
4980 /* MGLS is a global flag to control all MGLS in GFX */
4981 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4982 /* 2 - RLC memory Light sleep */
4983 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4984 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4985 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4986 if (def != data)
4987 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4988 }
4989 /* 3 - CP memory Light sleep */
4990 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4991 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4992 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4993 if (def != data)
4994 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4995 }
4996 }
4997 } else {
4998 /* 1 - MGCG_OVERRIDE */
4999 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5000
5001 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
5002 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
5003
5004 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5005 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5006 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
5007 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
5008
5009 if (def != data)
5010 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5011
5012 /* 2 - disable MGLS in RLC */
5013 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
5014 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5015 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5016 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
5017 }
5018
5019 /* 3 - disable MGLS in CP */
5020 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
5021 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5022 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5023 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
5024 }
5025 }
5026 }
5027
gfx_v9_0_update_3d_clock_gating(struct amdgpu_device * adev,bool enable)5028 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
5029 bool enable)
5030 {
5031 uint32_t data, def;
5032
5033 if (!adev->gfx.num_gfx_rings)
5034 return;
5035
5036 /* Enable 3D CGCG/CGLS */
5037 if (enable) {
5038 /* write cmd to clear cgcg/cgls ov */
5039 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5040 /* unset CGCG override */
5041 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5042 /* update CGCG and CGLS override bits */
5043 if (def != data)
5044 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5045
5046 /* enable 3Dcgcg FSM(0x0000363f) */
5047 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5048
5049 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5050 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5051 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5052 else
5053 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
5054
5055 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5056 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5057 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5058 if (def != data)
5059 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5060
5061 /* set IDLE_POLL_COUNT(0x00900100) */
5062 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5063 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5064 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5065 if (def != data)
5066 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5067 } else {
5068 /* Disable CGCG/CGLS */
5069 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5070 /* disable cgcg, cgls should be disabled */
5071 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5072 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5073 /* disable cgcg and cgls in FSM */
5074 if (def != data)
5075 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5076 }
5077 }
5078
gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5079 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5080 bool enable)
5081 {
5082 uint32_t def, data;
5083
5084 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5085 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5086 /* unset CGCG override */
5087 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5088 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5089 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5090 else
5091 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5092 /* update CGCG and CGLS override bits */
5093 if (def != data)
5094 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5095
5096 /* enable cgcg FSM(0x0000363F) */
5097 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5098
5099 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))
5100 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5101 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5102 else
5103 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5104 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5105 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5106 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5107 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5108 if (def != data)
5109 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5110
5111 /* set IDLE_POLL_COUNT(0x00900100) */
5112 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5113 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5114 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5115 if (def != data)
5116 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5117 } else {
5118 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5119 /* reset CGCG/CGLS bits */
5120 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5121 /* disable cgcg and cgls in FSM */
5122 if (def != data)
5123 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5124 }
5125 }
5126
gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5127 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5128 bool enable)
5129 {
5130 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5131 if (enable) {
5132 /* CGCG/CGLS should be enabled after MGCG/MGLS
5133 * === MGCG + MGLS ===
5134 */
5135 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5136 /* === CGCG /CGLS for GFX 3D Only === */
5137 gfx_v9_0_update_3d_clock_gating(adev, enable);
5138 /* === CGCG + CGLS === */
5139 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5140 } else {
5141 /* CGCG/CGLS should be disabled before MGCG/MGLS
5142 * === CGCG + CGLS ===
5143 */
5144 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5145 /* === CGCG /CGLS for GFX 3D Only === */
5146 gfx_v9_0_update_3d_clock_gating(adev, enable);
5147 /* === MGCG + MGLS === */
5148 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5149 }
5150 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5151 return 0;
5152 }
5153
gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device * adev,unsigned int vmid)5154 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
5155 unsigned int vmid)
5156 {
5157 u32 reg, data;
5158
5159 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5160 if (amdgpu_sriov_is_pp_one_vf(adev))
5161 data = RREG32_NO_KIQ(reg);
5162 else
5163 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
5164
5165 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5166 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5167
5168 if (amdgpu_sriov_is_pp_one_vf(adev))
5169 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5170 else
5171 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5172 }
5173
gfx_v9_0_update_spm_vmid(struct amdgpu_device * adev,struct amdgpu_ring * ring,unsigned int vmid)5174 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
5175 {
5176 amdgpu_gfx_off_ctrl(adev, false);
5177
5178 gfx_v9_0_update_spm_vmid_internal(adev, vmid);
5179
5180 amdgpu_gfx_off_ctrl(adev, true);
5181 }
5182
gfx_v9_0_check_rlcg_range(struct amdgpu_device * adev,uint32_t offset,struct soc15_reg_rlcg * entries,int arr_size)5183 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5184 uint32_t offset,
5185 struct soc15_reg_rlcg *entries, int arr_size)
5186 {
5187 int i;
5188 uint32_t reg;
5189
5190 if (!entries)
5191 return false;
5192
5193 for (i = 0; i < arr_size; i++) {
5194 const struct soc15_reg_rlcg *entry;
5195
5196 entry = &entries[i];
5197 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5198 if (offset == reg)
5199 return true;
5200 }
5201
5202 return false;
5203 }
5204
gfx_v9_0_is_rlcg_access_range(struct amdgpu_device * adev,u32 offset)5205 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5206 {
5207 return gfx_v9_0_check_rlcg_range(adev, offset,
5208 (void *)rlcg_access_gc_9_0,
5209 ARRAY_SIZE(rlcg_access_gc_9_0));
5210 }
5211
5212 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5213 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5214 .set_safe_mode = gfx_v9_0_set_safe_mode,
5215 .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5216 .init = gfx_v9_0_rlc_init,
5217 .get_csb_size = gfx_v9_0_get_csb_size,
5218 .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5219 .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5220 .resume = gfx_v9_0_rlc_resume,
5221 .stop = gfx_v9_0_rlc_stop,
5222 .reset = gfx_v9_0_rlc_reset,
5223 .start = gfx_v9_0_rlc_start,
5224 .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5225 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5226 };
5227
gfx_v9_0_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)5228 static int gfx_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5229 enum amd_powergating_state state)
5230 {
5231 struct amdgpu_device *adev = ip_block->adev;
5232 bool enable = (state == AMD_PG_STATE_GATE);
5233
5234 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5235 case IP_VERSION(9, 2, 2):
5236 case IP_VERSION(9, 1, 0):
5237 case IP_VERSION(9, 3, 0):
5238 if (!enable)
5239 amdgpu_gfx_off_ctrl_immediate(adev, false);
5240
5241 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5242 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5243 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5244 } else {
5245 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5246 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5247 }
5248
5249 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5250 gfx_v9_0_enable_cp_power_gating(adev, true);
5251 else
5252 gfx_v9_0_enable_cp_power_gating(adev, false);
5253
5254 /* update gfx cgpg state */
5255 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5256
5257 /* update mgcg state */
5258 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5259
5260 if (enable)
5261 amdgpu_gfx_off_ctrl_immediate(adev, true);
5262 break;
5263 case IP_VERSION(9, 2, 1):
5264 amdgpu_gfx_off_ctrl_immediate(adev, enable);
5265 break;
5266 default:
5267 break;
5268 }
5269
5270 return 0;
5271 }
5272
gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)5273 static int gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5274 enum amd_clockgating_state state)
5275 {
5276 struct amdgpu_device *adev = ip_block->adev;
5277
5278 if (amdgpu_sriov_vf(adev))
5279 return 0;
5280
5281 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5282 case IP_VERSION(9, 0, 1):
5283 case IP_VERSION(9, 2, 1):
5284 case IP_VERSION(9, 4, 0):
5285 case IP_VERSION(9, 2, 2):
5286 case IP_VERSION(9, 1, 0):
5287 case IP_VERSION(9, 4, 1):
5288 case IP_VERSION(9, 3, 0):
5289 case IP_VERSION(9, 4, 2):
5290 gfx_v9_0_update_gfx_clock_gating(adev,
5291 state == AMD_CG_STATE_GATE);
5292 break;
5293 default:
5294 break;
5295 }
5296 return 0;
5297 }
5298
gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)5299 static void gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
5300 {
5301 struct amdgpu_device *adev = ip_block->adev;
5302 int data;
5303
5304 if (amdgpu_sriov_vf(adev))
5305 *flags = 0;
5306
5307 /* AMD_CG_SUPPORT_GFX_MGCG */
5308 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5309 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5310 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5311
5312 /* AMD_CG_SUPPORT_GFX_CGCG */
5313 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5314 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5315 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5316
5317 /* AMD_CG_SUPPORT_GFX_CGLS */
5318 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5319 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5320
5321 /* AMD_CG_SUPPORT_GFX_RLC_LS */
5322 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5323 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5324 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5325
5326 /* AMD_CG_SUPPORT_GFX_CP_LS */
5327 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5328 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5329 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5330
5331 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {
5332 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5333 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5334 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5335 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5336
5337 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5338 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5339 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5340 }
5341 }
5342
gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5343 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5344 {
5345 return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5346 }
5347
gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5348 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5349 {
5350 struct amdgpu_device *adev = ring->adev;
5351 u64 wptr;
5352
5353 /* XXX check if swapping is necessary on BE */
5354 if (ring->use_doorbell) {
5355 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5356 } else {
5357 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5358 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5359 }
5360
5361 return wptr;
5362 }
5363
gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5364 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5365 {
5366 struct amdgpu_device *adev = ring->adev;
5367
5368 if (ring->use_doorbell) {
5369 /* XXX check if swapping is necessary on BE */
5370 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5371 WDOORBELL64(ring->doorbell_index, ring->wptr);
5372 } else {
5373 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5374 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5375 }
5376 }
5377
gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5378 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5379 {
5380 struct amdgpu_device *adev = ring->adev;
5381 u32 ref_and_mask, reg_mem_engine;
5382 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5383
5384 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5385 switch (ring->me) {
5386 case 1:
5387 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5388 break;
5389 case 2:
5390 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5391 break;
5392 default:
5393 return;
5394 }
5395 reg_mem_engine = 0;
5396 } else {
5397 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5398 reg_mem_engine = 1; /* pfp */
5399 }
5400
5401 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5402 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5403 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5404 ref_and_mask, ref_and_mask, 0x20);
5405 }
5406
gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5407 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5408 struct amdgpu_job *job,
5409 struct amdgpu_ib *ib,
5410 uint32_t flags)
5411 {
5412 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5413 u32 header, control = 0;
5414
5415 if (ib->flags & AMDGPU_IB_FLAG_CE)
5416 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5417 else
5418 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5419
5420 control |= ib->length_dw | (vmid << 24);
5421
5422 if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5423 control |= INDIRECT_BUFFER_PRE_ENB(1);
5424
5425 if (flags & AMDGPU_IB_PREEMPTED)
5426 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5427
5428 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5429 gfx_v9_0_ring_emit_de_meta(ring,
5430 (!amdgpu_sriov_vf(ring->adev) &&
5431 flags & AMDGPU_IB_PREEMPTED) ?
5432 true : false,
5433 job->gds_size > 0 && job->gds_base != 0);
5434 }
5435
5436 amdgpu_ring_write(ring, header);
5437 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5438 amdgpu_ring_write(ring,
5439 #ifdef __BIG_ENDIAN
5440 (2 << 0) |
5441 #endif
5442 lower_32_bits(ib->gpu_addr));
5443 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5444 amdgpu_ring_ib_on_emit_cntl(ring);
5445 amdgpu_ring_write(ring, control);
5446 }
5447
gfx_v9_0_ring_patch_cntl(struct amdgpu_ring * ring,unsigned offset)5448 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5449 unsigned offset)
5450 {
5451 u32 control = ring->ring[offset];
5452
5453 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5454 ring->ring[offset] = control;
5455 }
5456
gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring * ring,unsigned offset)5457 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5458 unsigned offset)
5459 {
5460 struct amdgpu_device *adev = ring->adev;
5461 void *ce_payload_cpu_addr;
5462 uint64_t payload_offset, payload_size;
5463
5464 payload_size = sizeof(struct v9_ce_ib_state);
5465
5466 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5467 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5468
5469 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5470 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5471 } else {
5472 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5473 (ring->buf_mask + 1 - offset) << 2);
5474 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5475 memcpy((void *)&ring->ring[0],
5476 ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5477 payload_size);
5478 }
5479 }
5480
gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring * ring,unsigned offset)5481 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5482 unsigned offset)
5483 {
5484 struct amdgpu_device *adev = ring->adev;
5485 void *de_payload_cpu_addr;
5486 uint64_t payload_offset, payload_size;
5487
5488 payload_size = sizeof(struct v9_de_ib_state);
5489
5490 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5491 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5492
5493 ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5494 IB_COMPLETION_STATUS_PREEMPTED;
5495
5496 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5497 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5498 } else {
5499 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5500 (ring->buf_mask + 1 - offset) << 2);
5501 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5502 memcpy((void *)&ring->ring[0],
5503 de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5504 payload_size);
5505 }
5506 }
5507
gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5508 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5509 struct amdgpu_job *job,
5510 struct amdgpu_ib *ib,
5511 uint32_t flags)
5512 {
5513 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5514 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5515
5516 /* Currently, there is a high possibility to get wave ID mismatch
5517 * between ME and GDS, leading to a hw deadlock, because ME generates
5518 * different wave IDs than the GDS expects. This situation happens
5519 * randomly when at least 5 compute pipes use GDS ordered append.
5520 * The wave IDs generated by ME are also wrong after suspend/resume.
5521 * Those are probably bugs somewhere else in the kernel driver.
5522 *
5523 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5524 * GDS to 0 for this ring (me/pipe).
5525 */
5526 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5527 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5528 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5529 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5530 }
5531
5532 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5533 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5534 amdgpu_ring_write(ring,
5535 #ifdef __BIG_ENDIAN
5536 (2 << 0) |
5537 #endif
5538 lower_32_bits(ib->gpu_addr));
5539 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5540 amdgpu_ring_write(ring, control);
5541 }
5542
gfx_v9_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)5543 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5544 u64 seq, unsigned flags)
5545 {
5546 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5547 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5548 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5549 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5550 uint32_t dw2 = 0;
5551
5552 /* RELEASE_MEM - flush caches, send int */
5553 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5554
5555 if (writeback) {
5556 dw2 = EOP_TC_NC_ACTION_EN;
5557 } else {
5558 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5559 EOP_TC_MD_ACTION_EN;
5560 }
5561 dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5562 EVENT_INDEX(5);
5563 if (exec)
5564 dw2 |= EOP_EXEC;
5565
5566 amdgpu_ring_write(ring, dw2);
5567 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5568
5569 /*
5570 * the address should be Qword aligned if 64bit write, Dword
5571 * aligned if only send 32bit data low (discard data high)
5572 */
5573 if (write64bit)
5574 BUG_ON(addr & 0x7);
5575 else
5576 BUG_ON(addr & 0x3);
5577 amdgpu_ring_write(ring, lower_32_bits(addr));
5578 amdgpu_ring_write(ring, upper_32_bits(addr));
5579 amdgpu_ring_write(ring, lower_32_bits(seq));
5580 amdgpu_ring_write(ring, upper_32_bits(seq));
5581 amdgpu_ring_write(ring, 0);
5582 }
5583
gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)5584 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5585 {
5586 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5587 uint32_t seq = ring->fence_drv.sync_seq;
5588 uint64_t addr = ring->fence_drv.gpu_addr;
5589
5590 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5591 lower_32_bits(addr), upper_32_bits(addr),
5592 seq, 0xffffffff, 4);
5593 }
5594
gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)5595 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5596 unsigned vmid, uint64_t pd_addr)
5597 {
5598 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5599
5600 /* compute doesn't have PFP */
5601 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5602 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5603 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5604 amdgpu_ring_write(ring, 0x0);
5605 }
5606 }
5607
gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5608 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5609 {
5610 return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5611 }
5612
gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5613 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5614 {
5615 u64 wptr;
5616
5617 /* XXX check if swapping is necessary on BE */
5618 if (ring->use_doorbell)
5619 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5620 else
5621 BUG();
5622 return wptr;
5623 }
5624
gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5625 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5626 {
5627 struct amdgpu_device *adev = ring->adev;
5628
5629 /* XXX check if swapping is necessary on BE */
5630 if (ring->use_doorbell) {
5631 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5632 WDOORBELL64(ring->doorbell_index, ring->wptr);
5633 } else{
5634 BUG(); /* only DOORBELL method supported on gfx9 now */
5635 }
5636 }
5637
gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)5638 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5639 u64 seq, unsigned int flags)
5640 {
5641 struct amdgpu_device *adev = ring->adev;
5642
5643 /* we only allocate 32bit for each seq wb address */
5644 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5645
5646 /* write fence seq to the "addr" */
5647 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5648 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5649 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5650 amdgpu_ring_write(ring, lower_32_bits(addr));
5651 amdgpu_ring_write(ring, upper_32_bits(addr));
5652 amdgpu_ring_write(ring, lower_32_bits(seq));
5653
5654 if (flags & AMDGPU_FENCE_FLAG_INT) {
5655 /* set register to trigger INT */
5656 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5657 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5658 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5659 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5660 amdgpu_ring_write(ring, 0);
5661 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5662 }
5663 }
5664
gfx_v9_ring_emit_sb(struct amdgpu_ring * ring)5665 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5666 {
5667 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5668 amdgpu_ring_write(ring, 0);
5669 }
5670
gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring * ring,bool resume)5671 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5672 {
5673 struct amdgpu_device *adev = ring->adev;
5674 struct v9_ce_ib_state ce_payload = {0};
5675 uint64_t offset, ce_payload_gpu_addr;
5676 void *ce_payload_cpu_addr;
5677 int cnt;
5678
5679 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5680
5681 offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5682 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5683 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5684
5685 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5686 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5687 WRITE_DATA_DST_SEL(8) |
5688 WR_CONFIRM) |
5689 WRITE_DATA_CACHE_POLICY(0));
5690 amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5691 amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5692
5693 amdgpu_ring_ib_on_emit_ce(ring);
5694
5695 if (resume)
5696 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,
5697 sizeof(ce_payload) >> 2);
5698 else
5699 amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
5700 sizeof(ce_payload) >> 2);
5701 }
5702
gfx_v9_0_ring_preempt_ib(struct amdgpu_ring * ring)5703 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5704 {
5705 int i, r = 0;
5706 struct amdgpu_device *adev = ring->adev;
5707 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5708 struct amdgpu_ring *kiq_ring = &kiq->ring;
5709 unsigned long flags;
5710
5711 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5712 return -EINVAL;
5713
5714 spin_lock_irqsave(&kiq->ring_lock, flags);
5715
5716 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5717 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5718 return -ENOMEM;
5719 }
5720
5721 /* assert preemption condition */
5722 amdgpu_ring_set_preempt_cond_exec(ring, false);
5723
5724 ring->trail_seq += 1;
5725 amdgpu_ring_alloc(ring, 13);
5726 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
5727 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5728
5729 /* assert IB preemption, emit the trailing fence */
5730 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5731 ring->trail_fence_gpu_addr,
5732 ring->trail_seq);
5733
5734 amdgpu_ring_commit(kiq_ring);
5735 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5736
5737 /* poll the trailing fence */
5738 for (i = 0; i < adev->usec_timeout; i++) {
5739 if (ring->trail_seq ==
5740 le32_to_cpu(*ring->trail_fence_cpu_addr))
5741 break;
5742 udelay(1);
5743 }
5744
5745 if (i >= adev->usec_timeout) {
5746 r = -EINVAL;
5747 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5748 }
5749
5750 /*reset the CP_VMID_PREEMPT after trailing fence*/
5751 amdgpu_ring_emit_wreg(ring,
5752 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5753 0x0);
5754 amdgpu_ring_commit(ring);
5755
5756 /* deassert preemption condition */
5757 amdgpu_ring_set_preempt_cond_exec(ring, true);
5758 return r;
5759 }
5760
gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring * ring,bool resume,bool usegds)5761 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5762 {
5763 struct amdgpu_device *adev = ring->adev;
5764 struct v9_de_ib_state de_payload = {0};
5765 uint64_t offset, gds_addr, de_payload_gpu_addr;
5766 void *de_payload_cpu_addr;
5767 int cnt;
5768
5769 offset = offsetof(struct v9_gfx_meta_data, de_payload);
5770 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5771 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5772
5773 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5774 AMDGPU_CSA_SIZE - adev->gds.gds_size,
5775 PAGE_SIZE);
5776
5777 if (usegds) {
5778 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5779 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5780 }
5781
5782 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5783 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5784 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5785 WRITE_DATA_DST_SEL(8) |
5786 WR_CONFIRM) |
5787 WRITE_DATA_CACHE_POLICY(0));
5788 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5789 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5790
5791 amdgpu_ring_ib_on_emit_de(ring);
5792 if (resume)
5793 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5794 sizeof(de_payload) >> 2);
5795 else
5796 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5797 sizeof(de_payload) >> 2);
5798 }
5799
gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)5800 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5801 bool secure)
5802 {
5803 uint32_t v = secure ? FRAME_TMZ : 0;
5804
5805 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5806 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5807 }
5808
gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)5809 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5810 {
5811 uint32_t dw2 = 0;
5812
5813 gfx_v9_0_ring_emit_ce_meta(ring,
5814 (!amdgpu_sriov_vf(ring->adev) &&
5815 flags & AMDGPU_IB_PREEMPTED) ? true : false);
5816
5817 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5818 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5819 /* set load_global_config & load_global_uconfig */
5820 dw2 |= 0x8001;
5821 /* set load_cs_sh_regs */
5822 dw2 |= 0x01000000;
5823 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5824 dw2 |= 0x10002;
5825
5826 /* set load_ce_ram if preamble presented */
5827 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5828 dw2 |= 0x10000000;
5829 } else {
5830 /* still load_ce_ram if this is the first time preamble presented
5831 * although there is no context switch happens.
5832 */
5833 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5834 dw2 |= 0x10000000;
5835 }
5836
5837 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5838 amdgpu_ring_write(ring, dw2);
5839 amdgpu_ring_write(ring, 0);
5840 }
5841
gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)5842 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5843 uint64_t addr)
5844 {
5845 unsigned ret;
5846 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5847 amdgpu_ring_write(ring, lower_32_bits(addr));
5848 amdgpu_ring_write(ring, upper_32_bits(addr));
5849 /* discard following DWs if *cond_exec_gpu_addr==0 */
5850 amdgpu_ring_write(ring, 0);
5851 ret = ring->wptr & ring->buf_mask;
5852 /* patch dummy value later */
5853 amdgpu_ring_write(ring, 0);
5854 return ret;
5855 }
5856
gfx_v9_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)5857 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5858 uint32_t reg_val_offs)
5859 {
5860 struct amdgpu_device *adev = ring->adev;
5861
5862 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5863 amdgpu_ring_write(ring, 0 | /* src: register*/
5864 (5 << 8) | /* dst: memory */
5865 (1 << 20)); /* write confirm */
5866 amdgpu_ring_write(ring, reg);
5867 amdgpu_ring_write(ring, 0);
5868 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5869 reg_val_offs * 4));
5870 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5871 reg_val_offs * 4));
5872 }
5873
gfx_v9_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)5874 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5875 uint32_t val)
5876 {
5877 uint32_t cmd = 0;
5878
5879 switch (ring->funcs->type) {
5880 case AMDGPU_RING_TYPE_GFX:
5881 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5882 break;
5883 case AMDGPU_RING_TYPE_KIQ:
5884 cmd = (1 << 16); /* no inc addr */
5885 break;
5886 default:
5887 cmd = WR_CONFIRM;
5888 break;
5889 }
5890 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5891 amdgpu_ring_write(ring, cmd);
5892 amdgpu_ring_write(ring, reg);
5893 amdgpu_ring_write(ring, 0);
5894 amdgpu_ring_write(ring, val);
5895 }
5896
gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)5897 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5898 uint32_t val, uint32_t mask)
5899 {
5900 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5901 }
5902
gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)5903 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5904 uint32_t reg0, uint32_t reg1,
5905 uint32_t ref, uint32_t mask)
5906 {
5907 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5908 struct amdgpu_device *adev = ring->adev;
5909 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5910 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5911
5912 if (fw_version_ok)
5913 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5914 ref, mask, 0x20);
5915 else
5916 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5917 ref, mask);
5918 }
5919
gfx_v9_0_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)5920 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5921 {
5922 struct amdgpu_device *adev = ring->adev;
5923 uint32_t value = 0;
5924
5925 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5926 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5927 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5928 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5929 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5930 WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5931 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5932 }
5933
gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,enum amdgpu_interrupt_state state)5934 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5935 enum amdgpu_interrupt_state state)
5936 {
5937 switch (state) {
5938 case AMDGPU_IRQ_STATE_DISABLE:
5939 case AMDGPU_IRQ_STATE_ENABLE:
5940 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5941 TIME_STAMP_INT_ENABLE,
5942 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5943 break;
5944 default:
5945 break;
5946 }
5947 }
5948
gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)5949 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5950 int me, int pipe,
5951 enum amdgpu_interrupt_state state)
5952 {
5953 u32 mec_int_cntl, mec_int_cntl_reg;
5954
5955 /*
5956 * amdgpu controls only the first MEC. That's why this function only
5957 * handles the setting of interrupts for this specific MEC. All other
5958 * pipes' interrupts are set by amdkfd.
5959 */
5960
5961 if (me == 1) {
5962 switch (pipe) {
5963 case 0:
5964 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5965 break;
5966 case 1:
5967 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5968 break;
5969 case 2:
5970 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5971 break;
5972 case 3:
5973 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5974 break;
5975 default:
5976 DRM_DEBUG("invalid pipe %d\n", pipe);
5977 return;
5978 }
5979 } else {
5980 DRM_DEBUG("invalid me %d\n", me);
5981 return;
5982 }
5983
5984 switch (state) {
5985 case AMDGPU_IRQ_STATE_DISABLE:
5986 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
5987 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5988 TIME_STAMP_INT_ENABLE, 0);
5989 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5990 break;
5991 case AMDGPU_IRQ_STATE_ENABLE:
5992 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5993 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5994 TIME_STAMP_INT_ENABLE, 1);
5995 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5996 break;
5997 default:
5998 break;
5999 }
6000 }
6001
gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device * adev,int me,int pipe)6002 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev,
6003 int me, int pipe)
6004 {
6005 /*
6006 * amdgpu controls only the first MEC. That's why this function only
6007 * handles the setting of interrupts for this specific MEC. All other
6008 * pipes' interrupts are set by amdkfd.
6009 */
6010 if (me != 1)
6011 return 0;
6012
6013 switch (pipe) {
6014 case 0:
6015 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6016 case 1:
6017 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6018 case 2:
6019 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6020 case 3:
6021 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6022 default:
6023 return 0;
6024 }
6025 }
6026
gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6027 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6028 struct amdgpu_irq_src *source,
6029 unsigned type,
6030 enum amdgpu_interrupt_state state)
6031 {
6032 u32 cp_int_cntl_reg, cp_int_cntl;
6033 int i, j;
6034
6035 switch (state) {
6036 case AMDGPU_IRQ_STATE_DISABLE:
6037 case AMDGPU_IRQ_STATE_ENABLE:
6038 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6039 PRIV_REG_INT_ENABLE,
6040 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6041 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6042 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6043 /* MECs start at 1 */
6044 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6045
6046 if (cp_int_cntl_reg) {
6047 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6048 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6049 PRIV_REG_INT_ENABLE,
6050 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6051 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6052 }
6053 }
6054 }
6055 break;
6056 default:
6057 break;
6058 }
6059
6060 return 0;
6061 }
6062
gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6063 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6064 struct amdgpu_irq_src *source,
6065 unsigned type,
6066 enum amdgpu_interrupt_state state)
6067 {
6068 u32 cp_int_cntl_reg, cp_int_cntl;
6069 int i, j;
6070
6071 switch (state) {
6072 case AMDGPU_IRQ_STATE_DISABLE:
6073 case AMDGPU_IRQ_STATE_ENABLE:
6074 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6075 OPCODE_ERROR_INT_ENABLE,
6076 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6077 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6078 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6079 /* MECs start at 1 */
6080 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6081
6082 if (cp_int_cntl_reg) {
6083 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6084 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6085 OPCODE_ERROR_INT_ENABLE,
6086 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6087 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6088 }
6089 }
6090 }
6091 break;
6092 default:
6093 break;
6094 }
6095
6096 return 0;
6097 }
6098
gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6099 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6100 struct amdgpu_irq_src *source,
6101 unsigned type,
6102 enum amdgpu_interrupt_state state)
6103 {
6104 switch (state) {
6105 case AMDGPU_IRQ_STATE_DISABLE:
6106 case AMDGPU_IRQ_STATE_ENABLE:
6107 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6108 PRIV_INSTR_INT_ENABLE,
6109 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6110 break;
6111 default:
6112 break;
6113 }
6114
6115 return 0;
6116 }
6117
6118 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
6119 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6120 CP_ECC_ERROR_INT_ENABLE, 1)
6121
6122 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
6123 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6124 CP_ECC_ERROR_INT_ENABLE, 0)
6125
gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6126 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
6127 struct amdgpu_irq_src *source,
6128 unsigned type,
6129 enum amdgpu_interrupt_state state)
6130 {
6131 switch (state) {
6132 case AMDGPU_IRQ_STATE_DISABLE:
6133 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6134 CP_ECC_ERROR_INT_ENABLE, 0);
6135 DISABLE_ECC_ON_ME_PIPE(1, 0);
6136 DISABLE_ECC_ON_ME_PIPE(1, 1);
6137 DISABLE_ECC_ON_ME_PIPE(1, 2);
6138 DISABLE_ECC_ON_ME_PIPE(1, 3);
6139 break;
6140
6141 case AMDGPU_IRQ_STATE_ENABLE:
6142 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6143 CP_ECC_ERROR_INT_ENABLE, 1);
6144 ENABLE_ECC_ON_ME_PIPE(1, 0);
6145 ENABLE_ECC_ON_ME_PIPE(1, 1);
6146 ENABLE_ECC_ON_ME_PIPE(1, 2);
6147 ENABLE_ECC_ON_ME_PIPE(1, 3);
6148 break;
6149 default:
6150 break;
6151 }
6152
6153 return 0;
6154 }
6155
6156
gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6157 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6158 struct amdgpu_irq_src *src,
6159 unsigned type,
6160 enum amdgpu_interrupt_state state)
6161 {
6162 switch (type) {
6163 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6164 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
6165 break;
6166 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6167 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6168 break;
6169 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6170 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6171 break;
6172 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6173 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6174 break;
6175 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6176 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6177 break;
6178 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6179 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6180 break;
6181 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6182 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6183 break;
6184 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6185 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6186 break;
6187 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6188 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6189 break;
6190 default:
6191 break;
6192 }
6193 return 0;
6194 }
6195
gfx_v9_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6196 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
6197 struct amdgpu_irq_src *source,
6198 struct amdgpu_iv_entry *entry)
6199 {
6200 int i;
6201 u8 me_id, pipe_id, queue_id;
6202 struct amdgpu_ring *ring;
6203
6204 DRM_DEBUG("IH: CP EOP\n");
6205 me_id = (entry->ring_id & 0x0c) >> 2;
6206 pipe_id = (entry->ring_id & 0x03) >> 0;
6207 queue_id = (entry->ring_id & 0x70) >> 4;
6208
6209 switch (me_id) {
6210 case 0:
6211 if (adev->gfx.num_gfx_rings) {
6212 if (!adev->gfx.mcbp) {
6213 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6214 } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {
6215 /* Fence signals are handled on the software rings*/
6216 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
6217 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);
6218 }
6219 }
6220 break;
6221 case 1:
6222 case 2:
6223 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6224 ring = &adev->gfx.compute_ring[i];
6225 /* Per-queue interrupt is supported for MEC starting from VI.
6226 * The interrupt can only be enabled/disabled per pipe instead of per queue.
6227 */
6228 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6229 amdgpu_fence_process(ring);
6230 }
6231 break;
6232 }
6233 return 0;
6234 }
6235
gfx_v9_0_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6236 static void gfx_v9_0_fault(struct amdgpu_device *adev,
6237 struct amdgpu_iv_entry *entry)
6238 {
6239 u8 me_id, pipe_id, queue_id;
6240 struct amdgpu_ring *ring;
6241 int i;
6242
6243 me_id = (entry->ring_id & 0x0c) >> 2;
6244 pipe_id = (entry->ring_id & 0x03) >> 0;
6245 queue_id = (entry->ring_id & 0x70) >> 4;
6246
6247 switch (me_id) {
6248 case 0:
6249 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6250 break;
6251 case 1:
6252 case 2:
6253 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6254 ring = &adev->gfx.compute_ring[i];
6255 if (ring->me == me_id && ring->pipe == pipe_id &&
6256 ring->queue == queue_id)
6257 drm_sched_fault(&ring->sched);
6258 }
6259 break;
6260 }
6261 }
6262
gfx_v9_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6263 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
6264 struct amdgpu_irq_src *source,
6265 struct amdgpu_iv_entry *entry)
6266 {
6267 DRM_ERROR("Illegal register access in command stream\n");
6268 gfx_v9_0_fault(adev, entry);
6269 return 0;
6270 }
6271
gfx_v9_0_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6272 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev,
6273 struct amdgpu_irq_src *source,
6274 struct amdgpu_iv_entry *entry)
6275 {
6276 DRM_ERROR("Illegal opcode in command stream\n");
6277 gfx_v9_0_fault(adev, entry);
6278 return 0;
6279 }
6280
gfx_v9_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6281 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
6282 struct amdgpu_irq_src *source,
6283 struct amdgpu_iv_entry *entry)
6284 {
6285 DRM_ERROR("Illegal instruction in command stream\n");
6286 gfx_v9_0_fault(adev, entry);
6287 return 0;
6288 }
6289
6290
6291 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
6292 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
6293 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
6294 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
6295 },
6296 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
6297 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
6298 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
6299 },
6300 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6301 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
6302 0, 0
6303 },
6304 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6305 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
6306 0, 0
6307 },
6308 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
6309 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
6310 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
6311 },
6312 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6313 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
6314 0, 0
6315 },
6316 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6317 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6318 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6319 },
6320 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6321 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6322 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6323 },
6324 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6325 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6326 0, 0
6327 },
6328 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6329 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6330 0, 0
6331 },
6332 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6333 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6334 0, 0
6335 },
6336 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6337 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6338 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6339 },
6340 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6341 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6342 0, 0
6343 },
6344 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6345 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6346 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6347 },
6348 { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6349 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6350 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6351 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6352 },
6353 { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6354 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6355 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6356 0, 0
6357 },
6358 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6359 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6360 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6361 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6362 },
6363 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6364 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6365 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6366 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6367 },
6368 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6369 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6370 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6371 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6372 },
6373 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6374 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6375 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6376 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6377 },
6378 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6379 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6380 0, 0
6381 },
6382 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6383 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6384 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6385 },
6386 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6387 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6388 0, 0
6389 },
6390 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6391 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6392 0, 0
6393 },
6394 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6395 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6396 0, 0
6397 },
6398 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6399 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6400 0, 0
6401 },
6402 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6403 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6404 0, 0
6405 },
6406 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6407 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6408 0, 0
6409 },
6410 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6411 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6412 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6413 },
6414 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6415 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6416 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6417 },
6418 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6419 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6420 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6421 },
6422 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6423 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6424 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6425 },
6426 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6427 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6428 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6429 },
6430 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6431 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6432 0, 0
6433 },
6434 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6435 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6436 0, 0
6437 },
6438 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6439 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6440 0, 0
6441 },
6442 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6443 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6444 0, 0
6445 },
6446 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6447 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6448 0, 0
6449 },
6450 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6451 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6452 0, 0
6453 },
6454 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6455 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6456 0, 0
6457 },
6458 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6459 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6460 0, 0
6461 },
6462 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6463 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6464 0, 0
6465 },
6466 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6467 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6468 0, 0
6469 },
6470 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6471 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6472 0, 0
6473 },
6474 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6475 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6476 0, 0
6477 },
6478 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6479 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6480 0, 0
6481 },
6482 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6483 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6484 0, 0
6485 },
6486 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6487 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6488 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6489 },
6490 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6491 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6492 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6493 },
6494 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6495 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6496 0, 0
6497 },
6498 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6499 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6500 0, 0
6501 },
6502 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6503 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6504 0, 0
6505 },
6506 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6507 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6508 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6509 },
6510 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6511 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6512 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6513 },
6514 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6515 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6516 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6517 },
6518 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6519 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6520 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6521 },
6522 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6523 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6524 0, 0
6525 },
6526 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6527 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6528 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6529 },
6530 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6531 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6532 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6533 },
6534 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6535 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6536 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6537 },
6538 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6539 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6540 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6541 },
6542 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6543 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6544 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6545 },
6546 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6547 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6548 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6549 },
6550 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6551 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6552 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6553 },
6554 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6555 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6556 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6557 },
6558 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6559 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6560 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6561 },
6562 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6563 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6564 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6565 },
6566 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6567 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6568 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6569 },
6570 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6571 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6572 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6573 },
6574 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6575 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6576 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6577 },
6578 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6579 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6580 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6581 },
6582 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6583 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6584 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6585 },
6586 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6587 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6588 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6589 },
6590 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6591 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6592 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6593 },
6594 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6595 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6596 0, 0
6597 },
6598 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6599 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6600 0, 0
6601 },
6602 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6603 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6604 0, 0
6605 },
6606 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6607 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6608 0, 0
6609 },
6610 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6611 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6612 0, 0
6613 },
6614 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6615 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6616 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6617 },
6618 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6619 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6620 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6621 },
6622 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6623 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6624 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6625 },
6626 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6627 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6628 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6629 },
6630 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6631 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6632 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6633 },
6634 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6635 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6636 0, 0
6637 },
6638 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6639 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6640 0, 0
6641 },
6642 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6643 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6644 0, 0
6645 },
6646 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6647 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6648 0, 0
6649 },
6650 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6651 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6652 0, 0
6653 },
6654 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6655 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6656 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6657 },
6658 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6659 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6660 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6661 },
6662 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6663 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6664 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6665 },
6666 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6667 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6668 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6669 },
6670 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6671 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6672 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6673 },
6674 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6675 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6676 0, 0
6677 },
6678 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6679 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6680 0, 0
6681 },
6682 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6683 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6684 0, 0
6685 },
6686 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6687 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6688 0, 0
6689 },
6690 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6691 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6692 0, 0
6693 },
6694 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6695 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6696 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6697 },
6698 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6699 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6700 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6701 },
6702 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6703 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6704 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6705 },
6706 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6707 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6708 0, 0
6709 },
6710 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6711 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6712 0, 0
6713 },
6714 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6715 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6716 0, 0
6717 },
6718 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6719 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6720 0, 0
6721 },
6722 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6723 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6724 0, 0
6725 },
6726 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6727 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6728 0, 0
6729 }
6730 };
6731
gfx_v9_0_ras_error_inject(struct amdgpu_device * adev,void * inject_if,uint32_t instance_mask)6732 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6733 void *inject_if, uint32_t instance_mask)
6734 {
6735 struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6736 int ret;
6737 struct ta_ras_trigger_error_input block_info = { 0 };
6738
6739 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6740 return -EINVAL;
6741
6742 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6743 return -EINVAL;
6744
6745 if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6746 return -EPERM;
6747
6748 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6749 info->head.type)) {
6750 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6751 ras_gfx_subblocks[info->head.sub_block_index].name,
6752 info->head.type);
6753 return -EPERM;
6754 }
6755
6756 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6757 info->head.type)) {
6758 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6759 ras_gfx_subblocks[info->head.sub_block_index].name,
6760 info->head.type);
6761 return -EPERM;
6762 }
6763
6764 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6765 block_info.sub_block_index =
6766 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6767 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6768 block_info.address = info->address;
6769 block_info.value = info->value;
6770
6771 mutex_lock(&adev->grbm_idx_mutex);
6772 ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);
6773 mutex_unlock(&adev->grbm_idx_mutex);
6774
6775 return ret;
6776 }
6777
6778 static const char * const vml2_mems[] = {
6779 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6780 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6781 "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6782 "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6783 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6784 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6785 "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6786 "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6787 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6788 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6789 "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6790 "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6791 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6792 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6793 "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6794 "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6795 };
6796
6797 static const char * const vml2_walker_mems[] = {
6798 "UTC_VML2_CACHE_PDE0_MEM0",
6799 "UTC_VML2_CACHE_PDE0_MEM1",
6800 "UTC_VML2_CACHE_PDE1_MEM0",
6801 "UTC_VML2_CACHE_PDE1_MEM1",
6802 "UTC_VML2_CACHE_PDE2_MEM0",
6803 "UTC_VML2_CACHE_PDE2_MEM1",
6804 "UTC_VML2_RDIF_LOG_FIFO",
6805 };
6806
6807 static const char * const atc_l2_cache_2m_mems[] = {
6808 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6809 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6810 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6811 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6812 };
6813
6814 static const char *atc_l2_cache_4k_mems[] = {
6815 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6816 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6817 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6818 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6819 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6820 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6821 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6822 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6823 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6824 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6825 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6826 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6827 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6828 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6829 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6830 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6831 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6832 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6833 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6834 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6835 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6836 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6837 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6838 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6839 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6840 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6841 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6842 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6843 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6844 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6845 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6846 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6847 };
6848
gfx_v9_0_query_utc_edc_status(struct amdgpu_device * adev,struct ras_err_data * err_data)6849 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6850 struct ras_err_data *err_data)
6851 {
6852 uint32_t i, data;
6853 uint32_t sec_count, ded_count;
6854
6855 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6856 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6857 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6858 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6859 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6860 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6861 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6862 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6863
6864 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6865 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6866 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6867
6868 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6869 if (sec_count) {
6870 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6871 "SEC %d\n", i, vml2_mems[i], sec_count);
6872 err_data->ce_count += sec_count;
6873 }
6874
6875 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6876 if (ded_count) {
6877 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6878 "DED %d\n", i, vml2_mems[i], ded_count);
6879 err_data->ue_count += ded_count;
6880 }
6881 }
6882
6883 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6884 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6885 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6886
6887 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6888 SEC_COUNT);
6889 if (sec_count) {
6890 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6891 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6892 err_data->ce_count += sec_count;
6893 }
6894
6895 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6896 DED_COUNT);
6897 if (ded_count) {
6898 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6899 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6900 err_data->ue_count += ded_count;
6901 }
6902 }
6903
6904 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6905 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6906 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6907
6908 sec_count = (data & 0x00006000L) >> 0xd;
6909 if (sec_count) {
6910 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6911 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6912 sec_count);
6913 err_data->ce_count += sec_count;
6914 }
6915 }
6916
6917 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6918 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6919 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6920
6921 sec_count = (data & 0x00006000L) >> 0xd;
6922 if (sec_count) {
6923 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6924 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6925 sec_count);
6926 err_data->ce_count += sec_count;
6927 }
6928
6929 ded_count = (data & 0x00018000L) >> 0xf;
6930 if (ded_count) {
6931 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6932 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6933 ded_count);
6934 err_data->ue_count += ded_count;
6935 }
6936 }
6937
6938 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6939 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6940 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6941 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6942
6943 return 0;
6944 }
6945
gfx_v9_0_ras_error_count(struct amdgpu_device * adev,const struct soc15_reg_entry * reg,uint32_t se_id,uint32_t inst_id,uint32_t value,uint32_t * sec_count,uint32_t * ded_count)6946 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6947 const struct soc15_reg_entry *reg,
6948 uint32_t se_id, uint32_t inst_id, uint32_t value,
6949 uint32_t *sec_count, uint32_t *ded_count)
6950 {
6951 uint32_t i;
6952 uint32_t sec_cnt, ded_cnt;
6953
6954 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6955 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6956 gfx_v9_0_ras_fields[i].seg != reg->seg ||
6957 gfx_v9_0_ras_fields[i].inst != reg->inst)
6958 continue;
6959
6960 sec_cnt = (value &
6961 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6962 gfx_v9_0_ras_fields[i].sec_count_shift;
6963 if (sec_cnt) {
6964 dev_info(adev->dev, "GFX SubBlock %s, "
6965 "Instance[%d][%d], SEC %d\n",
6966 gfx_v9_0_ras_fields[i].name,
6967 se_id, inst_id,
6968 sec_cnt);
6969 *sec_count += sec_cnt;
6970 }
6971
6972 ded_cnt = (value &
6973 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6974 gfx_v9_0_ras_fields[i].ded_count_shift;
6975 if (ded_cnt) {
6976 dev_info(adev->dev, "GFX SubBlock %s, "
6977 "Instance[%d][%d], DED %d\n",
6978 gfx_v9_0_ras_fields[i].name,
6979 se_id, inst_id,
6980 ded_cnt);
6981 *ded_count += ded_cnt;
6982 }
6983 }
6984
6985 return 0;
6986 }
6987
gfx_v9_0_reset_ras_error_count(struct amdgpu_device * adev)6988 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6989 {
6990 int i, j, k;
6991
6992 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6993 return;
6994
6995 /* read back registers to clear the counters */
6996 mutex_lock(&adev->grbm_idx_mutex);
6997 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6998 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6999 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7000 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
7001 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7002 }
7003 }
7004 }
7005 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
7006 mutex_unlock(&adev->grbm_idx_mutex);
7007
7008 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7009 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
7010 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7011 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
7012 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7013 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
7014 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7015 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
7016
7017 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
7018 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
7019 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
7020 }
7021
7022 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
7023 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
7024 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
7025 }
7026
7027 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
7028 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
7029 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
7030 }
7031
7032 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
7033 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
7034 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
7035 }
7036
7037 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7038 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7039 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7040 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7041 }
7042
gfx_v9_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)7043 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
7044 void *ras_error_status)
7045 {
7046 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
7047 uint32_t sec_count = 0, ded_count = 0;
7048 uint32_t i, j, k;
7049 uint32_t reg_value;
7050
7051 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7052 return;
7053
7054 err_data->ue_count = 0;
7055 err_data->ce_count = 0;
7056
7057 mutex_lock(&adev->grbm_idx_mutex);
7058
7059 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7060 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7061 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7062 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
7063 reg_value =
7064 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7065 if (reg_value)
7066 gfx_v9_0_ras_error_count(adev,
7067 &gfx_v9_0_edc_counter_regs[i],
7068 j, k, reg_value,
7069 &sec_count, &ded_count);
7070 }
7071 }
7072 }
7073
7074 err_data->ce_count += sec_count;
7075 err_data->ue_count += ded_count;
7076
7077 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7078 mutex_unlock(&adev->grbm_idx_mutex);
7079
7080 gfx_v9_0_query_utc_edc_status(adev, err_data);
7081 }
7082
gfx_v9_0_emit_mem_sync(struct amdgpu_ring * ring)7083 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
7084 {
7085 const unsigned int cp_coher_cntl =
7086 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
7087 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
7088 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
7089 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
7090 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
7091
7092 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
7093 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
7094 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
7095 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
7096 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
7097 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
7098 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
7099 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
7100 }
7101
gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring * ring,uint32_t pipe,bool enable)7102 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
7103 uint32_t pipe, bool enable)
7104 {
7105 struct amdgpu_device *adev = ring->adev;
7106 uint32_t val;
7107 uint32_t wcl_cs_reg;
7108
7109 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
7110 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
7111
7112 switch (pipe) {
7113 case 0:
7114 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
7115 break;
7116 case 1:
7117 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
7118 break;
7119 case 2:
7120 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
7121 break;
7122 case 3:
7123 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
7124 break;
7125 default:
7126 DRM_DEBUG("invalid pipe %d\n", pipe);
7127 return;
7128 }
7129
7130 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
7131
7132 }
gfx_v9_0_emit_wave_limit(struct amdgpu_ring * ring,bool enable)7133 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
7134 {
7135 struct amdgpu_device *adev = ring->adev;
7136 uint32_t val;
7137 int i;
7138
7139
7140 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
7141 * number of gfx waves. Setting 5 bit will make sure gfx only gets
7142 * around 25% of gpu resources.
7143 */
7144 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
7145 amdgpu_ring_emit_wreg(ring,
7146 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
7147 val);
7148
7149 /* Restrict waves for normal/low priority compute queues as well
7150 * to get best QoS for high priority compute jobs.
7151 *
7152 * amdgpu controls only 1st ME(0-3 CS pipes).
7153 */
7154 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
7155 if (i != ring->pipe)
7156 gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
7157
7158 }
7159 }
7160
gfx_v9_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)7161 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
7162 {
7163 /* Header itself is a NOP packet */
7164 if (num_nop == 1) {
7165 amdgpu_ring_write(ring, ring->funcs->nop);
7166 return;
7167 }
7168
7169 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
7170 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
7171
7172 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
7173 amdgpu_ring_insert_nop(ring, num_nop - 1);
7174 }
7175
gfx_v9_0_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)7176 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,
7177 unsigned int vmid,
7178 struct amdgpu_fence *timedout_fence)
7179 {
7180 struct amdgpu_device *adev = ring->adev;
7181 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7182 struct amdgpu_ring *kiq_ring = &kiq->ring;
7183 unsigned long flags;
7184 int i, r;
7185
7186 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7187 return -EINVAL;
7188
7189 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
7190
7191 spin_lock_irqsave(&kiq->ring_lock, flags);
7192
7193 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
7194 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7195 return -ENOMEM;
7196 }
7197
7198 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
7199 0, 0);
7200 amdgpu_ring_commit(kiq_ring);
7201
7202 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7203
7204 r = amdgpu_ring_test_ring(kiq_ring);
7205 if (r)
7206 return r;
7207
7208 /* make sure dequeue is complete*/
7209 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
7210 mutex_lock(&adev->srbm_mutex);
7211 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
7212 for (i = 0; i < adev->usec_timeout; i++) {
7213 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
7214 break;
7215 udelay(1);
7216 }
7217 if (i >= adev->usec_timeout)
7218 r = -ETIMEDOUT;
7219 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7220 mutex_unlock(&adev->srbm_mutex);
7221 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
7222 if (r) {
7223 dev_err(adev->dev, "fail to wait on hqd deactive\n");
7224 return r;
7225 }
7226
7227 r = gfx_v9_0_kcq_init_queue(ring, true);
7228 if (r) {
7229 dev_err(adev->dev, "fail to init kcq\n");
7230 return r;
7231 }
7232 spin_lock_irqsave(&kiq->ring_lock, flags);
7233 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
7234 if (r) {
7235 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7236 return -ENOMEM;
7237 }
7238 kiq->pmf->kiq_map_queues(kiq_ring, ring);
7239 amdgpu_ring_commit(kiq_ring);
7240 r = amdgpu_ring_test_ring(kiq_ring);
7241 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7242 if (r) {
7243 DRM_ERROR("fail to remap queue\n");
7244 return r;
7245 }
7246 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
7247 }
7248
gfx_v9_ip_print(struct amdgpu_ip_block * ip_block,struct drm_printer * p)7249 static void gfx_v9_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
7250 {
7251 struct amdgpu_device *adev = ip_block->adev;
7252 uint32_t i, j, k, reg, index = 0;
7253 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7254
7255 if (!adev->gfx.ip_dump_core)
7256 return;
7257
7258 for (i = 0; i < reg_count; i++)
7259 drm_printf(p, "%-50s \t 0x%08x\n",
7260 gc_reg_list_9[i].reg_name,
7261 adev->gfx.ip_dump_core[i]);
7262
7263 /* print compute queue registers for all instances */
7264 if (!adev->gfx.ip_dump_compute_queues)
7265 return;
7266
7267 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7268 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
7269 adev->gfx.mec.num_mec,
7270 adev->gfx.mec.num_pipe_per_mec,
7271 adev->gfx.mec.num_queue_per_pipe);
7272
7273 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7274 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7275 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7276 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
7277 for (reg = 0; reg < reg_count; reg++) {
7278 if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP)
7279 drm_printf(p, "%-50s \t 0x%08x\n",
7280 "mmCP_MEC_ME2_HEADER_DUMP",
7281 adev->gfx.ip_dump_compute_queues[index + reg]);
7282 else
7283 drm_printf(p, "%-50s \t 0x%08x\n",
7284 gc_cp_reg_list_9[reg].reg_name,
7285 adev->gfx.ip_dump_compute_queues[index + reg]);
7286 }
7287 index += reg_count;
7288 }
7289 }
7290 }
7291
7292 }
7293
gfx_v9_ip_dump(struct amdgpu_ip_block * ip_block)7294 static void gfx_v9_ip_dump(struct amdgpu_ip_block *ip_block)
7295 {
7296 struct amdgpu_device *adev = ip_block->adev;
7297 uint32_t i, j, k, reg, index = 0;
7298 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7299
7300 if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)
7301 return;
7302
7303 amdgpu_gfx_off_ctrl(adev, false);
7304 for (i = 0; i < reg_count; i++)
7305 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));
7306 amdgpu_gfx_off_ctrl(adev, true);
7307
7308 /* dump compute queue registers for all instances */
7309 if (!adev->gfx.ip_dump_compute_queues)
7310 return;
7311
7312 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7313 amdgpu_gfx_off_ctrl(adev, false);
7314 mutex_lock(&adev->srbm_mutex);
7315 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7316 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7317 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7318 /* ME0 is for GFX so start from 1 for CP */
7319 soc15_grbm_select(adev, 1 + i, j, k, 0, 0);
7320
7321 for (reg = 0; reg < reg_count; reg++) {
7322 if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP)
7323 adev->gfx.ip_dump_compute_queues[index + reg] =
7324 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME2_HEADER_DUMP));
7325 else
7326 adev->gfx.ip_dump_compute_queues[index + reg] =
7327 RREG32(SOC15_REG_ENTRY_OFFSET(
7328 gc_cp_reg_list_9[reg]));
7329 }
7330 index += reg_count;
7331 }
7332 }
7333 }
7334 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7335 mutex_unlock(&adev->srbm_mutex);
7336 amdgpu_gfx_off_ctrl(adev, true);
7337
7338 }
7339
gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring * ring)7340 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7341 {
7342 struct amdgpu_device *adev = ring->adev;
7343
7344 /* Emit the cleaner shader */
7345 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
7346 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7347 else
7348 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER_9_0, 0));
7349
7350 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */
7351 }
7352
gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring * ring)7353 static void gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring *ring)
7354 {
7355 struct amdgpu_device *adev = ring->adev;
7356 struct amdgpu_ip_block *gfx_block =
7357 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7358
7359 amdgpu_gfx_enforce_isolation_ring_begin_use(ring);
7360
7361 /* Raven and PCO APUs seem to have stability issues
7362 * with compute and gfxoff and gfx pg. Disable gfx pg during
7363 * submission and allow again afterwards.
7364 */
7365 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7366 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_UNGATE);
7367 }
7368
gfx_v9_0_ring_end_use_compute(struct amdgpu_ring * ring)7369 static void gfx_v9_0_ring_end_use_compute(struct amdgpu_ring *ring)
7370 {
7371 struct amdgpu_device *adev = ring->adev;
7372 struct amdgpu_ip_block *gfx_block =
7373 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7374
7375 /* Raven and PCO APUs seem to have stability issues
7376 * with compute and gfxoff and gfx pg. Disable gfx pg during
7377 * submission and allow again afterwards.
7378 */
7379 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7380 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_GATE);
7381
7382 amdgpu_gfx_enforce_isolation_ring_end_use(ring);
7383 }
7384
7385 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
7386 .name = "gfx_v9_0",
7387 .early_init = gfx_v9_0_early_init,
7388 .late_init = gfx_v9_0_late_init,
7389 .sw_init = gfx_v9_0_sw_init,
7390 .sw_fini = gfx_v9_0_sw_fini,
7391 .hw_init = gfx_v9_0_hw_init,
7392 .hw_fini = gfx_v9_0_hw_fini,
7393 .suspend = gfx_v9_0_suspend,
7394 .resume = gfx_v9_0_resume,
7395 .is_idle = gfx_v9_0_is_idle,
7396 .wait_for_idle = gfx_v9_0_wait_for_idle,
7397 .soft_reset = gfx_v9_0_soft_reset,
7398 .set_clockgating_state = gfx_v9_0_set_clockgating_state,
7399 .set_powergating_state = gfx_v9_0_set_powergating_state,
7400 .get_clockgating_state = gfx_v9_0_get_clockgating_state,
7401 .dump_ip_state = gfx_v9_ip_dump,
7402 .print_ip_state = gfx_v9_ip_print,
7403 };
7404
7405 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
7406 .type = AMDGPU_RING_TYPE_GFX,
7407 .align_mask = 0xff,
7408 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7409 .support_64bit_ptrs = true,
7410 .secure_submission_supported = true,
7411 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
7412 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
7413 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
7414 .emit_frame_size = /* totally 242 maximum if 16 IBs */
7415 5 + /* COND_EXEC */
7416 7 + /* PIPELINE_SYNC */
7417 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7418 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7419 2 + /* VM_FLUSH */
7420 8 + /* FENCE for VM_FLUSH */
7421 20 + /* GDS switch */
7422 4 + /* double SWITCH_BUFFER,
7423 the first COND_EXEC jump to the place just
7424 prior to this double SWITCH_BUFFER */
7425 5 + /* COND_EXEC */
7426 7 + /* HDP_flush */
7427 4 + /* VGT_flush */
7428 14 + /* CE_META */
7429 31 + /* DE_META */
7430 3 + /* CNTX_CTRL */
7431 5 + /* HDP_INVL */
7432 8 + 8 + /* FENCE x2 */
7433 2 + /* SWITCH_BUFFER */
7434 7 + /* gfx_v9_0_emit_mem_sync */
7435 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7436 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7437 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7438 .emit_fence = gfx_v9_0_ring_emit_fence,
7439 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7440 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7441 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7442 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7443 .test_ring = gfx_v9_0_ring_test_ring,
7444 .insert_nop = gfx_v9_ring_insert_nop,
7445 .pad_ib = amdgpu_ring_generic_pad_ib,
7446 .emit_switch_buffer = gfx_v9_ring_emit_sb,
7447 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7448 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7449 .preempt_ib = gfx_v9_0_ring_preempt_ib,
7450 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7451 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7452 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7453 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7454 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7455 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7456 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7457 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7458 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7459 };
7460
7461 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
7462 .type = AMDGPU_RING_TYPE_GFX,
7463 .align_mask = 0xff,
7464 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7465 .support_64bit_ptrs = true,
7466 .secure_submission_supported = true,
7467 .get_rptr = amdgpu_sw_ring_get_rptr_gfx,
7468 .get_wptr = amdgpu_sw_ring_get_wptr_gfx,
7469 .set_wptr = amdgpu_sw_ring_set_wptr_gfx,
7470 .emit_frame_size = /* totally 242 maximum if 16 IBs */
7471 5 + /* COND_EXEC */
7472 7 + /* PIPELINE_SYNC */
7473 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7474 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7475 2 + /* VM_FLUSH */
7476 8 + /* FENCE for VM_FLUSH */
7477 20 + /* GDS switch */
7478 4 + /* double SWITCH_BUFFER,
7479 * the first COND_EXEC jump to the place just
7480 * prior to this double SWITCH_BUFFER
7481 */
7482 5 + /* COND_EXEC */
7483 7 + /* HDP_flush */
7484 4 + /* VGT_flush */
7485 14 + /* CE_META */
7486 31 + /* DE_META */
7487 3 + /* CNTX_CTRL */
7488 5 + /* HDP_INVL */
7489 8 + 8 + /* FENCE x2 */
7490 2 + /* SWITCH_BUFFER */
7491 7 + /* gfx_v9_0_emit_mem_sync */
7492 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7493 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7494 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7495 .emit_fence = gfx_v9_0_ring_emit_fence,
7496 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7497 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7498 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7499 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7500 .test_ring = gfx_v9_0_ring_test_ring,
7501 .test_ib = gfx_v9_0_ring_test_ib,
7502 .insert_nop = gfx_v9_ring_insert_nop,
7503 .pad_ib = amdgpu_ring_generic_pad_ib,
7504 .emit_switch_buffer = gfx_v9_ring_emit_sb,
7505 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7506 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7507 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7508 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7509 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7510 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7511 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7512 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7513 .patch_cntl = gfx_v9_0_ring_patch_cntl,
7514 .patch_de = gfx_v9_0_ring_patch_de_meta,
7515 .patch_ce = gfx_v9_0_ring_patch_ce_meta,
7516 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7517 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7518 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7519 };
7520
7521 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
7522 .type = AMDGPU_RING_TYPE_COMPUTE,
7523 .align_mask = 0xff,
7524 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7525 .support_64bit_ptrs = true,
7526 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7527 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7528 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7529 .emit_frame_size =
7530 20 + /* gfx_v9_0_ring_emit_gds_switch */
7531 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7532 5 + /* hdp invalidate */
7533 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7534 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7535 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7536 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
7537 7 + /* gfx_v9_0_emit_mem_sync */
7538 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
7539 15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
7540 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7541 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7542 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
7543 .emit_fence = gfx_v9_0_ring_emit_fence,
7544 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7545 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7546 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7547 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7548 .test_ring = gfx_v9_0_ring_test_ring,
7549 .test_ib = gfx_v9_0_ring_test_ib,
7550 .insert_nop = gfx_v9_ring_insert_nop,
7551 .pad_ib = amdgpu_ring_generic_pad_ib,
7552 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7553 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7554 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7555 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7556 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7557 .emit_wave_limit = gfx_v9_0_emit_wave_limit,
7558 .reset = gfx_v9_0_reset_kcq,
7559 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7560 .begin_use = gfx_v9_0_ring_begin_use_compute,
7561 .end_use = gfx_v9_0_ring_end_use_compute,
7562 };
7563
7564 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7565 .type = AMDGPU_RING_TYPE_KIQ,
7566 .align_mask = 0xff,
7567 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7568 .support_64bit_ptrs = true,
7569 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7570 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7571 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7572 .emit_frame_size =
7573 20 + /* gfx_v9_0_ring_emit_gds_switch */
7574 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7575 5 + /* hdp invalidate */
7576 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7577 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7578 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7579 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7580 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7581 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7582 .test_ring = gfx_v9_0_ring_test_ring,
7583 .insert_nop = amdgpu_ring_insert_nop,
7584 .pad_ib = amdgpu_ring_generic_pad_ib,
7585 .emit_rreg = gfx_v9_0_ring_emit_rreg,
7586 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7587 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7588 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7589 };
7590
gfx_v9_0_set_ring_funcs(struct amdgpu_device * adev)7591 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7592 {
7593 int i;
7594
7595 adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7596
7597 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7598 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7599
7600 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
7601 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7602 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7603 }
7604
7605 for (i = 0; i < adev->gfx.num_compute_rings; i++)
7606 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7607 }
7608
7609 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7610 .set = gfx_v9_0_set_eop_interrupt_state,
7611 .process = gfx_v9_0_eop_irq,
7612 };
7613
7614 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7615 .set = gfx_v9_0_set_priv_reg_fault_state,
7616 .process = gfx_v9_0_priv_reg_irq,
7617 };
7618
7619 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = {
7620 .set = gfx_v9_0_set_bad_op_fault_state,
7621 .process = gfx_v9_0_bad_op_irq,
7622 };
7623
7624 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7625 .set = gfx_v9_0_set_priv_inst_fault_state,
7626 .process = gfx_v9_0_priv_inst_irq,
7627 };
7628
7629 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7630 .set = gfx_v9_0_set_cp_ecc_error_state,
7631 .process = amdgpu_gfx_cp_ecc_error_irq,
7632 };
7633
7634
gfx_v9_0_set_irq_funcs(struct amdgpu_device * adev)7635 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7636 {
7637 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7638 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7639
7640 adev->gfx.priv_reg_irq.num_types = 1;
7641 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7642
7643 adev->gfx.bad_op_irq.num_types = 1;
7644 adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;
7645
7646 adev->gfx.priv_inst_irq.num_types = 1;
7647 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7648
7649 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7650 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7651 }
7652
gfx_v9_0_set_rlc_funcs(struct amdgpu_device * adev)7653 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7654 {
7655 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7656 case IP_VERSION(9, 0, 1):
7657 case IP_VERSION(9, 2, 1):
7658 case IP_VERSION(9, 4, 0):
7659 case IP_VERSION(9, 2, 2):
7660 case IP_VERSION(9, 1, 0):
7661 case IP_VERSION(9, 4, 1):
7662 case IP_VERSION(9, 3, 0):
7663 case IP_VERSION(9, 4, 2):
7664 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7665 break;
7666 default:
7667 break;
7668 }
7669 }
7670
gfx_v9_0_set_gds_init(struct amdgpu_device * adev)7671 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7672 {
7673 /* init asci gds info */
7674 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7675 case IP_VERSION(9, 0, 1):
7676 case IP_VERSION(9, 2, 1):
7677 case IP_VERSION(9, 4, 0):
7678 adev->gds.gds_size = 0x10000;
7679 break;
7680 case IP_VERSION(9, 2, 2):
7681 case IP_VERSION(9, 1, 0):
7682 case IP_VERSION(9, 4, 1):
7683 adev->gds.gds_size = 0x1000;
7684 break;
7685 case IP_VERSION(9, 4, 2):
7686 /* aldebaran removed all the GDS internal memory,
7687 * only support GWS opcode in kernel, like barrier
7688 * semaphore.etc */
7689 adev->gds.gds_size = 0;
7690 break;
7691 default:
7692 adev->gds.gds_size = 0x10000;
7693 break;
7694 }
7695
7696 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7697 case IP_VERSION(9, 0, 1):
7698 case IP_VERSION(9, 4, 0):
7699 adev->gds.gds_compute_max_wave_id = 0x7ff;
7700 break;
7701 case IP_VERSION(9, 2, 1):
7702 adev->gds.gds_compute_max_wave_id = 0x27f;
7703 break;
7704 case IP_VERSION(9, 2, 2):
7705 case IP_VERSION(9, 1, 0):
7706 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7707 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7708 else
7709 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7710 break;
7711 case IP_VERSION(9, 4, 1):
7712 adev->gds.gds_compute_max_wave_id = 0xfff;
7713 break;
7714 case IP_VERSION(9, 4, 2):
7715 /* deprecated for Aldebaran, no usage at all */
7716 adev->gds.gds_compute_max_wave_id = 0;
7717 break;
7718 default:
7719 /* this really depends on the chip */
7720 adev->gds.gds_compute_max_wave_id = 0x7ff;
7721 break;
7722 }
7723
7724 adev->gds.gws_size = 64;
7725 adev->gds.oa_size = 16;
7726 }
7727
gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device * adev,u32 bitmap)7728 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7729 u32 bitmap)
7730 {
7731 u32 data;
7732
7733 if (!bitmap)
7734 return;
7735
7736 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7737 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7738
7739 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7740 }
7741
gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device * adev)7742 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7743 {
7744 u32 data, mask;
7745
7746 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7747 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7748
7749 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7750 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7751
7752 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7753
7754 return (~data) & mask;
7755 }
7756
gfx_v9_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7757 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7758 struct amdgpu_cu_info *cu_info)
7759 {
7760 int i, j, k, counter, active_cu_number = 0;
7761 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7762 unsigned disable_masks[4 * 4];
7763
7764 if (!adev || !cu_info)
7765 return -EINVAL;
7766
7767 /*
7768 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7769 */
7770 if (adev->gfx.config.max_shader_engines *
7771 adev->gfx.config.max_sh_per_se > 16)
7772 return -EINVAL;
7773
7774 amdgpu_gfx_parse_disable_cu(disable_masks,
7775 adev->gfx.config.max_shader_engines,
7776 adev->gfx.config.max_sh_per_se);
7777
7778 mutex_lock(&adev->grbm_idx_mutex);
7779 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7780 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7781 mask = 1;
7782 ao_bitmap = 0;
7783 counter = 0;
7784 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7785 gfx_v9_0_set_user_cu_inactive_bitmap(
7786 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7787 bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7788
7789 /*
7790 * The bitmap(and ao_cu_bitmap) in cu_info structure is
7791 * 4x4 size array, and it's usually suitable for Vega
7792 * ASICs which has 4*2 SE/SH layout.
7793 * But for Arcturus, SE/SH layout is changed to 8*1.
7794 * To mostly reduce the impact, we make it compatible
7795 * with current bitmap array as below:
7796 * SE4,SH0 --> bitmap[0][1]
7797 * SE5,SH0 --> bitmap[1][1]
7798 * SE6,SH0 --> bitmap[2][1]
7799 * SE7,SH0 --> bitmap[3][1]
7800 */
7801 cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7802
7803 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7804 if (bitmap & mask) {
7805 if (counter < adev->gfx.config.max_cu_per_sh)
7806 ao_bitmap |= mask;
7807 counter ++;
7808 }
7809 mask <<= 1;
7810 }
7811 active_cu_number += counter;
7812 if (i < 2 && j < 2)
7813 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7814 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7815 }
7816 }
7817 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7818 mutex_unlock(&adev->grbm_idx_mutex);
7819
7820 cu_info->number = active_cu_number;
7821 cu_info->ao_cu_mask = ao_cu_mask;
7822 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7823
7824 return 0;
7825 }
7826
7827 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7828 {
7829 .type = AMD_IP_BLOCK_TYPE_GFX,
7830 .major = 9,
7831 .minor = 0,
7832 .rev = 0,
7833 .funcs = &gfx_v9_0_ip_funcs,
7834 };
7835