1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29
30 #include "amdgpu.h"
31 #include "amdgpu_gfx.h"
32 #include "soc15.h"
33 #include "soc15d.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_pm.h"
36
37 #include "gc/gc_9_0_offset.h"
38 #include "gc/gc_9_0_sh_mask.h"
39
40 #include "vega10_enum.h"
41
42 #include "soc15_common.h"
43 #include "clearstate_gfx9.h"
44 #include "v9_structs.h"
45
46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
47
48 #include "amdgpu_ras.h"
49
50 #include "amdgpu_ring_mux.h"
51 #include "gfx_v9_4.h"
52 #include "gfx_v9_0.h"
53 #include "gfx_v9_0_cleaner_shader.h"
54 #include "gfx_v9_4_2.h"
55
56 #include "asic_reg/pwr/pwr_10_0_offset.h"
57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h"
58 #include "asic_reg/gc/gc_9_0_default.h"
59
60 #define GFX9_NUM_GFX_RINGS 1
61 #define GFX9_NUM_SW_GFX_RINGS 2
62 #define GFX9_MEC_HPD_SIZE 4096
63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
65
66 #define mmGCEA_PROBE_MAP 0x070c
67 #define mmGCEA_PROBE_MAP_BASE_IDX 0
68
69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
71 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
75
76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
78 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
82
83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
85 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
89
90 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
92 MODULE_FIRMWARE("amdgpu/raven_me.bin");
93 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
96
97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
104
105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
107 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
112
113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
115
116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
118 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
121
122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin");
123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin");
124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin");
125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin");
126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin");
127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
128
129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
134
135 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03
136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
137 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04
138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
139 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09
140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
141 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
143 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
145 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
147
148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025
149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1
150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026
151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1
152
153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = {
154 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS),
155 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2),
156 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1),
157 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2),
158 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1),
159 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1),
160 SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT),
161 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT),
162 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT),
163 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS),
164 SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR),
165 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE),
166 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR),
167 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR),
168 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE),
169 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR),
170 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR),
171 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE),
172 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR),
173 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR),
174 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE),
175 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
176 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR),
177 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ),
178 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ),
179 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ),
180 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ),
181 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO),
182 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI),
183 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ),
184 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO),
185 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI),
186 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ),
187 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO),
188 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI),
189 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ),
190 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO),
191 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI),
192 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ),
193 SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS),
194 SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS),
195 SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS),
196 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT),
197 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT),
198 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS),
199 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL),
200 SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS),
201 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS),
202 SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS),
203 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS),
204 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS),
205 SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS),
206 SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS),
207 SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS),
208 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL),
209 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS),
210 SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG),
211 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL),
212 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR),
213 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR),
214 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR),
215 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR),
216 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR),
217 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS),
218 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT),
219 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND),
220 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE),
221 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1),
222 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2),
223 SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE),
224 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE),
225 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE),
226 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT),
227 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6),
228 /* SE status registers */
229 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0),
230 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1),
231 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2),
232 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3),
233 /* packet headers */
234 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
235 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
236 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
237 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
238 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
239 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
240 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
241 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP),
242 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
243 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
244 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
245 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
246 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
247 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
248 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
249 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP),
250 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
251 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
252 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
253 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
254 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
255 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
256 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP),
257 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP)
258 };
259
260 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = {
261 /* compute queue registers */
262 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID),
263 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE),
264 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE),
265 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY),
266 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY),
267 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM),
268 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE),
269 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI),
270 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR),
271 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR),
272 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI),
273 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL),
274 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL),
275 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR),
276 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI),
277 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR),
278 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL),
279 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST),
280 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR),
281 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI),
282 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL),
283 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR),
284 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR),
285 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS),
286 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO),
287 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI),
288 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL),
289 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET),
290 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE),
291 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET),
292 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE),
293 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE),
294 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR),
295 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM),
296 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO),
297 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI),
298 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS),
299 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
300 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
301 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
302 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
303 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
304 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
305 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP),
306 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP)
307 };
308
309 enum ta_ras_gfx_subblock {
310 /*CPC*/
311 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
312 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
313 TA_RAS_BLOCK__GFX_CPC_UCODE,
314 TA_RAS_BLOCK__GFX_DC_STATE_ME1,
315 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
316 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
317 TA_RAS_BLOCK__GFX_DC_STATE_ME2,
318 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
319 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
320 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
321 /* CPF*/
322 TA_RAS_BLOCK__GFX_CPF_INDEX_START,
323 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
324 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
325 TA_RAS_BLOCK__GFX_CPF_TAG,
326 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
327 /* CPG*/
328 TA_RAS_BLOCK__GFX_CPG_INDEX_START,
329 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
330 TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
331 TA_RAS_BLOCK__GFX_CPG_TAG,
332 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
333 /* GDS*/
334 TA_RAS_BLOCK__GFX_GDS_INDEX_START,
335 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
336 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
337 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
338 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
339 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
340 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
341 /* SPI*/
342 TA_RAS_BLOCK__GFX_SPI_SR_MEM,
343 /* SQ*/
344 TA_RAS_BLOCK__GFX_SQ_INDEX_START,
345 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
346 TA_RAS_BLOCK__GFX_SQ_LDS_D,
347 TA_RAS_BLOCK__GFX_SQ_LDS_I,
348 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
349 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
350 /* SQC (3 ranges)*/
351 TA_RAS_BLOCK__GFX_SQC_INDEX_START,
352 /* SQC range 0*/
353 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
354 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
355 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
356 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
357 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
358 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
359 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
360 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
361 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
362 TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
363 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
364 /* SQC range 1*/
365 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
366 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
367 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
368 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
369 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
370 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
371 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
372 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
373 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
374 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
375 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
376 TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
377 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
378 /* SQC range 2*/
379 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
380 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
381 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
382 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
383 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
384 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
385 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
386 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
387 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
388 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
389 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
390 TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
391 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
392 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
393 /* TA*/
394 TA_RAS_BLOCK__GFX_TA_INDEX_START,
395 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
396 TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
397 TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
398 TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
399 TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
400 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
401 /* TCA*/
402 TA_RAS_BLOCK__GFX_TCA_INDEX_START,
403 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
404 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
405 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
406 /* TCC (5 sub-ranges)*/
407 TA_RAS_BLOCK__GFX_TCC_INDEX_START,
408 /* TCC range 0*/
409 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
410 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
411 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
412 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
413 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
414 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
415 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
416 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
417 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
418 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
419 /* TCC range 1*/
420 TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
421 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
422 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
423 TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
424 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
425 /* TCC range 2*/
426 TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
427 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
428 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
429 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
430 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
431 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
432 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
433 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
434 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
435 TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
436 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
437 /* TCC range 3*/
438 TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
439 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
440 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
441 TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
442 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
443 /* TCC range 4*/
444 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
445 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
446 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
447 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
448 TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
449 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
450 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
451 /* TCI*/
452 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
453 /* TCP*/
454 TA_RAS_BLOCK__GFX_TCP_INDEX_START,
455 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
456 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
457 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
458 TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
459 TA_RAS_BLOCK__GFX_TCP_DB_RAM,
460 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
461 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
462 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
463 /* TD*/
464 TA_RAS_BLOCK__GFX_TD_INDEX_START,
465 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
466 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
467 TA_RAS_BLOCK__GFX_TD_CS_FIFO,
468 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
469 /* EA (3 sub-ranges)*/
470 TA_RAS_BLOCK__GFX_EA_INDEX_START,
471 /* EA range 0*/
472 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
473 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
474 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
475 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
476 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
477 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
478 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
479 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
480 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
481 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
482 /* EA range 1*/
483 TA_RAS_BLOCK__GFX_EA_INDEX1_START,
484 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
485 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
486 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
487 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
488 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
489 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
490 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
491 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
492 /* EA range 2*/
493 TA_RAS_BLOCK__GFX_EA_INDEX2_START,
494 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
495 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
496 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
497 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
498 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
499 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
500 /* UTC VM L2 bank*/
501 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
502 /* UTC VM walker*/
503 TA_RAS_BLOCK__UTC_VML2_WALKER,
504 /* UTC ATC L2 2MB cache*/
505 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
506 /* UTC ATC L2 4KB cache*/
507 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
508 TA_RAS_BLOCK__GFX_MAX
509 };
510
511 struct ras_gfx_subblock {
512 unsigned char *name;
513 int ta_subblock;
514 int hw_supported_error_type;
515 int sw_supported_error_type;
516 };
517
518 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
519 [AMDGPU_RAS_BLOCK__##subblock] = { \
520 #subblock, \
521 TA_RAS_BLOCK__##subblock, \
522 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
523 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
524 }
525
526 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
527 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
528 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
529 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
530 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
531 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
532 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
533 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
534 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
535 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
536 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
537 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
538 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
539 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
540 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
541 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
542 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
543 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
544 0),
545 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
546 0),
547 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
548 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
549 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
550 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
551 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
552 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
553 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
554 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
555 0, 0),
556 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
557 0),
558 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
559 0, 0),
560 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
561 0),
562 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
563 0, 0),
564 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
565 0),
566 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
567 1),
568 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
569 0, 0, 0),
570 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
571 0),
572 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
573 0),
574 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
575 0),
576 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
577 0),
578 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
579 0),
580 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
581 0, 0),
582 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
583 0),
584 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
585 0),
586 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
587 0, 0, 0),
588 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
589 0),
590 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
591 0),
592 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
593 0),
594 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
595 0),
596 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
597 0),
598 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
599 0, 0),
600 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
601 0),
602 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
603 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
604 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
605 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
606 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
607 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
608 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
609 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
610 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
611 1),
612 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
613 1),
614 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
615 1),
616 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
617 0),
618 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
619 0),
620 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
621 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
622 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
623 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
624 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
625 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
626 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
627 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
628 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
629 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
630 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
631 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
632 0),
633 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
634 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
635 0),
636 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
637 0, 0),
638 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
639 0),
640 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
641 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
642 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
643 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
644 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
645 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
646 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
647 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
648 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
649 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
650 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
651 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
652 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
653 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
654 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
655 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
656 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
657 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
658 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
659 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
660 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
661 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
662 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
663 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
664 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
665 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
666 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
667 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
668 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
669 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
670 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
671 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
672 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
673 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
674 };
675
676 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
677 {
678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87),
688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f),
689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
698 };
699
700 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
701 {
702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
707 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
708 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
709 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
710 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
711 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
712 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
713 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
714 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
715 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
716 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
717 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
718 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
719 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
720 };
721
722 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
723 {
724 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
725 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
726 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
727 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
728 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
729 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
730 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
731 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
732 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
733 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
734 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
735 };
736
737 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
738 {
739 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
740 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
741 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
742 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
743 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
744 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
745 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
746 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
747 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
748 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
749 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
750 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
751 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
752 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
753 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
754 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
755 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
756 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
757 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
758 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
759 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
760 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
761 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
762 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
763 };
764
765 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
766 {
767 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
768 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
769 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
770 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
771 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
772 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
773 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
774 };
775
776 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
777 {
778 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
779 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
780 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
781 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
782 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
783 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
784 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
785 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
786 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
787 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
788 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
789 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
790 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
791 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
792 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
793 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
794 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
795 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
796 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
797 };
798
799 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
800 {
801 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
802 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
803 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
804 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
805 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
806 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
807 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
808 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
809 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
810 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
811 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
812 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
813 };
814
815 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
816 {
817 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
818 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
819 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
820 };
821
822 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
823 {
824 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
825 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
826 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
827 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
828 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
829 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
830 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
831 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
832 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
833 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
834 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
835 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
836 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
837 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
838 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
839 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
840 };
841
842 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
843 {
844 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
845 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
846 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
847 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
848 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
849 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
850 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
851 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
852 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
853 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
854 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
855 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
856 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
857 };
858
859 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
860 {
861 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
862 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
863 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
864 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
865 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
866 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
867 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
868 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
869 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
870 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
871 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000)
872 };
873
874 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
875 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
876 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
877 };
878
879 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
880 {
881 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
882 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
883 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
884 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
885 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
886 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
887 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
888 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
889 };
890
891 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
892 {
893 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
894 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
895 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
896 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
897 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
898 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
899 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
900 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
901 };
902
903 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
904 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
905 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
906 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
907
908 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
909 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
910 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
911 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
912 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
913 struct amdgpu_cu_info *cu_info);
914 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
915 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds);
916 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
917 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
918 void *ras_error_status);
919 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
920 void *inject_if, uint32_t instance_mask);
921 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
922 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
923 unsigned int vmid);
924 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
925 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
926
gfx_v9_0_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)927 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
928 uint64_t queue_mask)
929 {
930 struct amdgpu_device *adev = kiq_ring->adev;
931 u64 shader_mc_addr;
932
933 /* Cleaner shader MC address */
934 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
935
936 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
937 amdgpu_ring_write(kiq_ring,
938 PACKET3_SET_RESOURCES_VMID_MASK(0) |
939 /* vmid_mask:0* queue_type:0 (KIQ) */
940 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
941 amdgpu_ring_write(kiq_ring,
942 lower_32_bits(queue_mask)); /* queue mask lo */
943 amdgpu_ring_write(kiq_ring,
944 upper_32_bits(queue_mask)); /* queue mask hi */
945 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
946 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
947 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
948 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
949 }
950
gfx_v9_0_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)951 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
952 struct amdgpu_ring *ring)
953 {
954 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
955 uint64_t wptr_addr = ring->wptr_gpu_addr;
956 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
957
958 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
959 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
960 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
961 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
962 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
963 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
964 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
965 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
966 /*queue_type: normal compute queue */
967 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
968 /* alloc format: all_on_one_pipe */
969 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
970 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
971 /* num_queues: must be 1 */
972 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
973 amdgpu_ring_write(kiq_ring,
974 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
975 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
976 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
977 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
978 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
979 }
980
gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)981 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
982 struct amdgpu_ring *ring,
983 enum amdgpu_unmap_queues_action action,
984 u64 gpu_addr, u64 seq)
985 {
986 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
987
988 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
989 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
990 PACKET3_UNMAP_QUEUES_ACTION(action) |
991 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
992 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
993 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
994 amdgpu_ring_write(kiq_ring,
995 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
996
997 if (action == PREEMPT_QUEUES_NO_UNMAP) {
998 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask));
999 amdgpu_ring_write(kiq_ring, 0);
1000 amdgpu_ring_write(kiq_ring, 0);
1001
1002 } else {
1003 amdgpu_ring_write(kiq_ring, 0);
1004 amdgpu_ring_write(kiq_ring, 0);
1005 amdgpu_ring_write(kiq_ring, 0);
1006 }
1007 }
1008
gfx_v9_0_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)1009 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
1010 struct amdgpu_ring *ring,
1011 u64 addr,
1012 u64 seq)
1013 {
1014 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
1015
1016 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
1017 amdgpu_ring_write(kiq_ring,
1018 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
1019 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
1020 PACKET3_QUERY_STATUS_COMMAND(2));
1021 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
1022 amdgpu_ring_write(kiq_ring,
1023 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
1024 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
1025 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
1026 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
1027 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
1028 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
1029 }
1030
gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)1031 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
1032 uint16_t pasid, uint32_t flush_type,
1033 bool all_hub)
1034 {
1035 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
1036 amdgpu_ring_write(kiq_ring,
1037 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
1038 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
1039 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
1040 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
1041 }
1042
1043
gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring * kiq_ring,uint32_t queue_type,uint32_t me_id,uint32_t pipe_id,uint32_t queue_id,uint32_t xcc_id,uint32_t vmid)1044 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
1045 uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
1046 uint32_t xcc_id, uint32_t vmid)
1047 {
1048 struct amdgpu_device *adev = kiq_ring->adev;
1049 unsigned i;
1050
1051 /* enter save mode */
1052 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1053 mutex_lock(&adev->srbm_mutex);
1054 soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0);
1055
1056 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
1057 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2);
1058 WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1);
1059 /* wait till dequeue take effects */
1060 for (i = 0; i < adev->usec_timeout; i++) {
1061 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
1062 break;
1063 udelay(1);
1064 }
1065 if (i >= adev->usec_timeout)
1066 dev_err(adev->dev, "fail to wait on hqd deactive\n");
1067 } else {
1068 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type);
1069 }
1070
1071 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
1072 mutex_unlock(&adev->srbm_mutex);
1073 /* exit safe mode */
1074 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1075 }
1076
1077 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
1078 .kiq_set_resources = gfx_v9_0_kiq_set_resources,
1079 .kiq_map_queues = gfx_v9_0_kiq_map_queues,
1080 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
1081 .kiq_query_status = gfx_v9_0_kiq_query_status,
1082 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
1083 .kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue,
1084 .set_resources_size = 8,
1085 .map_queues_size = 7,
1086 .unmap_queues_size = 6,
1087 .query_status_size = 7,
1088 .invalidate_tlbs_size = 2,
1089 };
1090
gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)1091 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
1092 {
1093 adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs;
1094 }
1095
gfx_v9_0_init_golden_registers(struct amdgpu_device * adev)1096 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
1097 {
1098 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1099 case IP_VERSION(9, 0, 1):
1100 soc15_program_register_sequence(adev,
1101 golden_settings_gc_9_0,
1102 ARRAY_SIZE(golden_settings_gc_9_0));
1103 soc15_program_register_sequence(adev,
1104 golden_settings_gc_9_0_vg10,
1105 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
1106 break;
1107 case IP_VERSION(9, 2, 1):
1108 soc15_program_register_sequence(adev,
1109 golden_settings_gc_9_2_1,
1110 ARRAY_SIZE(golden_settings_gc_9_2_1));
1111 soc15_program_register_sequence(adev,
1112 golden_settings_gc_9_2_1_vg12,
1113 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
1114 break;
1115 case IP_VERSION(9, 4, 0):
1116 soc15_program_register_sequence(adev,
1117 golden_settings_gc_9_0,
1118 ARRAY_SIZE(golden_settings_gc_9_0));
1119 soc15_program_register_sequence(adev,
1120 golden_settings_gc_9_0_vg20,
1121 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
1122 break;
1123 case IP_VERSION(9, 4, 1):
1124 soc15_program_register_sequence(adev,
1125 golden_settings_gc_9_4_1_arct,
1126 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
1127 break;
1128 case IP_VERSION(9, 2, 2):
1129 case IP_VERSION(9, 1, 0):
1130 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
1131 ARRAY_SIZE(golden_settings_gc_9_1));
1132 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1133 soc15_program_register_sequence(adev,
1134 golden_settings_gc_9_1_rv2,
1135 ARRAY_SIZE(golden_settings_gc_9_1_rv2));
1136 else
1137 soc15_program_register_sequence(adev,
1138 golden_settings_gc_9_1_rv1,
1139 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
1140 break;
1141 case IP_VERSION(9, 3, 0):
1142 soc15_program_register_sequence(adev,
1143 golden_settings_gc_9_1_rn,
1144 ARRAY_SIZE(golden_settings_gc_9_1_rn));
1145 return; /* for renoir, don't need common goldensetting */
1146 case IP_VERSION(9, 4, 2):
1147 gfx_v9_4_2_init_golden_registers(adev,
1148 adev->smuio.funcs->get_die_id(adev));
1149 break;
1150 default:
1151 break;
1152 }
1153
1154 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1155 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)))
1156 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
1157 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
1158 }
1159
gfx_v9_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)1160 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
1161 bool wc, uint32_t reg, uint32_t val)
1162 {
1163 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1164 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1165 WRITE_DATA_DST_SEL(0) |
1166 (wc ? WR_CONFIRM : 0));
1167 amdgpu_ring_write(ring, reg);
1168 amdgpu_ring_write(ring, 0);
1169 amdgpu_ring_write(ring, val);
1170 }
1171
gfx_v9_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)1172 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1173 int mem_space, int opt, uint32_t addr0,
1174 uint32_t addr1, uint32_t ref, uint32_t mask,
1175 uint32_t inv)
1176 {
1177 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1178 amdgpu_ring_write(ring,
1179 /* memory (1) or register (0) */
1180 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1181 WAIT_REG_MEM_OPERATION(opt) | /* wait */
1182 WAIT_REG_MEM_FUNCTION(3) | /* equal */
1183 WAIT_REG_MEM_ENGINE(eng_sel)));
1184
1185 if (mem_space)
1186 BUG_ON(addr0 & 0x3); /* Dword align */
1187 amdgpu_ring_write(ring, addr0);
1188 amdgpu_ring_write(ring, addr1);
1189 amdgpu_ring_write(ring, ref);
1190 amdgpu_ring_write(ring, mask);
1191 amdgpu_ring_write(ring, inv); /* poll interval */
1192 }
1193
gfx_v9_0_ring_test_ring(struct amdgpu_ring * ring)1194 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1195 {
1196 struct amdgpu_device *adev = ring->adev;
1197 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1198 uint32_t tmp = 0;
1199 unsigned i;
1200 int r;
1201
1202 WREG32(scratch, 0xCAFEDEAD);
1203 r = amdgpu_ring_alloc(ring, 3);
1204 if (r)
1205 return r;
1206
1207 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1208 amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START);
1209 amdgpu_ring_write(ring, 0xDEADBEEF);
1210 amdgpu_ring_commit(ring);
1211
1212 for (i = 0; i < adev->usec_timeout; i++) {
1213 tmp = RREG32(scratch);
1214 if (tmp == 0xDEADBEEF)
1215 break;
1216 udelay(1);
1217 }
1218
1219 if (i >= adev->usec_timeout)
1220 r = -ETIMEDOUT;
1221 return r;
1222 }
1223
gfx_v9_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)1224 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1225 {
1226 struct amdgpu_device *adev = ring->adev;
1227 struct amdgpu_ib ib;
1228 struct dma_fence *f = NULL;
1229
1230 unsigned index;
1231 uint64_t gpu_addr;
1232 uint32_t tmp;
1233 long r;
1234
1235 r = amdgpu_device_wb_get(adev, &index);
1236 if (r)
1237 return r;
1238
1239 gpu_addr = adev->wb.gpu_addr + (index * 4);
1240 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1241 memset(&ib, 0, sizeof(ib));
1242
1243 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
1244 if (r)
1245 goto err1;
1246
1247 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1248 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1249 ib.ptr[2] = lower_32_bits(gpu_addr);
1250 ib.ptr[3] = upper_32_bits(gpu_addr);
1251 ib.ptr[4] = 0xDEADBEEF;
1252 ib.length_dw = 5;
1253
1254 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1255 if (r)
1256 goto err2;
1257
1258 r = dma_fence_wait_timeout(f, false, timeout);
1259 if (r == 0) {
1260 r = -ETIMEDOUT;
1261 goto err2;
1262 } else if (r < 0) {
1263 goto err2;
1264 }
1265
1266 tmp = adev->wb.wb[index];
1267 if (tmp == 0xDEADBEEF)
1268 r = 0;
1269 else
1270 r = -EINVAL;
1271
1272 err2:
1273 amdgpu_ib_free(&ib, NULL);
1274 dma_fence_put(f);
1275 err1:
1276 amdgpu_device_wb_free(adev, index);
1277 return r;
1278 }
1279
1280
gfx_v9_0_free_microcode(struct amdgpu_device * adev)1281 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1282 {
1283 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1284 amdgpu_ucode_release(&adev->gfx.me_fw);
1285 amdgpu_ucode_release(&adev->gfx.ce_fw);
1286 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1287 amdgpu_ucode_release(&adev->gfx.mec_fw);
1288 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1289
1290 kfree(adev->gfx.rlc.register_list_format);
1291 }
1292
gfx_v9_0_check_fw_write_wait(struct amdgpu_device * adev)1293 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1294 {
1295 adev->gfx.me_fw_write_wait = false;
1296 adev->gfx.mec_fw_write_wait = false;
1297
1298 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) &&
1299 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) &&
1300 ((adev->gfx.mec_fw_version < 0x000001a5) ||
1301 (adev->gfx.mec_feature_version < 46) ||
1302 (adev->gfx.pfp_fw_version < 0x000000b7) ||
1303 (adev->gfx.pfp_feature_version < 46)))
1304 DRM_WARN_ONCE("CP firmware version too old, please update!");
1305
1306 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1307 case IP_VERSION(9, 0, 1):
1308 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1309 (adev->gfx.me_feature_version >= 42) &&
1310 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1311 (adev->gfx.pfp_feature_version >= 42))
1312 adev->gfx.me_fw_write_wait = true;
1313
1314 if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1315 (adev->gfx.mec_feature_version >= 42))
1316 adev->gfx.mec_fw_write_wait = true;
1317 break;
1318 case IP_VERSION(9, 2, 1):
1319 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1320 (adev->gfx.me_feature_version >= 44) &&
1321 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1322 (adev->gfx.pfp_feature_version >= 44))
1323 adev->gfx.me_fw_write_wait = true;
1324
1325 if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1326 (adev->gfx.mec_feature_version >= 44))
1327 adev->gfx.mec_fw_write_wait = true;
1328 break;
1329 case IP_VERSION(9, 4, 0):
1330 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1331 (adev->gfx.me_feature_version >= 44) &&
1332 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1333 (adev->gfx.pfp_feature_version >= 44))
1334 adev->gfx.me_fw_write_wait = true;
1335
1336 if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1337 (adev->gfx.mec_feature_version >= 44))
1338 adev->gfx.mec_fw_write_wait = true;
1339 break;
1340 case IP_VERSION(9, 1, 0):
1341 case IP_VERSION(9, 2, 2):
1342 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1343 (adev->gfx.me_feature_version >= 42) &&
1344 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1345 (adev->gfx.pfp_feature_version >= 42))
1346 adev->gfx.me_fw_write_wait = true;
1347
1348 if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1349 (adev->gfx.mec_feature_version >= 42))
1350 adev->gfx.mec_fw_write_wait = true;
1351 break;
1352 default:
1353 adev->gfx.me_fw_write_wait = true;
1354 adev->gfx.mec_fw_write_wait = true;
1355 break;
1356 }
1357 }
1358
1359 struct amdgpu_gfxoff_quirk {
1360 u16 chip_vendor;
1361 u16 chip_device;
1362 u16 subsys_vendor;
1363 u16 subsys_device;
1364 u8 revision;
1365 };
1366
1367 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1368 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1369 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1370 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1371 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1372 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */
1373 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 },
1374 /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */
1375 { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 },
1376 /* https://bbs.openkylin.top/t/topic/171497 */
1377 { 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 },
1378 /* HP 705G4 DM with R5 2400G */
1379 { 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 },
1380 { 0, 0, 0, 0, 0 },
1381 };
1382
gfx_v9_0_should_disable_gfxoff(struct pci_dev * pdev)1383 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1384 {
1385 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1386
1387 while (p && p->chip_device != 0) {
1388 if (pdev->vendor == p->chip_vendor &&
1389 pdev->device == p->chip_device &&
1390 pdev->subsystem_vendor == p->subsys_vendor &&
1391 pdev->subsystem_device == p->subsys_device &&
1392 pdev->revision == p->revision) {
1393 return true;
1394 }
1395 ++p;
1396 }
1397 return false;
1398 }
1399
is_raven_kicker(struct amdgpu_device * adev)1400 static bool is_raven_kicker(struct amdgpu_device *adev)
1401 {
1402 if (adev->pm.fw_version >= 0x41e2b)
1403 return true;
1404 else
1405 return false;
1406 }
1407
check_if_enlarge_doorbell_range(struct amdgpu_device * adev)1408 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev)
1409 {
1410 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) &&
1411 (adev->gfx.me_fw_version >= 0x000000a5) &&
1412 (adev->gfx.me_feature_version >= 52))
1413 return true;
1414 else
1415 return false;
1416 }
1417
gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device * adev)1418 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1419 {
1420 if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1421 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1422
1423 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1424 case IP_VERSION(9, 0, 1):
1425 case IP_VERSION(9, 2, 1):
1426 case IP_VERSION(9, 4, 0):
1427 break;
1428 case IP_VERSION(9, 2, 2):
1429 case IP_VERSION(9, 1, 0):
1430 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) ||
1431 (adev->apu_flags & AMD_APU_IS_PICASSO)) &&
1432 ((!is_raven_kicker(adev) &&
1433 adev->gfx.rlc_fw_version < 531) ||
1434 (adev->gfx.rlc_feature_version < 1) ||
1435 !adev->gfx.rlc.is_rlc_v2_1))
1436 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1437
1438 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1439 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1440 AMD_PG_SUPPORT_CP |
1441 AMD_PG_SUPPORT_RLC_SMU_HS;
1442 break;
1443 case IP_VERSION(9, 3, 0):
1444 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1445 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1446 AMD_PG_SUPPORT_CP |
1447 AMD_PG_SUPPORT_RLC_SMU_HS;
1448 break;
1449 default:
1450 break;
1451 }
1452 }
1453
gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device * adev,char * chip_name)1454 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1455 char *chip_name)
1456 {
1457 int err;
1458
1459 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
1460 AMDGPU_UCODE_REQUIRED,
1461 "amdgpu/%s_pfp.bin", chip_name);
1462 if (err)
1463 goto out;
1464 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
1465
1466 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1467 AMDGPU_UCODE_REQUIRED,
1468 "amdgpu/%s_me.bin", chip_name);
1469 if (err)
1470 goto out;
1471 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
1472
1473 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1474 AMDGPU_UCODE_REQUIRED,
1475 "amdgpu/%s_ce.bin", chip_name);
1476 if (err)
1477 goto out;
1478 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
1479
1480 out:
1481 if (err) {
1482 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1483 amdgpu_ucode_release(&adev->gfx.me_fw);
1484 amdgpu_ucode_release(&adev->gfx.ce_fw);
1485 }
1486 return err;
1487 }
1488
gfx_v9_0_init_rlc_microcode(struct amdgpu_device * adev,char * chip_name)1489 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1490 char *chip_name)
1491 {
1492 int err;
1493 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1494 uint16_t version_major;
1495 uint16_t version_minor;
1496 uint32_t smu_version;
1497
1498 /*
1499 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1500 * instead of picasso_rlc.bin.
1501 * Judgment method:
1502 * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1503 * or revision >= 0xD8 && revision <= 0xDF
1504 * otherwise is PCO FP5
1505 */
1506 if (!strcmp(chip_name, "picasso") &&
1507 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1508 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1509 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1510 AMDGPU_UCODE_REQUIRED,
1511 "amdgpu/%s_rlc_am4.bin", chip_name);
1512 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1513 (smu_version >= 0x41e2b))
1514 /**
1515 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1516 */
1517 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1518 AMDGPU_UCODE_REQUIRED,
1519 "amdgpu/%s_kicker_rlc.bin", chip_name);
1520 else
1521 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1522 AMDGPU_UCODE_REQUIRED,
1523 "amdgpu/%s_rlc.bin", chip_name);
1524 if (err)
1525 goto out;
1526
1527 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1528 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1529 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1530 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
1531 out:
1532 if (err)
1533 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1534
1535 return err;
1536 }
1537
gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device * adev)1538 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev)
1539 {
1540 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
1541 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1542 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0))
1543 return false;
1544
1545 return true;
1546 }
1547
gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device * adev,char * chip_name)1548 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1549 char *chip_name)
1550 {
1551 int err;
1552
1553 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1554 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1555 AMDGPU_UCODE_REQUIRED,
1556 "amdgpu/%s_sjt_mec.bin", chip_name);
1557 else
1558 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1559 AMDGPU_UCODE_REQUIRED,
1560 "amdgpu/%s_mec.bin", chip_name);
1561 if (err)
1562 goto out;
1563
1564 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
1565 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
1566
1567 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
1568 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
1569 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1570 AMDGPU_UCODE_REQUIRED,
1571 "amdgpu/%s_sjt_mec2.bin", chip_name);
1572 else
1573 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1574 AMDGPU_UCODE_REQUIRED,
1575 "amdgpu/%s_mec2.bin", chip_name);
1576 if (!err) {
1577 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
1578 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
1579 } else {
1580 err = 0;
1581 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1582 }
1583 } else {
1584 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
1585 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
1586 }
1587
1588 gfx_v9_0_check_if_need_gfxoff(adev);
1589 gfx_v9_0_check_fw_write_wait(adev);
1590
1591 out:
1592 if (err)
1593 amdgpu_ucode_release(&adev->gfx.mec_fw);
1594 return err;
1595 }
1596
gfx_v9_0_init_microcode(struct amdgpu_device * adev)1597 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1598 {
1599 char ucode_prefix[30];
1600 int r;
1601
1602 DRM_DEBUG("\n");
1603 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
1604
1605 /* No CPG in Arcturus */
1606 if (adev->gfx.num_gfx_rings) {
1607 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix);
1608 if (r)
1609 return r;
1610 }
1611
1612 r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix);
1613 if (r)
1614 return r;
1615
1616 r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix);
1617 if (r)
1618 return r;
1619
1620 return r;
1621 }
1622
gfx_v9_0_get_csb_size(struct amdgpu_device * adev)1623 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1624 {
1625 u32 count = 0;
1626 const struct cs_section_def *sect = NULL;
1627 const struct cs_extent_def *ext = NULL;
1628
1629 /* begin clear state */
1630 count += 2;
1631 /* context control state */
1632 count += 3;
1633
1634 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1635 for (ext = sect->section; ext->extent != NULL; ++ext) {
1636 if (sect->id == SECT_CONTEXT)
1637 count += 2 + ext->reg_count;
1638 else
1639 return 0;
1640 }
1641 }
1642
1643 /* end clear state */
1644 count += 2;
1645 /* clear state */
1646 count += 2;
1647
1648 return count;
1649 }
1650
gfx_v9_0_get_csb_buffer(struct amdgpu_device * adev,volatile u32 * buffer)1651 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1652 volatile u32 *buffer)
1653 {
1654 u32 count = 0;
1655
1656 if (adev->gfx.rlc.cs_data == NULL)
1657 return;
1658 if (buffer == NULL)
1659 return;
1660
1661 count = amdgpu_gfx_csb_preamble_start(buffer);
1662 count = amdgpu_gfx_csb_data_parser(adev, buffer, count);
1663 amdgpu_gfx_csb_preamble_end(buffer, count);
1664 }
1665
gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device * adev)1666 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1667 {
1668 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1669 uint32_t pg_always_on_cu_num = 2;
1670 uint32_t always_on_cu_num;
1671 uint32_t i, j, k;
1672 uint32_t mask, cu_bitmap, counter;
1673
1674 if (adev->flags & AMD_IS_APU)
1675 always_on_cu_num = 4;
1676 else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1))
1677 always_on_cu_num = 8;
1678 else
1679 always_on_cu_num = 12;
1680
1681 mutex_lock(&adev->grbm_idx_mutex);
1682 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1683 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1684 mask = 1;
1685 cu_bitmap = 0;
1686 counter = 0;
1687 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
1688
1689 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1690 if (cu_info->bitmap[0][i][j] & mask) {
1691 if (counter == pg_always_on_cu_num)
1692 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1693 if (counter < always_on_cu_num)
1694 cu_bitmap |= mask;
1695 else
1696 break;
1697 counter++;
1698 }
1699 mask <<= 1;
1700 }
1701
1702 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1703 cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1704 }
1705 }
1706 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1707 mutex_unlock(&adev->grbm_idx_mutex);
1708 }
1709
gfx_v9_0_init_lbpw(struct amdgpu_device * adev)1710 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1711 {
1712 uint32_t data;
1713
1714 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1715 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1716 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1717 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1718 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1719
1720 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1721 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1722
1723 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1724 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1725
1726 mutex_lock(&adev->grbm_idx_mutex);
1727 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1728 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1729 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1730
1731 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1732 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1733 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1734 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1735 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1736
1737 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1738 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1739 data &= 0x0000FFFF;
1740 data |= 0x00C00000;
1741 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1742
1743 /*
1744 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1745 * programmed in gfx_v9_0_init_always_on_cu_mask()
1746 */
1747
1748 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1749 * but used for RLC_LB_CNTL configuration */
1750 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1751 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1752 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1753 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1754 mutex_unlock(&adev->grbm_idx_mutex);
1755
1756 gfx_v9_0_init_always_on_cu_mask(adev);
1757 }
1758
gfx_v9_4_init_lbpw(struct amdgpu_device * adev)1759 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1760 {
1761 uint32_t data;
1762
1763 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1764 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1765 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1766 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1767 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1768
1769 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1770 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1771
1772 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1773 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1774
1775 mutex_lock(&adev->grbm_idx_mutex);
1776 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1777 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1778 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1779
1780 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1781 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1782 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1783 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1784 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1785
1786 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1787 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1788 data &= 0x0000FFFF;
1789 data |= 0x00C00000;
1790 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1791
1792 /*
1793 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1794 * programmed in gfx_v9_0_init_always_on_cu_mask()
1795 */
1796
1797 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1798 * but used for RLC_LB_CNTL configuration */
1799 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1800 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1801 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1802 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1803 mutex_unlock(&adev->grbm_idx_mutex);
1804
1805 gfx_v9_0_init_always_on_cu_mask(adev);
1806 }
1807
gfx_v9_0_enable_lbpw(struct amdgpu_device * adev,bool enable)1808 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1809 {
1810 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1811 }
1812
gfx_v9_0_cp_jump_table_num(struct amdgpu_device * adev)1813 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1814 {
1815 if (gfx_v9_0_load_mec2_fw_bin_support(adev))
1816 return 5;
1817 else
1818 return 4;
1819 }
1820
gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)1821 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1822 {
1823 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1824
1825 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
1826 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
1827 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1);
1828 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2);
1829 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3);
1830 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL);
1831 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX);
1832 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT);
1833 adev->gfx.rlc.rlcg_reg_access_supported = true;
1834 }
1835
gfx_v9_0_rlc_init(struct amdgpu_device * adev)1836 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1837 {
1838 const struct cs_section_def *cs_data;
1839 int r;
1840
1841 adev->gfx.rlc.cs_data = gfx9_cs_data;
1842
1843 cs_data = adev->gfx.rlc.cs_data;
1844
1845 if (cs_data) {
1846 /* init clear state block */
1847 r = amdgpu_gfx_rlc_init_csb(adev);
1848 if (r)
1849 return r;
1850 }
1851
1852 if (adev->flags & AMD_IS_APU) {
1853 /* TODO: double check the cp_table_size for RV */
1854 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1855 r = amdgpu_gfx_rlc_init_cpt(adev);
1856 if (r)
1857 return r;
1858 }
1859
1860 return 0;
1861 }
1862
gfx_v9_0_mec_fini(struct amdgpu_device * adev)1863 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1864 {
1865 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1866 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1867 }
1868
gfx_v9_0_mec_init(struct amdgpu_device * adev)1869 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1870 {
1871 int r;
1872 u32 *hpd;
1873 const __le32 *fw_data;
1874 unsigned fw_size;
1875 u32 *fw;
1876 size_t mec_hpd_size;
1877
1878 const struct gfx_firmware_header_v1_0 *mec_hdr;
1879
1880 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1881
1882 /* take ownership of the relevant compute queues */
1883 amdgpu_gfx_compute_queue_acquire(adev);
1884 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1885 if (mec_hpd_size) {
1886 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1887 AMDGPU_GEM_DOMAIN_VRAM |
1888 AMDGPU_GEM_DOMAIN_GTT,
1889 &adev->gfx.mec.hpd_eop_obj,
1890 &adev->gfx.mec.hpd_eop_gpu_addr,
1891 (void **)&hpd);
1892 if (r) {
1893 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1894 gfx_v9_0_mec_fini(adev);
1895 return r;
1896 }
1897
1898 memset(hpd, 0, mec_hpd_size);
1899
1900 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1901 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1902 }
1903
1904 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1905
1906 fw_data = (const __le32 *)
1907 (adev->gfx.mec_fw->data +
1908 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1909 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1910
1911 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1912 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1913 &adev->gfx.mec.mec_fw_obj,
1914 &adev->gfx.mec.mec_fw_gpu_addr,
1915 (void **)&fw);
1916 if (r) {
1917 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1918 gfx_v9_0_mec_fini(adev);
1919 return r;
1920 }
1921
1922 memcpy(fw, fw_data, fw_size);
1923
1924 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1925 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1926
1927 return 0;
1928 }
1929
wave_read_ind(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t address)1930 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1931 {
1932 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1933 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1934 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1935 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1936 (SQ_IND_INDEX__FORCE_READ_MASK));
1937 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1938 }
1939
wave_read_regs(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)1940 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1941 uint32_t wave, uint32_t thread,
1942 uint32_t regno, uint32_t num, uint32_t *out)
1943 {
1944 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1945 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1946 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1947 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1948 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1949 (SQ_IND_INDEX__FORCE_READ_MASK) |
1950 (SQ_IND_INDEX__AUTO_INCR_MASK));
1951 while (num--)
1952 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1953 }
1954
gfx_v9_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)1955 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1956 {
1957 /* type 1 wave data */
1958 dst[(*no_fields)++] = 1;
1959 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1960 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1961 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1962 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1963 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1964 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1965 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1966 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1967 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1968 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1969 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1970 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1971 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1972 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1973 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
1974 }
1975
gfx_v9_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)1976 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1977 uint32_t wave, uint32_t start,
1978 uint32_t size, uint32_t *dst)
1979 {
1980 wave_read_regs(
1981 adev, simd, wave, 0,
1982 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1983 }
1984
gfx_v9_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)1985 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1986 uint32_t wave, uint32_t thread,
1987 uint32_t start, uint32_t size,
1988 uint32_t *dst)
1989 {
1990 wave_read_regs(
1991 adev, simd, wave, thread,
1992 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1993 }
1994
gfx_v9_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)1995 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1996 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1997 {
1998 soc15_grbm_select(adev, me, pipe, q, vm, 0);
1999 }
2000
2001 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2002 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2003 .select_se_sh = &gfx_v9_0_select_se_sh,
2004 .read_wave_data = &gfx_v9_0_read_wave_data,
2005 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2006 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2007 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2008 };
2009
2010 const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = {
2011 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2012 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2013 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2014 };
2015
2016 static struct amdgpu_gfx_ras gfx_v9_0_ras = {
2017 .ras_block = {
2018 .hw_ops = &gfx_v9_0_ras_ops,
2019 },
2020 };
2021
gfx_v9_0_gpu_early_init(struct amdgpu_device * adev)2022 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2023 {
2024 u32 gb_addr_config;
2025 int err;
2026
2027 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2028 case IP_VERSION(9, 0, 1):
2029 adev->gfx.config.max_hw_contexts = 8;
2030 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2031 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2032 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2033 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2034 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2035 break;
2036 case IP_VERSION(9, 2, 1):
2037 adev->gfx.config.max_hw_contexts = 8;
2038 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2039 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2040 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2041 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2042 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2043 DRM_INFO("fix gfx.config for vega12\n");
2044 break;
2045 case IP_VERSION(9, 4, 0):
2046 adev->gfx.ras = &gfx_v9_0_ras;
2047 adev->gfx.config.max_hw_contexts = 8;
2048 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2049 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2050 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2051 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2052 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2053 gb_addr_config &= ~0xf3e777ff;
2054 gb_addr_config |= 0x22014042;
2055 /* check vbios table if gpu info is not available */
2056 err = amdgpu_atomfirmware_get_gfx_info(adev);
2057 if (err)
2058 return err;
2059 break;
2060 case IP_VERSION(9, 2, 2):
2061 case IP_VERSION(9, 1, 0):
2062 adev->gfx.config.max_hw_contexts = 8;
2063 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2064 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2065 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2066 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2067 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2068 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2069 else
2070 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2071 break;
2072 case IP_VERSION(9, 4, 1):
2073 adev->gfx.ras = &gfx_v9_4_ras;
2074 adev->gfx.config.max_hw_contexts = 8;
2075 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2076 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2077 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2078 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2079 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2080 gb_addr_config &= ~0xf3e777ff;
2081 gb_addr_config |= 0x22014042;
2082 break;
2083 case IP_VERSION(9, 3, 0):
2084 adev->gfx.config.max_hw_contexts = 8;
2085 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2086 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2087 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2088 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2089 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2090 gb_addr_config &= ~0xf3e777ff;
2091 gb_addr_config |= 0x22010042;
2092 break;
2093 case IP_VERSION(9, 4, 2):
2094 adev->gfx.ras = &gfx_v9_4_2_ras;
2095 adev->gfx.config.max_hw_contexts = 8;
2096 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2097 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2098 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2099 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2100 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2101 gb_addr_config &= ~0xf3e777ff;
2102 gb_addr_config |= 0x22014042;
2103 /* check vbios table if gpu info is not available */
2104 err = amdgpu_atomfirmware_get_gfx_info(adev);
2105 if (err)
2106 return err;
2107 break;
2108 default:
2109 BUG();
2110 break;
2111 }
2112
2113 adev->gfx.config.gb_addr_config = gb_addr_config;
2114
2115 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2116 REG_GET_FIELD(
2117 adev->gfx.config.gb_addr_config,
2118 GB_ADDR_CONFIG,
2119 NUM_PIPES);
2120
2121 adev->gfx.config.max_tile_pipes =
2122 adev->gfx.config.gb_addr_config_fields.num_pipes;
2123
2124 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2125 REG_GET_FIELD(
2126 adev->gfx.config.gb_addr_config,
2127 GB_ADDR_CONFIG,
2128 NUM_BANKS);
2129 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2130 REG_GET_FIELD(
2131 adev->gfx.config.gb_addr_config,
2132 GB_ADDR_CONFIG,
2133 MAX_COMPRESSED_FRAGS);
2134 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2135 REG_GET_FIELD(
2136 adev->gfx.config.gb_addr_config,
2137 GB_ADDR_CONFIG,
2138 NUM_RB_PER_SE);
2139 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2140 REG_GET_FIELD(
2141 adev->gfx.config.gb_addr_config,
2142 GB_ADDR_CONFIG,
2143 NUM_SHADER_ENGINES);
2144 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2145 REG_GET_FIELD(
2146 adev->gfx.config.gb_addr_config,
2147 GB_ADDR_CONFIG,
2148 PIPE_INTERLEAVE_SIZE));
2149
2150 return 0;
2151 }
2152
gfx_v9_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)2153 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2154 int mec, int pipe, int queue)
2155 {
2156 unsigned irq_type;
2157 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2158 unsigned int hw_prio;
2159
2160 ring = &adev->gfx.compute_ring[ring_id];
2161
2162 /* mec0 is me1 */
2163 ring->me = mec + 1;
2164 ring->pipe = pipe;
2165 ring->queue = queue;
2166
2167 ring->ring_obj = NULL;
2168 ring->use_doorbell = true;
2169 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2170 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2171 + (ring_id * GFX9_MEC_HPD_SIZE);
2172 ring->vm_hub = AMDGPU_GFXHUB(0);
2173 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2174
2175 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2176 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2177 + ring->pipe;
2178 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
2179 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
2180 /* type-2 packets are deprecated on MEC, use type-3 instead */
2181 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
2182 hw_prio, NULL);
2183 }
2184
gfx_v9_0_alloc_ip_dump(struct amdgpu_device * adev)2185 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev)
2186 {
2187 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
2188 uint32_t *ptr;
2189 uint32_t inst;
2190
2191 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
2192 if (!ptr) {
2193 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
2194 adev->gfx.ip_dump_core = NULL;
2195 } else {
2196 adev->gfx.ip_dump_core = ptr;
2197 }
2198
2199 /* Allocate memory for compute queue registers for all the instances */
2200 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
2201 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
2202 adev->gfx.mec.num_queue_per_pipe;
2203
2204 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
2205 if (!ptr) {
2206 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
2207 adev->gfx.ip_dump_compute_queues = NULL;
2208 } else {
2209 adev->gfx.ip_dump_compute_queues = ptr;
2210 }
2211 }
2212
gfx_v9_0_sw_init(struct amdgpu_ip_block * ip_block)2213 static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
2214 {
2215 int i, j, k, r, ring_id;
2216 int xcc_id = 0;
2217 struct amdgpu_ring *ring;
2218 struct amdgpu_device *adev = ip_block->adev;
2219 unsigned int hw_prio;
2220
2221 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2222 case IP_VERSION(9, 0, 1):
2223 case IP_VERSION(9, 2, 1):
2224 case IP_VERSION(9, 4, 0):
2225 case IP_VERSION(9, 2, 2):
2226 case IP_VERSION(9, 1, 0):
2227 case IP_VERSION(9, 4, 1):
2228 case IP_VERSION(9, 3, 0):
2229 case IP_VERSION(9, 4, 2):
2230 adev->gfx.mec.num_mec = 2;
2231 break;
2232 default:
2233 adev->gfx.mec.num_mec = 1;
2234 break;
2235 }
2236
2237 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2238 case IP_VERSION(9, 0, 1):
2239 case IP_VERSION(9, 2, 1):
2240 case IP_VERSION(9, 4, 0):
2241 case IP_VERSION(9, 2, 2):
2242 case IP_VERSION(9, 1, 0):
2243 case IP_VERSION(9, 3, 0):
2244 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2245 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2246 if (adev->gfx.me_fw_version >= 167 &&
2247 adev->gfx.pfp_fw_version >= 196 &&
2248 adev->gfx.mec_fw_version >= 474) {
2249 adev->gfx.enable_cleaner_shader = true;
2250 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2251 if (r) {
2252 adev->gfx.enable_cleaner_shader = false;
2253 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2254 }
2255 }
2256 break;
2257 case IP_VERSION(9, 4, 2):
2258 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex;
2259 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex);
2260 if (adev->gfx.mec_fw_version >= 88) {
2261 adev->gfx.enable_cleaner_shader = true;
2262 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
2263 if (r) {
2264 adev->gfx.enable_cleaner_shader = false;
2265 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
2266 }
2267 }
2268 break;
2269 default:
2270 adev->gfx.enable_cleaner_shader = false;
2271 break;
2272 }
2273
2274 adev->gfx.mec.num_pipe_per_mec = 4;
2275 adev->gfx.mec.num_queue_per_pipe = 8;
2276
2277 /* EOP Event */
2278 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2279 if (r)
2280 return r;
2281
2282 /* Bad opcode Event */
2283 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
2284 GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
2285 &adev->gfx.bad_op_irq);
2286 if (r)
2287 return r;
2288
2289 /* Privileged reg */
2290 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2291 &adev->gfx.priv_reg_irq);
2292 if (r)
2293 return r;
2294
2295 /* Privileged inst */
2296 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2297 &adev->gfx.priv_inst_irq);
2298 if (r)
2299 return r;
2300
2301 /* ECC error */
2302 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2303 &adev->gfx.cp_ecc_error_irq);
2304 if (r)
2305 return r;
2306
2307 /* FUE error */
2308 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2309 &adev->gfx.cp_ecc_error_irq);
2310 if (r)
2311 return r;
2312
2313 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2314
2315 if (adev->gfx.rlc.funcs) {
2316 if (adev->gfx.rlc.funcs->init) {
2317 r = adev->gfx.rlc.funcs->init(adev);
2318 if (r) {
2319 dev_err(adev->dev, "Failed to init rlc BOs!\n");
2320 return r;
2321 }
2322 }
2323 }
2324
2325 r = gfx_v9_0_mec_init(adev);
2326 if (r) {
2327 DRM_ERROR("Failed to init MEC BOs!\n");
2328 return r;
2329 }
2330
2331 /* set up the gfx ring */
2332 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2333 ring = &adev->gfx.gfx_ring[i];
2334 ring->ring_obj = NULL;
2335 if (!i)
2336 sprintf(ring->name, "gfx");
2337 else
2338 sprintf(ring->name, "gfx_%d", i);
2339 ring->use_doorbell = true;
2340 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2341
2342 /* disable scheduler on the real ring */
2343 ring->no_scheduler = adev->gfx.mcbp;
2344 ring->vm_hub = AMDGPU_GFXHUB(0);
2345 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2346 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
2347 AMDGPU_RING_PRIO_DEFAULT, NULL);
2348 if (r)
2349 return r;
2350 }
2351
2352 /* set up the software rings */
2353 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2354 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2355 ring = &adev->gfx.sw_gfx_ring[i];
2356 ring->ring_obj = NULL;
2357 sprintf(ring->name, amdgpu_sw_ring_name(i));
2358 ring->use_doorbell = true;
2359 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2360 ring->is_sw_ring = true;
2361 hw_prio = amdgpu_sw_ring_priority(i);
2362 ring->vm_hub = AMDGPU_GFXHUB(0);
2363 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2364 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio,
2365 NULL);
2366 if (r)
2367 return r;
2368 ring->wptr = 0;
2369 }
2370
2371 /* init the muxer and add software rings */
2372 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0],
2373 GFX9_NUM_SW_GFX_RINGS);
2374 if (r) {
2375 DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r);
2376 return r;
2377 }
2378 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) {
2379 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer,
2380 &adev->gfx.sw_gfx_ring[i]);
2381 if (r) {
2382 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r);
2383 return r;
2384 }
2385 }
2386 }
2387
2388 /* set up the compute queues - allocate horizontally across pipes */
2389 ring_id = 0;
2390 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2391 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2392 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2393 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
2394 k, j))
2395 continue;
2396
2397 r = gfx_v9_0_compute_ring_init(adev,
2398 ring_id,
2399 i, k, j);
2400 if (r)
2401 return r;
2402
2403 ring_id++;
2404 }
2405 }
2406 }
2407
2408 /* TODO: Add queue reset mask when FW fully supports it */
2409 adev->gfx.gfx_supported_reset =
2410 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
2411 adev->gfx.compute_supported_reset =
2412 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
2413
2414 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0);
2415 if (r) {
2416 DRM_ERROR("Failed to init KIQ BOs!\n");
2417 return r;
2418 }
2419
2420 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2421 if (r)
2422 return r;
2423
2424 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2425 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0);
2426 if (r)
2427 return r;
2428
2429 adev->gfx.ce_ram_size = 0x8000;
2430
2431 r = gfx_v9_0_gpu_early_init(adev);
2432 if (r)
2433 return r;
2434
2435 if (amdgpu_gfx_ras_sw_init(adev)) {
2436 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
2437 return -EINVAL;
2438 }
2439
2440 gfx_v9_0_alloc_ip_dump(adev);
2441
2442 r = amdgpu_gfx_sysfs_init(adev);
2443 if (r)
2444 return r;
2445
2446 return 0;
2447 }
2448
2449
gfx_v9_0_sw_fini(struct amdgpu_ip_block * ip_block)2450 static int gfx_v9_0_sw_fini(struct amdgpu_ip_block *ip_block)
2451 {
2452 int i;
2453 struct amdgpu_device *adev = ip_block->adev;
2454
2455 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
2456 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
2457 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]);
2458 amdgpu_ring_mux_fini(&adev->gfx.muxer);
2459 }
2460
2461 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2462 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2463 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2464 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2465
2466 amdgpu_gfx_mqd_sw_fini(adev, 0);
2467 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
2468 amdgpu_gfx_kiq_fini(adev, 0);
2469
2470 amdgpu_gfx_cleaner_shader_sw_fini(adev);
2471
2472 gfx_v9_0_mec_fini(adev);
2473 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2474 &adev->gfx.rlc.clear_state_gpu_addr,
2475 (void **)&adev->gfx.rlc.cs_ptr);
2476 if (adev->flags & AMD_IS_APU) {
2477 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2478 &adev->gfx.rlc.cp_table_gpu_addr,
2479 (void **)&adev->gfx.rlc.cp_table_ptr);
2480 }
2481 gfx_v9_0_free_microcode(adev);
2482
2483 amdgpu_gfx_sysfs_fini(adev);
2484
2485 kfree(adev->gfx.ip_dump_core);
2486 kfree(adev->gfx.ip_dump_compute_queues);
2487
2488 return 0;
2489 }
2490
2491
gfx_v9_0_tiling_mode_table_init(struct amdgpu_device * adev)2492 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2493 {
2494 /* TODO */
2495 }
2496
gfx_v9_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)2497 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num,
2498 u32 instance, int xcc_id)
2499 {
2500 u32 data;
2501
2502 if (instance == 0xffffffff)
2503 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2504 else
2505 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2506
2507 if (se_num == 0xffffffff)
2508 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2509 else
2510 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2511
2512 if (sh_num == 0xffffffff)
2513 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2514 else
2515 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2516
2517 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2518 }
2519
gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device * adev)2520 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2521 {
2522 u32 data, mask;
2523
2524 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2525 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2526
2527 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2528 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2529
2530 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2531 adev->gfx.config.max_sh_per_se);
2532
2533 return (~data) & mask;
2534 }
2535
gfx_v9_0_setup_rb(struct amdgpu_device * adev)2536 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2537 {
2538 int i, j;
2539 u32 data;
2540 u32 active_rbs = 0;
2541 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2542 adev->gfx.config.max_sh_per_se;
2543
2544 mutex_lock(&adev->grbm_idx_mutex);
2545 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2546 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2547 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2548 data = gfx_v9_0_get_rb_active_bitmap(adev);
2549 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2550 rb_bitmap_width_per_sh);
2551 }
2552 }
2553 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2554 mutex_unlock(&adev->grbm_idx_mutex);
2555
2556 adev->gfx.config.backend_enable_mask = active_rbs;
2557 adev->gfx.config.num_rbs = hweight32(active_rbs);
2558 }
2559
gfx_v9_0_debug_trap_config_init(struct amdgpu_device * adev,uint32_t first_vmid,uint32_t last_vmid)2560 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev,
2561 uint32_t first_vmid,
2562 uint32_t last_vmid)
2563 {
2564 uint32_t data;
2565 uint32_t trap_config_vmid_mask = 0;
2566 int i;
2567
2568 /* Calculate trap config vmid mask */
2569 for (i = first_vmid; i < last_vmid; i++)
2570 trap_config_vmid_mask |= (1 << i);
2571
2572 data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG,
2573 VMID_SEL, trap_config_vmid_mask);
2574 data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG,
2575 TRAP_EN, 1);
2576 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data);
2577 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0);
2578
2579 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0);
2580 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0);
2581 }
2582
2583 #define DEFAULT_SH_MEM_BASES (0x6000)
gfx_v9_0_init_compute_vmid(struct amdgpu_device * adev)2584 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2585 {
2586 int i;
2587 uint32_t sh_mem_config;
2588 uint32_t sh_mem_bases;
2589
2590 /*
2591 * Configure apertures:
2592 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2593 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2594 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2595 */
2596 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2597
2598 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2599 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2600 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2601
2602 mutex_lock(&adev->srbm_mutex);
2603 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2604 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2605 /* CP and shaders */
2606 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2607 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2608 }
2609 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2610 mutex_unlock(&adev->srbm_mutex);
2611
2612 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2613 access. These should be enabled by FW for target VMIDs. */
2614 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2615 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2616 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2617 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2618 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2619 }
2620 }
2621
gfx_v9_0_init_gds_vmid(struct amdgpu_device * adev)2622 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2623 {
2624 int vmid;
2625
2626 /*
2627 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2628 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2629 * the driver can enable them for graphics. VMID0 should maintain
2630 * access so that HWS firmware can save/restore entries.
2631 */
2632 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
2633 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2634 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2635 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2636 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2637 }
2638 }
2639
gfx_v9_0_init_sq_config(struct amdgpu_device * adev)2640 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2641 {
2642 uint32_t tmp;
2643
2644 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
2645 case IP_VERSION(9, 4, 1):
2646 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2647 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT,
2648 !READ_ONCE(adev->barrier_has_auto_waitcnt));
2649 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2650 break;
2651 default:
2652 break;
2653 }
2654 }
2655
gfx_v9_0_constants_init(struct amdgpu_device * adev)2656 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2657 {
2658 u32 tmp;
2659 int i;
2660
2661 if (!amdgpu_sriov_vf(adev) ||
2662 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) {
2663 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2664 }
2665
2666 gfx_v9_0_tiling_mode_table_init(adev);
2667
2668 if (adev->gfx.num_gfx_rings)
2669 gfx_v9_0_setup_rb(adev);
2670 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2671 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2672
2673 /* XXX SH_MEM regs */
2674 /* where to put LDS, scratch, GPUVM in FSA64 space */
2675 mutex_lock(&adev->srbm_mutex);
2676 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2677 soc15_grbm_select(adev, 0, 0, 0, i, 0);
2678 /* CP and shaders */
2679 if (i == 0) {
2680 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2681 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2682 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2683 !!adev->gmc.noretry);
2684 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2685 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2686 } else {
2687 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2688 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2689 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2690 !!adev->gmc.noretry);
2691 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2692 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2693 (adev->gmc.private_aperture_start >> 48));
2694 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2695 (adev->gmc.shared_aperture_start >> 48));
2696 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2697 }
2698 }
2699 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
2700
2701 mutex_unlock(&adev->srbm_mutex);
2702
2703 gfx_v9_0_init_compute_vmid(adev);
2704 gfx_v9_0_init_gds_vmid(adev);
2705 gfx_v9_0_init_sq_config(adev);
2706 }
2707
gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device * adev)2708 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2709 {
2710 u32 i, j, k;
2711 u32 mask;
2712
2713 mutex_lock(&adev->grbm_idx_mutex);
2714 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2715 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2716 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
2717 for (k = 0; k < adev->usec_timeout; k++) {
2718 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2719 break;
2720 udelay(1);
2721 }
2722 if (k == adev->usec_timeout) {
2723 amdgpu_gfx_select_se_sh(adev, 0xffffffff,
2724 0xffffffff, 0xffffffff, 0);
2725 mutex_unlock(&adev->grbm_idx_mutex);
2726 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2727 i, j);
2728 return;
2729 }
2730 }
2731 }
2732 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
2733 mutex_unlock(&adev->grbm_idx_mutex);
2734
2735 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2736 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2737 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2738 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2739 for (k = 0; k < adev->usec_timeout; k++) {
2740 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2741 break;
2742 udelay(1);
2743 }
2744 }
2745
gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2746 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2747 bool enable)
2748 {
2749 u32 tmp;
2750
2751 /* These interrupts should be enabled to drive DS clock */
2752
2753 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2754
2755 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2756 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2757 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2758 if (adev->gfx.num_gfx_rings)
2759 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2760
2761 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2762 }
2763
gfx_v9_0_init_csb(struct amdgpu_device * adev)2764 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2765 {
2766 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2767 /* csib */
2768 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2769 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2770 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2771 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2772 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2773 adev->gfx.rlc.clear_state_size);
2774 }
2775
gfx_v9_1_parse_ind_reg_list(int * register_list_format,int indirect_offset,int list_size,int * unique_indirect_regs,int unique_indirect_reg_count,int * indirect_start_offsets,int * indirect_start_offsets_count,int max_start_offsets_count)2776 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2777 int indirect_offset,
2778 int list_size,
2779 int *unique_indirect_regs,
2780 int unique_indirect_reg_count,
2781 int *indirect_start_offsets,
2782 int *indirect_start_offsets_count,
2783 int max_start_offsets_count)
2784 {
2785 int idx;
2786
2787 for (; indirect_offset < list_size; indirect_offset++) {
2788 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2789 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2790 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2791
2792 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2793 indirect_offset += 2;
2794
2795 /* look for the matching indice */
2796 for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2797 if (unique_indirect_regs[idx] ==
2798 register_list_format[indirect_offset] ||
2799 !unique_indirect_regs[idx])
2800 break;
2801 }
2802
2803 BUG_ON(idx >= unique_indirect_reg_count);
2804
2805 if (!unique_indirect_regs[idx])
2806 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2807
2808 indirect_offset++;
2809 }
2810 }
2811 }
2812
gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device * adev)2813 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2814 {
2815 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2816 int unique_indirect_reg_count = 0;
2817
2818 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2819 int indirect_start_offsets_count = 0;
2820
2821 int list_size = 0;
2822 int i = 0, j = 0;
2823 u32 tmp = 0;
2824
2825 u32 *register_list_format =
2826 kmemdup(adev->gfx.rlc.register_list_format,
2827 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2828 if (!register_list_format)
2829 return -ENOMEM;
2830
2831 /* setup unique_indirect_regs array and indirect_start_offsets array */
2832 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2833 gfx_v9_1_parse_ind_reg_list(register_list_format,
2834 adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2835 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2836 unique_indirect_regs,
2837 unique_indirect_reg_count,
2838 indirect_start_offsets,
2839 &indirect_start_offsets_count,
2840 ARRAY_SIZE(indirect_start_offsets));
2841
2842 /* enable auto inc in case it is disabled */
2843 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2844 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2845 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2846
2847 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2848 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2849 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2850 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2851 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2852 adev->gfx.rlc.register_restore[i]);
2853
2854 /* load indirect register */
2855 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2856 adev->gfx.rlc.reg_list_format_start);
2857
2858 /* direct register portion */
2859 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2860 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2861 register_list_format[i]);
2862
2863 /* indirect register portion */
2864 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2865 if (register_list_format[i] == 0xFFFFFFFF) {
2866 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2867 continue;
2868 }
2869
2870 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2871 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2872
2873 for (j = 0; j < unique_indirect_reg_count; j++) {
2874 if (register_list_format[i] == unique_indirect_regs[j]) {
2875 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2876 break;
2877 }
2878 }
2879
2880 BUG_ON(j >= unique_indirect_reg_count);
2881
2882 i++;
2883 }
2884
2885 /* set save/restore list size */
2886 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2887 list_size = list_size >> 1;
2888 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2889 adev->gfx.rlc.reg_restore_list_size);
2890 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2891
2892 /* write the starting offsets to RLC scratch ram */
2893 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2894 adev->gfx.rlc.starting_offsets_start);
2895 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2896 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2897 indirect_start_offsets[i]);
2898
2899 /* load unique indirect regs*/
2900 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2901 if (unique_indirect_regs[i] != 0) {
2902 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2903 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2904 unique_indirect_regs[i] & 0x3FFFF);
2905
2906 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2907 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2908 unique_indirect_regs[i] >> 20);
2909 }
2910 }
2911
2912 kfree(register_list_format);
2913 return 0;
2914 }
2915
gfx_v9_0_enable_save_restore_machine(struct amdgpu_device * adev)2916 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2917 {
2918 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2919 }
2920
pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device * adev,bool enable)2921 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2922 bool enable)
2923 {
2924 uint32_t data = 0;
2925 uint32_t default_data = 0;
2926
2927 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2928 if (enable) {
2929 /* enable GFXIP control over CGPG */
2930 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2931 if(default_data != data)
2932 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2933
2934 /* update status */
2935 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2936 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2937 if(default_data != data)
2938 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2939 } else {
2940 /* restore GFXIP control over GCPG */
2941 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2942 if(default_data != data)
2943 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2944 }
2945 }
2946
gfx_v9_0_init_gfx_power_gating(struct amdgpu_device * adev)2947 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2948 {
2949 uint32_t data = 0;
2950
2951 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2952 AMD_PG_SUPPORT_GFX_SMG |
2953 AMD_PG_SUPPORT_GFX_DMG)) {
2954 /* init IDLE_POLL_COUNT = 60 */
2955 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2956 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2957 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2958 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2959
2960 /* init RLC PG Delay */
2961 data = 0;
2962 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2963 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2964 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2965 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2966 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2967
2968 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2969 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2970 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2971 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2972
2973 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2974 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2975 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2976 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2977
2978 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2979 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2980
2981 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2982 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2983 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2984 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0))
2985 pwr_10_0_gfxip_control_over_cgpg(adev, true);
2986 }
2987 }
2988
gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device * adev,bool enable)2989 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2990 bool enable)
2991 {
2992 uint32_t data = 0;
2993 uint32_t default_data = 0;
2994
2995 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2996 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2997 SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2998 enable ? 1 : 0);
2999 if (default_data != data)
3000 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3001 }
3002
gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device * adev,bool enable)3003 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
3004 bool enable)
3005 {
3006 uint32_t data = 0;
3007 uint32_t default_data = 0;
3008
3009 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3010 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3011 SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
3012 enable ? 1 : 0);
3013 if(default_data != data)
3014 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3015 }
3016
gfx_v9_0_enable_cp_power_gating(struct amdgpu_device * adev,bool enable)3017 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
3018 bool enable)
3019 {
3020 uint32_t data = 0;
3021 uint32_t default_data = 0;
3022
3023 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3024 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3025 CP_PG_DISABLE,
3026 enable ? 0 : 1);
3027 if(default_data != data)
3028 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3029 }
3030
gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)3031 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
3032 bool enable)
3033 {
3034 uint32_t data, default_data;
3035
3036 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3037 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3038 GFX_POWER_GATING_ENABLE,
3039 enable ? 1 : 0);
3040 if(default_data != data)
3041 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3042 }
3043
gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device * adev,bool enable)3044 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
3045 bool enable)
3046 {
3047 uint32_t data, default_data;
3048
3049 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3050 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3051 GFX_PIPELINE_PG_ENABLE,
3052 enable ? 1 : 0);
3053 if(default_data != data)
3054 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3055
3056 if (!enable)
3057 /* read any GFX register to wake up GFX */
3058 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
3059 }
3060
gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device * adev,bool enable)3061 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
3062 bool enable)
3063 {
3064 uint32_t data, default_data;
3065
3066 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3067 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3068 STATIC_PER_CU_PG_ENABLE,
3069 enable ? 1 : 0);
3070 if(default_data != data)
3071 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3072 }
3073
gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device * adev,bool enable)3074 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
3075 bool enable)
3076 {
3077 uint32_t data, default_data;
3078
3079 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
3080 data = REG_SET_FIELD(data, RLC_PG_CNTL,
3081 DYN_PER_CU_PG_ENABLE,
3082 enable ? 1 : 0);
3083 if(default_data != data)
3084 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
3085 }
3086
gfx_v9_0_init_pg(struct amdgpu_device * adev)3087 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
3088 {
3089 gfx_v9_0_init_csb(adev);
3090
3091 /*
3092 * Rlc save restore list is workable since v2_1.
3093 * And it's needed by gfxoff feature.
3094 */
3095 if (adev->gfx.rlc.is_rlc_v2_1) {
3096 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3097 IP_VERSION(9, 2, 1) ||
3098 (adev->apu_flags & AMD_APU_IS_RAVEN2))
3099 gfx_v9_1_init_rlc_save_restore_list(adev);
3100 gfx_v9_0_enable_save_restore_machine(adev);
3101 }
3102
3103 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3104 AMD_PG_SUPPORT_GFX_SMG |
3105 AMD_PG_SUPPORT_GFX_DMG |
3106 AMD_PG_SUPPORT_CP |
3107 AMD_PG_SUPPORT_GDS |
3108 AMD_PG_SUPPORT_RLC_SMU_HS)) {
3109 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE,
3110 adev->gfx.rlc.cp_table_gpu_addr >> 8);
3111 gfx_v9_0_init_gfx_power_gating(adev);
3112 }
3113 }
3114
gfx_v9_0_rlc_stop(struct amdgpu_device * adev)3115 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
3116 {
3117 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
3118 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3119 gfx_v9_0_wait_for_rlc_serdes(adev);
3120 }
3121
gfx_v9_0_rlc_reset(struct amdgpu_device * adev)3122 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
3123 {
3124 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3125 udelay(50);
3126 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3127 udelay(50);
3128 }
3129
gfx_v9_0_rlc_start(struct amdgpu_device * adev)3130 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3131 {
3132 #ifdef AMDGPU_RLC_DEBUG_RETRY
3133 u32 rlc_ucode_ver;
3134 #endif
3135
3136 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3137 udelay(50);
3138
3139 /* carrizo do enable cp interrupt after cp inited */
3140 if (!(adev->flags & AMD_IS_APU)) {
3141 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3142 udelay(50);
3143 }
3144
3145 #ifdef AMDGPU_RLC_DEBUG_RETRY
3146 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3147 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3148 if(rlc_ucode_ver == 0x108) {
3149 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3150 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3151 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3152 * default is 0x9C4 to create a 100us interval */
3153 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3154 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3155 * to disable the page fault retry interrupts, default is
3156 * 0x100 (256) */
3157 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3158 }
3159 #endif
3160 }
3161
gfx_v9_0_rlc_load_microcode(struct amdgpu_device * adev)3162 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3163 {
3164 const struct rlc_firmware_header_v2_0 *hdr;
3165 const __le32 *fw_data;
3166 unsigned i, fw_size;
3167
3168 if (!adev->gfx.rlc_fw)
3169 return -EINVAL;
3170
3171 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3172 amdgpu_ucode_print_rlc_hdr(&hdr->header);
3173
3174 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3175 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3176 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3177
3178 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3179 RLCG_UCODE_LOADING_START_ADDRESS);
3180 for (i = 0; i < fw_size; i++)
3181 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3182 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3183
3184 return 0;
3185 }
3186
gfx_v9_0_rlc_resume(struct amdgpu_device * adev)3187 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3188 {
3189 int r;
3190
3191 if (amdgpu_sriov_vf(adev)) {
3192 gfx_v9_0_init_csb(adev);
3193 return 0;
3194 }
3195
3196 adev->gfx.rlc.funcs->stop(adev);
3197
3198 /* disable CG */
3199 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3200
3201 gfx_v9_0_init_pg(adev);
3202
3203 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3204 /* legacy rlc firmware loading */
3205 r = gfx_v9_0_rlc_load_microcode(adev);
3206 if (r)
3207 return r;
3208 }
3209
3210 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3211 case IP_VERSION(9, 2, 2):
3212 case IP_VERSION(9, 1, 0):
3213 gfx_v9_0_init_lbpw(adev);
3214 if (amdgpu_lbpw == 0)
3215 gfx_v9_0_enable_lbpw(adev, false);
3216 else
3217 gfx_v9_0_enable_lbpw(adev, true);
3218 break;
3219 case IP_VERSION(9, 4, 0):
3220 gfx_v9_4_init_lbpw(adev);
3221 if (amdgpu_lbpw > 0)
3222 gfx_v9_0_enable_lbpw(adev, true);
3223 else
3224 gfx_v9_0_enable_lbpw(adev, false);
3225 break;
3226 default:
3227 break;
3228 }
3229
3230 gfx_v9_0_update_spm_vmid_internal(adev, 0xf);
3231
3232 adev->gfx.rlc.funcs->start(adev);
3233
3234 return 0;
3235 }
3236
gfx_v9_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)3237 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3238 {
3239 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3240
3241 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_INVALIDATE_ICACHE, enable ? 0 : 1);
3242 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_INVALIDATE_ICACHE, enable ? 0 : 1);
3243 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_INVALIDATE_ICACHE, enable ? 0 : 1);
3244 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE0_RESET, enable ? 0 : 1);
3245 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE1_RESET, enable ? 0 : 1);
3246 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, enable ? 0 : 1);
3247 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, enable ? 0 : 1);
3248 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, enable ? 0 : 1);
3249 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, enable ? 0 : 1);
3250 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3251 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3252 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3253 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3254 udelay(50);
3255 }
3256
gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3257 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3258 {
3259 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3260 const struct gfx_firmware_header_v1_0 *ce_hdr;
3261 const struct gfx_firmware_header_v1_0 *me_hdr;
3262 const __le32 *fw_data;
3263 unsigned i, fw_size;
3264
3265 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3266 return -EINVAL;
3267
3268 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3269 adev->gfx.pfp_fw->data;
3270 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3271 adev->gfx.ce_fw->data;
3272 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3273 adev->gfx.me_fw->data;
3274
3275 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3276 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3277 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3278
3279 gfx_v9_0_cp_gfx_enable(adev, false);
3280
3281 /* PFP */
3282 fw_data = (const __le32 *)
3283 (adev->gfx.pfp_fw->data +
3284 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3285 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3286 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3287 for (i = 0; i < fw_size; i++)
3288 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3289 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3290
3291 /* CE */
3292 fw_data = (const __le32 *)
3293 (adev->gfx.ce_fw->data +
3294 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3295 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3296 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3297 for (i = 0; i < fw_size; i++)
3298 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3299 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3300
3301 /* ME */
3302 fw_data = (const __le32 *)
3303 (adev->gfx.me_fw->data +
3304 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3305 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3306 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3307 for (i = 0; i < fw_size; i++)
3308 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3309 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3310
3311 return 0;
3312 }
3313
gfx_v9_0_cp_gfx_start(struct amdgpu_device * adev)3314 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3315 {
3316 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3317 const struct cs_section_def *sect = NULL;
3318 const struct cs_extent_def *ext = NULL;
3319 int r, i, tmp;
3320
3321 /* init the CP */
3322 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3323 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3324
3325 gfx_v9_0_cp_gfx_enable(adev, true);
3326
3327 /* Now only limit the quirk on the APU gfx9 series and already
3328 * confirmed that the APU gfx10/gfx11 needn't such update.
3329 */
3330 if (adev->flags & AMD_IS_APU &&
3331 adev->in_s3 && !pm_resume_via_firmware()) {
3332 DRM_INFO("Will skip the CSB packet resubmit\n");
3333 return 0;
3334 }
3335 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3336 if (r) {
3337 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3338 return r;
3339 }
3340
3341 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3342 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3343
3344 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3345 amdgpu_ring_write(ring, 0x80000000);
3346 amdgpu_ring_write(ring, 0x80000000);
3347
3348 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3349 for (ext = sect->section; ext->extent != NULL; ++ext) {
3350 if (sect->id == SECT_CONTEXT) {
3351 amdgpu_ring_write(ring,
3352 PACKET3(PACKET3_SET_CONTEXT_REG,
3353 ext->reg_count));
3354 amdgpu_ring_write(ring,
3355 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3356 for (i = 0; i < ext->reg_count; i++)
3357 amdgpu_ring_write(ring, ext->extent[i]);
3358 }
3359 }
3360 }
3361
3362 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3363 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3364
3365 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3366 amdgpu_ring_write(ring, 0);
3367
3368 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3369 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3370 amdgpu_ring_write(ring, 0x8000);
3371 amdgpu_ring_write(ring, 0x8000);
3372
3373 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3374 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3375 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3376 amdgpu_ring_write(ring, tmp);
3377 amdgpu_ring_write(ring, 0);
3378
3379 amdgpu_ring_commit(ring);
3380
3381 return 0;
3382 }
3383
gfx_v9_0_cp_gfx_resume(struct amdgpu_device * adev)3384 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3385 {
3386 struct amdgpu_ring *ring;
3387 u32 tmp;
3388 u32 rb_bufsz;
3389 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3390
3391 /* Set the write pointer delay */
3392 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3393
3394 /* set the RB to use vmid 0 */
3395 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3396
3397 /* Set ring buffer size */
3398 ring = &adev->gfx.gfx_ring[0];
3399 rb_bufsz = order_base_2(ring->ring_size / 8);
3400 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3401 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3402 #ifdef __BIG_ENDIAN
3403 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3404 #endif
3405 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3406
3407 /* Initialize the ring buffer's write pointers */
3408 ring->wptr = 0;
3409 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3410 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3411
3412 /* set the wb address whether it's enabled or not */
3413 rptr_addr = ring->rptr_gpu_addr;
3414 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3415 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3416
3417 wptr_gpu_addr = ring->wptr_gpu_addr;
3418 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3419 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3420
3421 mdelay(1);
3422 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3423
3424 rb_addr = ring->gpu_addr >> 8;
3425 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3426 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3427
3428 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3429 if (ring->use_doorbell) {
3430 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3431 DOORBELL_OFFSET, ring->doorbell_index);
3432 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3433 DOORBELL_EN, 1);
3434 } else {
3435 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3436 }
3437 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3438
3439 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3440 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3441 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3442
3443 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3444 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3445
3446
3447 /* start the ring */
3448 gfx_v9_0_cp_gfx_start(adev);
3449
3450 return 0;
3451 }
3452
gfx_v9_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3453 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3454 {
3455 if (enable) {
3456 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3457 } else {
3458 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3459 (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
3460 CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
3461 CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
3462 CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
3463 CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
3464 CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
3465 CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
3466 CP_MEC_CNTL__MEC_ME1_HALT_MASK |
3467 CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3468 adev->gfx.kiq[0].ring.sched.ready = false;
3469 }
3470 udelay(50);
3471 }
3472
gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device * adev)3473 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3474 {
3475 const struct gfx_firmware_header_v1_0 *mec_hdr;
3476 const __le32 *fw_data;
3477 unsigned i;
3478 u32 tmp;
3479
3480 if (!adev->gfx.mec_fw)
3481 return -EINVAL;
3482
3483 gfx_v9_0_cp_compute_enable(adev, false);
3484
3485 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3486 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3487
3488 fw_data = (const __le32 *)
3489 (adev->gfx.mec_fw->data +
3490 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3491 tmp = 0;
3492 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3493 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3494 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3495
3496 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3497 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3498 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3499 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3500
3501 /* MEC1 */
3502 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3503 mec_hdr->jt_offset);
3504 for (i = 0; i < mec_hdr->jt_size; i++)
3505 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3506 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3507
3508 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3509 adev->gfx.mec_fw_version);
3510 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3511
3512 return 0;
3513 }
3514
3515 /* KIQ functions */
gfx_v9_0_kiq_setting(struct amdgpu_ring * ring)3516 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3517 {
3518 uint32_t tmp;
3519 struct amdgpu_device *adev = ring->adev;
3520
3521 /* tell RLC which is KIQ queue */
3522 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3523 tmp &= 0xffffff00;
3524 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3525 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp | 0x80);
3526 }
3527
gfx_v9_0_mqd_set_priority(struct amdgpu_ring * ring,struct v9_mqd * mqd)3528 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3529 {
3530 struct amdgpu_device *adev = ring->adev;
3531
3532 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3533 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
3534 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3535 mqd->cp_hqd_queue_priority =
3536 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3537 }
3538 }
3539 }
3540
gfx_v9_0_mqd_init(struct amdgpu_ring * ring)3541 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3542 {
3543 struct amdgpu_device *adev = ring->adev;
3544 struct v9_mqd *mqd = ring->mqd_ptr;
3545 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3546 uint32_t tmp;
3547
3548 mqd->header = 0xC0310800;
3549 mqd->compute_pipelinestat_enable = 0x00000001;
3550 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3551 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3552 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3553 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3554 mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3555 mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3556 mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3557 mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3558 mqd->compute_misc_reserved = 0x00000003;
3559
3560 mqd->dynamic_cu_mask_addr_lo =
3561 lower_32_bits(ring->mqd_gpu_addr
3562 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3563 mqd->dynamic_cu_mask_addr_hi =
3564 upper_32_bits(ring->mqd_gpu_addr
3565 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3566
3567 eop_base_addr = ring->eop_gpu_addr >> 8;
3568 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3569 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3570
3571 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3572 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3573 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3574 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3575
3576 mqd->cp_hqd_eop_control = tmp;
3577
3578 /* enable doorbell? */
3579 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3580
3581 if (ring->use_doorbell) {
3582 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3583 DOORBELL_OFFSET, ring->doorbell_index);
3584 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3585 DOORBELL_EN, 1);
3586 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3587 DOORBELL_SOURCE, 0);
3588 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3589 DOORBELL_HIT, 0);
3590 } else {
3591 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3592 DOORBELL_EN, 0);
3593 }
3594
3595 mqd->cp_hqd_pq_doorbell_control = tmp;
3596
3597 /* disable the queue if it's active */
3598 ring->wptr = 0;
3599 mqd->cp_hqd_dequeue_request = 0;
3600 mqd->cp_hqd_pq_rptr = 0;
3601 mqd->cp_hqd_pq_wptr_lo = 0;
3602 mqd->cp_hqd_pq_wptr_hi = 0;
3603
3604 /* set the pointer to the MQD */
3605 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3606 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3607
3608 /* set MQD vmid to 0 */
3609 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3610 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3611 mqd->cp_mqd_control = tmp;
3612
3613 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3614 hqd_gpu_addr = ring->gpu_addr >> 8;
3615 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3616 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3617
3618 /* set up the HQD, this is similar to CP_RB0_CNTL */
3619 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3620 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3621 (order_base_2(ring->ring_size / 4) - 1));
3622 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3623 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
3624 #ifdef __BIG_ENDIAN
3625 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3626 #endif
3627 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3628 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3629 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3630 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3631 mqd->cp_hqd_pq_control = tmp;
3632
3633 /* set the wb address whether it's enabled or not */
3634 wb_gpu_addr = ring->rptr_gpu_addr;
3635 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3636 mqd->cp_hqd_pq_rptr_report_addr_hi =
3637 upper_32_bits(wb_gpu_addr) & 0xffff;
3638
3639 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3640 wb_gpu_addr = ring->wptr_gpu_addr;
3641 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3642 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3643
3644 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3645 ring->wptr = 0;
3646 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3647
3648 /* set the vmid for the queue */
3649 mqd->cp_hqd_vmid = 0;
3650
3651 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3652 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3653 mqd->cp_hqd_persistent_state = tmp;
3654
3655 /* set MIN_IB_AVAIL_SIZE */
3656 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3657 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3658 mqd->cp_hqd_ib_control = tmp;
3659
3660 /* set static priority for a queue/ring */
3661 gfx_v9_0_mqd_set_priority(ring, mqd);
3662 mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
3663
3664 /* map_queues packet doesn't need activate the queue,
3665 * so only kiq need set this field.
3666 */
3667 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3668 mqd->cp_hqd_active = 1;
3669
3670 return 0;
3671 }
3672
gfx_v9_0_kiq_init_register(struct amdgpu_ring * ring)3673 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3674 {
3675 struct amdgpu_device *adev = ring->adev;
3676 struct v9_mqd *mqd = ring->mqd_ptr;
3677 int j;
3678
3679 /* disable wptr polling */
3680 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3681
3682 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3683 mqd->cp_hqd_eop_base_addr_lo);
3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3685 mqd->cp_hqd_eop_base_addr_hi);
3686
3687 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3688 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3689 mqd->cp_hqd_eop_control);
3690
3691 /* enable doorbell? */
3692 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3693 mqd->cp_hqd_pq_doorbell_control);
3694
3695 /* disable the queue if it's active */
3696 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3697 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3698 for (j = 0; j < adev->usec_timeout; j++) {
3699 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3700 break;
3701 udelay(1);
3702 }
3703 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3704 mqd->cp_hqd_dequeue_request);
3705 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3706 mqd->cp_hqd_pq_rptr);
3707 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3708 mqd->cp_hqd_pq_wptr_lo);
3709 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3710 mqd->cp_hqd_pq_wptr_hi);
3711 }
3712
3713 /* set the pointer to the MQD */
3714 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3715 mqd->cp_mqd_base_addr_lo);
3716 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3717 mqd->cp_mqd_base_addr_hi);
3718
3719 /* set MQD vmid to 0 */
3720 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3721 mqd->cp_mqd_control);
3722
3723 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3724 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3725 mqd->cp_hqd_pq_base_lo);
3726 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3727 mqd->cp_hqd_pq_base_hi);
3728
3729 /* set up the HQD, this is similar to CP_RB0_CNTL */
3730 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3731 mqd->cp_hqd_pq_control);
3732
3733 /* set the wb address whether it's enabled or not */
3734 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3735 mqd->cp_hqd_pq_rptr_report_addr_lo);
3736 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3737 mqd->cp_hqd_pq_rptr_report_addr_hi);
3738
3739 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3740 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3741 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3742 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3743 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3744
3745 /* enable the doorbell if requested */
3746 if (ring->use_doorbell) {
3747 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3748 (adev->doorbell_index.kiq * 2) << 2);
3749 /* If GC has entered CGPG, ringing doorbell > first page
3750 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to
3751 * workaround this issue. And this change has to align with firmware
3752 * update.
3753 */
3754 if (check_if_enlarge_doorbell_range(adev))
3755 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3756 (adev->doorbell.size - 4));
3757 else
3758 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3759 (adev->doorbell_index.userqueue_end * 2) << 2);
3760 }
3761
3762 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3763 mqd->cp_hqd_pq_doorbell_control);
3764
3765 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3766 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3767 mqd->cp_hqd_pq_wptr_lo);
3768 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3769 mqd->cp_hqd_pq_wptr_hi);
3770
3771 /* set the vmid for the queue */
3772 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3773
3774 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3775 mqd->cp_hqd_persistent_state);
3776
3777 /* activate the queue */
3778 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3779 mqd->cp_hqd_active);
3780
3781 if (ring->use_doorbell)
3782 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3783
3784 return 0;
3785 }
3786
gfx_v9_0_kiq_fini_register(struct amdgpu_ring * ring)3787 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3788 {
3789 struct amdgpu_device *adev = ring->adev;
3790 int j;
3791
3792 /* disable the queue if it's active */
3793 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3794
3795 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3796
3797 for (j = 0; j < adev->usec_timeout; j++) {
3798 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3799 break;
3800 udelay(1);
3801 }
3802
3803 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3804 DRM_DEBUG("KIQ dequeue request failed.\n");
3805
3806 /* Manual disable if dequeue request times out */
3807 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3808 }
3809
3810 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3811 0);
3812 }
3813
3814 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3815 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3816 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3817 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3818 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3819 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3820 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3821 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3822
3823 return 0;
3824 }
3825
gfx_v9_0_kiq_init_queue(struct amdgpu_ring * ring)3826 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3827 {
3828 struct amdgpu_device *adev = ring->adev;
3829 struct v9_mqd *mqd = ring->mqd_ptr;
3830 struct v9_mqd *tmp_mqd;
3831
3832 gfx_v9_0_kiq_setting(ring);
3833
3834 /* GPU could be in bad state during probe, driver trigger the reset
3835 * after load the SMU, in this case , the mqd is not be initialized.
3836 * driver need to re-init the mqd.
3837 * check mqd->cp_hqd_pq_control since this value should not be 0
3838 */
3839 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup;
3840 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){
3841 /* for GPU_RESET case , reset MQD to a clean status */
3842 if (adev->gfx.kiq[0].mqd_backup)
3843 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation));
3844
3845 /* reset ring buffer */
3846 ring->wptr = 0;
3847 amdgpu_ring_clear_ring(ring);
3848
3849 mutex_lock(&adev->srbm_mutex);
3850 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3851 gfx_v9_0_kiq_init_register(ring);
3852 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3853 mutex_unlock(&adev->srbm_mutex);
3854 } else {
3855 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3856 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3857 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3858 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
3859 amdgpu_ring_clear_ring(ring);
3860 mutex_lock(&adev->srbm_mutex);
3861 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3862 gfx_v9_0_mqd_init(ring);
3863 gfx_v9_0_kiq_init_register(ring);
3864 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3865 mutex_unlock(&adev->srbm_mutex);
3866
3867 if (adev->gfx.kiq[0].mqd_backup)
3868 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
3869 }
3870
3871 return 0;
3872 }
3873
gfx_v9_0_kcq_init_queue(struct amdgpu_ring * ring,bool restore)3874 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore)
3875 {
3876 struct amdgpu_device *adev = ring->adev;
3877 struct v9_mqd *mqd = ring->mqd_ptr;
3878 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3879 struct v9_mqd *tmp_mqd;
3880
3881 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
3882 * is not be initialized before
3883 */
3884 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
3885
3886 if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
3887 (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
3888 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3889 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3890 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3891 mutex_lock(&adev->srbm_mutex);
3892 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
3893 gfx_v9_0_mqd_init(ring);
3894 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3895 mutex_unlock(&adev->srbm_mutex);
3896
3897 if (adev->gfx.mec.mqd_backup[mqd_idx])
3898 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3899 } else {
3900 /* restore MQD to a clean status */
3901 if (adev->gfx.mec.mqd_backup[mqd_idx])
3902 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3903 /* reset ring buffer */
3904 ring->wptr = 0;
3905 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
3906 amdgpu_ring_clear_ring(ring);
3907 }
3908
3909 return 0;
3910 }
3911
gfx_v9_0_kiq_resume(struct amdgpu_device * adev)3912 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3913 {
3914 gfx_v9_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
3915 return 0;
3916 }
3917
gfx_v9_0_kcq_resume(struct amdgpu_device * adev)3918 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3919 {
3920 int i, r;
3921
3922 gfx_v9_0_cp_compute_enable(adev, true);
3923
3924 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3925 r = gfx_v9_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
3926 if (r)
3927 return r;
3928 }
3929
3930 return amdgpu_gfx_enable_kcq(adev, 0);
3931 }
3932
gfx_v9_0_cp_resume(struct amdgpu_device * adev)3933 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3934 {
3935 int r, i;
3936 struct amdgpu_ring *ring;
3937
3938 if (!(adev->flags & AMD_IS_APU))
3939 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3940
3941 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3942 if (adev->gfx.num_gfx_rings) {
3943 /* legacy firmware loading */
3944 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3945 if (r)
3946 return r;
3947 }
3948
3949 r = gfx_v9_0_cp_compute_load_microcode(adev);
3950 if (r)
3951 return r;
3952 }
3953
3954 if (adev->gfx.num_gfx_rings)
3955 gfx_v9_0_cp_gfx_enable(adev, false);
3956 gfx_v9_0_cp_compute_enable(adev, false);
3957
3958 r = gfx_v9_0_kiq_resume(adev);
3959 if (r)
3960 return r;
3961
3962 if (adev->gfx.num_gfx_rings) {
3963 r = gfx_v9_0_cp_gfx_resume(adev);
3964 if (r)
3965 return r;
3966 }
3967
3968 r = gfx_v9_0_kcq_resume(adev);
3969 if (r)
3970 return r;
3971
3972 if (adev->gfx.num_gfx_rings) {
3973 ring = &adev->gfx.gfx_ring[0];
3974 r = amdgpu_ring_test_helper(ring);
3975 if (r)
3976 return r;
3977 }
3978
3979 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3980 ring = &adev->gfx.compute_ring[i];
3981 amdgpu_ring_test_helper(ring);
3982 }
3983
3984 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3985
3986 return 0;
3987 }
3988
gfx_v9_0_init_tcp_config(struct amdgpu_device * adev)3989 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3990 {
3991 u32 tmp;
3992
3993 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) &&
3994 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))
3995 return;
3996
3997 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3998 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3999 adev->df.hash_status.hash_64k);
4000 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
4001 adev->df.hash_status.hash_2m);
4002 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
4003 adev->df.hash_status.hash_1g);
4004 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
4005 }
4006
gfx_v9_0_cp_enable(struct amdgpu_device * adev,bool enable)4007 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
4008 {
4009 if (adev->gfx.num_gfx_rings)
4010 gfx_v9_0_cp_gfx_enable(adev, enable);
4011 gfx_v9_0_cp_compute_enable(adev, enable);
4012 }
4013
gfx_v9_0_hw_init(struct amdgpu_ip_block * ip_block)4014 static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
4015 {
4016 int r;
4017 struct amdgpu_device *adev = ip_block->adev;
4018
4019 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4020 adev->gfx.cleaner_shader_ptr);
4021
4022 if (!amdgpu_sriov_vf(adev))
4023 gfx_v9_0_init_golden_registers(adev);
4024
4025 gfx_v9_0_constants_init(adev);
4026
4027 gfx_v9_0_init_tcp_config(adev);
4028
4029 r = adev->gfx.rlc.funcs->resume(adev);
4030 if (r)
4031 return r;
4032
4033 r = gfx_v9_0_cp_resume(adev);
4034 if (r)
4035 return r;
4036
4037 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) &&
4038 !amdgpu_sriov_vf(adev))
4039 gfx_v9_4_2_set_power_brake_sequence(adev);
4040
4041 return r;
4042 }
4043
gfx_v9_0_hw_fini(struct amdgpu_ip_block * ip_block)4044 static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
4045 {
4046 struct amdgpu_device *adev = ip_block->adev;
4047
4048 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4049 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4050 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4051 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4052 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4053
4054 /* DF freeze and kcq disable will fail */
4055 if (!amdgpu_ras_intr_triggered())
4056 /* disable KCQ to avoid CPC touch memory not valid anymore */
4057 amdgpu_gfx_disable_kcq(adev, 0);
4058
4059 if (amdgpu_sriov_vf(adev)) {
4060 gfx_v9_0_cp_gfx_enable(adev, false);
4061 /* must disable polling for SRIOV when hw finished, otherwise
4062 * CPC engine may still keep fetching WB address which is already
4063 * invalid after sw finished and trigger DMAR reading error in
4064 * hypervisor side.
4065 */
4066 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4067 return 0;
4068 }
4069
4070 /* Use deinitialize sequence from CAIL when unbinding device from driver,
4071 * otherwise KIQ is hanging when binding back
4072 */
4073 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4074 mutex_lock(&adev->srbm_mutex);
4075 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me,
4076 adev->gfx.kiq[0].ring.pipe,
4077 adev->gfx.kiq[0].ring.queue, 0, 0);
4078 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring);
4079 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
4080 mutex_unlock(&adev->srbm_mutex);
4081 }
4082
4083 gfx_v9_0_cp_enable(adev, false);
4084
4085 /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */
4086 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) ||
4087 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) {
4088 dev_dbg(adev->dev, "Skipping RLC halt\n");
4089 return 0;
4090 }
4091
4092 adev->gfx.rlc.funcs->stop(adev);
4093 return 0;
4094 }
4095
gfx_v9_0_suspend(struct amdgpu_ip_block * ip_block)4096 static int gfx_v9_0_suspend(struct amdgpu_ip_block *ip_block)
4097 {
4098 return gfx_v9_0_hw_fini(ip_block);
4099 }
4100
gfx_v9_0_resume(struct amdgpu_ip_block * ip_block)4101 static int gfx_v9_0_resume(struct amdgpu_ip_block *ip_block)
4102 {
4103 return gfx_v9_0_hw_init(ip_block);
4104 }
4105
gfx_v9_0_is_idle(struct amdgpu_ip_block * ip_block)4106 static bool gfx_v9_0_is_idle(struct amdgpu_ip_block *ip_block)
4107 {
4108 struct amdgpu_device *adev = ip_block->adev;
4109
4110 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
4111 GRBM_STATUS, GUI_ACTIVE))
4112 return false;
4113 else
4114 return true;
4115 }
4116
gfx_v9_0_wait_for_idle(struct amdgpu_ip_block * ip_block)4117 static int gfx_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4118 {
4119 unsigned i;
4120 struct amdgpu_device *adev = ip_block->adev;
4121
4122 for (i = 0; i < adev->usec_timeout; i++) {
4123 if (gfx_v9_0_is_idle(ip_block))
4124 return 0;
4125 udelay(1);
4126 }
4127 return -ETIMEDOUT;
4128 }
4129
gfx_v9_0_soft_reset(struct amdgpu_ip_block * ip_block)4130 static int gfx_v9_0_soft_reset(struct amdgpu_ip_block *ip_block)
4131 {
4132 u32 grbm_soft_reset = 0;
4133 u32 tmp;
4134 struct amdgpu_device *adev = ip_block->adev;
4135
4136 /* GRBM_STATUS */
4137 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4138 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4139 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4140 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4141 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4142 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4143 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4144 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4145 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4146 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4147 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4148 }
4149
4150 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4151 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4152 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4153 }
4154
4155 /* GRBM_STATUS2 */
4156 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4157 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4158 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4159 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4160
4161
4162 if (grbm_soft_reset) {
4163 /* stop the rlc */
4164 adev->gfx.rlc.funcs->stop(adev);
4165
4166 if (adev->gfx.num_gfx_rings)
4167 /* Disable GFX parsing/prefetching */
4168 gfx_v9_0_cp_gfx_enable(adev, false);
4169
4170 /* Disable MEC parsing/prefetching */
4171 gfx_v9_0_cp_compute_enable(adev, false);
4172
4173 if (grbm_soft_reset) {
4174 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4175 tmp |= grbm_soft_reset;
4176 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4177 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4178 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4179
4180 udelay(50);
4181
4182 tmp &= ~grbm_soft_reset;
4183 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4184 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4185 }
4186
4187 /* Wait a little for things to settle down */
4188 udelay(50);
4189 }
4190 return 0;
4191 }
4192
gfx_v9_0_kiq_read_clock(struct amdgpu_device * adev)4193 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4194 {
4195 signed long r, cnt = 0;
4196 unsigned long flags;
4197 uint32_t seq, reg_val_offs = 0;
4198 uint64_t value = 0;
4199 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
4200 struct amdgpu_ring *ring = &kiq->ring;
4201
4202 BUG_ON(!ring->funcs->emit_rreg);
4203
4204 spin_lock_irqsave(&kiq->ring_lock, flags);
4205 if (amdgpu_device_wb_get(adev, ®_val_offs)) {
4206 pr_err("critical bug! too many kiq readers\n");
4207 goto failed_unlock;
4208 }
4209 amdgpu_ring_alloc(ring, 32);
4210 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4211 amdgpu_ring_write(ring, 9 | /* src: register*/
4212 (5 << 8) | /* dst: memory */
4213 (1 << 16) | /* count sel */
4214 (1 << 20)); /* write confirm */
4215 amdgpu_ring_write(ring, 0);
4216 amdgpu_ring_write(ring, 0);
4217 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4218 reg_val_offs * 4));
4219 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4220 reg_val_offs * 4));
4221 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
4222 if (r)
4223 goto failed_undo;
4224
4225 amdgpu_ring_commit(ring);
4226 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4227
4228 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4229
4230 /* don't wait anymore for gpu reset case because this way may
4231 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4232 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4233 * never return if we keep waiting in virt_kiq_rreg, which cause
4234 * gpu_recover() hang there.
4235 *
4236 * also don't wait anymore for IRQ context
4237 * */
4238 if (r < 1 && (amdgpu_in_reset(adev)))
4239 goto failed_kiq_read;
4240
4241 might_sleep();
4242 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4243 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4244 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4245 }
4246
4247 if (cnt > MAX_KIQ_REG_TRY)
4248 goto failed_kiq_read;
4249
4250 mb();
4251 value = (uint64_t)adev->wb.wb[reg_val_offs] |
4252 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
4253 amdgpu_device_wb_free(adev, reg_val_offs);
4254 return value;
4255
4256 failed_undo:
4257 amdgpu_ring_undo(ring);
4258 failed_unlock:
4259 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4260 failed_kiq_read:
4261 if (reg_val_offs)
4262 amdgpu_device_wb_free(adev, reg_val_offs);
4263 pr_err("failed to read gpu clock\n");
4264 return ~0;
4265 }
4266
gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device * adev)4267 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4268 {
4269 uint64_t clock, clock_lo, clock_hi, hi_check;
4270
4271 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
4272 case IP_VERSION(9, 3, 0):
4273 preempt_disable();
4274 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4275 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4276 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir);
4277 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over
4278 * roughly every 42 seconds.
4279 */
4280 if (hi_check != clock_hi) {
4281 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir);
4282 clock_hi = hi_check;
4283 }
4284 preempt_enable();
4285 clock = clock_lo | (clock_hi << 32ULL);
4286 break;
4287 default:
4288 amdgpu_gfx_off_ctrl(adev, false);
4289 mutex_lock(&adev->gfx.gpu_clock_mutex);
4290 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
4291 IP_VERSION(9, 0, 1) &&
4292 amdgpu_sriov_runtime(adev)) {
4293 clock = gfx_v9_0_kiq_read_clock(adev);
4294 } else {
4295 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4296 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4297 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4298 }
4299 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4300 amdgpu_gfx_off_ctrl(adev, true);
4301 break;
4302 }
4303 return clock;
4304 }
4305
gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)4306 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4307 uint32_t vmid,
4308 uint32_t gds_base, uint32_t gds_size,
4309 uint32_t gws_base, uint32_t gws_size,
4310 uint32_t oa_base, uint32_t oa_size)
4311 {
4312 struct amdgpu_device *adev = ring->adev;
4313
4314 /* GDS Base */
4315 gfx_v9_0_write_data_to_reg(ring, 0, false,
4316 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4317 gds_base);
4318
4319 /* GDS Size */
4320 gfx_v9_0_write_data_to_reg(ring, 0, false,
4321 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4322 gds_size);
4323
4324 /* GWS */
4325 gfx_v9_0_write_data_to_reg(ring, 0, false,
4326 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4327 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4328
4329 /* OA */
4330 gfx_v9_0_write_data_to_reg(ring, 0, false,
4331 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4332 (1 << (oa_size + oa_base)) - (1 << oa_base));
4333 }
4334
4335 static const u32 vgpr_init_compute_shader[] =
4336 {
4337 0xb07c0000, 0xbe8000ff,
4338 0x000000f8, 0xbf110800,
4339 0x7e000280, 0x7e020280,
4340 0x7e040280, 0x7e060280,
4341 0x7e080280, 0x7e0a0280,
4342 0x7e0c0280, 0x7e0e0280,
4343 0x80808800, 0xbe803200,
4344 0xbf84fff5, 0xbf9c0000,
4345 0xd28c0001, 0x0001007f,
4346 0xd28d0001, 0x0002027e,
4347 0x10020288, 0xb8810904,
4348 0xb7814000, 0xd1196a01,
4349 0x00000301, 0xbe800087,
4350 0xbefc00c1, 0xd89c4000,
4351 0x00020201, 0xd89cc080,
4352 0x00040401, 0x320202ff,
4353 0x00000800, 0x80808100,
4354 0xbf84fff8, 0x7e020280,
4355 0xbf810000, 0x00000000,
4356 };
4357
4358 static const u32 sgpr_init_compute_shader[] =
4359 {
4360 0xb07c0000, 0xbe8000ff,
4361 0x0000005f, 0xbee50080,
4362 0xbe812c65, 0xbe822c65,
4363 0xbe832c65, 0xbe842c65,
4364 0xbe852c65, 0xb77c0005,
4365 0x80808500, 0xbf84fff8,
4366 0xbe800080, 0xbf810000,
4367 };
4368
4369 static const u32 vgpr_init_compute_shader_arcturus[] = {
4370 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4371 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4372 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4373 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4374 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4375 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4376 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4377 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4378 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4379 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4380 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4381 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4382 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4383 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4384 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4385 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4386 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4387 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4388 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4389 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4390 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4391 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4392 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4393 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4394 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4395 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4396 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4397 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4398 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4399 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4400 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4401 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4402 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4403 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4404 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4405 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4406 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4407 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4408 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4409 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4410 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4411 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4412 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4413 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4414 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4415 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4416 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4417 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4418 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4419 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4420 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4421 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4422 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4423 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4424 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4425 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4426 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4427 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4428 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4429 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4430 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4431 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4432 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4433 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4434 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4435 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4436 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4437 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4438 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4439 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4440 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4441 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4442 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4443 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4444 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4445 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4446 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4447 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4448 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4449 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4450 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4451 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4452 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4453 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4454 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4455 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4456 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4457 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4458 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4459 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4460 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4461 0xbf84fff8, 0xbf810000,
4462 };
4463
4464 /* When below register arrays changed, please update gpr_reg_size,
4465 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4466 to cover all gfx9 ASICs */
4467 static const struct soc15_reg_entry vgpr_init_regs[] = {
4468 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4469 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4470 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4471 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4472 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4473 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4474 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4475 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4476 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4477 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4478 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4479 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4480 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4481 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4482 };
4483
4484 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4485 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4486 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4487 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4488 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4489 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4490 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4491 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4492 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4493 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4494 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4495 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4496 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4497 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4498 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4499 };
4500
4501 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4502 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4503 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4504 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4505 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4506 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4507 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4508 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4509 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4510 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4511 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4512 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4513 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4514 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4515 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4516 };
4517
4518 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4519 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4520 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4521 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4522 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4523 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4524 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4525 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4526 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4527 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4528 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4529 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4530 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4531 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4532 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4533 };
4534
4535 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4536 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4537 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4538 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4539 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4540 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4541 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4542 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4543 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4544 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4545 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4546 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4547 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4548 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4549 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4550 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4551 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4552 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4553 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4554 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4555 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4556 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4557 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4558 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4559 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4560 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4561 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4562 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4563 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4564 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4565 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4566 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4567 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4568 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4569 };
4570
gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device * adev)4571 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4572 {
4573 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4574 int i, r;
4575
4576 /* only support when RAS is enabled */
4577 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4578 return 0;
4579
4580 r = amdgpu_ring_alloc(ring, 7);
4581 if (r) {
4582 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4583 ring->name, r);
4584 return r;
4585 }
4586
4587 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4588 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4589
4590 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4591 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4592 PACKET3_DMA_DATA_DST_SEL(1) |
4593 PACKET3_DMA_DATA_SRC_SEL(2) |
4594 PACKET3_DMA_DATA_ENGINE(0)));
4595 amdgpu_ring_write(ring, 0);
4596 amdgpu_ring_write(ring, 0);
4597 amdgpu_ring_write(ring, 0);
4598 amdgpu_ring_write(ring, 0);
4599 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4600 adev->gds.gds_size);
4601
4602 amdgpu_ring_commit(ring);
4603
4604 for (i = 0; i < adev->usec_timeout; i++) {
4605 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4606 break;
4607 udelay(1);
4608 }
4609
4610 if (i >= adev->usec_timeout)
4611 r = -ETIMEDOUT;
4612
4613 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4614
4615 return r;
4616 }
4617
gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device * adev)4618 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4619 {
4620 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4621 struct amdgpu_ib ib;
4622 struct dma_fence *f = NULL;
4623 int r, i;
4624 unsigned total_size, vgpr_offset, sgpr_offset;
4625 u64 gpu_addr;
4626
4627 int compute_dim_x = adev->gfx.config.max_shader_engines *
4628 adev->gfx.config.max_cu_per_sh *
4629 adev->gfx.config.max_sh_per_se;
4630 int sgpr_work_group_size = 5;
4631 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4632 int vgpr_init_shader_size;
4633 const u32 *vgpr_init_shader_ptr;
4634 const struct soc15_reg_entry *vgpr_init_regs_ptr;
4635
4636 /* only support when RAS is enabled */
4637 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4638 return 0;
4639
4640 /* bail if the compute ring is not ready */
4641 if (!ring->sched.ready)
4642 return 0;
4643
4644 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
4645 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4646 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4647 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4648 } else {
4649 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4650 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4651 vgpr_init_regs_ptr = vgpr_init_regs;
4652 }
4653
4654 total_size =
4655 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4656 total_size +=
4657 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4658 total_size +=
4659 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4660 total_size = ALIGN(total_size, 256);
4661 vgpr_offset = total_size;
4662 total_size += ALIGN(vgpr_init_shader_size, 256);
4663 sgpr_offset = total_size;
4664 total_size += sizeof(sgpr_init_compute_shader);
4665
4666 /* allocate an indirect buffer to put the commands in */
4667 memset(&ib, 0, sizeof(ib));
4668 r = amdgpu_ib_get(adev, NULL, total_size,
4669 AMDGPU_IB_POOL_DIRECT, &ib);
4670 if (r) {
4671 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4672 return r;
4673 }
4674
4675 /* load the compute shaders */
4676 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4677 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4678
4679 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4680 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4681
4682 /* init the ib length to 0 */
4683 ib.length_dw = 0;
4684
4685 /* VGPR */
4686 /* write the register state for the compute dispatch */
4687 for (i = 0; i < gpr_reg_size; i++) {
4688 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4689 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4690 - PACKET3_SET_SH_REG_START;
4691 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4692 }
4693 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4694 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4695 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4696 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4697 - PACKET3_SET_SH_REG_START;
4698 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4699 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4700
4701 /* write dispatch packet */
4702 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4703 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4704 ib.ptr[ib.length_dw++] = 1; /* y */
4705 ib.ptr[ib.length_dw++] = 1; /* z */
4706 ib.ptr[ib.length_dw++] =
4707 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4708
4709 /* write CS partial flush packet */
4710 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4711 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4712
4713 /* SGPR1 */
4714 /* write the register state for the compute dispatch */
4715 for (i = 0; i < gpr_reg_size; i++) {
4716 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4717 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4718 - PACKET3_SET_SH_REG_START;
4719 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4720 }
4721 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4722 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4723 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4724 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4725 - PACKET3_SET_SH_REG_START;
4726 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4727 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4728
4729 /* write dispatch packet */
4730 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4731 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4732 ib.ptr[ib.length_dw++] = 1; /* y */
4733 ib.ptr[ib.length_dw++] = 1; /* z */
4734 ib.ptr[ib.length_dw++] =
4735 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4736
4737 /* write CS partial flush packet */
4738 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4739 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4740
4741 /* SGPR2 */
4742 /* write the register state for the compute dispatch */
4743 for (i = 0; i < gpr_reg_size; i++) {
4744 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4745 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4746 - PACKET3_SET_SH_REG_START;
4747 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4748 }
4749 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4750 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4751 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4752 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4753 - PACKET3_SET_SH_REG_START;
4754 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4755 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4756
4757 /* write dispatch packet */
4758 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4759 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4760 ib.ptr[ib.length_dw++] = 1; /* y */
4761 ib.ptr[ib.length_dw++] = 1; /* z */
4762 ib.ptr[ib.length_dw++] =
4763 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4764
4765 /* write CS partial flush packet */
4766 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4767 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4768
4769 /* shedule the ib on the ring */
4770 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4771 if (r) {
4772 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4773 goto fail;
4774 }
4775
4776 /* wait for the GPU to finish processing the IB */
4777 r = dma_fence_wait(f, false);
4778 if (r) {
4779 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4780 goto fail;
4781 }
4782
4783 fail:
4784 amdgpu_ib_free(&ib, NULL);
4785 dma_fence_put(f);
4786
4787 return r;
4788 }
4789
gfx_v9_0_early_init(struct amdgpu_ip_block * ip_block)4790 static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block)
4791 {
4792 struct amdgpu_device *adev = ip_block->adev;
4793
4794 adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
4795
4796 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
4797 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4798 adev->gfx.num_gfx_rings = 0;
4799 else
4800 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4801 adev->gfx.xcc_mask = 1;
4802 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
4803 AMDGPU_MAX_COMPUTE_RINGS);
4804 gfx_v9_0_set_kiq_pm4_funcs(adev);
4805 gfx_v9_0_set_ring_funcs(adev);
4806 gfx_v9_0_set_irq_funcs(adev);
4807 gfx_v9_0_set_gds_init(adev);
4808 gfx_v9_0_set_rlc_funcs(adev);
4809
4810 /* init rlcg reg access ctrl */
4811 gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
4812
4813 return gfx_v9_0_init_microcode(adev);
4814 }
4815
gfx_v9_0_ecc_late_init(struct amdgpu_ip_block * ip_block)4816 static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block)
4817 {
4818 struct amdgpu_device *adev = ip_block->adev;
4819 int r;
4820
4821 /*
4822 * Temp workaround to fix the issue that CP firmware fails to
4823 * update read pointer when CPDMA is writing clearing operation
4824 * to GDS in suspend/resume sequence on several cards. So just
4825 * limit this operation in cold boot sequence.
4826 */
4827 if ((!adev->in_suspend) &&
4828 (adev->gds.gds_size)) {
4829 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4830 if (r)
4831 return r;
4832 }
4833
4834 /* requires IBs so do in late init after IB pool is initialized */
4835 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4836 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
4837 else
4838 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4839
4840 if (r)
4841 return r;
4842
4843 if (adev->gfx.ras &&
4844 adev->gfx.ras->enable_watchdog_timer)
4845 adev->gfx.ras->enable_watchdog_timer(adev);
4846
4847 return 0;
4848 }
4849
gfx_v9_0_late_init(struct amdgpu_ip_block * ip_block)4850 static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block)
4851 {
4852 struct amdgpu_device *adev = ip_block->adev;
4853 int r;
4854
4855 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4856 if (r)
4857 return r;
4858
4859 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4860 if (r)
4861 return r;
4862
4863 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
4864 if (r)
4865 return r;
4866
4867 r = gfx_v9_0_ecc_late_init(ip_block);
4868 if (r)
4869 return r;
4870
4871 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
4872 gfx_v9_4_2_debug_trap_config_init(adev,
4873 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4874 else
4875 gfx_v9_0_debug_trap_config_init(adev,
4876 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID);
4877
4878 return 0;
4879 }
4880
gfx_v9_0_is_rlc_enabled(struct amdgpu_device * adev)4881 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4882 {
4883 uint32_t rlc_setting;
4884
4885 /* if RLC is not enabled, do nothing */
4886 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4887 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4888 return false;
4889
4890 return true;
4891 }
4892
gfx_v9_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)4893 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
4894 {
4895 uint32_t data;
4896 unsigned i;
4897
4898 data = RLC_SAFE_MODE__CMD_MASK;
4899 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4900 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4901
4902 /* wait for RLC_SAFE_MODE */
4903 for (i = 0; i < adev->usec_timeout; i++) {
4904 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4905 break;
4906 udelay(1);
4907 }
4908 }
4909
gfx_v9_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)4910 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
4911 {
4912 uint32_t data;
4913
4914 data = RLC_SAFE_MODE__CMD_MASK;
4915 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4916 }
4917
gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)4918 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4919 bool enable)
4920 {
4921 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4922
4923 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4924 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4925 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4926 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4927 } else {
4928 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4929 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4930 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4931 }
4932
4933 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4934 }
4935
gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device * adev,bool enable)4936 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4937 bool enable)
4938 {
4939 /* TODO: double check if we need to perform under safe mode */
4940 /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4941
4942 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4943 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4944 else
4945 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4946
4947 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4948 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4949 else
4950 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4951
4952 /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4953 }
4954
gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)4955 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4956 bool enable)
4957 {
4958 uint32_t data, def;
4959
4960 /* It is disabled by HW by default */
4961 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4962 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4963 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4964
4965 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
4966 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4967
4968 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4969 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4970 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4971
4972 /* only for Vega10 & Raven1 */
4973 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4974
4975 if (def != data)
4976 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4977
4978 /* MGLS is a global flag to control all MGLS in GFX */
4979 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4980 /* 2 - RLC memory Light sleep */
4981 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4982 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4983 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4984 if (def != data)
4985 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4986 }
4987 /* 3 - CP memory Light sleep */
4988 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4989 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4990 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4991 if (def != data)
4992 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4993 }
4994 }
4995 } else {
4996 /* 1 - MGCG_OVERRIDE */
4997 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4998
4999 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1))
5000 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
5001
5002 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5003 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5004 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
5005 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
5006
5007 if (def != data)
5008 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5009
5010 /* 2 - disable MGLS in RLC */
5011 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
5012 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5013 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5014 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
5015 }
5016
5017 /* 3 - disable MGLS in CP */
5018 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
5019 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5020 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5021 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
5022 }
5023 }
5024 }
5025
gfx_v9_0_update_3d_clock_gating(struct amdgpu_device * adev,bool enable)5026 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
5027 bool enable)
5028 {
5029 uint32_t data, def;
5030
5031 if (!adev->gfx.num_gfx_rings)
5032 return;
5033
5034 /* Enable 3D CGCG/CGLS */
5035 if (enable) {
5036 /* write cmd to clear cgcg/cgls ov */
5037 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5038 /* unset CGCG override */
5039 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5040 /* update CGCG and CGLS override bits */
5041 if (def != data)
5042 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5043
5044 /* enable 3Dcgcg FSM(0x0000363f) */
5045 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5046
5047 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5048 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5049 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5050 else
5051 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT;
5052
5053 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5054 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5055 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5056 if (def != data)
5057 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5058
5059 /* set IDLE_POLL_COUNT(0x00900100) */
5060 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5061 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5062 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5063 if (def != data)
5064 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5065 } else {
5066 /* Disable CGCG/CGLS */
5067 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
5068 /* disable cgcg, cgls should be disabled */
5069 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
5070 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
5071 /* disable cgcg and cgls in FSM */
5072 if (def != data)
5073 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
5074 }
5075 }
5076
gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5077 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5078 bool enable)
5079 {
5080 uint32_t def, data;
5081
5082 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5083 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
5084 /* unset CGCG override */
5085 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5086 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5087 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5088 else
5089 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5090 /* update CGCG and CGLS override bits */
5091 if (def != data)
5092 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
5093
5094 /* enable cgcg FSM(0x0000363F) */
5095 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5096
5097 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1))
5098 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5099 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5100 else
5101 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5102 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5103 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5104 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5105 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5106 if (def != data)
5107 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5108
5109 /* set IDLE_POLL_COUNT(0x00900100) */
5110 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
5111 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5112 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5113 if (def != data)
5114 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
5115 } else {
5116 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
5117 /* reset CGCG/CGLS bits */
5118 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5119 /* disable cgcg and cgls in FSM */
5120 if (def != data)
5121 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
5122 }
5123 }
5124
gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5125 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5126 bool enable)
5127 {
5128 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5129 if (enable) {
5130 /* CGCG/CGLS should be enabled after MGCG/MGLS
5131 * === MGCG + MGLS ===
5132 */
5133 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5134 /* === CGCG /CGLS for GFX 3D Only === */
5135 gfx_v9_0_update_3d_clock_gating(adev, enable);
5136 /* === CGCG + CGLS === */
5137 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5138 } else {
5139 /* CGCG/CGLS should be disabled before MGCG/MGLS
5140 * === CGCG + CGLS ===
5141 */
5142 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
5143 /* === CGCG /CGLS for GFX 3D Only === */
5144 gfx_v9_0_update_3d_clock_gating(adev, enable);
5145 /* === MGCG + MGLS === */
5146 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
5147 }
5148 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5149 return 0;
5150 }
5151
gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device * adev,unsigned int vmid)5152 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev,
5153 unsigned int vmid)
5154 {
5155 u32 reg, data;
5156
5157 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL);
5158 if (amdgpu_sriov_is_pp_one_vf(adev))
5159 data = RREG32_NO_KIQ(reg);
5160 else
5161 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
5162
5163 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
5164 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5165
5166 if (amdgpu_sriov_is_pp_one_vf(adev))
5167 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data);
5168 else
5169 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
5170 }
5171
gfx_v9_0_update_spm_vmid(struct amdgpu_device * adev,struct amdgpu_ring * ring,unsigned int vmid)5172 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid)
5173 {
5174 amdgpu_gfx_off_ctrl(adev, false);
5175
5176 gfx_v9_0_update_spm_vmid_internal(adev, vmid);
5177
5178 amdgpu_gfx_off_ctrl(adev, true);
5179 }
5180
gfx_v9_0_check_rlcg_range(struct amdgpu_device * adev,uint32_t offset,struct soc15_reg_rlcg * entries,int arr_size)5181 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
5182 uint32_t offset,
5183 struct soc15_reg_rlcg *entries, int arr_size)
5184 {
5185 int i;
5186 uint32_t reg;
5187
5188 if (!entries)
5189 return false;
5190
5191 for (i = 0; i < arr_size; i++) {
5192 const struct soc15_reg_rlcg *entry;
5193
5194 entry = &entries[i];
5195 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
5196 if (offset == reg)
5197 return true;
5198 }
5199
5200 return false;
5201 }
5202
gfx_v9_0_is_rlcg_access_range(struct amdgpu_device * adev,u32 offset)5203 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
5204 {
5205 return gfx_v9_0_check_rlcg_range(adev, offset,
5206 (void *)rlcg_access_gc_9_0,
5207 ARRAY_SIZE(rlcg_access_gc_9_0));
5208 }
5209
5210 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5211 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5212 .set_safe_mode = gfx_v9_0_set_safe_mode,
5213 .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5214 .init = gfx_v9_0_rlc_init,
5215 .get_csb_size = gfx_v9_0_get_csb_size,
5216 .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5217 .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5218 .resume = gfx_v9_0_rlc_resume,
5219 .stop = gfx_v9_0_rlc_stop,
5220 .reset = gfx_v9_0_rlc_reset,
5221 .start = gfx_v9_0_rlc_start,
5222 .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5223 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5224 };
5225
gfx_v9_0_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)5226 static int gfx_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5227 enum amd_powergating_state state)
5228 {
5229 struct amdgpu_device *adev = ip_block->adev;
5230 bool enable = (state == AMD_PG_STATE_GATE);
5231
5232 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5233 case IP_VERSION(9, 2, 2):
5234 case IP_VERSION(9, 1, 0):
5235 case IP_VERSION(9, 3, 0):
5236 if (!enable)
5237 amdgpu_gfx_off_ctrl_immediate(adev, false);
5238
5239 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5240 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5241 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5242 } else {
5243 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5244 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5245 }
5246
5247 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5248 gfx_v9_0_enable_cp_power_gating(adev, true);
5249 else
5250 gfx_v9_0_enable_cp_power_gating(adev, false);
5251
5252 /* update gfx cgpg state */
5253 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5254
5255 /* update mgcg state */
5256 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5257
5258 if (enable)
5259 amdgpu_gfx_off_ctrl_immediate(adev, true);
5260 break;
5261 case IP_VERSION(9, 2, 1):
5262 amdgpu_gfx_off_ctrl_immediate(adev, enable);
5263 break;
5264 default:
5265 break;
5266 }
5267
5268 return 0;
5269 }
5270
gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)5271 static int gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5272 enum amd_clockgating_state state)
5273 {
5274 struct amdgpu_device *adev = ip_block->adev;
5275
5276 if (amdgpu_sriov_vf(adev))
5277 return 0;
5278
5279 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5280 case IP_VERSION(9, 0, 1):
5281 case IP_VERSION(9, 2, 1):
5282 case IP_VERSION(9, 4, 0):
5283 case IP_VERSION(9, 2, 2):
5284 case IP_VERSION(9, 1, 0):
5285 case IP_VERSION(9, 4, 1):
5286 case IP_VERSION(9, 3, 0):
5287 case IP_VERSION(9, 4, 2):
5288 gfx_v9_0_update_gfx_clock_gating(adev,
5289 state == AMD_CG_STATE_GATE);
5290 break;
5291 default:
5292 break;
5293 }
5294 return 0;
5295 }
5296
gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)5297 static void gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
5298 {
5299 struct amdgpu_device *adev = ip_block->adev;
5300 int data;
5301
5302 if (amdgpu_sriov_vf(adev))
5303 *flags = 0;
5304
5305 /* AMD_CG_SUPPORT_GFX_MGCG */
5306 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5307 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5308 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5309
5310 /* AMD_CG_SUPPORT_GFX_CGCG */
5311 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5312 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5313 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5314
5315 /* AMD_CG_SUPPORT_GFX_CGLS */
5316 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5317 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5318
5319 /* AMD_CG_SUPPORT_GFX_RLC_LS */
5320 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5321 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5322 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5323
5324 /* AMD_CG_SUPPORT_GFX_CP_LS */
5325 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5326 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5327 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5328
5329 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) {
5330 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5331 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5332 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5333 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5334
5335 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5336 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5337 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5338 }
5339 }
5340
gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5341 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5342 {
5343 return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/
5344 }
5345
gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5346 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5347 {
5348 struct amdgpu_device *adev = ring->adev;
5349 u64 wptr;
5350
5351 /* XXX check if swapping is necessary on BE */
5352 if (ring->use_doorbell) {
5353 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5354 } else {
5355 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5356 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5357 }
5358
5359 return wptr;
5360 }
5361
gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5362 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5363 {
5364 struct amdgpu_device *adev = ring->adev;
5365
5366 if (ring->use_doorbell) {
5367 /* XXX check if swapping is necessary on BE */
5368 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5369 WDOORBELL64(ring->doorbell_index, ring->wptr);
5370 } else {
5371 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5372 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5373 }
5374 }
5375
gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5376 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5377 {
5378 struct amdgpu_device *adev = ring->adev;
5379 u32 ref_and_mask, reg_mem_engine;
5380 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5381
5382 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5383 switch (ring->me) {
5384 case 1:
5385 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5386 break;
5387 case 2:
5388 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5389 break;
5390 default:
5391 return;
5392 }
5393 reg_mem_engine = 0;
5394 } else {
5395 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5396 reg_mem_engine = 1; /* pfp */
5397 }
5398
5399 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5400 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5401 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5402 ref_and_mask, ref_and_mask, 0x20);
5403 }
5404
gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5405 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5406 struct amdgpu_job *job,
5407 struct amdgpu_ib *ib,
5408 uint32_t flags)
5409 {
5410 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5411 u32 header, control = 0;
5412
5413 if (ib->flags & AMDGPU_IB_FLAG_CE)
5414 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5415 else
5416 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5417
5418 control |= ib->length_dw | (vmid << 24);
5419
5420 if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
5421 control |= INDIRECT_BUFFER_PRE_ENB(1);
5422
5423 if (flags & AMDGPU_IB_PREEMPTED)
5424 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5425
5426 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5427 gfx_v9_0_ring_emit_de_meta(ring,
5428 (!amdgpu_sriov_vf(ring->adev) &&
5429 flags & AMDGPU_IB_PREEMPTED) ?
5430 true : false,
5431 job->gds_size > 0 && job->gds_base != 0);
5432 }
5433
5434 amdgpu_ring_write(ring, header);
5435 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5436 amdgpu_ring_write(ring,
5437 #ifdef __BIG_ENDIAN
5438 (2 << 0) |
5439 #endif
5440 lower_32_bits(ib->gpu_addr));
5441 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5442 amdgpu_ring_ib_on_emit_cntl(ring);
5443 amdgpu_ring_write(ring, control);
5444 }
5445
gfx_v9_0_ring_patch_cntl(struct amdgpu_ring * ring,unsigned offset)5446 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring,
5447 unsigned offset)
5448 {
5449 u32 control = ring->ring[offset];
5450
5451 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5452 ring->ring[offset] = control;
5453 }
5454
gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring * ring,unsigned offset)5455 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring,
5456 unsigned offset)
5457 {
5458 struct amdgpu_device *adev = ring->adev;
5459 void *ce_payload_cpu_addr;
5460 uint64_t payload_offset, payload_size;
5461
5462 payload_size = sizeof(struct v9_ce_ib_state);
5463
5464 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5465 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5466
5467 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5468 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size);
5469 } else {
5470 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr,
5471 (ring->buf_mask + 1 - offset) << 2);
5472 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5473 memcpy((void *)&ring->ring[0],
5474 ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5475 payload_size);
5476 }
5477 }
5478
gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring * ring,unsigned offset)5479 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring,
5480 unsigned offset)
5481 {
5482 struct amdgpu_device *adev = ring->adev;
5483 void *de_payload_cpu_addr;
5484 uint64_t payload_offset, payload_size;
5485
5486 payload_size = sizeof(struct v9_de_ib_state);
5487
5488 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload);
5489 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset;
5490
5491 ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status =
5492 IB_COMPLETION_STATUS_PREEMPTED;
5493
5494 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) {
5495 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size);
5496 } else {
5497 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr,
5498 (ring->buf_mask + 1 - offset) << 2);
5499 payload_size -= (ring->buf_mask + 1 - offset) << 2;
5500 memcpy((void *)&ring->ring[0],
5501 de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2),
5502 payload_size);
5503 }
5504 }
5505
gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5506 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5507 struct amdgpu_job *job,
5508 struct amdgpu_ib *ib,
5509 uint32_t flags)
5510 {
5511 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5512 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5513
5514 /* Currently, there is a high possibility to get wave ID mismatch
5515 * between ME and GDS, leading to a hw deadlock, because ME generates
5516 * different wave IDs than the GDS expects. This situation happens
5517 * randomly when at least 5 compute pipes use GDS ordered append.
5518 * The wave IDs generated by ME are also wrong after suspend/resume.
5519 * Those are probably bugs somewhere else in the kernel driver.
5520 *
5521 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5522 * GDS to 0 for this ring (me/pipe).
5523 */
5524 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5525 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5526 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5527 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5528 }
5529
5530 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5531 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5532 amdgpu_ring_write(ring,
5533 #ifdef __BIG_ENDIAN
5534 (2 << 0) |
5535 #endif
5536 lower_32_bits(ib->gpu_addr));
5537 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5538 amdgpu_ring_write(ring, control);
5539 }
5540
gfx_v9_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)5541 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5542 u64 seq, unsigned flags)
5543 {
5544 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5545 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5546 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5547 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
5548 uint32_t dw2 = 0;
5549
5550 /* RELEASE_MEM - flush caches, send int */
5551 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5552
5553 if (writeback) {
5554 dw2 = EOP_TC_NC_ACTION_EN;
5555 } else {
5556 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN |
5557 EOP_TC_MD_ACTION_EN;
5558 }
5559 dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5560 EVENT_INDEX(5);
5561 if (exec)
5562 dw2 |= EOP_EXEC;
5563
5564 amdgpu_ring_write(ring, dw2);
5565 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5566
5567 /*
5568 * the address should be Qword aligned if 64bit write, Dword
5569 * aligned if only send 32bit data low (discard data high)
5570 */
5571 if (write64bit)
5572 BUG_ON(addr & 0x7);
5573 else
5574 BUG_ON(addr & 0x3);
5575 amdgpu_ring_write(ring, lower_32_bits(addr));
5576 amdgpu_ring_write(ring, upper_32_bits(addr));
5577 amdgpu_ring_write(ring, lower_32_bits(seq));
5578 amdgpu_ring_write(ring, upper_32_bits(seq));
5579 amdgpu_ring_write(ring, 0);
5580 }
5581
gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)5582 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5583 {
5584 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5585 uint32_t seq = ring->fence_drv.sync_seq;
5586 uint64_t addr = ring->fence_drv.gpu_addr;
5587
5588 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5589 lower_32_bits(addr), upper_32_bits(addr),
5590 seq, 0xffffffff, 4);
5591 }
5592
gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)5593 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5594 unsigned vmid, uint64_t pd_addr)
5595 {
5596 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5597
5598 /* compute doesn't have PFP */
5599 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5600 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5601 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5602 amdgpu_ring_write(ring, 0x0);
5603 }
5604 }
5605
gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5606 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5607 {
5608 return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */
5609 }
5610
gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5611 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5612 {
5613 u64 wptr;
5614
5615 /* XXX check if swapping is necessary on BE */
5616 if (ring->use_doorbell)
5617 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5618 else
5619 BUG();
5620 return wptr;
5621 }
5622
gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5623 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5624 {
5625 struct amdgpu_device *adev = ring->adev;
5626
5627 /* XXX check if swapping is necessary on BE */
5628 if (ring->use_doorbell) {
5629 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr);
5630 WDOORBELL64(ring->doorbell_index, ring->wptr);
5631 } else{
5632 BUG(); /* only DOORBELL method supported on gfx9 now */
5633 }
5634 }
5635
gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)5636 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5637 u64 seq, unsigned int flags)
5638 {
5639 struct amdgpu_device *adev = ring->adev;
5640
5641 /* we only allocate 32bit for each seq wb address */
5642 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5643
5644 /* write fence seq to the "addr" */
5645 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5646 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5647 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5648 amdgpu_ring_write(ring, lower_32_bits(addr));
5649 amdgpu_ring_write(ring, upper_32_bits(addr));
5650 amdgpu_ring_write(ring, lower_32_bits(seq));
5651
5652 if (flags & AMDGPU_FENCE_FLAG_INT) {
5653 /* set register to trigger INT */
5654 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5655 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5656 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5657 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5658 amdgpu_ring_write(ring, 0);
5659 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5660 }
5661 }
5662
gfx_v9_ring_emit_sb(struct amdgpu_ring * ring)5663 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5664 {
5665 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5666 amdgpu_ring_write(ring, 0);
5667 }
5668
gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring * ring,bool resume)5669 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
5670 {
5671 struct amdgpu_device *adev = ring->adev;
5672 struct v9_ce_ib_state ce_payload = {0};
5673 uint64_t offset, ce_payload_gpu_addr;
5674 void *ce_payload_cpu_addr;
5675 int cnt;
5676
5677 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5678
5679 offset = offsetof(struct v9_gfx_meta_data, ce_payload);
5680 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5681 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5682
5683 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5684 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5685 WRITE_DATA_DST_SEL(8) |
5686 WR_CONFIRM) |
5687 WRITE_DATA_CACHE_POLICY(0));
5688 amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr));
5689 amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr));
5690
5691 amdgpu_ring_ib_on_emit_ce(ring);
5692
5693 if (resume)
5694 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr,
5695 sizeof(ce_payload) >> 2);
5696 else
5697 amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
5698 sizeof(ce_payload) >> 2);
5699 }
5700
gfx_v9_0_ring_preempt_ib(struct amdgpu_ring * ring)5701 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring)
5702 {
5703 int i, r = 0;
5704 struct amdgpu_device *adev = ring->adev;
5705 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5706 struct amdgpu_ring *kiq_ring = &kiq->ring;
5707 unsigned long flags;
5708
5709 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5710 return -EINVAL;
5711
5712 spin_lock_irqsave(&kiq->ring_lock, flags);
5713
5714 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5715 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5716 return -ENOMEM;
5717 }
5718
5719 /* assert preemption condition */
5720 amdgpu_ring_set_preempt_cond_exec(ring, false);
5721
5722 ring->trail_seq += 1;
5723 amdgpu_ring_alloc(ring, 13);
5724 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
5725 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT);
5726
5727 /* assert IB preemption, emit the trailing fence */
5728 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5729 ring->trail_fence_gpu_addr,
5730 ring->trail_seq);
5731
5732 amdgpu_ring_commit(kiq_ring);
5733 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5734
5735 /* poll the trailing fence */
5736 for (i = 0; i < adev->usec_timeout; i++) {
5737 if (ring->trail_seq ==
5738 le32_to_cpu(*ring->trail_fence_cpu_addr))
5739 break;
5740 udelay(1);
5741 }
5742
5743 if (i >= adev->usec_timeout) {
5744 r = -EINVAL;
5745 DRM_WARN("ring %d timeout to preempt ib\n", ring->idx);
5746 }
5747
5748 /*reset the CP_VMID_PREEMPT after trailing fence*/
5749 amdgpu_ring_emit_wreg(ring,
5750 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT),
5751 0x0);
5752 amdgpu_ring_commit(ring);
5753
5754 /* deassert preemption condition */
5755 amdgpu_ring_set_preempt_cond_exec(ring, true);
5756 return r;
5757 }
5758
gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring * ring,bool resume,bool usegds)5759 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds)
5760 {
5761 struct amdgpu_device *adev = ring->adev;
5762 struct v9_de_ib_state de_payload = {0};
5763 uint64_t offset, gds_addr, de_payload_gpu_addr;
5764 void *de_payload_cpu_addr;
5765 int cnt;
5766
5767 offset = offsetof(struct v9_gfx_meta_data, de_payload);
5768 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5769 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5770
5771 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5772 AMDGPU_CSA_SIZE - adev->gds.gds_size,
5773 PAGE_SIZE);
5774
5775 if (usegds) {
5776 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5777 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5778 }
5779
5780 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5781 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5782 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5783 WRITE_DATA_DST_SEL(8) |
5784 WR_CONFIRM) |
5785 WRITE_DATA_CACHE_POLICY(0));
5786 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
5787 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
5788
5789 amdgpu_ring_ib_on_emit_de(ring);
5790 if (resume)
5791 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
5792 sizeof(de_payload) >> 2);
5793 else
5794 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
5795 sizeof(de_payload) >> 2);
5796 }
5797
gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)5798 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
5799 bool secure)
5800 {
5801 uint32_t v = secure ? FRAME_TMZ : 0;
5802
5803 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5804 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
5805 }
5806
gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)5807 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5808 {
5809 uint32_t dw2 = 0;
5810
5811 gfx_v9_0_ring_emit_ce_meta(ring,
5812 (!amdgpu_sriov_vf(ring->adev) &&
5813 flags & AMDGPU_IB_PREEMPTED) ? true : false);
5814
5815 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5816 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5817 /* set load_global_config & load_global_uconfig */
5818 dw2 |= 0x8001;
5819 /* set load_cs_sh_regs */
5820 dw2 |= 0x01000000;
5821 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5822 dw2 |= 0x10002;
5823
5824 /* set load_ce_ram if preamble presented */
5825 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5826 dw2 |= 0x10000000;
5827 } else {
5828 /* still load_ce_ram if this is the first time preamble presented
5829 * although there is no context switch happens.
5830 */
5831 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5832 dw2 |= 0x10000000;
5833 }
5834
5835 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5836 amdgpu_ring_write(ring, dw2);
5837 amdgpu_ring_write(ring, 0);
5838 }
5839
gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)5840 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5841 uint64_t addr)
5842 {
5843 unsigned ret;
5844 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5845 amdgpu_ring_write(ring, lower_32_bits(addr));
5846 amdgpu_ring_write(ring, upper_32_bits(addr));
5847 /* discard following DWs if *cond_exec_gpu_addr==0 */
5848 amdgpu_ring_write(ring, 0);
5849 ret = ring->wptr & ring->buf_mask;
5850 /* patch dummy value later */
5851 amdgpu_ring_write(ring, 0);
5852 return ret;
5853 }
5854
gfx_v9_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)5855 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
5856 uint32_t reg_val_offs)
5857 {
5858 struct amdgpu_device *adev = ring->adev;
5859
5860 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5861 amdgpu_ring_write(ring, 0 | /* src: register*/
5862 (5 << 8) | /* dst: memory */
5863 (1 << 20)); /* write confirm */
5864 amdgpu_ring_write(ring, reg);
5865 amdgpu_ring_write(ring, 0);
5866 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5867 reg_val_offs * 4));
5868 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5869 reg_val_offs * 4));
5870 }
5871
gfx_v9_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)5872 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5873 uint32_t val)
5874 {
5875 uint32_t cmd = 0;
5876
5877 switch (ring->funcs->type) {
5878 case AMDGPU_RING_TYPE_GFX:
5879 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5880 break;
5881 case AMDGPU_RING_TYPE_KIQ:
5882 cmd = (1 << 16); /* no inc addr */
5883 break;
5884 default:
5885 cmd = WR_CONFIRM;
5886 break;
5887 }
5888 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5889 amdgpu_ring_write(ring, cmd);
5890 amdgpu_ring_write(ring, reg);
5891 amdgpu_ring_write(ring, 0);
5892 amdgpu_ring_write(ring, val);
5893 }
5894
gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)5895 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5896 uint32_t val, uint32_t mask)
5897 {
5898 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5899 }
5900
gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)5901 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5902 uint32_t reg0, uint32_t reg1,
5903 uint32_t ref, uint32_t mask)
5904 {
5905 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5906 struct amdgpu_device *adev = ring->adev;
5907 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5908 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5909
5910 if (fw_version_ok)
5911 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5912 ref, mask, 0x20);
5913 else
5914 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5915 ref, mask);
5916 }
5917
gfx_v9_0_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)5918 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5919 {
5920 struct amdgpu_device *adev = ring->adev;
5921 uint32_t value = 0;
5922
5923 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5924 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5925 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5926 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5927 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5928 WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5929 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5930 }
5931
gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,enum amdgpu_interrupt_state state)5932 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5933 enum amdgpu_interrupt_state state)
5934 {
5935 switch (state) {
5936 case AMDGPU_IRQ_STATE_DISABLE:
5937 case AMDGPU_IRQ_STATE_ENABLE:
5938 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5939 TIME_STAMP_INT_ENABLE,
5940 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5941 break;
5942 default:
5943 break;
5944 }
5945 }
5946
gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)5947 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5948 int me, int pipe,
5949 enum amdgpu_interrupt_state state)
5950 {
5951 u32 mec_int_cntl, mec_int_cntl_reg;
5952
5953 /*
5954 * amdgpu controls only the first MEC. That's why this function only
5955 * handles the setting of interrupts for this specific MEC. All other
5956 * pipes' interrupts are set by amdkfd.
5957 */
5958
5959 if (me == 1) {
5960 switch (pipe) {
5961 case 0:
5962 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5963 break;
5964 case 1:
5965 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5966 break;
5967 case 2:
5968 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5969 break;
5970 case 3:
5971 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5972 break;
5973 default:
5974 DRM_DEBUG("invalid pipe %d\n", pipe);
5975 return;
5976 }
5977 } else {
5978 DRM_DEBUG("invalid me %d\n", me);
5979 return;
5980 }
5981
5982 switch (state) {
5983 case AMDGPU_IRQ_STATE_DISABLE:
5984 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg);
5985 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5986 TIME_STAMP_INT_ENABLE, 0);
5987 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5988 break;
5989 case AMDGPU_IRQ_STATE_ENABLE:
5990 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
5991 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5992 TIME_STAMP_INT_ENABLE, 1);
5993 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
5994 break;
5995 default:
5996 break;
5997 }
5998 }
5999
gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device * adev,int me,int pipe)6000 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev,
6001 int me, int pipe)
6002 {
6003 /*
6004 * amdgpu controls only the first MEC. That's why this function only
6005 * handles the setting of interrupts for this specific MEC. All other
6006 * pipes' interrupts are set by amdkfd.
6007 */
6008 if (me != 1)
6009 return 0;
6010
6011 switch (pipe) {
6012 case 0:
6013 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
6014 case 1:
6015 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
6016 case 2:
6017 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
6018 case 3:
6019 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
6020 default:
6021 return 0;
6022 }
6023 }
6024
gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6025 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6026 struct amdgpu_irq_src *source,
6027 unsigned type,
6028 enum amdgpu_interrupt_state state)
6029 {
6030 u32 cp_int_cntl_reg, cp_int_cntl;
6031 int i, j;
6032
6033 switch (state) {
6034 case AMDGPU_IRQ_STATE_DISABLE:
6035 case AMDGPU_IRQ_STATE_ENABLE:
6036 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6037 PRIV_REG_INT_ENABLE,
6038 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6039 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6040 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6041 /* MECs start at 1 */
6042 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6043
6044 if (cp_int_cntl_reg) {
6045 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6046 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6047 PRIV_REG_INT_ENABLE,
6048 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6049 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6050 }
6051 }
6052 }
6053 break;
6054 default:
6055 break;
6056 }
6057
6058 return 0;
6059 }
6060
gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6061 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6062 struct amdgpu_irq_src *source,
6063 unsigned type,
6064 enum amdgpu_interrupt_state state)
6065 {
6066 u32 cp_int_cntl_reg, cp_int_cntl;
6067 int i, j;
6068
6069 switch (state) {
6070 case AMDGPU_IRQ_STATE_DISABLE:
6071 case AMDGPU_IRQ_STATE_ENABLE:
6072 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6073 OPCODE_ERROR_INT_ENABLE,
6074 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6075 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6076 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6077 /* MECs start at 1 */
6078 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j);
6079
6080 if (cp_int_cntl_reg) {
6081 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6082 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6083 OPCODE_ERROR_INT_ENABLE,
6084 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6085 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6086 }
6087 }
6088 }
6089 break;
6090 default:
6091 break;
6092 }
6093
6094 return 0;
6095 }
6096
gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6097 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6098 struct amdgpu_irq_src *source,
6099 unsigned type,
6100 enum amdgpu_interrupt_state state)
6101 {
6102 switch (state) {
6103 case AMDGPU_IRQ_STATE_DISABLE:
6104 case AMDGPU_IRQ_STATE_ENABLE:
6105 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6106 PRIV_INSTR_INT_ENABLE,
6107 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6108 break;
6109 default:
6110 break;
6111 }
6112
6113 return 0;
6114 }
6115
6116 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
6117 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6118 CP_ECC_ERROR_INT_ENABLE, 1)
6119
6120 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
6121 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
6122 CP_ECC_ERROR_INT_ENABLE, 0)
6123
gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6124 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
6125 struct amdgpu_irq_src *source,
6126 unsigned type,
6127 enum amdgpu_interrupt_state state)
6128 {
6129 switch (state) {
6130 case AMDGPU_IRQ_STATE_DISABLE:
6131 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6132 CP_ECC_ERROR_INT_ENABLE, 0);
6133 DISABLE_ECC_ON_ME_PIPE(1, 0);
6134 DISABLE_ECC_ON_ME_PIPE(1, 1);
6135 DISABLE_ECC_ON_ME_PIPE(1, 2);
6136 DISABLE_ECC_ON_ME_PIPE(1, 3);
6137 break;
6138
6139 case AMDGPU_IRQ_STATE_ENABLE:
6140 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
6141 CP_ECC_ERROR_INT_ENABLE, 1);
6142 ENABLE_ECC_ON_ME_PIPE(1, 0);
6143 ENABLE_ECC_ON_ME_PIPE(1, 1);
6144 ENABLE_ECC_ON_ME_PIPE(1, 2);
6145 ENABLE_ECC_ON_ME_PIPE(1, 3);
6146 break;
6147 default:
6148 break;
6149 }
6150
6151 return 0;
6152 }
6153
6154
gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6155 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6156 struct amdgpu_irq_src *src,
6157 unsigned type,
6158 enum amdgpu_interrupt_state state)
6159 {
6160 switch (type) {
6161 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6162 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
6163 break;
6164 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6165 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6166 break;
6167 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6168 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6169 break;
6170 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6171 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6172 break;
6173 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6174 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6175 break;
6176 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6177 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6178 break;
6179 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6180 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6181 break;
6182 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6183 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6184 break;
6185 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6186 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6187 break;
6188 default:
6189 break;
6190 }
6191 return 0;
6192 }
6193
gfx_v9_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6194 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
6195 struct amdgpu_irq_src *source,
6196 struct amdgpu_iv_entry *entry)
6197 {
6198 int i;
6199 u8 me_id, pipe_id, queue_id;
6200 struct amdgpu_ring *ring;
6201
6202 DRM_DEBUG("IH: CP EOP\n");
6203 me_id = (entry->ring_id & 0x0c) >> 2;
6204 pipe_id = (entry->ring_id & 0x03) >> 0;
6205 queue_id = (entry->ring_id & 0x70) >> 4;
6206
6207 switch (me_id) {
6208 case 0:
6209 if (adev->gfx.num_gfx_rings) {
6210 if (!adev->gfx.mcbp) {
6211 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6212 } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) {
6213 /* Fence signals are handled on the software rings*/
6214 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
6215 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]);
6216 }
6217 }
6218 break;
6219 case 1:
6220 case 2:
6221 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6222 ring = &adev->gfx.compute_ring[i];
6223 /* Per-queue interrupt is supported for MEC starting from VI.
6224 * The interrupt can only be enabled/disabled per pipe instead of per queue.
6225 */
6226 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6227 amdgpu_fence_process(ring);
6228 }
6229 break;
6230 }
6231 return 0;
6232 }
6233
gfx_v9_0_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6234 static void gfx_v9_0_fault(struct amdgpu_device *adev,
6235 struct amdgpu_iv_entry *entry)
6236 {
6237 u8 me_id, pipe_id, queue_id;
6238 struct amdgpu_ring *ring;
6239 int i;
6240
6241 me_id = (entry->ring_id & 0x0c) >> 2;
6242 pipe_id = (entry->ring_id & 0x03) >> 0;
6243 queue_id = (entry->ring_id & 0x70) >> 4;
6244
6245 switch (me_id) {
6246 case 0:
6247 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6248 break;
6249 case 1:
6250 case 2:
6251 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6252 ring = &adev->gfx.compute_ring[i];
6253 if (ring->me == me_id && ring->pipe == pipe_id &&
6254 ring->queue == queue_id)
6255 drm_sched_fault(&ring->sched);
6256 }
6257 break;
6258 }
6259 }
6260
gfx_v9_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6261 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
6262 struct amdgpu_irq_src *source,
6263 struct amdgpu_iv_entry *entry)
6264 {
6265 DRM_ERROR("Illegal register access in command stream\n");
6266 gfx_v9_0_fault(adev, entry);
6267 return 0;
6268 }
6269
gfx_v9_0_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6270 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev,
6271 struct amdgpu_irq_src *source,
6272 struct amdgpu_iv_entry *entry)
6273 {
6274 DRM_ERROR("Illegal opcode in command stream\n");
6275 gfx_v9_0_fault(adev, entry);
6276 return 0;
6277 }
6278
gfx_v9_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6279 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
6280 struct amdgpu_irq_src *source,
6281 struct amdgpu_iv_entry *entry)
6282 {
6283 DRM_ERROR("Illegal instruction in command stream\n");
6284 gfx_v9_0_fault(adev, entry);
6285 return 0;
6286 }
6287
6288
6289 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
6290 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
6291 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
6292 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
6293 },
6294 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
6295 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
6296 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
6297 },
6298 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6299 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
6300 0, 0
6301 },
6302 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
6303 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
6304 0, 0
6305 },
6306 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
6307 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
6308 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
6309 },
6310 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6311 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
6312 0, 0
6313 },
6314 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
6315 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
6316 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
6317 },
6318 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
6319 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
6320 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
6321 },
6322 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
6323 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
6324 0, 0
6325 },
6326 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
6327 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
6328 0, 0
6329 },
6330 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
6331 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
6332 0, 0
6333 },
6334 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6335 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
6336 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
6337 },
6338 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
6339 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
6340 0, 0
6341 },
6342 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6343 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
6344 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
6345 },
6346 { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
6347 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6348 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
6349 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
6350 },
6351 { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
6352 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
6353 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
6354 0, 0
6355 },
6356 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
6357 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6358 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
6359 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
6360 },
6361 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
6362 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6363 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
6364 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
6365 },
6366 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
6367 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6368 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
6369 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
6370 },
6371 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
6372 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
6373 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
6374 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
6375 },
6376 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
6377 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
6378 0, 0
6379 },
6380 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6381 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
6382 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
6383 },
6384 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6385 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
6386 0, 0
6387 },
6388 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6389 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
6390 0, 0
6391 },
6392 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6393 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
6394 0, 0
6395 },
6396 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
6397 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
6398 0, 0
6399 },
6400 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6401 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
6402 0, 0
6403 },
6404 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
6405 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
6406 0, 0
6407 },
6408 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6409 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
6410 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
6411 },
6412 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6413 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
6414 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
6415 },
6416 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6417 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
6418 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
6419 },
6420 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6421 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
6422 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
6423 },
6424 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6425 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
6426 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
6427 },
6428 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6429 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
6430 0, 0
6431 },
6432 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6433 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
6434 0, 0
6435 },
6436 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6437 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
6438 0, 0
6439 },
6440 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6441 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
6442 0, 0
6443 },
6444 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6445 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
6446 0, 0
6447 },
6448 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
6449 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
6450 0, 0
6451 },
6452 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6453 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
6454 0, 0
6455 },
6456 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6457 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6458 0, 0
6459 },
6460 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6461 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6462 0, 0
6463 },
6464 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6465 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6466 0, 0
6467 },
6468 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6469 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6470 0, 0
6471 },
6472 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6473 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6474 0, 0
6475 },
6476 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6477 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6478 0, 0
6479 },
6480 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6481 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6482 0, 0
6483 },
6484 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6485 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6486 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6487 },
6488 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6489 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6490 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6491 },
6492 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6493 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6494 0, 0
6495 },
6496 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6497 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6498 0, 0
6499 },
6500 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6501 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6502 0, 0
6503 },
6504 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6505 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6506 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6507 },
6508 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6509 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6510 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6511 },
6512 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6513 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6514 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6515 },
6516 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6517 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6518 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6519 },
6520 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6521 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6522 0, 0
6523 },
6524 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6525 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6526 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6527 },
6528 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6529 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6530 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6531 },
6532 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6533 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6534 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6535 },
6536 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6537 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6538 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6539 },
6540 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6541 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6542 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6543 },
6544 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6545 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6546 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6547 },
6548 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6549 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6550 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6551 },
6552 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6553 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6554 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6555 },
6556 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6557 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6558 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6559 },
6560 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6561 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6562 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6563 },
6564 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6565 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6566 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6567 },
6568 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6569 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6570 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6571 },
6572 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6573 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6574 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6575 },
6576 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6577 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6578 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6579 },
6580 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6581 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6582 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6583 },
6584 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6585 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6586 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6587 },
6588 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6589 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6590 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6591 },
6592 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6593 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6594 0, 0
6595 },
6596 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6597 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6598 0, 0
6599 },
6600 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6601 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6602 0, 0
6603 },
6604 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6605 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6606 0, 0
6607 },
6608 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6609 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6610 0, 0
6611 },
6612 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6613 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6614 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6615 },
6616 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6617 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6618 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6619 },
6620 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6621 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6622 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6623 },
6624 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6625 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6626 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6627 },
6628 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6629 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6630 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6631 },
6632 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6633 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6634 0, 0
6635 },
6636 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6637 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6638 0, 0
6639 },
6640 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6641 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6642 0, 0
6643 },
6644 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6645 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6646 0, 0
6647 },
6648 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6649 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6650 0, 0
6651 },
6652 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6653 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6654 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6655 },
6656 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6657 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6658 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6659 },
6660 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6661 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6662 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6663 },
6664 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6665 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6666 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6667 },
6668 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6669 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6670 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6671 },
6672 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6673 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6674 0, 0
6675 },
6676 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6677 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6678 0, 0
6679 },
6680 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6681 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6682 0, 0
6683 },
6684 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6685 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6686 0, 0
6687 },
6688 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6689 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6690 0, 0
6691 },
6692 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6693 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6694 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6695 },
6696 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6697 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6698 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6699 },
6700 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6701 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6702 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6703 },
6704 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6705 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6706 0, 0
6707 },
6708 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6709 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6710 0, 0
6711 },
6712 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6713 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6714 0, 0
6715 },
6716 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6717 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6718 0, 0
6719 },
6720 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6721 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6722 0, 0
6723 },
6724 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6725 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6726 0, 0
6727 }
6728 };
6729
gfx_v9_0_ras_error_inject(struct amdgpu_device * adev,void * inject_if,uint32_t instance_mask)6730 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6731 void *inject_if, uint32_t instance_mask)
6732 {
6733 struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6734 int ret;
6735 struct ta_ras_trigger_error_input block_info = { 0 };
6736
6737 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6738 return -EINVAL;
6739
6740 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6741 return -EINVAL;
6742
6743 if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6744 return -EPERM;
6745
6746 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6747 info->head.type)) {
6748 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6749 ras_gfx_subblocks[info->head.sub_block_index].name,
6750 info->head.type);
6751 return -EPERM;
6752 }
6753
6754 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6755 info->head.type)) {
6756 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6757 ras_gfx_subblocks[info->head.sub_block_index].name,
6758 info->head.type);
6759 return -EPERM;
6760 }
6761
6762 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6763 block_info.sub_block_index =
6764 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6765 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6766 block_info.address = info->address;
6767 block_info.value = info->value;
6768
6769 mutex_lock(&adev->grbm_idx_mutex);
6770 ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask);
6771 mutex_unlock(&adev->grbm_idx_mutex);
6772
6773 return ret;
6774 }
6775
6776 static const char * const vml2_mems[] = {
6777 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6778 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6779 "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6780 "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6781 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6782 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6783 "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6784 "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6785 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6786 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6787 "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6788 "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6789 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6790 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6791 "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6792 "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6793 };
6794
6795 static const char * const vml2_walker_mems[] = {
6796 "UTC_VML2_CACHE_PDE0_MEM0",
6797 "UTC_VML2_CACHE_PDE0_MEM1",
6798 "UTC_VML2_CACHE_PDE1_MEM0",
6799 "UTC_VML2_CACHE_PDE1_MEM1",
6800 "UTC_VML2_CACHE_PDE2_MEM0",
6801 "UTC_VML2_CACHE_PDE2_MEM1",
6802 "UTC_VML2_RDIF_LOG_FIFO",
6803 };
6804
6805 static const char * const atc_l2_cache_2m_mems[] = {
6806 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6807 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6808 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6809 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6810 };
6811
6812 static const char *atc_l2_cache_4k_mems[] = {
6813 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6814 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6815 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6816 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6817 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6818 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6819 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6820 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6821 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6822 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6823 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6824 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6825 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6826 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6827 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6828 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6829 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6830 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6831 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6832 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6833 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6834 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6835 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6836 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6837 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6838 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6839 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6840 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6841 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6842 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6843 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6844 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6845 };
6846
gfx_v9_0_query_utc_edc_status(struct amdgpu_device * adev,struct ras_err_data * err_data)6847 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6848 struct ras_err_data *err_data)
6849 {
6850 uint32_t i, data;
6851 uint32_t sec_count, ded_count;
6852
6853 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6854 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6855 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6856 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6857 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6858 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6859 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6860 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6861
6862 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6863 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6864 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6865
6866 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6867 if (sec_count) {
6868 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6869 "SEC %d\n", i, vml2_mems[i], sec_count);
6870 err_data->ce_count += sec_count;
6871 }
6872
6873 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6874 if (ded_count) {
6875 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6876 "DED %d\n", i, vml2_mems[i], ded_count);
6877 err_data->ue_count += ded_count;
6878 }
6879 }
6880
6881 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6882 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6883 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6884
6885 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6886 SEC_COUNT);
6887 if (sec_count) {
6888 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6889 "SEC %d\n", i, vml2_walker_mems[i], sec_count);
6890 err_data->ce_count += sec_count;
6891 }
6892
6893 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6894 DED_COUNT);
6895 if (ded_count) {
6896 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6897 "DED %d\n", i, vml2_walker_mems[i], ded_count);
6898 err_data->ue_count += ded_count;
6899 }
6900 }
6901
6902 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6903 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6904 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6905
6906 sec_count = (data & 0x00006000L) >> 0xd;
6907 if (sec_count) {
6908 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6909 "SEC %d\n", i, atc_l2_cache_2m_mems[i],
6910 sec_count);
6911 err_data->ce_count += sec_count;
6912 }
6913 }
6914
6915 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6916 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6917 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6918
6919 sec_count = (data & 0x00006000L) >> 0xd;
6920 if (sec_count) {
6921 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6922 "SEC %d\n", i, atc_l2_cache_4k_mems[i],
6923 sec_count);
6924 err_data->ce_count += sec_count;
6925 }
6926
6927 ded_count = (data & 0x00018000L) >> 0xf;
6928 if (ded_count) {
6929 dev_info(adev->dev, "Instance[%d]: SubBlock %s, "
6930 "DED %d\n", i, atc_l2_cache_4k_mems[i],
6931 ded_count);
6932 err_data->ue_count += ded_count;
6933 }
6934 }
6935
6936 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6937 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6938 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6939 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6940
6941 return 0;
6942 }
6943
gfx_v9_0_ras_error_count(struct amdgpu_device * adev,const struct soc15_reg_entry * reg,uint32_t se_id,uint32_t inst_id,uint32_t value,uint32_t * sec_count,uint32_t * ded_count)6944 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev,
6945 const struct soc15_reg_entry *reg,
6946 uint32_t se_id, uint32_t inst_id, uint32_t value,
6947 uint32_t *sec_count, uint32_t *ded_count)
6948 {
6949 uint32_t i;
6950 uint32_t sec_cnt, ded_cnt;
6951
6952 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6953 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6954 gfx_v9_0_ras_fields[i].seg != reg->seg ||
6955 gfx_v9_0_ras_fields[i].inst != reg->inst)
6956 continue;
6957
6958 sec_cnt = (value &
6959 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6960 gfx_v9_0_ras_fields[i].sec_count_shift;
6961 if (sec_cnt) {
6962 dev_info(adev->dev, "GFX SubBlock %s, "
6963 "Instance[%d][%d], SEC %d\n",
6964 gfx_v9_0_ras_fields[i].name,
6965 se_id, inst_id,
6966 sec_cnt);
6967 *sec_count += sec_cnt;
6968 }
6969
6970 ded_cnt = (value &
6971 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6972 gfx_v9_0_ras_fields[i].ded_count_shift;
6973 if (ded_cnt) {
6974 dev_info(adev->dev, "GFX SubBlock %s, "
6975 "Instance[%d][%d], DED %d\n",
6976 gfx_v9_0_ras_fields[i].name,
6977 se_id, inst_id,
6978 ded_cnt);
6979 *ded_count += ded_cnt;
6980 }
6981 }
6982
6983 return 0;
6984 }
6985
gfx_v9_0_reset_ras_error_count(struct amdgpu_device * adev)6986 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6987 {
6988 int i, j, k;
6989
6990 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6991 return;
6992
6993 /* read back registers to clear the counters */
6994 mutex_lock(&adev->grbm_idx_mutex);
6995 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6996 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6997 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6998 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0);
6999 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7000 }
7001 }
7002 }
7003 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
7004 mutex_unlock(&adev->grbm_idx_mutex);
7005
7006 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7007 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
7008 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7009 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
7010 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7011 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
7012 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7013 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
7014
7015 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
7016 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
7017 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
7018 }
7019
7020 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
7021 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
7022 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
7023 }
7024
7025 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
7026 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
7027 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
7028 }
7029
7030 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
7031 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
7032 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
7033 }
7034
7035 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
7036 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
7037 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
7038 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
7039 }
7040
gfx_v9_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)7041 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
7042 void *ras_error_status)
7043 {
7044 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
7045 uint32_t sec_count = 0, ded_count = 0;
7046 uint32_t i, j, k;
7047 uint32_t reg_value;
7048
7049 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
7050 return;
7051
7052 err_data->ue_count = 0;
7053 err_data->ce_count = 0;
7054
7055 mutex_lock(&adev->grbm_idx_mutex);
7056
7057 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
7058 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
7059 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
7060 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0);
7061 reg_value =
7062 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
7063 if (reg_value)
7064 gfx_v9_0_ras_error_count(adev,
7065 &gfx_v9_0_edc_counter_regs[i],
7066 j, k, reg_value,
7067 &sec_count, &ded_count);
7068 }
7069 }
7070 }
7071
7072 err_data->ce_count += sec_count;
7073 err_data->ue_count += ded_count;
7074
7075 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7076 mutex_unlock(&adev->grbm_idx_mutex);
7077
7078 gfx_v9_0_query_utc_edc_status(adev, err_data);
7079 }
7080
gfx_v9_0_emit_mem_sync(struct amdgpu_ring * ring)7081 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring)
7082 {
7083 const unsigned int cp_coher_cntl =
7084 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
7085 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
7086 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
7087 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
7088 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
7089
7090 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
7091 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
7092 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
7093 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
7094 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
7095 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
7096 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
7097 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
7098 }
7099
gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring * ring,uint32_t pipe,bool enable)7100 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
7101 uint32_t pipe, bool enable)
7102 {
7103 struct amdgpu_device *adev = ring->adev;
7104 uint32_t val;
7105 uint32_t wcl_cs_reg;
7106
7107 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
7108 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT;
7109
7110 switch (pipe) {
7111 case 0:
7112 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0);
7113 break;
7114 case 1:
7115 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1);
7116 break;
7117 case 2:
7118 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2);
7119 break;
7120 case 3:
7121 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3);
7122 break;
7123 default:
7124 DRM_DEBUG("invalid pipe %d\n", pipe);
7125 return;
7126 }
7127
7128 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
7129
7130 }
gfx_v9_0_emit_wave_limit(struct amdgpu_ring * ring,bool enable)7131 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
7132 {
7133 struct amdgpu_device *adev = ring->adev;
7134 uint32_t val;
7135 int i;
7136
7137
7138 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
7139 * number of gfx waves. Setting 5 bit will make sure gfx only gets
7140 * around 25% of gpu resources.
7141 */
7142 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
7143 amdgpu_ring_emit_wreg(ring,
7144 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX),
7145 val);
7146
7147 /* Restrict waves for normal/low priority compute queues as well
7148 * to get best QoS for high priority compute jobs.
7149 *
7150 * amdgpu controls only 1st ME(0-3 CS pipes).
7151 */
7152 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
7153 if (i != ring->pipe)
7154 gfx_v9_0_emit_wave_limit_cs(ring, i, enable);
7155
7156 }
7157 }
7158
gfx_v9_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)7159 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
7160 {
7161 /* Header itself is a NOP packet */
7162 if (num_nop == 1) {
7163 amdgpu_ring_write(ring, ring->funcs->nop);
7164 return;
7165 }
7166
7167 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
7168 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
7169
7170 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
7171 amdgpu_ring_insert_nop(ring, num_nop - 1);
7172 }
7173
gfx_v9_0_reset_kgq(struct amdgpu_ring * ring,unsigned int vmid)7174 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)
7175 {
7176 struct amdgpu_device *adev = ring->adev;
7177 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7178 struct amdgpu_ring *kiq_ring = &kiq->ring;
7179 unsigned long flags;
7180 u32 tmp;
7181 int r;
7182
7183 if (amdgpu_sriov_vf(adev))
7184 return -EINVAL;
7185
7186 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7187 return -EINVAL;
7188
7189 spin_lock_irqsave(&kiq->ring_lock, flags);
7190
7191 if (amdgpu_ring_alloc(kiq_ring, 5)) {
7192 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7193 return -ENOMEM;
7194 }
7195
7196 tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid);
7197 gfx_v9_0_ring_emit_wreg(kiq_ring,
7198 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp);
7199 amdgpu_ring_commit(kiq_ring);
7200
7201 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7202
7203 r = amdgpu_ring_test_ring(kiq_ring);
7204 if (r)
7205 return r;
7206
7207 if (amdgpu_ring_alloc(ring, 7 + 7 + 5))
7208 return -ENOMEM;
7209 gfx_v9_0_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
7210 ring->fence_drv.sync_seq, AMDGPU_FENCE_FLAG_EXEC);
7211 gfx_v9_0_ring_emit_reg_wait(ring,
7212 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0, 0xffff);
7213 gfx_v9_0_ring_emit_wreg(ring,
7214 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0);
7215
7216 return amdgpu_ring_test_ring(ring);
7217 }
7218
gfx_v9_0_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid)7219 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring,
7220 unsigned int vmid)
7221 {
7222 struct amdgpu_device *adev = ring->adev;
7223 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
7224 struct amdgpu_ring *kiq_ring = &kiq->ring;
7225 unsigned long flags;
7226 int i, r;
7227
7228 if (amdgpu_sriov_vf(adev))
7229 return -EINVAL;
7230
7231 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
7232 return -EINVAL;
7233
7234 spin_lock_irqsave(&kiq->ring_lock, flags);
7235
7236 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
7237 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7238 return -ENOMEM;
7239 }
7240
7241 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
7242 0, 0);
7243 amdgpu_ring_commit(kiq_ring);
7244
7245 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7246
7247 r = amdgpu_ring_test_ring(kiq_ring);
7248 if (r)
7249 return r;
7250
7251 /* make sure dequeue is complete*/
7252 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
7253 mutex_lock(&adev->srbm_mutex);
7254 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0);
7255 for (i = 0; i < adev->usec_timeout; i++) {
7256 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
7257 break;
7258 udelay(1);
7259 }
7260 if (i >= adev->usec_timeout)
7261 r = -ETIMEDOUT;
7262 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7263 mutex_unlock(&adev->srbm_mutex);
7264 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
7265 if (r) {
7266 dev_err(adev->dev, "fail to wait on hqd deactive\n");
7267 return r;
7268 }
7269
7270 r = gfx_v9_0_kcq_init_queue(ring, true);
7271 if (r) {
7272 dev_err(adev->dev, "fail to init kcq\n");
7273 return r;
7274 }
7275 spin_lock_irqsave(&kiq->ring_lock, flags);
7276 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
7277 if (r) {
7278 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7279 return -ENOMEM;
7280 }
7281 kiq->pmf->kiq_map_queues(kiq_ring, ring);
7282 amdgpu_ring_commit(kiq_ring);
7283 spin_unlock_irqrestore(&kiq->ring_lock, flags);
7284 r = amdgpu_ring_test_ring(kiq_ring);
7285 if (r) {
7286 DRM_ERROR("fail to remap queue\n");
7287 return r;
7288 }
7289 return amdgpu_ring_test_ring(ring);
7290 }
7291
gfx_v9_ip_print(struct amdgpu_ip_block * ip_block,struct drm_printer * p)7292 static void gfx_v9_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
7293 {
7294 struct amdgpu_device *adev = ip_block->adev;
7295 uint32_t i, j, k, reg, index = 0;
7296 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7297
7298 if (!adev->gfx.ip_dump_core)
7299 return;
7300
7301 for (i = 0; i < reg_count; i++)
7302 drm_printf(p, "%-50s \t 0x%08x\n",
7303 gc_reg_list_9[i].reg_name,
7304 adev->gfx.ip_dump_core[i]);
7305
7306 /* print compute queue registers for all instances */
7307 if (!adev->gfx.ip_dump_compute_queues)
7308 return;
7309
7310 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7311 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
7312 adev->gfx.mec.num_mec,
7313 adev->gfx.mec.num_pipe_per_mec,
7314 adev->gfx.mec.num_queue_per_pipe);
7315
7316 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7317 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7318 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7319 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
7320 for (reg = 0; reg < reg_count; reg++) {
7321 if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP)
7322 drm_printf(p, "%-50s \t 0x%08x\n",
7323 "mmCP_MEC_ME2_HEADER_DUMP",
7324 adev->gfx.ip_dump_compute_queues[index + reg]);
7325 else
7326 drm_printf(p, "%-50s \t 0x%08x\n",
7327 gc_cp_reg_list_9[reg].reg_name,
7328 adev->gfx.ip_dump_compute_queues[index + reg]);
7329 }
7330 index += reg_count;
7331 }
7332 }
7333 }
7334
7335 }
7336
gfx_v9_ip_dump(struct amdgpu_ip_block * ip_block)7337 static void gfx_v9_ip_dump(struct amdgpu_ip_block *ip_block)
7338 {
7339 struct amdgpu_device *adev = ip_block->adev;
7340 uint32_t i, j, k, reg, index = 0;
7341 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9);
7342
7343 if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings)
7344 return;
7345
7346 amdgpu_gfx_off_ctrl(adev, false);
7347 for (i = 0; i < reg_count; i++)
7348 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i]));
7349 amdgpu_gfx_off_ctrl(adev, true);
7350
7351 /* dump compute queue registers for all instances */
7352 if (!adev->gfx.ip_dump_compute_queues)
7353 return;
7354
7355 reg_count = ARRAY_SIZE(gc_cp_reg_list_9);
7356 amdgpu_gfx_off_ctrl(adev, false);
7357 mutex_lock(&adev->srbm_mutex);
7358 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7359 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7360 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7361 /* ME0 is for GFX so start from 1 for CP */
7362 soc15_grbm_select(adev, 1 + i, j, k, 0, 0);
7363
7364 for (reg = 0; reg < reg_count; reg++) {
7365 if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP)
7366 adev->gfx.ip_dump_compute_queues[index + reg] =
7367 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME2_HEADER_DUMP));
7368 else
7369 adev->gfx.ip_dump_compute_queues[index + reg] =
7370 RREG32(SOC15_REG_ENTRY_OFFSET(
7371 gc_cp_reg_list_9[reg]));
7372 }
7373 index += reg_count;
7374 }
7375 }
7376 }
7377 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
7378 mutex_unlock(&adev->srbm_mutex);
7379 amdgpu_gfx_off_ctrl(adev, true);
7380
7381 }
7382
gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring * ring)7383 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7384 {
7385 struct amdgpu_device *adev = ring->adev;
7386
7387 /* Emit the cleaner shader */
7388 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
7389 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7390 else
7391 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER_9_0, 0));
7392
7393 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */
7394 }
7395
gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring * ring)7396 static void gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring *ring)
7397 {
7398 struct amdgpu_device *adev = ring->adev;
7399 struct amdgpu_ip_block *gfx_block =
7400 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7401
7402 amdgpu_gfx_enforce_isolation_ring_begin_use(ring);
7403
7404 /* Raven and PCO APUs seem to have stability issues
7405 * with compute and gfxoff and gfx pg. Disable gfx pg during
7406 * submission and allow again afterwards.
7407 */
7408 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7409 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_UNGATE);
7410 }
7411
gfx_v9_0_ring_end_use_compute(struct amdgpu_ring * ring)7412 static void gfx_v9_0_ring_end_use_compute(struct amdgpu_ring *ring)
7413 {
7414 struct amdgpu_device *adev = ring->adev;
7415 struct amdgpu_ip_block *gfx_block =
7416 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
7417
7418 /* Raven and PCO APUs seem to have stability issues
7419 * with compute and gfxoff and gfx pg. Disable gfx pg during
7420 * submission and allow again afterwards.
7421 */
7422 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0))
7423 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_GATE);
7424
7425 amdgpu_gfx_enforce_isolation_ring_end_use(ring);
7426 }
7427
7428 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
7429 .name = "gfx_v9_0",
7430 .early_init = gfx_v9_0_early_init,
7431 .late_init = gfx_v9_0_late_init,
7432 .sw_init = gfx_v9_0_sw_init,
7433 .sw_fini = gfx_v9_0_sw_fini,
7434 .hw_init = gfx_v9_0_hw_init,
7435 .hw_fini = gfx_v9_0_hw_fini,
7436 .suspend = gfx_v9_0_suspend,
7437 .resume = gfx_v9_0_resume,
7438 .is_idle = gfx_v9_0_is_idle,
7439 .wait_for_idle = gfx_v9_0_wait_for_idle,
7440 .soft_reset = gfx_v9_0_soft_reset,
7441 .set_clockgating_state = gfx_v9_0_set_clockgating_state,
7442 .set_powergating_state = gfx_v9_0_set_powergating_state,
7443 .get_clockgating_state = gfx_v9_0_get_clockgating_state,
7444 .dump_ip_state = gfx_v9_ip_dump,
7445 .print_ip_state = gfx_v9_ip_print,
7446 };
7447
7448 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
7449 .type = AMDGPU_RING_TYPE_GFX,
7450 .align_mask = 0xff,
7451 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7452 .support_64bit_ptrs = true,
7453 .secure_submission_supported = true,
7454 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
7455 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
7456 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
7457 .emit_frame_size = /* totally 242 maximum if 16 IBs */
7458 5 + /* COND_EXEC */
7459 7 + /* PIPELINE_SYNC */
7460 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7461 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7462 2 + /* VM_FLUSH */
7463 8 + /* FENCE for VM_FLUSH */
7464 20 + /* GDS switch */
7465 4 + /* double SWITCH_BUFFER,
7466 the first COND_EXEC jump to the place just
7467 prior to this double SWITCH_BUFFER */
7468 5 + /* COND_EXEC */
7469 7 + /* HDP_flush */
7470 4 + /* VGT_flush */
7471 14 + /* CE_META */
7472 31 + /* DE_META */
7473 3 + /* CNTX_CTRL */
7474 5 + /* HDP_INVL */
7475 8 + 8 + /* FENCE x2 */
7476 2 + /* SWITCH_BUFFER */
7477 7 + /* gfx_v9_0_emit_mem_sync */
7478 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7479 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7480 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7481 .emit_fence = gfx_v9_0_ring_emit_fence,
7482 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7483 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7484 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7485 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7486 .test_ring = gfx_v9_0_ring_test_ring,
7487 .insert_nop = gfx_v9_ring_insert_nop,
7488 .pad_ib = amdgpu_ring_generic_pad_ib,
7489 .emit_switch_buffer = gfx_v9_ring_emit_sb,
7490 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7491 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7492 .preempt_ib = gfx_v9_0_ring_preempt_ib,
7493 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7494 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7495 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7496 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7497 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7498 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7499 .reset = gfx_v9_0_reset_kgq,
7500 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7501 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7502 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7503 };
7504
7505 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = {
7506 .type = AMDGPU_RING_TYPE_GFX,
7507 .align_mask = 0xff,
7508 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7509 .support_64bit_ptrs = true,
7510 .secure_submission_supported = true,
7511 .get_rptr = amdgpu_sw_ring_get_rptr_gfx,
7512 .get_wptr = amdgpu_sw_ring_get_wptr_gfx,
7513 .set_wptr = amdgpu_sw_ring_set_wptr_gfx,
7514 .emit_frame_size = /* totally 242 maximum if 16 IBs */
7515 5 + /* COND_EXEC */
7516 7 + /* PIPELINE_SYNC */
7517 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7518 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7519 2 + /* VM_FLUSH */
7520 8 + /* FENCE for VM_FLUSH */
7521 20 + /* GDS switch */
7522 4 + /* double SWITCH_BUFFER,
7523 * the first COND_EXEC jump to the place just
7524 * prior to this double SWITCH_BUFFER
7525 */
7526 5 + /* COND_EXEC */
7527 7 + /* HDP_flush */
7528 4 + /* VGT_flush */
7529 14 + /* CE_META */
7530 31 + /* DE_META */
7531 3 + /* CNTX_CTRL */
7532 5 + /* HDP_INVL */
7533 8 + 8 + /* FENCE x2 */
7534 2 + /* SWITCH_BUFFER */
7535 7 + /* gfx_v9_0_emit_mem_sync */
7536 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7537 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
7538 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
7539 .emit_fence = gfx_v9_0_ring_emit_fence,
7540 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7541 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7542 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7543 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7544 .test_ring = gfx_v9_0_ring_test_ring,
7545 .test_ib = gfx_v9_0_ring_test_ib,
7546 .insert_nop = gfx_v9_ring_insert_nop,
7547 .pad_ib = amdgpu_ring_generic_pad_ib,
7548 .emit_switch_buffer = gfx_v9_ring_emit_sb,
7549 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
7550 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
7551 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl,
7552 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7553 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7554 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7555 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7556 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7557 .patch_cntl = gfx_v9_0_ring_patch_cntl,
7558 .patch_de = gfx_v9_0_ring_patch_de_meta,
7559 .patch_ce = gfx_v9_0_ring_patch_ce_meta,
7560 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7561 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
7562 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
7563 };
7564
7565 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
7566 .type = AMDGPU_RING_TYPE_COMPUTE,
7567 .align_mask = 0xff,
7568 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7569 .support_64bit_ptrs = true,
7570 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7571 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7572 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7573 .emit_frame_size =
7574 20 + /* gfx_v9_0_ring_emit_gds_switch */
7575 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7576 5 + /* hdp invalidate */
7577 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7578 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7579 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7580 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
7581 7 + /* gfx_v9_0_emit_mem_sync */
7582 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
7583 15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
7584 2, /* gfx_v9_0_ring_emit_cleaner_shader */
7585 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7586 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
7587 .emit_fence = gfx_v9_0_ring_emit_fence,
7588 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
7589 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
7590 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
7591 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
7592 .test_ring = gfx_v9_0_ring_test_ring,
7593 .test_ib = gfx_v9_0_ring_test_ib,
7594 .insert_nop = gfx_v9_ring_insert_nop,
7595 .pad_ib = amdgpu_ring_generic_pad_ib,
7596 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7597 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7598 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7599 .soft_recovery = gfx_v9_0_ring_soft_recovery,
7600 .emit_mem_sync = gfx_v9_0_emit_mem_sync,
7601 .emit_wave_limit = gfx_v9_0_emit_wave_limit,
7602 .reset = gfx_v9_0_reset_kcq,
7603 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader,
7604 .begin_use = gfx_v9_0_ring_begin_use_compute,
7605 .end_use = gfx_v9_0_ring_end_use_compute,
7606 };
7607
7608 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
7609 .type = AMDGPU_RING_TYPE_KIQ,
7610 .align_mask = 0xff,
7611 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7612 .support_64bit_ptrs = true,
7613 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
7614 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
7615 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
7616 .emit_frame_size =
7617 20 + /* gfx_v9_0_ring_emit_gds_switch */
7618 7 + /* gfx_v9_0_ring_emit_hdp_flush */
7619 5 + /* hdp invalidate */
7620 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
7621 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7622 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7623 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7624 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
7625 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
7626 .test_ring = gfx_v9_0_ring_test_ring,
7627 .insert_nop = amdgpu_ring_insert_nop,
7628 .pad_ib = amdgpu_ring_generic_pad_ib,
7629 .emit_rreg = gfx_v9_0_ring_emit_rreg,
7630 .emit_wreg = gfx_v9_0_ring_emit_wreg,
7631 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
7632 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
7633 };
7634
gfx_v9_0_set_ring_funcs(struct amdgpu_device * adev)7635 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
7636 {
7637 int i;
7638
7639 adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq;
7640
7641 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7642 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
7643
7644 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) {
7645 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++)
7646 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx;
7647 }
7648
7649 for (i = 0; i < adev->gfx.num_compute_rings; i++)
7650 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
7651 }
7652
7653 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
7654 .set = gfx_v9_0_set_eop_interrupt_state,
7655 .process = gfx_v9_0_eop_irq,
7656 };
7657
7658 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
7659 .set = gfx_v9_0_set_priv_reg_fault_state,
7660 .process = gfx_v9_0_priv_reg_irq,
7661 };
7662
7663 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = {
7664 .set = gfx_v9_0_set_bad_op_fault_state,
7665 .process = gfx_v9_0_bad_op_irq,
7666 };
7667
7668 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
7669 .set = gfx_v9_0_set_priv_inst_fault_state,
7670 .process = gfx_v9_0_priv_inst_irq,
7671 };
7672
7673 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
7674 .set = gfx_v9_0_set_cp_ecc_error_state,
7675 .process = amdgpu_gfx_cp_ecc_error_irq,
7676 };
7677
7678
gfx_v9_0_set_irq_funcs(struct amdgpu_device * adev)7679 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
7680 {
7681 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7682 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
7683
7684 adev->gfx.priv_reg_irq.num_types = 1;
7685 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
7686
7687 adev->gfx.bad_op_irq.num_types = 1;
7688 adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs;
7689
7690 adev->gfx.priv_inst_irq.num_types = 1;
7691 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
7692
7693 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
7694 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
7695 }
7696
gfx_v9_0_set_rlc_funcs(struct amdgpu_device * adev)7697 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
7698 {
7699 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7700 case IP_VERSION(9, 0, 1):
7701 case IP_VERSION(9, 2, 1):
7702 case IP_VERSION(9, 4, 0):
7703 case IP_VERSION(9, 2, 2):
7704 case IP_VERSION(9, 1, 0):
7705 case IP_VERSION(9, 4, 1):
7706 case IP_VERSION(9, 3, 0):
7707 case IP_VERSION(9, 4, 2):
7708 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
7709 break;
7710 default:
7711 break;
7712 }
7713 }
7714
gfx_v9_0_set_gds_init(struct amdgpu_device * adev)7715 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
7716 {
7717 /* init asci gds info */
7718 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7719 case IP_VERSION(9, 0, 1):
7720 case IP_VERSION(9, 2, 1):
7721 case IP_VERSION(9, 4, 0):
7722 adev->gds.gds_size = 0x10000;
7723 break;
7724 case IP_VERSION(9, 2, 2):
7725 case IP_VERSION(9, 1, 0):
7726 case IP_VERSION(9, 4, 1):
7727 adev->gds.gds_size = 0x1000;
7728 break;
7729 case IP_VERSION(9, 4, 2):
7730 /* aldebaran removed all the GDS internal memory,
7731 * only support GWS opcode in kernel, like barrier
7732 * semaphore.etc */
7733 adev->gds.gds_size = 0;
7734 break;
7735 default:
7736 adev->gds.gds_size = 0x10000;
7737 break;
7738 }
7739
7740 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
7741 case IP_VERSION(9, 0, 1):
7742 case IP_VERSION(9, 4, 0):
7743 adev->gds.gds_compute_max_wave_id = 0x7ff;
7744 break;
7745 case IP_VERSION(9, 2, 1):
7746 adev->gds.gds_compute_max_wave_id = 0x27f;
7747 break;
7748 case IP_VERSION(9, 2, 2):
7749 case IP_VERSION(9, 1, 0):
7750 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
7751 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
7752 else
7753 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
7754 break;
7755 case IP_VERSION(9, 4, 1):
7756 adev->gds.gds_compute_max_wave_id = 0xfff;
7757 break;
7758 case IP_VERSION(9, 4, 2):
7759 /* deprecated for Aldebaran, no usage at all */
7760 adev->gds.gds_compute_max_wave_id = 0;
7761 break;
7762 default:
7763 /* this really depends on the chip */
7764 adev->gds.gds_compute_max_wave_id = 0x7ff;
7765 break;
7766 }
7767
7768 adev->gds.gws_size = 64;
7769 adev->gds.oa_size = 16;
7770 }
7771
gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device * adev,u32 bitmap)7772 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7773 u32 bitmap)
7774 {
7775 u32 data;
7776
7777 if (!bitmap)
7778 return;
7779
7780 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7781 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7782
7783 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
7784 }
7785
gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device * adev)7786 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7787 {
7788 u32 data, mask;
7789
7790 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
7791 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
7792
7793 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7794 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7795
7796 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7797
7798 return (~data) & mask;
7799 }
7800
gfx_v9_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7801 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
7802 struct amdgpu_cu_info *cu_info)
7803 {
7804 int i, j, k, counter, active_cu_number = 0;
7805 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7806 unsigned disable_masks[4 * 4];
7807
7808 if (!adev || !cu_info)
7809 return -EINVAL;
7810
7811 /*
7812 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
7813 */
7814 if (adev->gfx.config.max_shader_engines *
7815 adev->gfx.config.max_sh_per_se > 16)
7816 return -EINVAL;
7817
7818 amdgpu_gfx_parse_disable_cu(disable_masks,
7819 adev->gfx.config.max_shader_engines,
7820 adev->gfx.config.max_sh_per_se);
7821
7822 mutex_lock(&adev->grbm_idx_mutex);
7823 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7824 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7825 mask = 1;
7826 ao_bitmap = 0;
7827 counter = 0;
7828 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0);
7829 gfx_v9_0_set_user_cu_inactive_bitmap(
7830 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
7831 bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
7832
7833 /*
7834 * The bitmap(and ao_cu_bitmap) in cu_info structure is
7835 * 4x4 size array, and it's usually suitable for Vega
7836 * ASICs which has 4*2 SE/SH layout.
7837 * But for Arcturus, SE/SH layout is changed to 8*1.
7838 * To mostly reduce the impact, we make it compatible
7839 * with current bitmap array as below:
7840 * SE4,SH0 --> bitmap[0][1]
7841 * SE5,SH0 --> bitmap[1][1]
7842 * SE6,SH0 --> bitmap[2][1]
7843 * SE7,SH0 --> bitmap[3][1]
7844 */
7845 cu_info->bitmap[0][i % 4][j + i / 4] = bitmap;
7846
7847 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7848 if (bitmap & mask) {
7849 if (counter < adev->gfx.config.max_cu_per_sh)
7850 ao_bitmap |= mask;
7851 counter ++;
7852 }
7853 mask <<= 1;
7854 }
7855 active_cu_number += counter;
7856 if (i < 2 && j < 2)
7857 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7858 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
7859 }
7860 }
7861 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7862 mutex_unlock(&adev->grbm_idx_mutex);
7863
7864 cu_info->number = active_cu_number;
7865 cu_info->ao_cu_mask = ao_cu_mask;
7866 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7867
7868 return 0;
7869 }
7870
7871 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
7872 {
7873 .type = AMD_IP_BLOCK_TYPE_GFX,
7874 .major = 9,
7875 .minor = 0,
7876 .rev = 0,
7877 .funcs = &gfx_v9_0_ip_funcs,
7878 };
7879