1 /*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/delay.h>
24 #include <linux/kernel.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 #include "amdgpu.h"
29 #include "amdgpu_gfx.h"
30 #include "amdgpu_psp.h"
31 #include "amdgpu_smu.h"
32 #include "imu_v11_0.h"
33 #include "soc21.h"
34 #include "nvd.h"
35
36 #include "gc/gc_11_0_0_offset.h"
37 #include "gc/gc_11_0_0_sh_mask.h"
38 #include "smuio/smuio_13_0_6_offset.h"
39 #include "smuio/smuio_13_0_6_sh_mask.h"
40 #include "navi10_enum.h"
41 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
42
43 #include "soc15.h"
44 #include "clearstate_gfx11.h"
45 #include "v11_structs.h"
46 #include "gfx_v11_0.h"
47 #include "gfx_v11_0_cleaner_shader.h"
48 #include "gfx_v11_0_3.h"
49 #include "nbio_v4_3.h"
50 #include "mes_v11_0.h"
51 #include "mes_userqueue.h"
52 #include "amdgpu_userq_fence.h"
53
54 #define GFX11_NUM_GFX_RINGS 2
55 #define GFX11_MEC_HPD_SIZE 2048
56
57 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
58 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388
59
60 #define regCGTT_WD_CLK_CTRL 0x5086
61 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1
62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e
63 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1
64 #define regPC_CONFIG_CNTL_1 0x194d
65 #define regPC_CONFIG_CNTL_1_BASE_IDX 1
66
67 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0 0x0030
68 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0_BASE_IDX 1
69 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0 0x0031
70 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0_BASE_IDX 1
71
72 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100
73 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000
74 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000
75 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01
76 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000
77 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000
78 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000
79
80 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006
81 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000
82 #define regCP_MQD_CONTROL_DEFAULT 0x00000100
83 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509
84 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000
85 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000
86 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501
87 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000
88
89 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
90 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
91 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
92 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
93 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_kicker.bin");
94 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin");
95 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
96 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
97 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
98 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
99 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
100 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
101 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
102 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
103 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
104 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
105 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
106 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
107 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
108 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
109 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
110 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
111 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
112 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin");
113 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin");
114 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin");
115 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin");
116 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin");
117 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin");
118 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin");
119 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin");
120 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin");
121 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin");
122 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin");
123 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin");
124 MODULE_FIRMWARE("amdgpu/gc_11_5_3_pfp.bin");
125 MODULE_FIRMWARE("amdgpu/gc_11_5_3_me.bin");
126 MODULE_FIRMWARE("amdgpu/gc_11_5_3_mec.bin");
127 MODULE_FIRMWARE("amdgpu/gc_11_5_3_rlc.bin");
128 MODULE_FIRMWARE("amdgpu/gc_11_5_4_pfp.bin");
129 MODULE_FIRMWARE("amdgpu/gc_11_5_4_me.bin");
130 MODULE_FIRMWARE("amdgpu/gc_11_5_4_mec.bin");
131 MODULE_FIRMWARE("amdgpu/gc_11_5_4_rlc.bin");
132 MODULE_FIRMWARE("amdgpu/gc_11_5_6_pfp.bin");
133 MODULE_FIRMWARE("amdgpu/gc_11_5_6_me.bin");
134 MODULE_FIRMWARE("amdgpu/gc_11_5_6_mec.bin");
135 MODULE_FIRMWARE("amdgpu/gc_11_5_6_rlc.bin");
136 MODULE_FIRMWARE("amdgpu/gc_11_7_0_pfp.bin");
137 MODULE_FIRMWARE("amdgpu/gc_11_7_0_me.bin");
138 MODULE_FIRMWARE("amdgpu/gc_11_7_0_mec.bin");
139 MODULE_FIRMWARE("amdgpu/gc_11_7_0_rlc.bin");
140 MODULE_FIRMWARE("amdgpu/gc_11_7_1_pfp.bin");
141 MODULE_FIRMWARE("amdgpu/gc_11_7_1_me.bin");
142 MODULE_FIRMWARE("amdgpu/gc_11_7_1_mec.bin");
143 MODULE_FIRMWARE("amdgpu/gc_11_7_1_rlc.bin");
144
145 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = {
146 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
147 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
148 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
149 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
150 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
151 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
152 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
153 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
154 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
155 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
156 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
157 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
158 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
159 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
160 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
161 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
162 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
163 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
164 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
165 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
166 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
167 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
168 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE),
169 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR),
170 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR),
171 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
172 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
173 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
174 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
175 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
176 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
177 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
178 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
179 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
180 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
181 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
182 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
183 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
184 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
185 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
186 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
187 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
188 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
189 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
190 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
191 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
192 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL),
193 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS),
194 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
195 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
196 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
197 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
198 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),
199 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
200 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
201 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
202 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
203 /* cp header registers */
204 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
205 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
206 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
207 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
208 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
209 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
210 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
211 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
212 /* SE status registers */
213 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
214 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
215 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
216 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3),
217 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4),
218 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5)
219 };
220
221 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = {
222 /* compute registers */
223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
232 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
233 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
234 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
235 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
236 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
237 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
238 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
239 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
240 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
241 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
242 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
243 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
244 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
245 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
246 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
247 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
248 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
249 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
250 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
251 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
252 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
253 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
254 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
255 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
256 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
257 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
258 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
259 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
260 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
261 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS),
262 /* cp header registers */
263 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
264 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
265 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
266 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
267 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
268 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
269 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
270 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
271 };
272
273 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = {
274 /* gfx queue registers */
275 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE),
276 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID),
277 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY),
278 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM),
279 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE),
280 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI),
281 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET),
282 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL),
283 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR),
284 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR),
285 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI),
286 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST),
287 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED),
288 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL),
289 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0),
290 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0),
291 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR),
292 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI),
293 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO),
294 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI),
295 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
296 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
297 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
298 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
299 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
300 /* cp header registers */
301 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
302 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
303 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
304 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
305 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
306 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
307 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
308 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
309 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
310 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
311 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
312 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
313 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
314 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
315 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
316 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
317 };
318
319 static const struct soc15_reg_golden golden_settings_gc_11_0[] = {
320 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000)
321 };
322
323 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
324 {
325 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
326 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
327 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
328 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
329 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
330 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
331 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
332 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
333 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
334 };
335
336 #define DEFAULT_SH_MEM_CONFIG \
337 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
338 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
339 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
340
341 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
342 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
343 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
344 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
345 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
346 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
347 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
348 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
349 struct amdgpu_cu_info *cu_info);
350 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
351 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
352 u32 sh_num, u32 instance, int xcc_id);
353 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
354
355 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
356 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
357 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
358 uint32_t val);
359 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
360 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
361 uint16_t pasid, uint32_t flush_type,
362 bool all_hub, uint8_t dst_sel);
363 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
364 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
365 static void gfx_v11_0_userq_priv_fault_work(struct work_struct *work);
366 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
367 bool enable);
368
gfx11_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)369 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
370 {
371 struct amdgpu_device *adev = kiq_ring->adev;
372 u64 shader_mc_addr;
373
374 /* Cleaner shader MC address */
375 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
376
377 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
378 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
379 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */
380 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
381 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
382 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
383 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
384 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
385 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
386 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
387 }
388
gfx11_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)389 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
390 struct amdgpu_ring *ring)
391 {
392 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
393 uint64_t wptr_addr = ring->wptr_gpu_addr;
394 uint32_t me = 0, eng_sel = 0;
395
396 switch (ring->funcs->type) {
397 case AMDGPU_RING_TYPE_COMPUTE:
398 me = 1;
399 eng_sel = 0;
400 break;
401 case AMDGPU_RING_TYPE_GFX:
402 me = 0;
403 eng_sel = 4;
404 break;
405 case AMDGPU_RING_TYPE_MES:
406 me = 2;
407 eng_sel = 5;
408 break;
409 default:
410 WARN_ON(1);
411 }
412
413 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
414 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
415 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
416 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
417 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
418 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
419 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
420 PACKET3_MAP_QUEUES_ME((me)) |
421 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
422 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
423 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
424 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
425 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
426 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
427 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
428 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
429 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
430 }
431
gfx11_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)432 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
433 struct amdgpu_ring *ring,
434 enum amdgpu_unmap_queues_action action,
435 u64 gpu_addr, u64 seq)
436 {
437 struct amdgpu_device *adev = kiq_ring->adev;
438 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
439
440 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
441 amdgpu_mes_unmap_legacy_queue(adev, ring, action,
442 gpu_addr, seq, 0);
443 return;
444 }
445
446 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
447 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
448 PACKET3_UNMAP_QUEUES_ACTION(action) |
449 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
450 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
451 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
452 amdgpu_ring_write(kiq_ring,
453 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
454
455 if (action == PREEMPT_QUEUES_NO_UNMAP) {
456 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
457 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
458 amdgpu_ring_write(kiq_ring, seq);
459 } else {
460 amdgpu_ring_write(kiq_ring, 0);
461 amdgpu_ring_write(kiq_ring, 0);
462 amdgpu_ring_write(kiq_ring, 0);
463 }
464 }
465
gfx11_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)466 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
467 struct amdgpu_ring *ring,
468 u64 addr,
469 u64 seq)
470 {
471 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
472
473 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
474 amdgpu_ring_write(kiq_ring,
475 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
476 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
477 PACKET3_QUERY_STATUS_COMMAND(2));
478 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
479 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
480 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
481 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
482 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
483 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
484 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
485 }
486
gfx11_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)487 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
488 uint16_t pasid, uint32_t flush_type,
489 bool all_hub)
490 {
491 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
492 }
493
494 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
495 .kiq_set_resources = gfx11_kiq_set_resources,
496 .kiq_map_queues = gfx11_kiq_map_queues,
497 .kiq_unmap_queues = gfx11_kiq_unmap_queues,
498 .kiq_query_status = gfx11_kiq_query_status,
499 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
500 .set_resources_size = 8,
501 .map_queues_size = 7,
502 .unmap_queues_size = 6,
503 .query_status_size = 7,
504 .invalidate_tlbs_size = 2,
505 };
506
gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)507 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
508 {
509 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs;
510 }
511
gfx_v11_0_init_golden_registers(struct amdgpu_device * adev)512 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
513 {
514 if (amdgpu_sriov_vf(adev))
515 return;
516
517 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
518 case IP_VERSION(11, 0, 1):
519 case IP_VERSION(11, 0, 4):
520 soc15_program_register_sequence(adev,
521 golden_settings_gc_11_0_1,
522 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
523 break;
524 default:
525 break;
526 }
527 soc15_program_register_sequence(adev,
528 golden_settings_gc_11_0,
529 (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
530
531 }
532
gfx_v11_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)533 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
534 bool wc, uint32_t reg, uint32_t val)
535 {
536 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
537 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
538 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
539 amdgpu_ring_write(ring, reg);
540 amdgpu_ring_write(ring, 0);
541 amdgpu_ring_write(ring, val);
542 }
543
gfx_v11_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)544 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
545 int mem_space, int opt, uint32_t addr0,
546 uint32_t addr1, uint32_t ref, uint32_t mask,
547 uint32_t inv)
548 {
549 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
550 amdgpu_ring_write(ring,
551 /* memory (1) or register (0) */
552 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
553 WAIT_REG_MEM_OPERATION(opt) | /* wait */
554 WAIT_REG_MEM_FUNCTION(3) | /* equal */
555 WAIT_REG_MEM_ENGINE(eng_sel)));
556
557 if (mem_space)
558 WARN_ON(addr0 & 0x3); /* Dword align */
559 amdgpu_ring_write(ring, addr0);
560 amdgpu_ring_write(ring, addr1);
561 amdgpu_ring_write(ring, ref);
562 amdgpu_ring_write(ring, mask);
563 amdgpu_ring_write(ring, inv); /* poll interval */
564 }
565
gfx_v11_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)566 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
567 {
568 /* Header itself is a NOP packet */
569 if (num_nop == 1) {
570 amdgpu_ring_write(ring, ring->funcs->nop);
571 return;
572 }
573
574 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
575 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
576
577 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
578 amdgpu_ring_insert_nop(ring, num_nop - 1);
579 }
580
gfx_v11_0_ring_test_ring(struct amdgpu_ring * ring)581 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
582 {
583 struct amdgpu_device *adev = ring->adev;
584 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
585 uint32_t tmp = 0;
586 unsigned i;
587 int r;
588
589 WREG32(scratch, 0xCAFEDEAD);
590 r = amdgpu_ring_alloc(ring, 5);
591 if (r) {
592 drm_err(adev_to_drm(adev), "cp failed to lock ring %d (%d).\n",
593 ring->idx, r);
594 return r;
595 }
596
597 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
598 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
599 } else {
600 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
601 amdgpu_ring_write(ring, scratch -
602 PACKET3_SET_UCONFIG_REG_START);
603 amdgpu_ring_write(ring, 0xDEADBEEF);
604 }
605 amdgpu_ring_commit(ring);
606
607 for (i = 0; i < adev->usec_timeout; i++) {
608 tmp = RREG32(scratch);
609 if (tmp == 0xDEADBEEF)
610 break;
611 if (amdgpu_emu_mode == 1)
612 msleep(1);
613 else
614 udelay(1);
615 }
616
617 if (i >= adev->usec_timeout)
618 r = -ETIMEDOUT;
619 return r;
620 }
621
gfx_v11_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)622 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
623 {
624 struct amdgpu_device *adev = ring->adev;
625 struct amdgpu_ib ib;
626 struct dma_fence *f = NULL;
627 unsigned index;
628 uint64_t gpu_addr;
629 uint32_t *cpu_ptr;
630 long r;
631
632 /* MES KIQ fw hasn't indirect buffer support for now */
633 if (adev->enable_mes_kiq &&
634 ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
635 return 0;
636
637 memset(&ib, 0, sizeof(ib));
638
639 r = amdgpu_wb_get(adev, &index);
640 if (r)
641 return r;
642
643 gpu_addr = adev->wb.gpu_addr + (index * 4);
644 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
645 cpu_ptr = &adev->wb.wb[index];
646
647 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
648 if (r) {
649 drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
650 goto err1;
651 }
652
653 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
654 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
655 ib.ptr[2] = lower_32_bits(gpu_addr);
656 ib.ptr[3] = upper_32_bits(gpu_addr);
657 ib.ptr[4] = 0xDEADBEEF;
658 ib.length_dw = 5;
659
660 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
661 if (r)
662 goto err2;
663
664 r = dma_fence_wait_timeout(f, false, timeout);
665 if (r == 0) {
666 r = -ETIMEDOUT;
667 goto err2;
668 } else if (r < 0) {
669 goto err2;
670 }
671
672 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
673 r = 0;
674 else
675 r = -EINVAL;
676 err2:
677 amdgpu_ib_free(&ib, NULL);
678 dma_fence_put(f);
679 err1:
680 amdgpu_wb_free(adev, index);
681 return r;
682 }
683
gfx_v11_0_free_microcode(struct amdgpu_device * adev)684 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
685 {
686 amdgpu_ucode_release(&adev->gfx.pfp_fw);
687 amdgpu_ucode_release(&adev->gfx.me_fw);
688 amdgpu_ucode_release(&adev->gfx.rlc_fw);
689 amdgpu_ucode_release(&adev->gfx.mec_fw);
690
691 kfree(adev->gfx.rlc.register_list_format);
692 }
693
gfx_v11_0_init_toc_microcode(struct amdgpu_device * adev,const char * ucode_prefix)694 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
695 {
696 const struct psp_firmware_header_v1_0 *toc_hdr;
697 int err = 0;
698
699 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
700 AMDGPU_UCODE_REQUIRED,
701 "amdgpu/%s_toc.bin", ucode_prefix);
702 if (err)
703 goto out;
704
705 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
706 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
707 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
708 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
709 adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
710 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
711 return 0;
712 out:
713 amdgpu_ucode_release(&adev->psp.toc_fw);
714 return err;
715 }
716
gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device * adev)717 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
718 {
719 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
720 case IP_VERSION(11, 0, 0):
721 case IP_VERSION(11, 0, 2):
722 case IP_VERSION(11, 0, 3):
723 if ((adev->gfx.me_fw_version >= 1505) &&
724 (adev->gfx.pfp_fw_version >= 1600) &&
725 (adev->gfx.mec_fw_version >= 512)) {
726 if (amdgpu_sriov_vf(adev))
727 adev->gfx.cp_gfx_shadow = true;
728 else
729 adev->gfx.cp_gfx_shadow = false;
730 }
731 break;
732 default:
733 adev->gfx.cp_gfx_shadow = false;
734 break;
735 }
736 }
737
gfx_v11_0_init_microcode(struct amdgpu_device * adev)738 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
739 {
740 char ucode_prefix[25];
741 int err;
742 const struct rlc_firmware_header_v2_0 *rlc_hdr;
743 uint16_t version_major;
744 uint16_t version_minor;
745
746 DRM_DEBUG("\n");
747
748 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
749 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
750 AMDGPU_UCODE_REQUIRED,
751 "amdgpu/%s_pfp.bin", ucode_prefix);
752 if (err)
753 goto out;
754 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
755 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
756 (union amdgpu_firmware_header *)
757 adev->gfx.pfp_fw->data, 2, 0);
758 if (adev->gfx.rs64_enable) {
759 dev_info(adev->dev, "CP RS64 enable\n");
760 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
761 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
762 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
763 } else {
764 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
765 }
766
767 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
768 AMDGPU_UCODE_REQUIRED,
769 "amdgpu/%s_me.bin", ucode_prefix);
770 if (err)
771 goto out;
772 if (adev->gfx.rs64_enable) {
773 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
774 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
775 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
776 } else {
777 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
778 }
779
780 if (!amdgpu_sriov_vf(adev)) {
781 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) &&
782 adev->pdev->revision == 0xCE)
783 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
784 AMDGPU_UCODE_REQUIRED,
785 "amdgpu/gc_11_0_0_rlc_1.bin");
786 else if (amdgpu_is_kicker_fw(adev))
787 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
788 AMDGPU_UCODE_REQUIRED,
789 "amdgpu/%s_rlc_kicker.bin", ucode_prefix);
790 else
791 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
792 AMDGPU_UCODE_REQUIRED,
793 "amdgpu/%s_rlc.bin", ucode_prefix);
794 if (err)
795 goto out;
796 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
797 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
798 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
799 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
800 if (err)
801 goto out;
802 }
803
804 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
805 AMDGPU_UCODE_REQUIRED,
806 "amdgpu/%s_mec.bin", ucode_prefix);
807 if (err)
808 goto out;
809 if (adev->gfx.rs64_enable) {
810 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
811 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
812 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
813 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
814 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
815 } else {
816 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
817 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
818 }
819
820 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
821 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
822
823 /* only one MEC for gfx 11.0.0. */
824 adev->gfx.mec2_fw = NULL;
825
826 gfx_v11_0_check_fw_cp_gfx_shadow(adev);
827
828 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) {
829 err = adev->gfx.imu.funcs->init_microcode(adev);
830 if (err)
831 DRM_ERROR("Failed to init imu firmware!\n");
832 return err;
833 }
834
835 out:
836 if (err) {
837 amdgpu_ucode_release(&adev->gfx.pfp_fw);
838 amdgpu_ucode_release(&adev->gfx.me_fw);
839 amdgpu_ucode_release(&adev->gfx.rlc_fw);
840 amdgpu_ucode_release(&adev->gfx.mec_fw);
841 }
842
843 return err;
844 }
845
gfx_v11_0_get_csb_size(struct amdgpu_device * adev)846 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
847 {
848 u32 count = 0;
849 const struct cs_section_def *sect = NULL;
850 const struct cs_extent_def *ext = NULL;
851
852 /* begin clear state */
853 count += 2;
854 /* context control state */
855 count += 3;
856
857 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
858 for (ext = sect->section; ext->extent != NULL; ++ext) {
859 if (sect->id == SECT_CONTEXT)
860 count += 2 + ext->reg_count;
861 else
862 return 0;
863 }
864 }
865
866 /* set PA_SC_TILE_STEERING_OVERRIDE */
867 count += 3;
868 /* end clear state */
869 count += 2;
870 /* clear state */
871 count += 2;
872
873 return count;
874 }
875
gfx_v11_0_get_csb_buffer(struct amdgpu_device * adev,u32 * buffer)876 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
877 {
878 u32 count = 0;
879 int ctx_reg_offset;
880
881 if (adev->gfx.rlc.cs_data == NULL)
882 return;
883 if (buffer == NULL)
884 return;
885
886 count = amdgpu_gfx_csb_preamble_start(buffer);
887 count = amdgpu_gfx_csb_data_parser(adev, buffer, count);
888
889 ctx_reg_offset = SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
890 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
891 buffer[count++] = cpu_to_le32(ctx_reg_offset);
892 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
893
894 amdgpu_gfx_csb_preamble_end(buffer, count);
895 }
896
gfx_v11_0_rlc_fini(struct amdgpu_device * adev)897 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
898 {
899 /* clear state block */
900 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
901 &adev->gfx.rlc.clear_state_gpu_addr,
902 (void **)&adev->gfx.rlc.cs_ptr);
903
904 /* jump table block */
905 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
906 &adev->gfx.rlc.cp_table_gpu_addr,
907 (void **)&adev->gfx.rlc.cp_table_ptr);
908 }
909
gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)910 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
911 {
912 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
913
914 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
915 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
916 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
917 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
918 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
919 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
920 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
921 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
922 adev->gfx.rlc.rlcg_reg_access_supported = true;
923 }
924
gfx_v11_0_rlc_init(struct amdgpu_device * adev)925 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
926 {
927 const struct cs_section_def *cs_data;
928 int r;
929
930 adev->gfx.rlc.cs_data = gfx11_cs_data;
931
932 cs_data = adev->gfx.rlc.cs_data;
933
934 if (cs_data) {
935 /* init clear state block */
936 r = amdgpu_gfx_rlc_init_csb(adev);
937 if (r)
938 return r;
939 }
940
941 /* init spm vmid with 0xf */
942 if (adev->gfx.rlc.funcs->update_spm_vmid)
943 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
944
945 return 0;
946 }
947
gfx_v11_0_mec_fini(struct amdgpu_device * adev)948 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
949 {
950 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
951 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
952 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
953 }
954
gfx_v11_0_me_init(struct amdgpu_device * adev)955 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
956 {
957 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
958
959 amdgpu_gfx_graphics_queue_acquire(adev);
960 }
961
gfx_v11_0_mec_init(struct amdgpu_device * adev)962 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
963 {
964 int r;
965 u32 *hpd;
966 size_t mec_hpd_size;
967
968 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
969
970 /* take ownership of the relevant compute queues */
971 amdgpu_gfx_compute_queue_acquire(adev);
972 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
973
974 if (mec_hpd_size) {
975 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
976 AMDGPU_GEM_DOMAIN_GTT,
977 &adev->gfx.mec.hpd_eop_obj,
978 &adev->gfx.mec.hpd_eop_gpu_addr,
979 (void **)&hpd);
980 if (r) {
981 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
982 gfx_v11_0_mec_fini(adev);
983 return r;
984 }
985
986 memset(hpd, 0, mec_hpd_size);
987
988 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
989 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
990 }
991
992 return 0;
993 }
994
wave_read_ind(struct amdgpu_device * adev,uint32_t wave,uint32_t address)995 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
996 {
997 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
998 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
999 (address << SQ_IND_INDEX__INDEX__SHIFT));
1000 return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
1001 }
1002
wave_read_regs(struct amdgpu_device * adev,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)1003 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
1004 uint32_t thread, uint32_t regno,
1005 uint32_t num, uint32_t *out)
1006 {
1007 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
1008 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1009 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1010 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
1011 (SQ_IND_INDEX__AUTO_INCR_MASK));
1012 while (num--)
1013 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
1014 }
1015
gfx_v11_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)1016 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1017 {
1018 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
1019 * field when performing a select_se_sh so it should be
1020 * zero here */
1021 WARN_ON(simd != 0);
1022
1023 /* type 3 wave data */
1024 dst[(*no_fields)++] = 3;
1025 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
1026 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
1027 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
1028 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
1029 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
1030 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
1031 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
1032 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
1033 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
1034 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
1035 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
1036 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
1037 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
1038 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
1039 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
1040 }
1041
gfx_v11_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)1042 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1043 uint32_t wave, uint32_t start,
1044 uint32_t size, uint32_t *dst)
1045 {
1046 WARN_ON(simd != 0);
1047
1048 wave_read_regs(
1049 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
1050 dst);
1051 }
1052
gfx_v11_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)1053 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1054 uint32_t wave, uint32_t thread,
1055 uint32_t start, uint32_t size,
1056 uint32_t *dst)
1057 {
1058 wave_read_regs(
1059 adev, wave, thread,
1060 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1061 }
1062
gfx_v11_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)1063 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
1064 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1065 {
1066 soc21_grbm_select(adev, me, pipe, q, vm);
1067 }
1068
1069 /* all sizes are in bytes */
1070 #define MQD_SHADOW_BASE_SIZE 73728
1071 #define MQD_SHADOW_BASE_ALIGNMENT 256
1072 #define MQD_FWWORKAREA_SIZE 484
1073 #define MQD_FWWORKAREA_ALIGNMENT 256
1074
gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device * adev,struct amdgpu_gfx_shadow_info * shadow_info)1075 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev,
1076 struct amdgpu_gfx_shadow_info *shadow_info)
1077 {
1078 /* for gfx */
1079 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
1080 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
1081 shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
1082 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
1083 /* for compute */
1084 shadow_info->eop_size = GFX11_MEC_HPD_SIZE;
1085 shadow_info->eop_alignment = 256;
1086 }
1087
gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device * adev,struct amdgpu_gfx_shadow_info * shadow_info,bool skip_check)1088 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
1089 struct amdgpu_gfx_shadow_info *shadow_info,
1090 bool skip_check)
1091 {
1092 if (adev->gfx.cp_gfx_shadow || skip_check) {
1093 gfx_v11_0_get_gfx_shadow_info_nocheck(adev, shadow_info);
1094 return 0;
1095 } else {
1096 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info));
1097 return -ENOTSUPP;
1098 }
1099 }
1100
1101 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
1102 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
1103 .select_se_sh = &gfx_v11_0_select_se_sh,
1104 .read_wave_data = &gfx_v11_0_read_wave_data,
1105 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
1106 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
1107 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
1108 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
1109 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
1110 .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
1111 };
1112
gfx_v11_0_gpu_early_init(struct amdgpu_device * adev)1113 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
1114 {
1115 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1116 case IP_VERSION(11, 0, 0):
1117 case IP_VERSION(11, 0, 2):
1118 adev->gfx.config.max_hw_contexts = 8;
1119 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1120 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1121 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1122 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1123 break;
1124 case IP_VERSION(11, 0, 3):
1125 adev->gfx.ras = &gfx_v11_0_3_ras;
1126 adev->gfx.config.max_hw_contexts = 8;
1127 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1128 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1129 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1130 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1131 break;
1132 case IP_VERSION(11, 0, 1):
1133 case IP_VERSION(11, 0, 4):
1134 case IP_VERSION(11, 5, 0):
1135 case IP_VERSION(11, 5, 1):
1136 case IP_VERSION(11, 5, 2):
1137 case IP_VERSION(11, 5, 3):
1138 case IP_VERSION(11, 5, 4):
1139 case IP_VERSION(11, 5, 6):
1140 case IP_VERSION(11, 7, 0):
1141 case IP_VERSION(11, 7, 1):
1142 adev->gfx.config.max_hw_contexts = 8;
1143 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1144 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1145 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1146 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
1147 break;
1148 default:
1149 dev_warn(adev->dev, "Unsupported GC version 0x%08x\n",
1150 amdgpu_ip_version(adev, GC_HWIP, 0));
1151 return -EINVAL;
1152 }
1153
1154 return 0;
1155 }
1156
gfx_v11_0_gfx_ring_init(struct amdgpu_device * adev,int ring_id,int me,int pipe,int queue)1157 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1158 int me, int pipe, int queue)
1159 {
1160 struct amdgpu_ring *ring;
1161 unsigned int irq_type;
1162 unsigned int hw_prio;
1163
1164 ring = &adev->gfx.gfx_ring[ring_id];
1165
1166 ring->me = me;
1167 ring->pipe = pipe;
1168 ring->queue = queue;
1169
1170 ring->ring_obj = NULL;
1171 ring->use_doorbell = true;
1172 if (adev->gfx.disable_kq) {
1173 ring->no_scheduler = true;
1174 ring->no_user_submission = true;
1175 }
1176
1177 if (!ring_id)
1178 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1179 else
1180 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1181 ring->vm_hub = AMDGPU_GFXHUB(0);
1182 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1183
1184 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1185 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ?
1186 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1187 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1188 hw_prio, NULL);
1189 }
1190
gfx_v11_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)1191 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1192 int mec, int pipe, int queue)
1193 {
1194 int r;
1195 unsigned irq_type;
1196 struct amdgpu_ring *ring;
1197 unsigned int hw_prio;
1198
1199 ring = &adev->gfx.compute_ring[ring_id];
1200
1201 /* mec0 is me1 */
1202 ring->me = mec + 1;
1203 ring->pipe = pipe;
1204 ring->queue = queue;
1205
1206 ring->ring_obj = NULL;
1207 ring->use_doorbell = true;
1208 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1209 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1210 + (ring_id * GFX11_MEC_HPD_SIZE);
1211 ring->vm_hub = AMDGPU_GFXHUB(0);
1212 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1213
1214 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1215 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1216 + ring->pipe;
1217 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1218 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1219 /* type-2 packets are deprecated on MEC, use type-3 instead */
1220 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1221 hw_prio, NULL);
1222 if (r)
1223 return r;
1224
1225 return 0;
1226 }
1227
1228 static struct {
1229 SOC21_FIRMWARE_ID id;
1230 unsigned int offset;
1231 unsigned int size;
1232 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
1233
gfx_v11_0_parse_rlc_toc(struct amdgpu_device * adev,void * rlc_toc)1234 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1235 {
1236 RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
1237
1238 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1239 (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1240 rlc_autoload_info[ucode->id].id = ucode->id;
1241 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1242 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1243
1244 ucode++;
1245 }
1246 }
1247
gfx_v11_0_calc_toc_total_size(struct amdgpu_device * adev)1248 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1249 {
1250 uint32_t total_size = 0;
1251 SOC21_FIRMWARE_ID id;
1252
1253 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1254
1255 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1256 total_size += rlc_autoload_info[id].size;
1257
1258 /* In case the offset in rlc toc ucode is aligned */
1259 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1260 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1261 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1262
1263 return total_size;
1264 }
1265
gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device * adev)1266 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1267 {
1268 int r;
1269 uint32_t total_size;
1270
1271 total_size = gfx_v11_0_calc_toc_total_size(adev);
1272
1273 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1274 AMDGPU_GEM_DOMAIN_VRAM |
1275 AMDGPU_GEM_DOMAIN_GTT,
1276 &adev->gfx.rlc.rlc_autoload_bo,
1277 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1278 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1279
1280 if (r) {
1281 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1282 return r;
1283 }
1284
1285 return 0;
1286 }
1287
gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device * adev,SOC21_FIRMWARE_ID id,const void * fw_data,uint32_t fw_size,uint32_t * fw_autoload_mask)1288 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1289 SOC21_FIRMWARE_ID id,
1290 const void *fw_data,
1291 uint32_t fw_size,
1292 uint32_t *fw_autoload_mask)
1293 {
1294 uint32_t toc_offset;
1295 uint32_t toc_fw_size;
1296 char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1297
1298 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1299 return;
1300
1301 toc_offset = rlc_autoload_info[id].offset;
1302 toc_fw_size = rlc_autoload_info[id].size;
1303
1304 if (fw_size == 0)
1305 fw_size = toc_fw_size;
1306
1307 if (fw_size > toc_fw_size)
1308 fw_size = toc_fw_size;
1309
1310 memcpy(ptr + toc_offset, fw_data, fw_size);
1311
1312 if (fw_size < toc_fw_size)
1313 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1314
1315 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1316 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1317 }
1318
gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1319 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1320 uint32_t *fw_autoload_mask)
1321 {
1322 void *data;
1323 uint32_t size;
1324 uint64_t *toc_ptr;
1325
1326 *(uint64_t *)fw_autoload_mask |= 0x1;
1327
1328 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1329
1330 data = adev->psp.toc.start_addr;
1331 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1332
1333 toc_ptr = (uint64_t *)data + size / 8 - 1;
1334 *toc_ptr = *(uint64_t *)fw_autoload_mask;
1335
1336 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1337 data, size, fw_autoload_mask);
1338 }
1339
gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1340 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1341 uint32_t *fw_autoload_mask)
1342 {
1343 const __le32 *fw_data;
1344 uint32_t fw_size;
1345 const struct gfx_firmware_header_v1_0 *cp_hdr;
1346 const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1347 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1348 const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1349 uint16_t version_major, version_minor;
1350
1351 if (adev->gfx.rs64_enable) {
1352 /* pfp ucode */
1353 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1354 adev->gfx.pfp_fw->data;
1355 /* instruction */
1356 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1357 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1358 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1359 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1360 fw_data, fw_size, fw_autoload_mask);
1361 /* data */
1362 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1363 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1364 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1365 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1366 fw_data, fw_size, fw_autoload_mask);
1367 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1368 fw_data, fw_size, fw_autoload_mask);
1369 /* me ucode */
1370 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1371 adev->gfx.me_fw->data;
1372 /* instruction */
1373 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1374 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1375 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1376 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1377 fw_data, fw_size, fw_autoload_mask);
1378 /* data */
1379 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1380 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1381 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1382 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1383 fw_data, fw_size, fw_autoload_mask);
1384 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1385 fw_data, fw_size, fw_autoload_mask);
1386 /* mec ucode */
1387 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1388 adev->gfx.mec_fw->data;
1389 /* instruction */
1390 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1391 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1392 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1393 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1394 fw_data, fw_size, fw_autoload_mask);
1395 /* data */
1396 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1397 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1398 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1399 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1400 fw_data, fw_size, fw_autoload_mask);
1401 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1402 fw_data, fw_size, fw_autoload_mask);
1403 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1404 fw_data, fw_size, fw_autoload_mask);
1405 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1406 fw_data, fw_size, fw_autoload_mask);
1407 } else {
1408 /* pfp ucode */
1409 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1410 adev->gfx.pfp_fw->data;
1411 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1412 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1413 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1414 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1415 fw_data, fw_size, fw_autoload_mask);
1416
1417 /* me ucode */
1418 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1419 adev->gfx.me_fw->data;
1420 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1421 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1422 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1423 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1424 fw_data, fw_size, fw_autoload_mask);
1425
1426 /* mec ucode */
1427 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1428 adev->gfx.mec_fw->data;
1429 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1430 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1431 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1432 cp_hdr->jt_size * 4;
1433 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1434 fw_data, fw_size, fw_autoload_mask);
1435 }
1436
1437 /* rlc ucode */
1438 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1439 adev->gfx.rlc_fw->data;
1440 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1441 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1442 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1443 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1444 fw_data, fw_size, fw_autoload_mask);
1445
1446 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1447 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1448 if (version_major == 2) {
1449 if (version_minor >= 2) {
1450 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1451
1452 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1453 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1454 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1455 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1456 fw_data, fw_size, fw_autoload_mask);
1457
1458 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1459 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1460 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1461 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1462 fw_data, fw_size, fw_autoload_mask);
1463 }
1464 }
1465 }
1466
gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1467 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1468 uint32_t *fw_autoload_mask)
1469 {
1470 const __le32 *fw_data;
1471 uint32_t fw_size;
1472 const struct sdma_firmware_header_v2_0 *sdma_hdr;
1473
1474 sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1475 adev->sdma.instance[0].fw->data;
1476 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1477 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1478 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1479
1480 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1481 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1482
1483 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1484 le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1485 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1486
1487 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1488 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1489 }
1490
gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1491 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1492 uint32_t *fw_autoload_mask)
1493 {
1494 const __le32 *fw_data;
1495 unsigned fw_size;
1496 const struct mes_firmware_header_v1_0 *mes_hdr;
1497 int pipe, ucode_id, data_id;
1498
1499 for (pipe = 0; pipe < 2; pipe++) {
1500 if (pipe==0) {
1501 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1502 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1503 } else {
1504 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1505 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1506 }
1507
1508 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1509 adev->mes.fw[pipe]->data;
1510
1511 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1512 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1513 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1514
1515 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1516 ucode_id, fw_data, fw_size, fw_autoload_mask);
1517
1518 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1519 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1520 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1521
1522 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1523 data_id, fw_data, fw_size, fw_autoload_mask);
1524 }
1525 }
1526
gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device * adev)1527 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1528 {
1529 uint32_t rlc_g_offset, rlc_g_size;
1530 uint64_t gpu_addr;
1531 uint32_t autoload_fw_id[2];
1532
1533 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1534
1535 /* RLC autoload sequence 2: copy ucode */
1536 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1537 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1538 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1539 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1540
1541 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1542 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1543 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1544
1545 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1546 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1547
1548 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1549
1550 /* RLC autoload sequence 3: load IMU fw */
1551 if (adev->gfx.imu.funcs->load_microcode)
1552 adev->gfx.imu.funcs->load_microcode(adev);
1553 /* RLC autoload sequence 4 init IMU fw */
1554 if (adev->gfx.imu.funcs->setup_imu)
1555 adev->gfx.imu.funcs->setup_imu(adev);
1556 if (adev->gfx.imu.funcs->start_imu)
1557 adev->gfx.imu.funcs->start_imu(adev);
1558
1559 /* RLC autoload sequence 5 disable gpa mode */
1560 gfx_v11_0_disable_gpa_mode(adev);
1561
1562 return 0;
1563 }
1564
gfx_v11_0_alloc_ip_dump(struct amdgpu_device * adev)1565 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
1566 {
1567 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
1568 uint32_t *ptr;
1569 uint32_t inst;
1570
1571 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
1572 if (!ptr) {
1573 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1574 adev->gfx.ip_dump_core = NULL;
1575 } else {
1576 adev->gfx.ip_dump_core = ptr;
1577 }
1578
1579 /* Allocate memory for compute queue registers for all the instances */
1580 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
1581 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1582 adev->gfx.mec.num_queue_per_pipe;
1583
1584 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1585 if (!ptr) {
1586 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1587 adev->gfx.ip_dump_compute_queues = NULL;
1588 } else {
1589 adev->gfx.ip_dump_compute_queues = ptr;
1590 }
1591
1592 /* Allocate memory for gfx queue registers for all the instances */
1593 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
1594 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1595 adev->gfx.me.num_queue_per_pipe;
1596
1597 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1598 if (!ptr) {
1599 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n");
1600 adev->gfx.ip_dump_gfx_queues = NULL;
1601 } else {
1602 adev->gfx.ip_dump_gfx_queues = ptr;
1603 }
1604 }
1605
gfx_v11_0_sw_init(struct amdgpu_ip_block * ip_block)1606 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
1607 {
1608 int i, j, k, r, ring_id;
1609 int xcc_id = 0;
1610 struct amdgpu_device *adev = ip_block->adev;
1611 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
1612
1613 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler);
1614
1615 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1616 case IP_VERSION(11, 0, 0):
1617 case IP_VERSION(11, 0, 1):
1618 case IP_VERSION(11, 0, 2):
1619 case IP_VERSION(11, 0, 3):
1620 case IP_VERSION(11, 0, 4):
1621 case IP_VERSION(11, 5, 0):
1622 case IP_VERSION(11, 5, 1):
1623 case IP_VERSION(11, 5, 2):
1624 case IP_VERSION(11, 5, 3):
1625 case IP_VERSION(11, 5, 4):
1626 case IP_VERSION(11, 5, 6):
1627 case IP_VERSION(11, 7, 0):
1628 case IP_VERSION(11, 7, 1):
1629 adev->gfx.me.num_me = 1;
1630 if (adev->gfx.rs64_enable)
1631 adev->gfx.me.num_pipe_per_me = 1;
1632 else
1633 adev->gfx.me.num_pipe_per_me = 2;
1634 adev->gfx.me.num_queue_per_pipe = 2;
1635 adev->gfx.mec.num_mec = 1;
1636 adev->gfx.mec.num_pipe_per_mec = 4;
1637 adev->gfx.mec.num_queue_per_pipe = 4;
1638 break;
1639 default:
1640 adev->gfx.me.num_me = 1;
1641 adev->gfx.me.num_pipe_per_me = 1;
1642 adev->gfx.me.num_queue_per_pipe = 1;
1643 adev->gfx.mec.num_mec = 1;
1644 adev->gfx.mec.num_pipe_per_mec = 4;
1645 adev->gfx.mec.num_queue_per_pipe = 8;
1646 break;
1647 }
1648
1649 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1650 case IP_VERSION(11, 0, 0):
1651 case IP_VERSION(11, 0, 2):
1652 case IP_VERSION(11, 0, 3):
1653 if (!adev->gfx.disable_uq &&
1654 adev->gfx.me_fw_version >= 2420 &&
1655 adev->gfx.pfp_fw_version >= 2580 &&
1656 adev->gfx.mec_fw_version >= 2650 &&
1657 adev->mes.fw_version[0] >= 120) {
1658 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs;
1659 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs;
1660 }
1661 break;
1662 case IP_VERSION(11, 0, 1):
1663 case IP_VERSION(11, 0, 4):
1664 case IP_VERSION(11, 5, 0):
1665 case IP_VERSION(11, 5, 1):
1666 case IP_VERSION(11, 5, 2):
1667 case IP_VERSION(11, 5, 3):
1668 /* add firmware version checks here */
1669 if (0 && !adev->gfx.disable_uq) {
1670 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs;
1671 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs;
1672 }
1673 break;
1674 default:
1675 break;
1676 }
1677
1678 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1679 case IP_VERSION(11, 0, 0):
1680 case IP_VERSION(11, 0, 2):
1681 case IP_VERSION(11, 0, 3):
1682 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1683 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1684 if (adev->gfx.me_fw_version >= 2280 &&
1685 adev->gfx.pfp_fw_version >= 2370 &&
1686 adev->gfx.mec_fw_version >= 2450 &&
1687 adev->mes.fw_version[0] >= 99) {
1688 adev->gfx.enable_cleaner_shader = true;
1689 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1690 if (r) {
1691 adev->gfx.enable_cleaner_shader = false;
1692 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1693 }
1694 }
1695 break;
1696 case IP_VERSION(11, 0, 1):
1697 case IP_VERSION(11, 0, 4):
1698 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1699 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1700 if (adev->gfx.pfp_fw_version >= 102 &&
1701 adev->gfx.mec_fw_version >= 66 &&
1702 adev->mes.fw_version[0] >= 128) {
1703 adev->gfx.enable_cleaner_shader = true;
1704 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1705 if (r) {
1706 adev->gfx.enable_cleaner_shader = false;
1707 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1708 }
1709 }
1710 break;
1711 case IP_VERSION(11, 5, 0):
1712 case IP_VERSION(11, 5, 1):
1713 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1714 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1715 if (adev->gfx.mec_fw_version >= 26 &&
1716 adev->mes.fw_version[0] >= 114) {
1717 adev->gfx.enable_cleaner_shader = true;
1718 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1719 if (r) {
1720 adev->gfx.enable_cleaner_shader = false;
1721 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1722 }
1723 }
1724 break;
1725 case IP_VERSION(11, 5, 2):
1726 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1727 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1728 if (adev->gfx.me_fw_version >= 12 &&
1729 adev->gfx.pfp_fw_version >= 15 &&
1730 adev->gfx.mec_fw_version >= 15) {
1731 adev->gfx.enable_cleaner_shader = true;
1732 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1733 if (r) {
1734 adev->gfx.enable_cleaner_shader = false;
1735 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1736 }
1737 }
1738 break;
1739 case IP_VERSION(11, 5, 3):
1740 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1741 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1742 if (adev->gfx.me_fw_version >= 7 &&
1743 adev->gfx.pfp_fw_version >= 8 &&
1744 adev->gfx.mec_fw_version >= 8) {
1745 adev->gfx.enable_cleaner_shader = true;
1746 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1747 if (r) {
1748 adev->gfx.enable_cleaner_shader = false;
1749 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1750 }
1751 }
1752 break;
1753 case IP_VERSION(11, 5, 4):
1754 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1755 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1756 if (adev->gfx.me_fw_version >= 4 &&
1757 adev->gfx.pfp_fw_version >= 7 &&
1758 adev->gfx.mec_fw_version >= 5) {
1759 adev->gfx.enable_cleaner_shader = true;
1760 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1761 if (r) {
1762 adev->gfx.enable_cleaner_shader = false;
1763 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1764 }
1765 }
1766 break;
1767 default:
1768 adev->gfx.enable_cleaner_shader = false;
1769 break;
1770 }
1771
1772 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1773 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) &&
1774 amdgpu_sriov_is_pp_one_vf(adev))
1775 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1776
1777 /* EOP Event */
1778 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1779 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1780 &adev->gfx.eop_irq);
1781 if (r)
1782 return r;
1783
1784 /* Bad opcode Event */
1785 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1786 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR,
1787 &adev->gfx.bad_op_irq);
1788 if (r)
1789 return r;
1790
1791 /* Privileged reg */
1792 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1793 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1794 &adev->gfx.priv_reg_irq);
1795 if (r)
1796 return r;
1797
1798 /* Privileged inst */
1799 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1800 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1801 &adev->gfx.priv_inst_irq);
1802 if (r)
1803 return r;
1804
1805 /* FED error */
1806 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1807 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1808 &adev->gfx.rlc_gc_fed_irq);
1809 if (r)
1810 return r;
1811
1812 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1813
1814 gfx_v11_0_me_init(adev);
1815
1816 r = gfx_v11_0_rlc_init(adev);
1817 if (r) {
1818 DRM_ERROR("Failed to init rlc BOs!\n");
1819 return r;
1820 }
1821
1822 r = gfx_v11_0_mec_init(adev);
1823 if (r) {
1824 DRM_ERROR("Failed to init MEC BOs!\n");
1825 return r;
1826 }
1827
1828 if (adev->gfx.num_gfx_rings) {
1829 ring_id = 0;
1830 /* set up the gfx ring */
1831 for (i = 0; i < adev->gfx.me.num_me; i++) {
1832 for (j = 0; j < num_queue_per_pipe; j++) {
1833 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1834 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1835 continue;
1836
1837 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1838 i, k, j);
1839 if (r)
1840 return r;
1841 ring_id++;
1842 }
1843 }
1844 }
1845 }
1846
1847 if (adev->gfx.num_compute_rings) {
1848 ring_id = 0;
1849 /* set up the compute queues - allocate horizontally across pipes */
1850 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1851 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1852 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1853 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1854 k, j))
1855 continue;
1856
1857 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1858 i, k, j);
1859 if (r)
1860 return r;
1861
1862 ring_id++;
1863 }
1864 }
1865 }
1866 }
1867
1868 adev->gfx.gfx_supported_reset =
1869 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
1870 adev->gfx.compute_supported_reset =
1871 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
1872 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1873 case IP_VERSION(11, 0, 0):
1874 case IP_VERSION(11, 0, 2):
1875 case IP_VERSION(11, 0, 3):
1876 if ((adev->gfx.me_fw_version >= 2280) &&
1877 (adev->gfx.mec_fw_version >= 2410) &&
1878 !amdgpu_sriov_vf(adev) &&
1879 !adev->debug_disable_gpu_ring_reset) {
1880 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1881 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1882 }
1883 break;
1884 default:
1885 if (!amdgpu_sriov_vf(adev) &&
1886 !adev->debug_disable_gpu_ring_reset) {
1887 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1888 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1889 }
1890 break;
1891 }
1892
1893 if (!adev->enable_mes_kiq) {
1894 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0);
1895 if (r) {
1896 DRM_ERROR("Failed to init KIQ BOs!\n");
1897 return r;
1898 }
1899
1900 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1901 if (r)
1902 return r;
1903 }
1904
1905 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0);
1906 if (r)
1907 return r;
1908
1909 /* allocate visible FB for rlc auto-loading fw */
1910 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1911 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1912 if (r)
1913 return r;
1914 }
1915
1916 r = gfx_v11_0_gpu_early_init(adev);
1917 if (r)
1918 return r;
1919
1920 if (amdgpu_gfx_ras_sw_init(adev)) {
1921 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1922 return -EINVAL;
1923 }
1924
1925 r = amdgpu_gfx_sysfs_init(adev);
1926 if (r)
1927 return r;
1928
1929 gfx_v11_0_alloc_ip_dump(adev);
1930
1931 adev->gfx.me.use_mmio_for_reset = false;
1932 adev->gfx.mec.use_mmio_for_reset = true;
1933
1934 mutex_init(&adev->gfx.mec.reset_mutex);
1935
1936 INIT_WORK(&adev->gfx.userq_priv_fault_work, gfx_v11_0_userq_priv_fault_work);
1937
1938 return 0;
1939 }
1940
gfx_v11_0_pfp_fini(struct amdgpu_device * adev)1941 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1942 {
1943 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1944 &adev->gfx.pfp.pfp_fw_gpu_addr,
1945 (void **)&adev->gfx.pfp.pfp_fw_ptr);
1946
1947 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1948 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1949 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1950 }
1951
gfx_v11_0_me_fini(struct amdgpu_device * adev)1952 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1953 {
1954 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1955 &adev->gfx.me.me_fw_gpu_addr,
1956 (void **)&adev->gfx.me.me_fw_ptr);
1957
1958 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1959 &adev->gfx.me.me_fw_data_gpu_addr,
1960 (void **)&adev->gfx.me.me_fw_data_ptr);
1961 }
1962
gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device * adev)1963 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1964 {
1965 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1966 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1967 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1968 }
1969
gfx_v11_0_sw_fini(struct amdgpu_ip_block * ip_block)1970 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block)
1971 {
1972 int i;
1973 struct amdgpu_device *adev = ip_block->adev;
1974
1975 cancel_work_sync(&adev->gfx.userq_priv_fault_work);
1976
1977 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1978 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1979 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1980 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1981
1982 amdgpu_gfx_mqd_sw_fini(adev, 0);
1983
1984 if (!adev->enable_mes_kiq) {
1985 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1986 amdgpu_gfx_kiq_fini(adev, 0);
1987 }
1988
1989 amdgpu_gfx_cleaner_shader_sw_fini(adev);
1990
1991 gfx_v11_0_pfp_fini(adev);
1992 gfx_v11_0_me_fini(adev);
1993 gfx_v11_0_rlc_fini(adev);
1994 gfx_v11_0_mec_fini(adev);
1995
1996 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1997 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1998
1999 gfx_v11_0_free_microcode(adev);
2000
2001 amdgpu_gfx_sysfs_fini(adev);
2002
2003 kfree(adev->gfx.ip_dump_core);
2004 kfree(adev->gfx.ip_dump_compute_queues);
2005 kfree(adev->gfx.ip_dump_gfx_queues);
2006
2007 return 0;
2008 }
2009
gfx_v11_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)2010 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
2011 u32 sh_num, u32 instance, int xcc_id)
2012 {
2013 u32 data;
2014
2015 if (instance == 0xffffffff)
2016 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
2017 INSTANCE_BROADCAST_WRITES, 1);
2018 else
2019 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
2020 instance);
2021
2022 if (se_num == 0xffffffff)
2023 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
2024 1);
2025 else
2026 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2027
2028 if (sh_num == 0xffffffff)
2029 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
2030 1);
2031 else
2032 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
2033
2034 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
2035 }
2036
gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device * adev)2037 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
2038 {
2039 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
2040
2041 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
2042 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
2043 CC_GC_SA_UNIT_DISABLE,
2044 SA_DISABLE);
2045 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
2046 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
2047 GC_USER_SA_UNIT_DISABLE,
2048 SA_DISABLE);
2049 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
2050 adev->gfx.config.max_shader_engines);
2051
2052 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
2053 }
2054
gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device * adev)2055 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2056 {
2057 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
2058 u32 rb_mask;
2059
2060 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
2061 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
2062 CC_RB_BACKEND_DISABLE,
2063 BACKEND_DISABLE);
2064 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
2065 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
2066 GC_USER_RB_BACKEND_DISABLE,
2067 BACKEND_DISABLE);
2068 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
2069 adev->gfx.config.max_shader_engines);
2070
2071 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
2072 }
2073
gfx_v11_0_setup_rb(struct amdgpu_device * adev)2074 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
2075 {
2076 u32 rb_bitmap_per_sa;
2077 u32 rb_bitmap_width_per_sa;
2078 u32 max_sa;
2079 u32 active_sa_bitmap;
2080 u32 global_active_rb_bitmap;
2081 u32 active_rb_bitmap = 0;
2082 u32 i;
2083
2084 /* query sa bitmap from SA_UNIT_DISABLE registers */
2085 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
2086 /* query rb bitmap from RB_BACKEND_DISABLE registers */
2087 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
2088
2089 /* generate active rb bitmap according to active sa bitmap */
2090 max_sa = adev->gfx.config.max_shader_engines *
2091 adev->gfx.config.max_sh_per_se;
2092 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
2093 adev->gfx.config.max_sh_per_se;
2094 rb_bitmap_per_sa = amdgpu_gfx_create_bitmask(rb_bitmap_width_per_sa);
2095
2096 for (i = 0; i < max_sa; i++) {
2097 if (active_sa_bitmap & (1 << i))
2098 active_rb_bitmap |= (rb_bitmap_per_sa << (i * rb_bitmap_width_per_sa));
2099 }
2100
2101 active_rb_bitmap &= global_active_rb_bitmap;
2102 adev->gfx.config.backend_enable_mask = active_rb_bitmap;
2103 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
2104 }
2105
2106 #define DEFAULT_SH_MEM_BASES (0x6000)
2107 #define LDS_APP_BASE 0x1
2108 #define SCRATCH_APP_BASE 0x2
2109
gfx_v11_0_init_compute_vmid(struct amdgpu_device * adev)2110 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
2111 {
2112 int i;
2113 uint32_t sh_mem_bases;
2114 uint32_t data;
2115
2116 /*
2117 * Configure apertures:
2118 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2119 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2120 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2121 */
2122 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
2123 SCRATCH_APP_BASE;
2124
2125 mutex_lock(&adev->srbm_mutex);
2126 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2127 soc21_grbm_select(adev, 0, 0, 0, i);
2128 /* CP and shaders */
2129 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
2130 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
2131
2132 /* Enable trap for each kfd vmid. */
2133 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
2134 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
2135 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
2136 }
2137 soc21_grbm_select(adev, 0, 0, 0, 0);
2138 mutex_unlock(&adev->srbm_mutex);
2139
2140 /*
2141 * Initialize all compute VMIDs to have no GDS, GWS, or OA
2142 * access. These should be enabled by FW for target VMIDs.
2143 */
2144 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2145 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
2146 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
2147 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
2148 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
2149 }
2150 }
2151
gfx_v11_0_init_gds_vmid(struct amdgpu_device * adev)2152 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
2153 {
2154 int vmid;
2155
2156 /*
2157 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2158 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2159 * the driver can enable them for graphics. VMID0 should maintain
2160 * access so that HWS firmware can save/restore entries.
2161 */
2162 for (vmid = 1; vmid < 16; vmid++) {
2163 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
2164 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
2165 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
2166 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
2167 }
2168 }
2169
gfx_v11_0_tcp_harvest(struct amdgpu_device * adev)2170 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
2171 {
2172 /* TODO: harvest feature to be added later. */
2173 }
2174
gfx_v11_0_get_tcc_info(struct amdgpu_device * adev)2175 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
2176 {
2177 /* TCCs are global (not instanced). */
2178 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
2179 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
2180
2181 adev->gfx.config.tcc_disabled_mask =
2182 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
2183 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
2184 }
2185
gfx_v11_0_constants_init(struct amdgpu_device * adev)2186 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
2187 {
2188 u32 tmp;
2189 int i;
2190
2191 if (!amdgpu_sriov_vf(adev))
2192 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2193
2194 gfx_v11_0_setup_rb(adev);
2195 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
2196 gfx_v11_0_get_tcc_info(adev);
2197 adev->gfx.config.pa_sc_tile_steering_override = 0;
2198
2199 /* Set whether texture coordinate truncation is conformant. */
2200 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
2201 adev->gfx.config.ta_cntl2_truncate_coord_mode =
2202 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
2203 /* Program DB_RING_CONTROL for multiple GFX pipes
2204 * Default power up value is 1.
2205 * Possible values:
2206 * 0 - split occlusion counters between gfx pipes
2207 * 1 - all occlusion counters to pipe 0
2208 * 2 - all occlusion counters to pipe 1
2209 */
2210 WREG32_FIELD15_PREREG(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL,
2211 (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1);
2212
2213 /* XXX SH_MEM regs */
2214 /* where to put LDS, scratch, GPUVM in FSA64 space */
2215 mutex_lock(&adev->srbm_mutex);
2216 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2217 soc21_grbm_select(adev, 0, 0, 0, i);
2218 /* CP and shaders */
2219 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
2220 if (i != 0) {
2221 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2222 (adev->gmc.private_aperture_start >> 48));
2223 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2224 (adev->gmc.shared_aperture_start >> 48));
2225 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
2226 }
2227 }
2228 soc21_grbm_select(adev, 0, 0, 0, 0);
2229
2230 mutex_unlock(&adev->srbm_mutex);
2231
2232 gfx_v11_0_init_compute_vmid(adev);
2233 gfx_v11_0_init_gds_vmid(adev);
2234 }
2235
gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device * adev,int me,int pipe)2236 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev,
2237 int me, int pipe)
2238 {
2239 if (me != 0)
2240 return 0;
2241
2242 switch (pipe) {
2243 case 0:
2244 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
2245 case 1:
2246 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
2247 default:
2248 return 0;
2249 }
2250 }
2251
gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device * adev,int me,int pipe)2252 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev,
2253 int me, int pipe)
2254 {
2255 /*
2256 * amdgpu controls only the first MEC. That's why this function only
2257 * handles the setting of interrupts for this specific MEC. All other
2258 * pipes' interrupts are set by amdkfd.
2259 */
2260 if (me != 1)
2261 return 0;
2262
2263 switch (pipe) {
2264 case 0:
2265 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
2266 case 1:
2267 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
2268 case 2:
2269 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
2270 case 3:
2271 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
2272 default:
2273 return 0;
2274 }
2275 }
2276
gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2277 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2278 bool enable)
2279 {
2280 u32 tmp, cp_int_cntl_reg;
2281 int i, j;
2282
2283 if (amdgpu_sriov_vf(adev))
2284 return;
2285
2286 for (i = 0; i < adev->gfx.me.num_me; i++) {
2287 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
2288 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
2289
2290 if (cp_int_cntl_reg) {
2291 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
2292 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
2293 enable ? 1 : 0);
2294 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
2295 enable ? 1 : 0);
2296 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
2297 enable ? 1 : 0);
2298 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
2299 enable ? 1 : 0);
2300 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp);
2301 }
2302 }
2303 }
2304 }
2305
gfx_v11_0_init_csb(struct amdgpu_device * adev)2306 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
2307 {
2308 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2309
2310 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
2311 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2312 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
2313 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2314 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
2315
2316 return 0;
2317 }
2318
gfx_v11_0_rlc_stop(struct amdgpu_device * adev)2319 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
2320 {
2321 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
2322
2323 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
2324 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
2325 }
2326
gfx_v11_0_rlc_reset(struct amdgpu_device * adev)2327 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
2328 {
2329 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2330 udelay(50);
2331 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2332 udelay(50);
2333 }
2334
gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device * adev,bool enable)2335 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
2336 bool enable)
2337 {
2338 uint32_t rlc_pg_cntl;
2339
2340 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
2341
2342 if (!enable) {
2343 /* RLC_PG_CNTL[23] = 0 (default)
2344 * RLC will wait for handshake acks with SMU
2345 * GFXOFF will be enabled
2346 * RLC_PG_CNTL[23] = 1
2347 * RLC will not issue any message to SMU
2348 * hence no handshake between SMU & RLC
2349 * GFXOFF will be disabled
2350 */
2351 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2352 } else
2353 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2354 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
2355 }
2356
gfx_v11_0_rlc_start(struct amdgpu_device * adev)2357 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
2358 {
2359 /* TODO: enable rlc & smu handshake until smu
2360 * and gfxoff feature works as expected */
2361 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
2362 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
2363
2364 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2365 udelay(50);
2366 }
2367
gfx_v11_0_rlc_enable_srm(struct amdgpu_device * adev)2368 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
2369 {
2370 uint32_t tmp;
2371
2372 /* enable Save Restore Machine */
2373 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
2374 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2375 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
2376 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
2377 }
2378
gfx_v11_0_load_rlcg_microcode(struct amdgpu_device * adev)2379 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
2380 {
2381 const struct rlc_firmware_header_v2_0 *hdr;
2382 const __le32 *fw_data;
2383 unsigned i, fw_size;
2384
2385 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2386 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2387 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2388 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2389
2390 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
2391 RLCG_UCODE_LOADING_START_ADDRESS);
2392
2393 for (i = 0; i < fw_size; i++)
2394 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
2395 le32_to_cpup(fw_data++));
2396
2397 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2398 }
2399
gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device * adev)2400 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
2401 {
2402 const struct rlc_firmware_header_v2_2 *hdr;
2403 const __le32 *fw_data;
2404 unsigned i, fw_size;
2405 u32 tmp;
2406
2407 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
2408
2409 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2410 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
2411 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
2412
2413 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
2414
2415 for (i = 0; i < fw_size; i++) {
2416 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2417 msleep(1);
2418 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
2419 le32_to_cpup(fw_data++));
2420 }
2421
2422 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2423
2424 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2425 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2426 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2427
2428 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
2429 for (i = 0; i < fw_size; i++) {
2430 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2431 msleep(1);
2432 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
2433 le32_to_cpup(fw_data++));
2434 }
2435
2436 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2437
2438 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
2439 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2440 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2441 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
2442 }
2443
gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device * adev)2444 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
2445 {
2446 const struct rlc_firmware_header_v2_3 *hdr;
2447 const __le32 *fw_data;
2448 unsigned i, fw_size;
2449 u32 tmp;
2450
2451 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
2452
2453 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2454 le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
2455 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
2456
2457 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
2458
2459 for (i = 0; i < fw_size; i++) {
2460 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2461 msleep(1);
2462 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
2463 le32_to_cpup(fw_data++));
2464 }
2465
2466 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
2467
2468 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
2469 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
2470 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
2471
2472 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2473 le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
2474 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
2475
2476 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
2477
2478 for (i = 0; i < fw_size; i++) {
2479 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2480 msleep(1);
2481 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
2482 le32_to_cpup(fw_data++));
2483 }
2484
2485 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
2486
2487 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
2488 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
2489 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
2490 }
2491
gfx_v11_0_rlc_load_microcode(struct amdgpu_device * adev)2492 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
2493 {
2494 const struct rlc_firmware_header_v2_0 *hdr;
2495 uint16_t version_major;
2496 uint16_t version_minor;
2497
2498 if (!adev->gfx.rlc_fw)
2499 return -EINVAL;
2500
2501 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2502 amdgpu_ucode_print_rlc_hdr(&hdr->header);
2503
2504 version_major = le16_to_cpu(hdr->header.header_version_major);
2505 version_minor = le16_to_cpu(hdr->header.header_version_minor);
2506
2507 if (version_major == 2) {
2508 gfx_v11_0_load_rlcg_microcode(adev);
2509 if (amdgpu_dpm == 1) {
2510 if (version_minor >= 2)
2511 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
2512 if (version_minor == 3)
2513 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
2514 }
2515
2516 return 0;
2517 }
2518
2519 return -EINVAL;
2520 }
2521
gfx_v11_0_rlc_resume(struct amdgpu_device * adev)2522 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2523 {
2524 int r;
2525
2526 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2527 gfx_v11_0_init_csb(adev);
2528
2529 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2530 gfx_v11_0_rlc_enable_srm(adev);
2531 } else {
2532 if (amdgpu_sriov_vf(adev)) {
2533 gfx_v11_0_init_csb(adev);
2534 return 0;
2535 }
2536
2537 adev->gfx.rlc.funcs->stop(adev);
2538
2539 /* disable CG */
2540 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2541
2542 /* disable PG */
2543 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2544
2545 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2546 /* legacy rlc firmware loading */
2547 r = gfx_v11_0_rlc_load_microcode(adev);
2548 if (r)
2549 return r;
2550 }
2551
2552 gfx_v11_0_init_csb(adev);
2553
2554 adev->gfx.rlc.funcs->start(adev);
2555 }
2556 return 0;
2557 }
2558
gfx_v11_0_config_me_cache(struct amdgpu_device * adev,uint64_t addr)2559 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2560 {
2561 uint32_t usec_timeout = 50000; /* wait for 50ms */
2562 uint32_t tmp;
2563 int i;
2564
2565 /* Trigger an invalidation of the L1 instruction caches */
2566 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2567 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2568 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2569
2570 /* Wait for invalidation complete */
2571 for (i = 0; i < usec_timeout; i++) {
2572 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2573 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2574 INVALIDATE_CACHE_COMPLETE))
2575 break;
2576 udelay(1);
2577 }
2578
2579 if (i >= usec_timeout) {
2580 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2581 return -EINVAL;
2582 }
2583
2584 if (amdgpu_emu_mode == 1)
2585 amdgpu_device_flush_hdp(adev, NULL);
2586
2587 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2588 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2589 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2590 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2591 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2592 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2593
2594 /* Program me ucode address into intruction cache address register */
2595 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2596 lower_32_bits(addr) & 0xFFFFF000);
2597 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2598 upper_32_bits(addr));
2599
2600 return 0;
2601 }
2602
gfx_v11_0_config_pfp_cache(struct amdgpu_device * adev,uint64_t addr)2603 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2604 {
2605 uint32_t usec_timeout = 50000; /* wait for 50ms */
2606 uint32_t tmp;
2607 int i;
2608
2609 /* Trigger an invalidation of the L1 instruction caches */
2610 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2611 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2612 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2613
2614 /* Wait for invalidation complete */
2615 for (i = 0; i < usec_timeout; i++) {
2616 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2617 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2618 INVALIDATE_CACHE_COMPLETE))
2619 break;
2620 udelay(1);
2621 }
2622
2623 if (i >= usec_timeout) {
2624 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2625 return -EINVAL;
2626 }
2627
2628 if (amdgpu_emu_mode == 1)
2629 amdgpu_device_flush_hdp(adev, NULL);
2630
2631 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2632 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2633 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2634 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2635 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2636 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2637
2638 /* Program pfp ucode address into intruction cache address register */
2639 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2640 lower_32_bits(addr) & 0xFFFFF000);
2641 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2642 upper_32_bits(addr));
2643
2644 return 0;
2645 }
2646
gfx_v11_0_config_mec_cache(struct amdgpu_device * adev,uint64_t addr)2647 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2648 {
2649 uint32_t usec_timeout = 50000; /* wait for 50ms */
2650 uint32_t tmp;
2651 int i;
2652
2653 /* Trigger an invalidation of the L1 instruction caches */
2654 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2655 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2656
2657 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2658
2659 /* Wait for invalidation complete */
2660 for (i = 0; i < usec_timeout; i++) {
2661 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2662 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2663 INVALIDATE_CACHE_COMPLETE))
2664 break;
2665 udelay(1);
2666 }
2667
2668 if (i >= usec_timeout) {
2669 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2670 return -EINVAL;
2671 }
2672
2673 if (amdgpu_emu_mode == 1)
2674 amdgpu_device_flush_hdp(adev, NULL);
2675
2676 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2677 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2678 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2679 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2680 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2681
2682 /* Program mec1 ucode address into intruction cache address register */
2683 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2684 lower_32_bits(addr) & 0xFFFFF000);
2685 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2686 upper_32_bits(addr));
2687
2688 return 0;
2689 }
2690
gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2691 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2692 {
2693 uint32_t usec_timeout = 50000; /* wait for 50ms */
2694 uint32_t tmp;
2695 unsigned i, pipe_id;
2696 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2697
2698 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2699 adev->gfx.pfp_fw->data;
2700
2701 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2702 lower_32_bits(addr));
2703 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2704 upper_32_bits(addr));
2705
2706 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2707 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2708 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2709 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2710 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2711
2712 /*
2713 * Programming any of the CP_PFP_IC_BASE registers
2714 * forces invalidation of the ME L1 I$. Wait for the
2715 * invalidation complete
2716 */
2717 for (i = 0; i < usec_timeout; i++) {
2718 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2719 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2720 INVALIDATE_CACHE_COMPLETE))
2721 break;
2722 udelay(1);
2723 }
2724
2725 if (i >= usec_timeout) {
2726 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2727 return -EINVAL;
2728 }
2729
2730 /* Prime the L1 instruction caches */
2731 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2732 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2733 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2734 /* Waiting for cache primed*/
2735 for (i = 0; i < usec_timeout; i++) {
2736 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2737 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2738 ICACHE_PRIMED))
2739 break;
2740 udelay(1);
2741 }
2742
2743 if (i >= usec_timeout) {
2744 dev_err(adev->dev, "failed to prime instruction cache\n");
2745 return -EINVAL;
2746 }
2747
2748 mutex_lock(&adev->srbm_mutex);
2749 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2750 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2751 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2752 (pfp_hdr->ucode_start_addr_hi << 30) |
2753 (pfp_hdr->ucode_start_addr_lo >> 2));
2754 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2755 pfp_hdr->ucode_start_addr_hi >> 2);
2756
2757 /*
2758 * Program CP_ME_CNTL to reset given PIPE to take
2759 * effect of CP_PFP_PRGRM_CNTR_START.
2760 */
2761 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2762 if (pipe_id == 0)
2763 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2764 PFP_PIPE0_RESET, 1);
2765 else
2766 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2767 PFP_PIPE1_RESET, 1);
2768 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2769
2770 /* Clear pfp pipe0 reset bit. */
2771 if (pipe_id == 0)
2772 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2773 PFP_PIPE0_RESET, 0);
2774 else
2775 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2776 PFP_PIPE1_RESET, 0);
2777 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2778
2779 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2780 lower_32_bits(addr2));
2781 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2782 upper_32_bits(addr2));
2783 }
2784 soc21_grbm_select(adev, 0, 0, 0, 0);
2785 mutex_unlock(&adev->srbm_mutex);
2786
2787 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2788 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2789 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2790 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2791
2792 /* Invalidate the data caches */
2793 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2794 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2795 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2796
2797 for (i = 0; i < usec_timeout; i++) {
2798 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2799 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2800 INVALIDATE_DCACHE_COMPLETE))
2801 break;
2802 udelay(1);
2803 }
2804
2805 if (i >= usec_timeout) {
2806 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2807 return -EINVAL;
2808 }
2809
2810 return 0;
2811 }
2812
gfx_v11_0_config_me_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2813 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2814 {
2815 uint32_t usec_timeout = 50000; /* wait for 50ms */
2816 uint32_t tmp;
2817 unsigned i, pipe_id;
2818 const struct gfx_firmware_header_v2_0 *me_hdr;
2819
2820 me_hdr = (const struct gfx_firmware_header_v2_0 *)
2821 adev->gfx.me_fw->data;
2822
2823 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2824 lower_32_bits(addr));
2825 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2826 upper_32_bits(addr));
2827
2828 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2829 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2830 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2831 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2832 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2833
2834 /*
2835 * Programming any of the CP_ME_IC_BASE registers
2836 * forces invalidation of the ME L1 I$. Wait for the
2837 * invalidation complete
2838 */
2839 for (i = 0; i < usec_timeout; i++) {
2840 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2841 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2842 INVALIDATE_CACHE_COMPLETE))
2843 break;
2844 udelay(1);
2845 }
2846
2847 if (i >= usec_timeout) {
2848 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2849 return -EINVAL;
2850 }
2851
2852 /* Prime the instruction caches */
2853 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2854 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2855 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2856
2857 /* Waiting for instruction cache primed*/
2858 for (i = 0; i < usec_timeout; i++) {
2859 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2860 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2861 ICACHE_PRIMED))
2862 break;
2863 udelay(1);
2864 }
2865
2866 if (i >= usec_timeout) {
2867 dev_err(adev->dev, "failed to prime instruction cache\n");
2868 return -EINVAL;
2869 }
2870
2871 mutex_lock(&adev->srbm_mutex);
2872 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2873 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2874 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2875 (me_hdr->ucode_start_addr_hi << 30) |
2876 (me_hdr->ucode_start_addr_lo >> 2) );
2877 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2878 me_hdr->ucode_start_addr_hi>>2);
2879
2880 /*
2881 * Program CP_ME_CNTL to reset given PIPE to take
2882 * effect of CP_PFP_PRGRM_CNTR_START.
2883 */
2884 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2885 if (pipe_id == 0)
2886 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2887 ME_PIPE0_RESET, 1);
2888 else
2889 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2890 ME_PIPE1_RESET, 1);
2891 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2892
2893 /* Clear pfp pipe0 reset bit. */
2894 if (pipe_id == 0)
2895 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2896 ME_PIPE0_RESET, 0);
2897 else
2898 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2899 ME_PIPE1_RESET, 0);
2900 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2901
2902 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2903 lower_32_bits(addr2));
2904 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2905 upper_32_bits(addr2));
2906 }
2907 soc21_grbm_select(adev, 0, 0, 0, 0);
2908 mutex_unlock(&adev->srbm_mutex);
2909
2910 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2911 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2912 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2913 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2914
2915 /* Invalidate the data caches */
2916 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2917 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2918 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2919
2920 for (i = 0; i < usec_timeout; i++) {
2921 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2922 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2923 INVALIDATE_DCACHE_COMPLETE))
2924 break;
2925 udelay(1);
2926 }
2927
2928 if (i >= usec_timeout) {
2929 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2930 return -EINVAL;
2931 }
2932
2933 return 0;
2934 }
2935
gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2936 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2937 {
2938 uint32_t usec_timeout = 50000; /* wait for 50ms */
2939 uint32_t tmp;
2940 unsigned i;
2941 const struct gfx_firmware_header_v2_0 *mec_hdr;
2942
2943 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2944 adev->gfx.mec_fw->data;
2945
2946 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2947 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2948 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2949 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2950 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2951
2952 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2953 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2954 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2955 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2956
2957 mutex_lock(&adev->srbm_mutex);
2958 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2959 soc21_grbm_select(adev, 1, i, 0, 0);
2960
2961 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2962 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2963 upper_32_bits(addr2));
2964
2965 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2966 mec_hdr->ucode_start_addr_lo >> 2 |
2967 mec_hdr->ucode_start_addr_hi << 30);
2968 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2969 mec_hdr->ucode_start_addr_hi >> 2);
2970
2971 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2972 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2973 upper_32_bits(addr));
2974 }
2975 mutex_unlock(&adev->srbm_mutex);
2976 soc21_grbm_select(adev, 0, 0, 0, 0);
2977
2978 /* Trigger an invalidation of the L1 instruction caches */
2979 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2980 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2981 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2982
2983 /* Wait for invalidation complete */
2984 for (i = 0; i < usec_timeout; i++) {
2985 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2986 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2987 INVALIDATE_DCACHE_COMPLETE))
2988 break;
2989 udelay(1);
2990 }
2991
2992 if (i >= usec_timeout) {
2993 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2994 return -EINVAL;
2995 }
2996
2997 /* Trigger an invalidation of the L1 instruction caches */
2998 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2999 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3000 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3001
3002 /* Wait for invalidation complete */
3003 for (i = 0; i < usec_timeout; i++) {
3004 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3005 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3006 INVALIDATE_CACHE_COMPLETE))
3007 break;
3008 udelay(1);
3009 }
3010
3011 if (i >= usec_timeout) {
3012 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3013 return -EINVAL;
3014 }
3015
3016 return 0;
3017 }
3018
gfx_v11_0_config_gfx_rs64(struct amdgpu_device * adev)3019 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
3020 {
3021 const struct gfx_firmware_header_v2_0 *pfp_hdr;
3022 const struct gfx_firmware_header_v2_0 *me_hdr;
3023 const struct gfx_firmware_header_v2_0 *mec_hdr;
3024 uint32_t pipe_id, tmp;
3025
3026 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
3027 adev->gfx.mec_fw->data;
3028 me_hdr = (const struct gfx_firmware_header_v2_0 *)
3029 adev->gfx.me_fw->data;
3030 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
3031 adev->gfx.pfp_fw->data;
3032
3033 /* config pfp program start addr */
3034 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
3035 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3036 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
3037 (pfp_hdr->ucode_start_addr_hi << 30) |
3038 (pfp_hdr->ucode_start_addr_lo >> 2));
3039 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3040 pfp_hdr->ucode_start_addr_hi >> 2);
3041 }
3042 soc21_grbm_select(adev, 0, 0, 0, 0);
3043
3044 /* reset pfp pipe */
3045 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3046 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
3047 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
3048 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3049
3050 /* clear pfp pipe reset */
3051 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
3052 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
3053 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3054
3055 /* config me program start addr */
3056 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
3057 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3058 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3059 (me_hdr->ucode_start_addr_hi << 30) |
3060 (me_hdr->ucode_start_addr_lo >> 2) );
3061 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3062 me_hdr->ucode_start_addr_hi>>2);
3063 }
3064 soc21_grbm_select(adev, 0, 0, 0, 0);
3065
3066 /* reset me pipe */
3067 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3068 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
3069 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
3070 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3071
3072 /* clear me pipe reset */
3073 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
3074 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
3075 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3076
3077 /* config mec program start addr */
3078 for (pipe_id = 0; pipe_id < 4; pipe_id++) {
3079 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
3080 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3081 mec_hdr->ucode_start_addr_lo >> 2 |
3082 mec_hdr->ucode_start_addr_hi << 30);
3083 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3084 mec_hdr->ucode_start_addr_hi >> 2);
3085 }
3086 soc21_grbm_select(adev, 0, 0, 0, 0);
3087
3088 /* reset mec pipe */
3089 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3090 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
3091 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
3092 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
3093 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
3094 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
3095
3096 /* clear mec pipe reset */
3097 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
3098 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
3099 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
3100 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
3101 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
3102 }
3103
gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device * adev)3104 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
3105 {
3106 uint32_t cp_status;
3107 uint32_t bootload_status;
3108 int i, r;
3109 uint64_t addr, addr2;
3110
3111 for (i = 0; i < adev->usec_timeout; i++) {
3112 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
3113
3114 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3115 IP_VERSION(11, 0, 1) ||
3116 amdgpu_ip_version(adev, GC_HWIP, 0) ==
3117 IP_VERSION(11, 0, 4) ||
3118 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) ||
3119 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) ||
3120 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) ||
3121 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3) ||
3122 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4) ||
3123 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6) ||
3124 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 0) ||
3125 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 1))
3126 bootload_status = RREG32_SOC15(GC, 0,
3127 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
3128 else
3129 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
3130
3131 if ((cp_status == 0) &&
3132 (REG_GET_FIELD(bootload_status,
3133 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
3134 break;
3135 }
3136 udelay(1);
3137 }
3138
3139 if (i >= adev->usec_timeout) {
3140 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
3141 return -ETIMEDOUT;
3142 }
3143
3144 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
3145 if (adev->gfx.rs64_enable) {
3146 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3147 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
3148 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
3149 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
3150 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
3151 if (r)
3152 return r;
3153 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3154 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
3155 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
3156 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
3157 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
3158 if (r)
3159 return r;
3160 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3161 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
3162 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
3163 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
3164 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
3165 if (r)
3166 return r;
3167 } else {
3168 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3169 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
3170 r = gfx_v11_0_config_me_cache(adev, addr);
3171 if (r)
3172 return r;
3173 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3174 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
3175 r = gfx_v11_0_config_pfp_cache(adev, addr);
3176 if (r)
3177 return r;
3178 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3179 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
3180 r = gfx_v11_0_config_mec_cache(adev, addr);
3181 if (r)
3182 return r;
3183 }
3184 }
3185
3186 return 0;
3187 }
3188
gfx_v11_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)3189 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3190 {
3191 int i;
3192 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3193
3194 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3195 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3196 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3197
3198 for (i = 0; i < adev->usec_timeout; i++) {
3199 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
3200 break;
3201 udelay(1);
3202 }
3203
3204 if (i >= adev->usec_timeout)
3205 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
3206
3207 return 0;
3208 }
3209
gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device * adev)3210 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
3211 {
3212 int r;
3213 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3214 const __le32 *fw_data;
3215 unsigned i, fw_size;
3216
3217 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3218 adev->gfx.pfp_fw->data;
3219
3220 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3221
3222 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
3223 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3224 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
3225
3226 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
3227 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3228 &adev->gfx.pfp.pfp_fw_obj,
3229 &adev->gfx.pfp.pfp_fw_gpu_addr,
3230 (void **)&adev->gfx.pfp.pfp_fw_ptr);
3231 if (r) {
3232 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
3233 gfx_v11_0_pfp_fini(adev);
3234 return r;
3235 }
3236
3237 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
3238
3239 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3240 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3241
3242 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
3243
3244 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
3245
3246 for (i = 0; i < pfp_hdr->jt_size; i++)
3247 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
3248 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
3249
3250 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3251
3252 return 0;
3253 }
3254
gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device * adev)3255 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
3256 {
3257 int r;
3258 const struct gfx_firmware_header_v2_0 *pfp_hdr;
3259 const __le32 *fw_ucode, *fw_data;
3260 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3261 uint32_t tmp;
3262 uint32_t usec_timeout = 50000; /* wait for 50ms */
3263
3264 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
3265 adev->gfx.pfp_fw->data;
3266
3267 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3268
3269 /* instruction */
3270 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
3271 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
3272 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
3273 /* data */
3274 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
3275 le32_to_cpu(pfp_hdr->data_offset_bytes));
3276 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
3277
3278 /* 64kb align */
3279 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3280 64 * 1024,
3281 AMDGPU_GEM_DOMAIN_VRAM |
3282 AMDGPU_GEM_DOMAIN_GTT,
3283 &adev->gfx.pfp.pfp_fw_obj,
3284 &adev->gfx.pfp.pfp_fw_gpu_addr,
3285 (void **)&adev->gfx.pfp.pfp_fw_ptr);
3286 if (r) {
3287 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
3288 gfx_v11_0_pfp_fini(adev);
3289 return r;
3290 }
3291
3292 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3293 64 * 1024,
3294 AMDGPU_GEM_DOMAIN_VRAM |
3295 AMDGPU_GEM_DOMAIN_GTT,
3296 &adev->gfx.pfp.pfp_fw_data_obj,
3297 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
3298 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
3299 if (r) {
3300 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
3301 gfx_v11_0_pfp_fini(adev);
3302 return r;
3303 }
3304
3305 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
3306 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
3307
3308 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3309 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
3310 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3311 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
3312
3313 if (amdgpu_emu_mode == 1)
3314 amdgpu_device_flush_hdp(adev, NULL);
3315
3316 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
3317 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3318 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
3319 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3320
3321 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
3322 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
3323 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
3324 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
3325 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
3326
3327 /*
3328 * Programming any of the CP_PFP_IC_BASE registers
3329 * forces invalidation of the ME L1 I$. Wait for the
3330 * invalidation complete
3331 */
3332 for (i = 0; i < usec_timeout; i++) {
3333 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3334 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3335 INVALIDATE_CACHE_COMPLETE))
3336 break;
3337 udelay(1);
3338 }
3339
3340 if (i >= usec_timeout) {
3341 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3342 return -EINVAL;
3343 }
3344
3345 /* Prime the L1 instruction caches */
3346 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3347 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
3348 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
3349 /* Waiting for cache primed*/
3350 for (i = 0; i < usec_timeout; i++) {
3351 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3352 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3353 ICACHE_PRIMED))
3354 break;
3355 udelay(1);
3356 }
3357
3358 if (i >= usec_timeout) {
3359 dev_err(adev->dev, "failed to prime instruction cache\n");
3360 return -EINVAL;
3361 }
3362
3363 mutex_lock(&adev->srbm_mutex);
3364 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3365 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3366 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
3367 (pfp_hdr->ucode_start_addr_hi << 30) |
3368 (pfp_hdr->ucode_start_addr_lo >> 2) );
3369 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3370 pfp_hdr->ucode_start_addr_hi>>2);
3371
3372 /*
3373 * Program CP_ME_CNTL to reset given PIPE to take
3374 * effect of CP_PFP_PRGRM_CNTR_START.
3375 */
3376 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3377 if (pipe_id == 0)
3378 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3379 PFP_PIPE0_RESET, 1);
3380 else
3381 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3382 PFP_PIPE1_RESET, 1);
3383 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3384
3385 /* Clear pfp pipe0 reset bit. */
3386 if (pipe_id == 0)
3387 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3388 PFP_PIPE0_RESET, 0);
3389 else
3390 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3391 PFP_PIPE1_RESET, 0);
3392 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3393
3394 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
3395 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3396 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
3397 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3398 }
3399 soc21_grbm_select(adev, 0, 0, 0, 0);
3400 mutex_unlock(&adev->srbm_mutex);
3401
3402 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3403 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3404 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3405 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3406
3407 /* Invalidate the data caches */
3408 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3409 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3410 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3411
3412 for (i = 0; i < usec_timeout; i++) {
3413 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3414 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3415 INVALIDATE_DCACHE_COMPLETE))
3416 break;
3417 udelay(1);
3418 }
3419
3420 if (i >= usec_timeout) {
3421 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3422 return -EINVAL;
3423 }
3424
3425 return 0;
3426 }
3427
gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device * adev)3428 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
3429 {
3430 int r;
3431 const struct gfx_firmware_header_v1_0 *me_hdr;
3432 const __le32 *fw_data;
3433 unsigned i, fw_size;
3434
3435 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3436 adev->gfx.me_fw->data;
3437
3438 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3439
3440 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3441 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3442 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
3443
3444 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
3445 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3446 &adev->gfx.me.me_fw_obj,
3447 &adev->gfx.me.me_fw_gpu_addr,
3448 (void **)&adev->gfx.me.me_fw_ptr);
3449 if (r) {
3450 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
3451 gfx_v11_0_me_fini(adev);
3452 return r;
3453 }
3454
3455 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
3456
3457 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3458 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3459
3460 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
3461
3462 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
3463
3464 for (i = 0; i < me_hdr->jt_size; i++)
3465 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
3466 le32_to_cpup(fw_data + me_hdr->jt_offset + i));
3467
3468 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
3469
3470 return 0;
3471 }
3472
gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device * adev)3473 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
3474 {
3475 int r;
3476 const struct gfx_firmware_header_v2_0 *me_hdr;
3477 const __le32 *fw_ucode, *fw_data;
3478 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3479 uint32_t tmp;
3480 uint32_t usec_timeout = 50000; /* wait for 50ms */
3481
3482 me_hdr = (const struct gfx_firmware_header_v2_0 *)
3483 adev->gfx.me_fw->data;
3484
3485 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3486
3487 /* instruction */
3488 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
3489 le32_to_cpu(me_hdr->ucode_offset_bytes));
3490 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
3491 /* data */
3492 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3493 le32_to_cpu(me_hdr->data_offset_bytes));
3494 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
3495
3496 /* 64kb align*/
3497 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3498 64 * 1024,
3499 AMDGPU_GEM_DOMAIN_VRAM |
3500 AMDGPU_GEM_DOMAIN_GTT,
3501 &adev->gfx.me.me_fw_obj,
3502 &adev->gfx.me.me_fw_gpu_addr,
3503 (void **)&adev->gfx.me.me_fw_ptr);
3504 if (r) {
3505 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
3506 gfx_v11_0_me_fini(adev);
3507 return r;
3508 }
3509
3510 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3511 64 * 1024,
3512 AMDGPU_GEM_DOMAIN_VRAM |
3513 AMDGPU_GEM_DOMAIN_GTT,
3514 &adev->gfx.me.me_fw_data_obj,
3515 &adev->gfx.me.me_fw_data_gpu_addr,
3516 (void **)&adev->gfx.me.me_fw_data_ptr);
3517 if (r) {
3518 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
3519 gfx_v11_0_pfp_fini(adev);
3520 return r;
3521 }
3522
3523 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
3524 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
3525
3526 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3527 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
3528 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3529 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3530
3531 if (amdgpu_emu_mode == 1)
3532 amdgpu_device_flush_hdp(adev, NULL);
3533
3534 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3535 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3536 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3537 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3538
3539 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3540 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3541 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3542 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3543 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3544
3545 /*
3546 * Programming any of the CP_ME_IC_BASE registers
3547 * forces invalidation of the ME L1 I$. Wait for the
3548 * invalidation complete
3549 */
3550 for (i = 0; i < usec_timeout; i++) {
3551 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3552 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3553 INVALIDATE_CACHE_COMPLETE))
3554 break;
3555 udelay(1);
3556 }
3557
3558 if (i >= usec_timeout) {
3559 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3560 return -EINVAL;
3561 }
3562
3563 /* Prime the instruction caches */
3564 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3565 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3566 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3567
3568 /* Waiting for instruction cache primed*/
3569 for (i = 0; i < usec_timeout; i++) {
3570 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3571 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3572 ICACHE_PRIMED))
3573 break;
3574 udelay(1);
3575 }
3576
3577 if (i >= usec_timeout) {
3578 dev_err(adev->dev, "failed to prime instruction cache\n");
3579 return -EINVAL;
3580 }
3581
3582 mutex_lock(&adev->srbm_mutex);
3583 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3584 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3585 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3586 (me_hdr->ucode_start_addr_hi << 30) |
3587 (me_hdr->ucode_start_addr_lo >> 2) );
3588 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3589 me_hdr->ucode_start_addr_hi>>2);
3590
3591 /*
3592 * Program CP_ME_CNTL to reset given PIPE to take
3593 * effect of CP_PFP_PRGRM_CNTR_START.
3594 */
3595 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3596 if (pipe_id == 0)
3597 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3598 ME_PIPE0_RESET, 1);
3599 else
3600 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3601 ME_PIPE1_RESET, 1);
3602 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3603
3604 /* Clear pfp pipe0 reset bit. */
3605 if (pipe_id == 0)
3606 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3607 ME_PIPE0_RESET, 0);
3608 else
3609 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3610 ME_PIPE1_RESET, 0);
3611 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3612
3613 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3614 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3615 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3616 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3617 }
3618 soc21_grbm_select(adev, 0, 0, 0, 0);
3619 mutex_unlock(&adev->srbm_mutex);
3620
3621 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3622 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3623 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3624 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3625
3626 /* Invalidate the data caches */
3627 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3628 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3629 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3630
3631 for (i = 0; i < usec_timeout; i++) {
3632 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3633 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3634 INVALIDATE_DCACHE_COMPLETE))
3635 break;
3636 udelay(1);
3637 }
3638
3639 if (i >= usec_timeout) {
3640 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3641 return -EINVAL;
3642 }
3643
3644 return 0;
3645 }
3646
gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3647 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3648 {
3649 int r;
3650
3651 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3652 return -EINVAL;
3653
3654 gfx_v11_0_cp_gfx_enable(adev, false);
3655
3656 if (adev->gfx.rs64_enable)
3657 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3658 else
3659 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3660 if (r) {
3661 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3662 return r;
3663 }
3664
3665 if (adev->gfx.rs64_enable)
3666 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3667 else
3668 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3669 if (r) {
3670 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3671 return r;
3672 }
3673
3674 return 0;
3675 }
3676
gfx_v11_0_cp_gfx_start(struct amdgpu_device * adev)3677 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3678 {
3679 struct amdgpu_ring *ring;
3680 const struct cs_section_def *sect = NULL;
3681 const struct cs_extent_def *ext = NULL;
3682 int r, i;
3683 int ctx_reg_offset;
3684
3685 /* init the CP */
3686 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3687 adev->gfx.config.max_hw_contexts - 1);
3688 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3689
3690 if (!amdgpu_async_gfx_ring)
3691 gfx_v11_0_cp_gfx_enable(adev, true);
3692
3693 ring = &adev->gfx.gfx_ring[0];
3694 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3695 if (r) {
3696 drm_err(&adev->ddev, "cp failed to lock ring (%d).\n", r);
3697 return r;
3698 }
3699
3700 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3701 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3702
3703 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3704 amdgpu_ring_write(ring, 0x80000000);
3705 amdgpu_ring_write(ring, 0x80000000);
3706
3707 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3708 for (ext = sect->section; ext->extent != NULL; ++ext) {
3709 if (sect->id == SECT_CONTEXT) {
3710 amdgpu_ring_write(ring,
3711 PACKET3(PACKET3_SET_CONTEXT_REG,
3712 ext->reg_count));
3713 amdgpu_ring_write(ring, ext->reg_index -
3714 PACKET3_SET_CONTEXT_REG_START);
3715 for (i = 0; i < ext->reg_count; i++)
3716 amdgpu_ring_write(ring, ext->extent[i]);
3717 }
3718 }
3719 }
3720
3721 ctx_reg_offset =
3722 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3723 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3724 amdgpu_ring_write(ring, ctx_reg_offset);
3725 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3726
3727 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3728 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3729
3730 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3731 amdgpu_ring_write(ring, 0);
3732
3733 amdgpu_ring_commit(ring);
3734
3735 /* submit cs packet to copy state 0 to next available state */
3736 if (adev->gfx.num_gfx_rings > 1) {
3737 /* maximum supported gfx ring is 2 */
3738 ring = &adev->gfx.gfx_ring[1];
3739 r = amdgpu_ring_alloc(ring, 2);
3740 if (r) {
3741 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r);
3742 return r;
3743 }
3744
3745 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3746 amdgpu_ring_write(ring, 0);
3747
3748 amdgpu_ring_commit(ring);
3749 }
3750 return 0;
3751 }
3752
gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device * adev,CP_PIPE_ID pipe)3753 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3754 CP_PIPE_ID pipe)
3755 {
3756 u32 tmp;
3757
3758 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3759 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3760
3761 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3762 }
3763
gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device * adev,struct amdgpu_ring * ring)3764 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3765 struct amdgpu_ring *ring)
3766 {
3767 u32 tmp;
3768
3769 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3770 if (ring->use_doorbell) {
3771 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3772 DOORBELL_OFFSET, ring->doorbell_index);
3773 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3774 DOORBELL_EN, 1);
3775 } else {
3776 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3777 DOORBELL_EN, 0);
3778 }
3779 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3780
3781 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3782 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3783 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3784
3785 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3786 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3787 }
3788
gfx_v11_0_cp_gfx_resume(struct amdgpu_device * adev)3789 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3790 {
3791 struct amdgpu_ring *ring;
3792 u32 tmp;
3793 u32 rb_bufsz;
3794 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3795
3796 /* Set the write pointer delay */
3797 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3798
3799 /* set the RB to use vmid 0 */
3800 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3801
3802 /* Init gfx ring 0 for pipe 0 */
3803 mutex_lock(&adev->srbm_mutex);
3804 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3805
3806 /* Set ring buffer size */
3807 ring = &adev->gfx.gfx_ring[0];
3808 rb_bufsz = order_base_2(ring->ring_size / 8);
3809 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3810 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3811 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3812
3813 /* Initialize the ring buffer's write pointers */
3814 ring->wptr = 0;
3815 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3816 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3817
3818 /* set the wb address whether it's enabled or not */
3819 rptr_addr = ring->rptr_gpu_addr;
3820 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3821 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3822 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3823
3824 wptr_gpu_addr = ring->wptr_gpu_addr;
3825 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3826 lower_32_bits(wptr_gpu_addr));
3827 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3828 upper_32_bits(wptr_gpu_addr));
3829
3830 mdelay(1);
3831 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3832
3833 rb_addr = ring->gpu_addr >> 8;
3834 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3835 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3836
3837 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3838
3839 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3840 mutex_unlock(&adev->srbm_mutex);
3841
3842 /* Init gfx ring 1 for pipe 1 */
3843 if (adev->gfx.num_gfx_rings > 1) {
3844 mutex_lock(&adev->srbm_mutex);
3845 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3846 /* maximum supported gfx ring is 2 */
3847 ring = &adev->gfx.gfx_ring[1];
3848 rb_bufsz = order_base_2(ring->ring_size / 8);
3849 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3850 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3851 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3852 /* Initialize the ring buffer's write pointers */
3853 ring->wptr = 0;
3854 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3855 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3856 /* Set the wb address whether it's enabled or not */
3857 rptr_addr = ring->rptr_gpu_addr;
3858 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3859 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3860 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3861 wptr_gpu_addr = ring->wptr_gpu_addr;
3862 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3863 lower_32_bits(wptr_gpu_addr));
3864 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3865 upper_32_bits(wptr_gpu_addr));
3866
3867 mdelay(1);
3868 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3869
3870 rb_addr = ring->gpu_addr >> 8;
3871 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3872 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3873 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3874
3875 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3876 mutex_unlock(&adev->srbm_mutex);
3877 }
3878 /* Switch to pipe 0 */
3879 mutex_lock(&adev->srbm_mutex);
3880 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3881 mutex_unlock(&adev->srbm_mutex);
3882
3883 /* start the ring */
3884 gfx_v11_0_cp_gfx_start(adev);
3885
3886 return 0;
3887 }
3888
gfx_v11_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3889 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3890 {
3891 u32 data;
3892
3893 if (adev->gfx.rs64_enable) {
3894 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3895 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3896 enable ? 0 : 1);
3897 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3898 enable ? 0 : 1);
3899 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3900 enable ? 0 : 1);
3901 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3902 enable ? 0 : 1);
3903 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3904 enable ? 0 : 1);
3905 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3906 enable ? 1 : 0);
3907 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3908 enable ? 1 : 0);
3909 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3910 enable ? 1 : 0);
3911 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3912 enable ? 1 : 0);
3913 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3914 enable ? 0 : 1);
3915 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3916 } else {
3917 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3918
3919 if (enable) {
3920 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3921 if (!adev->enable_mes_kiq)
3922 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3923 MEC_ME2_HALT, 0);
3924 } else {
3925 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3926 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3927 }
3928 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3929 }
3930
3931 udelay(50);
3932 }
3933
gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device * adev)3934 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3935 {
3936 const struct gfx_firmware_header_v1_0 *mec_hdr;
3937 const __le32 *fw_data;
3938 unsigned i, fw_size;
3939 u32 *fw = NULL;
3940 int r;
3941
3942 if (!adev->gfx.mec_fw)
3943 return -EINVAL;
3944
3945 gfx_v11_0_cp_compute_enable(adev, false);
3946
3947 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3948 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3949
3950 fw_data = (const __le32 *)
3951 (adev->gfx.mec_fw->data +
3952 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3953 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3954
3955 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3956 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3957 &adev->gfx.mec.mec_fw_obj,
3958 &adev->gfx.mec.mec_fw_gpu_addr,
3959 (void **)&fw);
3960 if (r) {
3961 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3962 gfx_v11_0_mec_fini(adev);
3963 return r;
3964 }
3965
3966 memcpy(fw, fw_data, fw_size);
3967
3968 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3969 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3970
3971 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3972
3973 /* MEC1 */
3974 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3975
3976 for (i = 0; i < mec_hdr->jt_size; i++)
3977 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3978 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3979
3980 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3981
3982 return 0;
3983 }
3984
gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device * adev)3985 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3986 {
3987 const struct gfx_firmware_header_v2_0 *mec_hdr;
3988 const __le32 *fw_ucode, *fw_data;
3989 u32 tmp, fw_ucode_size, fw_data_size;
3990 u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3991 u32 *fw_ucode_ptr, *fw_data_ptr;
3992 int r;
3993
3994 if (!adev->gfx.mec_fw)
3995 return -EINVAL;
3996
3997 gfx_v11_0_cp_compute_enable(adev, false);
3998
3999 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
4000 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
4001
4002 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
4003 le32_to_cpu(mec_hdr->ucode_offset_bytes));
4004 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
4005
4006 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
4007 le32_to_cpu(mec_hdr->data_offset_bytes));
4008 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
4009
4010 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
4011 64 * 1024,
4012 AMDGPU_GEM_DOMAIN_VRAM |
4013 AMDGPU_GEM_DOMAIN_GTT,
4014 &adev->gfx.mec.mec_fw_obj,
4015 &adev->gfx.mec.mec_fw_gpu_addr,
4016 (void **)&fw_ucode_ptr);
4017 if (r) {
4018 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
4019 gfx_v11_0_mec_fini(adev);
4020 return r;
4021 }
4022
4023 r = amdgpu_bo_create_reserved(adev, fw_data_size,
4024 64 * 1024,
4025 AMDGPU_GEM_DOMAIN_VRAM |
4026 AMDGPU_GEM_DOMAIN_GTT,
4027 &adev->gfx.mec.mec_fw_data_obj,
4028 &adev->gfx.mec.mec_fw_data_gpu_addr,
4029 (void **)&fw_data_ptr);
4030 if (r) {
4031 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
4032 gfx_v11_0_mec_fini(adev);
4033 return r;
4034 }
4035
4036 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
4037 memcpy(fw_data_ptr, fw_data, fw_data_size);
4038
4039 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
4040 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
4041 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
4042 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
4043
4044 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
4045 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
4046 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
4047 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
4048 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
4049
4050 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
4051 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
4052 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
4053 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
4054
4055 mutex_lock(&adev->srbm_mutex);
4056 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
4057 soc21_grbm_select(adev, 1, i, 0, 0);
4058
4059 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
4060 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
4061 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
4062
4063 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
4064 mec_hdr->ucode_start_addr_lo >> 2 |
4065 mec_hdr->ucode_start_addr_hi << 30);
4066 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
4067 mec_hdr->ucode_start_addr_hi >> 2);
4068
4069 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
4070 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
4071 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
4072 }
4073 mutex_unlock(&adev->srbm_mutex);
4074 soc21_grbm_select(adev, 0, 0, 0, 0);
4075
4076 /* Trigger an invalidation of the L1 instruction caches */
4077 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
4078 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
4079 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
4080
4081 /* Wait for invalidation complete */
4082 for (i = 0; i < usec_timeout; i++) {
4083 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
4084 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
4085 INVALIDATE_DCACHE_COMPLETE))
4086 break;
4087 udelay(1);
4088 }
4089
4090 if (i >= usec_timeout) {
4091 dev_err(adev->dev, "failed to invalidate instruction cache\n");
4092 return -EINVAL;
4093 }
4094
4095 /* Trigger an invalidation of the L1 instruction caches */
4096 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
4097 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
4098 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
4099
4100 /* Wait for invalidation complete */
4101 for (i = 0; i < usec_timeout; i++) {
4102 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
4103 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
4104 INVALIDATE_CACHE_COMPLETE))
4105 break;
4106 udelay(1);
4107 }
4108
4109 if (i >= usec_timeout) {
4110 dev_err(adev->dev, "failed to invalidate instruction cache\n");
4111 return -EINVAL;
4112 }
4113
4114 return 0;
4115 }
4116
gfx_v11_0_kiq_setting(struct amdgpu_ring * ring)4117 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
4118 {
4119 uint32_t tmp;
4120 struct amdgpu_device *adev = ring->adev;
4121
4122 /* tell RLC which is KIQ queue */
4123 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
4124 tmp &= 0xffffff00;
4125 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4126 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80);
4127 }
4128
gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device * adev)4129 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
4130 {
4131 /* set graphics engine doorbell range */
4132 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
4133 (adev->doorbell_index.gfx_ring0 * 2) << 2);
4134 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
4135 (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
4136
4137 /* set compute engine doorbell range */
4138 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4139 (adev->doorbell_index.kiq * 2) << 2);
4140 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4141 (adev->doorbell_index.userqueue_end * 2) << 2);
4142 }
4143
gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device * adev,struct v11_gfx_mqd * mqd,struct amdgpu_mqd_prop * prop)4144 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev,
4145 struct v11_gfx_mqd *mqd,
4146 struct amdgpu_mqd_prop *prop)
4147 {
4148 bool priority = 0;
4149 u32 tmp;
4150
4151 /* set up default queue priority level
4152 * 0x0 = low priority, 0x1 = high priority
4153 */
4154 if (prop->hqd_queue_priority == AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM)
4155 priority = 1;
4156
4157 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT;
4158 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority);
4159 mqd->cp_gfx_hqd_queue_priority = tmp;
4160 }
4161
gfx_v11_0_gfx_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)4162 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
4163 struct amdgpu_mqd_prop *prop)
4164 {
4165 struct v11_gfx_mqd *mqd = m;
4166 uint64_t hqd_gpu_addr, wb_gpu_addr;
4167 uint32_t tmp;
4168 uint32_t rb_bufsz;
4169
4170 /* set up gfx hqd wptr */
4171 mqd->cp_gfx_hqd_wptr = 0;
4172 mqd->cp_gfx_hqd_wptr_hi = 0;
4173
4174 /* set the pointer to the MQD */
4175 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
4176 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4177
4178 /* set up mqd control */
4179 tmp = regCP_GFX_MQD_CONTROL_DEFAULT;
4180 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
4181 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
4182 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
4183 mqd->cp_gfx_mqd_control = tmp;
4184
4185 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
4186 tmp = regCP_GFX_HQD_VMID_DEFAULT;
4187 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
4188 mqd->cp_gfx_hqd_vmid = 0;
4189
4190 /* set up gfx queue priority */
4191 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop);
4192
4193 /* set up time quantum */
4194 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT;
4195 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
4196 mqd->cp_gfx_hqd_quantum = tmp;
4197
4198 /* set up gfx hqd base. this is similar as CP_RB_BASE */
4199 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4200 mqd->cp_gfx_hqd_base = hqd_gpu_addr;
4201 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
4202
4203 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
4204 wb_gpu_addr = prop->rptr_gpu_addr;
4205 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
4206 mqd->cp_gfx_hqd_rptr_addr_hi =
4207 upper_32_bits(wb_gpu_addr) & 0xffff;
4208
4209 /* set up rb_wptr_poll addr */
4210 wb_gpu_addr = prop->wptr_gpu_addr;
4211 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4212 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4213
4214 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
4215 rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
4216 tmp = regCP_GFX_HQD_CNTL_DEFAULT;
4217 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
4218 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
4219 #ifdef __BIG_ENDIAN
4220 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
4221 #endif
4222 if (prop->tmz_queue)
4223 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, TMZ_MATCH, 1);
4224 if (!prop->kernel_queue)
4225 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_NON_PRIV, 1);
4226 mqd->cp_gfx_hqd_cntl = tmp;
4227
4228 /* set up cp_doorbell_control */
4229 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT;
4230 if (prop->use_doorbell) {
4231 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4232 DOORBELL_OFFSET, prop->doorbell_index);
4233 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4234 DOORBELL_EN, 1);
4235 } else
4236 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4237 DOORBELL_EN, 0);
4238 mqd->cp_rb_doorbell_control = tmp;
4239
4240 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4241 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT;
4242
4243 /* active the queue */
4244 mqd->cp_gfx_hqd_active = 1;
4245
4246 /* set gfx UQ items */
4247 mqd->shadow_base_lo = lower_32_bits(prop->shadow_addr);
4248 mqd->shadow_base_hi = upper_32_bits(prop->shadow_addr);
4249 mqd->gds_bkup_base_lo = lower_32_bits(prop->gds_bkup_addr);
4250 mqd->gds_bkup_base_hi = upper_32_bits(prop->gds_bkup_addr);
4251 mqd->fw_work_area_base_lo = lower_32_bits(prop->csa_addr);
4252 mqd->fw_work_area_base_hi = upper_32_bits(prop->csa_addr);
4253 mqd->fence_address_lo = lower_32_bits(prop->fence_address);
4254 mqd->fence_address_hi = upper_32_bits(prop->fence_address);
4255
4256 return 0;
4257 }
4258
gfx_v11_0_kgq_init_queue(struct amdgpu_ring * ring)4259 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring)
4260 {
4261 struct amdgpu_device *adev = ring->adev;
4262 struct v11_gfx_mqd *mqd = ring->mqd_ptr;
4263 int mqd_idx = ring - &adev->gfx.gfx_ring[0];
4264
4265 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4266 memset((void *)mqd, 0, sizeof(*mqd));
4267 mutex_lock(&adev->srbm_mutex);
4268 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4269 amdgpu_ring_init_mqd(ring);
4270 soc21_grbm_select(adev, 0, 0, 0, 0);
4271 mutex_unlock(&adev->srbm_mutex);
4272 if (adev->gfx.me.mqd_backup[mqd_idx])
4273 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4274 } else {
4275 /* restore mqd with the backup copy */
4276 if (adev->gfx.me.mqd_backup[mqd_idx])
4277 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
4278 /* reset the ring */
4279 ring->wptr = 0;
4280 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4281 amdgpu_ring_clear_ring(ring);
4282 }
4283
4284 return 0;
4285 }
4286
gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device * adev)4287 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
4288 {
4289 int r, i;
4290
4291 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4292 r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i]);
4293 if (r)
4294 return r;
4295 }
4296
4297 r = amdgpu_gfx_enable_kgq(adev, 0);
4298 if (r)
4299 return r;
4300
4301 return gfx_v11_0_cp_gfx_start(adev);
4302 }
4303
gfx_v11_0_compute_mqd_set_cu_mask(struct amdgpu_device * adev,struct v11_compute_mqd * mqd,struct amdgpu_mqd_prop * prop)4304 static void gfx_v11_0_compute_mqd_set_cu_mask(struct amdgpu_device *adev,
4305 struct v11_compute_mqd *mqd,
4306 struct amdgpu_mqd_prop *prop)
4307 {
4308 uint32_t se_mask[8] = {0};
4309 uint32_t wa_mask;
4310 bool has_wa_flag = prop->cu_flags & (AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE |
4311 AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE);
4312
4313 if (!has_wa_flag && (!prop->cu_mask || !prop->cu_mask_count))
4314 return;
4315
4316 if (has_wa_flag) {
4317 wa_mask = (prop->cu_flags & AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE) ?
4318 0xffff : 0xffffffff;
4319 mqd->compute_static_thread_mgmt_se0 = wa_mask;
4320 mqd->compute_static_thread_mgmt_se1 = wa_mask;
4321 mqd->compute_static_thread_mgmt_se2 = wa_mask;
4322 mqd->compute_static_thread_mgmt_se3 = wa_mask;
4323 return;
4324 }
4325
4326 amdgpu_gfx_mqd_symmetrically_map_cu_mask(adev, prop->cu_mask,
4327 prop->cu_mask_count, se_mask);
4328
4329 mqd->compute_static_thread_mgmt_se0 = se_mask[0];
4330 mqd->compute_static_thread_mgmt_se1 = se_mask[1];
4331 mqd->compute_static_thread_mgmt_se2 = se_mask[2];
4332 mqd->compute_static_thread_mgmt_se3 = se_mask[3];
4333 }
4334
gfx_v11_0_compute_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)4335 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
4336 struct amdgpu_mqd_prop *prop)
4337 {
4338 struct v11_compute_mqd *mqd = m;
4339 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4340 uint32_t tmp;
4341
4342 mqd->header = 0xC0310800;
4343 mqd->compute_pipelinestat_enable = 0x00000001;
4344 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4345 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4346 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4347 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4348 mqd->compute_misc_reserved = 0x00000007;
4349
4350 eop_base_addr = prop->eop_gpu_addr >> 8;
4351 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4352 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4353
4354 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4355 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
4356 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4357 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
4358
4359 mqd->cp_hqd_eop_control = tmp;
4360
4361 /* enable doorbell? */
4362 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
4363
4364 if (prop->use_doorbell) {
4365 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4366 DOORBELL_OFFSET, prop->doorbell_index);
4367 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4368 DOORBELL_EN, 1);
4369 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4370 DOORBELL_SOURCE, 0);
4371 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4372 DOORBELL_HIT, 0);
4373 } else {
4374 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4375 DOORBELL_EN, 0);
4376 }
4377
4378 mqd->cp_hqd_pq_doorbell_control = tmp;
4379
4380 /* disable the queue if it's active */
4381 mqd->cp_hqd_dequeue_request = 0;
4382 mqd->cp_hqd_pq_rptr = 0;
4383 mqd->cp_hqd_pq_wptr_lo = 0;
4384 mqd->cp_hqd_pq_wptr_hi = 0;
4385
4386 /* set the pointer to the MQD */
4387 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
4388 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4389
4390 /* set MQD vmid to 0 */
4391 tmp = regCP_MQD_CONTROL_DEFAULT;
4392 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4393 mqd->cp_mqd_control = tmp;
4394
4395 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4396 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4397 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4398 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4399
4400 /* set up the HQD, this is similar to CP_RB0_CNTL */
4401 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
4402 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4403 (order_base_2(prop->queue_size / 4) - 1));
4404 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4405 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4406 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
4407 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH,
4408 prop->allow_tunneling);
4409 if (prop->kernel_queue) {
4410 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4411 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4412 }
4413 if (prop->tmz_queue)
4414 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TMZ, 1);
4415 mqd->cp_hqd_pq_control = tmp;
4416
4417 /* set the wb address whether it's enabled or not */
4418 wb_gpu_addr = prop->rptr_gpu_addr;
4419 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4420 mqd->cp_hqd_pq_rptr_report_addr_hi =
4421 upper_32_bits(wb_gpu_addr) & 0xffff;
4422
4423 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4424 wb_gpu_addr = prop->wptr_gpu_addr;
4425 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4426 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4427
4428 tmp = 0;
4429 /* enable the doorbell if requested */
4430 if (prop->use_doorbell) {
4431 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
4432 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4433 DOORBELL_OFFSET, prop->doorbell_index);
4434
4435 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4436 DOORBELL_EN, 1);
4437 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4438 DOORBELL_SOURCE, 0);
4439 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4440 DOORBELL_HIT, 0);
4441 }
4442
4443 mqd->cp_hqd_pq_doorbell_control = tmp;
4444
4445 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4446 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT;
4447
4448 /* set the vmid for the queue */
4449 mqd->cp_hqd_vmid = 0;
4450
4451 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
4452 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
4453 mqd->cp_hqd_persistent_state = tmp;
4454
4455 /* set MIN_IB_AVAIL_SIZE */
4456 tmp = regCP_HQD_IB_CONTROL_DEFAULT;
4457 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4458 mqd->cp_hqd_ib_control = tmp;
4459
4460 /* set static priority for a compute queue/ring */
4461 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
4462 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
4463
4464 tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
4465 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
4466 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
4467 mqd->cp_hqd_quantum = tmp;
4468
4469 mqd->cp_hqd_active = prop->hqd_active;
4470
4471 /* set UQ fenceaddress */
4472 mqd->fence_address_lo = lower_32_bits(prop->fence_address);
4473 mqd->fence_address_hi = upper_32_bits(prop->fence_address);
4474 /* set CU mask */
4475 gfx_v11_0_compute_mqd_set_cu_mask(adev, mqd, prop);
4476
4477 return 0;
4478 }
4479
gfx_v11_0_kiq_init_register(struct amdgpu_ring * ring)4480 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
4481 {
4482 struct amdgpu_device *adev = ring->adev;
4483 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4484 int j;
4485
4486 /* inactivate the queue */
4487 if (amdgpu_sriov_vf(adev))
4488 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
4489
4490 /* disable wptr polling */
4491 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4492
4493 /* write the EOP addr */
4494 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
4495 mqd->cp_hqd_eop_base_addr_lo);
4496 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
4497 mqd->cp_hqd_eop_base_addr_hi);
4498
4499 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4500 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
4501 mqd->cp_hqd_eop_control);
4502
4503 /* enable doorbell? */
4504 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4505 mqd->cp_hqd_pq_doorbell_control);
4506
4507 /* disable the queue if it's active */
4508 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
4509 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
4510 for (j = 0; j < adev->usec_timeout; j++) {
4511 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
4512 break;
4513 udelay(1);
4514 }
4515 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
4516 mqd->cp_hqd_dequeue_request);
4517 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
4518 mqd->cp_hqd_pq_rptr);
4519 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4520 mqd->cp_hqd_pq_wptr_lo);
4521 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4522 mqd->cp_hqd_pq_wptr_hi);
4523 }
4524
4525 /* set the pointer to the MQD */
4526 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
4527 mqd->cp_mqd_base_addr_lo);
4528 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
4529 mqd->cp_mqd_base_addr_hi);
4530
4531 /* set MQD vmid to 0 */
4532 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
4533 mqd->cp_mqd_control);
4534
4535 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4536 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
4537 mqd->cp_hqd_pq_base_lo);
4538 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
4539 mqd->cp_hqd_pq_base_hi);
4540
4541 /* set up the HQD, this is similar to CP_RB0_CNTL */
4542 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
4543 mqd->cp_hqd_pq_control);
4544
4545 /* set the wb address whether it's enabled or not */
4546 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
4547 mqd->cp_hqd_pq_rptr_report_addr_lo);
4548 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4549 mqd->cp_hqd_pq_rptr_report_addr_hi);
4550
4551 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4552 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
4553 mqd->cp_hqd_pq_wptr_poll_addr_lo);
4554 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
4555 mqd->cp_hqd_pq_wptr_poll_addr_hi);
4556
4557 /* enable the doorbell if requested */
4558 if (ring->use_doorbell) {
4559 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4560 (adev->doorbell_index.kiq * 2) << 2);
4561 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4562 (adev->doorbell_index.userqueue_end * 2) << 2);
4563 }
4564
4565 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4566 mqd->cp_hqd_pq_doorbell_control);
4567
4568 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4569 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4570 mqd->cp_hqd_pq_wptr_lo);
4571 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4572 mqd->cp_hqd_pq_wptr_hi);
4573
4574 /* set the vmid for the queue */
4575 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4576
4577 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4578 mqd->cp_hqd_persistent_state);
4579
4580 /* activate the queue */
4581 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4582 mqd->cp_hqd_active);
4583
4584 if (ring->use_doorbell)
4585 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4586
4587 return 0;
4588 }
4589
gfx_v11_0_kiq_init_queue(struct amdgpu_ring * ring)4590 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4591 {
4592 struct amdgpu_device *adev = ring->adev;
4593 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4594
4595 gfx_v11_0_kiq_setting(ring);
4596
4597 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4598 /* reset MQD to a clean status */
4599 if (adev->gfx.kiq[0].mqd_backup)
4600 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd));
4601
4602 /* reset ring buffer */
4603 ring->wptr = 0;
4604 amdgpu_ring_clear_ring(ring);
4605
4606 mutex_lock(&adev->srbm_mutex);
4607 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4608 gfx_v11_0_kiq_init_register(ring);
4609 soc21_grbm_select(adev, 0, 0, 0, 0);
4610 mutex_unlock(&adev->srbm_mutex);
4611 } else {
4612 memset((void *)mqd, 0, sizeof(*mqd));
4613 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4614 amdgpu_ring_clear_ring(ring);
4615 mutex_lock(&adev->srbm_mutex);
4616 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4617 amdgpu_ring_init_mqd(ring);
4618 gfx_v11_0_kiq_init_register(ring);
4619 soc21_grbm_select(adev, 0, 0, 0, 0);
4620 mutex_unlock(&adev->srbm_mutex);
4621
4622 if (adev->gfx.kiq[0].mqd_backup)
4623 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd));
4624 }
4625
4626 return 0;
4627 }
4628
gfx_v11_0_kcq_init_queue(struct amdgpu_ring * ring)4629 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring)
4630 {
4631 struct amdgpu_device *adev = ring->adev;
4632 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4633 int mqd_idx = ring - &adev->gfx.compute_ring[0];
4634
4635 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4636 memset((void *)mqd, 0, sizeof(*mqd));
4637 mutex_lock(&adev->srbm_mutex);
4638 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4639 amdgpu_ring_init_mqd(ring);
4640 soc21_grbm_select(adev, 0, 0, 0, 0);
4641 mutex_unlock(&adev->srbm_mutex);
4642
4643 if (adev->gfx.mec.mqd_backup[mqd_idx])
4644 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4645 } else {
4646 /* restore MQD to a clean status */
4647 if (adev->gfx.mec.mqd_backup[mqd_idx])
4648 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4649 /* reset ring buffer */
4650 ring->wptr = 0;
4651 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4652 amdgpu_ring_clear_ring(ring);
4653 }
4654
4655 return 0;
4656 }
4657
gfx_v11_0_kiq_resume(struct amdgpu_device * adev)4658 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4659 {
4660 gfx_v11_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
4661 return 0;
4662 }
4663
gfx_v11_0_kcq_resume(struct amdgpu_device * adev)4664 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4665 {
4666 int i, r;
4667
4668 if (!amdgpu_async_gfx_ring)
4669 gfx_v11_0_cp_compute_enable(adev, true);
4670
4671 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4672 r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i]);
4673 if (r)
4674 return r;
4675 }
4676
4677 return amdgpu_gfx_enable_kcq(adev, 0);
4678 }
4679
gfx_v11_0_cp_resume(struct amdgpu_device * adev)4680 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4681 {
4682 int r, i;
4683 struct amdgpu_ring *ring;
4684
4685 if (!(adev->flags & AMD_IS_APU))
4686 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4687
4688 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4689 /* legacy firmware loading */
4690 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4691 if (r)
4692 return r;
4693
4694 if (adev->gfx.rs64_enable)
4695 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4696 else
4697 r = gfx_v11_0_cp_compute_load_microcode(adev);
4698 if (r)
4699 return r;
4700 }
4701
4702 gfx_v11_0_cp_set_doorbell_range(adev);
4703
4704 if (amdgpu_async_gfx_ring) {
4705 gfx_v11_0_cp_compute_enable(adev, true);
4706 gfx_v11_0_cp_gfx_enable(adev, true);
4707 }
4708
4709 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4710 r = amdgpu_mes_kiq_hw_init(adev, 0);
4711 else
4712 r = gfx_v11_0_kiq_resume(adev);
4713 if (r)
4714 return r;
4715
4716 r = gfx_v11_0_kcq_resume(adev);
4717 if (r)
4718 return r;
4719
4720 if (!amdgpu_async_gfx_ring) {
4721 r = gfx_v11_0_cp_gfx_resume(adev);
4722 if (r)
4723 return r;
4724 } else {
4725 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4726 if (r)
4727 return r;
4728 }
4729
4730 if (adev->gfx.disable_kq) {
4731 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4732 ring = &adev->gfx.gfx_ring[i];
4733 /* we don't want to set ring->ready */
4734 r = amdgpu_ring_test_ring(ring);
4735 if (r)
4736 return r;
4737 }
4738 if (amdgpu_async_gfx_ring)
4739 amdgpu_gfx_disable_kgq(adev, 0);
4740 } else {
4741 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4742 ring = &adev->gfx.gfx_ring[i];
4743 r = amdgpu_ring_test_helper(ring);
4744 if (r)
4745 return r;
4746 }
4747 }
4748
4749 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4750 ring = &adev->gfx.compute_ring[i];
4751 r = amdgpu_ring_test_helper(ring);
4752 if (r)
4753 return r;
4754 }
4755
4756 return 0;
4757 }
4758
gfx_v11_0_cp_enable(struct amdgpu_device * adev,bool enable)4759 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4760 {
4761 gfx_v11_0_cp_gfx_enable(adev, enable);
4762 gfx_v11_0_cp_compute_enable(adev, enable);
4763 }
4764
gfx_v11_0_gfxhub_enable(struct amdgpu_device * adev)4765 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4766 {
4767 int r;
4768 bool value;
4769
4770 r = adev->gfxhub.funcs->gart_enable(adev);
4771 if (r)
4772 return r;
4773
4774 amdgpu_device_flush_hdp(adev, NULL);
4775
4776 value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS;
4777
4778 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4779 /* TODO investigate why this and the hdp flush above is needed,
4780 * are we missing a flush somewhere else? */
4781 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
4782
4783 return 0;
4784 }
4785
gfx_v11_0_select_cp_fw_arch(struct amdgpu_device * adev)4786 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4787 {
4788 u32 tmp;
4789
4790 /* select RS64 */
4791 if (adev->gfx.rs64_enable) {
4792 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4793 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4794 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4795
4796 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4797 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4798 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4799 }
4800
4801 if (amdgpu_emu_mode == 1)
4802 msleep(100);
4803 }
4804
get_gb_addr_config(struct amdgpu_device * adev)4805 static int get_gb_addr_config(struct amdgpu_device * adev)
4806 {
4807 u32 gb_addr_config;
4808
4809 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4810 if (gb_addr_config == 0)
4811 return -EINVAL;
4812
4813 adev->gfx.config.gb_addr_config_fields.num_pkrs =
4814 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4815
4816 adev->gfx.config.gb_addr_config = gb_addr_config;
4817
4818 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4819 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4820 GB_ADDR_CONFIG, NUM_PIPES);
4821
4822 adev->gfx.config.max_tile_pipes =
4823 adev->gfx.config.gb_addr_config_fields.num_pipes;
4824
4825 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4826 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4827 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4828 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4829 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4830 GB_ADDR_CONFIG, NUM_RB_PER_SE);
4831 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4832 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4833 GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4834 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4835 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4836 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4837
4838 return 0;
4839 }
4840
gfx_v11_0_disable_gpa_mode(struct amdgpu_device * adev)4841 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4842 {
4843 uint32_t data;
4844
4845 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4846 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4847 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4848
4849 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4850 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4851 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4852 }
4853
gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device * adev,bool enable)4854 static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev,
4855 bool enable)
4856 {
4857 unsigned int irq_type;
4858 int m, p, r;
4859
4860 if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) {
4861 for (m = 0; m < adev->gfx.me.num_me; m++) {
4862 for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) {
4863 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
4864 if (enable)
4865 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
4866 else
4867 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
4868 if (r) {
4869 if (!enable)
4870 return r;
4871 goto err_gfx;
4872 }
4873 }
4874 }
4875 }
4876
4877 if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) {
4878 for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
4879 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
4880 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
4881 + (m * adev->gfx.mec.num_pipe_per_mec)
4882 + p;
4883 if (enable)
4884 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
4885 else
4886 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
4887 if (r) {
4888 if (!enable)
4889 return r;
4890 goto err_compute;
4891 }
4892 }
4893 }
4894 }
4895
4896 return 0;
4897
4898 err_compute:
4899 for (p--; p >= 0; p--) {
4900 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
4901 + (m * adev->gfx.mec.num_pipe_per_mec) + p;
4902 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
4903 }
4904 for (m--; m >= 0; m--) {
4905 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
4906 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
4907 + (m * adev->gfx.mec.num_pipe_per_mec) + p;
4908 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
4909 }
4910 }
4911 m = adev->gfx.me.num_me;
4912 err_gfx:
4913 for (p--; p >= 0; p--) {
4914 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
4915 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
4916 }
4917 for (m--; m >= 0; m--) {
4918 for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) {
4919 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
4920 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
4921 }
4922 }
4923 return r;
4924 }
4925
gfx_v11_0_hw_init(struct amdgpu_ip_block * ip_block)4926 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
4927 {
4928 int r;
4929 struct amdgpu_device *adev = ip_block->adev;
4930
4931 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4932 adev->gfx.cleaner_shader_ptr);
4933
4934 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4935 if (adev->gfx.imu.funcs) {
4936 /* RLC autoload sequence 1: Program rlc ram */
4937 if (adev->gfx.imu.funcs->program_rlc_ram)
4938 adev->gfx.imu.funcs->program_rlc_ram(adev);
4939 /* rlc autoload firmware */
4940 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4941 if (r)
4942 return r;
4943 }
4944 } else {
4945 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4946 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4947 if (adev->gfx.imu.funcs->load_microcode)
4948 adev->gfx.imu.funcs->load_microcode(adev);
4949 if (adev->gfx.imu.funcs->setup_imu)
4950 adev->gfx.imu.funcs->setup_imu(adev);
4951 if (adev->gfx.imu.funcs->start_imu)
4952 adev->gfx.imu.funcs->start_imu(adev);
4953 }
4954
4955 /* disable gpa mode in backdoor loading */
4956 gfx_v11_0_disable_gpa_mode(adev);
4957 }
4958 }
4959
4960 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4961 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4962 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4963 if (r) {
4964 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4965 return r;
4966 }
4967 }
4968
4969 adev->gfx.is_poweron = true;
4970
4971 if(get_gb_addr_config(adev))
4972 drm_warn(adev_to_drm(adev), "Invalid gb_addr_config !\n");
4973
4974 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4975 adev->gfx.rs64_enable)
4976 gfx_v11_0_config_gfx_rs64(adev);
4977
4978 r = gfx_v11_0_gfxhub_enable(adev);
4979 if (r)
4980 return r;
4981
4982 if (!amdgpu_emu_mode)
4983 gfx_v11_0_init_golden_registers(adev);
4984
4985 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4986 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4987 /**
4988 * For gfx 11, rlc firmware loading relies on smu firmware is
4989 * loaded firstly, so in direct type, it has to load smc ucode
4990 * here before rlc.
4991 */
4992 r = amdgpu_pm_load_smu_firmware(adev, NULL);
4993 if (r)
4994 return r;
4995 }
4996
4997 gfx_v11_0_constants_init(adev);
4998
4999 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
5000 gfx_v11_0_select_cp_fw_arch(adev);
5001
5002 if (adev->nbio.funcs->gc_doorbell_init)
5003 adev->nbio.funcs->gc_doorbell_init(adev);
5004
5005 r = gfx_v11_0_rlc_resume(adev);
5006 if (r)
5007 return r;
5008
5009 /*
5010 * init golden registers and rlc resume may override some registers,
5011 * reconfig them here
5012 */
5013 gfx_v11_0_tcp_harvest(adev);
5014
5015 r = gfx_v11_0_cp_resume(adev);
5016 if (r)
5017 return r;
5018
5019 /* get IMU version from HW if it's not set */
5020 if (!adev->gfx.imu_fw_version)
5021 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0);
5022
5023 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5024 if (r)
5025 return r;
5026
5027 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5028 if (r)
5029 goto err_priv_inst;
5030
5031 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
5032 if (r)
5033 goto err_bad_op;
5034
5035 r = gfx_v11_0_set_userq_eop_interrupts(adev, true);
5036 if (r)
5037 goto err_userq_eop;
5038
5039 return 0;
5040
5041 err_userq_eop:
5042 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
5043 err_bad_op:
5044 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
5045 err_priv_inst:
5046 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
5047 return r;
5048 }
5049
gfx_v11_0_hw_fini(struct amdgpu_ip_block * ip_block)5050 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
5051 {
5052 struct amdgpu_device *adev = ip_block->adev;
5053
5054 cancel_delayed_work_sync(&adev->gfx.idle_work);
5055
5056 gfx_v11_0_set_userq_eop_interrupts(adev, false);
5057 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
5058 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
5059 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
5060
5061 if (!adev->no_hw_access) {
5062 if (amdgpu_async_gfx_ring &&
5063 !adev->gfx.disable_kq) {
5064 if (amdgpu_gfx_disable_kgq(adev, 0))
5065 DRM_ERROR("KGQ disable failed\n");
5066 }
5067
5068 if (amdgpu_gfx_disable_kcq(adev, 0))
5069 DRM_ERROR("KCQ disable failed\n");
5070
5071 amdgpu_mes_kiq_hw_fini(adev, 0);
5072 }
5073
5074 if (amdgpu_sriov_vf(adev))
5075 /* Remove the steps disabling CPG and clearing KIQ position,
5076 * so that CP could perform IDLE-SAVE during switch. Those
5077 * steps are necessary to avoid a DMAR error in gfx9 but it is
5078 * not reproduced on gfx11.
5079 */
5080 return 0;
5081
5082 gfx_v11_0_cp_enable(adev, false);
5083 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
5084
5085 adev->gfxhub.funcs->gart_disable(adev);
5086
5087 adev->gfx.is_poweron = false;
5088
5089 return 0;
5090 }
5091
gfx_v11_0_suspend(struct amdgpu_ip_block * ip_block)5092 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block)
5093 {
5094 return gfx_v11_0_hw_fini(ip_block);
5095 }
5096
gfx_v11_0_resume(struct amdgpu_ip_block * ip_block)5097 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block)
5098 {
5099 return gfx_v11_0_hw_init(ip_block);
5100 }
5101
gfx_v11_0_is_idle(struct amdgpu_ip_block * ip_block)5102 static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block)
5103 {
5104 struct amdgpu_device *adev = ip_block->adev;
5105
5106 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
5107 GRBM_STATUS, GUI_ACTIVE))
5108 return false;
5109 else
5110 return true;
5111 }
5112
gfx_v11_0_wait_for_idle(struct amdgpu_ip_block * ip_block)5113 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
5114 {
5115 unsigned i;
5116 u32 tmp;
5117 struct amdgpu_device *adev = ip_block->adev;
5118
5119 for (i = 0; i < adev->usec_timeout; i++) {
5120 /* read MC_STATUS */
5121 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
5122 GRBM_STATUS__GUI_ACTIVE_MASK;
5123
5124 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
5125 return 0;
5126 udelay(1);
5127 }
5128 return -ETIMEDOUT;
5129 }
5130
gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device * adev,bool req)5131 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev,
5132 bool req)
5133 {
5134 u32 i, tmp, val;
5135
5136 for (i = 0; i < adev->usec_timeout; i++) {
5137 /* Request with MeId=2, PipeId=0 */
5138 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
5139 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
5140 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
5141
5142 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
5143 if (req) {
5144 if (val == tmp)
5145 break;
5146 } else {
5147 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
5148 REQUEST, 1);
5149
5150 /* unlocked or locked by firmware */
5151 if (val != tmp)
5152 break;
5153 }
5154 udelay(1);
5155 }
5156
5157 if (i >= adev->usec_timeout)
5158 return -EINVAL;
5159
5160 return 0;
5161 }
5162
gfx_v11_0_soft_reset(struct amdgpu_ip_block * ip_block)5163 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block)
5164 {
5165 u32 grbm_soft_reset = 0;
5166 u32 tmp;
5167 int r, i, j, k;
5168 struct amdgpu_device *adev = ip_block->adev;
5169
5170 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5171
5172 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5173 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
5174 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
5175 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
5176 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
5177 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
5178
5179 mutex_lock(&adev->srbm_mutex);
5180 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
5181 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
5182 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
5183 soc21_grbm_select(adev, i, k, j, 0);
5184
5185 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
5186 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
5187 }
5188 }
5189 }
5190 for (i = 0; i < adev->gfx.me.num_me; ++i) {
5191 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
5192 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
5193 soc21_grbm_select(adev, i, k, j, 0);
5194
5195 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
5196 }
5197 }
5198 }
5199 soc21_grbm_select(adev, 0, 0, 0, 0);
5200 mutex_unlock(&adev->srbm_mutex);
5201
5202 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */
5203 mutex_lock(&adev->gfx.reset_sem_mutex);
5204 r = gfx_v11_0_request_gfx_index_mutex(adev, true);
5205 if (r) {
5206 mutex_unlock(&adev->gfx.reset_sem_mutex);
5207 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n");
5208 return r;
5209 }
5210
5211 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
5212
5213 // Read CP_VMID_RESET register three times.
5214 // to get sufficient time for GFX_HQD_ACTIVE reach 0
5215 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
5216 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
5217 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
5218
5219 /* release the gfx mutex */
5220 r = gfx_v11_0_request_gfx_index_mutex(adev, false);
5221 mutex_unlock(&adev->gfx.reset_sem_mutex);
5222 if (r) {
5223 DRM_ERROR("Failed to release the gfx mutex during soft reset\n");
5224 return r;
5225 }
5226
5227 for (i = 0; i < adev->usec_timeout; i++) {
5228 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
5229 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
5230 break;
5231 udelay(1);
5232 }
5233 if (i >= adev->usec_timeout) {
5234 printk("Failed to wait all pipes clean\n");
5235 return -EINVAL;
5236 }
5237
5238 /********** trigger soft reset ***********/
5239 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
5240 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5241 SOFT_RESET_CP, 1);
5242 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5243 SOFT_RESET_GFX, 1);
5244 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5245 SOFT_RESET_CPF, 1);
5246 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5247 SOFT_RESET_CPC, 1);
5248 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5249 SOFT_RESET_CPG, 1);
5250 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
5251 /********** exit soft reset ***********/
5252 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
5253 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5254 SOFT_RESET_CP, 0);
5255 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5256 SOFT_RESET_GFX, 0);
5257 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5258 SOFT_RESET_CPF, 0);
5259 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5260 SOFT_RESET_CPC, 0);
5261 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5262 SOFT_RESET_CPG, 0);
5263 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
5264
5265 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
5266 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
5267 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
5268
5269 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
5270 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
5271
5272 for (i = 0; i < adev->usec_timeout; i++) {
5273 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
5274 break;
5275 udelay(1);
5276 }
5277 if (i >= adev->usec_timeout) {
5278 printk("Failed to wait CP_VMID_RESET to 0\n");
5279 return -EINVAL;
5280 }
5281
5282 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5283 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5284 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5285 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5286 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5287 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
5288
5289 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5290
5291 r = gfx_v11_0_cp_resume(adev);
5292 if (r)
5293 return r;
5294
5295 /**
5296 * GFX soft reset impacts MES, resume MES after GFX soft reset is finished
5297 */
5298 return amdgpu_mes_resume(adev, 0);
5299 }
5300
gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device * adev)5301 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
5302 {
5303 uint64_t clock;
5304 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
5305
5306 if (amdgpu_sriov_vf(adev)) {
5307 amdgpu_gfx_off_ctrl(adev, false);
5308 mutex_lock(&adev->gfx.gpu_clock_mutex);
5309 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
5310 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
5311 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
5312 if (clock_counter_hi_pre != clock_counter_hi_after)
5313 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
5314 mutex_unlock(&adev->gfx.gpu_clock_mutex);
5315 amdgpu_gfx_off_ctrl(adev, true);
5316 } else {
5317 preempt_disable();
5318 if (amdgpu_ip_version(adev, SMUIO_HWIP, 0) < IP_VERSION(15, 0, 0)) {
5319 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0,
5320 regGOLDEN_TSC_COUNT_UPPER);
5321 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0,
5322 regGOLDEN_TSC_COUNT_LOWER);
5323 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0,
5324 regGOLDEN_TSC_COUNT_UPPER);
5325 if (clock_counter_hi_pre != clock_counter_hi_after)
5326 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0,
5327 regGOLDEN_TSC_COUNT_LOWER);
5328 } else {
5329 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0,
5330 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0);
5331 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0,
5332 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0);
5333 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0,
5334 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0);
5335 if (clock_counter_hi_pre != clock_counter_hi_after)
5336 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0,
5337 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0);
5338 }
5339 preempt_enable();
5340 }
5341 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
5342
5343 return clock;
5344 }
5345
gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)5346 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5347 uint32_t vmid,
5348 uint32_t gds_base, uint32_t gds_size,
5349 uint32_t gws_base, uint32_t gws_size,
5350 uint32_t oa_base, uint32_t oa_size)
5351 {
5352 struct amdgpu_device *adev = ring->adev;
5353
5354 /* GDS Base */
5355 gfx_v11_0_write_data_to_reg(ring, 0, false,
5356 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
5357 gds_base);
5358
5359 /* GDS Size */
5360 gfx_v11_0_write_data_to_reg(ring, 0, false,
5361 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
5362 gds_size);
5363
5364 /* GWS */
5365 gfx_v11_0_write_data_to_reg(ring, 0, false,
5366 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
5367 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5368
5369 /* OA */
5370 gfx_v11_0_write_data_to_reg(ring, 0, false,
5371 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
5372 (1 << (oa_size + oa_base)) - (1 << oa_base));
5373 }
5374
gfx_v11_0_early_init(struct amdgpu_ip_block * ip_block)5375 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
5376 {
5377 struct amdgpu_device *adev = ip_block->adev;
5378 int r;
5379
5380 switch (amdgpu_user_queue) {
5381 case -1:
5382 case 0:
5383 default:
5384 adev->gfx.disable_kq = false;
5385 adev->gfx.disable_uq = true;
5386 break;
5387 case 1:
5388 adev->gfx.disable_kq = false;
5389 adev->gfx.disable_uq = false;
5390 break;
5391 case 2:
5392 adev->gfx.disable_kq = true;
5393 adev->gfx.disable_uq = false;
5394 break;
5395 }
5396
5397 adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
5398
5399 gfx_v11_0_set_imu_funcs(adev);
5400 r = gfx_v11_0_init_microcode(adev);
5401 if (r)
5402 return r;
5403
5404 if (adev->gfx.disable_kq) {
5405 /* We need one GFX ring temporarily to set up
5406 * the clear state.
5407 */
5408 adev->gfx.num_gfx_rings = 1;
5409 adev->gfx.num_compute_rings = 0;
5410 } else {
5411 /* rs64 only supports one gfx pipe */
5412 if (adev->gfx.rs64_enable)
5413 adev->gfx.num_gfx_rings = 1;
5414 else
5415 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
5416 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5417 AMDGPU_MAX_COMPUTE_RINGS);
5418 }
5419
5420 gfx_v11_0_set_kiq_pm4_funcs(adev);
5421 gfx_v11_0_set_ring_funcs(adev);
5422 gfx_v11_0_set_irq_funcs(adev);
5423 gfx_v11_0_set_gds_init(adev);
5424 gfx_v11_0_set_rlc_funcs(adev);
5425 gfx_v11_0_set_mqd_funcs(adev);
5426
5427 gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
5428
5429 amdgpu_init_rlc_reg_funcs(adev);
5430
5431 return 0;
5432 }
5433
gfx_v11_0_is_rlc_enabled(struct amdgpu_device * adev)5434 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
5435 {
5436 uint32_t rlc_cntl;
5437
5438 /* if RLC is not enabled, do nothing */
5439 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
5440 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
5441 }
5442
gfx_v11_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)5443 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
5444 {
5445 uint32_t data;
5446 unsigned i;
5447
5448 data = RLC_SAFE_MODE__CMD_MASK;
5449 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5450
5451 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
5452
5453 /* wait for RLC_SAFE_MODE */
5454 for (i = 0; i < adev->usec_timeout; i++) {
5455 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
5456 RLC_SAFE_MODE, CMD))
5457 break;
5458 udelay(1);
5459 }
5460 }
5461
gfx_v11_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)5462 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
5463 {
5464 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
5465 }
5466
gfx_v11_0_update_perf_clk(struct amdgpu_device * adev,bool enable)5467 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
5468 bool enable)
5469 {
5470 uint32_t def, data;
5471
5472 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
5473 return;
5474
5475 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5476
5477 if (enable)
5478 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5479 else
5480 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5481
5482 if (def != data)
5483 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5484 }
5485
gfx_v11_0_update_sram_fgcg(struct amdgpu_device * adev,bool enable)5486 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
5487 bool enable)
5488 {
5489 uint32_t def, data;
5490
5491 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
5492 return;
5493
5494 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5495
5496 if (enable)
5497 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5498 else
5499 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5500
5501 if (def != data)
5502 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5503 }
5504
gfx_v11_0_update_repeater_fgcg(struct amdgpu_device * adev,bool enable)5505 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
5506 bool enable)
5507 {
5508 uint32_t def, data;
5509
5510 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
5511 return;
5512
5513 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5514
5515 if (enable)
5516 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5517 else
5518 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5519
5520 if (def != data)
5521 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5522 }
5523
gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)5524 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5525 bool enable)
5526 {
5527 uint32_t data, def;
5528
5529 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
5530 return;
5531
5532 /* It is disabled by HW by default */
5533 if (enable) {
5534 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5535 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
5536 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5537
5538 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5539 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5540 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5541
5542 if (def != data)
5543 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5544 }
5545 } else {
5546 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5547 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5548
5549 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5550 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5551 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5552
5553 if (def != data)
5554 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5555 }
5556 }
5557 }
5558
gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5559 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5560 bool enable)
5561 {
5562 uint32_t def, data;
5563
5564 if (!(adev->cg_flags &
5565 (AMD_CG_SUPPORT_GFX_CGCG |
5566 AMD_CG_SUPPORT_GFX_CGLS |
5567 AMD_CG_SUPPORT_GFX_3D_CGCG |
5568 AMD_CG_SUPPORT_GFX_3D_CGLS)))
5569 return;
5570
5571 if (enable) {
5572 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5573
5574 /* unset CGCG override */
5575 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5576 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5577 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5578 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5579 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
5580 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5581 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5582
5583 /* update CGCG override bits */
5584 if (def != data)
5585 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5586
5587 /* enable cgcg FSM(0x0000363F) */
5588 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5589
5590 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5591 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
5592 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5593 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5594 }
5595
5596 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5597 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
5598 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5599 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5600 }
5601
5602 if (def != data)
5603 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5604
5605 /* Program RLC_CGCG_CGLS_CTRL_3D */
5606 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5607
5608 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5609 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
5610 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5611 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5612 }
5613
5614 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5615 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
5616 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5617 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5618 }
5619
5620 if (def != data)
5621 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5622
5623 /* set IDLE_POLL_COUNT(0x00900100) */
5624 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
5625
5626 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
5627 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5628 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5629
5630 if (def != data)
5631 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
5632
5633 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5634 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5635 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5636 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5637 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5638 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
5639
5640 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5641 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5642 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5643
5644 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5645 if (adev->sdma.num_instances > 1) {
5646 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5647 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5648 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5649 }
5650 } else {
5651 /* Program RLC_CGCG_CGLS_CTRL */
5652 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5653
5654 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5655 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5656
5657 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5658 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5659
5660 if (def != data)
5661 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5662
5663 /* Program RLC_CGCG_CGLS_CTRL_3D */
5664 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5665
5666 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5667 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5668 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5669 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5670
5671 if (def != data)
5672 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5673
5674 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5675 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5676 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5677
5678 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5679 if (adev->sdma.num_instances > 1) {
5680 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5681 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5682 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5683 }
5684 }
5685 }
5686
gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5687 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5688 bool enable)
5689 {
5690 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5691
5692 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5693
5694 gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5695
5696 gfx_v11_0_update_repeater_fgcg(adev, enable);
5697
5698 gfx_v11_0_update_sram_fgcg(adev, enable);
5699
5700 gfx_v11_0_update_perf_clk(adev, enable);
5701
5702 if (adev->cg_flags &
5703 (AMD_CG_SUPPORT_GFX_MGCG |
5704 AMD_CG_SUPPORT_GFX_CGLS |
5705 AMD_CG_SUPPORT_GFX_CGCG |
5706 AMD_CG_SUPPORT_GFX_3D_CGCG |
5707 AMD_CG_SUPPORT_GFX_3D_CGLS))
5708 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5709
5710 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5711
5712 return 0;
5713 }
5714
gfx_v11_0_update_spm_vmid(struct amdgpu_device * adev,int xcc_id,struct amdgpu_ring * ring,unsigned vmid)5715 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
5716 struct amdgpu_ring *ring, unsigned vmid)
5717 {
5718 u32 reg, pre_data, data;
5719
5720 amdgpu_gfx_off_ctrl(adev, false);
5721 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5722 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
5723 pre_data = RREG32_NO_KIQ(reg);
5724 else
5725 pre_data = RREG32(reg);
5726
5727 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
5728 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5729
5730 if (pre_data != data) {
5731 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
5732 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5733 } else
5734 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5735 }
5736 amdgpu_gfx_off_ctrl(adev, true);
5737
5738 if (ring
5739 && amdgpu_sriov_is_pp_one_vf(adev)
5740 && (pre_data != data)
5741 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
5742 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
5743 amdgpu_ring_emit_wreg(ring, reg, data);
5744 }
5745 }
5746
5747 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5748 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5749 .set_safe_mode = gfx_v11_0_set_safe_mode,
5750 .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5751 .init = gfx_v11_0_rlc_init,
5752 .get_csb_size = gfx_v11_0_get_csb_size,
5753 .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5754 .resume = gfx_v11_0_rlc_resume,
5755 .stop = gfx_v11_0_rlc_stop,
5756 .reset = gfx_v11_0_rlc_reset,
5757 .start = gfx_v11_0_rlc_start,
5758 .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5759 };
5760
gfx_v11_cntl_power_gating(struct amdgpu_device * adev,bool enable)5761 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5762 {
5763 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5764
5765 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5766 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5767 else
5768 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5769
5770 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5771
5772 // Program RLC_PG_DELAY3 for CGPG hysteresis
5773 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5774 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5775 case IP_VERSION(11, 0, 1):
5776 case IP_VERSION(11, 0, 4):
5777 case IP_VERSION(11, 5, 0):
5778 case IP_VERSION(11, 5, 1):
5779 case IP_VERSION(11, 5, 2):
5780 case IP_VERSION(11, 5, 3):
5781 case IP_VERSION(11, 5, 4):
5782 case IP_VERSION(11, 5, 6):
5783 case IP_VERSION(11, 7, 0):
5784 case IP_VERSION(11, 7, 1):
5785 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5786 break;
5787 default:
5788 break;
5789 }
5790 }
5791 }
5792
gfx_v11_cntl_pg(struct amdgpu_device * adev,bool enable)5793 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5794 {
5795 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5796
5797 gfx_v11_cntl_power_gating(adev, enable);
5798
5799 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5800 }
5801
gfx_v11_0_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)5802 static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5803 enum amd_powergating_state state)
5804 {
5805 struct amdgpu_device *adev = ip_block->adev;
5806 bool enable = (state == AMD_PG_STATE_GATE);
5807
5808 if (amdgpu_sriov_vf(adev))
5809 return 0;
5810
5811 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5812 case IP_VERSION(11, 0, 0):
5813 case IP_VERSION(11, 0, 2):
5814 case IP_VERSION(11, 0, 3):
5815 amdgpu_gfx_off_ctrl(adev, enable);
5816 break;
5817 case IP_VERSION(11, 0, 1):
5818 case IP_VERSION(11, 0, 4):
5819 case IP_VERSION(11, 5, 0):
5820 case IP_VERSION(11, 5, 1):
5821 case IP_VERSION(11, 5, 2):
5822 case IP_VERSION(11, 5, 3):
5823 case IP_VERSION(11, 5, 4):
5824 case IP_VERSION(11, 5, 6):
5825 case IP_VERSION(11, 7, 0):
5826 case IP_VERSION(11, 7, 1):
5827 if (!enable)
5828 amdgpu_gfx_off_ctrl(adev, false);
5829
5830 gfx_v11_cntl_pg(adev, enable);
5831
5832 if (enable)
5833 amdgpu_gfx_off_ctrl(adev, true);
5834
5835 break;
5836 default:
5837 break;
5838 }
5839
5840 return 0;
5841 }
5842
gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)5843 static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5844 enum amd_clockgating_state state)
5845 {
5846 struct amdgpu_device *adev = ip_block->adev;
5847
5848 if (amdgpu_sriov_vf(adev))
5849 return 0;
5850
5851 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5852 case IP_VERSION(11, 0, 0):
5853 case IP_VERSION(11, 0, 1):
5854 case IP_VERSION(11, 0, 2):
5855 case IP_VERSION(11, 0, 3):
5856 case IP_VERSION(11, 0, 4):
5857 case IP_VERSION(11, 5, 0):
5858 case IP_VERSION(11, 5, 1):
5859 case IP_VERSION(11, 5, 2):
5860 case IP_VERSION(11, 5, 3):
5861 case IP_VERSION(11, 5, 4):
5862 case IP_VERSION(11, 5, 6):
5863 case IP_VERSION(11, 7, 0):
5864 case IP_VERSION(11, 7, 1):
5865 gfx_v11_0_update_gfx_clock_gating(adev,
5866 state == AMD_CG_STATE_GATE);
5867 break;
5868 default:
5869 break;
5870 }
5871
5872 return 0;
5873 }
5874
gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)5875 static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
5876 {
5877 struct amdgpu_device *adev = ip_block->adev;
5878 int data;
5879
5880 /* AMD_CG_SUPPORT_GFX_MGCG */
5881 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5882 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5883 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5884
5885 /* AMD_CG_SUPPORT_REPEATER_FGCG */
5886 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5887 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5888
5889 /* AMD_CG_SUPPORT_GFX_FGCG */
5890 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5891 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5892
5893 /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5894 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5895 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5896
5897 /* AMD_CG_SUPPORT_GFX_CGCG */
5898 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5899 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5900 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5901
5902 /* AMD_CG_SUPPORT_GFX_CGLS */
5903 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5904 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5905
5906 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5907 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5908 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5909 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5910
5911 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5912 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5913 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5914 }
5915
gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5916 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5917 {
5918 /* gfx11 is 32bit rptr*/
5919 return *(uint32_t *)ring->rptr_cpu_addr;
5920 }
5921
gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5922 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5923 {
5924 struct amdgpu_device *adev = ring->adev;
5925 u64 wptr;
5926
5927 /* XXX check if swapping is necessary on BE */
5928 if (ring->use_doorbell) {
5929 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5930 } else {
5931 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5932 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5933 }
5934
5935 return wptr;
5936 }
5937
gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5938 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5939 {
5940 struct amdgpu_device *adev = ring->adev;
5941
5942 if (ring->use_doorbell) {
5943 /* XXX check if swapping is necessary on BE */
5944 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5945 ring->wptr);
5946 WDOORBELL64(ring->doorbell_index, ring->wptr);
5947 } else {
5948 WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5949 lower_32_bits(ring->wptr));
5950 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5951 upper_32_bits(ring->wptr));
5952 }
5953 }
5954
gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5955 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5956 {
5957 /* gfx11 hardware is 32bit rptr */
5958 return *(uint32_t *)ring->rptr_cpu_addr;
5959 }
5960
gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5961 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5962 {
5963 struct amdgpu_device *adev = ring->adev;
5964 u64 wptr;
5965
5966 /* XXX check if swapping is necessary on BE */
5967 if (ring->use_doorbell) {
5968 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5969 } else {
5970 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
5971 wptr = 0;
5972 }
5973 return wptr;
5974 }
5975
gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5976 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5977 {
5978 struct amdgpu_device *adev = ring->adev;
5979
5980 /* XXX check if swapping is necessary on BE */
5981 if (ring->use_doorbell) {
5982 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5983 ring->wptr);
5984 WDOORBELL64(ring->doorbell_index, ring->wptr);
5985 } else {
5986 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
5987 }
5988 }
5989
gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5990 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5991 {
5992 struct amdgpu_device *adev = ring->adev;
5993 u32 ref_and_mask, reg_mem_engine;
5994
5995 if (!adev->gfx.funcs->get_hdp_flush_mask) {
5996 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
5997 return;
5998 }
5999
6000 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine);
6001 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
6002 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
6003 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
6004 ref_and_mask, ref_and_mask, 0x20);
6005 }
6006
gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)6007 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
6008 struct amdgpu_job *job,
6009 struct amdgpu_ib *ib,
6010 uint32_t flags)
6011 {
6012 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6013 u32 header, control = 0;
6014
6015 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
6016
6017 control |= ib->length_dw | (vmid << 24);
6018
6019 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
6020 control |= INDIRECT_BUFFER_PRE_ENB(1);
6021
6022 if (flags & AMDGPU_IB_PREEMPTED)
6023 control |= INDIRECT_BUFFER_PRE_RESUME(1);
6024
6025 if (vmid && !ring->adev->gfx.rs64_enable)
6026 gfx_v11_0_ring_emit_de_meta(ring,
6027 !amdgpu_sriov_vf(ring->adev) && (flags & AMDGPU_IB_PREEMPTED));
6028 }
6029
6030 amdgpu_ring_write(ring, header);
6031 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
6032 amdgpu_ring_write(ring,
6033 #ifdef __BIG_ENDIAN
6034 (2 << 0) |
6035 #endif
6036 lower_32_bits(ib->gpu_addr));
6037 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
6038 amdgpu_ring_write(ring, control);
6039 }
6040
gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)6041 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
6042 struct amdgpu_job *job,
6043 struct amdgpu_ib *ib,
6044 uint32_t flags)
6045 {
6046 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6047 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
6048
6049 /* Currently, there is a high possibility to get wave ID mismatch
6050 * between ME and GDS, leading to a hw deadlock, because ME generates
6051 * different wave IDs than the GDS expects. This situation happens
6052 * randomly when at least 5 compute pipes use GDS ordered append.
6053 * The wave IDs generated by ME are also wrong after suspend/resume.
6054 * Those are probably bugs somewhere else in the kernel driver.
6055 *
6056 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
6057 * GDS to 0 for this ring (me/pipe).
6058 */
6059 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
6060 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
6061 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
6062 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
6063 }
6064
6065 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
6066 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
6067 amdgpu_ring_write(ring,
6068 #ifdef __BIG_ENDIAN
6069 (2 << 0) |
6070 #endif
6071 lower_32_bits(ib->gpu_addr));
6072 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
6073 amdgpu_ring_write(ring, control);
6074 }
6075
gfx_v11_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)6076 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
6077 u64 seq, unsigned flags)
6078 {
6079 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6080 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6081
6082 /* RELEASE_MEM - flush caches, send int */
6083 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
6084 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
6085 PACKET3_RELEASE_MEM_GCR_GL2_WB |
6086 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */
6087 PACKET3_RELEASE_MEM_GCR_GLM_WB |
6088 PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
6089 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6090 PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
6091 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
6092 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
6093
6094 /*
6095 * the address should be Qword aligned if 64bit write, Dword
6096 * aligned if only send 32bit data low (discard data high)
6097 */
6098 if (write64bit)
6099 WARN_ON(addr & 0x7);
6100 else
6101 WARN_ON(addr & 0x3);
6102 amdgpu_ring_write(ring, lower_32_bits(addr));
6103 amdgpu_ring_write(ring, upper_32_bits(addr));
6104 amdgpu_ring_write(ring, lower_32_bits(seq));
6105 amdgpu_ring_write(ring, upper_32_bits(seq));
6106 amdgpu_ring_write(ring, 0);
6107 }
6108
gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)6109 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
6110 {
6111 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6112 uint32_t seq = ring->fence_drv.sync_seq;
6113 uint64_t addr = ring->fence_drv.gpu_addr;
6114
6115 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
6116 upper_32_bits(addr), seq, 0xffffffff, 4);
6117 }
6118
gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring * ring,uint16_t pasid,uint32_t flush_type,bool all_hub,uint8_t dst_sel)6119 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
6120 uint16_t pasid, uint32_t flush_type,
6121 bool all_hub, uint8_t dst_sel)
6122 {
6123 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
6124 amdgpu_ring_write(ring,
6125 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
6126 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
6127 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
6128 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
6129 }
6130
gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)6131 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
6132 unsigned vmid, uint64_t pd_addr)
6133 {
6134 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
6135
6136 /* compute doesn't have PFP */
6137 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
6138 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6139 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6140 amdgpu_ring_write(ring, 0x0);
6141 }
6142
6143 /* Make sure that we can't skip the SET_Q_MODE packets when the VM
6144 * changed in any way.
6145 */
6146 ring->set_q_mode_offs = 0;
6147 ring->set_q_mode_ptr = NULL;
6148 }
6149
gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)6150 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6151 u64 seq, unsigned int flags)
6152 {
6153 struct amdgpu_device *adev = ring->adev;
6154
6155 /* write fence seq to the "addr" */
6156 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6157 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6158 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6159 amdgpu_ring_write(ring, lower_32_bits(addr));
6160 amdgpu_ring_write(ring, upper_32_bits(addr));
6161 amdgpu_ring_write(ring, lower_32_bits(seq));
6162
6163 if (flags & AMDGPU_FENCE_FLAG_INT) {
6164 /* set register to trigger INT */
6165 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6166 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6167 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6168 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
6169 amdgpu_ring_write(ring, 0);
6170 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6171 }
6172 }
6173
gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)6174 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
6175 uint32_t flags)
6176 {
6177 uint32_t dw2 = 0;
6178
6179 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6180 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
6181 /* set load_global_config & load_global_uconfig */
6182 dw2 |= 0x8001;
6183 /* set load_cs_sh_regs */
6184 dw2 |= 0x01000000;
6185 /* set load_per_context_state & load_gfx_sh_regs for GFX */
6186 dw2 |= 0x10002;
6187 }
6188
6189 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6190 amdgpu_ring_write(ring, dw2);
6191 amdgpu_ring_write(ring, 0);
6192 }
6193
gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)6194 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
6195 uint64_t addr)
6196 {
6197 unsigned ret;
6198
6199 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
6200 amdgpu_ring_write(ring, lower_32_bits(addr));
6201 amdgpu_ring_write(ring, upper_32_bits(addr));
6202 /* discard following DWs if *cond_exec_gpu_addr==0 */
6203 amdgpu_ring_write(ring, 0);
6204 ret = ring->wptr & ring->buf_mask;
6205 /* patch dummy value later */
6206 amdgpu_ring_write(ring, 0);
6207
6208 return ret;
6209 }
6210
gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring * ring,u64 shadow_va,u64 csa_va,u64 gds_va,bool init_shadow,int vmid)6211 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
6212 u64 shadow_va, u64 csa_va,
6213 u64 gds_va, bool init_shadow,
6214 int vmid)
6215 {
6216 struct amdgpu_device *adev = ring->adev;
6217 unsigned int offs, end;
6218
6219 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj)
6220 return;
6221
6222 /*
6223 * The logic here isn't easy to understand because we need to keep state
6224 * accross multiple executions of the function as well as between the
6225 * CPU and GPU. The general idea is that the newly written GPU command
6226 * has a condition on the previous one and only executed if really
6227 * necessary.
6228 */
6229
6230 /*
6231 * The dw in the NOP controls if the next SET_Q_MODE packet should be
6232 * executed or not. Reserve 64bits just to be on the save side.
6233 */
6234 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1));
6235 offs = ring->wptr & ring->buf_mask;
6236
6237 /*
6238 * We start with skipping the prefix SET_Q_MODE and always executing
6239 * the postfix SET_Q_MODE packet. This is changed below with a
6240 * WRITE_DATA command when the postfix executed.
6241 */
6242 amdgpu_ring_write(ring, shadow_va ? 1 : 0);
6243 amdgpu_ring_write(ring, 0);
6244
6245 if (ring->set_q_mode_offs) {
6246 uint64_t addr;
6247
6248 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
6249 addr += ring->set_q_mode_offs << 2;
6250 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr);
6251 }
6252
6253 /*
6254 * When the postfix SET_Q_MODE packet executes we need to make sure that the
6255 * next prefix SET_Q_MODE packet executes as well.
6256 */
6257 if (!shadow_va) {
6258 uint64_t addr;
6259
6260 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
6261 addr += offs << 2;
6262 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6263 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
6264 amdgpu_ring_write(ring, lower_32_bits(addr));
6265 amdgpu_ring_write(ring, upper_32_bits(addr));
6266 amdgpu_ring_write(ring, 0x1);
6267 }
6268
6269 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7));
6270 amdgpu_ring_write(ring, lower_32_bits(shadow_va));
6271 amdgpu_ring_write(ring, upper_32_bits(shadow_va));
6272 amdgpu_ring_write(ring, lower_32_bits(gds_va));
6273 amdgpu_ring_write(ring, upper_32_bits(gds_va));
6274 amdgpu_ring_write(ring, lower_32_bits(csa_va));
6275 amdgpu_ring_write(ring, upper_32_bits(csa_va));
6276 amdgpu_ring_write(ring, shadow_va ?
6277 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0);
6278 amdgpu_ring_write(ring, init_shadow ?
6279 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0);
6280
6281 if (ring->set_q_mode_offs)
6282 amdgpu_ring_patch_cond_exec(ring, end);
6283
6284 if (shadow_va) {
6285 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid;
6286
6287 /*
6288 * If the tokens match try to skip the last postfix SET_Q_MODE
6289 * packet to avoid saving/restoring the state all the time.
6290 */
6291 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token)
6292 *ring->set_q_mode_ptr = 0;
6293
6294 ring->set_q_mode_token = token;
6295 } else {
6296 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs];
6297 }
6298
6299 ring->set_q_mode_offs = offs;
6300 }
6301
gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring * ring,bool resume)6302 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
6303 {
6304 struct amdgpu_device *adev = ring->adev;
6305 struct v10_de_ib_state de_payload = {0};
6306 uint64_t offset, gds_addr, de_payload_gpu_addr;
6307 void *de_payload_cpu_addr;
6308 int cnt;
6309
6310 offset = offsetof(struct v10_gfx_meta_data, de_payload);
6311 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
6312 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
6313
6314 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
6315 AMDGPU_CSA_SIZE - adev->gds.gds_size,
6316 PAGE_SIZE);
6317
6318 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
6319 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
6320
6321 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
6322 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
6323 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
6324 WRITE_DATA_DST_SEL(8) |
6325 WR_CONFIRM) |
6326 WRITE_DATA_CACHE_POLICY(0));
6327 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
6328 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
6329
6330 if (resume)
6331 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
6332 sizeof(de_payload) >> 2);
6333 else
6334 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
6335 sizeof(de_payload) >> 2);
6336 }
6337
gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)6338 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
6339 bool secure)
6340 {
6341 uint32_t v = secure ? FRAME_TMZ : 0;
6342
6343 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
6344 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
6345 }
6346
gfx_v11_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)6347 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6348 uint32_t reg_val_offs)
6349 {
6350 struct amdgpu_device *adev = ring->adev;
6351
6352 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6353 amdgpu_ring_write(ring, 0 | /* src: register*/
6354 (5 << 8) | /* dst: memory */
6355 (1 << 20)); /* write confirm */
6356 amdgpu_ring_write(ring, reg);
6357 amdgpu_ring_write(ring, 0);
6358 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6359 reg_val_offs * 4));
6360 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6361 reg_val_offs * 4));
6362 }
6363
gfx_v11_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)6364 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6365 uint32_t val)
6366 {
6367 uint32_t cmd = 0;
6368
6369 switch (ring->funcs->type) {
6370 case AMDGPU_RING_TYPE_GFX:
6371 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6372 break;
6373 case AMDGPU_RING_TYPE_KIQ:
6374 cmd = (1 << 16); /* no inc addr */
6375 break;
6376 default:
6377 cmd = WR_CONFIRM;
6378 break;
6379 }
6380 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6381 amdgpu_ring_write(ring, cmd);
6382 amdgpu_ring_write(ring, reg);
6383 amdgpu_ring_write(ring, 0);
6384 amdgpu_ring_write(ring, val);
6385 }
6386
gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)6387 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
6388 uint32_t val, uint32_t mask)
6389 {
6390 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
6391 }
6392
gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)6393 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
6394 uint32_t reg0, uint32_t reg1,
6395 uint32_t ref, uint32_t mask)
6396 {
6397 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6398
6399 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
6400 ref, mask, 0x20);
6401 }
6402
6403 static void
gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,uint32_t me,uint32_t pipe,enum amdgpu_interrupt_state state)6404 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6405 uint32_t me, uint32_t pipe,
6406 enum amdgpu_interrupt_state state)
6407 {
6408 uint32_t cp_int_cntl, cp_int_cntl_reg;
6409
6410 if (!me) {
6411 switch (pipe) {
6412 case 0:
6413 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
6414 break;
6415 case 1:
6416 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
6417 break;
6418 default:
6419 DRM_DEBUG("invalid pipe %d\n", pipe);
6420 return;
6421 }
6422 } else {
6423 DRM_DEBUG("invalid me %d\n", me);
6424 return;
6425 }
6426
6427 switch (state) {
6428 case AMDGPU_IRQ_STATE_DISABLE:
6429 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6430 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6431 TIME_STAMP_INT_ENABLE, 0);
6432 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6433 GENERIC0_INT_ENABLE, 0);
6434 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6435 break;
6436 case AMDGPU_IRQ_STATE_ENABLE:
6437 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6438 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6439 TIME_STAMP_INT_ENABLE, 1);
6440 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6441 GENERIC0_INT_ENABLE, 1);
6442 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6443 break;
6444 default:
6445 break;
6446 }
6447 }
6448
gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)6449 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6450 int me, int pipe,
6451 enum amdgpu_interrupt_state state)
6452 {
6453 u32 mec_int_cntl, mec_int_cntl_reg;
6454
6455 /*
6456 * amdgpu controls only the first MEC. That's why this function only
6457 * handles the setting of interrupts for this specific MEC. All other
6458 * pipes' interrupts are set by amdkfd.
6459 */
6460
6461 if (me == 1) {
6462 switch (pipe) {
6463 case 0:
6464 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6465 break;
6466 case 1:
6467 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
6468 break;
6469 case 2:
6470 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
6471 break;
6472 case 3:
6473 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
6474 break;
6475 default:
6476 DRM_DEBUG("invalid pipe %d\n", pipe);
6477 return;
6478 }
6479 } else {
6480 DRM_DEBUG("invalid me %d\n", me);
6481 return;
6482 }
6483
6484 switch (state) {
6485 case AMDGPU_IRQ_STATE_DISABLE:
6486 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6487 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6488 TIME_STAMP_INT_ENABLE, 0);
6489 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6490 GENERIC0_INT_ENABLE, 0);
6491 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6492 break;
6493 case AMDGPU_IRQ_STATE_ENABLE:
6494 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6495 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6496 TIME_STAMP_INT_ENABLE, 1);
6497 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6498 GENERIC0_INT_ENABLE, 1);
6499 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6500 break;
6501 default:
6502 break;
6503 }
6504 }
6505
gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6506 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6507 struct amdgpu_irq_src *src,
6508 unsigned type,
6509 enum amdgpu_interrupt_state state)
6510 {
6511 switch (type) {
6512 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6513 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
6514 break;
6515 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
6516 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
6517 break;
6518 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6519 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6520 break;
6521 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6522 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6523 break;
6524 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6525 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6526 break;
6527 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6528 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6529 break;
6530 default:
6531 break;
6532 }
6533 return 0;
6534 }
6535
gfx_v11_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6536 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
6537 struct amdgpu_irq_src *source,
6538 struct amdgpu_iv_entry *entry)
6539 {
6540 u32 doorbell_offset = entry->src_data[0];
6541
6542 DRM_DEBUG("IH: CP EOP\n");
6543
6544 if (!adev->gfx.disable_kq) {
6545 u8 me_id = (entry->ring_id & 0x0c) >> 2;
6546 u8 pipe_id = (entry->ring_id & 0x03) >> 0;
6547 u8 queue_id = (entry->ring_id & 0x70) >> 4;
6548 struct amdgpu_ring *ring;
6549 int i;
6550
6551 switch (me_id) {
6552 case 0:
6553 /*
6554 * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0,
6555 * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask).
6556 * Require a strict (me,pipe,queue) match so userq gfx
6557 * EOPs fall through to amdgpu_userq_process_fence_irq().
6558 */
6559 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6560 ring = &adev->gfx.gfx_ring[i];
6561 if ((ring->me == me_id) &&
6562 (ring->pipe == pipe_id) &&
6563 (ring->queue == queue_id)) {
6564 amdgpu_fence_process(ring);
6565 return 0;
6566 }
6567 }
6568 break;
6569 case 1:
6570 case 2:
6571 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6572 ring = &adev->gfx.compute_ring[i];
6573 /* Per-queue interrupt is supported for MEC starting from VI.
6574 * The interrupt can only be enabled/disabled per pipe instead
6575 * of per queue.
6576 */
6577 if ((ring->me == me_id) &&
6578 (ring->pipe == pipe_id) &&
6579 (ring->queue == queue_id)) {
6580 amdgpu_fence_process(ring);
6581 return 0;
6582 }
6583 }
6584 break;
6585 default:
6586 break;
6587 }
6588 }
6589
6590 if (adev->enable_mes && doorbell_offset)
6591 amdgpu_userq_process_fence_irq(adev, doorbell_offset);
6592
6593 return 0;
6594 }
6595
gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6596 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6597 struct amdgpu_irq_src *source,
6598 unsigned int type,
6599 enum amdgpu_interrupt_state state)
6600 {
6601 u32 cp_int_cntl_reg, cp_int_cntl;
6602 int i, j;
6603
6604 switch (state) {
6605 case AMDGPU_IRQ_STATE_DISABLE:
6606 case AMDGPU_IRQ_STATE_ENABLE:
6607 for (i = 0; i < adev->gfx.me.num_me; i++) {
6608 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6609 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6610
6611 if (cp_int_cntl_reg) {
6612 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6613 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6614 PRIV_REG_INT_ENABLE,
6615 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6616 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6617 }
6618 }
6619 }
6620 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6621 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6622 /* MECs start at 1 */
6623 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6624
6625 if (cp_int_cntl_reg) {
6626 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6627 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6628 PRIV_REG_INT_ENABLE,
6629 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6630 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6631 }
6632 }
6633 }
6634 break;
6635 default:
6636 break;
6637 }
6638
6639 return 0;
6640 }
6641
gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6642 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6643 struct amdgpu_irq_src *source,
6644 unsigned type,
6645 enum amdgpu_interrupt_state state)
6646 {
6647 u32 cp_int_cntl_reg, cp_int_cntl;
6648 int i, j;
6649
6650 switch (state) {
6651 case AMDGPU_IRQ_STATE_DISABLE:
6652 case AMDGPU_IRQ_STATE_ENABLE:
6653 for (i = 0; i < adev->gfx.me.num_me; i++) {
6654 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6655 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6656
6657 if (cp_int_cntl_reg) {
6658 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6659 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6660 OPCODE_ERROR_INT_ENABLE,
6661 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6662 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6663 }
6664 }
6665 }
6666 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6667 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6668 /* MECs start at 1 */
6669 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6670
6671 if (cp_int_cntl_reg) {
6672 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6673 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6674 OPCODE_ERROR_INT_ENABLE,
6675 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6676 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6677 }
6678 }
6679 }
6680 break;
6681 default:
6682 break;
6683 }
6684 return 0;
6685 }
6686
gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6687 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6688 struct amdgpu_irq_src *source,
6689 unsigned int type,
6690 enum amdgpu_interrupt_state state)
6691 {
6692 u32 cp_int_cntl_reg, cp_int_cntl;
6693 int i, j;
6694
6695 switch (state) {
6696 case AMDGPU_IRQ_STATE_DISABLE:
6697 case AMDGPU_IRQ_STATE_ENABLE:
6698 for (i = 0; i < adev->gfx.me.num_me; i++) {
6699 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6700 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6701
6702 if (cp_int_cntl_reg) {
6703 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6704 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6705 PRIV_INSTR_INT_ENABLE,
6706 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6707 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6708 }
6709 }
6710 }
6711 break;
6712 default:
6713 break;
6714 }
6715
6716 return 0;
6717 }
6718
6719 /*
6720 * A gfx exception IV carries no doorbell. For each faulted slot, read the
6721 * doorbell back from the HQD (left in place by the fatal fault), look up the
6722 * user queue and kick its per-queue reset.
6723 */
gfx_v11_0_userq_priv_fault_work(struct work_struct * work)6724 static void gfx_v11_0_userq_priv_fault_work(struct work_struct *work)
6725 {
6726 struct amdgpu_device *adev =
6727 container_of(work, struct amdgpu_device, gfx.userq_priv_fault_work);
6728 unsigned long slots = xchg(&adev->gfx.userq_priv_fault_slots, 0);
6729 unsigned int id;
6730
6731 for_each_set_bit(id, &slots, BITS_PER_LONG) {
6732 u8 pipe = id & 0x3;
6733 u8 queue = (id >> 2) & 0x7;
6734 struct amdgpu_usermode_queue *q;
6735 u32 db_ctrl, doorbell;
6736
6737 amdgpu_gfx_off_ctrl(adev, false);
6738 mutex_lock(&adev->srbm_mutex);
6739 soc21_grbm_select(adev, 0, pipe, queue, 0);
6740 db_ctrl = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
6741 soc21_grbm_select(adev, 0, 0, 0, 0);
6742 mutex_unlock(&adev->srbm_mutex);
6743 amdgpu_gfx_off_ctrl(adev, true);
6744
6745 doorbell = (db_ctrl & CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >>
6746 CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
6747 q = xa_load(&adev->userq_doorbell_xa, doorbell);
6748 if (q)
6749 amdgpu_userq_start_hang_detect_work(q);
6750 }
6751 }
6752
gfx_v11_0_handle_priv_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6753 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6754 struct amdgpu_iv_entry *entry)
6755 {
6756 u8 me_id, pipe_id, queue_id;
6757
6758 me_id = (entry->ring_id & 0x0c) >> 2;
6759 pipe_id = (entry->ring_id & 0x03) >> 0;
6760 queue_id = (entry->ring_id & 0x70) >> 4;
6761
6762 amdgpu_gfx_handle_priv_fault(adev, entry, me_id, pipe_id, queue_id);
6763 }
6764
gfx_v11_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6765 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6766 struct amdgpu_irq_src *source,
6767 struct amdgpu_iv_entry *entry)
6768 {
6769 DRM_ERROR("Illegal register access in command stream\n");
6770 gfx_v11_0_handle_priv_fault(adev, entry);
6771 return 0;
6772 }
6773
gfx_v11_0_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6774 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev,
6775 struct amdgpu_irq_src *source,
6776 struct amdgpu_iv_entry *entry)
6777 {
6778 DRM_ERROR("Illegal opcode in command stream\n");
6779 gfx_v11_0_handle_priv_fault(adev, entry);
6780 return 0;
6781 }
6782
gfx_v11_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6783 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6784 struct amdgpu_irq_src *source,
6785 struct amdgpu_iv_entry *entry)
6786 {
6787 DRM_ERROR("Illegal instruction in command stream\n");
6788 gfx_v11_0_handle_priv_fault(adev, entry);
6789 return 0;
6790 }
6791
gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6792 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6793 struct amdgpu_irq_src *source,
6794 struct amdgpu_iv_entry *entry)
6795 {
6796 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6797 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6798
6799 return 0;
6800 }
6801
6802 #if 0
6803 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6804 struct amdgpu_irq_src *src,
6805 unsigned int type,
6806 enum amdgpu_interrupt_state state)
6807 {
6808 uint32_t tmp, target;
6809 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring);
6810
6811 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6812 target += ring->pipe;
6813
6814 switch (type) {
6815 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6816 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6817 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6818 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6819 GENERIC2_INT_ENABLE, 0);
6820 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6821
6822 tmp = RREG32_SOC15_IP(GC, target);
6823 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6824 GENERIC2_INT_ENABLE, 0);
6825 WREG32_SOC15_IP(GC, target, tmp);
6826 } else {
6827 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6828 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6829 GENERIC2_INT_ENABLE, 1);
6830 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6831
6832 tmp = RREG32_SOC15_IP(GC, target);
6833 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6834 GENERIC2_INT_ENABLE, 1);
6835 WREG32_SOC15_IP(GC, target, tmp);
6836 }
6837 break;
6838 default:
6839 /* kiq only support GENERIC2_INT now */
6840 break;
6841 }
6842 return 0;
6843 }
6844 #endif
6845
gfx_v11_0_emit_mem_sync(struct amdgpu_ring * ring)6846 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6847 {
6848 const unsigned int gcr_cntl =
6849 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6850 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6851 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6852 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6853 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6854 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6855 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6856 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6857
6858 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6859 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6860 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6861 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
6862 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
6863 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6864 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
6865 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6866 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6867 }
6868
gfx_v11_0_reset_kgq(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)6869 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring,
6870 unsigned int vmid,
6871 struct amdgpu_fence *timedout_fence)
6872 {
6873 struct amdgpu_device *adev = ring->adev;
6874 bool use_mmio = adev->gfx.me.use_mmio_for_reset;
6875
6876 return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio);
6877 }
6878
gfx_v11_0_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)6879 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
6880 unsigned int vmid,
6881 struct amdgpu_fence *timedout_fence)
6882 {
6883 struct amdgpu_device *adev = ring->adev;
6884
6885 return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL);
6886 }
6887
gfx_v11_ip_print(struct amdgpu_ip_block * ip_block,struct drm_printer * p)6888 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
6889 {
6890 struct amdgpu_device *adev = ip_block->adev;
6891 uint32_t i, j, k, reg, index = 0;
6892 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6893
6894 if (!adev->gfx.ip_dump_core)
6895 return;
6896
6897 for (i = 0; i < reg_count; i++)
6898 drm_printf(p, "%-50s \t 0x%08x\n",
6899 gc_reg_list_11_0[i].reg_name,
6900 adev->gfx.ip_dump_core[i]);
6901
6902 /* print compute queue registers for all instances */
6903 if (!adev->gfx.ip_dump_compute_queues)
6904 return;
6905
6906 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6907 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
6908 adev->gfx.mec.num_mec,
6909 adev->gfx.mec.num_pipe_per_mec,
6910 adev->gfx.mec.num_queue_per_pipe);
6911
6912 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6913 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6914 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6915 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
6916 for (reg = 0; reg < reg_count; reg++) {
6917 if (i && gc_cp_reg_list_11[reg].reg_offset == regCP_MEC_ME1_HEADER_DUMP)
6918 drm_printf(p, "%-50s \t 0x%08x\n",
6919 "regCP_MEC_ME2_HEADER_DUMP",
6920 adev->gfx.ip_dump_compute_queues[index + reg]);
6921 else
6922 drm_printf(p, "%-50s \t 0x%08x\n",
6923 gc_cp_reg_list_11[reg].reg_name,
6924 adev->gfx.ip_dump_compute_queues[index + reg]);
6925 }
6926 index += reg_count;
6927 }
6928 }
6929 }
6930
6931 /* print gfx queue registers for all instances */
6932 if (!adev->gfx.ip_dump_gfx_queues)
6933 return;
6934
6935 index = 0;
6936 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6937 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n",
6938 adev->gfx.me.num_me,
6939 adev->gfx.me.num_pipe_per_me,
6940 adev->gfx.me.num_queue_per_pipe);
6941
6942 for (i = 0; i < adev->gfx.me.num_me; i++) {
6943 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6944 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6945 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k);
6946 for (reg = 0; reg < reg_count; reg++) {
6947 drm_printf(p, "%-50s \t 0x%08x\n",
6948 gc_gfx_queue_reg_list_11[reg].reg_name,
6949 adev->gfx.ip_dump_gfx_queues[index + reg]);
6950 }
6951 index += reg_count;
6952 }
6953 }
6954 }
6955 }
6956
gfx_v11_ip_dump(struct amdgpu_ip_block * ip_block)6957 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block)
6958 {
6959 struct amdgpu_device *adev = ip_block->adev;
6960 uint32_t i, j, k, reg, index = 0;
6961 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6962
6963 if (!adev->gfx.ip_dump_core)
6964 return;
6965
6966 amdgpu_gfx_off_ctrl(adev, false);
6967 for (i = 0; i < reg_count; i++)
6968 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i]));
6969 amdgpu_gfx_off_ctrl(adev, true);
6970
6971 /* dump compute queue registers for all instances */
6972 if (!adev->gfx.ip_dump_compute_queues)
6973 return;
6974
6975 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6976 amdgpu_gfx_off_ctrl(adev, false);
6977 mutex_lock(&adev->srbm_mutex);
6978 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6979 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6980 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6981 /* ME0 is for GFX so start from 1 for CP */
6982 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
6983 for (reg = 0; reg < reg_count; reg++) {
6984 if (i &&
6985 gc_cp_reg_list_11[reg].reg_offset ==
6986 regCP_MEC_ME1_HEADER_DUMP)
6987 adev->gfx.ip_dump_compute_queues[index + reg] =
6988 RREG32(SOC15_REG_OFFSET(GC, 0,
6989 regCP_MEC_ME2_HEADER_DUMP));
6990 else
6991 adev->gfx.ip_dump_compute_queues[index + reg] =
6992 RREG32(SOC15_REG_ENTRY_OFFSET(
6993 gc_cp_reg_list_11[reg]));
6994 }
6995 index += reg_count;
6996 }
6997 }
6998 }
6999 soc21_grbm_select(adev, 0, 0, 0, 0);
7000 mutex_unlock(&adev->srbm_mutex);
7001 amdgpu_gfx_off_ctrl(adev, true);
7002
7003 /* dump gfx queue registers for all instances */
7004 if (!adev->gfx.ip_dump_gfx_queues)
7005 return;
7006
7007 index = 0;
7008 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
7009 amdgpu_gfx_off_ctrl(adev, false);
7010 mutex_lock(&adev->srbm_mutex);
7011 for (i = 0; i < adev->gfx.me.num_me; i++) {
7012 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
7013 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
7014 soc21_grbm_select(adev, i, j, k, 0);
7015
7016 for (reg = 0; reg < reg_count; reg++) {
7017 adev->gfx.ip_dump_gfx_queues[index + reg] =
7018 RREG32(SOC15_REG_ENTRY_OFFSET(
7019 gc_gfx_queue_reg_list_11[reg]));
7020 }
7021 index += reg_count;
7022 }
7023 }
7024 }
7025 soc21_grbm_select(adev, 0, 0, 0, 0);
7026 mutex_unlock(&adev->srbm_mutex);
7027 amdgpu_gfx_off_ctrl(adev, true);
7028 }
7029
gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring * ring)7030 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7031 {
7032 /* Emit the cleaner shader */
7033 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7034 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */
7035 }
7036
gfx_v11_0_ring_begin_use(struct amdgpu_ring * ring)7037 static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring)
7038 {
7039 amdgpu_gfx_profile_ring_begin_use(ring);
7040
7041 amdgpu_gfx_enforce_isolation_ring_begin_use(ring);
7042 }
7043
gfx_v11_0_ring_end_use(struct amdgpu_ring * ring)7044 static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring)
7045 {
7046 amdgpu_gfx_profile_ring_end_use(ring);
7047
7048 amdgpu_gfx_enforce_isolation_ring_end_use(ring);
7049 }
7050
7051 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
7052 .name = "gfx_v11_0",
7053 .early_init = gfx_v11_0_early_init,
7054 .sw_init = gfx_v11_0_sw_init,
7055 .sw_fini = gfx_v11_0_sw_fini,
7056 .hw_init = gfx_v11_0_hw_init,
7057 .hw_fini = gfx_v11_0_hw_fini,
7058 .suspend = gfx_v11_0_suspend,
7059 .resume = gfx_v11_0_resume,
7060 .is_idle = gfx_v11_0_is_idle,
7061 .wait_for_idle = gfx_v11_0_wait_for_idle,
7062 .soft_reset = gfx_v11_0_soft_reset,
7063 .set_clockgating_state = gfx_v11_0_set_clockgating_state,
7064 .set_powergating_state = gfx_v11_0_set_powergating_state,
7065 .get_clockgating_state = gfx_v11_0_get_clockgating_state,
7066 .dump_ip_state = gfx_v11_ip_dump,
7067 .print_ip_state = gfx_v11_ip_print,
7068 };
7069
7070 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
7071 .type = AMDGPU_RING_TYPE_GFX,
7072 .align_mask = 0xff,
7073 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7074 .support_64bit_ptrs = true,
7075 .secure_submission_supported = true,
7076 .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
7077 .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
7078 .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
7079 .emit_frame_size = /* totally 247 maximum if 16 IBs */
7080 5 + /* update_spm_vmid */
7081 5 + /* COND_EXEC */
7082 22 + /* SET_Q_PREEMPTION_MODE */
7083 7 + /* PIPELINE_SYNC */
7084 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7085 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7086 4 + /* VM_FLUSH */
7087 8 + /* FENCE for VM_FLUSH */
7088 20 + /* GDS switch */
7089 5 + /* COND_EXEC */
7090 7 + /* HDP_flush */
7091 4 + /* VGT_flush */
7092 31 + /* DE_META */
7093 3 + /* CNTX_CTRL */
7094 5 + /* HDP_INVL */
7095 22 + /* SET_Q_PREEMPTION_MODE */
7096 8 + 8 + /* FENCE x2 */
7097 8 + /* gfx_v11_0_emit_mem_sync */
7098 2, /* gfx_v11_0_ring_emit_cleaner_shader */
7099 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
7100 .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
7101 .emit_fence = gfx_v11_0_ring_emit_fence,
7102 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
7103 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
7104 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
7105 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
7106 .test_ring = gfx_v11_0_ring_test_ring,
7107 .test_ib = gfx_v11_0_ring_test_ib,
7108 .insert_nop = gfx_v11_ring_insert_nop,
7109 .pad_ib = amdgpu_ring_generic_pad_ib,
7110 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
7111 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
7112 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
7113 .preempt_ib = amdgpu_gfx_ring_preempt_ib,
7114 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
7115 .emit_wreg = gfx_v11_0_ring_emit_wreg,
7116 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
7117 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
7118 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
7119 .reset = gfx_v11_0_reset_kgq,
7120 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader,
7121 .begin_use = gfx_v11_0_ring_begin_use,
7122 .end_use = gfx_v11_0_ring_end_use,
7123 };
7124
7125 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
7126 .type = AMDGPU_RING_TYPE_COMPUTE,
7127 .align_mask = 0xff,
7128 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7129 .support_64bit_ptrs = true,
7130 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
7131 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
7132 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
7133 .emit_frame_size =
7134 5 + /* update_spm_vmid */
7135 20 + /* gfx_v11_0_ring_emit_gds_switch */
7136 7 + /* gfx_v11_0_ring_emit_hdp_flush */
7137 5 + /* hdp invalidate */
7138 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
7139 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7140 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7141 2 + /* gfx_v11_0_ring_emit_vm_flush */
7142 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
7143 8 + /* gfx_v11_0_emit_mem_sync */
7144 2, /* gfx_v11_0_ring_emit_cleaner_shader */
7145 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
7146 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
7147 .emit_fence = gfx_v11_0_ring_emit_fence,
7148 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
7149 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
7150 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
7151 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
7152 .test_ring = gfx_v11_0_ring_test_ring,
7153 .test_ib = gfx_v11_0_ring_test_ib,
7154 .insert_nop = gfx_v11_ring_insert_nop,
7155 .pad_ib = amdgpu_ring_generic_pad_ib,
7156 .emit_wreg = gfx_v11_0_ring_emit_wreg,
7157 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
7158 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
7159 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
7160 .reset = gfx_v11_0_reset_kcq,
7161 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader,
7162 .begin_use = gfx_v11_0_ring_begin_use,
7163 .end_use = gfx_v11_0_ring_end_use,
7164 };
7165
7166 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
7167 .type = AMDGPU_RING_TYPE_KIQ,
7168 .align_mask = 0xff,
7169 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7170 .support_64bit_ptrs = true,
7171 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
7172 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
7173 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
7174 .emit_frame_size =
7175 20 + /* gfx_v11_0_ring_emit_gds_switch */
7176 7 + /* gfx_v11_0_ring_emit_hdp_flush */
7177 5 + /*hdp invalidate */
7178 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
7179 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7180 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7181 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7182 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
7183 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
7184 .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
7185 .test_ring = gfx_v11_0_ring_test_ring,
7186 .test_ib = gfx_v11_0_ring_test_ib,
7187 .insert_nop = amdgpu_ring_insert_nop,
7188 .pad_ib = amdgpu_ring_generic_pad_ib,
7189 .emit_rreg = gfx_v11_0_ring_emit_rreg,
7190 .emit_wreg = gfx_v11_0_ring_emit_wreg,
7191 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
7192 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
7193 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
7194 };
7195
gfx_v11_0_set_ring_funcs(struct amdgpu_device * adev)7196 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
7197 {
7198 int i;
7199
7200 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq;
7201
7202 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7203 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
7204
7205 for (i = 0; i < adev->gfx.num_compute_rings; i++)
7206 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
7207 }
7208
7209 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
7210 .set = gfx_v11_0_set_eop_interrupt_state,
7211 .process = gfx_v11_0_eop_irq,
7212 };
7213
7214 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
7215 .set = gfx_v11_0_set_priv_reg_fault_state,
7216 .process = gfx_v11_0_priv_reg_irq,
7217 };
7218
7219 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = {
7220 .set = gfx_v11_0_set_bad_op_fault_state,
7221 .process = gfx_v11_0_bad_op_irq,
7222 };
7223
7224 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
7225 .set = gfx_v11_0_set_priv_inst_fault_state,
7226 .process = gfx_v11_0_priv_inst_irq,
7227 };
7228
7229 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
7230 .process = gfx_v11_0_rlc_gc_fed_irq,
7231 };
7232
gfx_v11_0_set_irq_funcs(struct amdgpu_device * adev)7233 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
7234 {
7235 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7236 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
7237
7238 adev->gfx.priv_reg_irq.num_types = 1;
7239 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
7240
7241 adev->gfx.bad_op_irq.num_types = 1;
7242 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs;
7243
7244 adev->gfx.priv_inst_irq.num_types = 1;
7245 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
7246
7247 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
7248 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
7249
7250 }
7251
gfx_v11_0_set_imu_funcs(struct amdgpu_device * adev)7252 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
7253 {
7254 if (adev->flags & AMD_IS_APU)
7255 adev->gfx.imu.mode = MISSION_MODE;
7256 else
7257 adev->gfx.imu.mode = DEBUG_MODE;
7258
7259 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
7260 }
7261
gfx_v11_0_set_rlc_funcs(struct amdgpu_device * adev)7262 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
7263 {
7264 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
7265 }
7266
gfx_v11_0_set_gds_init(struct amdgpu_device * adev)7267 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
7268 {
7269 unsigned total_cu = adev->gfx.config.max_cu_per_sh *
7270 adev->gfx.config.max_sh_per_se *
7271 adev->gfx.config.max_shader_engines;
7272
7273 adev->gds.gds_size = 0x1000;
7274 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
7275 adev->gds.gws_size = 64;
7276 adev->gds.oa_size = 16;
7277 }
7278
gfx_v11_0_set_mqd_funcs(struct amdgpu_device * adev)7279 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
7280 {
7281 /* set gfx eng mqd */
7282 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
7283 sizeof(struct v11_gfx_mqd);
7284 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
7285 gfx_v11_0_gfx_mqd_init;
7286 /* set compute eng mqd */
7287 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
7288 sizeof(struct v11_compute_mqd);
7289 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
7290 gfx_v11_0_compute_mqd_init;
7291 }
7292
gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device * adev,u32 bitmap)7293 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
7294 u32 bitmap)
7295 {
7296 u32 data;
7297
7298 if (!bitmap)
7299 return;
7300
7301 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7302 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7303
7304 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
7305 }
7306
gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device * adev)7307 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
7308 {
7309 u32 data, wgp_bitmask;
7310 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
7311 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
7312
7313 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7314 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7315
7316 wgp_bitmask =
7317 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
7318
7319 return (~data) & wgp_bitmask;
7320 }
7321
gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device * adev)7322 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
7323 {
7324 u32 wgp_idx, wgp_active_bitmap;
7325 u32 cu_bitmap_per_wgp, cu_active_bitmap;
7326
7327 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
7328 cu_active_bitmap = 0;
7329
7330 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
7331 /* if there is one WGP enabled, it means 2 CUs will be enabled */
7332 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
7333 if (wgp_active_bitmap & (1 << wgp_idx))
7334 cu_active_bitmap |= cu_bitmap_per_wgp;
7335 }
7336
7337 return cu_active_bitmap;
7338 }
7339
gfx_v11_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7340 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
7341 struct amdgpu_cu_info *cu_info)
7342 {
7343 int i, j, k, counter, active_cu_number = 0;
7344 u32 mask, bitmap;
7345 unsigned disable_masks[8 * 2];
7346
7347 if (!adev || !cu_info)
7348 return -EINVAL;
7349
7350 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 8, 2);
7351
7352 mutex_lock(&adev->grbm_idx_mutex);
7353 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7354 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7355 bitmap = i * adev->gfx.config.max_sh_per_se + j;
7356 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
7357 continue;
7358 mask = 1;
7359 counter = 0;
7360 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0);
7361 if (i < 8 && j < 2)
7362 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
7363 adev, disable_masks[i * 2 + j]);
7364 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
7365
7366 /**
7367 * GFX11 could support more than 4 SEs, while the bitmap
7368 * in cu_info struct is 4x4 and ioctl interface struct
7369 * drm_amdgpu_info_device should keep stable.
7370 * So we use last two columns of bitmap to store cu mask for
7371 * SEs 4 to 7, the layout of the bitmap is as below:
7372 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
7373 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
7374 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
7375 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
7376 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
7377 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
7378 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
7379 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
7380 */
7381 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
7382
7383 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
7384 if (bitmap & mask)
7385 counter++;
7386
7387 mask <<= 1;
7388 }
7389 active_cu_number += counter;
7390 }
7391 }
7392 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7393 mutex_unlock(&adev->grbm_idx_mutex);
7394
7395 cu_info->number = active_cu_number;
7396 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7397
7398 return 0;
7399 }
7400
7401 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
7402 {
7403 .type = AMD_IP_BLOCK_TYPE_GFX,
7404 .major = 11,
7405 .minor = 0,
7406 .rev = 0,
7407 .funcs = &gfx_v11_0_ip_funcs,
7408 };
7409