1 /*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/delay.h>
24 #include <linux/kernel.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 #include "amdgpu.h"
29 #include "amdgpu_gfx.h"
30 #include "amdgpu_psp.h"
31 #include "amdgpu_smu.h"
32 #include "imu_v11_0.h"
33 #include "soc21.h"
34 #include "nvd.h"
35
36 #include "gc/gc_11_0_0_offset.h"
37 #include "gc/gc_11_0_0_sh_mask.h"
38 #include "smuio/smuio_13_0_6_offset.h"
39 #include "smuio/smuio_13_0_6_sh_mask.h"
40 #include "navi10_enum.h"
41 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
42
43 #include "soc15.h"
44 #include "clearstate_gfx11.h"
45 #include "v11_structs.h"
46 #include "gfx_v11_0.h"
47 #include "gfx_v11_0_cleaner_shader.h"
48 #include "gfx_v11_0_3.h"
49 #include "nbio_v4_3.h"
50 #include "mes_v11_0.h"
51 #include "mes_userqueue.h"
52 #include "amdgpu_userq_fence.h"
53
54 #define GFX11_NUM_GFX_RINGS 1
55 #define GFX11_MEC_HPD_SIZE 2048
56
57 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
58 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388
59
60 #define regCGTT_WD_CLK_CTRL 0x5086
61 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1
62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e
63 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1
64 #define regPC_CONFIG_CNTL_1 0x194d
65 #define regPC_CONFIG_CNTL_1_BASE_IDX 1
66
67 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100
68 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000
69 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000
70 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01
71 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000
72 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000
73 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000
74
75 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006
76 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000
77 #define regCP_MQD_CONTROL_DEFAULT 0x00000100
78 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509
79 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000
80 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000
81 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501
82 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000
83
84 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
85 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
86 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
87 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
88 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_kicker.bin");
89 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin");
90 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
91 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
92 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
93 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
94 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
95 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
96 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
97 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
98 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
99 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
100 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
101 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
102 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
103 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
104 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
105 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
106 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
107 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin");
108 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin");
109 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin");
110 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin");
111 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin");
112 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin");
113 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin");
114 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin");
115 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin");
116 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin");
117 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin");
118 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin");
119 MODULE_FIRMWARE("amdgpu/gc_11_5_3_pfp.bin");
120 MODULE_FIRMWARE("amdgpu/gc_11_5_3_me.bin");
121 MODULE_FIRMWARE("amdgpu/gc_11_5_3_mec.bin");
122 MODULE_FIRMWARE("amdgpu/gc_11_5_3_rlc.bin");
123
124 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = {
125 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
126 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
127 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
128 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
129 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
130 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
131 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
132 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
133 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
134 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
135 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
136 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
137 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
138 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
139 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
140 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
141 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
142 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
143 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
144 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
145 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
146 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
147 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE),
148 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR),
149 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR),
150 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
151 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
152 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
153 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
154 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
155 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
156 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
157 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
158 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
159 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
160 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
161 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
162 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
163 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
164 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
165 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
166 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
167 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
168 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
169 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
170 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
171 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL),
172 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS),
173 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
174 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
175 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
176 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
177 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),
178 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
179 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
180 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
181 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
182 /* cp header registers */
183 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
184 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
185 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
186 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
187 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
188 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
189 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
190 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
191 /* SE status registers */
192 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
193 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
194 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
195 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3),
196 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4),
197 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5)
198 };
199
200 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = {
201 /* compute registers */
202 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
203 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
204 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
205 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
206 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
207 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
208 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
209 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
210 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
211 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
212 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
213 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
214 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
215 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
216 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
217 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
218 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
219 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
220 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
221 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
222 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
232 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
233 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
234 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
235 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
236 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
237 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
238 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
239 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
240 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS),
241 /* cp header registers */
242 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
243 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
244 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
245 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
246 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
247 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
248 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
249 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
250 };
251
252 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = {
253 /* gfx queue registers */
254 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE),
255 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID),
256 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY),
257 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM),
258 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE),
259 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI),
260 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET),
261 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL),
262 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR),
263 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR),
264 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI),
265 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST),
266 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED),
267 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL),
268 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0),
269 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0),
270 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR),
271 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI),
272 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO),
273 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI),
274 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
275 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
276 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
277 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
278 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
279 /* cp header registers */
280 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
281 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
282 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
283 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
284 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
285 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
286 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
287 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
288 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
289 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
290 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
291 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
292 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
293 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
294 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
295 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
296 };
297
298 static const struct soc15_reg_golden golden_settings_gc_11_0[] = {
299 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000)
300 };
301
302 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
303 {
304 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
305 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
306 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
307 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
308 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
309 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
310 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
311 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
312 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
313 };
314
315 #define DEFAULT_SH_MEM_CONFIG \
316 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
317 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
318 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
319
320 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
321 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
322 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
323 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
324 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
325 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
326 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
327 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
328 struct amdgpu_cu_info *cu_info);
329 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
330 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
331 u32 sh_num, u32 instance, int xcc_id);
332 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
333
334 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
335 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
336 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
337 uint32_t val);
338 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
339 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
340 uint16_t pasid, uint32_t flush_type,
341 bool all_hub, uint8_t dst_sel);
342 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
343 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
344 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
345 bool enable);
346
gfx11_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)347 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
348 {
349 struct amdgpu_device *adev = kiq_ring->adev;
350 u64 shader_mc_addr;
351
352 /* Cleaner shader MC address */
353 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
354
355 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
356 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
357 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */
358 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
359 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
360 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
361 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
362 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
363 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
364 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
365 }
366
gfx11_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)367 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
368 struct amdgpu_ring *ring)
369 {
370 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
371 uint64_t wptr_addr = ring->wptr_gpu_addr;
372 uint32_t me = 0, eng_sel = 0;
373
374 switch (ring->funcs->type) {
375 case AMDGPU_RING_TYPE_COMPUTE:
376 me = 1;
377 eng_sel = 0;
378 break;
379 case AMDGPU_RING_TYPE_GFX:
380 me = 0;
381 eng_sel = 4;
382 break;
383 case AMDGPU_RING_TYPE_MES:
384 me = 2;
385 eng_sel = 5;
386 break;
387 default:
388 WARN_ON(1);
389 }
390
391 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
392 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
393 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
394 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
395 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
396 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
397 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
398 PACKET3_MAP_QUEUES_ME((me)) |
399 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
400 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
401 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
402 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
403 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
404 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
405 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
406 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
407 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
408 }
409
gfx11_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)410 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
411 struct amdgpu_ring *ring,
412 enum amdgpu_unmap_queues_action action,
413 u64 gpu_addr, u64 seq)
414 {
415 struct amdgpu_device *adev = kiq_ring->adev;
416 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
417
418 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
419 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
420 return;
421 }
422
423 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
424 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
425 PACKET3_UNMAP_QUEUES_ACTION(action) |
426 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
427 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
428 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
429 amdgpu_ring_write(kiq_ring,
430 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
431
432 if (action == PREEMPT_QUEUES_NO_UNMAP) {
433 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
434 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
435 amdgpu_ring_write(kiq_ring, seq);
436 } else {
437 amdgpu_ring_write(kiq_ring, 0);
438 amdgpu_ring_write(kiq_ring, 0);
439 amdgpu_ring_write(kiq_ring, 0);
440 }
441 }
442
gfx11_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)443 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
444 struct amdgpu_ring *ring,
445 u64 addr,
446 u64 seq)
447 {
448 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
449
450 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
451 amdgpu_ring_write(kiq_ring,
452 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
453 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
454 PACKET3_QUERY_STATUS_COMMAND(2));
455 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
456 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
457 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
458 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
459 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
460 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
461 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
462 }
463
gfx11_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)464 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
465 uint16_t pasid, uint32_t flush_type,
466 bool all_hub)
467 {
468 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
469 }
470
471 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
472 .kiq_set_resources = gfx11_kiq_set_resources,
473 .kiq_map_queues = gfx11_kiq_map_queues,
474 .kiq_unmap_queues = gfx11_kiq_unmap_queues,
475 .kiq_query_status = gfx11_kiq_query_status,
476 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
477 .set_resources_size = 8,
478 .map_queues_size = 7,
479 .unmap_queues_size = 6,
480 .query_status_size = 7,
481 .invalidate_tlbs_size = 2,
482 };
483
gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)484 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
485 {
486 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs;
487 }
488
gfx_v11_0_init_golden_registers(struct amdgpu_device * adev)489 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
490 {
491 if (amdgpu_sriov_vf(adev))
492 return;
493
494 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
495 case IP_VERSION(11, 0, 1):
496 case IP_VERSION(11, 0, 4):
497 soc15_program_register_sequence(adev,
498 golden_settings_gc_11_0_1,
499 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
500 break;
501 default:
502 break;
503 }
504 soc15_program_register_sequence(adev,
505 golden_settings_gc_11_0,
506 (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
507
508 }
509
gfx_v11_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)510 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
511 bool wc, uint32_t reg, uint32_t val)
512 {
513 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
514 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
515 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
516 amdgpu_ring_write(ring, reg);
517 amdgpu_ring_write(ring, 0);
518 amdgpu_ring_write(ring, val);
519 }
520
gfx_v11_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)521 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
522 int mem_space, int opt, uint32_t addr0,
523 uint32_t addr1, uint32_t ref, uint32_t mask,
524 uint32_t inv)
525 {
526 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
527 amdgpu_ring_write(ring,
528 /* memory (1) or register (0) */
529 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
530 WAIT_REG_MEM_OPERATION(opt) | /* wait */
531 WAIT_REG_MEM_FUNCTION(3) | /* equal */
532 WAIT_REG_MEM_ENGINE(eng_sel)));
533
534 if (mem_space)
535 BUG_ON(addr0 & 0x3); /* Dword align */
536 amdgpu_ring_write(ring, addr0);
537 amdgpu_ring_write(ring, addr1);
538 amdgpu_ring_write(ring, ref);
539 amdgpu_ring_write(ring, mask);
540 amdgpu_ring_write(ring, inv); /* poll interval */
541 }
542
gfx_v11_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)543 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
544 {
545 /* Header itself is a NOP packet */
546 if (num_nop == 1) {
547 amdgpu_ring_write(ring, ring->funcs->nop);
548 return;
549 }
550
551 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
552 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
553
554 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
555 amdgpu_ring_insert_nop(ring, num_nop - 1);
556 }
557
gfx_v11_0_ring_test_ring(struct amdgpu_ring * ring)558 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
559 {
560 struct amdgpu_device *adev = ring->adev;
561 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
562 uint32_t tmp = 0;
563 unsigned i;
564 int r;
565
566 WREG32(scratch, 0xCAFEDEAD);
567 r = amdgpu_ring_alloc(ring, 5);
568 if (r) {
569 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
570 ring->idx, r);
571 return r;
572 }
573
574 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
575 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
576 } else {
577 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
578 amdgpu_ring_write(ring, scratch -
579 PACKET3_SET_UCONFIG_REG_START);
580 amdgpu_ring_write(ring, 0xDEADBEEF);
581 }
582 amdgpu_ring_commit(ring);
583
584 for (i = 0; i < adev->usec_timeout; i++) {
585 tmp = RREG32(scratch);
586 if (tmp == 0xDEADBEEF)
587 break;
588 if (amdgpu_emu_mode == 1)
589 msleep(1);
590 else
591 udelay(1);
592 }
593
594 if (i >= adev->usec_timeout)
595 r = -ETIMEDOUT;
596 return r;
597 }
598
gfx_v11_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)599 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
600 {
601 struct amdgpu_device *adev = ring->adev;
602 struct amdgpu_ib ib;
603 struct dma_fence *f = NULL;
604 unsigned index;
605 uint64_t gpu_addr;
606 uint32_t *cpu_ptr;
607 long r;
608
609 /* MES KIQ fw hasn't indirect buffer support for now */
610 if (adev->enable_mes_kiq &&
611 ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
612 return 0;
613
614 memset(&ib, 0, sizeof(ib));
615
616 r = amdgpu_device_wb_get(adev, &index);
617 if (r)
618 return r;
619
620 gpu_addr = adev->wb.gpu_addr + (index * 4);
621 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
622 cpu_ptr = &adev->wb.wb[index];
623
624 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
625 if (r) {
626 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
627 goto err1;
628 }
629
630 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
631 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
632 ib.ptr[2] = lower_32_bits(gpu_addr);
633 ib.ptr[3] = upper_32_bits(gpu_addr);
634 ib.ptr[4] = 0xDEADBEEF;
635 ib.length_dw = 5;
636
637 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
638 if (r)
639 goto err2;
640
641 r = dma_fence_wait_timeout(f, false, timeout);
642 if (r == 0) {
643 r = -ETIMEDOUT;
644 goto err2;
645 } else if (r < 0) {
646 goto err2;
647 }
648
649 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
650 r = 0;
651 else
652 r = -EINVAL;
653 err2:
654 amdgpu_ib_free(&ib, NULL);
655 dma_fence_put(f);
656 err1:
657 amdgpu_device_wb_free(adev, index);
658 return r;
659 }
660
gfx_v11_0_free_microcode(struct amdgpu_device * adev)661 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
662 {
663 amdgpu_ucode_release(&adev->gfx.pfp_fw);
664 amdgpu_ucode_release(&adev->gfx.me_fw);
665 amdgpu_ucode_release(&adev->gfx.rlc_fw);
666 amdgpu_ucode_release(&adev->gfx.mec_fw);
667
668 kfree(adev->gfx.rlc.register_list_format);
669 }
670
gfx_v11_0_init_toc_microcode(struct amdgpu_device * adev,const char * ucode_prefix)671 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
672 {
673 const struct psp_firmware_header_v1_0 *toc_hdr;
674 int err = 0;
675
676 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
677 AMDGPU_UCODE_REQUIRED,
678 "amdgpu/%s_toc.bin", ucode_prefix);
679 if (err)
680 goto out;
681
682 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
683 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
684 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
685 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
686 adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
687 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
688 return 0;
689 out:
690 amdgpu_ucode_release(&adev->psp.toc_fw);
691 return err;
692 }
693
gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device * adev)694 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
695 {
696 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
697 case IP_VERSION(11, 0, 0):
698 case IP_VERSION(11, 0, 2):
699 case IP_VERSION(11, 0, 3):
700 if ((adev->gfx.me_fw_version >= 1505) &&
701 (adev->gfx.pfp_fw_version >= 1600) &&
702 (adev->gfx.mec_fw_version >= 512)) {
703 if (amdgpu_sriov_vf(adev))
704 adev->gfx.cp_gfx_shadow = true;
705 else
706 adev->gfx.cp_gfx_shadow = false;
707 }
708 break;
709 default:
710 adev->gfx.cp_gfx_shadow = false;
711 break;
712 }
713 }
714
gfx_v11_0_init_microcode(struct amdgpu_device * adev)715 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
716 {
717 char ucode_prefix[25];
718 int err;
719 const struct rlc_firmware_header_v2_0 *rlc_hdr;
720 uint16_t version_major;
721 uint16_t version_minor;
722
723 DRM_DEBUG("\n");
724
725 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
726 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
727 AMDGPU_UCODE_REQUIRED,
728 "amdgpu/%s_pfp.bin", ucode_prefix);
729 if (err)
730 goto out;
731 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
732 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
733 (union amdgpu_firmware_header *)
734 adev->gfx.pfp_fw->data, 2, 0);
735 if (adev->gfx.rs64_enable) {
736 dev_info(adev->dev, "CP RS64 enable\n");
737 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
738 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
739 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
740 } else {
741 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
742 }
743
744 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
745 AMDGPU_UCODE_REQUIRED,
746 "amdgpu/%s_me.bin", ucode_prefix);
747 if (err)
748 goto out;
749 if (adev->gfx.rs64_enable) {
750 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
751 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
752 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
753 } else {
754 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
755 }
756
757 if (!amdgpu_sriov_vf(adev)) {
758 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) &&
759 adev->pdev->revision == 0xCE)
760 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
761 AMDGPU_UCODE_REQUIRED,
762 "amdgpu/gc_11_0_0_rlc_1.bin");
763 else if (amdgpu_is_kicker_fw(adev))
764 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
765 AMDGPU_UCODE_REQUIRED,
766 "amdgpu/%s_rlc_kicker.bin", ucode_prefix);
767 else
768 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
769 AMDGPU_UCODE_REQUIRED,
770 "amdgpu/%s_rlc.bin", ucode_prefix);
771 if (err)
772 goto out;
773 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
774 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
775 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
776 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
777 if (err)
778 goto out;
779 }
780
781 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
782 AMDGPU_UCODE_REQUIRED,
783 "amdgpu/%s_mec.bin", ucode_prefix);
784 if (err)
785 goto out;
786 if (adev->gfx.rs64_enable) {
787 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
788 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
789 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
790 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
791 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
792 } else {
793 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
794 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
795 }
796
797 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
798 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
799
800 /* only one MEC for gfx 11.0.0. */
801 adev->gfx.mec2_fw = NULL;
802
803 gfx_v11_0_check_fw_cp_gfx_shadow(adev);
804
805 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) {
806 err = adev->gfx.imu.funcs->init_microcode(adev);
807 if (err)
808 DRM_ERROR("Failed to init imu firmware!\n");
809 return err;
810 }
811
812 out:
813 if (err) {
814 amdgpu_ucode_release(&adev->gfx.pfp_fw);
815 amdgpu_ucode_release(&adev->gfx.me_fw);
816 amdgpu_ucode_release(&adev->gfx.rlc_fw);
817 amdgpu_ucode_release(&adev->gfx.mec_fw);
818 }
819
820 return err;
821 }
822
gfx_v11_0_get_csb_size(struct amdgpu_device * adev)823 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
824 {
825 u32 count = 0;
826 const struct cs_section_def *sect = NULL;
827 const struct cs_extent_def *ext = NULL;
828
829 /* begin clear state */
830 count += 2;
831 /* context control state */
832 count += 3;
833
834 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
835 for (ext = sect->section; ext->extent != NULL; ++ext) {
836 if (sect->id == SECT_CONTEXT)
837 count += 2 + ext->reg_count;
838 else
839 return 0;
840 }
841 }
842
843 /* set PA_SC_TILE_STEERING_OVERRIDE */
844 count += 3;
845 /* end clear state */
846 count += 2;
847 /* clear state */
848 count += 2;
849
850 return count;
851 }
852
gfx_v11_0_get_csb_buffer(struct amdgpu_device * adev,u32 * buffer)853 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
854 {
855 u32 count = 0;
856 int ctx_reg_offset;
857
858 if (adev->gfx.rlc.cs_data == NULL)
859 return;
860 if (buffer == NULL)
861 return;
862
863 count = amdgpu_gfx_csb_preamble_start(buffer);
864 count = amdgpu_gfx_csb_data_parser(adev, buffer, count);
865
866 ctx_reg_offset = SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
867 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
868 buffer[count++] = cpu_to_le32(ctx_reg_offset);
869 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
870
871 amdgpu_gfx_csb_preamble_end(buffer, count);
872 }
873
gfx_v11_0_rlc_fini(struct amdgpu_device * adev)874 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
875 {
876 /* clear state block */
877 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
878 &adev->gfx.rlc.clear_state_gpu_addr,
879 (void **)&adev->gfx.rlc.cs_ptr);
880
881 /* jump table block */
882 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
883 &adev->gfx.rlc.cp_table_gpu_addr,
884 (void **)&adev->gfx.rlc.cp_table_ptr);
885 }
886
gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)887 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
888 {
889 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
890
891 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
892 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
893 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
894 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
895 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
896 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
897 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
898 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
899 adev->gfx.rlc.rlcg_reg_access_supported = true;
900 }
901
gfx_v11_0_rlc_init(struct amdgpu_device * adev)902 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
903 {
904 const struct cs_section_def *cs_data;
905 int r;
906
907 adev->gfx.rlc.cs_data = gfx11_cs_data;
908
909 cs_data = adev->gfx.rlc.cs_data;
910
911 if (cs_data) {
912 /* init clear state block */
913 r = amdgpu_gfx_rlc_init_csb(adev);
914 if (r)
915 return r;
916 }
917
918 /* init spm vmid with 0xf */
919 if (adev->gfx.rlc.funcs->update_spm_vmid)
920 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
921
922 return 0;
923 }
924
gfx_v11_0_mec_fini(struct amdgpu_device * adev)925 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
926 {
927 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
928 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
929 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
930 }
931
gfx_v11_0_me_init(struct amdgpu_device * adev)932 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
933 {
934 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
935
936 amdgpu_gfx_graphics_queue_acquire(adev);
937 }
938
gfx_v11_0_mec_init(struct amdgpu_device * adev)939 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
940 {
941 int r;
942 u32 *hpd;
943 size_t mec_hpd_size;
944
945 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
946
947 /* take ownership of the relevant compute queues */
948 amdgpu_gfx_compute_queue_acquire(adev);
949 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
950
951 if (mec_hpd_size) {
952 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
953 AMDGPU_GEM_DOMAIN_GTT,
954 &adev->gfx.mec.hpd_eop_obj,
955 &adev->gfx.mec.hpd_eop_gpu_addr,
956 (void **)&hpd);
957 if (r) {
958 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
959 gfx_v11_0_mec_fini(adev);
960 return r;
961 }
962
963 memset(hpd, 0, mec_hpd_size);
964
965 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
966 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
967 }
968
969 return 0;
970 }
971
wave_read_ind(struct amdgpu_device * adev,uint32_t wave,uint32_t address)972 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
973 {
974 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
975 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
976 (address << SQ_IND_INDEX__INDEX__SHIFT));
977 return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
978 }
979
wave_read_regs(struct amdgpu_device * adev,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)980 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
981 uint32_t thread, uint32_t regno,
982 uint32_t num, uint32_t *out)
983 {
984 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
985 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
986 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
987 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
988 (SQ_IND_INDEX__AUTO_INCR_MASK));
989 while (num--)
990 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
991 }
992
gfx_v11_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)993 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
994 {
995 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
996 * field when performing a select_se_sh so it should be
997 * zero here */
998 WARN_ON(simd != 0);
999
1000 /* type 3 wave data */
1001 dst[(*no_fields)++] = 3;
1002 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
1003 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
1004 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
1005 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
1006 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
1007 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
1008 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
1009 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
1010 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
1011 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
1012 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
1013 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
1014 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
1015 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
1016 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
1017 }
1018
gfx_v11_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)1019 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1020 uint32_t wave, uint32_t start,
1021 uint32_t size, uint32_t *dst)
1022 {
1023 WARN_ON(simd != 0);
1024
1025 wave_read_regs(
1026 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
1027 dst);
1028 }
1029
gfx_v11_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)1030 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1031 uint32_t wave, uint32_t thread,
1032 uint32_t start, uint32_t size,
1033 uint32_t *dst)
1034 {
1035 wave_read_regs(
1036 adev, wave, thread,
1037 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1038 }
1039
gfx_v11_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)1040 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
1041 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1042 {
1043 soc21_grbm_select(adev, me, pipe, q, vm);
1044 }
1045
1046 /* all sizes are in bytes */
1047 #define MQD_SHADOW_BASE_SIZE 73728
1048 #define MQD_SHADOW_BASE_ALIGNMENT 256
1049 #define MQD_FWWORKAREA_SIZE 484
1050 #define MQD_FWWORKAREA_ALIGNMENT 256
1051
gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device * adev,struct amdgpu_gfx_shadow_info * shadow_info)1052 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev,
1053 struct amdgpu_gfx_shadow_info *shadow_info)
1054 {
1055 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
1056 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
1057 shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
1058 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
1059 }
1060
gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device * adev,struct amdgpu_gfx_shadow_info * shadow_info,bool skip_check)1061 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
1062 struct amdgpu_gfx_shadow_info *shadow_info,
1063 bool skip_check)
1064 {
1065 if (adev->gfx.cp_gfx_shadow || skip_check) {
1066 gfx_v11_0_get_gfx_shadow_info_nocheck(adev, shadow_info);
1067 return 0;
1068 } else {
1069 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info));
1070 return -ENOTSUPP;
1071 }
1072 }
1073
1074 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
1075 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
1076 .select_se_sh = &gfx_v11_0_select_se_sh,
1077 .read_wave_data = &gfx_v11_0_read_wave_data,
1078 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
1079 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
1080 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
1081 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
1082 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
1083 };
1084
gfx_v11_0_gpu_early_init(struct amdgpu_device * adev)1085 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
1086 {
1087 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1088 case IP_VERSION(11, 0, 0):
1089 case IP_VERSION(11, 0, 2):
1090 adev->gfx.config.max_hw_contexts = 8;
1091 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1092 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1093 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1094 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1095 break;
1096 case IP_VERSION(11, 0, 3):
1097 adev->gfx.ras = &gfx_v11_0_3_ras;
1098 adev->gfx.config.max_hw_contexts = 8;
1099 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1100 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1101 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1102 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1103 break;
1104 case IP_VERSION(11, 0, 1):
1105 case IP_VERSION(11, 0, 4):
1106 case IP_VERSION(11, 5, 0):
1107 case IP_VERSION(11, 5, 1):
1108 case IP_VERSION(11, 5, 2):
1109 case IP_VERSION(11, 5, 3):
1110 adev->gfx.config.max_hw_contexts = 8;
1111 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1112 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1113 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1114 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
1115 break;
1116 default:
1117 BUG();
1118 break;
1119 }
1120
1121 return 0;
1122 }
1123
gfx_v11_0_gfx_ring_init(struct amdgpu_device * adev,int ring_id,int me,int pipe,int queue)1124 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1125 int me, int pipe, int queue)
1126 {
1127 struct amdgpu_ring *ring;
1128 unsigned int irq_type;
1129 unsigned int hw_prio;
1130
1131 ring = &adev->gfx.gfx_ring[ring_id];
1132
1133 ring->me = me;
1134 ring->pipe = pipe;
1135 ring->queue = queue;
1136
1137 ring->ring_obj = NULL;
1138 ring->use_doorbell = true;
1139 if (adev->gfx.disable_kq) {
1140 ring->no_scheduler = true;
1141 ring->no_user_submission = true;
1142 }
1143
1144 if (!ring_id)
1145 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1146 else
1147 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1148 ring->vm_hub = AMDGPU_GFXHUB(0);
1149 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1150
1151 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1152 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ?
1153 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1154 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1155 hw_prio, NULL);
1156 }
1157
gfx_v11_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)1158 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1159 int mec, int pipe, int queue)
1160 {
1161 int r;
1162 unsigned irq_type;
1163 struct amdgpu_ring *ring;
1164 unsigned int hw_prio;
1165
1166 ring = &adev->gfx.compute_ring[ring_id];
1167
1168 /* mec0 is me1 */
1169 ring->me = mec + 1;
1170 ring->pipe = pipe;
1171 ring->queue = queue;
1172
1173 ring->ring_obj = NULL;
1174 ring->use_doorbell = true;
1175 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1176 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1177 + (ring_id * GFX11_MEC_HPD_SIZE);
1178 ring->vm_hub = AMDGPU_GFXHUB(0);
1179 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1180
1181 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1182 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1183 + ring->pipe;
1184 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1185 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1186 /* type-2 packets are deprecated on MEC, use type-3 instead */
1187 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1188 hw_prio, NULL);
1189 if (r)
1190 return r;
1191
1192 return 0;
1193 }
1194
1195 static struct {
1196 SOC21_FIRMWARE_ID id;
1197 unsigned int offset;
1198 unsigned int size;
1199 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
1200
gfx_v11_0_parse_rlc_toc(struct amdgpu_device * adev,void * rlc_toc)1201 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1202 {
1203 RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
1204
1205 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1206 (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1207 rlc_autoload_info[ucode->id].id = ucode->id;
1208 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1209 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1210
1211 ucode++;
1212 }
1213 }
1214
gfx_v11_0_calc_toc_total_size(struct amdgpu_device * adev)1215 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1216 {
1217 uint32_t total_size = 0;
1218 SOC21_FIRMWARE_ID id;
1219
1220 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1221
1222 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1223 total_size += rlc_autoload_info[id].size;
1224
1225 /* In case the offset in rlc toc ucode is aligned */
1226 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1227 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1228 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1229
1230 return total_size;
1231 }
1232
gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device * adev)1233 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1234 {
1235 int r;
1236 uint32_t total_size;
1237
1238 total_size = gfx_v11_0_calc_toc_total_size(adev);
1239
1240 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1241 AMDGPU_GEM_DOMAIN_VRAM |
1242 AMDGPU_GEM_DOMAIN_GTT,
1243 &adev->gfx.rlc.rlc_autoload_bo,
1244 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1245 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1246
1247 if (r) {
1248 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1249 return r;
1250 }
1251
1252 return 0;
1253 }
1254
gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device * adev,SOC21_FIRMWARE_ID id,const void * fw_data,uint32_t fw_size,uint32_t * fw_autoload_mask)1255 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1256 SOC21_FIRMWARE_ID id,
1257 const void *fw_data,
1258 uint32_t fw_size,
1259 uint32_t *fw_autoload_mask)
1260 {
1261 uint32_t toc_offset;
1262 uint32_t toc_fw_size;
1263 char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1264
1265 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1266 return;
1267
1268 toc_offset = rlc_autoload_info[id].offset;
1269 toc_fw_size = rlc_autoload_info[id].size;
1270
1271 if (fw_size == 0)
1272 fw_size = toc_fw_size;
1273
1274 if (fw_size > toc_fw_size)
1275 fw_size = toc_fw_size;
1276
1277 memcpy(ptr + toc_offset, fw_data, fw_size);
1278
1279 if (fw_size < toc_fw_size)
1280 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1281
1282 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1283 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1284 }
1285
gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1286 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1287 uint32_t *fw_autoload_mask)
1288 {
1289 void *data;
1290 uint32_t size;
1291 uint64_t *toc_ptr;
1292
1293 *(uint64_t *)fw_autoload_mask |= 0x1;
1294
1295 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1296
1297 data = adev->psp.toc.start_addr;
1298 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1299
1300 toc_ptr = (uint64_t *)data + size / 8 - 1;
1301 *toc_ptr = *(uint64_t *)fw_autoload_mask;
1302
1303 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1304 data, size, fw_autoload_mask);
1305 }
1306
gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1307 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1308 uint32_t *fw_autoload_mask)
1309 {
1310 const __le32 *fw_data;
1311 uint32_t fw_size;
1312 const struct gfx_firmware_header_v1_0 *cp_hdr;
1313 const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1314 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1315 const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1316 uint16_t version_major, version_minor;
1317
1318 if (adev->gfx.rs64_enable) {
1319 /* pfp ucode */
1320 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1321 adev->gfx.pfp_fw->data;
1322 /* instruction */
1323 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1324 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1325 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1326 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1327 fw_data, fw_size, fw_autoload_mask);
1328 /* data */
1329 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1330 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1331 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1332 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1333 fw_data, fw_size, fw_autoload_mask);
1334 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1335 fw_data, fw_size, fw_autoload_mask);
1336 /* me ucode */
1337 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1338 adev->gfx.me_fw->data;
1339 /* instruction */
1340 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1341 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1342 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1343 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1344 fw_data, fw_size, fw_autoload_mask);
1345 /* data */
1346 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1347 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1348 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1349 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1350 fw_data, fw_size, fw_autoload_mask);
1351 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1352 fw_data, fw_size, fw_autoload_mask);
1353 /* mec ucode */
1354 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1355 adev->gfx.mec_fw->data;
1356 /* instruction */
1357 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1358 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1359 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1360 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1361 fw_data, fw_size, fw_autoload_mask);
1362 /* data */
1363 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1364 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1365 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1366 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1367 fw_data, fw_size, fw_autoload_mask);
1368 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1369 fw_data, fw_size, fw_autoload_mask);
1370 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1371 fw_data, fw_size, fw_autoload_mask);
1372 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1373 fw_data, fw_size, fw_autoload_mask);
1374 } else {
1375 /* pfp ucode */
1376 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1377 adev->gfx.pfp_fw->data;
1378 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1379 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1380 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1381 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1382 fw_data, fw_size, fw_autoload_mask);
1383
1384 /* me ucode */
1385 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1386 adev->gfx.me_fw->data;
1387 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1388 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1389 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1390 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1391 fw_data, fw_size, fw_autoload_mask);
1392
1393 /* mec ucode */
1394 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1395 adev->gfx.mec_fw->data;
1396 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1397 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1398 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1399 cp_hdr->jt_size * 4;
1400 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1401 fw_data, fw_size, fw_autoload_mask);
1402 }
1403
1404 /* rlc ucode */
1405 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1406 adev->gfx.rlc_fw->data;
1407 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1408 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1409 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1410 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1411 fw_data, fw_size, fw_autoload_mask);
1412
1413 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1414 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1415 if (version_major == 2) {
1416 if (version_minor >= 2) {
1417 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1418
1419 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1420 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1421 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1422 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1423 fw_data, fw_size, fw_autoload_mask);
1424
1425 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1426 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1427 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1428 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1429 fw_data, fw_size, fw_autoload_mask);
1430 }
1431 }
1432 }
1433
gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1434 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1435 uint32_t *fw_autoload_mask)
1436 {
1437 const __le32 *fw_data;
1438 uint32_t fw_size;
1439 const struct sdma_firmware_header_v2_0 *sdma_hdr;
1440
1441 sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1442 adev->sdma.instance[0].fw->data;
1443 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1444 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1445 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1446
1447 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1448 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1449
1450 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1451 le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1452 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1453
1454 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1455 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1456 }
1457
gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1458 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1459 uint32_t *fw_autoload_mask)
1460 {
1461 const __le32 *fw_data;
1462 unsigned fw_size;
1463 const struct mes_firmware_header_v1_0 *mes_hdr;
1464 int pipe, ucode_id, data_id;
1465
1466 for (pipe = 0; pipe < 2; pipe++) {
1467 if (pipe==0) {
1468 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1469 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1470 } else {
1471 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1472 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1473 }
1474
1475 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1476 adev->mes.fw[pipe]->data;
1477
1478 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1479 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1480 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1481
1482 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1483 ucode_id, fw_data, fw_size, fw_autoload_mask);
1484
1485 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1486 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1487 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1488
1489 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1490 data_id, fw_data, fw_size, fw_autoload_mask);
1491 }
1492 }
1493
gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device * adev)1494 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1495 {
1496 uint32_t rlc_g_offset, rlc_g_size;
1497 uint64_t gpu_addr;
1498 uint32_t autoload_fw_id[2];
1499
1500 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1501
1502 /* RLC autoload sequence 2: copy ucode */
1503 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1504 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1505 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1506 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1507
1508 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1509 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1510 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1511
1512 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1513 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1514
1515 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1516
1517 /* RLC autoload sequence 3: load IMU fw */
1518 if (adev->gfx.imu.funcs->load_microcode)
1519 adev->gfx.imu.funcs->load_microcode(adev);
1520 /* RLC autoload sequence 4 init IMU fw */
1521 if (adev->gfx.imu.funcs->setup_imu)
1522 adev->gfx.imu.funcs->setup_imu(adev);
1523 if (adev->gfx.imu.funcs->start_imu)
1524 adev->gfx.imu.funcs->start_imu(adev);
1525
1526 /* RLC autoload sequence 5 disable gpa mode */
1527 gfx_v11_0_disable_gpa_mode(adev);
1528
1529 return 0;
1530 }
1531
gfx_v11_0_alloc_ip_dump(struct amdgpu_device * adev)1532 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
1533 {
1534 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
1535 uint32_t *ptr;
1536 uint32_t inst;
1537
1538 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
1539 if (!ptr) {
1540 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1541 adev->gfx.ip_dump_core = NULL;
1542 } else {
1543 adev->gfx.ip_dump_core = ptr;
1544 }
1545
1546 /* Allocate memory for compute queue registers for all the instances */
1547 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
1548 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1549 adev->gfx.mec.num_queue_per_pipe;
1550
1551 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1552 if (!ptr) {
1553 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1554 adev->gfx.ip_dump_compute_queues = NULL;
1555 } else {
1556 adev->gfx.ip_dump_compute_queues = ptr;
1557 }
1558
1559 /* Allocate memory for gfx queue registers for all the instances */
1560 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
1561 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1562 adev->gfx.me.num_queue_per_pipe;
1563
1564 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1565 if (!ptr) {
1566 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n");
1567 adev->gfx.ip_dump_gfx_queues = NULL;
1568 } else {
1569 adev->gfx.ip_dump_gfx_queues = ptr;
1570 }
1571 }
1572
gfx_v11_0_sw_init(struct amdgpu_ip_block * ip_block)1573 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
1574 {
1575 int i, j, k, r, ring_id;
1576 int xcc_id = 0;
1577 struct amdgpu_device *adev = ip_block->adev;
1578 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
1579
1580 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler);
1581
1582 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1583 case IP_VERSION(11, 0, 0):
1584 case IP_VERSION(11, 0, 1):
1585 case IP_VERSION(11, 0, 2):
1586 case IP_VERSION(11, 0, 3):
1587 case IP_VERSION(11, 0, 4):
1588 case IP_VERSION(11, 5, 0):
1589 case IP_VERSION(11, 5, 1):
1590 case IP_VERSION(11, 5, 2):
1591 case IP_VERSION(11, 5, 3):
1592 adev->gfx.me.num_me = 1;
1593 adev->gfx.me.num_pipe_per_me = 1;
1594 adev->gfx.me.num_queue_per_pipe = 2;
1595 adev->gfx.mec.num_mec = 1;
1596 adev->gfx.mec.num_pipe_per_mec = 4;
1597 adev->gfx.mec.num_queue_per_pipe = 4;
1598 break;
1599 default:
1600 adev->gfx.me.num_me = 1;
1601 adev->gfx.me.num_pipe_per_me = 1;
1602 adev->gfx.me.num_queue_per_pipe = 1;
1603 adev->gfx.mec.num_mec = 1;
1604 adev->gfx.mec.num_pipe_per_mec = 4;
1605 adev->gfx.mec.num_queue_per_pipe = 8;
1606 break;
1607 }
1608
1609 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1610 case IP_VERSION(11, 0, 0):
1611 case IP_VERSION(11, 0, 2):
1612 case IP_VERSION(11, 0, 3):
1613 if (!adev->gfx.disable_uq &&
1614 adev->gfx.me_fw_version >= 2420 &&
1615 adev->gfx.pfp_fw_version >= 2580 &&
1616 adev->gfx.mec_fw_version >= 2650 &&
1617 adev->mes.fw_version[0] >= 120) {
1618 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs;
1619 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs;
1620 }
1621 break;
1622 case IP_VERSION(11, 0, 1):
1623 case IP_VERSION(11, 0, 4):
1624 case IP_VERSION(11, 5, 0):
1625 case IP_VERSION(11, 5, 1):
1626 case IP_VERSION(11, 5, 2):
1627 case IP_VERSION(11, 5, 3):
1628 /* add firmware version checks here */
1629 if (0 && !adev->gfx.disable_uq) {
1630 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs;
1631 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs;
1632 }
1633 break;
1634 default:
1635 break;
1636 }
1637
1638 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1639 case IP_VERSION(11, 0, 0):
1640 case IP_VERSION(11, 0, 2):
1641 case IP_VERSION(11, 0, 3):
1642 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1643 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1644 if (adev->gfx.me_fw_version >= 2280 &&
1645 adev->gfx.pfp_fw_version >= 2370 &&
1646 adev->gfx.mec_fw_version >= 2450 &&
1647 adev->mes.fw_version[0] >= 99) {
1648 adev->gfx.enable_cleaner_shader = true;
1649 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1650 if (r) {
1651 adev->gfx.enable_cleaner_shader = false;
1652 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1653 }
1654 }
1655 break;
1656 case IP_VERSION(11, 0, 1):
1657 case IP_VERSION(11, 0, 4):
1658 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1659 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1660 if (adev->gfx.pfp_fw_version >= 102 &&
1661 adev->gfx.mec_fw_version >= 66 &&
1662 adev->mes.fw_version[0] >= 128) {
1663 adev->gfx.enable_cleaner_shader = true;
1664 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1665 if (r) {
1666 adev->gfx.enable_cleaner_shader = false;
1667 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1668 }
1669 }
1670 break;
1671 case IP_VERSION(11, 5, 0):
1672 case IP_VERSION(11, 5, 1):
1673 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1674 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1675 if (adev->gfx.mec_fw_version >= 26 &&
1676 adev->mes.fw_version[0] >= 114) {
1677 adev->gfx.enable_cleaner_shader = true;
1678 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1679 if (r) {
1680 adev->gfx.enable_cleaner_shader = false;
1681 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1682 }
1683 }
1684 break;
1685 case IP_VERSION(11, 5, 2):
1686 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1687 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1688 if (adev->gfx.me_fw_version >= 12 &&
1689 adev->gfx.pfp_fw_version >= 15 &&
1690 adev->gfx.mec_fw_version >= 15) {
1691 adev->gfx.enable_cleaner_shader = true;
1692 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1693 if (r) {
1694 adev->gfx.enable_cleaner_shader = false;
1695 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1696 }
1697 }
1698 break;
1699 case IP_VERSION(11, 5, 3):
1700 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex;
1701 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex);
1702 if (adev->gfx.me_fw_version >= 7 &&
1703 adev->gfx.pfp_fw_version >= 8 &&
1704 adev->gfx.mec_fw_version >= 8) {
1705 adev->gfx.enable_cleaner_shader = true;
1706 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
1707 if (r) {
1708 adev->gfx.enable_cleaner_shader = false;
1709 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1710 }
1711 }
1712 break;
1713 default:
1714 adev->gfx.enable_cleaner_shader = false;
1715 break;
1716 }
1717
1718 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1719 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) &&
1720 amdgpu_sriov_is_pp_one_vf(adev))
1721 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1722
1723 /* EOP Event */
1724 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1725 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1726 &adev->gfx.eop_irq);
1727 if (r)
1728 return r;
1729
1730 /* Bad opcode Event */
1731 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1732 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR,
1733 &adev->gfx.bad_op_irq);
1734 if (r)
1735 return r;
1736
1737 /* Privileged reg */
1738 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1739 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1740 &adev->gfx.priv_reg_irq);
1741 if (r)
1742 return r;
1743
1744 /* Privileged inst */
1745 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1746 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1747 &adev->gfx.priv_inst_irq);
1748 if (r)
1749 return r;
1750
1751 /* FED error */
1752 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1753 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1754 &adev->gfx.rlc_gc_fed_irq);
1755 if (r)
1756 return r;
1757
1758 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1759
1760 gfx_v11_0_me_init(adev);
1761
1762 r = gfx_v11_0_rlc_init(adev);
1763 if (r) {
1764 DRM_ERROR("Failed to init rlc BOs!\n");
1765 return r;
1766 }
1767
1768 r = gfx_v11_0_mec_init(adev);
1769 if (r) {
1770 DRM_ERROR("Failed to init MEC BOs!\n");
1771 return r;
1772 }
1773
1774 if (adev->gfx.num_gfx_rings) {
1775 ring_id = 0;
1776 /* set up the gfx ring */
1777 for (i = 0; i < adev->gfx.me.num_me; i++) {
1778 for (j = 0; j < num_queue_per_pipe; j++) {
1779 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1780 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1781 continue;
1782
1783 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1784 i, k, j);
1785 if (r)
1786 return r;
1787 ring_id++;
1788 }
1789 }
1790 }
1791 }
1792
1793 if (adev->gfx.num_compute_rings) {
1794 ring_id = 0;
1795 /* set up the compute queues - allocate horizontally across pipes */
1796 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1797 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1798 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1799 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1800 k, j))
1801 continue;
1802
1803 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1804 i, k, j);
1805 if (r)
1806 return r;
1807
1808 ring_id++;
1809 }
1810 }
1811 }
1812 }
1813
1814 adev->gfx.gfx_supported_reset =
1815 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
1816 adev->gfx.compute_supported_reset =
1817 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
1818 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1819 case IP_VERSION(11, 0, 0):
1820 case IP_VERSION(11, 0, 2):
1821 case IP_VERSION(11, 0, 3):
1822 if ((adev->gfx.me_fw_version >= 2280) &&
1823 (adev->gfx.mec_fw_version >= 2410) &&
1824 !amdgpu_sriov_vf(adev)) {
1825 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1826 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1827 }
1828 break;
1829 default:
1830 if (!amdgpu_sriov_vf(adev)) {
1831 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1832 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1833 }
1834 break;
1835 }
1836
1837 if (!adev->enable_mes_kiq) {
1838 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0);
1839 if (r) {
1840 DRM_ERROR("Failed to init KIQ BOs!\n");
1841 return r;
1842 }
1843
1844 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1845 if (r)
1846 return r;
1847 }
1848
1849 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0);
1850 if (r)
1851 return r;
1852
1853 /* allocate visible FB for rlc auto-loading fw */
1854 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1855 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1856 if (r)
1857 return r;
1858 }
1859
1860 r = gfx_v11_0_gpu_early_init(adev);
1861 if (r)
1862 return r;
1863
1864 if (amdgpu_gfx_ras_sw_init(adev)) {
1865 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1866 return -EINVAL;
1867 }
1868
1869 gfx_v11_0_alloc_ip_dump(adev);
1870
1871 r = amdgpu_gfx_sysfs_init(adev);
1872 if (r)
1873 return r;
1874
1875 return 0;
1876 }
1877
gfx_v11_0_pfp_fini(struct amdgpu_device * adev)1878 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1879 {
1880 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1881 &adev->gfx.pfp.pfp_fw_gpu_addr,
1882 (void **)&adev->gfx.pfp.pfp_fw_ptr);
1883
1884 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1885 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1886 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1887 }
1888
gfx_v11_0_me_fini(struct amdgpu_device * adev)1889 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1890 {
1891 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1892 &adev->gfx.me.me_fw_gpu_addr,
1893 (void **)&adev->gfx.me.me_fw_ptr);
1894
1895 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1896 &adev->gfx.me.me_fw_data_gpu_addr,
1897 (void **)&adev->gfx.me.me_fw_data_ptr);
1898 }
1899
gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device * adev)1900 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1901 {
1902 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1903 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1904 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1905 }
1906
gfx_v11_0_sw_fini(struct amdgpu_ip_block * ip_block)1907 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block)
1908 {
1909 int i;
1910 struct amdgpu_device *adev = ip_block->adev;
1911
1912 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1913 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1914 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1915 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1916
1917 amdgpu_gfx_mqd_sw_fini(adev, 0);
1918
1919 if (!adev->enable_mes_kiq) {
1920 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1921 amdgpu_gfx_kiq_fini(adev, 0);
1922 }
1923
1924 amdgpu_gfx_cleaner_shader_sw_fini(adev);
1925
1926 gfx_v11_0_pfp_fini(adev);
1927 gfx_v11_0_me_fini(adev);
1928 gfx_v11_0_rlc_fini(adev);
1929 gfx_v11_0_mec_fini(adev);
1930
1931 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1932 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1933
1934 gfx_v11_0_free_microcode(adev);
1935
1936 amdgpu_gfx_sysfs_fini(adev);
1937
1938 kfree(adev->gfx.ip_dump_core);
1939 kfree(adev->gfx.ip_dump_compute_queues);
1940 kfree(adev->gfx.ip_dump_gfx_queues);
1941
1942 return 0;
1943 }
1944
gfx_v11_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)1945 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1946 u32 sh_num, u32 instance, int xcc_id)
1947 {
1948 u32 data;
1949
1950 if (instance == 0xffffffff)
1951 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1952 INSTANCE_BROADCAST_WRITES, 1);
1953 else
1954 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1955 instance);
1956
1957 if (se_num == 0xffffffff)
1958 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1959 1);
1960 else
1961 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1962
1963 if (sh_num == 0xffffffff)
1964 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1965 1);
1966 else
1967 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1968
1969 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1970 }
1971
gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device * adev)1972 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1973 {
1974 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1975
1976 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
1977 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1978 CC_GC_SA_UNIT_DISABLE,
1979 SA_DISABLE);
1980 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
1981 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1982 GC_USER_SA_UNIT_DISABLE,
1983 SA_DISABLE);
1984 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1985 adev->gfx.config.max_shader_engines);
1986
1987 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1988 }
1989
gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device * adev)1990 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1991 {
1992 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1993 u32 rb_mask;
1994
1995 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1996 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1997 CC_RB_BACKEND_DISABLE,
1998 BACKEND_DISABLE);
1999 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
2000 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
2001 GC_USER_RB_BACKEND_DISABLE,
2002 BACKEND_DISABLE);
2003 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
2004 adev->gfx.config.max_shader_engines);
2005
2006 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
2007 }
2008
gfx_v11_0_setup_rb(struct amdgpu_device * adev)2009 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
2010 {
2011 u32 rb_bitmap_per_sa;
2012 u32 rb_bitmap_width_per_sa;
2013 u32 max_sa;
2014 u32 active_sa_bitmap;
2015 u32 global_active_rb_bitmap;
2016 u32 active_rb_bitmap = 0;
2017 u32 i;
2018
2019 /* query sa bitmap from SA_UNIT_DISABLE registers */
2020 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
2021 /* query rb bitmap from RB_BACKEND_DISABLE registers */
2022 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
2023
2024 /* generate active rb bitmap according to active sa bitmap */
2025 max_sa = adev->gfx.config.max_shader_engines *
2026 adev->gfx.config.max_sh_per_se;
2027 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
2028 adev->gfx.config.max_sh_per_se;
2029 rb_bitmap_per_sa = amdgpu_gfx_create_bitmask(rb_bitmap_width_per_sa);
2030
2031 for (i = 0; i < max_sa; i++) {
2032 if (active_sa_bitmap & (1 << i))
2033 active_rb_bitmap |= (rb_bitmap_per_sa << (i * rb_bitmap_width_per_sa));
2034 }
2035
2036 active_rb_bitmap &= global_active_rb_bitmap;
2037 adev->gfx.config.backend_enable_mask = active_rb_bitmap;
2038 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
2039 }
2040
2041 #define DEFAULT_SH_MEM_BASES (0x6000)
2042 #define LDS_APP_BASE 0x1
2043 #define SCRATCH_APP_BASE 0x2
2044
gfx_v11_0_init_compute_vmid(struct amdgpu_device * adev)2045 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
2046 {
2047 int i;
2048 uint32_t sh_mem_bases;
2049 uint32_t data;
2050
2051 /*
2052 * Configure apertures:
2053 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2054 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2055 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2056 */
2057 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
2058 SCRATCH_APP_BASE;
2059
2060 mutex_lock(&adev->srbm_mutex);
2061 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2062 soc21_grbm_select(adev, 0, 0, 0, i);
2063 /* CP and shaders */
2064 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
2065 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
2066
2067 /* Enable trap for each kfd vmid. */
2068 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
2069 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
2070 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
2071 }
2072 soc21_grbm_select(adev, 0, 0, 0, 0);
2073 mutex_unlock(&adev->srbm_mutex);
2074
2075 /*
2076 * Initialize all compute VMIDs to have no GDS, GWS, or OA
2077 * access. These should be enabled by FW for target VMIDs.
2078 */
2079 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
2080 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
2081 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
2082 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
2083 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
2084 }
2085 }
2086
gfx_v11_0_init_gds_vmid(struct amdgpu_device * adev)2087 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
2088 {
2089 int vmid;
2090
2091 /*
2092 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2093 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2094 * the driver can enable them for graphics. VMID0 should maintain
2095 * access so that HWS firmware can save/restore entries.
2096 */
2097 for (vmid = 1; vmid < 16; vmid++) {
2098 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
2099 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
2100 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
2101 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
2102 }
2103 }
2104
gfx_v11_0_tcp_harvest(struct amdgpu_device * adev)2105 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
2106 {
2107 /* TODO: harvest feature to be added later. */
2108 }
2109
gfx_v11_0_get_tcc_info(struct amdgpu_device * adev)2110 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
2111 {
2112 /* TCCs are global (not instanced). */
2113 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
2114 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
2115
2116 adev->gfx.config.tcc_disabled_mask =
2117 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
2118 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
2119 }
2120
gfx_v11_0_constants_init(struct amdgpu_device * adev)2121 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
2122 {
2123 u32 tmp;
2124 int i;
2125
2126 if (!amdgpu_sriov_vf(adev))
2127 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2128
2129 gfx_v11_0_setup_rb(adev);
2130 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
2131 gfx_v11_0_get_tcc_info(adev);
2132 adev->gfx.config.pa_sc_tile_steering_override = 0;
2133
2134 /* Set whether texture coordinate truncation is conformant. */
2135 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
2136 adev->gfx.config.ta_cntl2_truncate_coord_mode =
2137 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
2138
2139 /* XXX SH_MEM regs */
2140 /* where to put LDS, scratch, GPUVM in FSA64 space */
2141 mutex_lock(&adev->srbm_mutex);
2142 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
2143 soc21_grbm_select(adev, 0, 0, 0, i);
2144 /* CP and shaders */
2145 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
2146 if (i != 0) {
2147 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2148 (adev->gmc.private_aperture_start >> 48));
2149 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2150 (adev->gmc.shared_aperture_start >> 48));
2151 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
2152 }
2153 }
2154 soc21_grbm_select(adev, 0, 0, 0, 0);
2155
2156 mutex_unlock(&adev->srbm_mutex);
2157
2158 gfx_v11_0_init_compute_vmid(adev);
2159 gfx_v11_0_init_gds_vmid(adev);
2160 }
2161
gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device * adev,int me,int pipe)2162 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev,
2163 int me, int pipe)
2164 {
2165 if (me != 0)
2166 return 0;
2167
2168 switch (pipe) {
2169 case 0:
2170 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
2171 case 1:
2172 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
2173 default:
2174 return 0;
2175 }
2176 }
2177
gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device * adev,int me,int pipe)2178 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev,
2179 int me, int pipe)
2180 {
2181 /*
2182 * amdgpu controls only the first MEC. That's why this function only
2183 * handles the setting of interrupts for this specific MEC. All other
2184 * pipes' interrupts are set by amdkfd.
2185 */
2186 if (me != 1)
2187 return 0;
2188
2189 switch (pipe) {
2190 case 0:
2191 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
2192 case 1:
2193 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
2194 case 2:
2195 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
2196 case 3:
2197 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
2198 default:
2199 return 0;
2200 }
2201 }
2202
gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2203 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2204 bool enable)
2205 {
2206 u32 tmp, cp_int_cntl_reg;
2207 int i, j;
2208
2209 if (amdgpu_sriov_vf(adev))
2210 return;
2211
2212 for (i = 0; i < adev->gfx.me.num_me; i++) {
2213 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
2214 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
2215
2216 if (cp_int_cntl_reg) {
2217 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
2218 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
2219 enable ? 1 : 0);
2220 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
2221 enable ? 1 : 0);
2222 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
2223 enable ? 1 : 0);
2224 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
2225 enable ? 1 : 0);
2226 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp);
2227 }
2228 }
2229 }
2230 }
2231
gfx_v11_0_init_csb(struct amdgpu_device * adev)2232 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
2233 {
2234 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2235
2236 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
2237 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2238 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
2239 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2240 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
2241
2242 return 0;
2243 }
2244
gfx_v11_0_rlc_stop(struct amdgpu_device * adev)2245 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
2246 {
2247 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
2248
2249 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
2250 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
2251 }
2252
gfx_v11_0_rlc_reset(struct amdgpu_device * adev)2253 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
2254 {
2255 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2256 udelay(50);
2257 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2258 udelay(50);
2259 }
2260
gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device * adev,bool enable)2261 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
2262 bool enable)
2263 {
2264 uint32_t rlc_pg_cntl;
2265
2266 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
2267
2268 if (!enable) {
2269 /* RLC_PG_CNTL[23] = 0 (default)
2270 * RLC will wait for handshake acks with SMU
2271 * GFXOFF will be enabled
2272 * RLC_PG_CNTL[23] = 1
2273 * RLC will not issue any message to SMU
2274 * hence no handshake between SMU & RLC
2275 * GFXOFF will be disabled
2276 */
2277 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2278 } else
2279 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2280 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
2281 }
2282
gfx_v11_0_rlc_start(struct amdgpu_device * adev)2283 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
2284 {
2285 /* TODO: enable rlc & smu handshake until smu
2286 * and gfxoff feature works as expected */
2287 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
2288 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
2289
2290 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2291 udelay(50);
2292 }
2293
gfx_v11_0_rlc_enable_srm(struct amdgpu_device * adev)2294 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
2295 {
2296 uint32_t tmp;
2297
2298 /* enable Save Restore Machine */
2299 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
2300 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2301 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
2302 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
2303 }
2304
gfx_v11_0_load_rlcg_microcode(struct amdgpu_device * adev)2305 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
2306 {
2307 const struct rlc_firmware_header_v2_0 *hdr;
2308 const __le32 *fw_data;
2309 unsigned i, fw_size;
2310
2311 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2312 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2313 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2314 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2315
2316 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
2317 RLCG_UCODE_LOADING_START_ADDRESS);
2318
2319 for (i = 0; i < fw_size; i++)
2320 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
2321 le32_to_cpup(fw_data++));
2322
2323 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2324 }
2325
gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device * adev)2326 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
2327 {
2328 const struct rlc_firmware_header_v2_2 *hdr;
2329 const __le32 *fw_data;
2330 unsigned i, fw_size;
2331 u32 tmp;
2332
2333 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
2334
2335 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2336 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
2337 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
2338
2339 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
2340
2341 for (i = 0; i < fw_size; i++) {
2342 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2343 msleep(1);
2344 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
2345 le32_to_cpup(fw_data++));
2346 }
2347
2348 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2349
2350 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2351 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2352 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2353
2354 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
2355 for (i = 0; i < fw_size; i++) {
2356 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2357 msleep(1);
2358 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
2359 le32_to_cpup(fw_data++));
2360 }
2361
2362 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2363
2364 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
2365 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2366 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2367 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
2368 }
2369
gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device * adev)2370 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
2371 {
2372 const struct rlc_firmware_header_v2_3 *hdr;
2373 const __le32 *fw_data;
2374 unsigned i, fw_size;
2375 u32 tmp;
2376
2377 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
2378
2379 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2380 le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
2381 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
2382
2383 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
2384
2385 for (i = 0; i < fw_size; i++) {
2386 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2387 msleep(1);
2388 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
2389 le32_to_cpup(fw_data++));
2390 }
2391
2392 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
2393
2394 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
2395 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
2396 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
2397
2398 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2399 le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
2400 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
2401
2402 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
2403
2404 for (i = 0; i < fw_size; i++) {
2405 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2406 msleep(1);
2407 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
2408 le32_to_cpup(fw_data++));
2409 }
2410
2411 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
2412
2413 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
2414 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
2415 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
2416 }
2417
gfx_v11_0_rlc_load_microcode(struct amdgpu_device * adev)2418 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
2419 {
2420 const struct rlc_firmware_header_v2_0 *hdr;
2421 uint16_t version_major;
2422 uint16_t version_minor;
2423
2424 if (!adev->gfx.rlc_fw)
2425 return -EINVAL;
2426
2427 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2428 amdgpu_ucode_print_rlc_hdr(&hdr->header);
2429
2430 version_major = le16_to_cpu(hdr->header.header_version_major);
2431 version_minor = le16_to_cpu(hdr->header.header_version_minor);
2432
2433 if (version_major == 2) {
2434 gfx_v11_0_load_rlcg_microcode(adev);
2435 if (amdgpu_dpm == 1) {
2436 if (version_minor >= 2)
2437 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
2438 if (version_minor == 3)
2439 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
2440 }
2441
2442 return 0;
2443 }
2444
2445 return -EINVAL;
2446 }
2447
gfx_v11_0_rlc_resume(struct amdgpu_device * adev)2448 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2449 {
2450 int r;
2451
2452 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2453 gfx_v11_0_init_csb(adev);
2454
2455 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2456 gfx_v11_0_rlc_enable_srm(adev);
2457 } else {
2458 if (amdgpu_sriov_vf(adev)) {
2459 gfx_v11_0_init_csb(adev);
2460 return 0;
2461 }
2462
2463 adev->gfx.rlc.funcs->stop(adev);
2464
2465 /* disable CG */
2466 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2467
2468 /* disable PG */
2469 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2470
2471 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2472 /* legacy rlc firmware loading */
2473 r = gfx_v11_0_rlc_load_microcode(adev);
2474 if (r)
2475 return r;
2476 }
2477
2478 gfx_v11_0_init_csb(adev);
2479
2480 adev->gfx.rlc.funcs->start(adev);
2481 }
2482 return 0;
2483 }
2484
gfx_v11_0_config_me_cache(struct amdgpu_device * adev,uint64_t addr)2485 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2486 {
2487 uint32_t usec_timeout = 50000; /* wait for 50ms */
2488 uint32_t tmp;
2489 int i;
2490
2491 /* Trigger an invalidation of the L1 instruction caches */
2492 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2493 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2494 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2495
2496 /* Wait for invalidation complete */
2497 for (i = 0; i < usec_timeout; i++) {
2498 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2499 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2500 INVALIDATE_CACHE_COMPLETE))
2501 break;
2502 udelay(1);
2503 }
2504
2505 if (i >= usec_timeout) {
2506 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2507 return -EINVAL;
2508 }
2509
2510 if (amdgpu_emu_mode == 1)
2511 amdgpu_device_flush_hdp(adev, NULL);
2512
2513 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2514 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2515 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2516 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2517 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2518 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2519
2520 /* Program me ucode address into intruction cache address register */
2521 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2522 lower_32_bits(addr) & 0xFFFFF000);
2523 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2524 upper_32_bits(addr));
2525
2526 return 0;
2527 }
2528
gfx_v11_0_config_pfp_cache(struct amdgpu_device * adev,uint64_t addr)2529 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2530 {
2531 uint32_t usec_timeout = 50000; /* wait for 50ms */
2532 uint32_t tmp;
2533 int i;
2534
2535 /* Trigger an invalidation of the L1 instruction caches */
2536 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2537 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2538 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2539
2540 /* Wait for invalidation complete */
2541 for (i = 0; i < usec_timeout; i++) {
2542 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2543 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2544 INVALIDATE_CACHE_COMPLETE))
2545 break;
2546 udelay(1);
2547 }
2548
2549 if (i >= usec_timeout) {
2550 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2551 return -EINVAL;
2552 }
2553
2554 if (amdgpu_emu_mode == 1)
2555 amdgpu_device_flush_hdp(adev, NULL);
2556
2557 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2558 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2559 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2560 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2561 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2562 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2563
2564 /* Program pfp ucode address into intruction cache address register */
2565 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2566 lower_32_bits(addr) & 0xFFFFF000);
2567 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2568 upper_32_bits(addr));
2569
2570 return 0;
2571 }
2572
gfx_v11_0_config_mec_cache(struct amdgpu_device * adev,uint64_t addr)2573 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2574 {
2575 uint32_t usec_timeout = 50000; /* wait for 50ms */
2576 uint32_t tmp;
2577 int i;
2578
2579 /* Trigger an invalidation of the L1 instruction caches */
2580 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2581 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2582
2583 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2584
2585 /* Wait for invalidation complete */
2586 for (i = 0; i < usec_timeout; i++) {
2587 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2588 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2589 INVALIDATE_CACHE_COMPLETE))
2590 break;
2591 udelay(1);
2592 }
2593
2594 if (i >= usec_timeout) {
2595 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2596 return -EINVAL;
2597 }
2598
2599 if (amdgpu_emu_mode == 1)
2600 amdgpu_device_flush_hdp(adev, NULL);
2601
2602 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2603 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2604 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2605 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2606 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2607
2608 /* Program mec1 ucode address into intruction cache address register */
2609 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2610 lower_32_bits(addr) & 0xFFFFF000);
2611 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2612 upper_32_bits(addr));
2613
2614 return 0;
2615 }
2616
gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2617 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2618 {
2619 uint32_t usec_timeout = 50000; /* wait for 50ms */
2620 uint32_t tmp;
2621 unsigned i, pipe_id;
2622 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2623
2624 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2625 adev->gfx.pfp_fw->data;
2626
2627 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2628 lower_32_bits(addr));
2629 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2630 upper_32_bits(addr));
2631
2632 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2633 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2634 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2635 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2636 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2637
2638 /*
2639 * Programming any of the CP_PFP_IC_BASE registers
2640 * forces invalidation of the ME L1 I$. Wait for the
2641 * invalidation complete
2642 */
2643 for (i = 0; i < usec_timeout; i++) {
2644 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2645 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2646 INVALIDATE_CACHE_COMPLETE))
2647 break;
2648 udelay(1);
2649 }
2650
2651 if (i >= usec_timeout) {
2652 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2653 return -EINVAL;
2654 }
2655
2656 /* Prime the L1 instruction caches */
2657 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2658 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2659 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2660 /* Waiting for cache primed*/
2661 for (i = 0; i < usec_timeout; i++) {
2662 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2663 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2664 ICACHE_PRIMED))
2665 break;
2666 udelay(1);
2667 }
2668
2669 if (i >= usec_timeout) {
2670 dev_err(adev->dev, "failed to prime instruction cache\n");
2671 return -EINVAL;
2672 }
2673
2674 mutex_lock(&adev->srbm_mutex);
2675 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2676 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2677 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2678 (pfp_hdr->ucode_start_addr_hi << 30) |
2679 (pfp_hdr->ucode_start_addr_lo >> 2));
2680 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2681 pfp_hdr->ucode_start_addr_hi >> 2);
2682
2683 /*
2684 * Program CP_ME_CNTL to reset given PIPE to take
2685 * effect of CP_PFP_PRGRM_CNTR_START.
2686 */
2687 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2688 if (pipe_id == 0)
2689 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2690 PFP_PIPE0_RESET, 1);
2691 else
2692 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2693 PFP_PIPE1_RESET, 1);
2694 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2695
2696 /* Clear pfp pipe0 reset bit. */
2697 if (pipe_id == 0)
2698 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2699 PFP_PIPE0_RESET, 0);
2700 else
2701 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2702 PFP_PIPE1_RESET, 0);
2703 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2704
2705 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2706 lower_32_bits(addr2));
2707 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2708 upper_32_bits(addr2));
2709 }
2710 soc21_grbm_select(adev, 0, 0, 0, 0);
2711 mutex_unlock(&adev->srbm_mutex);
2712
2713 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2714 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2715 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2716 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2717
2718 /* Invalidate the data caches */
2719 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2720 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2721 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2722
2723 for (i = 0; i < usec_timeout; i++) {
2724 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2725 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2726 INVALIDATE_DCACHE_COMPLETE))
2727 break;
2728 udelay(1);
2729 }
2730
2731 if (i >= usec_timeout) {
2732 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2733 return -EINVAL;
2734 }
2735
2736 return 0;
2737 }
2738
gfx_v11_0_config_me_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2739 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2740 {
2741 uint32_t usec_timeout = 50000; /* wait for 50ms */
2742 uint32_t tmp;
2743 unsigned i, pipe_id;
2744 const struct gfx_firmware_header_v2_0 *me_hdr;
2745
2746 me_hdr = (const struct gfx_firmware_header_v2_0 *)
2747 adev->gfx.me_fw->data;
2748
2749 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2750 lower_32_bits(addr));
2751 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2752 upper_32_bits(addr));
2753
2754 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2755 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2756 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2757 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2758 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2759
2760 /*
2761 * Programming any of the CP_ME_IC_BASE registers
2762 * forces invalidation of the ME L1 I$. Wait for the
2763 * invalidation complete
2764 */
2765 for (i = 0; i < usec_timeout; i++) {
2766 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2767 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2768 INVALIDATE_CACHE_COMPLETE))
2769 break;
2770 udelay(1);
2771 }
2772
2773 if (i >= usec_timeout) {
2774 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2775 return -EINVAL;
2776 }
2777
2778 /* Prime the instruction caches */
2779 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2780 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2781 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2782
2783 /* Waiting for instruction cache primed*/
2784 for (i = 0; i < usec_timeout; i++) {
2785 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2786 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2787 ICACHE_PRIMED))
2788 break;
2789 udelay(1);
2790 }
2791
2792 if (i >= usec_timeout) {
2793 dev_err(adev->dev, "failed to prime instruction cache\n");
2794 return -EINVAL;
2795 }
2796
2797 mutex_lock(&adev->srbm_mutex);
2798 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2799 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2800 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2801 (me_hdr->ucode_start_addr_hi << 30) |
2802 (me_hdr->ucode_start_addr_lo >> 2) );
2803 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2804 me_hdr->ucode_start_addr_hi>>2);
2805
2806 /*
2807 * Program CP_ME_CNTL to reset given PIPE to take
2808 * effect of CP_PFP_PRGRM_CNTR_START.
2809 */
2810 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2811 if (pipe_id == 0)
2812 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2813 ME_PIPE0_RESET, 1);
2814 else
2815 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2816 ME_PIPE1_RESET, 1);
2817 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2818
2819 /* Clear pfp pipe0 reset bit. */
2820 if (pipe_id == 0)
2821 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2822 ME_PIPE0_RESET, 0);
2823 else
2824 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2825 ME_PIPE1_RESET, 0);
2826 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2827
2828 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2829 lower_32_bits(addr2));
2830 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2831 upper_32_bits(addr2));
2832 }
2833 soc21_grbm_select(adev, 0, 0, 0, 0);
2834 mutex_unlock(&adev->srbm_mutex);
2835
2836 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2837 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2838 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2839 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2840
2841 /* Invalidate the data caches */
2842 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2843 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2844 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2845
2846 for (i = 0; i < usec_timeout; i++) {
2847 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2848 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2849 INVALIDATE_DCACHE_COMPLETE))
2850 break;
2851 udelay(1);
2852 }
2853
2854 if (i >= usec_timeout) {
2855 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2856 return -EINVAL;
2857 }
2858
2859 return 0;
2860 }
2861
gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2862 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2863 {
2864 uint32_t usec_timeout = 50000; /* wait for 50ms */
2865 uint32_t tmp;
2866 unsigned i;
2867 const struct gfx_firmware_header_v2_0 *mec_hdr;
2868
2869 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2870 adev->gfx.mec_fw->data;
2871
2872 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2873 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2874 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2875 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2876 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2877
2878 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2879 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2880 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2881 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2882
2883 mutex_lock(&adev->srbm_mutex);
2884 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2885 soc21_grbm_select(adev, 1, i, 0, 0);
2886
2887 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2888 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2889 upper_32_bits(addr2));
2890
2891 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2892 mec_hdr->ucode_start_addr_lo >> 2 |
2893 mec_hdr->ucode_start_addr_hi << 30);
2894 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2895 mec_hdr->ucode_start_addr_hi >> 2);
2896
2897 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2898 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2899 upper_32_bits(addr));
2900 }
2901 mutex_unlock(&adev->srbm_mutex);
2902 soc21_grbm_select(adev, 0, 0, 0, 0);
2903
2904 /* Trigger an invalidation of the L1 instruction caches */
2905 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2906 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2907 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2908
2909 /* Wait for invalidation complete */
2910 for (i = 0; i < usec_timeout; i++) {
2911 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2912 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2913 INVALIDATE_DCACHE_COMPLETE))
2914 break;
2915 udelay(1);
2916 }
2917
2918 if (i >= usec_timeout) {
2919 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2920 return -EINVAL;
2921 }
2922
2923 /* Trigger an invalidation of the L1 instruction caches */
2924 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2925 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2926 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2927
2928 /* Wait for invalidation complete */
2929 for (i = 0; i < usec_timeout; i++) {
2930 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2931 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2932 INVALIDATE_CACHE_COMPLETE))
2933 break;
2934 udelay(1);
2935 }
2936
2937 if (i >= usec_timeout) {
2938 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2939 return -EINVAL;
2940 }
2941
2942 return 0;
2943 }
2944
gfx_v11_0_config_gfx_rs64(struct amdgpu_device * adev)2945 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2946 {
2947 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2948 const struct gfx_firmware_header_v2_0 *me_hdr;
2949 const struct gfx_firmware_header_v2_0 *mec_hdr;
2950 uint32_t pipe_id, tmp;
2951
2952 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2953 adev->gfx.mec_fw->data;
2954 me_hdr = (const struct gfx_firmware_header_v2_0 *)
2955 adev->gfx.me_fw->data;
2956 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2957 adev->gfx.pfp_fw->data;
2958
2959 /* config pfp program start addr */
2960 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2961 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2962 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2963 (pfp_hdr->ucode_start_addr_hi << 30) |
2964 (pfp_hdr->ucode_start_addr_lo >> 2));
2965 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2966 pfp_hdr->ucode_start_addr_hi >> 2);
2967 }
2968 soc21_grbm_select(adev, 0, 0, 0, 0);
2969
2970 /* reset pfp pipe */
2971 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2972 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2973 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2974 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2975
2976 /* clear pfp pipe reset */
2977 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2978 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2979 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2980
2981 /* config me program start addr */
2982 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2983 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2984 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2985 (me_hdr->ucode_start_addr_hi << 30) |
2986 (me_hdr->ucode_start_addr_lo >> 2) );
2987 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2988 me_hdr->ucode_start_addr_hi>>2);
2989 }
2990 soc21_grbm_select(adev, 0, 0, 0, 0);
2991
2992 /* reset me pipe */
2993 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2994 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2995 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2996 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2997
2998 /* clear me pipe reset */
2999 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
3000 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
3001 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3002
3003 /* config mec program start addr */
3004 for (pipe_id = 0; pipe_id < 4; pipe_id++) {
3005 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
3006 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3007 mec_hdr->ucode_start_addr_lo >> 2 |
3008 mec_hdr->ucode_start_addr_hi << 30);
3009 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3010 mec_hdr->ucode_start_addr_hi >> 2);
3011 }
3012 soc21_grbm_select(adev, 0, 0, 0, 0);
3013
3014 /* reset mec pipe */
3015 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3016 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
3017 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
3018 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
3019 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
3020 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
3021
3022 /* clear mec pipe reset */
3023 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
3024 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
3025 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
3026 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
3027 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
3028 }
3029
gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device * adev)3030 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
3031 {
3032 uint32_t cp_status;
3033 uint32_t bootload_status;
3034 int i, r;
3035 uint64_t addr, addr2;
3036
3037 for (i = 0; i < adev->usec_timeout; i++) {
3038 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
3039
3040 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
3041 IP_VERSION(11, 0, 1) ||
3042 amdgpu_ip_version(adev, GC_HWIP, 0) ==
3043 IP_VERSION(11, 0, 4) ||
3044 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) ||
3045 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) ||
3046 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) ||
3047 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3))
3048 bootload_status = RREG32_SOC15(GC, 0,
3049 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
3050 else
3051 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
3052
3053 if ((cp_status == 0) &&
3054 (REG_GET_FIELD(bootload_status,
3055 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
3056 break;
3057 }
3058 udelay(1);
3059 }
3060
3061 if (i >= adev->usec_timeout) {
3062 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
3063 return -ETIMEDOUT;
3064 }
3065
3066 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
3067 if (adev->gfx.rs64_enable) {
3068 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3069 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
3070 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
3071 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
3072 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
3073 if (r)
3074 return r;
3075 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3076 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
3077 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
3078 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
3079 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
3080 if (r)
3081 return r;
3082 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3083 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
3084 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
3085 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
3086 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
3087 if (r)
3088 return r;
3089 } else {
3090 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3091 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
3092 r = gfx_v11_0_config_me_cache(adev, addr);
3093 if (r)
3094 return r;
3095 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3096 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
3097 r = gfx_v11_0_config_pfp_cache(adev, addr);
3098 if (r)
3099 return r;
3100 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
3101 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
3102 r = gfx_v11_0_config_mec_cache(adev, addr);
3103 if (r)
3104 return r;
3105 }
3106 }
3107
3108 return 0;
3109 }
3110
gfx_v11_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)3111 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3112 {
3113 int i;
3114 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3115
3116 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3117 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3118 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3119
3120 for (i = 0; i < adev->usec_timeout; i++) {
3121 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
3122 break;
3123 udelay(1);
3124 }
3125
3126 if (i >= adev->usec_timeout)
3127 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
3128
3129 return 0;
3130 }
3131
gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device * adev)3132 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
3133 {
3134 int r;
3135 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3136 const __le32 *fw_data;
3137 unsigned i, fw_size;
3138
3139 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3140 adev->gfx.pfp_fw->data;
3141
3142 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3143
3144 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
3145 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3146 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
3147
3148 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
3149 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3150 &adev->gfx.pfp.pfp_fw_obj,
3151 &adev->gfx.pfp.pfp_fw_gpu_addr,
3152 (void **)&adev->gfx.pfp.pfp_fw_ptr);
3153 if (r) {
3154 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
3155 gfx_v11_0_pfp_fini(adev);
3156 return r;
3157 }
3158
3159 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
3160
3161 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3162 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3163
3164 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
3165
3166 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
3167
3168 for (i = 0; i < pfp_hdr->jt_size; i++)
3169 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
3170 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
3171
3172 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3173
3174 return 0;
3175 }
3176
gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device * adev)3177 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
3178 {
3179 int r;
3180 const struct gfx_firmware_header_v2_0 *pfp_hdr;
3181 const __le32 *fw_ucode, *fw_data;
3182 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3183 uint32_t tmp;
3184 uint32_t usec_timeout = 50000; /* wait for 50ms */
3185
3186 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
3187 adev->gfx.pfp_fw->data;
3188
3189 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3190
3191 /* instruction */
3192 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
3193 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
3194 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
3195 /* data */
3196 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
3197 le32_to_cpu(pfp_hdr->data_offset_bytes));
3198 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
3199
3200 /* 64kb align */
3201 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3202 64 * 1024,
3203 AMDGPU_GEM_DOMAIN_VRAM |
3204 AMDGPU_GEM_DOMAIN_GTT,
3205 &adev->gfx.pfp.pfp_fw_obj,
3206 &adev->gfx.pfp.pfp_fw_gpu_addr,
3207 (void **)&adev->gfx.pfp.pfp_fw_ptr);
3208 if (r) {
3209 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
3210 gfx_v11_0_pfp_fini(adev);
3211 return r;
3212 }
3213
3214 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3215 64 * 1024,
3216 AMDGPU_GEM_DOMAIN_VRAM |
3217 AMDGPU_GEM_DOMAIN_GTT,
3218 &adev->gfx.pfp.pfp_fw_data_obj,
3219 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
3220 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
3221 if (r) {
3222 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
3223 gfx_v11_0_pfp_fini(adev);
3224 return r;
3225 }
3226
3227 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
3228 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
3229
3230 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3231 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
3232 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3233 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
3234
3235 if (amdgpu_emu_mode == 1)
3236 amdgpu_device_flush_hdp(adev, NULL);
3237
3238 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
3239 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3240 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
3241 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3242
3243 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
3244 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
3245 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
3246 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
3247 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
3248
3249 /*
3250 * Programming any of the CP_PFP_IC_BASE registers
3251 * forces invalidation of the ME L1 I$. Wait for the
3252 * invalidation complete
3253 */
3254 for (i = 0; i < usec_timeout; i++) {
3255 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3256 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3257 INVALIDATE_CACHE_COMPLETE))
3258 break;
3259 udelay(1);
3260 }
3261
3262 if (i >= usec_timeout) {
3263 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3264 return -EINVAL;
3265 }
3266
3267 /* Prime the L1 instruction caches */
3268 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3269 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
3270 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
3271 /* Waiting for cache primed*/
3272 for (i = 0; i < usec_timeout; i++) {
3273 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3274 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3275 ICACHE_PRIMED))
3276 break;
3277 udelay(1);
3278 }
3279
3280 if (i >= usec_timeout) {
3281 dev_err(adev->dev, "failed to prime instruction cache\n");
3282 return -EINVAL;
3283 }
3284
3285 mutex_lock(&adev->srbm_mutex);
3286 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3287 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3288 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
3289 (pfp_hdr->ucode_start_addr_hi << 30) |
3290 (pfp_hdr->ucode_start_addr_lo >> 2) );
3291 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3292 pfp_hdr->ucode_start_addr_hi>>2);
3293
3294 /*
3295 * Program CP_ME_CNTL to reset given PIPE to take
3296 * effect of CP_PFP_PRGRM_CNTR_START.
3297 */
3298 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3299 if (pipe_id == 0)
3300 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3301 PFP_PIPE0_RESET, 1);
3302 else
3303 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3304 PFP_PIPE1_RESET, 1);
3305 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3306
3307 /* Clear pfp pipe0 reset bit. */
3308 if (pipe_id == 0)
3309 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3310 PFP_PIPE0_RESET, 0);
3311 else
3312 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3313 PFP_PIPE1_RESET, 0);
3314 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3315
3316 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
3317 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3318 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
3319 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3320 }
3321 soc21_grbm_select(adev, 0, 0, 0, 0);
3322 mutex_unlock(&adev->srbm_mutex);
3323
3324 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3325 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3326 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3327 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3328
3329 /* Invalidate the data caches */
3330 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3331 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3332 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3333
3334 for (i = 0; i < usec_timeout; i++) {
3335 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3336 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3337 INVALIDATE_DCACHE_COMPLETE))
3338 break;
3339 udelay(1);
3340 }
3341
3342 if (i >= usec_timeout) {
3343 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3344 return -EINVAL;
3345 }
3346
3347 return 0;
3348 }
3349
gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device * adev)3350 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
3351 {
3352 int r;
3353 const struct gfx_firmware_header_v1_0 *me_hdr;
3354 const __le32 *fw_data;
3355 unsigned i, fw_size;
3356
3357 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3358 adev->gfx.me_fw->data;
3359
3360 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3361
3362 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3363 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3364 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
3365
3366 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
3367 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3368 &adev->gfx.me.me_fw_obj,
3369 &adev->gfx.me.me_fw_gpu_addr,
3370 (void **)&adev->gfx.me.me_fw_ptr);
3371 if (r) {
3372 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
3373 gfx_v11_0_me_fini(adev);
3374 return r;
3375 }
3376
3377 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
3378
3379 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3380 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3381
3382 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
3383
3384 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
3385
3386 for (i = 0; i < me_hdr->jt_size; i++)
3387 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
3388 le32_to_cpup(fw_data + me_hdr->jt_offset + i));
3389
3390 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
3391
3392 return 0;
3393 }
3394
gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device * adev)3395 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
3396 {
3397 int r;
3398 const struct gfx_firmware_header_v2_0 *me_hdr;
3399 const __le32 *fw_ucode, *fw_data;
3400 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3401 uint32_t tmp;
3402 uint32_t usec_timeout = 50000; /* wait for 50ms */
3403
3404 me_hdr = (const struct gfx_firmware_header_v2_0 *)
3405 adev->gfx.me_fw->data;
3406
3407 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3408
3409 /* instruction */
3410 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
3411 le32_to_cpu(me_hdr->ucode_offset_bytes));
3412 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
3413 /* data */
3414 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3415 le32_to_cpu(me_hdr->data_offset_bytes));
3416 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
3417
3418 /* 64kb align*/
3419 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3420 64 * 1024,
3421 AMDGPU_GEM_DOMAIN_VRAM |
3422 AMDGPU_GEM_DOMAIN_GTT,
3423 &adev->gfx.me.me_fw_obj,
3424 &adev->gfx.me.me_fw_gpu_addr,
3425 (void **)&adev->gfx.me.me_fw_ptr);
3426 if (r) {
3427 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
3428 gfx_v11_0_me_fini(adev);
3429 return r;
3430 }
3431
3432 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3433 64 * 1024,
3434 AMDGPU_GEM_DOMAIN_VRAM |
3435 AMDGPU_GEM_DOMAIN_GTT,
3436 &adev->gfx.me.me_fw_data_obj,
3437 &adev->gfx.me.me_fw_data_gpu_addr,
3438 (void **)&adev->gfx.me.me_fw_data_ptr);
3439 if (r) {
3440 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
3441 gfx_v11_0_pfp_fini(adev);
3442 return r;
3443 }
3444
3445 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
3446 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
3447
3448 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3449 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
3450 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3451 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3452
3453 if (amdgpu_emu_mode == 1)
3454 amdgpu_device_flush_hdp(adev, NULL);
3455
3456 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3457 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3458 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3459 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3460
3461 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3462 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3463 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3464 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3465 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3466
3467 /*
3468 * Programming any of the CP_ME_IC_BASE registers
3469 * forces invalidation of the ME L1 I$. Wait for the
3470 * invalidation complete
3471 */
3472 for (i = 0; i < usec_timeout; i++) {
3473 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3474 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3475 INVALIDATE_CACHE_COMPLETE))
3476 break;
3477 udelay(1);
3478 }
3479
3480 if (i >= usec_timeout) {
3481 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3482 return -EINVAL;
3483 }
3484
3485 /* Prime the instruction caches */
3486 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3487 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3488 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3489
3490 /* Waiting for instruction cache primed*/
3491 for (i = 0; i < usec_timeout; i++) {
3492 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3493 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3494 ICACHE_PRIMED))
3495 break;
3496 udelay(1);
3497 }
3498
3499 if (i >= usec_timeout) {
3500 dev_err(adev->dev, "failed to prime instruction cache\n");
3501 return -EINVAL;
3502 }
3503
3504 mutex_lock(&adev->srbm_mutex);
3505 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3506 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3507 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3508 (me_hdr->ucode_start_addr_hi << 30) |
3509 (me_hdr->ucode_start_addr_lo >> 2) );
3510 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3511 me_hdr->ucode_start_addr_hi>>2);
3512
3513 /*
3514 * Program CP_ME_CNTL to reset given PIPE to take
3515 * effect of CP_PFP_PRGRM_CNTR_START.
3516 */
3517 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3518 if (pipe_id == 0)
3519 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3520 ME_PIPE0_RESET, 1);
3521 else
3522 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3523 ME_PIPE1_RESET, 1);
3524 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3525
3526 /* Clear pfp pipe0 reset bit. */
3527 if (pipe_id == 0)
3528 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3529 ME_PIPE0_RESET, 0);
3530 else
3531 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3532 ME_PIPE1_RESET, 0);
3533 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3534
3535 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3536 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3537 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3538 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3539 }
3540 soc21_grbm_select(adev, 0, 0, 0, 0);
3541 mutex_unlock(&adev->srbm_mutex);
3542
3543 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3544 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3545 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3546 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3547
3548 /* Invalidate the data caches */
3549 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3550 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3551 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3552
3553 for (i = 0; i < usec_timeout; i++) {
3554 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3555 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3556 INVALIDATE_DCACHE_COMPLETE))
3557 break;
3558 udelay(1);
3559 }
3560
3561 if (i >= usec_timeout) {
3562 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3563 return -EINVAL;
3564 }
3565
3566 return 0;
3567 }
3568
gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3569 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3570 {
3571 int r;
3572
3573 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3574 return -EINVAL;
3575
3576 gfx_v11_0_cp_gfx_enable(adev, false);
3577
3578 if (adev->gfx.rs64_enable)
3579 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3580 else
3581 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3582 if (r) {
3583 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3584 return r;
3585 }
3586
3587 if (adev->gfx.rs64_enable)
3588 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3589 else
3590 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3591 if (r) {
3592 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3593 return r;
3594 }
3595
3596 return 0;
3597 }
3598
gfx_v11_0_cp_gfx_start(struct amdgpu_device * adev)3599 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3600 {
3601 struct amdgpu_ring *ring;
3602 const struct cs_section_def *sect = NULL;
3603 const struct cs_extent_def *ext = NULL;
3604 int r, i;
3605 int ctx_reg_offset;
3606
3607 /* init the CP */
3608 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3609 adev->gfx.config.max_hw_contexts - 1);
3610 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3611
3612 if (!amdgpu_async_gfx_ring)
3613 gfx_v11_0_cp_gfx_enable(adev, true);
3614
3615 ring = &adev->gfx.gfx_ring[0];
3616 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3617 if (r) {
3618 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3619 return r;
3620 }
3621
3622 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3623 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3624
3625 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3626 amdgpu_ring_write(ring, 0x80000000);
3627 amdgpu_ring_write(ring, 0x80000000);
3628
3629 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3630 for (ext = sect->section; ext->extent != NULL; ++ext) {
3631 if (sect->id == SECT_CONTEXT) {
3632 amdgpu_ring_write(ring,
3633 PACKET3(PACKET3_SET_CONTEXT_REG,
3634 ext->reg_count));
3635 amdgpu_ring_write(ring, ext->reg_index -
3636 PACKET3_SET_CONTEXT_REG_START);
3637 for (i = 0; i < ext->reg_count; i++)
3638 amdgpu_ring_write(ring, ext->extent[i]);
3639 }
3640 }
3641 }
3642
3643 ctx_reg_offset =
3644 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3645 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3646 amdgpu_ring_write(ring, ctx_reg_offset);
3647 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3648
3649 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3650 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3651
3652 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3653 amdgpu_ring_write(ring, 0);
3654
3655 amdgpu_ring_commit(ring);
3656
3657 /* submit cs packet to copy state 0 to next available state */
3658 if (adev->gfx.num_gfx_rings > 1) {
3659 /* maximum supported gfx ring is 2 */
3660 ring = &adev->gfx.gfx_ring[1];
3661 r = amdgpu_ring_alloc(ring, 2);
3662 if (r) {
3663 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3664 return r;
3665 }
3666
3667 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3668 amdgpu_ring_write(ring, 0);
3669
3670 amdgpu_ring_commit(ring);
3671 }
3672 return 0;
3673 }
3674
gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device * adev,CP_PIPE_ID pipe)3675 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3676 CP_PIPE_ID pipe)
3677 {
3678 u32 tmp;
3679
3680 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3681 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3682
3683 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3684 }
3685
gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device * adev,struct amdgpu_ring * ring)3686 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3687 struct amdgpu_ring *ring)
3688 {
3689 u32 tmp;
3690
3691 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3692 if (ring->use_doorbell) {
3693 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3694 DOORBELL_OFFSET, ring->doorbell_index);
3695 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3696 DOORBELL_EN, 1);
3697 } else {
3698 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3699 DOORBELL_EN, 0);
3700 }
3701 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3702
3703 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3704 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3705 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3706
3707 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3708 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3709 }
3710
gfx_v11_0_cp_gfx_resume(struct amdgpu_device * adev)3711 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3712 {
3713 struct amdgpu_ring *ring;
3714 u32 tmp;
3715 u32 rb_bufsz;
3716 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3717
3718 /* Set the write pointer delay */
3719 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3720
3721 /* set the RB to use vmid 0 */
3722 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3723
3724 /* Init gfx ring 0 for pipe 0 */
3725 mutex_lock(&adev->srbm_mutex);
3726 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3727
3728 /* Set ring buffer size */
3729 ring = &adev->gfx.gfx_ring[0];
3730 rb_bufsz = order_base_2(ring->ring_size / 8);
3731 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3732 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3733 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3734
3735 /* Initialize the ring buffer's write pointers */
3736 ring->wptr = 0;
3737 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3738 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3739
3740 /* set the wb address whether it's enabled or not */
3741 rptr_addr = ring->rptr_gpu_addr;
3742 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3743 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3744 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3745
3746 wptr_gpu_addr = ring->wptr_gpu_addr;
3747 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3748 lower_32_bits(wptr_gpu_addr));
3749 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3750 upper_32_bits(wptr_gpu_addr));
3751
3752 mdelay(1);
3753 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3754
3755 rb_addr = ring->gpu_addr >> 8;
3756 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3757 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3758
3759 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3760
3761 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3762 mutex_unlock(&adev->srbm_mutex);
3763
3764 /* Init gfx ring 1 for pipe 1 */
3765 if (adev->gfx.num_gfx_rings > 1) {
3766 mutex_lock(&adev->srbm_mutex);
3767 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3768 /* maximum supported gfx ring is 2 */
3769 ring = &adev->gfx.gfx_ring[1];
3770 rb_bufsz = order_base_2(ring->ring_size / 8);
3771 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3772 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3773 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3774 /* Initialize the ring buffer's write pointers */
3775 ring->wptr = 0;
3776 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3777 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3778 /* Set the wb address whether it's enabled or not */
3779 rptr_addr = ring->rptr_gpu_addr;
3780 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3781 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3782 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3783 wptr_gpu_addr = ring->wptr_gpu_addr;
3784 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3785 lower_32_bits(wptr_gpu_addr));
3786 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3787 upper_32_bits(wptr_gpu_addr));
3788
3789 mdelay(1);
3790 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3791
3792 rb_addr = ring->gpu_addr >> 8;
3793 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3794 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3795 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3796
3797 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3798 mutex_unlock(&adev->srbm_mutex);
3799 }
3800 /* Switch to pipe 0 */
3801 mutex_lock(&adev->srbm_mutex);
3802 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3803 mutex_unlock(&adev->srbm_mutex);
3804
3805 /* start the ring */
3806 gfx_v11_0_cp_gfx_start(adev);
3807
3808 return 0;
3809 }
3810
gfx_v11_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3811 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3812 {
3813 u32 data;
3814
3815 if (adev->gfx.rs64_enable) {
3816 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3817 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3818 enable ? 0 : 1);
3819 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3820 enable ? 0 : 1);
3821 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3822 enable ? 0 : 1);
3823 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3824 enable ? 0 : 1);
3825 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3826 enable ? 0 : 1);
3827 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3828 enable ? 1 : 0);
3829 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3830 enable ? 1 : 0);
3831 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3832 enable ? 1 : 0);
3833 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3834 enable ? 1 : 0);
3835 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3836 enable ? 0 : 1);
3837 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3838 } else {
3839 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3840
3841 if (enable) {
3842 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3843 if (!adev->enable_mes_kiq)
3844 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3845 MEC_ME2_HALT, 0);
3846 } else {
3847 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3848 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3849 }
3850 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3851 }
3852
3853 udelay(50);
3854 }
3855
gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device * adev)3856 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3857 {
3858 const struct gfx_firmware_header_v1_0 *mec_hdr;
3859 const __le32 *fw_data;
3860 unsigned i, fw_size;
3861 u32 *fw = NULL;
3862 int r;
3863
3864 if (!adev->gfx.mec_fw)
3865 return -EINVAL;
3866
3867 gfx_v11_0_cp_compute_enable(adev, false);
3868
3869 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3870 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3871
3872 fw_data = (const __le32 *)
3873 (adev->gfx.mec_fw->data +
3874 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3875 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3876
3877 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3878 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3879 &adev->gfx.mec.mec_fw_obj,
3880 &adev->gfx.mec.mec_fw_gpu_addr,
3881 (void **)&fw);
3882 if (r) {
3883 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3884 gfx_v11_0_mec_fini(adev);
3885 return r;
3886 }
3887
3888 memcpy(fw, fw_data, fw_size);
3889
3890 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3891 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3892
3893 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3894
3895 /* MEC1 */
3896 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3897
3898 for (i = 0; i < mec_hdr->jt_size; i++)
3899 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3900 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3901
3902 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3903
3904 return 0;
3905 }
3906
gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device * adev)3907 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3908 {
3909 const struct gfx_firmware_header_v2_0 *mec_hdr;
3910 const __le32 *fw_ucode, *fw_data;
3911 u32 tmp, fw_ucode_size, fw_data_size;
3912 u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3913 u32 *fw_ucode_ptr, *fw_data_ptr;
3914 int r;
3915
3916 if (!adev->gfx.mec_fw)
3917 return -EINVAL;
3918
3919 gfx_v11_0_cp_compute_enable(adev, false);
3920
3921 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3922 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3923
3924 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3925 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3926 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3927
3928 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3929 le32_to_cpu(mec_hdr->data_offset_bytes));
3930 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3931
3932 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3933 64 * 1024,
3934 AMDGPU_GEM_DOMAIN_VRAM |
3935 AMDGPU_GEM_DOMAIN_GTT,
3936 &adev->gfx.mec.mec_fw_obj,
3937 &adev->gfx.mec.mec_fw_gpu_addr,
3938 (void **)&fw_ucode_ptr);
3939 if (r) {
3940 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3941 gfx_v11_0_mec_fini(adev);
3942 return r;
3943 }
3944
3945 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3946 64 * 1024,
3947 AMDGPU_GEM_DOMAIN_VRAM |
3948 AMDGPU_GEM_DOMAIN_GTT,
3949 &adev->gfx.mec.mec_fw_data_obj,
3950 &adev->gfx.mec.mec_fw_data_gpu_addr,
3951 (void **)&fw_data_ptr);
3952 if (r) {
3953 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3954 gfx_v11_0_mec_fini(adev);
3955 return r;
3956 }
3957
3958 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3959 memcpy(fw_data_ptr, fw_data, fw_data_size);
3960
3961 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3962 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3963 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3964 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3965
3966 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3967 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3968 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3969 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3970 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3971
3972 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3973 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3974 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3975 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3976
3977 mutex_lock(&adev->srbm_mutex);
3978 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3979 soc21_grbm_select(adev, 1, i, 0, 0);
3980
3981 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3982 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3983 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3984
3985 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3986 mec_hdr->ucode_start_addr_lo >> 2 |
3987 mec_hdr->ucode_start_addr_hi << 30);
3988 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3989 mec_hdr->ucode_start_addr_hi >> 2);
3990
3991 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3992 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3993 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3994 }
3995 mutex_unlock(&adev->srbm_mutex);
3996 soc21_grbm_select(adev, 0, 0, 0, 0);
3997
3998 /* Trigger an invalidation of the L1 instruction caches */
3999 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
4000 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
4001 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
4002
4003 /* Wait for invalidation complete */
4004 for (i = 0; i < usec_timeout; i++) {
4005 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
4006 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
4007 INVALIDATE_DCACHE_COMPLETE))
4008 break;
4009 udelay(1);
4010 }
4011
4012 if (i >= usec_timeout) {
4013 dev_err(adev->dev, "failed to invalidate instruction cache\n");
4014 return -EINVAL;
4015 }
4016
4017 /* Trigger an invalidation of the L1 instruction caches */
4018 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
4019 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
4020 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
4021
4022 /* Wait for invalidation complete */
4023 for (i = 0; i < usec_timeout; i++) {
4024 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
4025 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
4026 INVALIDATE_CACHE_COMPLETE))
4027 break;
4028 udelay(1);
4029 }
4030
4031 if (i >= usec_timeout) {
4032 dev_err(adev->dev, "failed to invalidate instruction cache\n");
4033 return -EINVAL;
4034 }
4035
4036 return 0;
4037 }
4038
gfx_v11_0_kiq_setting(struct amdgpu_ring * ring)4039 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
4040 {
4041 uint32_t tmp;
4042 struct amdgpu_device *adev = ring->adev;
4043
4044 /* tell RLC which is KIQ queue */
4045 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
4046 tmp &= 0xffffff00;
4047 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4048 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80);
4049 }
4050
gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device * adev)4051 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
4052 {
4053 /* set graphics engine doorbell range */
4054 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
4055 (adev->doorbell_index.gfx_ring0 * 2) << 2);
4056 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
4057 (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
4058
4059 /* set compute engine doorbell range */
4060 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4061 (adev->doorbell_index.kiq * 2) << 2);
4062 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4063 (adev->doorbell_index.userqueue_end * 2) << 2);
4064 }
4065
gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device * adev,struct v11_gfx_mqd * mqd,struct amdgpu_mqd_prop * prop)4066 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev,
4067 struct v11_gfx_mqd *mqd,
4068 struct amdgpu_mqd_prop *prop)
4069 {
4070 bool priority = 0;
4071 u32 tmp;
4072
4073 /* set up default queue priority level
4074 * 0x0 = low priority, 0x1 = high priority
4075 */
4076 if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH)
4077 priority = 1;
4078
4079 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT;
4080 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority);
4081 mqd->cp_gfx_hqd_queue_priority = tmp;
4082 }
4083
gfx_v11_0_gfx_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)4084 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
4085 struct amdgpu_mqd_prop *prop)
4086 {
4087 struct v11_gfx_mqd *mqd = m;
4088 uint64_t hqd_gpu_addr, wb_gpu_addr;
4089 uint32_t tmp;
4090 uint32_t rb_bufsz;
4091
4092 /* set up gfx hqd wptr */
4093 mqd->cp_gfx_hqd_wptr = 0;
4094 mqd->cp_gfx_hqd_wptr_hi = 0;
4095
4096 /* set the pointer to the MQD */
4097 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
4098 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4099
4100 /* set up mqd control */
4101 tmp = regCP_GFX_MQD_CONTROL_DEFAULT;
4102 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
4103 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
4104 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
4105 mqd->cp_gfx_mqd_control = tmp;
4106
4107 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
4108 tmp = regCP_GFX_HQD_VMID_DEFAULT;
4109 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
4110 mqd->cp_gfx_hqd_vmid = 0;
4111
4112 /* set up gfx queue priority */
4113 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop);
4114
4115 /* set up time quantum */
4116 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT;
4117 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
4118 mqd->cp_gfx_hqd_quantum = tmp;
4119
4120 /* set up gfx hqd base. this is similar as CP_RB_BASE */
4121 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4122 mqd->cp_gfx_hqd_base = hqd_gpu_addr;
4123 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
4124
4125 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
4126 wb_gpu_addr = prop->rptr_gpu_addr;
4127 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
4128 mqd->cp_gfx_hqd_rptr_addr_hi =
4129 upper_32_bits(wb_gpu_addr) & 0xffff;
4130
4131 /* set up rb_wptr_poll addr */
4132 wb_gpu_addr = prop->wptr_gpu_addr;
4133 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4134 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4135
4136 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
4137 rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
4138 tmp = regCP_GFX_HQD_CNTL_DEFAULT;
4139 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
4140 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
4141 #ifdef __BIG_ENDIAN
4142 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
4143 #endif
4144 if (prop->tmz_queue)
4145 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, TMZ_MATCH, 1);
4146 if (!prop->kernel_queue)
4147 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_NON_PRIV, 1);
4148 mqd->cp_gfx_hqd_cntl = tmp;
4149
4150 /* set up cp_doorbell_control */
4151 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT;
4152 if (prop->use_doorbell) {
4153 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4154 DOORBELL_OFFSET, prop->doorbell_index);
4155 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4156 DOORBELL_EN, 1);
4157 } else
4158 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4159 DOORBELL_EN, 0);
4160 mqd->cp_rb_doorbell_control = tmp;
4161
4162 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4163 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT;
4164
4165 /* active the queue */
4166 mqd->cp_gfx_hqd_active = 1;
4167
4168 /* set gfx UQ items */
4169 mqd->shadow_base_lo = lower_32_bits(prop->shadow_addr);
4170 mqd->shadow_base_hi = upper_32_bits(prop->shadow_addr);
4171 mqd->gds_bkup_base_lo = lower_32_bits(prop->gds_bkup_addr);
4172 mqd->gds_bkup_base_hi = upper_32_bits(prop->gds_bkup_addr);
4173 mqd->fw_work_area_base_lo = lower_32_bits(prop->csa_addr);
4174 mqd->fw_work_area_base_hi = upper_32_bits(prop->csa_addr);
4175 mqd->fence_address_lo = lower_32_bits(prop->fence_address);
4176 mqd->fence_address_hi = upper_32_bits(prop->fence_address);
4177
4178 return 0;
4179 }
4180
gfx_v11_0_kgq_init_queue(struct amdgpu_ring * ring,bool reset)4181 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset)
4182 {
4183 struct amdgpu_device *adev = ring->adev;
4184 struct v11_gfx_mqd *mqd = ring->mqd_ptr;
4185 int mqd_idx = ring - &adev->gfx.gfx_ring[0];
4186
4187 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
4188 memset((void *)mqd, 0, sizeof(*mqd));
4189 mutex_lock(&adev->srbm_mutex);
4190 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4191 amdgpu_ring_init_mqd(ring);
4192 soc21_grbm_select(adev, 0, 0, 0, 0);
4193 mutex_unlock(&adev->srbm_mutex);
4194 if (adev->gfx.me.mqd_backup[mqd_idx])
4195 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4196 } else {
4197 /* restore mqd with the backup copy */
4198 if (adev->gfx.me.mqd_backup[mqd_idx])
4199 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
4200 /* reset the ring */
4201 ring->wptr = 0;
4202 *ring->wptr_cpu_addr = 0;
4203 amdgpu_ring_clear_ring(ring);
4204 }
4205
4206 return 0;
4207 }
4208
gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device * adev)4209 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
4210 {
4211 int r, i;
4212
4213 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4214 r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i], false);
4215 if (r)
4216 return r;
4217 }
4218
4219 r = amdgpu_gfx_enable_kgq(adev, 0);
4220 if (r)
4221 return r;
4222
4223 return gfx_v11_0_cp_gfx_start(adev);
4224 }
4225
gfx_v11_0_compute_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)4226 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
4227 struct amdgpu_mqd_prop *prop)
4228 {
4229 struct v11_compute_mqd *mqd = m;
4230 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4231 uint32_t tmp;
4232
4233 mqd->header = 0xC0310800;
4234 mqd->compute_pipelinestat_enable = 0x00000001;
4235 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4236 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4237 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4238 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4239 mqd->compute_misc_reserved = 0x00000007;
4240
4241 eop_base_addr = prop->eop_gpu_addr >> 8;
4242 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4243 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4244
4245 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4246 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
4247 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4248 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
4249
4250 mqd->cp_hqd_eop_control = tmp;
4251
4252 /* enable doorbell? */
4253 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
4254
4255 if (prop->use_doorbell) {
4256 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4257 DOORBELL_OFFSET, prop->doorbell_index);
4258 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4259 DOORBELL_EN, 1);
4260 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4261 DOORBELL_SOURCE, 0);
4262 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4263 DOORBELL_HIT, 0);
4264 } else {
4265 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4266 DOORBELL_EN, 0);
4267 }
4268
4269 mqd->cp_hqd_pq_doorbell_control = tmp;
4270
4271 /* disable the queue if it's active */
4272 mqd->cp_hqd_dequeue_request = 0;
4273 mqd->cp_hqd_pq_rptr = 0;
4274 mqd->cp_hqd_pq_wptr_lo = 0;
4275 mqd->cp_hqd_pq_wptr_hi = 0;
4276
4277 /* set the pointer to the MQD */
4278 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
4279 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4280
4281 /* set MQD vmid to 0 */
4282 tmp = regCP_MQD_CONTROL_DEFAULT;
4283 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4284 mqd->cp_mqd_control = tmp;
4285
4286 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4287 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4288 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4289 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4290
4291 /* set up the HQD, this is similar to CP_RB0_CNTL */
4292 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
4293 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4294 (order_base_2(prop->queue_size / 4) - 1));
4295 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4296 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4297 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
4298 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH,
4299 prop->allow_tunneling);
4300 if (prop->kernel_queue) {
4301 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4302 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4303 }
4304 if (prop->tmz_queue)
4305 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TMZ, 1);
4306 mqd->cp_hqd_pq_control = tmp;
4307
4308 /* set the wb address whether it's enabled or not */
4309 wb_gpu_addr = prop->rptr_gpu_addr;
4310 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4311 mqd->cp_hqd_pq_rptr_report_addr_hi =
4312 upper_32_bits(wb_gpu_addr) & 0xffff;
4313
4314 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4315 wb_gpu_addr = prop->wptr_gpu_addr;
4316 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4317 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4318
4319 tmp = 0;
4320 /* enable the doorbell if requested */
4321 if (prop->use_doorbell) {
4322 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
4323 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4324 DOORBELL_OFFSET, prop->doorbell_index);
4325
4326 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4327 DOORBELL_EN, 1);
4328 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4329 DOORBELL_SOURCE, 0);
4330 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4331 DOORBELL_HIT, 0);
4332 }
4333
4334 mqd->cp_hqd_pq_doorbell_control = tmp;
4335
4336 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4337 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT;
4338
4339 /* set the vmid for the queue */
4340 mqd->cp_hqd_vmid = 0;
4341
4342 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
4343 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
4344 mqd->cp_hqd_persistent_state = tmp;
4345
4346 /* set MIN_IB_AVAIL_SIZE */
4347 tmp = regCP_HQD_IB_CONTROL_DEFAULT;
4348 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4349 mqd->cp_hqd_ib_control = tmp;
4350
4351 /* set static priority for a compute queue/ring */
4352 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
4353 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
4354
4355 mqd->cp_hqd_active = prop->hqd_active;
4356
4357 /* set UQ fenceaddress */
4358 mqd->fence_address_lo = lower_32_bits(prop->fence_address);
4359 mqd->fence_address_hi = upper_32_bits(prop->fence_address);
4360
4361 return 0;
4362 }
4363
gfx_v11_0_kiq_init_register(struct amdgpu_ring * ring)4364 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
4365 {
4366 struct amdgpu_device *adev = ring->adev;
4367 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4368 int j;
4369
4370 /* inactivate the queue */
4371 if (amdgpu_sriov_vf(adev))
4372 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
4373
4374 /* disable wptr polling */
4375 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4376
4377 /* write the EOP addr */
4378 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
4379 mqd->cp_hqd_eop_base_addr_lo);
4380 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
4381 mqd->cp_hqd_eop_base_addr_hi);
4382
4383 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4384 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
4385 mqd->cp_hqd_eop_control);
4386
4387 /* enable doorbell? */
4388 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4389 mqd->cp_hqd_pq_doorbell_control);
4390
4391 /* disable the queue if it's active */
4392 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
4393 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
4394 for (j = 0; j < adev->usec_timeout; j++) {
4395 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
4396 break;
4397 udelay(1);
4398 }
4399 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
4400 mqd->cp_hqd_dequeue_request);
4401 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
4402 mqd->cp_hqd_pq_rptr);
4403 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4404 mqd->cp_hqd_pq_wptr_lo);
4405 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4406 mqd->cp_hqd_pq_wptr_hi);
4407 }
4408
4409 /* set the pointer to the MQD */
4410 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
4411 mqd->cp_mqd_base_addr_lo);
4412 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
4413 mqd->cp_mqd_base_addr_hi);
4414
4415 /* set MQD vmid to 0 */
4416 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
4417 mqd->cp_mqd_control);
4418
4419 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4420 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
4421 mqd->cp_hqd_pq_base_lo);
4422 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
4423 mqd->cp_hqd_pq_base_hi);
4424
4425 /* set up the HQD, this is similar to CP_RB0_CNTL */
4426 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
4427 mqd->cp_hqd_pq_control);
4428
4429 /* set the wb address whether it's enabled or not */
4430 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
4431 mqd->cp_hqd_pq_rptr_report_addr_lo);
4432 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4433 mqd->cp_hqd_pq_rptr_report_addr_hi);
4434
4435 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4436 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
4437 mqd->cp_hqd_pq_wptr_poll_addr_lo);
4438 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
4439 mqd->cp_hqd_pq_wptr_poll_addr_hi);
4440
4441 /* enable the doorbell if requested */
4442 if (ring->use_doorbell) {
4443 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4444 (adev->doorbell_index.kiq * 2) << 2);
4445 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4446 (adev->doorbell_index.userqueue_end * 2) << 2);
4447 }
4448
4449 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4450 mqd->cp_hqd_pq_doorbell_control);
4451
4452 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4453 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4454 mqd->cp_hqd_pq_wptr_lo);
4455 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4456 mqd->cp_hqd_pq_wptr_hi);
4457
4458 /* set the vmid for the queue */
4459 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4460
4461 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4462 mqd->cp_hqd_persistent_state);
4463
4464 /* activate the queue */
4465 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4466 mqd->cp_hqd_active);
4467
4468 if (ring->use_doorbell)
4469 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4470
4471 return 0;
4472 }
4473
gfx_v11_0_kiq_init_queue(struct amdgpu_ring * ring)4474 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4475 {
4476 struct amdgpu_device *adev = ring->adev;
4477 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4478
4479 gfx_v11_0_kiq_setting(ring);
4480
4481 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4482 /* reset MQD to a clean status */
4483 if (adev->gfx.kiq[0].mqd_backup)
4484 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd));
4485
4486 /* reset ring buffer */
4487 ring->wptr = 0;
4488 amdgpu_ring_clear_ring(ring);
4489
4490 mutex_lock(&adev->srbm_mutex);
4491 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4492 gfx_v11_0_kiq_init_register(ring);
4493 soc21_grbm_select(adev, 0, 0, 0, 0);
4494 mutex_unlock(&adev->srbm_mutex);
4495 } else {
4496 memset((void *)mqd, 0, sizeof(*mqd));
4497 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4498 amdgpu_ring_clear_ring(ring);
4499 mutex_lock(&adev->srbm_mutex);
4500 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4501 amdgpu_ring_init_mqd(ring);
4502 gfx_v11_0_kiq_init_register(ring);
4503 soc21_grbm_select(adev, 0, 0, 0, 0);
4504 mutex_unlock(&adev->srbm_mutex);
4505
4506 if (adev->gfx.kiq[0].mqd_backup)
4507 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd));
4508 }
4509
4510 return 0;
4511 }
4512
gfx_v11_0_kcq_init_queue(struct amdgpu_ring * ring,bool reset)4513 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset)
4514 {
4515 struct amdgpu_device *adev = ring->adev;
4516 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4517 int mqd_idx = ring - &adev->gfx.compute_ring[0];
4518
4519 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
4520 memset((void *)mqd, 0, sizeof(*mqd));
4521 mutex_lock(&adev->srbm_mutex);
4522 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4523 amdgpu_ring_init_mqd(ring);
4524 soc21_grbm_select(adev, 0, 0, 0, 0);
4525 mutex_unlock(&adev->srbm_mutex);
4526
4527 if (adev->gfx.mec.mqd_backup[mqd_idx])
4528 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4529 } else {
4530 /* restore MQD to a clean status */
4531 if (adev->gfx.mec.mqd_backup[mqd_idx])
4532 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4533 /* reset ring buffer */
4534 ring->wptr = 0;
4535 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4536 amdgpu_ring_clear_ring(ring);
4537 }
4538
4539 return 0;
4540 }
4541
gfx_v11_0_kiq_resume(struct amdgpu_device * adev)4542 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4543 {
4544 gfx_v11_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
4545 return 0;
4546 }
4547
gfx_v11_0_kcq_resume(struct amdgpu_device * adev)4548 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4549 {
4550 int i, r;
4551
4552 if (!amdgpu_async_gfx_ring)
4553 gfx_v11_0_cp_compute_enable(adev, true);
4554
4555 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4556 r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
4557 if (r)
4558 return r;
4559 }
4560
4561 return amdgpu_gfx_enable_kcq(adev, 0);
4562 }
4563
gfx_v11_0_cp_resume(struct amdgpu_device * adev)4564 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4565 {
4566 int r, i;
4567 struct amdgpu_ring *ring;
4568
4569 if (!(adev->flags & AMD_IS_APU))
4570 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4571
4572 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4573 /* legacy firmware loading */
4574 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4575 if (r)
4576 return r;
4577
4578 if (adev->gfx.rs64_enable)
4579 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4580 else
4581 r = gfx_v11_0_cp_compute_load_microcode(adev);
4582 if (r)
4583 return r;
4584 }
4585
4586 gfx_v11_0_cp_set_doorbell_range(adev);
4587
4588 if (amdgpu_async_gfx_ring) {
4589 gfx_v11_0_cp_compute_enable(adev, true);
4590 gfx_v11_0_cp_gfx_enable(adev, true);
4591 }
4592
4593 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4594 r = amdgpu_mes_kiq_hw_init(adev);
4595 else
4596 r = gfx_v11_0_kiq_resume(adev);
4597 if (r)
4598 return r;
4599
4600 r = gfx_v11_0_kcq_resume(adev);
4601 if (r)
4602 return r;
4603
4604 if (!amdgpu_async_gfx_ring) {
4605 r = gfx_v11_0_cp_gfx_resume(adev);
4606 if (r)
4607 return r;
4608 } else {
4609 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4610 if (r)
4611 return r;
4612 }
4613
4614 if (adev->gfx.disable_kq) {
4615 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4616 ring = &adev->gfx.gfx_ring[i];
4617 /* we don't want to set ring->ready */
4618 r = amdgpu_ring_test_ring(ring);
4619 if (r)
4620 return r;
4621 }
4622 if (amdgpu_async_gfx_ring)
4623 amdgpu_gfx_disable_kgq(adev, 0);
4624 } else {
4625 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4626 ring = &adev->gfx.gfx_ring[i];
4627 r = amdgpu_ring_test_helper(ring);
4628 if (r)
4629 return r;
4630 }
4631 }
4632
4633 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4634 ring = &adev->gfx.compute_ring[i];
4635 r = amdgpu_ring_test_helper(ring);
4636 if (r)
4637 return r;
4638 }
4639
4640 return 0;
4641 }
4642
gfx_v11_0_cp_enable(struct amdgpu_device * adev,bool enable)4643 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4644 {
4645 gfx_v11_0_cp_gfx_enable(adev, enable);
4646 gfx_v11_0_cp_compute_enable(adev, enable);
4647 }
4648
gfx_v11_0_gfxhub_enable(struct amdgpu_device * adev)4649 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4650 {
4651 int r;
4652 bool value;
4653
4654 r = adev->gfxhub.funcs->gart_enable(adev);
4655 if (r)
4656 return r;
4657
4658 amdgpu_device_flush_hdp(adev, NULL);
4659
4660 value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS;
4661
4662 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4663 /* TODO investigate why this and the hdp flush above is needed,
4664 * are we missing a flush somewhere else? */
4665 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
4666
4667 return 0;
4668 }
4669
gfx_v11_0_select_cp_fw_arch(struct amdgpu_device * adev)4670 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4671 {
4672 u32 tmp;
4673
4674 /* select RS64 */
4675 if (adev->gfx.rs64_enable) {
4676 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4677 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4678 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4679
4680 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4681 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4682 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4683 }
4684
4685 if (amdgpu_emu_mode == 1)
4686 msleep(100);
4687 }
4688
get_gb_addr_config(struct amdgpu_device * adev)4689 static int get_gb_addr_config(struct amdgpu_device * adev)
4690 {
4691 u32 gb_addr_config;
4692
4693 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4694 if (gb_addr_config == 0)
4695 return -EINVAL;
4696
4697 adev->gfx.config.gb_addr_config_fields.num_pkrs =
4698 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4699
4700 adev->gfx.config.gb_addr_config = gb_addr_config;
4701
4702 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4703 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4704 GB_ADDR_CONFIG, NUM_PIPES);
4705
4706 adev->gfx.config.max_tile_pipes =
4707 adev->gfx.config.gb_addr_config_fields.num_pipes;
4708
4709 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4710 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4711 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4712 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4713 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4714 GB_ADDR_CONFIG, NUM_RB_PER_SE);
4715 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4716 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4717 GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4718 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4719 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4720 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4721
4722 return 0;
4723 }
4724
gfx_v11_0_disable_gpa_mode(struct amdgpu_device * adev)4725 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4726 {
4727 uint32_t data;
4728
4729 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4730 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4731 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4732
4733 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4734 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4735 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4736 }
4737
gfx_v11_0_hw_init(struct amdgpu_ip_block * ip_block)4738 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
4739 {
4740 int r;
4741 struct amdgpu_device *adev = ip_block->adev;
4742
4743 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
4744 adev->gfx.cleaner_shader_ptr);
4745
4746 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4747 if (adev->gfx.imu.funcs) {
4748 /* RLC autoload sequence 1: Program rlc ram */
4749 if (adev->gfx.imu.funcs->program_rlc_ram)
4750 adev->gfx.imu.funcs->program_rlc_ram(adev);
4751 /* rlc autoload firmware */
4752 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4753 if (r)
4754 return r;
4755 }
4756 } else {
4757 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4758 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4759 if (adev->gfx.imu.funcs->load_microcode)
4760 adev->gfx.imu.funcs->load_microcode(adev);
4761 if (adev->gfx.imu.funcs->setup_imu)
4762 adev->gfx.imu.funcs->setup_imu(adev);
4763 if (adev->gfx.imu.funcs->start_imu)
4764 adev->gfx.imu.funcs->start_imu(adev);
4765 }
4766
4767 /* disable gpa mode in backdoor loading */
4768 gfx_v11_0_disable_gpa_mode(adev);
4769 }
4770 }
4771
4772 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4773 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4774 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4775 if (r) {
4776 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4777 return r;
4778 }
4779 }
4780
4781 adev->gfx.is_poweron = true;
4782
4783 if(get_gb_addr_config(adev))
4784 DRM_WARN("Invalid gb_addr_config !\n");
4785
4786 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4787 adev->gfx.rs64_enable)
4788 gfx_v11_0_config_gfx_rs64(adev);
4789
4790 r = gfx_v11_0_gfxhub_enable(adev);
4791 if (r)
4792 return r;
4793
4794 if (!amdgpu_emu_mode)
4795 gfx_v11_0_init_golden_registers(adev);
4796
4797 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4798 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4799 /**
4800 * For gfx 11, rlc firmware loading relies on smu firmware is
4801 * loaded firstly, so in direct type, it has to load smc ucode
4802 * here before rlc.
4803 */
4804 r = amdgpu_pm_load_smu_firmware(adev, NULL);
4805 if (r)
4806 return r;
4807 }
4808
4809 gfx_v11_0_constants_init(adev);
4810
4811 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4812 gfx_v11_0_select_cp_fw_arch(adev);
4813
4814 if (adev->nbio.funcs->gc_doorbell_init)
4815 adev->nbio.funcs->gc_doorbell_init(adev);
4816
4817 r = gfx_v11_0_rlc_resume(adev);
4818 if (r)
4819 return r;
4820
4821 /*
4822 * init golden registers and rlc resume may override some registers,
4823 * reconfig them here
4824 */
4825 gfx_v11_0_tcp_harvest(adev);
4826
4827 r = gfx_v11_0_cp_resume(adev);
4828 if (r)
4829 return r;
4830
4831 /* get IMU version from HW if it's not set */
4832 if (!adev->gfx.imu_fw_version)
4833 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0);
4834
4835 return r;
4836 }
4837
gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device * adev,bool enable)4838 static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev,
4839 bool enable)
4840 {
4841 unsigned int irq_type;
4842 int m, p, r;
4843
4844 if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) {
4845 for (m = 0; m < adev->gfx.me.num_me; m++) {
4846 for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) {
4847 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
4848 if (enable)
4849 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
4850 irq_type);
4851 else
4852 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
4853 irq_type);
4854 if (r)
4855 return r;
4856 }
4857 }
4858 }
4859
4860 if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) {
4861 for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
4862 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
4863 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
4864 + (m * adev->gfx.mec.num_pipe_per_mec)
4865 + p;
4866 if (enable)
4867 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
4868 irq_type);
4869 else
4870 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
4871 irq_type);
4872 if (r)
4873 return r;
4874 }
4875 }
4876 }
4877
4878 return 0;
4879 }
4880
gfx_v11_0_hw_fini(struct amdgpu_ip_block * ip_block)4881 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
4882 {
4883 struct amdgpu_device *adev = ip_block->adev;
4884
4885 cancel_delayed_work_sync(&adev->gfx.idle_work);
4886
4887 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4888 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4889 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4890 gfx_v11_0_set_userq_eop_interrupts(adev, false);
4891
4892 if (!adev->no_hw_access) {
4893 if (amdgpu_async_gfx_ring &&
4894 !adev->gfx.disable_kq) {
4895 if (amdgpu_gfx_disable_kgq(adev, 0))
4896 DRM_ERROR("KGQ disable failed\n");
4897 }
4898
4899 if (amdgpu_gfx_disable_kcq(adev, 0))
4900 DRM_ERROR("KCQ disable failed\n");
4901
4902 amdgpu_mes_kiq_hw_fini(adev);
4903 }
4904
4905 if (amdgpu_sriov_vf(adev))
4906 /* Remove the steps disabling CPG and clearing KIQ position,
4907 * so that CP could perform IDLE-SAVE during switch. Those
4908 * steps are necessary to avoid a DMAR error in gfx9 but it is
4909 * not reproduced on gfx11.
4910 */
4911 return 0;
4912
4913 gfx_v11_0_cp_enable(adev, false);
4914 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4915
4916 adev->gfxhub.funcs->gart_disable(adev);
4917
4918 adev->gfx.is_poweron = false;
4919
4920 return 0;
4921 }
4922
gfx_v11_0_suspend(struct amdgpu_ip_block * ip_block)4923 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block)
4924 {
4925 return gfx_v11_0_hw_fini(ip_block);
4926 }
4927
gfx_v11_0_resume(struct amdgpu_ip_block * ip_block)4928 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block)
4929 {
4930 return gfx_v11_0_hw_init(ip_block);
4931 }
4932
gfx_v11_0_is_idle(struct amdgpu_ip_block * ip_block)4933 static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block)
4934 {
4935 struct amdgpu_device *adev = ip_block->adev;
4936
4937 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4938 GRBM_STATUS, GUI_ACTIVE))
4939 return false;
4940 else
4941 return true;
4942 }
4943
gfx_v11_0_wait_for_idle(struct amdgpu_ip_block * ip_block)4944 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4945 {
4946 unsigned i;
4947 u32 tmp;
4948 struct amdgpu_device *adev = ip_block->adev;
4949
4950 for (i = 0; i < adev->usec_timeout; i++) {
4951 /* read MC_STATUS */
4952 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4953 GRBM_STATUS__GUI_ACTIVE_MASK;
4954
4955 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4956 return 0;
4957 udelay(1);
4958 }
4959 return -ETIMEDOUT;
4960 }
4961
gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device * adev,bool req)4962 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev,
4963 bool req)
4964 {
4965 u32 i, tmp, val;
4966
4967 for (i = 0; i < adev->usec_timeout; i++) {
4968 /* Request with MeId=2, PipeId=0 */
4969 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
4970 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
4971 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
4972
4973 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
4974 if (req) {
4975 if (val == tmp)
4976 break;
4977 } else {
4978 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
4979 REQUEST, 1);
4980
4981 /* unlocked or locked by firmware */
4982 if (val != tmp)
4983 break;
4984 }
4985 udelay(1);
4986 }
4987
4988 if (i >= adev->usec_timeout)
4989 return -EINVAL;
4990
4991 return 0;
4992 }
4993
gfx_v11_0_soft_reset(struct amdgpu_ip_block * ip_block)4994 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block)
4995 {
4996 u32 grbm_soft_reset = 0;
4997 u32 tmp;
4998 int r, i, j, k;
4999 struct amdgpu_device *adev = ip_block->adev;
5000
5001 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5002
5003 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5004 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
5005 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
5006 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
5007 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
5008 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
5009
5010 mutex_lock(&adev->srbm_mutex);
5011 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
5012 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
5013 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
5014 soc21_grbm_select(adev, i, k, j, 0);
5015
5016 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
5017 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
5018 }
5019 }
5020 }
5021 for (i = 0; i < adev->gfx.me.num_me; ++i) {
5022 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
5023 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
5024 soc21_grbm_select(adev, i, k, j, 0);
5025
5026 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
5027 }
5028 }
5029 }
5030 soc21_grbm_select(adev, 0, 0, 0, 0);
5031 mutex_unlock(&adev->srbm_mutex);
5032
5033 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */
5034 mutex_lock(&adev->gfx.reset_sem_mutex);
5035 r = gfx_v11_0_request_gfx_index_mutex(adev, true);
5036 if (r) {
5037 mutex_unlock(&adev->gfx.reset_sem_mutex);
5038 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n");
5039 return r;
5040 }
5041
5042 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
5043
5044 // Read CP_VMID_RESET register three times.
5045 // to get sufficient time for GFX_HQD_ACTIVE reach 0
5046 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
5047 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
5048 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
5049
5050 /* release the gfx mutex */
5051 r = gfx_v11_0_request_gfx_index_mutex(adev, false);
5052 mutex_unlock(&adev->gfx.reset_sem_mutex);
5053 if (r) {
5054 DRM_ERROR("Failed to release the gfx mutex during soft reset\n");
5055 return r;
5056 }
5057
5058 for (i = 0; i < adev->usec_timeout; i++) {
5059 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
5060 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
5061 break;
5062 udelay(1);
5063 }
5064 if (i >= adev->usec_timeout) {
5065 printk("Failed to wait all pipes clean\n");
5066 return -EINVAL;
5067 }
5068
5069 /********** trigger soft reset ***********/
5070 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
5071 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5072 SOFT_RESET_CP, 1);
5073 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5074 SOFT_RESET_GFX, 1);
5075 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5076 SOFT_RESET_CPF, 1);
5077 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5078 SOFT_RESET_CPC, 1);
5079 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5080 SOFT_RESET_CPG, 1);
5081 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
5082 /********** exit soft reset ***********/
5083 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
5084 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5085 SOFT_RESET_CP, 0);
5086 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5087 SOFT_RESET_GFX, 0);
5088 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5089 SOFT_RESET_CPF, 0);
5090 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5091 SOFT_RESET_CPC, 0);
5092 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5093 SOFT_RESET_CPG, 0);
5094 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
5095
5096 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
5097 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
5098 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
5099
5100 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
5101 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
5102
5103 for (i = 0; i < adev->usec_timeout; i++) {
5104 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
5105 break;
5106 udelay(1);
5107 }
5108 if (i >= adev->usec_timeout) {
5109 printk("Failed to wait CP_VMID_RESET to 0\n");
5110 return -EINVAL;
5111 }
5112
5113 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5114 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5115 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5116 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5117 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5118 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
5119
5120 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5121
5122 return gfx_v11_0_cp_resume(adev);
5123 }
5124
gfx_v11_0_check_soft_reset(struct amdgpu_ip_block * ip_block)5125 static bool gfx_v11_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
5126 {
5127 int i, r;
5128 struct amdgpu_device *adev = ip_block->adev;
5129 struct amdgpu_ring *ring;
5130 long tmo = msecs_to_jiffies(1000);
5131
5132 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
5133 ring = &adev->gfx.gfx_ring[i];
5134 r = amdgpu_ring_test_ib(ring, tmo);
5135 if (r)
5136 return true;
5137 }
5138
5139 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5140 ring = &adev->gfx.compute_ring[i];
5141 r = amdgpu_ring_test_ib(ring, tmo);
5142 if (r)
5143 return true;
5144 }
5145
5146 return false;
5147 }
5148
gfx_v11_0_post_soft_reset(struct amdgpu_ip_block * ip_block)5149 static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
5150 {
5151 struct amdgpu_device *adev = ip_block->adev;
5152 /**
5153 * GFX soft reset will impact MES, need resume MES when do GFX soft reset
5154 */
5155 return amdgpu_mes_resume(adev);
5156 }
5157
gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device * adev)5158 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
5159 {
5160 uint64_t clock;
5161 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
5162
5163 if (amdgpu_sriov_vf(adev)) {
5164 amdgpu_gfx_off_ctrl(adev, false);
5165 mutex_lock(&adev->gfx.gpu_clock_mutex);
5166 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
5167 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
5168 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
5169 if (clock_counter_hi_pre != clock_counter_hi_after)
5170 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
5171 mutex_unlock(&adev->gfx.gpu_clock_mutex);
5172 amdgpu_gfx_off_ctrl(adev, true);
5173 } else {
5174 preempt_disable();
5175 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
5176 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
5177 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
5178 if (clock_counter_hi_pre != clock_counter_hi_after)
5179 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
5180 preempt_enable();
5181 }
5182 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
5183
5184 return clock;
5185 }
5186
gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)5187 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5188 uint32_t vmid,
5189 uint32_t gds_base, uint32_t gds_size,
5190 uint32_t gws_base, uint32_t gws_size,
5191 uint32_t oa_base, uint32_t oa_size)
5192 {
5193 struct amdgpu_device *adev = ring->adev;
5194
5195 /* GDS Base */
5196 gfx_v11_0_write_data_to_reg(ring, 0, false,
5197 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
5198 gds_base);
5199
5200 /* GDS Size */
5201 gfx_v11_0_write_data_to_reg(ring, 0, false,
5202 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
5203 gds_size);
5204
5205 /* GWS */
5206 gfx_v11_0_write_data_to_reg(ring, 0, false,
5207 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
5208 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5209
5210 /* OA */
5211 gfx_v11_0_write_data_to_reg(ring, 0, false,
5212 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
5213 (1 << (oa_size + oa_base)) - (1 << oa_base));
5214 }
5215
gfx_v11_0_early_init(struct amdgpu_ip_block * ip_block)5216 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
5217 {
5218 struct amdgpu_device *adev = ip_block->adev;
5219
5220 switch (amdgpu_user_queue) {
5221 case -1:
5222 case 0:
5223 default:
5224 adev->gfx.disable_kq = false;
5225 adev->gfx.disable_uq = true;
5226 break;
5227 case 1:
5228 adev->gfx.disable_kq = false;
5229 adev->gfx.disable_uq = false;
5230 break;
5231 case 2:
5232 adev->gfx.disable_kq = true;
5233 adev->gfx.disable_uq = false;
5234 break;
5235 }
5236
5237 adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
5238
5239 if (adev->gfx.disable_kq) {
5240 /* We need one GFX ring temporarily to set up
5241 * the clear state.
5242 */
5243 adev->gfx.num_gfx_rings = 1;
5244 adev->gfx.num_compute_rings = 0;
5245 } else {
5246 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
5247 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5248 AMDGPU_MAX_COMPUTE_RINGS);
5249 }
5250
5251 gfx_v11_0_set_kiq_pm4_funcs(adev);
5252 gfx_v11_0_set_ring_funcs(adev);
5253 gfx_v11_0_set_irq_funcs(adev);
5254 gfx_v11_0_set_gds_init(adev);
5255 gfx_v11_0_set_rlc_funcs(adev);
5256 gfx_v11_0_set_mqd_funcs(adev);
5257 gfx_v11_0_set_imu_funcs(adev);
5258
5259 gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
5260
5261 return gfx_v11_0_init_microcode(adev);
5262 }
5263
gfx_v11_0_late_init(struct amdgpu_ip_block * ip_block)5264 static int gfx_v11_0_late_init(struct amdgpu_ip_block *ip_block)
5265 {
5266 struct amdgpu_device *adev = ip_block->adev;
5267 int r;
5268
5269 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5270 if (r)
5271 return r;
5272
5273 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5274 if (r)
5275 return r;
5276
5277 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
5278 if (r)
5279 return r;
5280
5281 r = gfx_v11_0_set_userq_eop_interrupts(adev, true);
5282 if (r)
5283 return r;
5284
5285 return 0;
5286 }
5287
gfx_v11_0_is_rlc_enabled(struct amdgpu_device * adev)5288 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
5289 {
5290 uint32_t rlc_cntl;
5291
5292 /* if RLC is not enabled, do nothing */
5293 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
5294 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
5295 }
5296
gfx_v11_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)5297 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
5298 {
5299 uint32_t data;
5300 unsigned i;
5301
5302 data = RLC_SAFE_MODE__CMD_MASK;
5303 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5304
5305 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
5306
5307 /* wait for RLC_SAFE_MODE */
5308 for (i = 0; i < adev->usec_timeout; i++) {
5309 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
5310 RLC_SAFE_MODE, CMD))
5311 break;
5312 udelay(1);
5313 }
5314 }
5315
gfx_v11_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)5316 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
5317 {
5318 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
5319 }
5320
gfx_v11_0_update_perf_clk(struct amdgpu_device * adev,bool enable)5321 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
5322 bool enable)
5323 {
5324 uint32_t def, data;
5325
5326 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
5327 return;
5328
5329 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5330
5331 if (enable)
5332 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5333 else
5334 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5335
5336 if (def != data)
5337 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5338 }
5339
gfx_v11_0_update_sram_fgcg(struct amdgpu_device * adev,bool enable)5340 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
5341 bool enable)
5342 {
5343 uint32_t def, data;
5344
5345 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
5346 return;
5347
5348 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5349
5350 if (enable)
5351 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5352 else
5353 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5354
5355 if (def != data)
5356 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5357 }
5358
gfx_v11_0_update_repeater_fgcg(struct amdgpu_device * adev,bool enable)5359 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
5360 bool enable)
5361 {
5362 uint32_t def, data;
5363
5364 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
5365 return;
5366
5367 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5368
5369 if (enable)
5370 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5371 else
5372 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5373
5374 if (def != data)
5375 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5376 }
5377
gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)5378 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5379 bool enable)
5380 {
5381 uint32_t data, def;
5382
5383 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
5384 return;
5385
5386 /* It is disabled by HW by default */
5387 if (enable) {
5388 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5389 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
5390 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5391
5392 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5393 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5394 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5395
5396 if (def != data)
5397 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5398 }
5399 } else {
5400 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5401 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5402
5403 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5404 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5405 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5406
5407 if (def != data)
5408 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5409 }
5410 }
5411 }
5412
gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5413 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5414 bool enable)
5415 {
5416 uint32_t def, data;
5417
5418 if (!(adev->cg_flags &
5419 (AMD_CG_SUPPORT_GFX_CGCG |
5420 AMD_CG_SUPPORT_GFX_CGLS |
5421 AMD_CG_SUPPORT_GFX_3D_CGCG |
5422 AMD_CG_SUPPORT_GFX_3D_CGLS)))
5423 return;
5424
5425 if (enable) {
5426 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5427
5428 /* unset CGCG override */
5429 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5430 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5431 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5432 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5433 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
5434 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5435 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5436
5437 /* update CGCG override bits */
5438 if (def != data)
5439 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5440
5441 /* enable cgcg FSM(0x0000363F) */
5442 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5443
5444 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5445 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
5446 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5447 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5448 }
5449
5450 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5451 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
5452 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5453 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5454 }
5455
5456 if (def != data)
5457 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5458
5459 /* Program RLC_CGCG_CGLS_CTRL_3D */
5460 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5461
5462 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5463 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
5464 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5465 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5466 }
5467
5468 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5469 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
5470 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5471 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5472 }
5473
5474 if (def != data)
5475 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5476
5477 /* set IDLE_POLL_COUNT(0x00900100) */
5478 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
5479
5480 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
5481 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5482 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5483
5484 if (def != data)
5485 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
5486
5487 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5488 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5489 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5490 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5491 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5492 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
5493
5494 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5495 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5496 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5497
5498 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5499 if (adev->sdma.num_instances > 1) {
5500 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5501 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5502 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5503 }
5504 } else {
5505 /* Program RLC_CGCG_CGLS_CTRL */
5506 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5507
5508 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5509 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5510
5511 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5512 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5513
5514 if (def != data)
5515 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5516
5517 /* Program RLC_CGCG_CGLS_CTRL_3D */
5518 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5519
5520 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5521 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5522 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5523 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5524
5525 if (def != data)
5526 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5527
5528 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5529 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5530 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5531
5532 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5533 if (adev->sdma.num_instances > 1) {
5534 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5535 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5536 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5537 }
5538 }
5539 }
5540
gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5541 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5542 bool enable)
5543 {
5544 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5545
5546 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5547
5548 gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5549
5550 gfx_v11_0_update_repeater_fgcg(adev, enable);
5551
5552 gfx_v11_0_update_sram_fgcg(adev, enable);
5553
5554 gfx_v11_0_update_perf_clk(adev, enable);
5555
5556 if (adev->cg_flags &
5557 (AMD_CG_SUPPORT_GFX_MGCG |
5558 AMD_CG_SUPPORT_GFX_CGLS |
5559 AMD_CG_SUPPORT_GFX_CGCG |
5560 AMD_CG_SUPPORT_GFX_3D_CGCG |
5561 AMD_CG_SUPPORT_GFX_3D_CGLS))
5562 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5563
5564 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5565
5566 return 0;
5567 }
5568
gfx_v11_0_update_spm_vmid(struct amdgpu_device * adev,struct amdgpu_ring * ring,unsigned vmid)5569 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
5570 {
5571 u32 reg, pre_data, data;
5572
5573 amdgpu_gfx_off_ctrl(adev, false);
5574 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5575 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
5576 pre_data = RREG32_NO_KIQ(reg);
5577 else
5578 pre_data = RREG32(reg);
5579
5580 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
5581 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5582
5583 if (pre_data != data) {
5584 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
5585 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5586 } else
5587 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5588 }
5589 amdgpu_gfx_off_ctrl(adev, true);
5590
5591 if (ring
5592 && amdgpu_sriov_is_pp_one_vf(adev)
5593 && (pre_data != data)
5594 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
5595 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
5596 amdgpu_ring_emit_wreg(ring, reg, data);
5597 }
5598 }
5599
5600 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5601 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5602 .set_safe_mode = gfx_v11_0_set_safe_mode,
5603 .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5604 .init = gfx_v11_0_rlc_init,
5605 .get_csb_size = gfx_v11_0_get_csb_size,
5606 .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5607 .resume = gfx_v11_0_rlc_resume,
5608 .stop = gfx_v11_0_rlc_stop,
5609 .reset = gfx_v11_0_rlc_reset,
5610 .start = gfx_v11_0_rlc_start,
5611 .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5612 };
5613
gfx_v11_cntl_power_gating(struct amdgpu_device * adev,bool enable)5614 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5615 {
5616 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5617
5618 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5619 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5620 else
5621 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5622
5623 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5624
5625 // Program RLC_PG_DELAY3 for CGPG hysteresis
5626 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5627 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5628 case IP_VERSION(11, 0, 1):
5629 case IP_VERSION(11, 0, 4):
5630 case IP_VERSION(11, 5, 0):
5631 case IP_VERSION(11, 5, 1):
5632 case IP_VERSION(11, 5, 2):
5633 case IP_VERSION(11, 5, 3):
5634 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5635 break;
5636 default:
5637 break;
5638 }
5639 }
5640 }
5641
gfx_v11_cntl_pg(struct amdgpu_device * adev,bool enable)5642 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5643 {
5644 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5645
5646 gfx_v11_cntl_power_gating(adev, enable);
5647
5648 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5649 }
5650
gfx_v11_0_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)5651 static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5652 enum amd_powergating_state state)
5653 {
5654 struct amdgpu_device *adev = ip_block->adev;
5655 bool enable = (state == AMD_PG_STATE_GATE);
5656
5657 if (amdgpu_sriov_vf(adev))
5658 return 0;
5659
5660 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5661 case IP_VERSION(11, 0, 0):
5662 case IP_VERSION(11, 0, 2):
5663 case IP_VERSION(11, 0, 3):
5664 amdgpu_gfx_off_ctrl(adev, enable);
5665 break;
5666 case IP_VERSION(11, 0, 1):
5667 case IP_VERSION(11, 0, 4):
5668 case IP_VERSION(11, 5, 0):
5669 case IP_VERSION(11, 5, 1):
5670 case IP_VERSION(11, 5, 2):
5671 case IP_VERSION(11, 5, 3):
5672 if (!enable)
5673 amdgpu_gfx_off_ctrl(adev, false);
5674
5675 gfx_v11_cntl_pg(adev, enable);
5676
5677 if (enable)
5678 amdgpu_gfx_off_ctrl(adev, true);
5679
5680 break;
5681 default:
5682 break;
5683 }
5684
5685 return 0;
5686 }
5687
gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)5688 static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5689 enum amd_clockgating_state state)
5690 {
5691 struct amdgpu_device *adev = ip_block->adev;
5692
5693 if (amdgpu_sriov_vf(adev))
5694 return 0;
5695
5696 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5697 case IP_VERSION(11, 0, 0):
5698 case IP_VERSION(11, 0, 1):
5699 case IP_VERSION(11, 0, 2):
5700 case IP_VERSION(11, 0, 3):
5701 case IP_VERSION(11, 0, 4):
5702 case IP_VERSION(11, 5, 0):
5703 case IP_VERSION(11, 5, 1):
5704 case IP_VERSION(11, 5, 2):
5705 case IP_VERSION(11, 5, 3):
5706 gfx_v11_0_update_gfx_clock_gating(adev,
5707 state == AMD_CG_STATE_GATE);
5708 break;
5709 default:
5710 break;
5711 }
5712
5713 return 0;
5714 }
5715
gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)5716 static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
5717 {
5718 struct amdgpu_device *adev = ip_block->adev;
5719 int data;
5720
5721 /* AMD_CG_SUPPORT_GFX_MGCG */
5722 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5723 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5724 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5725
5726 /* AMD_CG_SUPPORT_REPEATER_FGCG */
5727 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5728 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5729
5730 /* AMD_CG_SUPPORT_GFX_FGCG */
5731 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5732 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5733
5734 /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5735 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5736 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5737
5738 /* AMD_CG_SUPPORT_GFX_CGCG */
5739 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5740 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5741 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5742
5743 /* AMD_CG_SUPPORT_GFX_CGLS */
5744 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5745 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5746
5747 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5748 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5749 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5750 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5751
5752 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5753 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5754 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5755 }
5756
gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5757 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5758 {
5759 /* gfx11 is 32bit rptr*/
5760 return *(uint32_t *)ring->rptr_cpu_addr;
5761 }
5762
gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5763 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5764 {
5765 struct amdgpu_device *adev = ring->adev;
5766 u64 wptr;
5767
5768 /* XXX check if swapping is necessary on BE */
5769 if (ring->use_doorbell) {
5770 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5771 } else {
5772 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5773 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5774 }
5775
5776 return wptr;
5777 }
5778
gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5779 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5780 {
5781 struct amdgpu_device *adev = ring->adev;
5782
5783 if (ring->use_doorbell) {
5784 /* XXX check if swapping is necessary on BE */
5785 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5786 ring->wptr);
5787 WDOORBELL64(ring->doorbell_index, ring->wptr);
5788 } else {
5789 WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5790 lower_32_bits(ring->wptr));
5791 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5792 upper_32_bits(ring->wptr));
5793 }
5794 }
5795
gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5796 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5797 {
5798 /* gfx11 hardware is 32bit rptr */
5799 return *(uint32_t *)ring->rptr_cpu_addr;
5800 }
5801
gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5802 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5803 {
5804 u64 wptr;
5805
5806 /* XXX check if swapping is necessary on BE */
5807 if (ring->use_doorbell)
5808 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5809 else
5810 BUG();
5811 return wptr;
5812 }
5813
gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5814 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5815 {
5816 struct amdgpu_device *adev = ring->adev;
5817
5818 /* XXX check if swapping is necessary on BE */
5819 if (ring->use_doorbell) {
5820 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5821 ring->wptr);
5822 WDOORBELL64(ring->doorbell_index, ring->wptr);
5823 } else {
5824 BUG(); /* only DOORBELL method supported on gfx11 now */
5825 }
5826 }
5827
gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5828 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5829 {
5830 struct amdgpu_device *adev = ring->adev;
5831 u32 ref_and_mask, reg_mem_engine;
5832 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5833
5834 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5835 switch (ring->me) {
5836 case 1:
5837 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5838 break;
5839 case 2:
5840 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5841 break;
5842 default:
5843 return;
5844 }
5845 reg_mem_engine = 0;
5846 } else {
5847 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
5848 reg_mem_engine = 1; /* pfp */
5849 }
5850
5851 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5852 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5853 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5854 ref_and_mask, ref_and_mask, 0x20);
5855 }
5856
gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5857 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5858 struct amdgpu_job *job,
5859 struct amdgpu_ib *ib,
5860 uint32_t flags)
5861 {
5862 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5863 u32 header, control = 0;
5864
5865 BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5866
5867 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5868
5869 control |= ib->length_dw | (vmid << 24);
5870
5871 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5872 control |= INDIRECT_BUFFER_PRE_ENB(1);
5873
5874 if (flags & AMDGPU_IB_PREEMPTED)
5875 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5876
5877 if (vmid)
5878 gfx_v11_0_ring_emit_de_meta(ring,
5879 (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5880 }
5881
5882 amdgpu_ring_write(ring, header);
5883 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5884 amdgpu_ring_write(ring,
5885 #ifdef __BIG_ENDIAN
5886 (2 << 0) |
5887 #endif
5888 lower_32_bits(ib->gpu_addr));
5889 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5890 amdgpu_ring_write(ring, control);
5891 }
5892
gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5893 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5894 struct amdgpu_job *job,
5895 struct amdgpu_ib *ib,
5896 uint32_t flags)
5897 {
5898 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5899 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5900
5901 /* Currently, there is a high possibility to get wave ID mismatch
5902 * between ME and GDS, leading to a hw deadlock, because ME generates
5903 * different wave IDs than the GDS expects. This situation happens
5904 * randomly when at least 5 compute pipes use GDS ordered append.
5905 * The wave IDs generated by ME are also wrong after suspend/resume.
5906 * Those are probably bugs somewhere else in the kernel driver.
5907 *
5908 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5909 * GDS to 0 for this ring (me/pipe).
5910 */
5911 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5912 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5913 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5914 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5915 }
5916
5917 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5918 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5919 amdgpu_ring_write(ring,
5920 #ifdef __BIG_ENDIAN
5921 (2 << 0) |
5922 #endif
5923 lower_32_bits(ib->gpu_addr));
5924 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5925 amdgpu_ring_write(ring, control);
5926 }
5927
gfx_v11_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)5928 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5929 u64 seq, unsigned flags)
5930 {
5931 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5932 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5933
5934 /* RELEASE_MEM - flush caches, send int */
5935 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5936 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5937 PACKET3_RELEASE_MEM_GCR_GL2_WB |
5938 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */
5939 PACKET3_RELEASE_MEM_GCR_GLM_WB |
5940 PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5941 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5942 PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5943 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5944 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5945
5946 /*
5947 * the address should be Qword aligned if 64bit write, Dword
5948 * aligned if only send 32bit data low (discard data high)
5949 */
5950 if (write64bit)
5951 BUG_ON(addr & 0x7);
5952 else
5953 BUG_ON(addr & 0x3);
5954 amdgpu_ring_write(ring, lower_32_bits(addr));
5955 amdgpu_ring_write(ring, upper_32_bits(addr));
5956 amdgpu_ring_write(ring, lower_32_bits(seq));
5957 amdgpu_ring_write(ring, upper_32_bits(seq));
5958 amdgpu_ring_write(ring, 0);
5959 }
5960
gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)5961 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5962 {
5963 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5964 uint32_t seq = ring->fence_drv.sync_seq;
5965 uint64_t addr = ring->fence_drv.gpu_addr;
5966
5967 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5968 upper_32_bits(addr), seq, 0xffffffff, 4);
5969 }
5970
gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring * ring,uint16_t pasid,uint32_t flush_type,bool all_hub,uint8_t dst_sel)5971 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5972 uint16_t pasid, uint32_t flush_type,
5973 bool all_hub, uint8_t dst_sel)
5974 {
5975 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5976 amdgpu_ring_write(ring,
5977 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5978 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5979 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5980 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5981 }
5982
gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)5983 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5984 unsigned vmid, uint64_t pd_addr)
5985 {
5986 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5987
5988 /* compute doesn't have PFP */
5989 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5990 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5991 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5992 amdgpu_ring_write(ring, 0x0);
5993 }
5994
5995 /* Make sure that we can't skip the SET_Q_MODE packets when the VM
5996 * changed in any way.
5997 */
5998 ring->set_q_mode_offs = 0;
5999 ring->set_q_mode_ptr = NULL;
6000 }
6001
gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)6002 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6003 u64 seq, unsigned int flags)
6004 {
6005 struct amdgpu_device *adev = ring->adev;
6006
6007 /* we only allocate 32bit for each seq wb address */
6008 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
6009
6010 /* write fence seq to the "addr" */
6011 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6012 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6013 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6014 amdgpu_ring_write(ring, lower_32_bits(addr));
6015 amdgpu_ring_write(ring, upper_32_bits(addr));
6016 amdgpu_ring_write(ring, lower_32_bits(seq));
6017
6018 if (flags & AMDGPU_FENCE_FLAG_INT) {
6019 /* set register to trigger INT */
6020 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6021 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6022 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6023 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
6024 amdgpu_ring_write(ring, 0);
6025 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6026 }
6027 }
6028
gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)6029 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
6030 uint32_t flags)
6031 {
6032 uint32_t dw2 = 0;
6033
6034 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6035 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
6036 /* set load_global_config & load_global_uconfig */
6037 dw2 |= 0x8001;
6038 /* set load_cs_sh_regs */
6039 dw2 |= 0x01000000;
6040 /* set load_per_context_state & load_gfx_sh_regs for GFX */
6041 dw2 |= 0x10002;
6042 }
6043
6044 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6045 amdgpu_ring_write(ring, dw2);
6046 amdgpu_ring_write(ring, 0);
6047 }
6048
gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)6049 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
6050 uint64_t addr)
6051 {
6052 unsigned ret;
6053
6054 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
6055 amdgpu_ring_write(ring, lower_32_bits(addr));
6056 amdgpu_ring_write(ring, upper_32_bits(addr));
6057 /* discard following DWs if *cond_exec_gpu_addr==0 */
6058 amdgpu_ring_write(ring, 0);
6059 ret = ring->wptr & ring->buf_mask;
6060 /* patch dummy value later */
6061 amdgpu_ring_write(ring, 0);
6062
6063 return ret;
6064 }
6065
gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring * ring,u64 shadow_va,u64 csa_va,u64 gds_va,bool init_shadow,int vmid)6066 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
6067 u64 shadow_va, u64 csa_va,
6068 u64 gds_va, bool init_shadow,
6069 int vmid)
6070 {
6071 struct amdgpu_device *adev = ring->adev;
6072 unsigned int offs, end;
6073
6074 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj)
6075 return;
6076
6077 /*
6078 * The logic here isn't easy to understand because we need to keep state
6079 * accross multiple executions of the function as well as between the
6080 * CPU and GPU. The general idea is that the newly written GPU command
6081 * has a condition on the previous one and only executed if really
6082 * necessary.
6083 */
6084
6085 /*
6086 * The dw in the NOP controls if the next SET_Q_MODE packet should be
6087 * executed or not. Reserve 64bits just to be on the save side.
6088 */
6089 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1));
6090 offs = ring->wptr & ring->buf_mask;
6091
6092 /*
6093 * We start with skipping the prefix SET_Q_MODE and always executing
6094 * the postfix SET_Q_MODE packet. This is changed below with a
6095 * WRITE_DATA command when the postfix executed.
6096 */
6097 amdgpu_ring_write(ring, shadow_va ? 1 : 0);
6098 amdgpu_ring_write(ring, 0);
6099
6100 if (ring->set_q_mode_offs) {
6101 uint64_t addr;
6102
6103 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
6104 addr += ring->set_q_mode_offs << 2;
6105 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr);
6106 }
6107
6108 /*
6109 * When the postfix SET_Q_MODE packet executes we need to make sure that the
6110 * next prefix SET_Q_MODE packet executes as well.
6111 */
6112 if (!shadow_va) {
6113 uint64_t addr;
6114
6115 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
6116 addr += offs << 2;
6117 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6118 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
6119 amdgpu_ring_write(ring, lower_32_bits(addr));
6120 amdgpu_ring_write(ring, upper_32_bits(addr));
6121 amdgpu_ring_write(ring, 0x1);
6122 }
6123
6124 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7));
6125 amdgpu_ring_write(ring, lower_32_bits(shadow_va));
6126 amdgpu_ring_write(ring, upper_32_bits(shadow_va));
6127 amdgpu_ring_write(ring, lower_32_bits(gds_va));
6128 amdgpu_ring_write(ring, upper_32_bits(gds_va));
6129 amdgpu_ring_write(ring, lower_32_bits(csa_va));
6130 amdgpu_ring_write(ring, upper_32_bits(csa_va));
6131 amdgpu_ring_write(ring, shadow_va ?
6132 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0);
6133 amdgpu_ring_write(ring, init_shadow ?
6134 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0);
6135
6136 if (ring->set_q_mode_offs)
6137 amdgpu_ring_patch_cond_exec(ring, end);
6138
6139 if (shadow_va) {
6140 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid;
6141
6142 /*
6143 * If the tokens match try to skip the last postfix SET_Q_MODE
6144 * packet to avoid saving/restoring the state all the time.
6145 */
6146 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token)
6147 *ring->set_q_mode_ptr = 0;
6148
6149 ring->set_q_mode_token = token;
6150 } else {
6151 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs];
6152 }
6153
6154 ring->set_q_mode_offs = offs;
6155 }
6156
gfx_v11_0_ring_preempt_ib(struct amdgpu_ring * ring)6157 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
6158 {
6159 int i, r = 0;
6160 struct amdgpu_device *adev = ring->adev;
6161 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
6162 struct amdgpu_ring *kiq_ring = &kiq->ring;
6163 unsigned long flags;
6164
6165 if (adev->enable_mes)
6166 return -EINVAL;
6167
6168 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
6169 return -EINVAL;
6170
6171 spin_lock_irqsave(&kiq->ring_lock, flags);
6172
6173 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
6174 spin_unlock_irqrestore(&kiq->ring_lock, flags);
6175 return -ENOMEM;
6176 }
6177
6178 /* assert preemption condition */
6179 amdgpu_ring_set_preempt_cond_exec(ring, false);
6180
6181 /* assert IB preemption, emit the trailing fence */
6182 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
6183 ring->trail_fence_gpu_addr,
6184 ++ring->trail_seq);
6185 amdgpu_ring_commit(kiq_ring);
6186
6187 spin_unlock_irqrestore(&kiq->ring_lock, flags);
6188
6189 /* poll the trailing fence */
6190 for (i = 0; i < adev->usec_timeout; i++) {
6191 if (ring->trail_seq ==
6192 le32_to_cpu(*(ring->trail_fence_cpu_addr)))
6193 break;
6194 udelay(1);
6195 }
6196
6197 if (i >= adev->usec_timeout) {
6198 r = -EINVAL;
6199 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
6200 }
6201
6202 /* deassert preemption condition */
6203 amdgpu_ring_set_preempt_cond_exec(ring, true);
6204 return r;
6205 }
6206
gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring * ring,bool resume)6207 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
6208 {
6209 struct amdgpu_device *adev = ring->adev;
6210 struct v10_de_ib_state de_payload = {0};
6211 uint64_t offset, gds_addr, de_payload_gpu_addr;
6212 void *de_payload_cpu_addr;
6213 int cnt;
6214
6215 offset = offsetof(struct v10_gfx_meta_data, de_payload);
6216 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
6217 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
6218
6219 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
6220 AMDGPU_CSA_SIZE - adev->gds.gds_size,
6221 PAGE_SIZE);
6222
6223 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
6224 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
6225
6226 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
6227 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
6228 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
6229 WRITE_DATA_DST_SEL(8) |
6230 WR_CONFIRM) |
6231 WRITE_DATA_CACHE_POLICY(0));
6232 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
6233 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
6234
6235 if (resume)
6236 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
6237 sizeof(de_payload) >> 2);
6238 else
6239 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
6240 sizeof(de_payload) >> 2);
6241 }
6242
gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)6243 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
6244 bool secure)
6245 {
6246 uint32_t v = secure ? FRAME_TMZ : 0;
6247
6248 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
6249 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
6250 }
6251
gfx_v11_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)6252 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6253 uint32_t reg_val_offs)
6254 {
6255 struct amdgpu_device *adev = ring->adev;
6256
6257 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6258 amdgpu_ring_write(ring, 0 | /* src: register*/
6259 (5 << 8) | /* dst: memory */
6260 (1 << 20)); /* write confirm */
6261 amdgpu_ring_write(ring, reg);
6262 amdgpu_ring_write(ring, 0);
6263 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6264 reg_val_offs * 4));
6265 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6266 reg_val_offs * 4));
6267 }
6268
gfx_v11_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)6269 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6270 uint32_t val)
6271 {
6272 uint32_t cmd = 0;
6273
6274 switch (ring->funcs->type) {
6275 case AMDGPU_RING_TYPE_GFX:
6276 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6277 break;
6278 case AMDGPU_RING_TYPE_KIQ:
6279 cmd = (1 << 16); /* no inc addr */
6280 break;
6281 default:
6282 cmd = WR_CONFIRM;
6283 break;
6284 }
6285 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6286 amdgpu_ring_write(ring, cmd);
6287 amdgpu_ring_write(ring, reg);
6288 amdgpu_ring_write(ring, 0);
6289 amdgpu_ring_write(ring, val);
6290 }
6291
gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)6292 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
6293 uint32_t val, uint32_t mask)
6294 {
6295 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
6296 }
6297
gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)6298 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
6299 uint32_t reg0, uint32_t reg1,
6300 uint32_t ref, uint32_t mask)
6301 {
6302 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6303
6304 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
6305 ref, mask, 0x20);
6306 }
6307
6308 static void
gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,uint32_t me,uint32_t pipe,enum amdgpu_interrupt_state state)6309 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6310 uint32_t me, uint32_t pipe,
6311 enum amdgpu_interrupt_state state)
6312 {
6313 uint32_t cp_int_cntl, cp_int_cntl_reg;
6314
6315 if (!me) {
6316 switch (pipe) {
6317 case 0:
6318 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
6319 break;
6320 case 1:
6321 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
6322 break;
6323 default:
6324 DRM_DEBUG("invalid pipe %d\n", pipe);
6325 return;
6326 }
6327 } else {
6328 DRM_DEBUG("invalid me %d\n", me);
6329 return;
6330 }
6331
6332 switch (state) {
6333 case AMDGPU_IRQ_STATE_DISABLE:
6334 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6335 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6336 TIME_STAMP_INT_ENABLE, 0);
6337 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6338 GENERIC0_INT_ENABLE, 0);
6339 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6340 break;
6341 case AMDGPU_IRQ_STATE_ENABLE:
6342 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6343 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6344 TIME_STAMP_INT_ENABLE, 1);
6345 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6346 GENERIC0_INT_ENABLE, 1);
6347 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6348 break;
6349 default:
6350 break;
6351 }
6352 }
6353
gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)6354 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6355 int me, int pipe,
6356 enum amdgpu_interrupt_state state)
6357 {
6358 u32 mec_int_cntl, mec_int_cntl_reg;
6359
6360 /*
6361 * amdgpu controls only the first MEC. That's why this function only
6362 * handles the setting of interrupts for this specific MEC. All other
6363 * pipes' interrupts are set by amdkfd.
6364 */
6365
6366 if (me == 1) {
6367 switch (pipe) {
6368 case 0:
6369 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6370 break;
6371 case 1:
6372 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
6373 break;
6374 case 2:
6375 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
6376 break;
6377 case 3:
6378 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
6379 break;
6380 default:
6381 DRM_DEBUG("invalid pipe %d\n", pipe);
6382 return;
6383 }
6384 } else {
6385 DRM_DEBUG("invalid me %d\n", me);
6386 return;
6387 }
6388
6389 switch (state) {
6390 case AMDGPU_IRQ_STATE_DISABLE:
6391 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6392 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6393 TIME_STAMP_INT_ENABLE, 0);
6394 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6395 GENERIC0_INT_ENABLE, 0);
6396 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6397 break;
6398 case AMDGPU_IRQ_STATE_ENABLE:
6399 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6400 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6401 TIME_STAMP_INT_ENABLE, 1);
6402 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6403 GENERIC0_INT_ENABLE, 1);
6404 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6405 break;
6406 default:
6407 break;
6408 }
6409 }
6410
gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6411 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6412 struct amdgpu_irq_src *src,
6413 unsigned type,
6414 enum amdgpu_interrupt_state state)
6415 {
6416 switch (type) {
6417 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6418 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
6419 break;
6420 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
6421 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
6422 break;
6423 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6424 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6425 break;
6426 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6427 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6428 break;
6429 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6430 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6431 break;
6432 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6433 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6434 break;
6435 default:
6436 break;
6437 }
6438 return 0;
6439 }
6440
gfx_v11_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6441 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
6442 struct amdgpu_irq_src *source,
6443 struct amdgpu_iv_entry *entry)
6444 {
6445 u32 doorbell_offset = entry->src_data[0];
6446 u8 me_id, pipe_id, queue_id;
6447 struct amdgpu_ring *ring;
6448 int i;
6449
6450 DRM_DEBUG("IH: CP EOP\n");
6451
6452 if (adev->enable_mes && doorbell_offset) {
6453 struct amdgpu_userq_fence_driver *fence_drv = NULL;
6454 struct xarray *xa = &adev->userq_xa;
6455 unsigned long flags;
6456
6457 xa_lock_irqsave(xa, flags);
6458 fence_drv = xa_load(xa, doorbell_offset);
6459 if (fence_drv)
6460 amdgpu_userq_fence_driver_process(fence_drv);
6461 xa_unlock_irqrestore(xa, flags);
6462 } else {
6463 me_id = (entry->ring_id & 0x0c) >> 2;
6464 pipe_id = (entry->ring_id & 0x03) >> 0;
6465 queue_id = (entry->ring_id & 0x70) >> 4;
6466
6467 switch (me_id) {
6468 case 0:
6469 if (pipe_id == 0)
6470 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6471 else
6472 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
6473 break;
6474 case 1:
6475 case 2:
6476 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6477 ring = &adev->gfx.compute_ring[i];
6478 /* Per-queue interrupt is supported for MEC starting from VI.
6479 * The interrupt can only be enabled/disabled per pipe instead
6480 * of per queue.
6481 */
6482 if ((ring->me == me_id) &&
6483 (ring->pipe == pipe_id) &&
6484 (ring->queue == queue_id))
6485 amdgpu_fence_process(ring);
6486 }
6487 break;
6488 }
6489 }
6490
6491 return 0;
6492 }
6493
gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6494 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6495 struct amdgpu_irq_src *source,
6496 unsigned int type,
6497 enum amdgpu_interrupt_state state)
6498 {
6499 u32 cp_int_cntl_reg, cp_int_cntl;
6500 int i, j;
6501
6502 switch (state) {
6503 case AMDGPU_IRQ_STATE_DISABLE:
6504 case AMDGPU_IRQ_STATE_ENABLE:
6505 for (i = 0; i < adev->gfx.me.num_me; i++) {
6506 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6507 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6508
6509 if (cp_int_cntl_reg) {
6510 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6511 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6512 PRIV_REG_INT_ENABLE,
6513 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6514 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6515 }
6516 }
6517 }
6518 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6519 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6520 /* MECs start at 1 */
6521 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6522
6523 if (cp_int_cntl_reg) {
6524 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6525 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6526 PRIV_REG_INT_ENABLE,
6527 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6528 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6529 }
6530 }
6531 }
6532 break;
6533 default:
6534 break;
6535 }
6536
6537 return 0;
6538 }
6539
gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6540 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6541 struct amdgpu_irq_src *source,
6542 unsigned type,
6543 enum amdgpu_interrupt_state state)
6544 {
6545 u32 cp_int_cntl_reg, cp_int_cntl;
6546 int i, j;
6547
6548 switch (state) {
6549 case AMDGPU_IRQ_STATE_DISABLE:
6550 case AMDGPU_IRQ_STATE_ENABLE:
6551 for (i = 0; i < adev->gfx.me.num_me; i++) {
6552 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6553 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6554
6555 if (cp_int_cntl_reg) {
6556 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6557 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6558 OPCODE_ERROR_INT_ENABLE,
6559 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6560 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6561 }
6562 }
6563 }
6564 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6565 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6566 /* MECs start at 1 */
6567 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6568
6569 if (cp_int_cntl_reg) {
6570 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6571 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6572 OPCODE_ERROR_INT_ENABLE,
6573 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6574 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6575 }
6576 }
6577 }
6578 break;
6579 default:
6580 break;
6581 }
6582 return 0;
6583 }
6584
gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6585 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6586 struct amdgpu_irq_src *source,
6587 unsigned int type,
6588 enum amdgpu_interrupt_state state)
6589 {
6590 u32 cp_int_cntl_reg, cp_int_cntl;
6591 int i, j;
6592
6593 switch (state) {
6594 case AMDGPU_IRQ_STATE_DISABLE:
6595 case AMDGPU_IRQ_STATE_ENABLE:
6596 for (i = 0; i < adev->gfx.me.num_me; i++) {
6597 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6598 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6599
6600 if (cp_int_cntl_reg) {
6601 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6602 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6603 PRIV_INSTR_INT_ENABLE,
6604 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6605 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6606 }
6607 }
6608 }
6609 break;
6610 default:
6611 break;
6612 }
6613
6614 return 0;
6615 }
6616
gfx_v11_0_handle_priv_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6617 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6618 struct amdgpu_iv_entry *entry)
6619 {
6620 u8 me_id, pipe_id, queue_id;
6621 struct amdgpu_ring *ring;
6622 int i;
6623
6624 me_id = (entry->ring_id & 0x0c) >> 2;
6625 pipe_id = (entry->ring_id & 0x03) >> 0;
6626 queue_id = (entry->ring_id & 0x70) >> 4;
6627
6628 if (!adev->gfx.disable_kq) {
6629 switch (me_id) {
6630 case 0:
6631 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6632 ring = &adev->gfx.gfx_ring[i];
6633 if (ring->me == me_id && ring->pipe == pipe_id &&
6634 ring->queue == queue_id)
6635 drm_sched_fault(&ring->sched);
6636 }
6637 break;
6638 case 1:
6639 case 2:
6640 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6641 ring = &adev->gfx.compute_ring[i];
6642 if (ring->me == me_id && ring->pipe == pipe_id &&
6643 ring->queue == queue_id)
6644 drm_sched_fault(&ring->sched);
6645 }
6646 break;
6647 default:
6648 BUG();
6649 break;
6650 }
6651 }
6652 }
6653
gfx_v11_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6654 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6655 struct amdgpu_irq_src *source,
6656 struct amdgpu_iv_entry *entry)
6657 {
6658 DRM_ERROR("Illegal register access in command stream\n");
6659 gfx_v11_0_handle_priv_fault(adev, entry);
6660 return 0;
6661 }
6662
gfx_v11_0_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6663 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev,
6664 struct amdgpu_irq_src *source,
6665 struct amdgpu_iv_entry *entry)
6666 {
6667 DRM_ERROR("Illegal opcode in command stream \n");
6668 gfx_v11_0_handle_priv_fault(adev, entry);
6669 return 0;
6670 }
6671
gfx_v11_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6672 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6673 struct amdgpu_irq_src *source,
6674 struct amdgpu_iv_entry *entry)
6675 {
6676 DRM_ERROR("Illegal instruction in command stream\n");
6677 gfx_v11_0_handle_priv_fault(adev, entry);
6678 return 0;
6679 }
6680
gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6681 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6682 struct amdgpu_irq_src *source,
6683 struct amdgpu_iv_entry *entry)
6684 {
6685 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6686 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6687
6688 return 0;
6689 }
6690
6691 #if 0
6692 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6693 struct amdgpu_irq_src *src,
6694 unsigned int type,
6695 enum amdgpu_interrupt_state state)
6696 {
6697 uint32_t tmp, target;
6698 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring);
6699
6700 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6701 target += ring->pipe;
6702
6703 switch (type) {
6704 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6705 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6706 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6707 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6708 GENERIC2_INT_ENABLE, 0);
6709 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6710
6711 tmp = RREG32_SOC15_IP(GC, target);
6712 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6713 GENERIC2_INT_ENABLE, 0);
6714 WREG32_SOC15_IP(GC, target, tmp);
6715 } else {
6716 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6717 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6718 GENERIC2_INT_ENABLE, 1);
6719 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6720
6721 tmp = RREG32_SOC15_IP(GC, target);
6722 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6723 GENERIC2_INT_ENABLE, 1);
6724 WREG32_SOC15_IP(GC, target, tmp);
6725 }
6726 break;
6727 default:
6728 BUG(); /* kiq only support GENERIC2_INT now */
6729 break;
6730 }
6731 return 0;
6732 }
6733 #endif
6734
gfx_v11_0_emit_mem_sync(struct amdgpu_ring * ring)6735 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6736 {
6737 const unsigned int gcr_cntl =
6738 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6739 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6740 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6741 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6742 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6743 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6744 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6745 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6746
6747 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6748 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6749 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6750 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
6751 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
6752 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6753 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
6754 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6755 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6756 }
6757
gfx_v11_pipe_reset_support(struct amdgpu_device * adev)6758 static bool gfx_v11_pipe_reset_support(struct amdgpu_device *adev)
6759 {
6760 /* Disable the pipe reset until the CPFW fully support it.*/
6761 dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n");
6762 return false;
6763 }
6764
6765
gfx_v11_reset_gfx_pipe(struct amdgpu_ring * ring)6766 static int gfx_v11_reset_gfx_pipe(struct amdgpu_ring *ring)
6767 {
6768 struct amdgpu_device *adev = ring->adev;
6769 uint32_t reset_pipe = 0, clean_pipe = 0;
6770 int r;
6771
6772 if (!gfx_v11_pipe_reset_support(adev))
6773 return -EOPNOTSUPP;
6774
6775 gfx_v11_0_set_safe_mode(adev, 0);
6776 mutex_lock(&adev->srbm_mutex);
6777 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
6778
6779 switch (ring->pipe) {
6780 case 0:
6781 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
6782 PFP_PIPE0_RESET, 1);
6783 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
6784 ME_PIPE0_RESET, 1);
6785 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
6786 PFP_PIPE0_RESET, 0);
6787 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
6788 ME_PIPE0_RESET, 0);
6789 break;
6790 case 1:
6791 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
6792 PFP_PIPE1_RESET, 1);
6793 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
6794 ME_PIPE1_RESET, 1);
6795 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
6796 PFP_PIPE1_RESET, 0);
6797 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
6798 ME_PIPE1_RESET, 0);
6799 break;
6800 default:
6801 break;
6802 }
6803
6804 WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe);
6805 WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe);
6806
6807 r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) -
6808 RS64_FW_UC_START_ADDR_LO;
6809 soc21_grbm_select(adev, 0, 0, 0, 0);
6810 mutex_unlock(&adev->srbm_mutex);
6811 gfx_v11_0_unset_safe_mode(adev, 0);
6812
6813 dev_info(adev->dev, "The ring %s pipe reset to the ME firmware start PC: %s\n", ring->name,
6814 r == 0 ? "successfully" : "failed");
6815 /* FIXME: Sometimes driver can't cache the ME firmware start PC correctly,
6816 * so the pipe reset status relies on the later gfx ring test result.
6817 */
6818 return 0;
6819 }
6820
gfx_v11_0_reset_kgq(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)6821 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring,
6822 unsigned int vmid,
6823 struct amdgpu_fence *timedout_fence)
6824 {
6825 struct amdgpu_device *adev = ring->adev;
6826 int r;
6827
6828 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
6829
6830 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false);
6831 if (r) {
6832
6833 dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r);
6834 r = gfx_v11_reset_gfx_pipe(ring);
6835 if (r)
6836 return r;
6837 }
6838
6839 r = gfx_v11_0_kgq_init_queue(ring, true);
6840 if (r) {
6841 dev_err(adev->dev, "failed to init kgq\n");
6842 return r;
6843 }
6844
6845 r = amdgpu_mes_map_legacy_queue(adev, ring);
6846 if (r) {
6847 dev_err(adev->dev, "failed to remap kgq\n");
6848 return r;
6849 }
6850
6851 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
6852 }
6853
gfx_v11_0_reset_compute_pipe(struct amdgpu_ring * ring)6854 static int gfx_v11_0_reset_compute_pipe(struct amdgpu_ring *ring)
6855 {
6856
6857 struct amdgpu_device *adev = ring->adev;
6858 uint32_t reset_pipe = 0, clean_pipe = 0;
6859 int r;
6860
6861 if (!gfx_v11_pipe_reset_support(adev))
6862 return -EOPNOTSUPP;
6863
6864 gfx_v11_0_set_safe_mode(adev, 0);
6865 mutex_lock(&adev->srbm_mutex);
6866 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
6867
6868 reset_pipe = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
6869 clean_pipe = reset_pipe;
6870
6871 if (adev->gfx.rs64_enable) {
6872
6873 switch (ring->pipe) {
6874 case 0:
6875 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
6876 MEC_PIPE0_RESET, 1);
6877 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
6878 MEC_PIPE0_RESET, 0);
6879 break;
6880 case 1:
6881 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
6882 MEC_PIPE1_RESET, 1);
6883 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
6884 MEC_PIPE1_RESET, 0);
6885 break;
6886 case 2:
6887 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
6888 MEC_PIPE2_RESET, 1);
6889 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
6890 MEC_PIPE2_RESET, 0);
6891 break;
6892 case 3:
6893 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
6894 MEC_PIPE3_RESET, 1);
6895 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
6896 MEC_PIPE3_RESET, 0);
6897 break;
6898 default:
6899 break;
6900 }
6901 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_pipe);
6902 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_pipe);
6903 r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) -
6904 RS64_FW_UC_START_ADDR_LO;
6905 } else {
6906 if (ring->me == 1) {
6907 switch (ring->pipe) {
6908 case 0:
6909 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6910 MEC_ME1_PIPE0_RESET, 1);
6911 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6912 MEC_ME1_PIPE0_RESET, 0);
6913 break;
6914 case 1:
6915 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6916 MEC_ME1_PIPE1_RESET, 1);
6917 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6918 MEC_ME1_PIPE1_RESET, 0);
6919 break;
6920 case 2:
6921 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6922 MEC_ME1_PIPE2_RESET, 1);
6923 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6924 MEC_ME1_PIPE2_RESET, 0);
6925 break;
6926 case 3:
6927 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6928 MEC_ME1_PIPE3_RESET, 1);
6929 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6930 MEC_ME1_PIPE3_RESET, 0);
6931 break;
6932 default:
6933 break;
6934 }
6935 /* mec1 fw pc: CP_MEC1_INSTR_PNTR */
6936 } else {
6937 switch (ring->pipe) {
6938 case 0:
6939 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6940 MEC_ME2_PIPE0_RESET, 1);
6941 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6942 MEC_ME2_PIPE0_RESET, 0);
6943 break;
6944 case 1:
6945 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6946 MEC_ME2_PIPE1_RESET, 1);
6947 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6948 MEC_ME2_PIPE1_RESET, 0);
6949 break;
6950 case 2:
6951 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6952 MEC_ME2_PIPE2_RESET, 1);
6953 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6954 MEC_ME2_PIPE2_RESET, 0);
6955 break;
6956 case 3:
6957 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
6958 MEC_ME2_PIPE3_RESET, 1);
6959 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
6960 MEC_ME2_PIPE3_RESET, 0);
6961 break;
6962 default:
6963 break;
6964 }
6965 /* mec2 fw pc: CP:CP_MEC2_INSTR_PNTR */
6966 }
6967 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_pipe);
6968 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_pipe);
6969 r = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_MEC1_INSTR_PNTR));
6970 }
6971
6972 soc21_grbm_select(adev, 0, 0, 0, 0);
6973 mutex_unlock(&adev->srbm_mutex);
6974 gfx_v11_0_unset_safe_mode(adev, 0);
6975
6976 dev_info(adev->dev, "The ring %s pipe resets to MEC FW start PC: %s\n", ring->name,
6977 r == 0 ? "successfully" : "failed");
6978 /*FIXME:Sometimes driver can't cache the MEC firmware start PC correctly, so the pipe
6979 * reset status relies on the compute ring test result.
6980 */
6981 return 0;
6982 }
6983
gfx_v11_0_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)6984 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
6985 unsigned int vmid,
6986 struct amdgpu_fence *timedout_fence)
6987 {
6988 struct amdgpu_device *adev = ring->adev;
6989 int r = 0;
6990
6991 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
6992
6993 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true);
6994 if (r) {
6995 dev_warn(adev->dev, "fail(%d) to reset kcq and try pipe reset\n", r);
6996 r = gfx_v11_0_reset_compute_pipe(ring);
6997 if (r)
6998 return r;
6999 }
7000
7001 r = gfx_v11_0_kcq_init_queue(ring, true);
7002 if (r) {
7003 dev_err(adev->dev, "fail to init kcq\n");
7004 return r;
7005 }
7006 r = amdgpu_mes_map_legacy_queue(adev, ring);
7007 if (r) {
7008 dev_err(adev->dev, "failed to remap kcq\n");
7009 return r;
7010 }
7011
7012 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
7013 }
7014
gfx_v11_ip_print(struct amdgpu_ip_block * ip_block,struct drm_printer * p)7015 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
7016 {
7017 struct amdgpu_device *adev = ip_block->adev;
7018 uint32_t i, j, k, reg, index = 0;
7019 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
7020
7021 if (!adev->gfx.ip_dump_core)
7022 return;
7023
7024 for (i = 0; i < reg_count; i++)
7025 drm_printf(p, "%-50s \t 0x%08x\n",
7026 gc_reg_list_11_0[i].reg_name,
7027 adev->gfx.ip_dump_core[i]);
7028
7029 /* print compute queue registers for all instances */
7030 if (!adev->gfx.ip_dump_compute_queues)
7031 return;
7032
7033 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
7034 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
7035 adev->gfx.mec.num_mec,
7036 adev->gfx.mec.num_pipe_per_mec,
7037 adev->gfx.mec.num_queue_per_pipe);
7038
7039 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7040 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7041 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7042 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
7043 for (reg = 0; reg < reg_count; reg++) {
7044 if (i && gc_cp_reg_list_11[reg].reg_offset == regCP_MEC_ME1_HEADER_DUMP)
7045 drm_printf(p, "%-50s \t 0x%08x\n",
7046 "regCP_MEC_ME2_HEADER_DUMP",
7047 adev->gfx.ip_dump_compute_queues[index + reg]);
7048 else
7049 drm_printf(p, "%-50s \t 0x%08x\n",
7050 gc_cp_reg_list_11[reg].reg_name,
7051 adev->gfx.ip_dump_compute_queues[index + reg]);
7052 }
7053 index += reg_count;
7054 }
7055 }
7056 }
7057
7058 /* print gfx queue registers for all instances */
7059 if (!adev->gfx.ip_dump_gfx_queues)
7060 return;
7061
7062 index = 0;
7063 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
7064 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n",
7065 adev->gfx.me.num_me,
7066 adev->gfx.me.num_pipe_per_me,
7067 adev->gfx.me.num_queue_per_pipe);
7068
7069 for (i = 0; i < adev->gfx.me.num_me; i++) {
7070 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
7071 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
7072 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k);
7073 for (reg = 0; reg < reg_count; reg++) {
7074 drm_printf(p, "%-50s \t 0x%08x\n",
7075 gc_gfx_queue_reg_list_11[reg].reg_name,
7076 adev->gfx.ip_dump_gfx_queues[index + reg]);
7077 }
7078 index += reg_count;
7079 }
7080 }
7081 }
7082 }
7083
gfx_v11_ip_dump(struct amdgpu_ip_block * ip_block)7084 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block)
7085 {
7086 struct amdgpu_device *adev = ip_block->adev;
7087 uint32_t i, j, k, reg, index = 0;
7088 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
7089
7090 if (!adev->gfx.ip_dump_core)
7091 return;
7092
7093 amdgpu_gfx_off_ctrl(adev, false);
7094 for (i = 0; i < reg_count; i++)
7095 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i]));
7096 amdgpu_gfx_off_ctrl(adev, true);
7097
7098 /* dump compute queue registers for all instances */
7099 if (!adev->gfx.ip_dump_compute_queues)
7100 return;
7101
7102 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
7103 amdgpu_gfx_off_ctrl(adev, false);
7104 mutex_lock(&adev->srbm_mutex);
7105 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
7106 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
7107 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
7108 /* ME0 is for GFX so start from 1 for CP */
7109 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
7110 for (reg = 0; reg < reg_count; reg++) {
7111 if (i &&
7112 gc_cp_reg_list_11[reg].reg_offset ==
7113 regCP_MEC_ME1_HEADER_DUMP)
7114 adev->gfx.ip_dump_compute_queues[index + reg] =
7115 RREG32(SOC15_REG_OFFSET(GC, 0,
7116 regCP_MEC_ME2_HEADER_DUMP));
7117 else
7118 adev->gfx.ip_dump_compute_queues[index + reg] =
7119 RREG32(SOC15_REG_ENTRY_OFFSET(
7120 gc_cp_reg_list_11[reg]));
7121 }
7122 index += reg_count;
7123 }
7124 }
7125 }
7126 soc21_grbm_select(adev, 0, 0, 0, 0);
7127 mutex_unlock(&adev->srbm_mutex);
7128 amdgpu_gfx_off_ctrl(adev, true);
7129
7130 /* dump gfx queue registers for all instances */
7131 if (!adev->gfx.ip_dump_gfx_queues)
7132 return;
7133
7134 index = 0;
7135 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
7136 amdgpu_gfx_off_ctrl(adev, false);
7137 mutex_lock(&adev->srbm_mutex);
7138 for (i = 0; i < adev->gfx.me.num_me; i++) {
7139 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
7140 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
7141 soc21_grbm_select(adev, i, j, k, 0);
7142
7143 for (reg = 0; reg < reg_count; reg++) {
7144 adev->gfx.ip_dump_gfx_queues[index + reg] =
7145 RREG32(SOC15_REG_ENTRY_OFFSET(
7146 gc_gfx_queue_reg_list_11[reg]));
7147 }
7148 index += reg_count;
7149 }
7150 }
7151 }
7152 soc21_grbm_select(adev, 0, 0, 0, 0);
7153 mutex_unlock(&adev->srbm_mutex);
7154 amdgpu_gfx_off_ctrl(adev, true);
7155 }
7156
gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring * ring)7157 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
7158 {
7159 /* Emit the cleaner shader */
7160 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
7161 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */
7162 }
7163
gfx_v11_0_ring_begin_use(struct amdgpu_ring * ring)7164 static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring)
7165 {
7166 amdgpu_gfx_profile_ring_begin_use(ring);
7167
7168 amdgpu_gfx_enforce_isolation_ring_begin_use(ring);
7169 }
7170
gfx_v11_0_ring_end_use(struct amdgpu_ring * ring)7171 static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring)
7172 {
7173 amdgpu_gfx_profile_ring_end_use(ring);
7174
7175 amdgpu_gfx_enforce_isolation_ring_end_use(ring);
7176 }
7177
7178 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
7179 .name = "gfx_v11_0",
7180 .early_init = gfx_v11_0_early_init,
7181 .late_init = gfx_v11_0_late_init,
7182 .sw_init = gfx_v11_0_sw_init,
7183 .sw_fini = gfx_v11_0_sw_fini,
7184 .hw_init = gfx_v11_0_hw_init,
7185 .hw_fini = gfx_v11_0_hw_fini,
7186 .suspend = gfx_v11_0_suspend,
7187 .resume = gfx_v11_0_resume,
7188 .is_idle = gfx_v11_0_is_idle,
7189 .wait_for_idle = gfx_v11_0_wait_for_idle,
7190 .soft_reset = gfx_v11_0_soft_reset,
7191 .check_soft_reset = gfx_v11_0_check_soft_reset,
7192 .post_soft_reset = gfx_v11_0_post_soft_reset,
7193 .set_clockgating_state = gfx_v11_0_set_clockgating_state,
7194 .set_powergating_state = gfx_v11_0_set_powergating_state,
7195 .get_clockgating_state = gfx_v11_0_get_clockgating_state,
7196 .dump_ip_state = gfx_v11_ip_dump,
7197 .print_ip_state = gfx_v11_ip_print,
7198 };
7199
7200 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
7201 .type = AMDGPU_RING_TYPE_GFX,
7202 .align_mask = 0xff,
7203 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7204 .support_64bit_ptrs = true,
7205 .secure_submission_supported = true,
7206 .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
7207 .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
7208 .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
7209 .emit_frame_size = /* totally 247 maximum if 16 IBs */
7210 5 + /* update_spm_vmid */
7211 5 + /* COND_EXEC */
7212 22 + /* SET_Q_PREEMPTION_MODE */
7213 7 + /* PIPELINE_SYNC */
7214 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7215 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7216 4 + /* VM_FLUSH */
7217 8 + /* FENCE for VM_FLUSH */
7218 20 + /* GDS switch */
7219 5 + /* COND_EXEC */
7220 7 + /* HDP_flush */
7221 4 + /* VGT_flush */
7222 31 + /* DE_META */
7223 3 + /* CNTX_CTRL */
7224 5 + /* HDP_INVL */
7225 22 + /* SET_Q_PREEMPTION_MODE */
7226 8 + 8 + /* FENCE x2 */
7227 8 + /* gfx_v11_0_emit_mem_sync */
7228 2, /* gfx_v11_0_ring_emit_cleaner_shader */
7229 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
7230 .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
7231 .emit_fence = gfx_v11_0_ring_emit_fence,
7232 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
7233 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
7234 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
7235 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
7236 .test_ring = gfx_v11_0_ring_test_ring,
7237 .test_ib = gfx_v11_0_ring_test_ib,
7238 .insert_nop = gfx_v11_ring_insert_nop,
7239 .pad_ib = amdgpu_ring_generic_pad_ib,
7240 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
7241 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
7242 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
7243 .preempt_ib = gfx_v11_0_ring_preempt_ib,
7244 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
7245 .emit_wreg = gfx_v11_0_ring_emit_wreg,
7246 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
7247 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
7248 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
7249 .reset = gfx_v11_0_reset_kgq,
7250 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader,
7251 .begin_use = gfx_v11_0_ring_begin_use,
7252 .end_use = gfx_v11_0_ring_end_use,
7253 };
7254
7255 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
7256 .type = AMDGPU_RING_TYPE_COMPUTE,
7257 .align_mask = 0xff,
7258 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7259 .support_64bit_ptrs = true,
7260 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
7261 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
7262 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
7263 .emit_frame_size =
7264 5 + /* update_spm_vmid */
7265 20 + /* gfx_v11_0_ring_emit_gds_switch */
7266 7 + /* gfx_v11_0_ring_emit_hdp_flush */
7267 5 + /* hdp invalidate */
7268 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
7269 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7270 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7271 2 + /* gfx_v11_0_ring_emit_vm_flush */
7272 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
7273 8 + /* gfx_v11_0_emit_mem_sync */
7274 2, /* gfx_v11_0_ring_emit_cleaner_shader */
7275 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
7276 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
7277 .emit_fence = gfx_v11_0_ring_emit_fence,
7278 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
7279 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
7280 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
7281 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
7282 .test_ring = gfx_v11_0_ring_test_ring,
7283 .test_ib = gfx_v11_0_ring_test_ib,
7284 .insert_nop = gfx_v11_ring_insert_nop,
7285 .pad_ib = amdgpu_ring_generic_pad_ib,
7286 .emit_wreg = gfx_v11_0_ring_emit_wreg,
7287 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
7288 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
7289 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
7290 .reset = gfx_v11_0_reset_kcq,
7291 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader,
7292 .begin_use = gfx_v11_0_ring_begin_use,
7293 .end_use = gfx_v11_0_ring_end_use,
7294 };
7295
7296 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
7297 .type = AMDGPU_RING_TYPE_KIQ,
7298 .align_mask = 0xff,
7299 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7300 .support_64bit_ptrs = true,
7301 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
7302 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
7303 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
7304 .emit_frame_size =
7305 20 + /* gfx_v11_0_ring_emit_gds_switch */
7306 7 + /* gfx_v11_0_ring_emit_hdp_flush */
7307 5 + /*hdp invalidate */
7308 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
7309 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
7310 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
7311 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7312 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
7313 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
7314 .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
7315 .test_ring = gfx_v11_0_ring_test_ring,
7316 .test_ib = gfx_v11_0_ring_test_ib,
7317 .insert_nop = amdgpu_ring_insert_nop,
7318 .pad_ib = amdgpu_ring_generic_pad_ib,
7319 .emit_rreg = gfx_v11_0_ring_emit_rreg,
7320 .emit_wreg = gfx_v11_0_ring_emit_wreg,
7321 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
7322 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
7323 };
7324
gfx_v11_0_set_ring_funcs(struct amdgpu_device * adev)7325 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
7326 {
7327 int i;
7328
7329 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq;
7330
7331 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7332 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
7333
7334 for (i = 0; i < adev->gfx.num_compute_rings; i++)
7335 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
7336 }
7337
7338 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
7339 .set = gfx_v11_0_set_eop_interrupt_state,
7340 .process = gfx_v11_0_eop_irq,
7341 };
7342
7343 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
7344 .set = gfx_v11_0_set_priv_reg_fault_state,
7345 .process = gfx_v11_0_priv_reg_irq,
7346 };
7347
7348 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = {
7349 .set = gfx_v11_0_set_bad_op_fault_state,
7350 .process = gfx_v11_0_bad_op_irq,
7351 };
7352
7353 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
7354 .set = gfx_v11_0_set_priv_inst_fault_state,
7355 .process = gfx_v11_0_priv_inst_irq,
7356 };
7357
7358 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
7359 .process = gfx_v11_0_rlc_gc_fed_irq,
7360 };
7361
gfx_v11_0_set_irq_funcs(struct amdgpu_device * adev)7362 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
7363 {
7364 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7365 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
7366
7367 adev->gfx.priv_reg_irq.num_types = 1;
7368 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
7369
7370 adev->gfx.bad_op_irq.num_types = 1;
7371 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs;
7372
7373 adev->gfx.priv_inst_irq.num_types = 1;
7374 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
7375
7376 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
7377 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
7378
7379 }
7380
gfx_v11_0_set_imu_funcs(struct amdgpu_device * adev)7381 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
7382 {
7383 if (adev->flags & AMD_IS_APU)
7384 adev->gfx.imu.mode = MISSION_MODE;
7385 else
7386 adev->gfx.imu.mode = DEBUG_MODE;
7387
7388 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
7389 }
7390
gfx_v11_0_set_rlc_funcs(struct amdgpu_device * adev)7391 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
7392 {
7393 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
7394 }
7395
gfx_v11_0_set_gds_init(struct amdgpu_device * adev)7396 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
7397 {
7398 unsigned total_cu = adev->gfx.config.max_cu_per_sh *
7399 adev->gfx.config.max_sh_per_se *
7400 adev->gfx.config.max_shader_engines;
7401
7402 adev->gds.gds_size = 0x1000;
7403 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
7404 adev->gds.gws_size = 64;
7405 adev->gds.oa_size = 16;
7406 }
7407
gfx_v11_0_set_mqd_funcs(struct amdgpu_device * adev)7408 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
7409 {
7410 /* set gfx eng mqd */
7411 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
7412 sizeof(struct v11_gfx_mqd);
7413 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
7414 gfx_v11_0_gfx_mqd_init;
7415 /* set compute eng mqd */
7416 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
7417 sizeof(struct v11_compute_mqd);
7418 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
7419 gfx_v11_0_compute_mqd_init;
7420 }
7421
gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device * adev,u32 bitmap)7422 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
7423 u32 bitmap)
7424 {
7425 u32 data;
7426
7427 if (!bitmap)
7428 return;
7429
7430 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7431 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7432
7433 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
7434 }
7435
gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device * adev)7436 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
7437 {
7438 u32 data, wgp_bitmask;
7439 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
7440 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
7441
7442 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7443 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7444
7445 wgp_bitmask =
7446 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
7447
7448 return (~data) & wgp_bitmask;
7449 }
7450
gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device * adev)7451 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
7452 {
7453 u32 wgp_idx, wgp_active_bitmap;
7454 u32 cu_bitmap_per_wgp, cu_active_bitmap;
7455
7456 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
7457 cu_active_bitmap = 0;
7458
7459 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
7460 /* if there is one WGP enabled, it means 2 CUs will be enabled */
7461 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
7462 if (wgp_active_bitmap & (1 << wgp_idx))
7463 cu_active_bitmap |= cu_bitmap_per_wgp;
7464 }
7465
7466 return cu_active_bitmap;
7467 }
7468
gfx_v11_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7469 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
7470 struct amdgpu_cu_info *cu_info)
7471 {
7472 int i, j, k, counter, active_cu_number = 0;
7473 u32 mask, bitmap;
7474 unsigned disable_masks[8 * 2];
7475
7476 if (!adev || !cu_info)
7477 return -EINVAL;
7478
7479 amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
7480
7481 mutex_lock(&adev->grbm_idx_mutex);
7482 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7483 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7484 bitmap = i * adev->gfx.config.max_sh_per_se + j;
7485 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
7486 continue;
7487 mask = 1;
7488 counter = 0;
7489 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0);
7490 if (i < 8 && j < 2)
7491 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
7492 adev, disable_masks[i * 2 + j]);
7493 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
7494
7495 /**
7496 * GFX11 could support more than 4 SEs, while the bitmap
7497 * in cu_info struct is 4x4 and ioctl interface struct
7498 * drm_amdgpu_info_device should keep stable.
7499 * So we use last two columns of bitmap to store cu mask for
7500 * SEs 4 to 7, the layout of the bitmap is as below:
7501 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
7502 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
7503 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
7504 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
7505 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
7506 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
7507 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
7508 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
7509 */
7510 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
7511
7512 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
7513 if (bitmap & mask)
7514 counter++;
7515
7516 mask <<= 1;
7517 }
7518 active_cu_number += counter;
7519 }
7520 }
7521 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7522 mutex_unlock(&adev->grbm_idx_mutex);
7523
7524 cu_info->number = active_cu_number;
7525 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7526
7527 return 0;
7528 }
7529
7530 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
7531 {
7532 .type = AMD_IP_BLOCK_TYPE_GFX,
7533 .major = 11,
7534 .minor = 0,
7535 .rev = 0,
7536 .funcs = &gfx_v11_0_ip_funcs,
7537 };
7538