1 /*
2 * Copyright 2021 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/delay.h>
24 #include <linux/kernel.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 #include "amdgpu.h"
29 #include "amdgpu_gfx.h"
30 #include "amdgpu_psp.h"
31 #include "amdgpu_smu.h"
32 #include "amdgpu_atomfirmware.h"
33 #include "imu_v11_0.h"
34 #include "soc21.h"
35 #include "nvd.h"
36
37 #include "gc/gc_11_0_0_offset.h"
38 #include "gc/gc_11_0_0_sh_mask.h"
39 #include "smuio/smuio_13_0_6_offset.h"
40 #include "smuio/smuio_13_0_6_sh_mask.h"
41 #include "navi10_enum.h"
42 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h"
43
44 #include "soc15.h"
45 #include "soc15d.h"
46 #include "clearstate_gfx11.h"
47 #include "v11_structs.h"
48 #include "gfx_v11_0.h"
49 #include "gfx_v11_0_3.h"
50 #include "nbio_v4_3.h"
51 #include "mes_v11_0.h"
52
53 #define GFX11_NUM_GFX_RINGS 1
54 #define GFX11_MEC_HPD_SIZE 2048
55
56 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
57 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388
58
59 #define regCGTT_WD_CLK_CTRL 0x5086
60 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1
61 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e
62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1
63 #define regPC_CONFIG_CNTL_1 0x194d
64 #define regPC_CONFIG_CNTL_1_BASE_IDX 1
65
66 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin");
67 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin");
68 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin");
69 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin");
70 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin");
71 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin");
72 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin");
73 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin");
74 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin");
75 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin");
76 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
77 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
78 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
79 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
80 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
81 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
82 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
83 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
84 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin");
85 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin");
86 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin");
87 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin");
88 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin");
89 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin");
90 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin");
91 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin");
92 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin");
93 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin");
94 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin");
95 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin");
96 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin");
97 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin");
98 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin");
99 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin");
100
101 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = {
102 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
103 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
104 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
105 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
106 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
107 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
108 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
109 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
110 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
111 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
112 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
113 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
114 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
115 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
116 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
117 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
118 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
119 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
120 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
121 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
122 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
123 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
124 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE),
125 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR),
126 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR),
127 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
128 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
129 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
130 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
131 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
132 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
133 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
134 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
135 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
136 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
137 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
138 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
139 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
140 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
141 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
142 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
143 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
144 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
145 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
146 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
147 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
148 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL),
149 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS),
150 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
151 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
152 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
153 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
154 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR),
155 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
156 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
157 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
158 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
159 /* cp header registers */
160 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
161 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP),
162 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP),
163 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
164 /* SE status registers */
165 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
166 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
167 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
168 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3),
169 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4),
170 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5)
171 };
172
173 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = {
174 /* compute registers */
175 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
176 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
177 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
178 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
179 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
180 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
181 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
182 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
183 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
184 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
185 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
186 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
187 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
188 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
189 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
190 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
191 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
192 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
193 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
194 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
195 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
196 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
197 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
198 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
199 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
200 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
201 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
202 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
203 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
204 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
205 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
206 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
207 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
208 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
209 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
210 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
211 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
212 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
213 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS)
214 };
215
216 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = {
217 /* gfx queue registers */
218 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE),
219 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID),
220 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY),
221 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM),
222 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE),
223 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI),
224 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET),
225 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL),
226 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR),
227 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR),
228 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI),
229 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST),
230 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED),
231 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL),
232 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0),
233 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0),
234 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR),
235 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI),
236 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO),
237 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI),
238 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
239 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
240 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
241 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
242 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ)
243 };
244
245 static const struct soc15_reg_golden golden_settings_gc_11_0[] = {
246 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000)
247 };
248
249 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
250 {
251 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010),
252 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010),
253 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200),
254 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988),
255 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007),
256 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008),
257 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100),
258 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000),
259 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a)
260 };
261
262 #define DEFAULT_SH_MEM_CONFIG \
263 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
264 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
265 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
266
267 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev);
268 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev);
269 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev);
270 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev);
271 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev);
272 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev);
273 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev);
274 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
275 struct amdgpu_cu_info *cu_info);
276 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev);
277 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
278 u32 sh_num, u32 instance, int xcc_id);
279 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
280
281 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
282 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure);
283 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
284 uint32_t val);
285 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
286 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
287 uint16_t pasid, uint32_t flush_type,
288 bool all_hub, uint8_t dst_sel);
289 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
290 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
291 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
292 bool enable);
293
gfx11_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)294 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
295 {
296 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
297 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
298 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */
299 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
300 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
301 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
302 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
303 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
304 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
305 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
306 }
307
gfx11_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)308 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring,
309 struct amdgpu_ring *ring)
310 {
311 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
312 uint64_t wptr_addr = ring->wptr_gpu_addr;
313 uint32_t me = 0, eng_sel = 0;
314
315 switch (ring->funcs->type) {
316 case AMDGPU_RING_TYPE_COMPUTE:
317 me = 1;
318 eng_sel = 0;
319 break;
320 case AMDGPU_RING_TYPE_GFX:
321 me = 0;
322 eng_sel = 4;
323 break;
324 case AMDGPU_RING_TYPE_MES:
325 me = 2;
326 eng_sel = 5;
327 break;
328 default:
329 WARN_ON(1);
330 }
331
332 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
333 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
334 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
335 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
336 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
337 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
338 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
339 PACKET3_MAP_QUEUES_ME((me)) |
340 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
341 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */
342 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
343 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
344 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
345 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
346 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
347 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
348 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
349 }
350
gfx11_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)351 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
352 struct amdgpu_ring *ring,
353 enum amdgpu_unmap_queues_action action,
354 u64 gpu_addr, u64 seq)
355 {
356 struct amdgpu_device *adev = kiq_ring->adev;
357 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
358
359 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
360 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, seq);
361 return;
362 }
363
364 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
365 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
366 PACKET3_UNMAP_QUEUES_ACTION(action) |
367 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
368 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
369 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
370 amdgpu_ring_write(kiq_ring,
371 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
372
373 if (action == PREEMPT_QUEUES_NO_UNMAP) {
374 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
375 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
376 amdgpu_ring_write(kiq_ring, seq);
377 } else {
378 amdgpu_ring_write(kiq_ring, 0);
379 amdgpu_ring_write(kiq_ring, 0);
380 amdgpu_ring_write(kiq_ring, 0);
381 }
382 }
383
gfx11_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)384 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring,
385 struct amdgpu_ring *ring,
386 u64 addr,
387 u64 seq)
388 {
389 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
390
391 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
392 amdgpu_ring_write(kiq_ring,
393 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
394 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
395 PACKET3_QUERY_STATUS_COMMAND(2));
396 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
397 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
398 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
399 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
400 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
401 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
402 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
403 }
404
gfx11_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)405 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
406 uint16_t pasid, uint32_t flush_type,
407 bool all_hub)
408 {
409 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
410 }
411
412 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = {
413 .kiq_set_resources = gfx11_kiq_set_resources,
414 .kiq_map_queues = gfx11_kiq_map_queues,
415 .kiq_unmap_queues = gfx11_kiq_unmap_queues,
416 .kiq_query_status = gfx11_kiq_query_status,
417 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs,
418 .set_resources_size = 8,
419 .map_queues_size = 7,
420 .unmap_queues_size = 6,
421 .query_status_size = 7,
422 .invalidate_tlbs_size = 2,
423 };
424
gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device * adev)425 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
426 {
427 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs;
428 }
429
gfx_v11_0_init_golden_registers(struct amdgpu_device * adev)430 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
431 {
432 if (amdgpu_sriov_vf(adev))
433 return;
434
435 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
436 case IP_VERSION(11, 0, 1):
437 case IP_VERSION(11, 0, 4):
438 soc15_program_register_sequence(adev,
439 golden_settings_gc_11_0_1,
440 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
441 break;
442 default:
443 break;
444 }
445 soc15_program_register_sequence(adev,
446 golden_settings_gc_11_0,
447 (const u32)ARRAY_SIZE(golden_settings_gc_11_0));
448
449 }
450
gfx_v11_0_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)451 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
452 bool wc, uint32_t reg, uint32_t val)
453 {
454 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
455 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
456 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
457 amdgpu_ring_write(ring, reg);
458 amdgpu_ring_write(ring, 0);
459 amdgpu_ring_write(ring, val);
460 }
461
gfx_v11_0_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)462 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
463 int mem_space, int opt, uint32_t addr0,
464 uint32_t addr1, uint32_t ref, uint32_t mask,
465 uint32_t inv)
466 {
467 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
468 amdgpu_ring_write(ring,
469 /* memory (1) or register (0) */
470 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
471 WAIT_REG_MEM_OPERATION(opt) | /* wait */
472 WAIT_REG_MEM_FUNCTION(3) | /* equal */
473 WAIT_REG_MEM_ENGINE(eng_sel)));
474
475 if (mem_space)
476 BUG_ON(addr0 & 0x3); /* Dword align */
477 amdgpu_ring_write(ring, addr0);
478 amdgpu_ring_write(ring, addr1);
479 amdgpu_ring_write(ring, ref);
480 amdgpu_ring_write(ring, mask);
481 amdgpu_ring_write(ring, inv); /* poll interval */
482 }
483
gfx_v11_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)484 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
485 {
486 int i;
487
488 /* Header itself is a NOP packet */
489 if (num_nop == 1) {
490 amdgpu_ring_write(ring, ring->funcs->nop);
491 return;
492 }
493
494 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
495 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
496
497 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
498 for (i = 1; i < num_nop; i++)
499 amdgpu_ring_write(ring, ring->funcs->nop);
500 }
501
gfx_v11_0_ring_test_ring(struct amdgpu_ring * ring)502 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
503 {
504 struct amdgpu_device *adev = ring->adev;
505 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
506 uint32_t tmp = 0;
507 unsigned i;
508 int r;
509
510 WREG32(scratch, 0xCAFEDEAD);
511 r = amdgpu_ring_alloc(ring, 5);
512 if (r) {
513 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
514 ring->idx, r);
515 return r;
516 }
517
518 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
519 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF);
520 } else {
521 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
522 amdgpu_ring_write(ring, scratch -
523 PACKET3_SET_UCONFIG_REG_START);
524 amdgpu_ring_write(ring, 0xDEADBEEF);
525 }
526 amdgpu_ring_commit(ring);
527
528 for (i = 0; i < adev->usec_timeout; i++) {
529 tmp = RREG32(scratch);
530 if (tmp == 0xDEADBEEF)
531 break;
532 if (amdgpu_emu_mode == 1)
533 msleep(1);
534 else
535 udelay(1);
536 }
537
538 if (i >= adev->usec_timeout)
539 r = -ETIMEDOUT;
540 return r;
541 }
542
gfx_v11_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)543 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
544 {
545 struct amdgpu_device *adev = ring->adev;
546 struct amdgpu_ib ib;
547 struct dma_fence *f = NULL;
548 unsigned index;
549 uint64_t gpu_addr;
550 volatile uint32_t *cpu_ptr;
551 long r;
552
553 /* MES KIQ fw hasn't indirect buffer support for now */
554 if (adev->enable_mes_kiq &&
555 ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
556 return 0;
557
558 memset(&ib, 0, sizeof(ib));
559
560 if (ring->is_mes_queue) {
561 uint32_t padding, offset;
562
563 offset = amdgpu_mes_ctx_get_offs(ring, AMDGPU_MES_CTX_IB_OFFS);
564 padding = amdgpu_mes_ctx_get_offs(ring,
565 AMDGPU_MES_CTX_PADDING_OFFS);
566
567 ib.gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
568 ib.ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
569
570 gpu_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, padding);
571 cpu_ptr = amdgpu_mes_ctx_get_offs_cpu_addr(ring, padding);
572 *cpu_ptr = cpu_to_le32(0xCAFEDEAD);
573 } else {
574 r = amdgpu_device_wb_get(adev, &index);
575 if (r)
576 return r;
577
578 gpu_addr = adev->wb.gpu_addr + (index * 4);
579 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
580 cpu_ptr = &adev->wb.wb[index];
581
582 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
583 if (r) {
584 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
585 goto err1;
586 }
587 }
588
589 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
590 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
591 ib.ptr[2] = lower_32_bits(gpu_addr);
592 ib.ptr[3] = upper_32_bits(gpu_addr);
593 ib.ptr[4] = 0xDEADBEEF;
594 ib.length_dw = 5;
595
596 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
597 if (r)
598 goto err2;
599
600 r = dma_fence_wait_timeout(f, false, timeout);
601 if (r == 0) {
602 r = -ETIMEDOUT;
603 goto err2;
604 } else if (r < 0) {
605 goto err2;
606 }
607
608 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
609 r = 0;
610 else
611 r = -EINVAL;
612 err2:
613 if (!ring->is_mes_queue)
614 amdgpu_ib_free(adev, &ib, NULL);
615 dma_fence_put(f);
616 err1:
617 if (!ring->is_mes_queue)
618 amdgpu_device_wb_free(adev, index);
619 return r;
620 }
621
gfx_v11_0_free_microcode(struct amdgpu_device * adev)622 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
623 {
624 amdgpu_ucode_release(&adev->gfx.pfp_fw);
625 amdgpu_ucode_release(&adev->gfx.me_fw);
626 amdgpu_ucode_release(&adev->gfx.rlc_fw);
627 amdgpu_ucode_release(&adev->gfx.mec_fw);
628
629 kfree(adev->gfx.rlc.register_list_format);
630 }
631
gfx_v11_0_init_toc_microcode(struct amdgpu_device * adev,const char * ucode_prefix)632 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
633 {
634 const struct psp_firmware_header_v1_0 *toc_hdr;
635 int err = 0;
636
637 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
638 "amdgpu/%s_toc.bin", ucode_prefix);
639 if (err)
640 goto out;
641
642 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
643 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
644 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
645 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
646 adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
647 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
648 return 0;
649 out:
650 amdgpu_ucode_release(&adev->psp.toc_fw);
651 return err;
652 }
653
gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device * adev)654 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev)
655 {
656 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
657 case IP_VERSION(11, 0, 0):
658 case IP_VERSION(11, 0, 2):
659 case IP_VERSION(11, 0, 3):
660 if ((adev->gfx.me_fw_version >= 1505) &&
661 (adev->gfx.pfp_fw_version >= 1600) &&
662 (adev->gfx.mec_fw_version >= 512)) {
663 if (amdgpu_sriov_vf(adev))
664 adev->gfx.cp_gfx_shadow = true;
665 else
666 adev->gfx.cp_gfx_shadow = false;
667 }
668 break;
669 default:
670 adev->gfx.cp_gfx_shadow = false;
671 break;
672 }
673 }
674
gfx_v11_0_init_microcode(struct amdgpu_device * adev)675 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
676 {
677 char ucode_prefix[25];
678 int err;
679 const struct rlc_firmware_header_v2_0 *rlc_hdr;
680 uint16_t version_major;
681 uint16_t version_minor;
682
683 DRM_DEBUG("\n");
684
685 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
686 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
687 "amdgpu/%s_pfp.bin", ucode_prefix);
688 if (err)
689 goto out;
690 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/
691 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version(
692 (union amdgpu_firmware_header *)
693 adev->gfx.pfp_fw->data, 2, 0);
694 if (adev->gfx.rs64_enable) {
695 dev_info(adev->dev, "CP RS64 enable\n");
696 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
697 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
698 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
699 } else {
700 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
701 }
702
703 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
704 "amdgpu/%s_me.bin", ucode_prefix);
705 if (err)
706 goto out;
707 if (adev->gfx.rs64_enable) {
708 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
709 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
710 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
711 } else {
712 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
713 }
714
715 if (!amdgpu_sriov_vf(adev)) {
716 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) &&
717 adev->pdev->revision == 0xCE)
718 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
719 "amdgpu/gc_11_0_0_rlc_1.bin");
720 else
721 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
722 "amdgpu/%s_rlc.bin", ucode_prefix);
723 if (err)
724 goto out;
725 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
726 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
727 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
728 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
729 if (err)
730 goto out;
731 }
732
733 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
734 "amdgpu/%s_mec.bin", ucode_prefix);
735 if (err)
736 goto out;
737 if (adev->gfx.rs64_enable) {
738 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
739 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
740 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
741 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
742 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
743 } else {
744 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
745 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
746 }
747
748 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
749 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix);
750
751 /* only one MEC for gfx 11.0.0. */
752 adev->gfx.mec2_fw = NULL;
753
754 gfx_v11_0_check_fw_cp_gfx_shadow(adev);
755
756 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) {
757 err = adev->gfx.imu.funcs->init_microcode(adev);
758 if (err)
759 DRM_ERROR("Failed to init imu firmware!\n");
760 return err;
761 }
762
763 out:
764 if (err) {
765 amdgpu_ucode_release(&adev->gfx.pfp_fw);
766 amdgpu_ucode_release(&adev->gfx.me_fw);
767 amdgpu_ucode_release(&adev->gfx.rlc_fw);
768 amdgpu_ucode_release(&adev->gfx.mec_fw);
769 }
770
771 return err;
772 }
773
gfx_v11_0_get_csb_size(struct amdgpu_device * adev)774 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev)
775 {
776 u32 count = 0;
777 const struct cs_section_def *sect = NULL;
778 const struct cs_extent_def *ext = NULL;
779
780 /* begin clear state */
781 count += 2;
782 /* context control state */
783 count += 3;
784
785 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
786 for (ext = sect->section; ext->extent != NULL; ++ext) {
787 if (sect->id == SECT_CONTEXT)
788 count += 2 + ext->reg_count;
789 else
790 return 0;
791 }
792 }
793
794 /* set PA_SC_TILE_STEERING_OVERRIDE */
795 count += 3;
796 /* end clear state */
797 count += 2;
798 /* clear state */
799 count += 2;
800
801 return count;
802 }
803
gfx_v11_0_get_csb_buffer(struct amdgpu_device * adev,volatile u32 * buffer)804 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev,
805 volatile u32 *buffer)
806 {
807 u32 count = 0, i;
808 const struct cs_section_def *sect = NULL;
809 const struct cs_extent_def *ext = NULL;
810 int ctx_reg_offset;
811
812 if (adev->gfx.rlc.cs_data == NULL)
813 return;
814 if (buffer == NULL)
815 return;
816
817 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
818 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
819
820 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
821 buffer[count++] = cpu_to_le32(0x80000000);
822 buffer[count++] = cpu_to_le32(0x80000000);
823
824 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
825 for (ext = sect->section; ext->extent != NULL; ++ext) {
826 if (sect->id == SECT_CONTEXT) {
827 buffer[count++] =
828 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
829 buffer[count++] = cpu_to_le32(ext->reg_index -
830 PACKET3_SET_CONTEXT_REG_START);
831 for (i = 0; i < ext->reg_count; i++)
832 buffer[count++] = cpu_to_le32(ext->extent[i]);
833 } else {
834 return;
835 }
836 }
837 }
838
839 ctx_reg_offset =
840 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
841 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
842 buffer[count++] = cpu_to_le32(ctx_reg_offset);
843 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
844
845 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
846 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
847
848 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
849 buffer[count++] = cpu_to_le32(0);
850 }
851
gfx_v11_0_rlc_fini(struct amdgpu_device * adev)852 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev)
853 {
854 /* clear state block */
855 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
856 &adev->gfx.rlc.clear_state_gpu_addr,
857 (void **)&adev->gfx.rlc.cs_ptr);
858
859 /* jump table block */
860 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
861 &adev->gfx.rlc.cp_table_gpu_addr,
862 (void **)&adev->gfx.rlc.cp_table_ptr);
863 }
864
gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)865 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
866 {
867 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
868
869 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0];
870 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
871 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1);
872 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2);
873 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3);
874 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL);
875 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX);
876 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0);
877 adev->gfx.rlc.rlcg_reg_access_supported = true;
878 }
879
gfx_v11_0_rlc_init(struct amdgpu_device * adev)880 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev)
881 {
882 const struct cs_section_def *cs_data;
883 int r;
884
885 adev->gfx.rlc.cs_data = gfx11_cs_data;
886
887 cs_data = adev->gfx.rlc.cs_data;
888
889 if (cs_data) {
890 /* init clear state block */
891 r = amdgpu_gfx_rlc_init_csb(adev);
892 if (r)
893 return r;
894 }
895
896 /* init spm vmid with 0xf */
897 if (adev->gfx.rlc.funcs->update_spm_vmid)
898 adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
899
900 return 0;
901 }
902
gfx_v11_0_mec_fini(struct amdgpu_device * adev)903 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev)
904 {
905 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
906 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
907 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
908 }
909
gfx_v11_0_me_init(struct amdgpu_device * adev)910 static void gfx_v11_0_me_init(struct amdgpu_device *adev)
911 {
912 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
913
914 amdgpu_gfx_graphics_queue_acquire(adev);
915 }
916
gfx_v11_0_mec_init(struct amdgpu_device * adev)917 static int gfx_v11_0_mec_init(struct amdgpu_device *adev)
918 {
919 int r;
920 u32 *hpd;
921 size_t mec_hpd_size;
922
923 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
924
925 /* take ownership of the relevant compute queues */
926 amdgpu_gfx_compute_queue_acquire(adev);
927 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE;
928
929 if (mec_hpd_size) {
930 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
931 AMDGPU_GEM_DOMAIN_GTT,
932 &adev->gfx.mec.hpd_eop_obj,
933 &adev->gfx.mec.hpd_eop_gpu_addr,
934 (void **)&hpd);
935 if (r) {
936 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
937 gfx_v11_0_mec_fini(adev);
938 return r;
939 }
940
941 memset(hpd, 0, mec_hpd_size);
942
943 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
944 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
945 }
946
947 return 0;
948 }
949
wave_read_ind(struct amdgpu_device * adev,uint32_t wave,uint32_t address)950 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
951 {
952 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
953 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
954 (address << SQ_IND_INDEX__INDEX__SHIFT));
955 return RREG32_SOC15(GC, 0, regSQ_IND_DATA);
956 }
957
wave_read_regs(struct amdgpu_device * adev,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)958 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
959 uint32_t thread, uint32_t regno,
960 uint32_t num, uint32_t *out)
961 {
962 WREG32_SOC15(GC, 0, regSQ_IND_INDEX,
963 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
964 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
965 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
966 (SQ_IND_INDEX__AUTO_INCR_MASK));
967 while (num--)
968 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA);
969 }
970
gfx_v11_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)971 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
972 {
973 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE
974 * field when performing a select_se_sh so it should be
975 * zero here */
976 WARN_ON(simd != 0);
977
978 /* type 3 wave data */
979 dst[(*no_fields)++] = 3;
980 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
981 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
982 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
983 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
984 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
985 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
986 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
987 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
988 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
989 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
990 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
991 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
992 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
993 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
994 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE);
995 }
996
gfx_v11_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)997 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
998 uint32_t wave, uint32_t start,
999 uint32_t size, uint32_t *dst)
1000 {
1001 WARN_ON(simd != 0);
1002
1003 wave_read_regs(
1004 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
1005 dst);
1006 }
1007
gfx_v11_0_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)1008 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
1009 uint32_t wave, uint32_t thread,
1010 uint32_t start, uint32_t size,
1011 uint32_t *dst)
1012 {
1013 wave_read_regs(
1014 adev, wave, thread,
1015 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1016 }
1017
gfx_v11_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)1018 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev,
1019 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
1020 {
1021 soc21_grbm_select(adev, me, pipe, q, vm);
1022 }
1023
1024 /* all sizes are in bytes */
1025 #define MQD_SHADOW_BASE_SIZE 73728
1026 #define MQD_SHADOW_BASE_ALIGNMENT 256
1027 #define MQD_FWWORKAREA_SIZE 484
1028 #define MQD_FWWORKAREA_ALIGNMENT 256
1029
gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device * adev,struct amdgpu_gfx_shadow_info * shadow_info)1030 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev,
1031 struct amdgpu_gfx_shadow_info *shadow_info)
1032 {
1033 if (adev->gfx.cp_gfx_shadow) {
1034 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE;
1035 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT;
1036 shadow_info->csa_size = MQD_FWWORKAREA_SIZE;
1037 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT;
1038 return 0;
1039 } else {
1040 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info));
1041 return -ENOTSUPP;
1042 }
1043 }
1044
1045 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
1046 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter,
1047 .select_se_sh = &gfx_v11_0_select_se_sh,
1048 .read_wave_data = &gfx_v11_0_read_wave_data,
1049 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
1050 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
1051 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
1052 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
1053 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info,
1054 };
1055
gfx_v11_0_gpu_early_init(struct amdgpu_device * adev)1056 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
1057 {
1058 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1059 case IP_VERSION(11, 0, 0):
1060 case IP_VERSION(11, 0, 2):
1061 adev->gfx.config.max_hw_contexts = 8;
1062 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1063 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1064 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1065 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1066 break;
1067 case IP_VERSION(11, 0, 3):
1068 adev->gfx.ras = &gfx_v11_0_3_ras;
1069 adev->gfx.config.max_hw_contexts = 8;
1070 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1071 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1072 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1073 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1074 break;
1075 case IP_VERSION(11, 0, 1):
1076 case IP_VERSION(11, 0, 4):
1077 case IP_VERSION(11, 5, 0):
1078 case IP_VERSION(11, 5, 1):
1079 case IP_VERSION(11, 5, 2):
1080 adev->gfx.config.max_hw_contexts = 8;
1081 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1082 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1083 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
1084 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300;
1085 break;
1086 default:
1087 BUG();
1088 break;
1089 }
1090
1091 return 0;
1092 }
1093
gfx_v11_0_gfx_ring_init(struct amdgpu_device * adev,int ring_id,int me,int pipe,int queue)1094 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1095 int me, int pipe, int queue)
1096 {
1097 struct amdgpu_ring *ring;
1098 unsigned int irq_type;
1099 unsigned int hw_prio;
1100
1101 ring = &adev->gfx.gfx_ring[ring_id];
1102
1103 ring->me = me;
1104 ring->pipe = pipe;
1105 ring->queue = queue;
1106
1107 ring->ring_obj = NULL;
1108 ring->use_doorbell = true;
1109
1110 if (!ring_id)
1111 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1112 else
1113 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1114 ring->vm_hub = AMDGPU_GFXHUB(0);
1115 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1116
1117 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1118 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ?
1119 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1120 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1121 hw_prio, NULL);
1122 }
1123
gfx_v11_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)1124 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1125 int mec, int pipe, int queue)
1126 {
1127 int r;
1128 unsigned irq_type;
1129 struct amdgpu_ring *ring;
1130 unsigned int hw_prio;
1131
1132 ring = &adev->gfx.compute_ring[ring_id];
1133
1134 /* mec0 is me1 */
1135 ring->me = mec + 1;
1136 ring->pipe = pipe;
1137 ring->queue = queue;
1138
1139 ring->ring_obj = NULL;
1140 ring->use_doorbell = true;
1141 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1142 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1143 + (ring_id * GFX11_MEC_HPD_SIZE);
1144 ring->vm_hub = AMDGPU_GFXHUB(0);
1145 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1146
1147 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1148 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1149 + ring->pipe;
1150 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1151 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1152 /* type-2 packets are deprecated on MEC, use type-3 instead */
1153 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1154 hw_prio, NULL);
1155 if (r)
1156 return r;
1157
1158 return 0;
1159 }
1160
1161 static struct {
1162 SOC21_FIRMWARE_ID id;
1163 unsigned int offset;
1164 unsigned int size;
1165 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX];
1166
gfx_v11_0_parse_rlc_toc(struct amdgpu_device * adev,void * rlc_toc)1167 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
1168 {
1169 RLC_TABLE_OF_CONTENT *ucode = rlc_toc;
1170
1171 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) &&
1172 (ucode->id < SOC21_FIRMWARE_ID_MAX)) {
1173 rlc_autoload_info[ucode->id].id = ucode->id;
1174 rlc_autoload_info[ucode->id].offset = ucode->offset * 4;
1175 rlc_autoload_info[ucode->id].size = ucode->size * 4;
1176
1177 ucode++;
1178 }
1179 }
1180
gfx_v11_0_calc_toc_total_size(struct amdgpu_device * adev)1181 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev)
1182 {
1183 uint32_t total_size = 0;
1184 SOC21_FIRMWARE_ID id;
1185
1186 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1187
1188 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++)
1189 total_size += rlc_autoload_info[id].size;
1190
1191 /* In case the offset in rlc toc ucode is aligned */
1192 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset)
1193 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset +
1194 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size;
1195
1196 return total_size;
1197 }
1198
gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device * adev)1199 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1200 {
1201 int r;
1202 uint32_t total_size;
1203
1204 total_size = gfx_v11_0_calc_toc_total_size(adev);
1205
1206 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1207 AMDGPU_GEM_DOMAIN_VRAM |
1208 AMDGPU_GEM_DOMAIN_GTT,
1209 &adev->gfx.rlc.rlc_autoload_bo,
1210 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1211 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1212
1213 if (r) {
1214 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1215 return r;
1216 }
1217
1218 return 0;
1219 }
1220
gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device * adev,SOC21_FIRMWARE_ID id,const void * fw_data,uint32_t fw_size,uint32_t * fw_autoload_mask)1221 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1222 SOC21_FIRMWARE_ID id,
1223 const void *fw_data,
1224 uint32_t fw_size,
1225 uint32_t *fw_autoload_mask)
1226 {
1227 uint32_t toc_offset;
1228 uint32_t toc_fw_size;
1229 char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1230
1231 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX)
1232 return;
1233
1234 toc_offset = rlc_autoload_info[id].offset;
1235 toc_fw_size = rlc_autoload_info[id].size;
1236
1237 if (fw_size == 0)
1238 fw_size = toc_fw_size;
1239
1240 if (fw_size > toc_fw_size)
1241 fw_size = toc_fw_size;
1242
1243 memcpy(ptr + toc_offset, fw_data, fw_size);
1244
1245 if (fw_size < toc_fw_size)
1246 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
1247
1248 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME))
1249 *(uint64_t *)fw_autoload_mask |= 1ULL << id;
1250 }
1251
gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1252 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev,
1253 uint32_t *fw_autoload_mask)
1254 {
1255 void *data;
1256 uint32_t size;
1257 uint64_t *toc_ptr;
1258
1259 *(uint64_t *)fw_autoload_mask |= 0x1;
1260
1261 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask);
1262
1263 data = adev->psp.toc.start_addr;
1264 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size;
1265
1266 toc_ptr = (uint64_t *)data + size / 8 - 1;
1267 *toc_ptr = *(uint64_t *)fw_autoload_mask;
1268
1269 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC,
1270 data, size, fw_autoload_mask);
1271 }
1272
gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1273 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev,
1274 uint32_t *fw_autoload_mask)
1275 {
1276 const __le32 *fw_data;
1277 uint32_t fw_size;
1278 const struct gfx_firmware_header_v1_0 *cp_hdr;
1279 const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1280 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1281 const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1282 uint16_t version_major, version_minor;
1283
1284 if (adev->gfx.rs64_enable) {
1285 /* pfp ucode */
1286 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1287 adev->gfx.pfp_fw->data;
1288 /* instruction */
1289 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1290 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1291 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1292 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP,
1293 fw_data, fw_size, fw_autoload_mask);
1294 /* data */
1295 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1296 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1297 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1298 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK,
1299 fw_data, fw_size, fw_autoload_mask);
1300 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK,
1301 fw_data, fw_size, fw_autoload_mask);
1302 /* me ucode */
1303 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1304 adev->gfx.me_fw->data;
1305 /* instruction */
1306 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1307 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1308 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1309 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME,
1310 fw_data, fw_size, fw_autoload_mask);
1311 /* data */
1312 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1313 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1314 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1315 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK,
1316 fw_data, fw_size, fw_autoload_mask);
1317 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK,
1318 fw_data, fw_size, fw_autoload_mask);
1319 /* mec ucode */
1320 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1321 adev->gfx.mec_fw->data;
1322 /* instruction */
1323 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1324 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1325 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1326 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC,
1327 fw_data, fw_size, fw_autoload_mask);
1328 /* data */
1329 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1330 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1331 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1332 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK,
1333 fw_data, fw_size, fw_autoload_mask);
1334 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK,
1335 fw_data, fw_size, fw_autoload_mask);
1336 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK,
1337 fw_data, fw_size, fw_autoload_mask);
1338 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK,
1339 fw_data, fw_size, fw_autoload_mask);
1340 } else {
1341 /* pfp ucode */
1342 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1343 adev->gfx.pfp_fw->data;
1344 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
1345 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1346 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1347 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP,
1348 fw_data, fw_size, fw_autoload_mask);
1349
1350 /* me ucode */
1351 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1352 adev->gfx.me_fw->data;
1353 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
1354 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1355 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1356 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME,
1357 fw_data, fw_size, fw_autoload_mask);
1358
1359 /* mec ucode */
1360 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1361 adev->gfx.mec_fw->data;
1362 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1363 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
1364 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1365 cp_hdr->jt_size * 4;
1366 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC,
1367 fw_data, fw_size, fw_autoload_mask);
1368 }
1369
1370 /* rlc ucode */
1371 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1372 adev->gfx.rlc_fw->data;
1373 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1374 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1375 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1376 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE,
1377 fw_data, fw_size, fw_autoload_mask);
1378
1379 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1380 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1381 if (version_major == 2) {
1382 if (version_minor >= 2) {
1383 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1384
1385 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1386 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1387 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1388 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE,
1389 fw_data, fw_size, fw_autoload_mask);
1390
1391 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1392 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1393 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1394 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT,
1395 fw_data, fw_size, fw_autoload_mask);
1396 }
1397 }
1398 }
1399
gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1400 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev,
1401 uint32_t *fw_autoload_mask)
1402 {
1403 const __le32 *fw_data;
1404 uint32_t fw_size;
1405 const struct sdma_firmware_header_v2_0 *sdma_hdr;
1406
1407 sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
1408 adev->sdma.instance[0].fw->data;
1409 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1410 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
1411 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes);
1412
1413 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1414 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask);
1415
1416 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1417 le32_to_cpu(sdma_hdr->ctl_ucode_offset));
1418 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes);
1419
1420 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1421 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask);
1422 }
1423
gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device * adev,uint32_t * fw_autoload_mask)1424 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev,
1425 uint32_t *fw_autoload_mask)
1426 {
1427 const __le32 *fw_data;
1428 unsigned fw_size;
1429 const struct mes_firmware_header_v1_0 *mes_hdr;
1430 int pipe, ucode_id, data_id;
1431
1432 for (pipe = 0; pipe < 2; pipe++) {
1433 if (pipe==0) {
1434 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0;
1435 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK;
1436 } else {
1437 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1;
1438 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK;
1439 }
1440
1441 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1442 adev->mes.fw[pipe]->data;
1443
1444 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1445 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1446 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1447
1448 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1449 ucode_id, fw_data, fw_size, fw_autoload_mask);
1450
1451 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1452 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1453 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1454
1455 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev,
1456 data_id, fw_data, fw_size, fw_autoload_mask);
1457 }
1458 }
1459
gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device * adev)1460 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1461 {
1462 uint32_t rlc_g_offset, rlc_g_size;
1463 uint64_t gpu_addr;
1464 uint32_t autoload_fw_id[2];
1465
1466 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2);
1467
1468 /* RLC autoload sequence 2: copy ucode */
1469 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id);
1470 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id);
1471 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id);
1472 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id);
1473
1474 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset;
1475 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size;
1476 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
1477
1478 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr));
1479 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr));
1480
1481 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size);
1482
1483 /* RLC autoload sequence 3: load IMU fw */
1484 if (adev->gfx.imu.funcs->load_microcode)
1485 adev->gfx.imu.funcs->load_microcode(adev);
1486 /* RLC autoload sequence 4 init IMU fw */
1487 if (adev->gfx.imu.funcs->setup_imu)
1488 adev->gfx.imu.funcs->setup_imu(adev);
1489 if (adev->gfx.imu.funcs->start_imu)
1490 adev->gfx.imu.funcs->start_imu(adev);
1491
1492 /* RLC autoload sequence 5 disable gpa mode */
1493 gfx_v11_0_disable_gpa_mode(adev);
1494
1495 return 0;
1496 }
1497
gfx_v11_0_alloc_ip_dump(struct amdgpu_device * adev)1498 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev)
1499 {
1500 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
1501 uint32_t *ptr;
1502 uint32_t inst;
1503
1504 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL);
1505 if (!ptr) {
1506 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1507 adev->gfx.ip_dump_core = NULL;
1508 } else {
1509 adev->gfx.ip_dump_core = ptr;
1510 }
1511
1512 /* Allocate memory for compute queue registers for all the instances */
1513 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
1514 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1515 adev->gfx.mec.num_queue_per_pipe;
1516
1517 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1518 if (!ptr) {
1519 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1520 adev->gfx.ip_dump_compute_queues = NULL;
1521 } else {
1522 adev->gfx.ip_dump_compute_queues = ptr;
1523 }
1524
1525 /* Allocate memory for gfx queue registers for all the instances */
1526 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
1527 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me *
1528 adev->gfx.me.num_queue_per_pipe;
1529
1530 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL);
1531 if (!ptr) {
1532 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n");
1533 adev->gfx.ip_dump_gfx_queues = NULL;
1534 } else {
1535 adev->gfx.ip_dump_gfx_queues = ptr;
1536 }
1537 }
1538
gfx_v11_0_sw_init(void * handle)1539 static int gfx_v11_0_sw_init(void *handle)
1540 {
1541 int i, j, k, r, ring_id = 0;
1542 int xcc_id = 0;
1543 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1544
1545 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1546 case IP_VERSION(11, 0, 0):
1547 case IP_VERSION(11, 0, 2):
1548 case IP_VERSION(11, 0, 3):
1549 adev->gfx.me.num_me = 1;
1550 adev->gfx.me.num_pipe_per_me = 1;
1551 adev->gfx.me.num_queue_per_pipe = 1;
1552 adev->gfx.mec.num_mec = 2;
1553 adev->gfx.mec.num_pipe_per_mec = 4;
1554 adev->gfx.mec.num_queue_per_pipe = 4;
1555 break;
1556 case IP_VERSION(11, 0, 1):
1557 case IP_VERSION(11, 0, 4):
1558 case IP_VERSION(11, 5, 0):
1559 case IP_VERSION(11, 5, 1):
1560 case IP_VERSION(11, 5, 2):
1561 adev->gfx.me.num_me = 1;
1562 adev->gfx.me.num_pipe_per_me = 1;
1563 adev->gfx.me.num_queue_per_pipe = 1;
1564 adev->gfx.mec.num_mec = 1;
1565 adev->gfx.mec.num_pipe_per_mec = 4;
1566 adev->gfx.mec.num_queue_per_pipe = 4;
1567 break;
1568 default:
1569 adev->gfx.me.num_me = 1;
1570 adev->gfx.me.num_pipe_per_me = 1;
1571 adev->gfx.me.num_queue_per_pipe = 1;
1572 adev->gfx.mec.num_mec = 1;
1573 adev->gfx.mec.num_pipe_per_mec = 4;
1574 adev->gfx.mec.num_queue_per_pipe = 8;
1575 break;
1576 }
1577
1578 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */
1579 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) &&
1580 amdgpu_sriov_is_pp_one_vf(adev))
1581 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG;
1582
1583 /* EOP Event */
1584 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1585 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT,
1586 &adev->gfx.eop_irq);
1587 if (r)
1588 return r;
1589
1590 /* Bad opcode Event */
1591 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1592 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR,
1593 &adev->gfx.bad_op_irq);
1594 if (r)
1595 return r;
1596
1597 /* Privileged reg */
1598 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1599 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT,
1600 &adev->gfx.priv_reg_irq);
1601 if (r)
1602 return r;
1603
1604 /* Privileged inst */
1605 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP,
1606 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT,
1607 &adev->gfx.priv_inst_irq);
1608 if (r)
1609 return r;
1610
1611 /* FED error */
1612 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1613 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT,
1614 &adev->gfx.rlc_gc_fed_irq);
1615 if (r)
1616 return r;
1617
1618 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1619
1620 gfx_v11_0_me_init(adev);
1621
1622 r = gfx_v11_0_rlc_init(adev);
1623 if (r) {
1624 DRM_ERROR("Failed to init rlc BOs!\n");
1625 return r;
1626 }
1627
1628 r = gfx_v11_0_mec_init(adev);
1629 if (r) {
1630 DRM_ERROR("Failed to init MEC BOs!\n");
1631 return r;
1632 }
1633
1634 /* set up the gfx ring */
1635 for (i = 0; i < adev->gfx.me.num_me; i++) {
1636 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1637 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1638 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1639 continue;
1640
1641 r = gfx_v11_0_gfx_ring_init(adev, ring_id,
1642 i, k, j);
1643 if (r)
1644 return r;
1645 ring_id++;
1646 }
1647 }
1648 }
1649
1650 ring_id = 0;
1651 /* set up the compute queues - allocate horizontally across pipes */
1652 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1653 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1654 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1655 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1656 k, j))
1657 continue;
1658
1659 r = gfx_v11_0_compute_ring_init(adev, ring_id,
1660 i, k, j);
1661 if (r)
1662 return r;
1663
1664 ring_id++;
1665 }
1666 }
1667 }
1668
1669 if (!adev->enable_mes_kiq) {
1670 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0);
1671 if (r) {
1672 DRM_ERROR("Failed to init KIQ BOs!\n");
1673 return r;
1674 }
1675
1676 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1677 if (r)
1678 return r;
1679 }
1680
1681 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0);
1682 if (r)
1683 return r;
1684
1685 /* allocate visible FB for rlc auto-loading fw */
1686 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1687 r = gfx_v11_0_rlc_autoload_buffer_init(adev);
1688 if (r)
1689 return r;
1690 }
1691
1692 r = gfx_v11_0_gpu_early_init(adev);
1693 if (r)
1694 return r;
1695
1696 if (amdgpu_gfx_ras_sw_init(adev)) {
1697 dev_err(adev->dev, "Failed to initialize gfx ras block!\n");
1698 return -EINVAL;
1699 }
1700
1701 gfx_v11_0_alloc_ip_dump(adev);
1702
1703 return 0;
1704 }
1705
gfx_v11_0_pfp_fini(struct amdgpu_device * adev)1706 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev)
1707 {
1708 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1709 &adev->gfx.pfp.pfp_fw_gpu_addr,
1710 (void **)&adev->gfx.pfp.pfp_fw_ptr);
1711
1712 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj,
1713 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
1714 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
1715 }
1716
gfx_v11_0_me_fini(struct amdgpu_device * adev)1717 static void gfx_v11_0_me_fini(struct amdgpu_device *adev)
1718 {
1719 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1720 &adev->gfx.me.me_fw_gpu_addr,
1721 (void **)&adev->gfx.me.me_fw_ptr);
1722
1723 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj,
1724 &adev->gfx.me.me_fw_data_gpu_addr,
1725 (void **)&adev->gfx.me.me_fw_data_ptr);
1726 }
1727
gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device * adev)1728 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1729 {
1730 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1731 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1732 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1733 }
1734
gfx_v11_0_sw_fini(void * handle)1735 static int gfx_v11_0_sw_fini(void *handle)
1736 {
1737 int i;
1738 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1739
1740 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1741 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1742 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1743 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1744
1745 amdgpu_gfx_mqd_sw_fini(adev, 0);
1746
1747 if (!adev->enable_mes_kiq) {
1748 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
1749 amdgpu_gfx_kiq_fini(adev, 0);
1750 }
1751
1752 gfx_v11_0_pfp_fini(adev);
1753 gfx_v11_0_me_fini(adev);
1754 gfx_v11_0_rlc_fini(adev);
1755 gfx_v11_0_mec_fini(adev);
1756
1757 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1758 gfx_v11_0_rlc_autoload_buffer_fini(adev);
1759
1760 gfx_v11_0_free_microcode(adev);
1761
1762 kfree(adev->gfx.ip_dump_core);
1763 kfree(adev->gfx.ip_dump_compute_queues);
1764 kfree(adev->gfx.ip_dump_gfx_queues);
1765
1766 return 0;
1767 }
1768
gfx_v11_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)1769 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1770 u32 sh_num, u32 instance, int xcc_id)
1771 {
1772 u32 data;
1773
1774 if (instance == 0xffffffff)
1775 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1776 INSTANCE_BROADCAST_WRITES, 1);
1777 else
1778 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1779 instance);
1780
1781 if (se_num == 0xffffffff)
1782 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1783 1);
1784 else
1785 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1786
1787 if (sh_num == 0xffffffff)
1788 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1789 1);
1790 else
1791 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1792
1793 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data);
1794 }
1795
gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device * adev)1796 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev)
1797 {
1798 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1799
1800 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE);
1801 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1802 CC_GC_SA_UNIT_DISABLE,
1803 SA_DISABLE);
1804 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE);
1805 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1806 GC_USER_SA_UNIT_DISABLE,
1807 SA_DISABLE);
1808 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1809 adev->gfx.config.max_shader_engines);
1810
1811 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1812 }
1813
gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device * adev)1814 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1815 {
1816 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1817 u32 rb_mask;
1818
1819 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE);
1820 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1821 CC_RB_BACKEND_DISABLE,
1822 BACKEND_DISABLE);
1823 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE);
1824 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1825 GC_USER_RB_BACKEND_DISABLE,
1826 BACKEND_DISABLE);
1827 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1828 adev->gfx.config.max_shader_engines);
1829
1830 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1831 }
1832
gfx_v11_0_setup_rb(struct amdgpu_device * adev)1833 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev)
1834 {
1835 u32 rb_bitmap_width_per_sa;
1836 u32 max_sa;
1837 u32 active_sa_bitmap;
1838 u32 global_active_rb_bitmap;
1839 u32 active_rb_bitmap = 0;
1840 u32 i;
1841
1842 /* query sa bitmap from SA_UNIT_DISABLE registers */
1843 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev);
1844 /* query rb bitmap from RB_BACKEND_DISABLE registers */
1845 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev);
1846
1847 /* generate active rb bitmap according to active sa bitmap */
1848 max_sa = adev->gfx.config.max_shader_engines *
1849 adev->gfx.config.max_sh_per_se;
1850 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1851 adev->gfx.config.max_sh_per_se;
1852 for (i = 0; i < max_sa; i++) {
1853 if (active_sa_bitmap & (1 << i))
1854 active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1855 }
1856
1857 active_rb_bitmap &= global_active_rb_bitmap;
1858 adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1859 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1860 }
1861
1862 #define DEFAULT_SH_MEM_BASES (0x6000)
1863 #define LDS_APP_BASE 0x1
1864 #define SCRATCH_APP_BASE 0x2
1865
gfx_v11_0_init_compute_vmid(struct amdgpu_device * adev)1866 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev)
1867 {
1868 int i;
1869 uint32_t sh_mem_bases;
1870 uint32_t data;
1871
1872 /*
1873 * Configure apertures:
1874 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
1875 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
1876 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
1877 */
1878 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) |
1879 SCRATCH_APP_BASE;
1880
1881 mutex_lock(&adev->srbm_mutex);
1882 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1883 soc21_grbm_select(adev, 0, 0, 0, i);
1884 /* CP and shaders */
1885 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1886 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases);
1887
1888 /* Enable trap for each kfd vmid. */
1889 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL);
1890 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1891 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data);
1892 }
1893 soc21_grbm_select(adev, 0, 0, 0, 0);
1894 mutex_unlock(&adev->srbm_mutex);
1895
1896 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
1897 acccess. These should be enabled by FW for target VMIDs. */
1898 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1899 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0);
1900 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0);
1901 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0);
1902 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0);
1903 }
1904 }
1905
gfx_v11_0_init_gds_vmid(struct amdgpu_device * adev)1906 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev)
1907 {
1908 int vmid;
1909
1910 /*
1911 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1912 * access. Compute VMIDs should be enabled by FW for target VMIDs,
1913 * the driver can enable them for graphics. VMID0 should maintain
1914 * access so that HWS firmware can save/restore entries.
1915 */
1916 for (vmid = 1; vmid < 16; vmid++) {
1917 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0);
1918 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0);
1919 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0);
1920 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0);
1921 }
1922 }
1923
gfx_v11_0_tcp_harvest(struct amdgpu_device * adev)1924 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev)
1925 {
1926 /* TODO: harvest feature to be added later. */
1927 }
1928
gfx_v11_0_get_tcc_info(struct amdgpu_device * adev)1929 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev)
1930 {
1931 /* TCCs are global (not instanced). */
1932 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) |
1933 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE);
1934
1935 adev->gfx.config.tcc_disabled_mask =
1936 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1937 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1938 }
1939
gfx_v11_0_constants_init(struct amdgpu_device * adev)1940 static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
1941 {
1942 u32 tmp;
1943 int i;
1944
1945 if (!amdgpu_sriov_vf(adev))
1946 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1947
1948 gfx_v11_0_setup_rb(adev);
1949 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info);
1950 gfx_v11_0_get_tcc_info(adev);
1951 adev->gfx.config.pa_sc_tile_steering_override = 0;
1952
1953 /* Set whether texture coordinate truncation is conformant. */
1954 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
1955 adev->gfx.config.ta_cntl2_truncate_coord_mode =
1956 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
1957
1958 /* XXX SH_MEM regs */
1959 /* where to put LDS, scratch, GPUVM in FSA64 space */
1960 mutex_lock(&adev->srbm_mutex);
1961 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1962 soc21_grbm_select(adev, 0, 0, 0, i);
1963 /* CP and shaders */
1964 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1965 if (i != 0) {
1966 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1967 (adev->gmc.private_aperture_start >> 48));
1968 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1969 (adev->gmc.shared_aperture_start >> 48));
1970 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp);
1971 }
1972 }
1973 soc21_grbm_select(adev, 0, 0, 0, 0);
1974
1975 mutex_unlock(&adev->srbm_mutex);
1976
1977 gfx_v11_0_init_compute_vmid(adev);
1978 gfx_v11_0_init_gds_vmid(adev);
1979 }
1980
gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device * adev,int me,int pipe)1981 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev,
1982 int me, int pipe)
1983 {
1984 if (me != 0)
1985 return 0;
1986
1987 switch (pipe) {
1988 case 0:
1989 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
1990 case 1:
1991 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
1992 default:
1993 return 0;
1994 }
1995 }
1996
gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device * adev,int me,int pipe)1997 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev,
1998 int me, int pipe)
1999 {
2000 /*
2001 * amdgpu controls only the first MEC. That's why this function only
2002 * handles the setting of interrupts for this specific MEC. All other
2003 * pipes' interrupts are set by amdkfd.
2004 */
2005 if (me != 1)
2006 return 0;
2007
2008 switch (pipe) {
2009 case 0:
2010 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
2011 case 1:
2012 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
2013 case 2:
2014 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
2015 case 3:
2016 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
2017 default:
2018 return 0;
2019 }
2020 }
2021
gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)2022 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2023 bool enable)
2024 {
2025 u32 tmp, cp_int_cntl_reg;
2026 int i, j;
2027
2028 if (amdgpu_sriov_vf(adev))
2029 return;
2030
2031 for (i = 0; i < adev->gfx.me.num_me; i++) {
2032 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
2033 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
2034
2035 if (cp_int_cntl_reg) {
2036 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
2037 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
2038 enable ? 1 : 0);
2039 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
2040 enable ? 1 : 0);
2041 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
2042 enable ? 1 : 0);
2043 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
2044 enable ? 1 : 0);
2045 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp);
2046 }
2047 }
2048 }
2049 }
2050
gfx_v11_0_init_csb(struct amdgpu_device * adev)2051 static int gfx_v11_0_init_csb(struct amdgpu_device *adev)
2052 {
2053 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2054
2055 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI,
2056 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2057 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO,
2058 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2059 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
2060
2061 return 0;
2062 }
2063
gfx_v11_0_rlc_stop(struct amdgpu_device * adev)2064 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev)
2065 {
2066 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL);
2067
2068 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
2069 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp);
2070 }
2071
gfx_v11_0_rlc_reset(struct amdgpu_device * adev)2072 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev)
2073 {
2074 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2075 udelay(50);
2076 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2077 udelay(50);
2078 }
2079
gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device * adev,bool enable)2080 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
2081 bool enable)
2082 {
2083 uint32_t rlc_pg_cntl;
2084
2085 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
2086
2087 if (!enable) {
2088 /* RLC_PG_CNTL[23] = 0 (default)
2089 * RLC will wait for handshake acks with SMU
2090 * GFXOFF will be enabled
2091 * RLC_PG_CNTL[23] = 1
2092 * RLC will not issue any message to SMU
2093 * hence no handshake between SMU & RLC
2094 * GFXOFF will be disabled
2095 */
2096 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2097 } else
2098 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
2099 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl);
2100 }
2101
gfx_v11_0_rlc_start(struct amdgpu_device * adev)2102 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev)
2103 {
2104 /* TODO: enable rlc & smu handshake until smu
2105 * and gfxoff feature works as expected */
2106 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
2107 gfx_v11_0_rlc_smu_handshake_cntl(adev, false);
2108
2109 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2110 udelay(50);
2111 }
2112
gfx_v11_0_rlc_enable_srm(struct amdgpu_device * adev)2113 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev)
2114 {
2115 uint32_t tmp;
2116
2117 /* enable Save Restore Machine */
2118 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL));
2119 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2120 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
2121 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp);
2122 }
2123
gfx_v11_0_load_rlcg_microcode(struct amdgpu_device * adev)2124 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev)
2125 {
2126 const struct rlc_firmware_header_v2_0 *hdr;
2127 const __le32 *fw_data;
2128 unsigned i, fw_size;
2129
2130 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2131 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2132 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2133 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2134
2135 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR,
2136 RLCG_UCODE_LOADING_START_ADDRESS);
2137
2138 for (i = 0; i < fw_size; i++)
2139 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA,
2140 le32_to_cpup(fw_data++));
2141
2142 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2143 }
2144
gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device * adev)2145 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev)
2146 {
2147 const struct rlc_firmware_header_v2_2 *hdr;
2148 const __le32 *fw_data;
2149 unsigned i, fw_size;
2150 u32 tmp;
2151
2152 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
2153
2154 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2155 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
2156 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
2157
2158 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0);
2159
2160 for (i = 0; i < fw_size; i++) {
2161 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2162 msleep(1);
2163 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA,
2164 le32_to_cpup(fw_data++));
2165 }
2166
2167 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2168
2169 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2170 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2171 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2172
2173 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0);
2174 for (i = 0; i < fw_size; i++) {
2175 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2176 msleep(1);
2177 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA,
2178 le32_to_cpup(fw_data++));
2179 }
2180
2181 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2182
2183 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL);
2184 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2185 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2186 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp);
2187 }
2188
gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device * adev)2189 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev)
2190 {
2191 const struct rlc_firmware_header_v2_3 *hdr;
2192 const __le32 *fw_data;
2193 unsigned i, fw_size;
2194 u32 tmp;
2195
2196 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
2197
2198 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2199 le32_to_cpu(hdr->rlcp_ucode_offset_bytes));
2200 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4;
2201
2202 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0);
2203
2204 for (i = 0; i < fw_size; i++) {
2205 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2206 msleep(1);
2207 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA,
2208 le32_to_cpup(fw_data++));
2209 }
2210
2211 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version);
2212
2213 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE);
2214 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
2215 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp);
2216
2217 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2218 le32_to_cpu(hdr->rlcv_ucode_offset_bytes));
2219 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4;
2220
2221 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0);
2222
2223 for (i = 0; i < fw_size; i++) {
2224 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2225 msleep(1);
2226 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA,
2227 le32_to_cpup(fw_data++));
2228 }
2229
2230 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version);
2231
2232 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL);
2233 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1);
2234 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp);
2235 }
2236
gfx_v11_0_rlc_load_microcode(struct amdgpu_device * adev)2237 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev)
2238 {
2239 const struct rlc_firmware_header_v2_0 *hdr;
2240 uint16_t version_major;
2241 uint16_t version_minor;
2242
2243 if (!adev->gfx.rlc_fw)
2244 return -EINVAL;
2245
2246 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2247 amdgpu_ucode_print_rlc_hdr(&hdr->header);
2248
2249 version_major = le16_to_cpu(hdr->header.header_version_major);
2250 version_minor = le16_to_cpu(hdr->header.header_version_minor);
2251
2252 if (version_major == 2) {
2253 gfx_v11_0_load_rlcg_microcode(adev);
2254 if (amdgpu_dpm == 1) {
2255 if (version_minor >= 2)
2256 gfx_v11_0_load_rlc_iram_dram_microcode(adev);
2257 if (version_minor == 3)
2258 gfx_v11_0_load_rlcp_rlcv_microcode(adev);
2259 }
2260
2261 return 0;
2262 }
2263
2264 return -EINVAL;
2265 }
2266
gfx_v11_0_rlc_resume(struct amdgpu_device * adev)2267 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev)
2268 {
2269 int r;
2270
2271 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2272 gfx_v11_0_init_csb(adev);
2273
2274 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2275 gfx_v11_0_rlc_enable_srm(adev);
2276 } else {
2277 if (amdgpu_sriov_vf(adev)) {
2278 gfx_v11_0_init_csb(adev);
2279 return 0;
2280 }
2281
2282 adev->gfx.rlc.funcs->stop(adev);
2283
2284 /* disable CG */
2285 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0);
2286
2287 /* disable PG */
2288 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0);
2289
2290 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2291 /* legacy rlc firmware loading */
2292 r = gfx_v11_0_rlc_load_microcode(adev);
2293 if (r)
2294 return r;
2295 }
2296
2297 gfx_v11_0_init_csb(adev);
2298
2299 adev->gfx.rlc.funcs->start(adev);
2300 }
2301 return 0;
2302 }
2303
gfx_v11_0_config_me_cache(struct amdgpu_device * adev,uint64_t addr)2304 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr)
2305 {
2306 uint32_t usec_timeout = 50000; /* wait for 50ms */
2307 uint32_t tmp;
2308 int i;
2309
2310 /* Trigger an invalidation of the L1 instruction caches */
2311 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2312 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2313 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2314
2315 /* Wait for invalidation complete */
2316 for (i = 0; i < usec_timeout; i++) {
2317 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2318 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2319 INVALIDATE_CACHE_COMPLETE))
2320 break;
2321 udelay(1);
2322 }
2323
2324 if (i >= usec_timeout) {
2325 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2326 return -EINVAL;
2327 }
2328
2329 if (amdgpu_emu_mode == 1)
2330 adev->hdp.funcs->flush_hdp(adev, NULL);
2331
2332 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2333 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2334 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2335 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2336 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2337 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2338
2339 /* Program me ucode address into intruction cache address register */
2340 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2341 lower_32_bits(addr) & 0xFFFFF000);
2342 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2343 upper_32_bits(addr));
2344
2345 return 0;
2346 }
2347
gfx_v11_0_config_pfp_cache(struct amdgpu_device * adev,uint64_t addr)2348 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr)
2349 {
2350 uint32_t usec_timeout = 50000; /* wait for 50ms */
2351 uint32_t tmp;
2352 int i;
2353
2354 /* Trigger an invalidation of the L1 instruction caches */
2355 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2356 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2357 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2358
2359 /* Wait for invalidation complete */
2360 for (i = 0; i < usec_timeout; i++) {
2361 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2362 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2363 INVALIDATE_CACHE_COMPLETE))
2364 break;
2365 udelay(1);
2366 }
2367
2368 if (i >= usec_timeout) {
2369 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2370 return -EINVAL;
2371 }
2372
2373 if (amdgpu_emu_mode == 1)
2374 adev->hdp.funcs->flush_hdp(adev, NULL);
2375
2376 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2377 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2378 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2379 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2380 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2381 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2382
2383 /* Program pfp ucode address into intruction cache address register */
2384 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2385 lower_32_bits(addr) & 0xFFFFF000);
2386 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2387 upper_32_bits(addr));
2388
2389 return 0;
2390 }
2391
gfx_v11_0_config_mec_cache(struct amdgpu_device * adev,uint64_t addr)2392 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr)
2393 {
2394 uint32_t usec_timeout = 50000; /* wait for 50ms */
2395 uint32_t tmp;
2396 int i;
2397
2398 /* Trigger an invalidation of the L1 instruction caches */
2399 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2400 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2401
2402 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2403
2404 /* Wait for invalidation complete */
2405 for (i = 0; i < usec_timeout; i++) {
2406 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2407 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2408 INVALIDATE_CACHE_COMPLETE))
2409 break;
2410 udelay(1);
2411 }
2412
2413 if (i >= usec_timeout) {
2414 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2415 return -EINVAL;
2416 }
2417
2418 if (amdgpu_emu_mode == 1)
2419 adev->hdp.funcs->flush_hdp(adev, NULL);
2420
2421 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2422 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2423 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2424 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2425 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2426
2427 /* Program mec1 ucode address into intruction cache address register */
2428 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO,
2429 lower_32_bits(addr) & 0xFFFFF000);
2430 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2431 upper_32_bits(addr));
2432
2433 return 0;
2434 }
2435
gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2436 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2437 {
2438 uint32_t usec_timeout = 50000; /* wait for 50ms */
2439 uint32_t tmp;
2440 unsigned i, pipe_id;
2441 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2442
2443 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2444 adev->gfx.pfp_fw->data;
2445
2446 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
2447 lower_32_bits(addr));
2448 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
2449 upper_32_bits(addr));
2450
2451 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
2452 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2453 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2454 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2455 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
2456
2457 /*
2458 * Programming any of the CP_PFP_IC_BASE registers
2459 * forces invalidation of the ME L1 I$. Wait for the
2460 * invalidation complete
2461 */
2462 for (i = 0; i < usec_timeout; i++) {
2463 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2464 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2465 INVALIDATE_CACHE_COMPLETE))
2466 break;
2467 udelay(1);
2468 }
2469
2470 if (i >= usec_timeout) {
2471 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2472 return -EINVAL;
2473 }
2474
2475 /* Prime the L1 instruction caches */
2476 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2477 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
2478 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
2479 /* Waiting for cache primed*/
2480 for (i = 0; i < usec_timeout; i++) {
2481 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
2482 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2483 ICACHE_PRIMED))
2484 break;
2485 udelay(1);
2486 }
2487
2488 if (i >= usec_timeout) {
2489 dev_err(adev->dev, "failed to prime instruction cache\n");
2490 return -EINVAL;
2491 }
2492
2493 mutex_lock(&adev->srbm_mutex);
2494 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2495 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2496 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2497 (pfp_hdr->ucode_start_addr_hi << 30) |
2498 (pfp_hdr->ucode_start_addr_lo >> 2));
2499 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2500 pfp_hdr->ucode_start_addr_hi >> 2);
2501
2502 /*
2503 * Program CP_ME_CNTL to reset given PIPE to take
2504 * effect of CP_PFP_PRGRM_CNTR_START.
2505 */
2506 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2507 if (pipe_id == 0)
2508 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2509 PFP_PIPE0_RESET, 1);
2510 else
2511 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2512 PFP_PIPE1_RESET, 1);
2513 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2514
2515 /* Clear pfp pipe0 reset bit. */
2516 if (pipe_id == 0)
2517 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2518 PFP_PIPE0_RESET, 0);
2519 else
2520 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2521 PFP_PIPE1_RESET, 0);
2522 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2523
2524 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
2525 lower_32_bits(addr2));
2526 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
2527 upper_32_bits(addr2));
2528 }
2529 soc21_grbm_select(adev, 0, 0, 0, 0);
2530 mutex_unlock(&adev->srbm_mutex);
2531
2532 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2533 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2534 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2535 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2536
2537 /* Invalidate the data caches */
2538 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2539 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2540 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2541
2542 for (i = 0; i < usec_timeout; i++) {
2543 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2544 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2545 INVALIDATE_DCACHE_COMPLETE))
2546 break;
2547 udelay(1);
2548 }
2549
2550 if (i >= usec_timeout) {
2551 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2552 return -EINVAL;
2553 }
2554
2555 return 0;
2556 }
2557
gfx_v11_0_config_me_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2558 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2559 {
2560 uint32_t usec_timeout = 50000; /* wait for 50ms */
2561 uint32_t tmp;
2562 unsigned i, pipe_id;
2563 const struct gfx_firmware_header_v2_0 *me_hdr;
2564
2565 me_hdr = (const struct gfx_firmware_header_v2_0 *)
2566 adev->gfx.me_fw->data;
2567
2568 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
2569 lower_32_bits(addr));
2570 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
2571 upper_32_bits(addr));
2572
2573 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
2574 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2575 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2576 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2577 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
2578
2579 /*
2580 * Programming any of the CP_ME_IC_BASE registers
2581 * forces invalidation of the ME L1 I$. Wait for the
2582 * invalidation complete
2583 */
2584 for (i = 0; i < usec_timeout; i++) {
2585 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2586 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2587 INVALIDATE_CACHE_COMPLETE))
2588 break;
2589 udelay(1);
2590 }
2591
2592 if (i >= usec_timeout) {
2593 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2594 return -EINVAL;
2595 }
2596
2597 /* Prime the instruction caches */
2598 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2599 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
2600 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
2601
2602 /* Waiting for instruction cache primed*/
2603 for (i = 0; i < usec_timeout; i++) {
2604 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
2605 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2606 ICACHE_PRIMED))
2607 break;
2608 udelay(1);
2609 }
2610
2611 if (i >= usec_timeout) {
2612 dev_err(adev->dev, "failed to prime instruction cache\n");
2613 return -EINVAL;
2614 }
2615
2616 mutex_lock(&adev->srbm_mutex);
2617 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
2618 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2619 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2620 (me_hdr->ucode_start_addr_hi << 30) |
2621 (me_hdr->ucode_start_addr_lo >> 2) );
2622 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2623 me_hdr->ucode_start_addr_hi>>2);
2624
2625 /*
2626 * Program CP_ME_CNTL to reset given PIPE to take
2627 * effect of CP_PFP_PRGRM_CNTR_START.
2628 */
2629 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2630 if (pipe_id == 0)
2631 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2632 ME_PIPE0_RESET, 1);
2633 else
2634 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2635 ME_PIPE1_RESET, 1);
2636 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2637
2638 /* Clear pfp pipe0 reset bit. */
2639 if (pipe_id == 0)
2640 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2641 ME_PIPE0_RESET, 0);
2642 else
2643 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
2644 ME_PIPE1_RESET, 0);
2645 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2646
2647 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
2648 lower_32_bits(addr2));
2649 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
2650 upper_32_bits(addr2));
2651 }
2652 soc21_grbm_select(adev, 0, 0, 0, 0);
2653 mutex_unlock(&adev->srbm_mutex);
2654
2655 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
2656 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
2657 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
2658 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
2659
2660 /* Invalidate the data caches */
2661 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2662 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2663 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
2664
2665 for (i = 0; i < usec_timeout; i++) {
2666 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
2667 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
2668 INVALIDATE_DCACHE_COMPLETE))
2669 break;
2670 udelay(1);
2671 }
2672
2673 if (i >= usec_timeout) {
2674 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
2675 return -EINVAL;
2676 }
2677
2678 return 0;
2679 }
2680
gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device * adev,uint64_t addr,uint64_t addr2)2681 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2)
2682 {
2683 uint32_t usec_timeout = 50000; /* wait for 50ms */
2684 uint32_t tmp;
2685 unsigned i;
2686 const struct gfx_firmware_header_v2_0 *mec_hdr;
2687
2688 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2689 adev->gfx.mec_fw->data;
2690
2691 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
2692 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2693 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2694 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2695 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
2696
2697 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
2698 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2699 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2700 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
2701
2702 mutex_lock(&adev->srbm_mutex);
2703 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2704 soc21_grbm_select(adev, 1, i, 0, 0);
2705
2706 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2);
2707 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
2708 upper_32_bits(addr2));
2709
2710 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2711 mec_hdr->ucode_start_addr_lo >> 2 |
2712 mec_hdr->ucode_start_addr_hi << 30);
2713 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2714 mec_hdr->ucode_start_addr_hi >> 2);
2715
2716 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr);
2717 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
2718 upper_32_bits(addr));
2719 }
2720 mutex_unlock(&adev->srbm_mutex);
2721 soc21_grbm_select(adev, 0, 0, 0, 0);
2722
2723 /* Trigger an invalidation of the L1 instruction caches */
2724 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2725 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2726 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
2727
2728 /* Wait for invalidation complete */
2729 for (i = 0; i < usec_timeout; i++) {
2730 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
2731 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2732 INVALIDATE_DCACHE_COMPLETE))
2733 break;
2734 udelay(1);
2735 }
2736
2737 if (i >= usec_timeout) {
2738 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2739 return -EINVAL;
2740 }
2741
2742 /* Trigger an invalidation of the L1 instruction caches */
2743 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2744 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2745 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
2746
2747 /* Wait for invalidation complete */
2748 for (i = 0; i < usec_timeout; i++) {
2749 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
2750 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2751 INVALIDATE_CACHE_COMPLETE))
2752 break;
2753 udelay(1);
2754 }
2755
2756 if (i >= usec_timeout) {
2757 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2758 return -EINVAL;
2759 }
2760
2761 return 0;
2762 }
2763
gfx_v11_0_config_gfx_rs64(struct amdgpu_device * adev)2764 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
2765 {
2766 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2767 const struct gfx_firmware_header_v2_0 *me_hdr;
2768 const struct gfx_firmware_header_v2_0 *mec_hdr;
2769 uint32_t pipe_id, tmp;
2770
2771 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2772 adev->gfx.mec_fw->data;
2773 me_hdr = (const struct gfx_firmware_header_v2_0 *)
2774 adev->gfx.me_fw->data;
2775 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
2776 adev->gfx.pfp_fw->data;
2777
2778 /* config pfp program start addr */
2779 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2780 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2781 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
2782 (pfp_hdr->ucode_start_addr_hi << 30) |
2783 (pfp_hdr->ucode_start_addr_lo >> 2));
2784 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
2785 pfp_hdr->ucode_start_addr_hi >> 2);
2786 }
2787 soc21_grbm_select(adev, 0, 0, 0, 0);
2788
2789 /* reset pfp pipe */
2790 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2791 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1);
2792 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1);
2793 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2794
2795 /* clear pfp pipe reset */
2796 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0);
2797 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0);
2798 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2799
2800 /* config me program start addr */
2801 for (pipe_id = 0; pipe_id < 2; pipe_id++) {
2802 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
2803 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
2804 (me_hdr->ucode_start_addr_hi << 30) |
2805 (me_hdr->ucode_start_addr_lo >> 2) );
2806 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
2807 me_hdr->ucode_start_addr_hi>>2);
2808 }
2809 soc21_grbm_select(adev, 0, 0, 0, 0);
2810
2811 /* reset me pipe */
2812 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2813 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1);
2814 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1);
2815 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2816
2817 /* clear me pipe reset */
2818 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0);
2819 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0);
2820 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2821
2822 /* config mec program start addr */
2823 for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2824 soc21_grbm_select(adev, 1, pipe_id, 0, 0);
2825 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
2826 mec_hdr->ucode_start_addr_lo >> 2 |
2827 mec_hdr->ucode_start_addr_hi << 30);
2828 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2829 mec_hdr->ucode_start_addr_hi >> 2);
2830 }
2831 soc21_grbm_select(adev, 0, 0, 0, 0);
2832
2833 /* reset mec pipe */
2834 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
2835 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
2836 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
2837 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
2838 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
2839 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2840
2841 /* clear mec pipe reset */
2842 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
2843 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
2844 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
2845 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
2846 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
2847 }
2848
gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device * adev)2849 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2850 {
2851 uint32_t cp_status;
2852 uint32_t bootload_status;
2853 int i, r;
2854 uint64_t addr, addr2;
2855
2856 for (i = 0; i < adev->usec_timeout; i++) {
2857 cp_status = RREG32_SOC15(GC, 0, regCP_STAT);
2858
2859 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
2860 IP_VERSION(11, 0, 1) ||
2861 amdgpu_ip_version(adev, GC_HWIP, 0) ==
2862 IP_VERSION(11, 0, 4) ||
2863 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) ||
2864 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) ||
2865 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2))
2866 bootload_status = RREG32_SOC15(GC, 0,
2867 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1);
2868 else
2869 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS);
2870
2871 if ((cp_status == 0) &&
2872 (REG_GET_FIELD(bootload_status,
2873 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2874 break;
2875 }
2876 udelay(1);
2877 }
2878
2879 if (i >= adev->usec_timeout) {
2880 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2881 return -ETIMEDOUT;
2882 }
2883
2884 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2885 if (adev->gfx.rs64_enable) {
2886 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2887 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset;
2888 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2889 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset;
2890 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2);
2891 if (r)
2892 return r;
2893 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2894 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset;
2895 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2896 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset;
2897 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2);
2898 if (r)
2899 return r;
2900 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2901 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset;
2902 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr +
2903 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset;
2904 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2);
2905 if (r)
2906 return r;
2907 } else {
2908 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2909 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset;
2910 r = gfx_v11_0_config_me_cache(adev, addr);
2911 if (r)
2912 return r;
2913 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2914 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset;
2915 r = gfx_v11_0_config_pfp_cache(adev, addr);
2916 if (r)
2917 return r;
2918 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2919 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset;
2920 r = gfx_v11_0_config_mec_cache(adev, addr);
2921 if (r)
2922 return r;
2923 }
2924 }
2925
2926 return 0;
2927 }
2928
gfx_v11_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)2929 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2930 {
2931 int i;
2932 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
2933
2934 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2935 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2936 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
2937
2938 for (i = 0; i < adev->usec_timeout; i++) {
2939 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0)
2940 break;
2941 udelay(1);
2942 }
2943
2944 if (i >= adev->usec_timeout)
2945 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2946
2947 return 0;
2948 }
2949
gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device * adev)2950 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2951 {
2952 int r;
2953 const struct gfx_firmware_header_v1_0 *pfp_hdr;
2954 const __le32 *fw_data;
2955 unsigned i, fw_size;
2956
2957 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2958 adev->gfx.pfp_fw->data;
2959
2960 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2961
2962 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2963 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2964 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
2965
2966 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
2967 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2968 &adev->gfx.pfp.pfp_fw_obj,
2969 &adev->gfx.pfp.pfp_fw_gpu_addr,
2970 (void **)&adev->gfx.pfp.pfp_fw_ptr);
2971 if (r) {
2972 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
2973 gfx_v11_0_pfp_fini(adev);
2974 return r;
2975 }
2976
2977 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
2978
2979 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2980 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2981
2982 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr);
2983
2984 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0);
2985
2986 for (i = 0; i < pfp_hdr->jt_size; i++)
2987 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA,
2988 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i));
2989
2990 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2991
2992 return 0;
2993 }
2994
gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device * adev)2995 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev)
2996 {
2997 int r;
2998 const struct gfx_firmware_header_v2_0 *pfp_hdr;
2999 const __le32 *fw_ucode, *fw_data;
3000 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3001 uint32_t tmp;
3002 uint32_t usec_timeout = 50000; /* wait for 50ms */
3003
3004 pfp_hdr = (const struct gfx_firmware_header_v2_0 *)
3005 adev->gfx.pfp_fw->data;
3006
3007 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3008
3009 /* instruction */
3010 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data +
3011 le32_to_cpu(pfp_hdr->ucode_offset_bytes));
3012 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes);
3013 /* data */
3014 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
3015 le32_to_cpu(pfp_hdr->data_offset_bytes));
3016 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes);
3017
3018 /* 64kb align */
3019 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3020 64 * 1024,
3021 AMDGPU_GEM_DOMAIN_VRAM |
3022 AMDGPU_GEM_DOMAIN_GTT,
3023 &adev->gfx.pfp.pfp_fw_obj,
3024 &adev->gfx.pfp.pfp_fw_gpu_addr,
3025 (void **)&adev->gfx.pfp.pfp_fw_ptr);
3026 if (r) {
3027 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r);
3028 gfx_v11_0_pfp_fini(adev);
3029 return r;
3030 }
3031
3032 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3033 64 * 1024,
3034 AMDGPU_GEM_DOMAIN_VRAM |
3035 AMDGPU_GEM_DOMAIN_GTT,
3036 &adev->gfx.pfp.pfp_fw_data_obj,
3037 &adev->gfx.pfp.pfp_fw_data_gpu_addr,
3038 (void **)&adev->gfx.pfp.pfp_fw_data_ptr);
3039 if (r) {
3040 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r);
3041 gfx_v11_0_pfp_fini(adev);
3042 return r;
3043 }
3044
3045 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size);
3046 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size);
3047
3048 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
3049 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj);
3050 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
3051 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj);
3052
3053 if (amdgpu_emu_mode == 1)
3054 adev->hdp.funcs->flush_hdp(adev, NULL);
3055
3056 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO,
3057 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3058 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI,
3059 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
3060
3061 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL);
3062 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
3063 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
3064 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
3065 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp);
3066
3067 /*
3068 * Programming any of the CP_PFP_IC_BASE registers
3069 * forces invalidation of the ME L1 I$. Wait for the
3070 * invalidation complete
3071 */
3072 for (i = 0; i < usec_timeout; i++) {
3073 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3074 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3075 INVALIDATE_CACHE_COMPLETE))
3076 break;
3077 udelay(1);
3078 }
3079
3080 if (i >= usec_timeout) {
3081 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3082 return -EINVAL;
3083 }
3084
3085 /* Prime the L1 instruction caches */
3086 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3087 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1);
3088 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp);
3089 /* Waiting for cache primed*/
3090 for (i = 0; i < usec_timeout; i++) {
3091 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL);
3092 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
3093 ICACHE_PRIMED))
3094 break;
3095 udelay(1);
3096 }
3097
3098 if (i >= usec_timeout) {
3099 dev_err(adev->dev, "failed to prime instruction cache\n");
3100 return -EINVAL;
3101 }
3102
3103 mutex_lock(&adev->srbm_mutex);
3104 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3105 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3106 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START,
3107 (pfp_hdr->ucode_start_addr_hi << 30) |
3108 (pfp_hdr->ucode_start_addr_lo >> 2) );
3109 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI,
3110 pfp_hdr->ucode_start_addr_hi>>2);
3111
3112 /*
3113 * Program CP_ME_CNTL to reset given PIPE to take
3114 * effect of CP_PFP_PRGRM_CNTR_START.
3115 */
3116 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3117 if (pipe_id == 0)
3118 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3119 PFP_PIPE0_RESET, 1);
3120 else
3121 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3122 PFP_PIPE1_RESET, 1);
3123 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3124
3125 /* Clear pfp pipe0 reset bit. */
3126 if (pipe_id == 0)
3127 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3128 PFP_PIPE0_RESET, 0);
3129 else
3130 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3131 PFP_PIPE1_RESET, 0);
3132 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3133
3134 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO,
3135 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3136 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI,
3137 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr));
3138 }
3139 soc21_grbm_select(adev, 0, 0, 0, 0);
3140 mutex_unlock(&adev->srbm_mutex);
3141
3142 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3143 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3144 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3145 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3146
3147 /* Invalidate the data caches */
3148 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3149 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3150 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3151
3152 for (i = 0; i < usec_timeout; i++) {
3153 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3154 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3155 INVALIDATE_DCACHE_COMPLETE))
3156 break;
3157 udelay(1);
3158 }
3159
3160 if (i >= usec_timeout) {
3161 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3162 return -EINVAL;
3163 }
3164
3165 return 0;
3166 }
3167
gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device * adev)3168 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
3169 {
3170 int r;
3171 const struct gfx_firmware_header_v1_0 *me_hdr;
3172 const __le32 *fw_data;
3173 unsigned i, fw_size;
3174
3175 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3176 adev->gfx.me_fw->data;
3177
3178 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3179
3180 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3181 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3182 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
3183
3184 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
3185 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3186 &adev->gfx.me.me_fw_obj,
3187 &adev->gfx.me.me_fw_gpu_addr,
3188 (void **)&adev->gfx.me.me_fw_ptr);
3189 if (r) {
3190 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
3191 gfx_v11_0_me_fini(adev);
3192 return r;
3193 }
3194
3195 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
3196
3197 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3198 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3199
3200 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr);
3201
3202 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0);
3203
3204 for (i = 0; i < me_hdr->jt_size; i++)
3205 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA,
3206 le32_to_cpup(fw_data + me_hdr->jt_offset + i));
3207
3208 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version);
3209
3210 return 0;
3211 }
3212
gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device * adev)3213 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev)
3214 {
3215 int r;
3216 const struct gfx_firmware_header_v2_0 *me_hdr;
3217 const __le32 *fw_ucode, *fw_data;
3218 unsigned i, pipe_id, fw_ucode_size, fw_data_size;
3219 uint32_t tmp;
3220 uint32_t usec_timeout = 50000; /* wait for 50ms */
3221
3222 me_hdr = (const struct gfx_firmware_header_v2_0 *)
3223 adev->gfx.me_fw->data;
3224
3225 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3226
3227 /* instruction */
3228 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data +
3229 le32_to_cpu(me_hdr->ucode_offset_bytes));
3230 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes);
3231 /* data */
3232 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
3233 le32_to_cpu(me_hdr->data_offset_bytes));
3234 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes);
3235
3236 /* 64kb align*/
3237 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3238 64 * 1024,
3239 AMDGPU_GEM_DOMAIN_VRAM |
3240 AMDGPU_GEM_DOMAIN_GTT,
3241 &adev->gfx.me.me_fw_obj,
3242 &adev->gfx.me.me_fw_gpu_addr,
3243 (void **)&adev->gfx.me.me_fw_ptr);
3244 if (r) {
3245 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r);
3246 gfx_v11_0_me_fini(adev);
3247 return r;
3248 }
3249
3250 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3251 64 * 1024,
3252 AMDGPU_GEM_DOMAIN_VRAM |
3253 AMDGPU_GEM_DOMAIN_GTT,
3254 &adev->gfx.me.me_fw_data_obj,
3255 &adev->gfx.me.me_fw_data_gpu_addr,
3256 (void **)&adev->gfx.me.me_fw_data_ptr);
3257 if (r) {
3258 dev_err(adev->dev, "(%d) failed to create me data bo\n", r);
3259 gfx_v11_0_pfp_fini(adev);
3260 return r;
3261 }
3262
3263 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size);
3264 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size);
3265
3266 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
3267 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj);
3268 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
3269 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj);
3270
3271 if (amdgpu_emu_mode == 1)
3272 adev->hdp.funcs->flush_hdp(adev, NULL);
3273
3274 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO,
3275 lower_32_bits(adev->gfx.me.me_fw_gpu_addr));
3276 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI,
3277 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
3278
3279 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL);
3280 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
3281 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
3282 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
3283 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp);
3284
3285 /*
3286 * Programming any of the CP_ME_IC_BASE registers
3287 * forces invalidation of the ME L1 I$. Wait for the
3288 * invalidation complete
3289 */
3290 for (i = 0; i < usec_timeout; i++) {
3291 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3292 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3293 INVALIDATE_CACHE_COMPLETE))
3294 break;
3295 udelay(1);
3296 }
3297
3298 if (i >= usec_timeout) {
3299 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3300 return -EINVAL;
3301 }
3302
3303 /* Prime the instruction caches */
3304 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3305 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1);
3306 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp);
3307
3308 /* Waiting for instruction cache primed*/
3309 for (i = 0; i < usec_timeout; i++) {
3310 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL);
3311 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
3312 ICACHE_PRIMED))
3313 break;
3314 udelay(1);
3315 }
3316
3317 if (i >= usec_timeout) {
3318 dev_err(adev->dev, "failed to prime instruction cache\n");
3319 return -EINVAL;
3320 }
3321
3322 mutex_lock(&adev->srbm_mutex);
3323 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) {
3324 soc21_grbm_select(adev, 0, pipe_id, 0, 0);
3325 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START,
3326 (me_hdr->ucode_start_addr_hi << 30) |
3327 (me_hdr->ucode_start_addr_lo >> 2) );
3328 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI,
3329 me_hdr->ucode_start_addr_hi>>2);
3330
3331 /*
3332 * Program CP_ME_CNTL to reset given PIPE to take
3333 * effect of CP_PFP_PRGRM_CNTR_START.
3334 */
3335 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL);
3336 if (pipe_id == 0)
3337 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3338 ME_PIPE0_RESET, 1);
3339 else
3340 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3341 ME_PIPE1_RESET, 1);
3342 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3343
3344 /* Clear pfp pipe0 reset bit. */
3345 if (pipe_id == 0)
3346 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3347 ME_PIPE0_RESET, 0);
3348 else
3349 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL,
3350 ME_PIPE1_RESET, 0);
3351 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp);
3352
3353 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO,
3354 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3355 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI,
3356 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr));
3357 }
3358 soc21_grbm_select(adev, 0, 0, 0, 0);
3359 mutex_unlock(&adev->srbm_mutex);
3360
3361 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL);
3362 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0);
3363 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0);
3364 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp);
3365
3366 /* Invalidate the data caches */
3367 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3368 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3369 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp);
3370
3371 for (i = 0; i < usec_timeout; i++) {
3372 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL);
3373 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL,
3374 INVALIDATE_DCACHE_COMPLETE))
3375 break;
3376 udelay(1);
3377 }
3378
3379 if (i >= usec_timeout) {
3380 dev_err(adev->dev, "failed to invalidate RS64 data cache\n");
3381 return -EINVAL;
3382 }
3383
3384 return 0;
3385 }
3386
gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device * adev)3387 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3388 {
3389 int r;
3390
3391 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw)
3392 return -EINVAL;
3393
3394 gfx_v11_0_cp_gfx_enable(adev, false);
3395
3396 if (adev->gfx.rs64_enable)
3397 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev);
3398 else
3399 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev);
3400 if (r) {
3401 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
3402 return r;
3403 }
3404
3405 if (adev->gfx.rs64_enable)
3406 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev);
3407 else
3408 r = gfx_v11_0_cp_gfx_load_me_microcode(adev);
3409 if (r) {
3410 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
3411 return r;
3412 }
3413
3414 return 0;
3415 }
3416
gfx_v11_0_cp_gfx_start(struct amdgpu_device * adev)3417 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev)
3418 {
3419 struct amdgpu_ring *ring;
3420 const struct cs_section_def *sect = NULL;
3421 const struct cs_extent_def *ext = NULL;
3422 int r, i;
3423 int ctx_reg_offset;
3424
3425 /* init the CP */
3426 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT,
3427 adev->gfx.config.max_hw_contexts - 1);
3428 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1);
3429
3430 if (!amdgpu_async_gfx_ring)
3431 gfx_v11_0_cp_gfx_enable(adev, true);
3432
3433 ring = &adev->gfx.gfx_ring[0];
3434 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev));
3435 if (r) {
3436 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3437 return r;
3438 }
3439
3440 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3441 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3442
3443 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3444 amdgpu_ring_write(ring, 0x80000000);
3445 amdgpu_ring_write(ring, 0x80000000);
3446
3447 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) {
3448 for (ext = sect->section; ext->extent != NULL; ++ext) {
3449 if (sect->id == SECT_CONTEXT) {
3450 amdgpu_ring_write(ring,
3451 PACKET3(PACKET3_SET_CONTEXT_REG,
3452 ext->reg_count));
3453 amdgpu_ring_write(ring, ext->reg_index -
3454 PACKET3_SET_CONTEXT_REG_START);
3455 for (i = 0; i < ext->reg_count; i++)
3456 amdgpu_ring_write(ring, ext->extent[i]);
3457 }
3458 }
3459 }
3460
3461 ctx_reg_offset =
3462 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
3463 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
3464 amdgpu_ring_write(ring, ctx_reg_offset);
3465 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
3466
3467 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3468 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3469
3470 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3471 amdgpu_ring_write(ring, 0);
3472
3473 amdgpu_ring_commit(ring);
3474
3475 /* submit cs packet to copy state 0 to next available state */
3476 if (adev->gfx.num_gfx_rings > 1) {
3477 /* maximum supported gfx ring is 2 */
3478 ring = &adev->gfx.gfx_ring[1];
3479 r = amdgpu_ring_alloc(ring, 2);
3480 if (r) {
3481 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3482 return r;
3483 }
3484
3485 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3486 amdgpu_ring_write(ring, 0);
3487
3488 amdgpu_ring_commit(ring);
3489 }
3490 return 0;
3491 }
3492
gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device * adev,CP_PIPE_ID pipe)3493 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
3494 CP_PIPE_ID pipe)
3495 {
3496 u32 tmp;
3497
3498 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL);
3499 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
3500
3501 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp);
3502 }
3503
gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device * adev,struct amdgpu_ring * ring)3504 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
3505 struct amdgpu_ring *ring)
3506 {
3507 u32 tmp;
3508
3509 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3510 if (ring->use_doorbell) {
3511 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3512 DOORBELL_OFFSET, ring->doorbell_index);
3513 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3514 DOORBELL_EN, 1);
3515 } else {
3516 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3517 DOORBELL_EN, 0);
3518 }
3519 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp);
3520
3521 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3522 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3523 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp);
3524
3525 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3526 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3527 }
3528
gfx_v11_0_cp_gfx_resume(struct amdgpu_device * adev)3529 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev)
3530 {
3531 struct amdgpu_ring *ring;
3532 u32 tmp;
3533 u32 rb_bufsz;
3534 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3535
3536 /* Set the write pointer delay */
3537 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0);
3538
3539 /* set the RB to use vmid 0 */
3540 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0);
3541
3542 /* Init gfx ring 0 for pipe 0 */
3543 mutex_lock(&adev->srbm_mutex);
3544 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3545
3546 /* Set ring buffer size */
3547 ring = &adev->gfx.gfx_ring[0];
3548 rb_bufsz = order_base_2(ring->ring_size / 8);
3549 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3550 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3551 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3552
3553 /* Initialize the ring buffer's write pointers */
3554 ring->wptr = 0;
3555 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr));
3556 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3557
3558 /* set the wb address wether it's enabled or not */
3559 rptr_addr = ring->rptr_gpu_addr;
3560 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3561 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3562 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3563
3564 wptr_gpu_addr = ring->wptr_gpu_addr;
3565 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3566 lower_32_bits(wptr_gpu_addr));
3567 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3568 upper_32_bits(wptr_gpu_addr));
3569
3570 mdelay(1);
3571 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp);
3572
3573 rb_addr = ring->gpu_addr >> 8;
3574 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr);
3575 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3576
3577 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1);
3578
3579 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3580 mutex_unlock(&adev->srbm_mutex);
3581
3582 /* Init gfx ring 1 for pipe 1 */
3583 if (adev->gfx.num_gfx_rings > 1) {
3584 mutex_lock(&adev->srbm_mutex);
3585 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
3586 /* maximum supported gfx ring is 2 */
3587 ring = &adev->gfx.gfx_ring[1];
3588 rb_bufsz = order_base_2(ring->ring_size / 8);
3589 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
3590 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
3591 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3592 /* Initialize the ring buffer's write pointers */
3593 ring->wptr = 0;
3594 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr));
3595 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
3596 /* Set the wb address wether it's enabled or not */
3597 rptr_addr = ring->rptr_gpu_addr;
3598 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
3599 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
3600 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3601 wptr_gpu_addr = ring->wptr_gpu_addr;
3602 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO,
3603 lower_32_bits(wptr_gpu_addr));
3604 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI,
3605 upper_32_bits(wptr_gpu_addr));
3606
3607 mdelay(1);
3608 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp);
3609
3610 rb_addr = ring->gpu_addr >> 8;
3611 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr);
3612 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr));
3613 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1);
3614
3615 gfx_v11_0_cp_gfx_set_doorbell(adev, ring);
3616 mutex_unlock(&adev->srbm_mutex);
3617 }
3618 /* Switch to pipe 0 */
3619 mutex_lock(&adev->srbm_mutex);
3620 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
3621 mutex_unlock(&adev->srbm_mutex);
3622
3623 /* start the ring */
3624 gfx_v11_0_cp_gfx_start(adev);
3625
3626 return 0;
3627 }
3628
gfx_v11_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)3629 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3630 {
3631 u32 data;
3632
3633 if (adev->gfx.rs64_enable) {
3634 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
3635 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
3636 enable ? 0 : 1);
3637 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
3638 enable ? 0 : 1);
3639 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
3640 enable ? 0 : 1);
3641 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
3642 enable ? 0 : 1);
3643 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
3644 enable ? 0 : 1);
3645 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
3646 enable ? 1 : 0);
3647 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
3648 enable ? 1 : 0);
3649 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
3650 enable ? 1 : 0);
3651 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
3652 enable ? 1 : 0);
3653 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
3654 enable ? 0 : 1);
3655 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data);
3656 } else {
3657 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
3658
3659 if (enable) {
3660 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0);
3661 if (!adev->enable_mes_kiq)
3662 data = REG_SET_FIELD(data, CP_MEC_CNTL,
3663 MEC_ME2_HALT, 0);
3664 } else {
3665 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1);
3666 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1);
3667 }
3668 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data);
3669 }
3670
3671 udelay(50);
3672 }
3673
gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device * adev)3674 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3675 {
3676 const struct gfx_firmware_header_v1_0 *mec_hdr;
3677 const __le32 *fw_data;
3678 unsigned i, fw_size;
3679 u32 *fw = NULL;
3680 int r;
3681
3682 if (!adev->gfx.mec_fw)
3683 return -EINVAL;
3684
3685 gfx_v11_0_cp_compute_enable(adev, false);
3686
3687 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3688 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3689
3690 fw_data = (const __le32 *)
3691 (adev->gfx.mec_fw->data +
3692 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3693 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
3694
3695 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
3696 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
3697 &adev->gfx.mec.mec_fw_obj,
3698 &adev->gfx.mec.mec_fw_gpu_addr,
3699 (void **)&fw);
3700 if (r) {
3701 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
3702 gfx_v11_0_mec_fini(adev);
3703 return r;
3704 }
3705
3706 memcpy(fw, fw_data, fw_size);
3707
3708 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3709 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3710
3711 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr);
3712
3713 /* MEC1 */
3714 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0);
3715
3716 for (i = 0; i < mec_hdr->jt_size; i++)
3717 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA,
3718 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3719
3720 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
3721
3722 return 0;
3723 }
3724
gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device * adev)3725 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev)
3726 {
3727 const struct gfx_firmware_header_v2_0 *mec_hdr;
3728 const __le32 *fw_ucode, *fw_data;
3729 u32 tmp, fw_ucode_size, fw_data_size;
3730 u32 i, usec_timeout = 50000; /* Wait for 50 ms */
3731 u32 *fw_ucode_ptr, *fw_data_ptr;
3732 int r;
3733
3734 if (!adev->gfx.mec_fw)
3735 return -EINVAL;
3736
3737 gfx_v11_0_cp_compute_enable(adev, false);
3738
3739 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
3740 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3741
3742 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
3743 le32_to_cpu(mec_hdr->ucode_offset_bytes));
3744 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
3745
3746 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
3747 le32_to_cpu(mec_hdr->data_offset_bytes));
3748 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
3749
3750 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
3751 64 * 1024,
3752 AMDGPU_GEM_DOMAIN_VRAM |
3753 AMDGPU_GEM_DOMAIN_GTT,
3754 &adev->gfx.mec.mec_fw_obj,
3755 &adev->gfx.mec.mec_fw_gpu_addr,
3756 (void **)&fw_ucode_ptr);
3757 if (r) {
3758 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3759 gfx_v11_0_mec_fini(adev);
3760 return r;
3761 }
3762
3763 r = amdgpu_bo_create_reserved(adev, fw_data_size,
3764 64 * 1024,
3765 AMDGPU_GEM_DOMAIN_VRAM |
3766 AMDGPU_GEM_DOMAIN_GTT,
3767 &adev->gfx.mec.mec_fw_data_obj,
3768 &adev->gfx.mec.mec_fw_data_gpu_addr,
3769 (void **)&fw_data_ptr);
3770 if (r) {
3771 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
3772 gfx_v11_0_mec_fini(adev);
3773 return r;
3774 }
3775
3776 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
3777 memcpy(fw_data_ptr, fw_data, fw_data_size);
3778
3779 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
3780 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
3781 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
3782 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
3783
3784 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL);
3785 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3786 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
3787 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3788 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp);
3789
3790 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL);
3791 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
3792 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
3793 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp);
3794
3795 mutex_lock(&adev->srbm_mutex);
3796 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3797 soc21_grbm_select(adev, 1, i, 0, 0);
3798
3799 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr);
3800 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI,
3801 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr));
3802
3803 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START,
3804 mec_hdr->ucode_start_addr_lo >> 2 |
3805 mec_hdr->ucode_start_addr_hi << 30);
3806 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI,
3807 mec_hdr->ucode_start_addr_hi >> 2);
3808
3809 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr);
3810 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI,
3811 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3812 }
3813 mutex_unlock(&adev->srbm_mutex);
3814 soc21_grbm_select(adev, 0, 0, 0, 0);
3815
3816 /* Trigger an invalidation of the L1 instruction caches */
3817 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3818 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
3819 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp);
3820
3821 /* Wait for invalidation complete */
3822 for (i = 0; i < usec_timeout; i++) {
3823 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL);
3824 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
3825 INVALIDATE_DCACHE_COMPLETE))
3826 break;
3827 udelay(1);
3828 }
3829
3830 if (i >= usec_timeout) {
3831 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3832 return -EINVAL;
3833 }
3834
3835 /* Trigger an invalidation of the L1 instruction caches */
3836 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3837 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
3838 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp);
3839
3840 /* Wait for invalidation complete */
3841 for (i = 0; i < usec_timeout; i++) {
3842 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL);
3843 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
3844 INVALIDATE_CACHE_COMPLETE))
3845 break;
3846 udelay(1);
3847 }
3848
3849 if (i >= usec_timeout) {
3850 dev_err(adev->dev, "failed to invalidate instruction cache\n");
3851 return -EINVAL;
3852 }
3853
3854 return 0;
3855 }
3856
gfx_v11_0_kiq_setting(struct amdgpu_ring * ring)3857 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring)
3858 {
3859 uint32_t tmp;
3860 struct amdgpu_device *adev = ring->adev;
3861
3862 /* tell RLC which is KIQ queue */
3863 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
3864 tmp &= 0xffffff00;
3865 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3866 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3867 tmp |= 0x80;
3868 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
3869 }
3870
gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device * adev)3871 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev)
3872 {
3873 /* set graphics engine doorbell range */
3874 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER,
3875 (adev->doorbell_index.gfx_ring0 * 2) << 2);
3876 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER,
3877 (adev->doorbell_index.gfx_userqueue_end * 2) << 2);
3878
3879 /* set compute engine doorbell range */
3880 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
3881 (adev->doorbell_index.kiq * 2) << 2);
3882 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
3883 (adev->doorbell_index.userqueue_end * 2) << 2);
3884 }
3885
gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device * adev,struct v11_gfx_mqd * mqd,struct amdgpu_mqd_prop * prop)3886 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev,
3887 struct v11_gfx_mqd *mqd,
3888 struct amdgpu_mqd_prop *prop)
3889 {
3890 bool priority = 0;
3891 u32 tmp;
3892
3893 /* set up default queue priority level
3894 * 0x0 = low priority, 0x1 = high priority
3895 */
3896 if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH)
3897 priority = 1;
3898
3899 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY);
3900 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority);
3901 mqd->cp_gfx_hqd_queue_priority = tmp;
3902 }
3903
gfx_v11_0_gfx_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)3904 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
3905 struct amdgpu_mqd_prop *prop)
3906 {
3907 struct v11_gfx_mqd *mqd = m;
3908 uint64_t hqd_gpu_addr, wb_gpu_addr;
3909 uint32_t tmp;
3910 uint32_t rb_bufsz;
3911
3912 /* set up gfx hqd wptr */
3913 mqd->cp_gfx_hqd_wptr = 0;
3914 mqd->cp_gfx_hqd_wptr_hi = 0;
3915
3916 /* set the pointer to the MQD */
3917 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc;
3918 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
3919
3920 /* set up mqd control */
3921 tmp = RREG32_SOC15(GC, 0, regCP_GFX_MQD_CONTROL);
3922 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3923 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3924 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3925 mqd->cp_gfx_mqd_control = tmp;
3926
3927 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */
3928 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_VMID);
3929 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3930 mqd->cp_gfx_hqd_vmid = 0;
3931
3932 /* set up gfx queue priority */
3933 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop);
3934
3935 /* set up time quantum */
3936 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_QUANTUM);
3937 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3938 mqd->cp_gfx_hqd_quantum = tmp;
3939
3940 /* set up gfx hqd base. this is similar as CP_RB_BASE */
3941 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
3942 mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3943 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3944
3945 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */
3946 wb_gpu_addr = prop->rptr_gpu_addr;
3947 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3948 mqd->cp_gfx_hqd_rptr_addr_hi =
3949 upper_32_bits(wb_gpu_addr) & 0xffff;
3950
3951 /* set up rb_wptr_poll addr */
3952 wb_gpu_addr = prop->wptr_gpu_addr;
3953 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3954 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3955
3956 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */
3957 rb_bufsz = order_base_2(prop->queue_size / 4) - 1;
3958 tmp = RREG32_SOC15(GC, 0, regCP_GFX_HQD_CNTL);
3959 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3960 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3961 #ifdef __BIG_ENDIAN
3962 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3963 #endif
3964 mqd->cp_gfx_hqd_cntl = tmp;
3965
3966 /* set up cp_doorbell_control */
3967 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL);
3968 if (prop->use_doorbell) {
3969 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3970 DOORBELL_OFFSET, prop->doorbell_index);
3971 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3972 DOORBELL_EN, 1);
3973 } else
3974 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3975 DOORBELL_EN, 0);
3976 mqd->cp_rb_doorbell_control = tmp;
3977
3978 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3979 mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, regCP_GFX_HQD_RPTR);
3980
3981 /* active the queue */
3982 mqd->cp_gfx_hqd_active = 1;
3983
3984 return 0;
3985 }
3986
gfx_v11_0_kgq_init_queue(struct amdgpu_ring * ring,bool reset)3987 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset)
3988 {
3989 struct amdgpu_device *adev = ring->adev;
3990 struct v11_gfx_mqd *mqd = ring->mqd_ptr;
3991 int mqd_idx = ring - &adev->gfx.gfx_ring[0];
3992
3993 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
3994 memset((void *)mqd, 0, sizeof(*mqd));
3995 mutex_lock(&adev->srbm_mutex);
3996 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3997 amdgpu_ring_init_mqd(ring);
3998 soc21_grbm_select(adev, 0, 0, 0, 0);
3999 mutex_unlock(&adev->srbm_mutex);
4000 if (adev->gfx.me.mqd_backup[mqd_idx])
4001 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4002 } else {
4003 /* restore mqd with the backup copy */
4004 if (adev->gfx.me.mqd_backup[mqd_idx])
4005 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd));
4006 /* reset the ring */
4007 ring->wptr = 0;
4008 *ring->wptr_cpu_addr = 0;
4009 amdgpu_ring_clear_ring(ring);
4010 }
4011
4012 return 0;
4013 }
4014
gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device * adev)4015 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
4016 {
4017 int r, i;
4018 struct amdgpu_ring *ring;
4019
4020 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4021 ring = &adev->gfx.gfx_ring[i];
4022
4023 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4024 if (unlikely(r != 0))
4025 return r;
4026
4027 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4028 if (!r) {
4029 r = gfx_v11_0_kgq_init_queue(ring, false);
4030 amdgpu_bo_kunmap(ring->mqd_obj);
4031 ring->mqd_ptr = NULL;
4032 }
4033 amdgpu_bo_unreserve(ring->mqd_obj);
4034 if (r)
4035 return r;
4036 }
4037
4038 r = amdgpu_gfx_enable_kgq(adev, 0);
4039 if (r)
4040 return r;
4041
4042 return gfx_v11_0_cp_gfx_start(adev);
4043 }
4044
gfx_v11_0_compute_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)4045 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
4046 struct amdgpu_mqd_prop *prop)
4047 {
4048 struct v11_compute_mqd *mqd = m;
4049 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4050 uint32_t tmp;
4051
4052 mqd->header = 0xC0310800;
4053 mqd->compute_pipelinestat_enable = 0x00000001;
4054 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4055 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4056 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4057 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4058 mqd->compute_misc_reserved = 0x00000007;
4059
4060 eop_base_addr = prop->eop_gpu_addr >> 8;
4061 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4062 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4063
4064 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4065 tmp = RREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL);
4066 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4067 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1));
4068
4069 mqd->cp_hqd_eop_control = tmp;
4070
4071 /* enable doorbell? */
4072 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4073
4074 if (prop->use_doorbell) {
4075 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4076 DOORBELL_OFFSET, prop->doorbell_index);
4077 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4078 DOORBELL_EN, 1);
4079 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4080 DOORBELL_SOURCE, 0);
4081 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4082 DOORBELL_HIT, 0);
4083 } else {
4084 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4085 DOORBELL_EN, 0);
4086 }
4087
4088 mqd->cp_hqd_pq_doorbell_control = tmp;
4089
4090 /* disable the queue if it's active */
4091 mqd->cp_hqd_dequeue_request = 0;
4092 mqd->cp_hqd_pq_rptr = 0;
4093 mqd->cp_hqd_pq_wptr_lo = 0;
4094 mqd->cp_hqd_pq_wptr_hi = 0;
4095
4096 /* set the pointer to the MQD */
4097 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
4098 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
4099
4100 /* set MQD vmid to 0 */
4101 tmp = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
4102 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4103 mqd->cp_mqd_control = tmp;
4104
4105 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4106 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
4107 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4108 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4109
4110 /* set up the HQD, this is similar to CP_RB0_CNTL */
4111 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL);
4112 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4113 (order_base_2(prop->queue_size / 4) - 1));
4114 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4115 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4116 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
4117 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH,
4118 prop->allow_tunneling);
4119 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4120 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4121 mqd->cp_hqd_pq_control = tmp;
4122
4123 /* set the wb address whether it's enabled or not */
4124 wb_gpu_addr = prop->rptr_gpu_addr;
4125 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4126 mqd->cp_hqd_pq_rptr_report_addr_hi =
4127 upper_32_bits(wb_gpu_addr) & 0xffff;
4128
4129 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4130 wb_gpu_addr = prop->wptr_gpu_addr;
4131 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4132 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4133
4134 tmp = 0;
4135 /* enable the doorbell if requested */
4136 if (prop->use_doorbell) {
4137 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
4138 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4139 DOORBELL_OFFSET, prop->doorbell_index);
4140
4141 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4142 DOORBELL_EN, 1);
4143 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4144 DOORBELL_SOURCE, 0);
4145 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4146 DOORBELL_HIT, 0);
4147 }
4148
4149 mqd->cp_hqd_pq_doorbell_control = tmp;
4150
4151 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4152 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR);
4153
4154 /* set the vmid for the queue */
4155 mqd->cp_hqd_vmid = 0;
4156
4157 tmp = RREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE);
4158 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55);
4159 mqd->cp_hqd_persistent_state = tmp;
4160
4161 /* set MIN_IB_AVAIL_SIZE */
4162 tmp = RREG32_SOC15(GC, 0, regCP_HQD_IB_CONTROL);
4163 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4164 mqd->cp_hqd_ib_control = tmp;
4165
4166 /* set static priority for a compute queue/ring */
4167 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
4168 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
4169
4170 mqd->cp_hqd_active = prop->hqd_active;
4171
4172 return 0;
4173 }
4174
gfx_v11_0_kiq_init_register(struct amdgpu_ring * ring)4175 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring)
4176 {
4177 struct amdgpu_device *adev = ring->adev;
4178 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4179 int j;
4180
4181 /* inactivate the queue */
4182 if (amdgpu_sriov_vf(adev))
4183 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
4184
4185 /* disable wptr polling */
4186 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4187
4188 /* write the EOP addr */
4189 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR,
4190 mqd->cp_hqd_eop_base_addr_lo);
4191 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI,
4192 mqd->cp_hqd_eop_base_addr_hi);
4193
4194 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4195 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL,
4196 mqd->cp_hqd_eop_control);
4197
4198 /* enable doorbell? */
4199 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4200 mqd->cp_hqd_pq_doorbell_control);
4201
4202 /* disable the queue if it's active */
4203 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
4204 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1);
4205 for (j = 0; j < adev->usec_timeout; j++) {
4206 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
4207 break;
4208 udelay(1);
4209 }
4210 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST,
4211 mqd->cp_hqd_dequeue_request);
4212 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR,
4213 mqd->cp_hqd_pq_rptr);
4214 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4215 mqd->cp_hqd_pq_wptr_lo);
4216 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4217 mqd->cp_hqd_pq_wptr_hi);
4218 }
4219
4220 /* set the pointer to the MQD */
4221 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR,
4222 mqd->cp_mqd_base_addr_lo);
4223 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI,
4224 mqd->cp_mqd_base_addr_hi);
4225
4226 /* set MQD vmid to 0 */
4227 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL,
4228 mqd->cp_mqd_control);
4229
4230 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4231 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE,
4232 mqd->cp_hqd_pq_base_lo);
4233 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI,
4234 mqd->cp_hqd_pq_base_hi);
4235
4236 /* set up the HQD, this is similar to CP_RB0_CNTL */
4237 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL,
4238 mqd->cp_hqd_pq_control);
4239
4240 /* set the wb address whether it's enabled or not */
4241 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
4242 mqd->cp_hqd_pq_rptr_report_addr_lo);
4243 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4244 mqd->cp_hqd_pq_rptr_report_addr_hi);
4245
4246 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4247 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
4248 mqd->cp_hqd_pq_wptr_poll_addr_lo);
4249 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
4250 mqd->cp_hqd_pq_wptr_poll_addr_hi);
4251
4252 /* enable the doorbell if requested */
4253 if (ring->use_doorbell) {
4254 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER,
4255 (adev->doorbell_index.kiq * 2) << 2);
4256 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER,
4257 (adev->doorbell_index.userqueue_end * 2) << 2);
4258 }
4259
4260 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
4261 mqd->cp_hqd_pq_doorbell_control);
4262
4263 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4264 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO,
4265 mqd->cp_hqd_pq_wptr_lo);
4266 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI,
4267 mqd->cp_hqd_pq_wptr_hi);
4268
4269 /* set the vmid for the queue */
4270 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid);
4271
4272 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE,
4273 mqd->cp_hqd_persistent_state);
4274
4275 /* activate the queue */
4276 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE,
4277 mqd->cp_hqd_active);
4278
4279 if (ring->use_doorbell)
4280 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4281
4282 return 0;
4283 }
4284
gfx_v11_0_kiq_init_queue(struct amdgpu_ring * ring)4285 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
4286 {
4287 struct amdgpu_device *adev = ring->adev;
4288 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4289
4290 gfx_v11_0_kiq_setting(ring);
4291
4292 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4293 /* reset MQD to a clean status */
4294 if (adev->gfx.kiq[0].mqd_backup)
4295 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd));
4296
4297 /* reset ring buffer */
4298 ring->wptr = 0;
4299 amdgpu_ring_clear_ring(ring);
4300
4301 mutex_lock(&adev->srbm_mutex);
4302 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4303 gfx_v11_0_kiq_init_register(ring);
4304 soc21_grbm_select(adev, 0, 0, 0, 0);
4305 mutex_unlock(&adev->srbm_mutex);
4306 } else {
4307 memset((void *)mqd, 0, sizeof(*mqd));
4308 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4309 amdgpu_ring_clear_ring(ring);
4310 mutex_lock(&adev->srbm_mutex);
4311 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4312 amdgpu_ring_init_mqd(ring);
4313 gfx_v11_0_kiq_init_register(ring);
4314 soc21_grbm_select(adev, 0, 0, 0, 0);
4315 mutex_unlock(&adev->srbm_mutex);
4316
4317 if (adev->gfx.kiq[0].mqd_backup)
4318 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd));
4319 }
4320
4321 return 0;
4322 }
4323
gfx_v11_0_kcq_init_queue(struct amdgpu_ring * ring,bool reset)4324 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset)
4325 {
4326 struct amdgpu_device *adev = ring->adev;
4327 struct v11_compute_mqd *mqd = ring->mqd_ptr;
4328 int mqd_idx = ring - &adev->gfx.compute_ring[0];
4329
4330 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
4331 memset((void *)mqd, 0, sizeof(*mqd));
4332 mutex_lock(&adev->srbm_mutex);
4333 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4334 amdgpu_ring_init_mqd(ring);
4335 soc21_grbm_select(adev, 0, 0, 0, 0);
4336 mutex_unlock(&adev->srbm_mutex);
4337
4338 if (adev->gfx.mec.mqd_backup[mqd_idx])
4339 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
4340 } else {
4341 /* restore MQD to a clean status */
4342 if (adev->gfx.mec.mqd_backup[mqd_idx])
4343 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
4344 /* reset ring buffer */
4345 ring->wptr = 0;
4346 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4347 amdgpu_ring_clear_ring(ring);
4348 }
4349
4350 return 0;
4351 }
4352
gfx_v11_0_kiq_resume(struct amdgpu_device * adev)4353 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev)
4354 {
4355 struct amdgpu_ring *ring;
4356 int r;
4357
4358 ring = &adev->gfx.kiq[0].ring;
4359
4360 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4361 if (unlikely(r != 0))
4362 return r;
4363
4364 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4365 if (unlikely(r != 0)) {
4366 amdgpu_bo_unreserve(ring->mqd_obj);
4367 return r;
4368 }
4369
4370 gfx_v11_0_kiq_init_queue(ring);
4371 amdgpu_bo_kunmap(ring->mqd_obj);
4372 ring->mqd_ptr = NULL;
4373 amdgpu_bo_unreserve(ring->mqd_obj);
4374 ring->sched.ready = true;
4375 return 0;
4376 }
4377
gfx_v11_0_kcq_resume(struct amdgpu_device * adev)4378 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
4379 {
4380 struct amdgpu_ring *ring = NULL;
4381 int r = 0, i;
4382
4383 if (!amdgpu_async_gfx_ring)
4384 gfx_v11_0_cp_compute_enable(adev, true);
4385
4386 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4387 ring = &adev->gfx.compute_ring[i];
4388
4389 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4390 if (unlikely(r != 0))
4391 goto done;
4392 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
4393 if (!r) {
4394 r = gfx_v11_0_kcq_init_queue(ring, false);
4395 amdgpu_bo_kunmap(ring->mqd_obj);
4396 ring->mqd_ptr = NULL;
4397 }
4398 amdgpu_bo_unreserve(ring->mqd_obj);
4399 if (r)
4400 goto done;
4401 }
4402
4403 r = amdgpu_gfx_enable_kcq(adev, 0);
4404 done:
4405 return r;
4406 }
4407
gfx_v11_0_cp_resume(struct amdgpu_device * adev)4408 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev)
4409 {
4410 int r, i;
4411 struct amdgpu_ring *ring;
4412
4413 if (!(adev->flags & AMD_IS_APU))
4414 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4415
4416 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4417 /* legacy firmware loading */
4418 r = gfx_v11_0_cp_gfx_load_microcode(adev);
4419 if (r)
4420 return r;
4421
4422 if (adev->gfx.rs64_enable)
4423 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev);
4424 else
4425 r = gfx_v11_0_cp_compute_load_microcode(adev);
4426 if (r)
4427 return r;
4428 }
4429
4430 gfx_v11_0_cp_set_doorbell_range(adev);
4431
4432 if (amdgpu_async_gfx_ring) {
4433 gfx_v11_0_cp_compute_enable(adev, true);
4434 gfx_v11_0_cp_gfx_enable(adev, true);
4435 }
4436
4437 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init)
4438 r = amdgpu_mes_kiq_hw_init(adev);
4439 else
4440 r = gfx_v11_0_kiq_resume(adev);
4441 if (r)
4442 return r;
4443
4444 r = gfx_v11_0_kcq_resume(adev);
4445 if (r)
4446 return r;
4447
4448 if (!amdgpu_async_gfx_ring) {
4449 r = gfx_v11_0_cp_gfx_resume(adev);
4450 if (r)
4451 return r;
4452 } else {
4453 r = gfx_v11_0_cp_async_gfx_ring_resume(adev);
4454 if (r)
4455 return r;
4456 }
4457
4458 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4459 ring = &adev->gfx.gfx_ring[i];
4460 r = amdgpu_ring_test_helper(ring);
4461 if (r)
4462 return r;
4463 }
4464
4465 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4466 ring = &adev->gfx.compute_ring[i];
4467 r = amdgpu_ring_test_helper(ring);
4468 if (r)
4469 return r;
4470 }
4471
4472 return 0;
4473 }
4474
gfx_v11_0_cp_enable(struct amdgpu_device * adev,bool enable)4475 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable)
4476 {
4477 gfx_v11_0_cp_gfx_enable(adev, enable);
4478 gfx_v11_0_cp_compute_enable(adev, enable);
4479 }
4480
gfx_v11_0_gfxhub_enable(struct amdgpu_device * adev)4481 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev)
4482 {
4483 int r;
4484 bool value;
4485
4486 r = adev->gfxhub.funcs->gart_enable(adev);
4487 if (r)
4488 return r;
4489
4490 adev->hdp.funcs->flush_hdp(adev, NULL);
4491
4492 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
4493 false : true;
4494
4495 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
4496 /* TODO investigate why this and the hdp flush above is needed,
4497 * are we missing a flush somewhere else? */
4498 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
4499
4500 return 0;
4501 }
4502
gfx_v11_0_select_cp_fw_arch(struct amdgpu_device * adev)4503 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev)
4504 {
4505 u32 tmp;
4506
4507 /* select RS64 */
4508 if (adev->gfx.rs64_enable) {
4509 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL);
4510 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1);
4511 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp);
4512
4513 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL);
4514 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1);
4515 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp);
4516 }
4517
4518 if (amdgpu_emu_mode == 1)
4519 msleep(100);
4520 }
4521
get_gb_addr_config(struct amdgpu_device * adev)4522 static int get_gb_addr_config(struct amdgpu_device * adev)
4523 {
4524 u32 gb_addr_config;
4525
4526 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG);
4527 if (gb_addr_config == 0)
4528 return -EINVAL;
4529
4530 adev->gfx.config.gb_addr_config_fields.num_pkrs =
4531 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS);
4532
4533 adev->gfx.config.gb_addr_config = gb_addr_config;
4534
4535 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
4536 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4537 GB_ADDR_CONFIG, NUM_PIPES);
4538
4539 adev->gfx.config.max_tile_pipes =
4540 adev->gfx.config.gb_addr_config_fields.num_pipes;
4541
4542 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
4543 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4544 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
4545 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
4546 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4547 GB_ADDR_CONFIG, NUM_RB_PER_SE);
4548 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
4549 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4550 GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
4551 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
4552 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
4553 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
4554
4555 return 0;
4556 }
4557
gfx_v11_0_disable_gpa_mode(struct amdgpu_device * adev)4558 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
4559 {
4560 uint32_t data;
4561
4562 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG);
4563 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
4564 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data);
4565
4566 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG);
4567 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
4568 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
4569 }
4570
gfx_v11_0_hw_init(void * handle)4571 static int gfx_v11_0_hw_init(void *handle)
4572 {
4573 int r;
4574 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4575
4576 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
4577 if (adev->gfx.imu.funcs) {
4578 /* RLC autoload sequence 1: Program rlc ram */
4579 if (adev->gfx.imu.funcs->program_rlc_ram)
4580 adev->gfx.imu.funcs->program_rlc_ram(adev);
4581 /* rlc autoload firmware */
4582 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev);
4583 if (r)
4584 return r;
4585 }
4586 } else {
4587 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
4588 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) {
4589 if (adev->gfx.imu.funcs->load_microcode)
4590 adev->gfx.imu.funcs->load_microcode(adev);
4591 if (adev->gfx.imu.funcs->setup_imu)
4592 adev->gfx.imu.funcs->setup_imu(adev);
4593 if (adev->gfx.imu.funcs->start_imu)
4594 adev->gfx.imu.funcs->start_imu(adev);
4595 }
4596
4597 /* disable gpa mode in backdoor loading */
4598 gfx_v11_0_disable_gpa_mode(adev);
4599 }
4600 }
4601
4602 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
4603 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
4604 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev);
4605 if (r) {
4606 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
4607 return r;
4608 }
4609 }
4610
4611 adev->gfx.is_poweron = true;
4612
4613 if(get_gb_addr_config(adev))
4614 DRM_WARN("Invalid gb_addr_config !\n");
4615
4616 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
4617 adev->gfx.rs64_enable)
4618 gfx_v11_0_config_gfx_rs64(adev);
4619
4620 r = gfx_v11_0_gfxhub_enable(adev);
4621 if (r)
4622 return r;
4623
4624 if (!amdgpu_emu_mode)
4625 gfx_v11_0_init_golden_registers(adev);
4626
4627 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
4628 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) {
4629 /**
4630 * For gfx 11, rlc firmware loading relies on smu firmware is
4631 * loaded firstly, so in direct type, it has to load smc ucode
4632 * here before rlc.
4633 */
4634 r = amdgpu_pm_load_smu_firmware(adev, NULL);
4635 if (r)
4636 return r;
4637 }
4638
4639 gfx_v11_0_constants_init(adev);
4640
4641 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
4642 gfx_v11_0_select_cp_fw_arch(adev);
4643
4644 if (adev->nbio.funcs->gc_doorbell_init)
4645 adev->nbio.funcs->gc_doorbell_init(adev);
4646
4647 r = gfx_v11_0_rlc_resume(adev);
4648 if (r)
4649 return r;
4650
4651 /*
4652 * init golden registers and rlc resume may override some registers,
4653 * reconfig them here
4654 */
4655 gfx_v11_0_tcp_harvest(adev);
4656
4657 r = gfx_v11_0_cp_resume(adev);
4658 if (r)
4659 return r;
4660
4661 /* get IMU version from HW if it's not set */
4662 if (!adev->gfx.imu_fw_version)
4663 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0);
4664
4665 return r;
4666 }
4667
gfx_v11_0_hw_fini(void * handle)4668 static int gfx_v11_0_hw_fini(void *handle)
4669 {
4670 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4671
4672 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4673 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4674 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
4675
4676 if (!adev->no_hw_access) {
4677 if (amdgpu_async_gfx_ring) {
4678 if (amdgpu_gfx_disable_kgq(adev, 0))
4679 DRM_ERROR("KGQ disable failed\n");
4680 }
4681
4682 if (amdgpu_gfx_disable_kcq(adev, 0))
4683 DRM_ERROR("KCQ disable failed\n");
4684
4685 amdgpu_mes_kiq_hw_fini(adev);
4686 }
4687
4688 if (amdgpu_sriov_vf(adev))
4689 /* Remove the steps disabling CPG and clearing KIQ position,
4690 * so that CP could perform IDLE-SAVE during switch. Those
4691 * steps are necessary to avoid a DMAR error in gfx9 but it is
4692 * not reproduced on gfx11.
4693 */
4694 return 0;
4695
4696 gfx_v11_0_cp_enable(adev, false);
4697 gfx_v11_0_enable_gui_idle_interrupt(adev, false);
4698
4699 adev->gfxhub.funcs->gart_disable(adev);
4700
4701 adev->gfx.is_poweron = false;
4702
4703 return 0;
4704 }
4705
gfx_v11_0_suspend(void * handle)4706 static int gfx_v11_0_suspend(void *handle)
4707 {
4708 return gfx_v11_0_hw_fini(handle);
4709 }
4710
gfx_v11_0_resume(void * handle)4711 static int gfx_v11_0_resume(void *handle)
4712 {
4713 return gfx_v11_0_hw_init(handle);
4714 }
4715
gfx_v11_0_is_idle(void * handle)4716 static bool gfx_v11_0_is_idle(void *handle)
4717 {
4718 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4719
4720 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
4721 GRBM_STATUS, GUI_ACTIVE))
4722 return false;
4723 else
4724 return true;
4725 }
4726
gfx_v11_0_wait_for_idle(void * handle)4727 static int gfx_v11_0_wait_for_idle(void *handle)
4728 {
4729 unsigned i;
4730 u32 tmp;
4731 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4732
4733 for (i = 0; i < adev->usec_timeout; i++) {
4734 /* read MC_STATUS */
4735 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) &
4736 GRBM_STATUS__GUI_ACTIVE_MASK;
4737
4738 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
4739 return 0;
4740 udelay(1);
4741 }
4742 return -ETIMEDOUT;
4743 }
4744
gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device * adev,bool req)4745 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev,
4746 bool req)
4747 {
4748 u32 i, tmp, val;
4749
4750 for (i = 0; i < adev->usec_timeout; i++) {
4751 /* Request with MeId=2, PipeId=0 */
4752 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req);
4753 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4);
4754 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp);
4755
4756 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX);
4757 if (req) {
4758 if (val == tmp)
4759 break;
4760 } else {
4761 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX,
4762 REQUEST, 1);
4763
4764 /* unlocked or locked by firmware */
4765 if (val != tmp)
4766 break;
4767 }
4768 udelay(1);
4769 }
4770
4771 if (i >= adev->usec_timeout)
4772 return -EINVAL;
4773
4774 return 0;
4775 }
4776
gfx_v11_0_soft_reset(void * handle)4777 static int gfx_v11_0_soft_reset(void *handle)
4778 {
4779 u32 grbm_soft_reset = 0;
4780 u32 tmp;
4781 int r, i, j, k;
4782 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4783
4784 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4785
4786 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4787 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0);
4788 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0);
4789 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0);
4790 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0);
4791 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4792
4793 mutex_lock(&adev->srbm_mutex);
4794 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4795 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4796 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4797 soc21_grbm_select(adev, i, k, j, 0);
4798
4799 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
4800 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
4801 }
4802 }
4803 }
4804 for (i = 0; i < adev->gfx.me.num_me; ++i) {
4805 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
4806 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
4807 soc21_grbm_select(adev, i, k, j, 0);
4808
4809 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1);
4810 }
4811 }
4812 }
4813 soc21_grbm_select(adev, 0, 0, 0, 0);
4814 mutex_unlock(&adev->srbm_mutex);
4815
4816 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */
4817 mutex_lock(&adev->gfx.reset_sem_mutex);
4818 r = gfx_v11_0_request_gfx_index_mutex(adev, true);
4819 if (r) {
4820 mutex_unlock(&adev->gfx.reset_sem_mutex);
4821 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n");
4822 return r;
4823 }
4824
4825 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe);
4826
4827 // Read CP_VMID_RESET register three times.
4828 // to get sufficient time for GFX_HQD_ACTIVE reach 0
4829 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4830 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4831 RREG32_SOC15(GC, 0, regCP_VMID_RESET);
4832
4833 /* release the gfx mutex */
4834 r = gfx_v11_0_request_gfx_index_mutex(adev, false);
4835 mutex_unlock(&adev->gfx.reset_sem_mutex);
4836 if (r) {
4837 DRM_ERROR("Failed to release the gfx mutex during soft reset\n");
4838 return r;
4839 }
4840
4841 for (i = 0; i < adev->usec_timeout; i++) {
4842 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) &&
4843 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE))
4844 break;
4845 udelay(1);
4846 }
4847 if (i >= adev->usec_timeout) {
4848 printk("Failed to wait all pipes clean\n");
4849 return -EINVAL;
4850 }
4851
4852 /********** trigger soft reset ***********/
4853 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4854 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4855 SOFT_RESET_CP, 1);
4856 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4857 SOFT_RESET_GFX, 1);
4858 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4859 SOFT_RESET_CPF, 1);
4860 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4861 SOFT_RESET_CPC, 1);
4862 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4863 SOFT_RESET_CPG, 1);
4864 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4865 /********** exit soft reset ***********/
4866 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
4867 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4868 SOFT_RESET_CP, 0);
4869 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4870 SOFT_RESET_GFX, 0);
4871 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4872 SOFT_RESET_CPF, 0);
4873 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4874 SOFT_RESET_CPC, 0);
4875 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4876 SOFT_RESET_CPG, 0);
4877 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset);
4878
4879 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL);
4880 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1);
4881 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp);
4882
4883 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0);
4884 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0);
4885
4886 for (i = 0; i < adev->usec_timeout; i++) {
4887 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET))
4888 break;
4889 udelay(1);
4890 }
4891 if (i >= adev->usec_timeout) {
4892 printk("Failed to wait CP_VMID_RESET to 0\n");
4893 return -EINVAL;
4894 }
4895
4896 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
4897 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
4898 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
4899 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
4900 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
4901 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp);
4902
4903 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4904
4905 return gfx_v11_0_cp_resume(adev);
4906 }
4907
gfx_v11_0_check_soft_reset(void * handle)4908 static bool gfx_v11_0_check_soft_reset(void *handle)
4909 {
4910 int i, r;
4911 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4912 struct amdgpu_ring *ring;
4913 long tmo = msecs_to_jiffies(1000);
4914
4915 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4916 ring = &adev->gfx.gfx_ring[i];
4917 r = amdgpu_ring_test_ib(ring, tmo);
4918 if (r)
4919 return true;
4920 }
4921
4922 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4923 ring = &adev->gfx.compute_ring[i];
4924 r = amdgpu_ring_test_ib(ring, tmo);
4925 if (r)
4926 return true;
4927 }
4928
4929 return false;
4930 }
4931
gfx_v11_0_post_soft_reset(void * handle)4932 static int gfx_v11_0_post_soft_reset(void *handle)
4933 {
4934 /**
4935 * GFX soft reset will impact MES, need resume MES when do GFX soft reset
4936 */
4937 return amdgpu_mes_resume((struct amdgpu_device *)handle);
4938 }
4939
gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device * adev)4940 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4941 {
4942 uint64_t clock;
4943 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after;
4944
4945 if (amdgpu_sriov_vf(adev)) {
4946 amdgpu_gfx_off_ctrl(adev, false);
4947 mutex_lock(&adev->gfx.gpu_clock_mutex);
4948 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4949 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4950 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI);
4951 if (clock_counter_hi_pre != clock_counter_hi_after)
4952 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO);
4953 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4954 amdgpu_gfx_off_ctrl(adev, true);
4955 } else {
4956 preempt_disable();
4957 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4958 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4959 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER);
4960 if (clock_counter_hi_pre != clock_counter_hi_after)
4961 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER);
4962 preempt_enable();
4963 }
4964 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL);
4965
4966 return clock;
4967 }
4968
gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)4969 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4970 uint32_t vmid,
4971 uint32_t gds_base, uint32_t gds_size,
4972 uint32_t gws_base, uint32_t gws_size,
4973 uint32_t oa_base, uint32_t oa_size)
4974 {
4975 struct amdgpu_device *adev = ring->adev;
4976
4977 /* GDS Base */
4978 gfx_v11_0_write_data_to_reg(ring, 0, false,
4979 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid,
4980 gds_base);
4981
4982 /* GDS Size */
4983 gfx_v11_0_write_data_to_reg(ring, 0, false,
4984 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid,
4985 gds_size);
4986
4987 /* GWS */
4988 gfx_v11_0_write_data_to_reg(ring, 0, false,
4989 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid,
4990 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4991
4992 /* OA */
4993 gfx_v11_0_write_data_to_reg(ring, 0, false,
4994 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid,
4995 (1 << (oa_size + oa_base)) - (1 << oa_base));
4996 }
4997
gfx_v11_0_early_init(void * handle)4998 static int gfx_v11_0_early_init(void *handle)
4999 {
5000 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5001
5002 adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
5003
5004 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
5005 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5006 AMDGPU_MAX_COMPUTE_RINGS);
5007
5008 gfx_v11_0_set_kiq_pm4_funcs(adev);
5009 gfx_v11_0_set_ring_funcs(adev);
5010 gfx_v11_0_set_irq_funcs(adev);
5011 gfx_v11_0_set_gds_init(adev);
5012 gfx_v11_0_set_rlc_funcs(adev);
5013 gfx_v11_0_set_mqd_funcs(adev);
5014 gfx_v11_0_set_imu_funcs(adev);
5015
5016 gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
5017
5018 return gfx_v11_0_init_microcode(adev);
5019 }
5020
gfx_v11_0_late_init(void * handle)5021 static int gfx_v11_0_late_init(void *handle)
5022 {
5023 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5024 int r;
5025
5026 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5027 if (r)
5028 return r;
5029
5030 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5031 if (r)
5032 return r;
5033
5034 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
5035 if (r)
5036 return r;
5037 return 0;
5038 }
5039
gfx_v11_0_is_rlc_enabled(struct amdgpu_device * adev)5040 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
5041 {
5042 uint32_t rlc_cntl;
5043
5044 /* if RLC is not enabled, do nothing */
5045 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL);
5046 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
5047 }
5048
gfx_v11_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)5049 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
5050 {
5051 uint32_t data;
5052 unsigned i;
5053
5054 data = RLC_SAFE_MODE__CMD_MASK;
5055 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5056
5057 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data);
5058
5059 /* wait for RLC_SAFE_MODE */
5060 for (i = 0; i < adev->usec_timeout; i++) {
5061 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE),
5062 RLC_SAFE_MODE, CMD))
5063 break;
5064 udelay(1);
5065 }
5066 }
5067
gfx_v11_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)5068 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
5069 {
5070 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
5071 }
5072
gfx_v11_0_update_perf_clk(struct amdgpu_device * adev,bool enable)5073 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev,
5074 bool enable)
5075 {
5076 uint32_t def, data;
5077
5078 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
5079 return;
5080
5081 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5082
5083 if (enable)
5084 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5085 else
5086 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
5087
5088 if (def != data)
5089 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5090 }
5091
gfx_v11_0_update_sram_fgcg(struct amdgpu_device * adev,bool enable)5092 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev,
5093 bool enable)
5094 {
5095 uint32_t def, data;
5096
5097 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
5098 return;
5099
5100 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5101
5102 if (enable)
5103 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5104 else
5105 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
5106
5107 if (def != data)
5108 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5109 }
5110
gfx_v11_0_update_repeater_fgcg(struct amdgpu_device * adev,bool enable)5111 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev,
5112 bool enable)
5113 {
5114 uint32_t def, data;
5115
5116 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
5117 return;
5118
5119 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5120
5121 if (enable)
5122 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5123 else
5124 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK;
5125
5126 if (def != data)
5127 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5128 }
5129
gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)5130 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5131 bool enable)
5132 {
5133 uint32_t data, def;
5134
5135 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
5136 return;
5137
5138 /* It is disabled by HW by default */
5139 if (enable) {
5140 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5141 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
5142 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5143
5144 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5145 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5146 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5147
5148 if (def != data)
5149 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5150 }
5151 } else {
5152 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5153 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5154
5155 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
5156 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
5157 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
5158
5159 if (def != data)
5160 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5161 }
5162 }
5163 }
5164
gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5165 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5166 bool enable)
5167 {
5168 uint32_t def, data;
5169
5170 if (!(adev->cg_flags &
5171 (AMD_CG_SUPPORT_GFX_CGCG |
5172 AMD_CG_SUPPORT_GFX_CGLS |
5173 AMD_CG_SUPPORT_GFX_3D_CGCG |
5174 AMD_CG_SUPPORT_GFX_3D_CGLS)))
5175 return;
5176
5177 if (enable) {
5178 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5179
5180 /* unset CGCG override */
5181 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5182 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
5183 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5184 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
5185 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
5186 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5187 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
5188
5189 /* update CGCG override bits */
5190 if (def != data)
5191 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data);
5192
5193 /* enable cgcg FSM(0x0000363F) */
5194 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5195
5196 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5197 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
5198 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5199 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5200 }
5201
5202 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5203 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
5204 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5205 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5206 }
5207
5208 if (def != data)
5209 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5210
5211 /* Program RLC_CGCG_CGLS_CTRL_3D */
5212 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5213
5214 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5215 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK;
5216 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
5217 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5218 }
5219
5220 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5221 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK;
5222 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
5223 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5224 }
5225
5226 if (def != data)
5227 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5228
5229 /* set IDLE_POLL_COUNT(0x00900100) */
5230 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL);
5231
5232 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK);
5233 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
5234 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
5235
5236 if (def != data)
5237 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data);
5238
5239 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL);
5240 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
5241 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
5242 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
5243 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
5244 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data);
5245
5246 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5247 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5248 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5249
5250 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5251 if (adev->sdma.num_instances > 1) {
5252 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5253 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1);
5254 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5255 }
5256 } else {
5257 /* Program RLC_CGCG_CGLS_CTRL */
5258 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5259
5260 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
5261 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5262
5263 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
5264 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5265
5266 if (def != data)
5267 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data);
5268
5269 /* Program RLC_CGCG_CGLS_CTRL_3D */
5270 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5271
5272 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)
5273 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
5274 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
5275 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
5276
5277 if (def != data)
5278 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data);
5279
5280 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL);
5281 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5282 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data);
5283
5284 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */
5285 if (adev->sdma.num_instances > 1) {
5286 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL);
5287 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK;
5288 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data);
5289 }
5290 }
5291 }
5292
gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5293 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5294 bool enable)
5295 {
5296 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5297
5298 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable);
5299
5300 gfx_v11_0_update_medium_grain_clock_gating(adev, enable);
5301
5302 gfx_v11_0_update_repeater_fgcg(adev, enable);
5303
5304 gfx_v11_0_update_sram_fgcg(adev, enable);
5305
5306 gfx_v11_0_update_perf_clk(adev, enable);
5307
5308 if (adev->cg_flags &
5309 (AMD_CG_SUPPORT_GFX_MGCG |
5310 AMD_CG_SUPPORT_GFX_CGLS |
5311 AMD_CG_SUPPORT_GFX_CGCG |
5312 AMD_CG_SUPPORT_GFX_3D_CGCG |
5313 AMD_CG_SUPPORT_GFX_3D_CGLS))
5314 gfx_v11_0_enable_gui_idle_interrupt(adev, enable);
5315
5316 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5317
5318 return 0;
5319 }
5320
gfx_v11_0_update_spm_vmid(struct amdgpu_device * adev,struct amdgpu_ring * ring,unsigned vmid)5321 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned vmid)
5322 {
5323 u32 reg, pre_data, data;
5324
5325 amdgpu_gfx_off_ctrl(adev, false);
5326 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
5327 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
5328 pre_data = RREG32_NO_KIQ(reg);
5329 else
5330 pre_data = RREG32(reg);
5331
5332 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
5333 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
5334
5335 if (pre_data != data) {
5336 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
5337 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
5338 } else
5339 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
5340 }
5341 amdgpu_gfx_off_ctrl(adev, true);
5342
5343 if (ring
5344 && amdgpu_sriov_is_pp_one_vf(adev)
5345 && (pre_data != data)
5346 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
5347 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
5348 amdgpu_ring_emit_wreg(ring, reg, data);
5349 }
5350 }
5351
5352 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
5353 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled,
5354 .set_safe_mode = gfx_v11_0_set_safe_mode,
5355 .unset_safe_mode = gfx_v11_0_unset_safe_mode,
5356 .init = gfx_v11_0_rlc_init,
5357 .get_csb_size = gfx_v11_0_get_csb_size,
5358 .get_csb_buffer = gfx_v11_0_get_csb_buffer,
5359 .resume = gfx_v11_0_rlc_resume,
5360 .stop = gfx_v11_0_rlc_stop,
5361 .reset = gfx_v11_0_rlc_reset,
5362 .start = gfx_v11_0_rlc_start,
5363 .update_spm_vmid = gfx_v11_0_update_spm_vmid,
5364 };
5365
gfx_v11_cntl_power_gating(struct amdgpu_device * adev,bool enable)5366 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable)
5367 {
5368 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL);
5369
5370 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
5371 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5372 else
5373 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
5374
5375 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data);
5376
5377 // Program RLC_PG_DELAY3 for CGPG hysteresis
5378 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
5379 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5380 case IP_VERSION(11, 0, 1):
5381 case IP_VERSION(11, 0, 4):
5382 case IP_VERSION(11, 5, 0):
5383 case IP_VERSION(11, 5, 1):
5384 case IP_VERSION(11, 5, 2):
5385 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1);
5386 break;
5387 default:
5388 break;
5389 }
5390 }
5391 }
5392
gfx_v11_cntl_pg(struct amdgpu_device * adev,bool enable)5393 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable)
5394 {
5395 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5396
5397 gfx_v11_cntl_power_gating(adev, enable);
5398
5399 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5400 }
5401
gfx_v11_0_set_powergating_state(void * handle,enum amd_powergating_state state)5402 static int gfx_v11_0_set_powergating_state(void *handle,
5403 enum amd_powergating_state state)
5404 {
5405 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5406 bool enable = (state == AMD_PG_STATE_GATE);
5407
5408 if (amdgpu_sriov_vf(adev))
5409 return 0;
5410
5411 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5412 case IP_VERSION(11, 0, 0):
5413 case IP_VERSION(11, 0, 2):
5414 case IP_VERSION(11, 0, 3):
5415 amdgpu_gfx_off_ctrl(adev, enable);
5416 break;
5417 case IP_VERSION(11, 0, 1):
5418 case IP_VERSION(11, 0, 4):
5419 case IP_VERSION(11, 5, 0):
5420 case IP_VERSION(11, 5, 1):
5421 case IP_VERSION(11, 5, 2):
5422 if (!enable)
5423 amdgpu_gfx_off_ctrl(adev, false);
5424
5425 gfx_v11_cntl_pg(adev, enable);
5426
5427 if (enable)
5428 amdgpu_gfx_off_ctrl(adev, true);
5429
5430 break;
5431 default:
5432 break;
5433 }
5434
5435 return 0;
5436 }
5437
gfx_v11_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)5438 static int gfx_v11_0_set_clockgating_state(void *handle,
5439 enum amd_clockgating_state state)
5440 {
5441 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5442
5443 if (amdgpu_sriov_vf(adev))
5444 return 0;
5445
5446 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
5447 case IP_VERSION(11, 0, 0):
5448 case IP_VERSION(11, 0, 1):
5449 case IP_VERSION(11, 0, 2):
5450 case IP_VERSION(11, 0, 3):
5451 case IP_VERSION(11, 0, 4):
5452 case IP_VERSION(11, 5, 0):
5453 case IP_VERSION(11, 5, 1):
5454 case IP_VERSION(11, 5, 2):
5455 gfx_v11_0_update_gfx_clock_gating(adev,
5456 state == AMD_CG_STATE_GATE);
5457 break;
5458 default:
5459 break;
5460 }
5461
5462 return 0;
5463 }
5464
gfx_v11_0_get_clockgating_state(void * handle,u64 * flags)5465 static void gfx_v11_0_get_clockgating_state(void *handle, u64 *flags)
5466 {
5467 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5468 int data;
5469
5470 /* AMD_CG_SUPPORT_GFX_MGCG */
5471 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE);
5472 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5473 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5474
5475 /* AMD_CG_SUPPORT_REPEATER_FGCG */
5476 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
5477 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
5478
5479 /* AMD_CG_SUPPORT_GFX_FGCG */
5480 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
5481 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
5482
5483 /* AMD_CG_SUPPORT_GFX_PERF_CLK */
5484 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
5485 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
5486
5487 /* AMD_CG_SUPPORT_GFX_CGCG */
5488 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL);
5489 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5490 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5491
5492 /* AMD_CG_SUPPORT_GFX_CGLS */
5493 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5494 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5495
5496 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5497 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D);
5498 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5499 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5500
5501 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5502 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5503 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5504 }
5505
gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring * ring)5506 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5507 {
5508 /* gfx11 is 32bit rptr*/
5509 return *(uint32_t *)ring->rptr_cpu_addr;
5510 }
5511
gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)5512 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5513 {
5514 struct amdgpu_device *adev = ring->adev;
5515 u64 wptr;
5516
5517 /* XXX check if swapping is necessary on BE */
5518 if (ring->use_doorbell) {
5519 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5520 } else {
5521 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR);
5522 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32;
5523 }
5524
5525 return wptr;
5526 }
5527
gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)5528 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5529 {
5530 struct amdgpu_device *adev = ring->adev;
5531
5532 if (ring->use_doorbell) {
5533 /* XXX check if swapping is necessary on BE */
5534 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5535 ring->wptr);
5536 WDOORBELL64(ring->doorbell_index, ring->wptr);
5537 } else {
5538 WREG32_SOC15(GC, 0, regCP_RB0_WPTR,
5539 lower_32_bits(ring->wptr));
5540 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI,
5541 upper_32_bits(ring->wptr));
5542 }
5543 }
5544
gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring * ring)5545 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5546 {
5547 /* gfx11 hardware is 32bit rptr */
5548 return *(uint32_t *)ring->rptr_cpu_addr;
5549 }
5550
gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring * ring)5551 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5552 {
5553 u64 wptr;
5554
5555 /* XXX check if swapping is necessary on BE */
5556 if (ring->use_doorbell)
5557 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
5558 else
5559 BUG();
5560 return wptr;
5561 }
5562
gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring * ring)5563 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5564 {
5565 struct amdgpu_device *adev = ring->adev;
5566
5567 /* XXX check if swapping is necessary on BE */
5568 if (ring->use_doorbell) {
5569 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
5570 ring->wptr);
5571 WDOORBELL64(ring->doorbell_index, ring->wptr);
5572 } else {
5573 BUG(); /* only DOORBELL method supported on gfx11 now */
5574 }
5575 }
5576
gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)5577 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5578 {
5579 struct amdgpu_device *adev = ring->adev;
5580 u32 ref_and_mask, reg_mem_engine;
5581 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5582
5583 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5584 switch (ring->me) {
5585 case 1:
5586 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5587 break;
5588 case 2:
5589 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5590 break;
5591 default:
5592 return;
5593 }
5594 reg_mem_engine = 0;
5595 } else {
5596 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0 << ring->pipe;
5597 reg_mem_engine = 1; /* pfp */
5598 }
5599
5600 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5601 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5602 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5603 ref_and_mask, ref_and_mask, 0x20);
5604 }
5605
gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5606 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5607 struct amdgpu_job *job,
5608 struct amdgpu_ib *ib,
5609 uint32_t flags)
5610 {
5611 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5612 u32 header, control = 0;
5613
5614 BUG_ON(ib->flags & AMDGPU_IB_FLAG_CE);
5615
5616 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5617
5618 control |= ib->length_dw | (vmid << 24);
5619
5620 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5621 control |= INDIRECT_BUFFER_PRE_ENB(1);
5622
5623 if (flags & AMDGPU_IB_PREEMPTED)
5624 control |= INDIRECT_BUFFER_PRE_RESUME(1);
5625
5626 if (vmid)
5627 gfx_v11_0_ring_emit_de_meta(ring,
5628 (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false);
5629 }
5630
5631 if (ring->is_mes_queue)
5632 /* inherit vmid from mqd */
5633 control |= 0x400000;
5634
5635 amdgpu_ring_write(ring, header);
5636 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5637 amdgpu_ring_write(ring,
5638 #ifdef __BIG_ENDIAN
5639 (2 << 0) |
5640 #endif
5641 lower_32_bits(ib->gpu_addr));
5642 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5643 amdgpu_ring_write(ring, control);
5644 }
5645
gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)5646 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5647 struct amdgpu_job *job,
5648 struct amdgpu_ib *ib,
5649 uint32_t flags)
5650 {
5651 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5652 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5653
5654 if (ring->is_mes_queue)
5655 /* inherit vmid from mqd */
5656 control |= 0x40000000;
5657
5658 /* Currently, there is a high possibility to get wave ID mismatch
5659 * between ME and GDS, leading to a hw deadlock, because ME generates
5660 * different wave IDs than the GDS expects. This situation happens
5661 * randomly when at least 5 compute pipes use GDS ordered append.
5662 * The wave IDs generated by ME are also wrong after suspend/resume.
5663 * Those are probably bugs somewhere else in the kernel driver.
5664 *
5665 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5666 * GDS to 0 for this ring (me/pipe).
5667 */
5668 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5669 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5670 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
5671 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5672 }
5673
5674 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5675 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5676 amdgpu_ring_write(ring,
5677 #ifdef __BIG_ENDIAN
5678 (2 << 0) |
5679 #endif
5680 lower_32_bits(ib->gpu_addr));
5681 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5682 amdgpu_ring_write(ring, control);
5683 }
5684
gfx_v11_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)5685 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5686 u64 seq, unsigned flags)
5687 {
5688 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5689 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5690
5691 /* RELEASE_MEM - flush caches, send int */
5692 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5693 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
5694 PACKET3_RELEASE_MEM_GCR_GL2_WB |
5695 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */
5696 PACKET3_RELEASE_MEM_GCR_GLM_WB |
5697 PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
5698 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5699 PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
5700 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
5701 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
5702
5703 /*
5704 * the address should be Qword aligned if 64bit write, Dword
5705 * aligned if only send 32bit data low (discard data high)
5706 */
5707 if (write64bit)
5708 BUG_ON(addr & 0x7);
5709 else
5710 BUG_ON(addr & 0x3);
5711 amdgpu_ring_write(ring, lower_32_bits(addr));
5712 amdgpu_ring_write(ring, upper_32_bits(addr));
5713 amdgpu_ring_write(ring, lower_32_bits(seq));
5714 amdgpu_ring_write(ring, upper_32_bits(seq));
5715 amdgpu_ring_write(ring, ring->is_mes_queue ?
5716 (ring->hw_queue_id | AMDGPU_FENCE_MES_QUEUE_FLAG) : 0);
5717 }
5718
gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)5719 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5720 {
5721 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5722 uint32_t seq = ring->fence_drv.sync_seq;
5723 uint64_t addr = ring->fence_drv.gpu_addr;
5724
5725 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
5726 upper_32_bits(addr), seq, 0xffffffff, 4);
5727 }
5728
gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring * ring,uint16_t pasid,uint32_t flush_type,bool all_hub,uint8_t dst_sel)5729 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring,
5730 uint16_t pasid, uint32_t flush_type,
5731 bool all_hub, uint8_t dst_sel)
5732 {
5733 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
5734 amdgpu_ring_write(ring,
5735 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
5736 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
5737 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
5738 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
5739 }
5740
gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)5741 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5742 unsigned vmid, uint64_t pd_addr)
5743 {
5744 if (ring->is_mes_queue)
5745 gfx_v11_0_ring_invalidate_tlbs(ring, 0, 0, false, 0);
5746 else
5747 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5748
5749 /* compute doesn't have PFP */
5750 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5751 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5752 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5753 amdgpu_ring_write(ring, 0x0);
5754 }
5755
5756 /* Make sure that we can't skip the SET_Q_MODE packets when the VM
5757 * changed in any way.
5758 */
5759 ring->set_q_mode_offs = 0;
5760 ring->set_q_mode_ptr = NULL;
5761 }
5762
gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)5763 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5764 u64 seq, unsigned int flags)
5765 {
5766 struct amdgpu_device *adev = ring->adev;
5767
5768 /* we only allocate 32bit for each seq wb address */
5769 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5770
5771 /* write fence seq to the "addr" */
5772 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5773 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5774 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5775 amdgpu_ring_write(ring, lower_32_bits(addr));
5776 amdgpu_ring_write(ring, upper_32_bits(addr));
5777 amdgpu_ring_write(ring, lower_32_bits(seq));
5778
5779 if (flags & AMDGPU_FENCE_FLAG_INT) {
5780 /* set register to trigger INT */
5781 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5782 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5783 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5784 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS));
5785 amdgpu_ring_write(ring, 0);
5786 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5787 }
5788 }
5789
gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)5790 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring,
5791 uint32_t flags)
5792 {
5793 uint32_t dw2 = 0;
5794
5795 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5796 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5797 /* set load_global_config & load_global_uconfig */
5798 dw2 |= 0x8001;
5799 /* set load_cs_sh_regs */
5800 dw2 |= 0x01000000;
5801 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5802 dw2 |= 0x10002;
5803 }
5804
5805 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5806 amdgpu_ring_write(ring, dw2);
5807 amdgpu_ring_write(ring, 0);
5808 }
5809
gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)5810 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
5811 uint64_t addr)
5812 {
5813 unsigned ret;
5814
5815 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5816 amdgpu_ring_write(ring, lower_32_bits(addr));
5817 amdgpu_ring_write(ring, upper_32_bits(addr));
5818 /* discard following DWs if *cond_exec_gpu_addr==0 */
5819 amdgpu_ring_write(ring, 0);
5820 ret = ring->wptr & ring->buf_mask;
5821 /* patch dummy value later */
5822 amdgpu_ring_write(ring, 0);
5823
5824 return ret;
5825 }
5826
gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring * ring,u64 shadow_va,u64 csa_va,u64 gds_va,bool init_shadow,int vmid)5827 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
5828 u64 shadow_va, u64 csa_va,
5829 u64 gds_va, bool init_shadow,
5830 int vmid)
5831 {
5832 struct amdgpu_device *adev = ring->adev;
5833 unsigned int offs, end;
5834
5835 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj)
5836 return;
5837
5838 /*
5839 * The logic here isn't easy to understand because we need to keep state
5840 * accross multiple executions of the function as well as between the
5841 * CPU and GPU. The general idea is that the newly written GPU command
5842 * has a condition on the previous one and only executed if really
5843 * necessary.
5844 */
5845
5846 /*
5847 * The dw in the NOP controls if the next SET_Q_MODE packet should be
5848 * executed or not. Reserve 64bits just to be on the save side.
5849 */
5850 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1));
5851 offs = ring->wptr & ring->buf_mask;
5852
5853 /*
5854 * We start with skipping the prefix SET_Q_MODE and always executing
5855 * the postfix SET_Q_MODE packet. This is changed below with a
5856 * WRITE_DATA command when the postfix executed.
5857 */
5858 amdgpu_ring_write(ring, shadow_va ? 1 : 0);
5859 amdgpu_ring_write(ring, 0);
5860
5861 if (ring->set_q_mode_offs) {
5862 uint64_t addr;
5863
5864 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5865 addr += ring->set_q_mode_offs << 2;
5866 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr);
5867 }
5868
5869 /*
5870 * When the postfix SET_Q_MODE packet executes we need to make sure that the
5871 * next prefix SET_Q_MODE packet executes as well.
5872 */
5873 if (!shadow_va) {
5874 uint64_t addr;
5875
5876 addr = amdgpu_bo_gpu_offset(ring->ring_obj);
5877 addr += offs << 2;
5878 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5879 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
5880 amdgpu_ring_write(ring, lower_32_bits(addr));
5881 amdgpu_ring_write(ring, upper_32_bits(addr));
5882 amdgpu_ring_write(ring, 0x1);
5883 }
5884
5885 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7));
5886 amdgpu_ring_write(ring, lower_32_bits(shadow_va));
5887 amdgpu_ring_write(ring, upper_32_bits(shadow_va));
5888 amdgpu_ring_write(ring, lower_32_bits(gds_va));
5889 amdgpu_ring_write(ring, upper_32_bits(gds_va));
5890 amdgpu_ring_write(ring, lower_32_bits(csa_va));
5891 amdgpu_ring_write(ring, upper_32_bits(csa_va));
5892 amdgpu_ring_write(ring, shadow_va ?
5893 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0);
5894 amdgpu_ring_write(ring, init_shadow ?
5895 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0);
5896
5897 if (ring->set_q_mode_offs)
5898 amdgpu_ring_patch_cond_exec(ring, end);
5899
5900 if (shadow_va) {
5901 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid;
5902
5903 /*
5904 * If the tokens match try to skip the last postfix SET_Q_MODE
5905 * packet to avoid saving/restoring the state all the time.
5906 */
5907 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token)
5908 *ring->set_q_mode_ptr = 0;
5909
5910 ring->set_q_mode_token = token;
5911 } else {
5912 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs];
5913 }
5914
5915 ring->set_q_mode_offs = offs;
5916 }
5917
gfx_v11_0_ring_preempt_ib(struct amdgpu_ring * ring)5918 static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
5919 {
5920 int i, r = 0;
5921 struct amdgpu_device *adev = ring->adev;
5922 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
5923 struct amdgpu_ring *kiq_ring = &kiq->ring;
5924 unsigned long flags;
5925
5926 if (adev->enable_mes)
5927 return -EINVAL;
5928
5929 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
5930 return -EINVAL;
5931
5932 spin_lock_irqsave(&kiq->ring_lock, flags);
5933
5934 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
5935 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5936 return -ENOMEM;
5937 }
5938
5939 /* assert preemption condition */
5940 amdgpu_ring_set_preempt_cond_exec(ring, false);
5941
5942 /* assert IB preemption, emit the trailing fence */
5943 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
5944 ring->trail_fence_gpu_addr,
5945 ++ring->trail_seq);
5946 amdgpu_ring_commit(kiq_ring);
5947
5948 spin_unlock_irqrestore(&kiq->ring_lock, flags);
5949
5950 /* poll the trailing fence */
5951 for (i = 0; i < adev->usec_timeout; i++) {
5952 if (ring->trail_seq ==
5953 le32_to_cpu(*(ring->trail_fence_cpu_addr)))
5954 break;
5955 udelay(1);
5956 }
5957
5958 if (i >= adev->usec_timeout) {
5959 r = -EINVAL;
5960 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
5961 }
5962
5963 /* deassert preemption condition */
5964 amdgpu_ring_set_preempt_cond_exec(ring, true);
5965 return r;
5966 }
5967
gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring * ring,bool resume)5968 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
5969 {
5970 struct amdgpu_device *adev = ring->adev;
5971 struct v10_de_ib_state de_payload = {0};
5972 uint64_t offset, gds_addr, de_payload_gpu_addr;
5973 void *de_payload_cpu_addr;
5974 int cnt;
5975
5976 if (ring->is_mes_queue) {
5977 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5978 gfx[0].gfx_meta_data) +
5979 offsetof(struct v10_gfx_meta_data, de_payload);
5980 de_payload_gpu_addr =
5981 amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5982 de_payload_cpu_addr =
5983 amdgpu_mes_ctx_get_offs_cpu_addr(ring, offset);
5984
5985 offset = offsetof(struct amdgpu_mes_ctx_meta_data,
5986 gfx[0].gds_backup) +
5987 offsetof(struct v10_gfx_meta_data, de_payload);
5988 gds_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
5989 } else {
5990 offset = offsetof(struct v10_gfx_meta_data, de_payload);
5991 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset;
5992 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset;
5993
5994 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) +
5995 AMDGPU_CSA_SIZE - adev->gds.gds_size,
5996 PAGE_SIZE);
5997 }
5998
5999 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
6000 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
6001
6002 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
6003 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
6004 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
6005 WRITE_DATA_DST_SEL(8) |
6006 WR_CONFIRM) |
6007 WRITE_DATA_CACHE_POLICY(0));
6008 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr));
6009 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr));
6010
6011 if (resume)
6012 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr,
6013 sizeof(de_payload) >> 2);
6014 else
6015 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
6016 sizeof(de_payload) >> 2);
6017 }
6018
gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring * ring,bool start,bool secure)6019 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start,
6020 bool secure)
6021 {
6022 uint32_t v = secure ? FRAME_TMZ : 0;
6023
6024 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
6025 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1));
6026 }
6027
gfx_v11_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)6028 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6029 uint32_t reg_val_offs)
6030 {
6031 struct amdgpu_device *adev = ring->adev;
6032
6033 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6034 amdgpu_ring_write(ring, 0 | /* src: register*/
6035 (5 << 8) | /* dst: memory */
6036 (1 << 20)); /* write confirm */
6037 amdgpu_ring_write(ring, reg);
6038 amdgpu_ring_write(ring, 0);
6039 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6040 reg_val_offs * 4));
6041 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6042 reg_val_offs * 4));
6043 }
6044
gfx_v11_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)6045 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6046 uint32_t val)
6047 {
6048 uint32_t cmd = 0;
6049
6050 switch (ring->funcs->type) {
6051 case AMDGPU_RING_TYPE_GFX:
6052 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6053 break;
6054 case AMDGPU_RING_TYPE_KIQ:
6055 cmd = (1 << 16); /* no inc addr */
6056 break;
6057 default:
6058 cmd = WR_CONFIRM;
6059 break;
6060 }
6061 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6062 amdgpu_ring_write(ring, cmd);
6063 amdgpu_ring_write(ring, reg);
6064 amdgpu_ring_write(ring, 0);
6065 amdgpu_ring_write(ring, val);
6066 }
6067
gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)6068 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
6069 uint32_t val, uint32_t mask)
6070 {
6071 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
6072 }
6073
gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)6074 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
6075 uint32_t reg0, uint32_t reg1,
6076 uint32_t ref, uint32_t mask)
6077 {
6078 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6079
6080 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
6081 ref, mask, 0x20);
6082 }
6083
gfx_v11_0_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)6084 static void gfx_v11_0_ring_soft_recovery(struct amdgpu_ring *ring,
6085 unsigned vmid)
6086 {
6087 struct amdgpu_device *adev = ring->adev;
6088 uint32_t value = 0;
6089
6090 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6091 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6092 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6093 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6094 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
6095 WREG32_SOC15(GC, 0, regSQ_CMD, value);
6096 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
6097 }
6098
6099 static void
gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,uint32_t me,uint32_t pipe,enum amdgpu_interrupt_state state)6100 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6101 uint32_t me, uint32_t pipe,
6102 enum amdgpu_interrupt_state state)
6103 {
6104 uint32_t cp_int_cntl, cp_int_cntl_reg;
6105
6106 if (!me) {
6107 switch (pipe) {
6108 case 0:
6109 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0);
6110 break;
6111 case 1:
6112 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1);
6113 break;
6114 default:
6115 DRM_DEBUG("invalid pipe %d\n", pipe);
6116 return;
6117 }
6118 } else {
6119 DRM_DEBUG("invalid me %d\n", me);
6120 return;
6121 }
6122
6123 switch (state) {
6124 case AMDGPU_IRQ_STATE_DISABLE:
6125 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6126 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6127 TIME_STAMP_INT_ENABLE, 0);
6128 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6129 GENERIC0_INT_ENABLE, 0);
6130 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6131 break;
6132 case AMDGPU_IRQ_STATE_ENABLE:
6133 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6134 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6135 TIME_STAMP_INT_ENABLE, 1);
6136 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6137 GENERIC0_INT_ENABLE, 1);
6138 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6139 break;
6140 default:
6141 break;
6142 }
6143 }
6144
gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)6145 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6146 int me, int pipe,
6147 enum amdgpu_interrupt_state state)
6148 {
6149 u32 mec_int_cntl, mec_int_cntl_reg;
6150
6151 /*
6152 * amdgpu controls only the first MEC. That's why this function only
6153 * handles the setting of interrupts for this specific MEC. All other
6154 * pipes' interrupts are set by amdkfd.
6155 */
6156
6157 if (me == 1) {
6158 switch (pipe) {
6159 case 0:
6160 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6161 break;
6162 case 1:
6163 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL);
6164 break;
6165 case 2:
6166 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL);
6167 break;
6168 case 3:
6169 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL);
6170 break;
6171 default:
6172 DRM_DEBUG("invalid pipe %d\n", pipe);
6173 return;
6174 }
6175 } else {
6176 DRM_DEBUG("invalid me %d\n", me);
6177 return;
6178 }
6179
6180 switch (state) {
6181 case AMDGPU_IRQ_STATE_DISABLE:
6182 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6183 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6184 TIME_STAMP_INT_ENABLE, 0);
6185 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6186 GENERIC0_INT_ENABLE, 0);
6187 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6188 break;
6189 case AMDGPU_IRQ_STATE_ENABLE:
6190 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg);
6191 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6192 TIME_STAMP_INT_ENABLE, 1);
6193 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6194 GENERIC0_INT_ENABLE, 1);
6195 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl);
6196 break;
6197 default:
6198 break;
6199 }
6200 }
6201
gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6202 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6203 struct amdgpu_irq_src *src,
6204 unsigned type,
6205 enum amdgpu_interrupt_state state)
6206 {
6207 switch (type) {
6208 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6209 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
6210 break;
6211 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
6212 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
6213 break;
6214 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6215 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6216 break;
6217 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6218 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6219 break;
6220 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6221 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6222 break;
6223 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6224 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6225 break;
6226 default:
6227 break;
6228 }
6229 return 0;
6230 }
6231
gfx_v11_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6232 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev,
6233 struct amdgpu_irq_src *source,
6234 struct amdgpu_iv_entry *entry)
6235 {
6236 int i;
6237 u8 me_id, pipe_id, queue_id;
6238 struct amdgpu_ring *ring;
6239 uint32_t mes_queue_id = entry->src_data[0];
6240
6241 DRM_DEBUG("IH: CP EOP\n");
6242
6243 if (adev->enable_mes && (mes_queue_id & AMDGPU_FENCE_MES_QUEUE_FLAG)) {
6244 struct amdgpu_mes_queue *queue;
6245
6246 mes_queue_id &= AMDGPU_FENCE_MES_QUEUE_ID_MASK;
6247
6248 spin_lock(&adev->mes.queue_id_lock);
6249 queue = idr_find(&adev->mes.queue_id_idr, mes_queue_id);
6250 if (queue) {
6251 DRM_DEBUG("process mes queue id = %d\n", mes_queue_id);
6252 amdgpu_fence_process(queue->ring);
6253 }
6254 spin_unlock(&adev->mes.queue_id_lock);
6255 } else {
6256 me_id = (entry->ring_id & 0x0c) >> 2;
6257 pipe_id = (entry->ring_id & 0x03) >> 0;
6258 queue_id = (entry->ring_id & 0x70) >> 4;
6259
6260 switch (me_id) {
6261 case 0:
6262 if (pipe_id == 0)
6263 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6264 else
6265 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
6266 break;
6267 case 1:
6268 case 2:
6269 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6270 ring = &adev->gfx.compute_ring[i];
6271 /* Per-queue interrupt is supported for MEC starting from VI.
6272 * The interrupt can only be enabled/disabled per pipe instead
6273 * of per queue.
6274 */
6275 if ((ring->me == me_id) &&
6276 (ring->pipe == pipe_id) &&
6277 (ring->queue == queue_id))
6278 amdgpu_fence_process(ring);
6279 }
6280 break;
6281 }
6282 }
6283
6284 return 0;
6285 }
6286
gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6287 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6288 struct amdgpu_irq_src *source,
6289 unsigned int type,
6290 enum amdgpu_interrupt_state state)
6291 {
6292 u32 cp_int_cntl_reg, cp_int_cntl;
6293 int i, j;
6294
6295 switch (state) {
6296 case AMDGPU_IRQ_STATE_DISABLE:
6297 case AMDGPU_IRQ_STATE_ENABLE:
6298 for (i = 0; i < adev->gfx.me.num_me; i++) {
6299 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6300 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6301
6302 if (cp_int_cntl_reg) {
6303 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6304 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6305 PRIV_REG_INT_ENABLE,
6306 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6307 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6308 }
6309 }
6310 }
6311 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6312 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6313 /* MECs start at 1 */
6314 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6315
6316 if (cp_int_cntl_reg) {
6317 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6318 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6319 PRIV_REG_INT_ENABLE,
6320 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6321 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6322 }
6323 }
6324 }
6325 break;
6326 default:
6327 break;
6328 }
6329
6330 return 0;
6331 }
6332
gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6333 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev,
6334 struct amdgpu_irq_src *source,
6335 unsigned type,
6336 enum amdgpu_interrupt_state state)
6337 {
6338 u32 cp_int_cntl_reg, cp_int_cntl;
6339 int i, j;
6340
6341 switch (state) {
6342 case AMDGPU_IRQ_STATE_DISABLE:
6343 case AMDGPU_IRQ_STATE_ENABLE:
6344 for (i = 0; i < adev->gfx.me.num_me; i++) {
6345 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6346 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6347
6348 if (cp_int_cntl_reg) {
6349 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6350 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6351 OPCODE_ERROR_INT_ENABLE,
6352 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6353 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6354 }
6355 }
6356 }
6357 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6358 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6359 /* MECs start at 1 */
6360 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j);
6361
6362 if (cp_int_cntl_reg) {
6363 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6364 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL,
6365 OPCODE_ERROR_INT_ENABLE,
6366 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6367 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6368 }
6369 }
6370 }
6371 break;
6372 default:
6373 break;
6374 }
6375 return 0;
6376 }
6377
gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6378 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6379 struct amdgpu_irq_src *source,
6380 unsigned int type,
6381 enum amdgpu_interrupt_state state)
6382 {
6383 u32 cp_int_cntl_reg, cp_int_cntl;
6384 int i, j;
6385
6386 switch (state) {
6387 case AMDGPU_IRQ_STATE_DISABLE:
6388 case AMDGPU_IRQ_STATE_ENABLE:
6389 for (i = 0; i < adev->gfx.me.num_me; i++) {
6390 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6391 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j);
6392
6393 if (cp_int_cntl_reg) {
6394 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg);
6395 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
6396 PRIV_INSTR_INT_ENABLE,
6397 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
6398 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl);
6399 }
6400 }
6401 }
6402 break;
6403 default:
6404 break;
6405 }
6406
6407 return 0;
6408 }
6409
gfx_v11_0_handle_priv_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6410 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
6411 struct amdgpu_iv_entry *entry)
6412 {
6413 u8 me_id, pipe_id, queue_id;
6414 struct amdgpu_ring *ring;
6415 int i;
6416
6417 me_id = (entry->ring_id & 0x0c) >> 2;
6418 pipe_id = (entry->ring_id & 0x03) >> 0;
6419 queue_id = (entry->ring_id & 0x70) >> 4;
6420
6421 switch (me_id) {
6422 case 0:
6423 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
6424 ring = &adev->gfx.gfx_ring[i];
6425 if (ring->me == me_id && ring->pipe == pipe_id &&
6426 ring->queue == queue_id)
6427 drm_sched_fault(&ring->sched);
6428 }
6429 break;
6430 case 1:
6431 case 2:
6432 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6433 ring = &adev->gfx.compute_ring[i];
6434 if (ring->me == me_id && ring->pipe == pipe_id &&
6435 ring->queue == queue_id)
6436 drm_sched_fault(&ring->sched);
6437 }
6438 break;
6439 default:
6440 BUG();
6441 break;
6442 }
6443 }
6444
gfx_v11_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6445 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
6446 struct amdgpu_irq_src *source,
6447 struct amdgpu_iv_entry *entry)
6448 {
6449 DRM_ERROR("Illegal register access in command stream\n");
6450 gfx_v11_0_handle_priv_fault(adev, entry);
6451 return 0;
6452 }
6453
gfx_v11_0_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6454 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev,
6455 struct amdgpu_irq_src *source,
6456 struct amdgpu_iv_entry *entry)
6457 {
6458 DRM_ERROR("Illegal opcode in command stream \n");
6459 gfx_v11_0_handle_priv_fault(adev, entry);
6460 return 0;
6461 }
6462
gfx_v11_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6463 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev,
6464 struct amdgpu_irq_src *source,
6465 struct amdgpu_iv_entry *entry)
6466 {
6467 DRM_ERROR("Illegal instruction in command stream\n");
6468 gfx_v11_0_handle_priv_fault(adev, entry);
6469 return 0;
6470 }
6471
gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6472 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev,
6473 struct amdgpu_irq_src *source,
6474 struct amdgpu_iv_entry *entry)
6475 {
6476 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq)
6477 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry);
6478
6479 return 0;
6480 }
6481
6482 #if 0
6483 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6484 struct amdgpu_irq_src *src,
6485 unsigned int type,
6486 enum amdgpu_interrupt_state state)
6487 {
6488 uint32_t tmp, target;
6489 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring);
6490
6491 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL);
6492 target += ring->pipe;
6493
6494 switch (type) {
6495 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6496 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6497 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6498 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6499 GENERIC2_INT_ENABLE, 0);
6500 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6501
6502 tmp = RREG32_SOC15_IP(GC, target);
6503 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6504 GENERIC2_INT_ENABLE, 0);
6505 WREG32_SOC15_IP(GC, target, tmp);
6506 } else {
6507 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL);
6508 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6509 GENERIC2_INT_ENABLE, 1);
6510 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp);
6511
6512 tmp = RREG32_SOC15_IP(GC, target);
6513 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL,
6514 GENERIC2_INT_ENABLE, 1);
6515 WREG32_SOC15_IP(GC, target, tmp);
6516 }
6517 break;
6518 default:
6519 BUG(); /* kiq only support GENERIC2_INT now */
6520 break;
6521 }
6522 return 0;
6523 }
6524 #endif
6525
gfx_v11_0_emit_mem_sync(struct amdgpu_ring * ring)6526 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
6527 {
6528 const unsigned int gcr_cntl =
6529 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
6530 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
6531 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) |
6532 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) |
6533 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) |
6534 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
6535 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
6536 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1);
6537
6538 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
6539 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
6540 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
6541 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
6542 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
6543 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6544 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
6545 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
6546 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
6547 }
6548
gfx_v11_0_reset_kgq(struct amdgpu_ring * ring,unsigned int vmid)6549 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, unsigned int vmid)
6550 {
6551 struct amdgpu_device *adev = ring->adev;
6552 int r;
6553
6554 if (amdgpu_sriov_vf(adev))
6555 return -EINVAL;
6556
6557 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, false);
6558 if (r)
6559 return r;
6560
6561 r = amdgpu_bo_reserve(ring->mqd_obj, false);
6562 if (unlikely(r != 0)) {
6563 dev_err(adev->dev, "fail to resv mqd_obj\n");
6564 return r;
6565 }
6566 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
6567 if (!r) {
6568 r = gfx_v11_0_kgq_init_queue(ring, true);
6569 amdgpu_bo_kunmap(ring->mqd_obj);
6570 ring->mqd_ptr = NULL;
6571 }
6572 amdgpu_bo_unreserve(ring->mqd_obj);
6573 if (r) {
6574 dev_err(adev->dev, "fail to unresv mqd_obj\n");
6575 return r;
6576 }
6577
6578 r = amdgpu_mes_map_legacy_queue(adev, ring);
6579 if (r) {
6580 dev_err(adev->dev, "failed to remap kgq\n");
6581 return r;
6582 }
6583
6584 return amdgpu_ring_test_ring(ring);
6585 }
6586
gfx_v11_0_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid)6587 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, unsigned int vmid)
6588 {
6589 struct amdgpu_device *adev = ring->adev;
6590 int i, r = 0;
6591
6592 if (amdgpu_sriov_vf(adev))
6593 return -EINVAL;
6594
6595 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
6596 mutex_lock(&adev->srbm_mutex);
6597 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
6598 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
6599 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
6600
6601 /* make sure dequeue is complete*/
6602 for (i = 0; i < adev->usec_timeout; i++) {
6603 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1))
6604 break;
6605 udelay(1);
6606 }
6607 if (i >= adev->usec_timeout)
6608 r = -ETIMEDOUT;
6609 soc21_grbm_select(adev, 0, 0, 0, 0);
6610 mutex_unlock(&adev->srbm_mutex);
6611 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
6612 if (r) {
6613 dev_err(adev->dev, "fail to wait on hqd deactivate\n");
6614 return r;
6615 }
6616
6617 r = amdgpu_bo_reserve(ring->mqd_obj, false);
6618 if (unlikely(r != 0)) {
6619 dev_err(adev->dev, "fail to resv mqd_obj\n");
6620 return r;
6621 }
6622 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
6623 if (!r) {
6624 r = gfx_v11_0_kcq_init_queue(ring, true);
6625 amdgpu_bo_kunmap(ring->mqd_obj);
6626 ring->mqd_ptr = NULL;
6627 }
6628 amdgpu_bo_unreserve(ring->mqd_obj);
6629 if (r) {
6630 dev_err(adev->dev, "fail to unresv mqd_obj\n");
6631 return r;
6632 }
6633 r = amdgpu_mes_map_legacy_queue(adev, ring);
6634 if (r) {
6635 dev_err(adev->dev, "failed to remap kcq\n");
6636 return r;
6637 }
6638
6639 return amdgpu_ring_test_ring(ring);
6640 }
6641
gfx_v11_ip_print(void * handle,struct drm_printer * p)6642 static void gfx_v11_ip_print(void *handle, struct drm_printer *p)
6643 {
6644 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6645 uint32_t i, j, k, reg, index = 0;
6646 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6647
6648 if (!adev->gfx.ip_dump_core)
6649 return;
6650
6651 for (i = 0; i < reg_count; i++)
6652 drm_printf(p, "%-50s \t 0x%08x\n",
6653 gc_reg_list_11_0[i].reg_name,
6654 adev->gfx.ip_dump_core[i]);
6655
6656 /* print compute queue registers for all instances */
6657 if (!adev->gfx.ip_dump_compute_queues)
6658 return;
6659
6660 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6661 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n",
6662 adev->gfx.mec.num_mec,
6663 adev->gfx.mec.num_pipe_per_mec,
6664 adev->gfx.mec.num_queue_per_pipe);
6665
6666 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6667 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6668 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6669 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k);
6670 for (reg = 0; reg < reg_count; reg++) {
6671 drm_printf(p, "%-50s \t 0x%08x\n",
6672 gc_cp_reg_list_11[reg].reg_name,
6673 adev->gfx.ip_dump_compute_queues[index + reg]);
6674 }
6675 index += reg_count;
6676 }
6677 }
6678 }
6679
6680 /* print gfx queue registers for all instances */
6681 if (!adev->gfx.ip_dump_gfx_queues)
6682 return;
6683
6684 index = 0;
6685 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6686 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n",
6687 adev->gfx.me.num_me,
6688 adev->gfx.me.num_pipe_per_me,
6689 adev->gfx.me.num_queue_per_pipe);
6690
6691 for (i = 0; i < adev->gfx.me.num_me; i++) {
6692 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6693 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6694 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k);
6695 for (reg = 0; reg < reg_count; reg++) {
6696 drm_printf(p, "%-50s \t 0x%08x\n",
6697 gc_gfx_queue_reg_list_11[reg].reg_name,
6698 adev->gfx.ip_dump_gfx_queues[index + reg]);
6699 }
6700 index += reg_count;
6701 }
6702 }
6703 }
6704 }
6705
gfx_v11_ip_dump(void * handle)6706 static void gfx_v11_ip_dump(void *handle)
6707 {
6708 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6709 uint32_t i, j, k, reg, index = 0;
6710 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0);
6711
6712 if (!adev->gfx.ip_dump_core)
6713 return;
6714
6715 amdgpu_gfx_off_ctrl(adev, false);
6716 for (i = 0; i < reg_count; i++)
6717 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i]));
6718 amdgpu_gfx_off_ctrl(adev, true);
6719
6720 /* dump compute queue registers for all instances */
6721 if (!adev->gfx.ip_dump_compute_queues)
6722 return;
6723
6724 reg_count = ARRAY_SIZE(gc_cp_reg_list_11);
6725 amdgpu_gfx_off_ctrl(adev, false);
6726 mutex_lock(&adev->srbm_mutex);
6727 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
6728 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
6729 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
6730 /* ME0 is for GFX so start from 1 for CP */
6731 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0);
6732 for (reg = 0; reg < reg_count; reg++) {
6733 adev->gfx.ip_dump_compute_queues[index + reg] =
6734 RREG32(SOC15_REG_ENTRY_OFFSET(
6735 gc_cp_reg_list_11[reg]));
6736 }
6737 index += reg_count;
6738 }
6739 }
6740 }
6741 soc21_grbm_select(adev, 0, 0, 0, 0);
6742 mutex_unlock(&adev->srbm_mutex);
6743 amdgpu_gfx_off_ctrl(adev, true);
6744
6745 /* dump gfx queue registers for all instances */
6746 if (!adev->gfx.ip_dump_gfx_queues)
6747 return;
6748
6749 index = 0;
6750 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11);
6751 amdgpu_gfx_off_ctrl(adev, false);
6752 mutex_lock(&adev->srbm_mutex);
6753 for (i = 0; i < adev->gfx.me.num_me; i++) {
6754 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) {
6755 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) {
6756 soc21_grbm_select(adev, i, j, k, 0);
6757
6758 for (reg = 0; reg < reg_count; reg++) {
6759 adev->gfx.ip_dump_gfx_queues[index + reg] =
6760 RREG32(SOC15_REG_ENTRY_OFFSET(
6761 gc_gfx_queue_reg_list_11[reg]));
6762 }
6763 index += reg_count;
6764 }
6765 }
6766 }
6767 soc21_grbm_select(adev, 0, 0, 0, 0);
6768 mutex_unlock(&adev->srbm_mutex);
6769 amdgpu_gfx_off_ctrl(adev, true);
6770 }
6771
6772 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
6773 .name = "gfx_v11_0",
6774 .early_init = gfx_v11_0_early_init,
6775 .late_init = gfx_v11_0_late_init,
6776 .sw_init = gfx_v11_0_sw_init,
6777 .sw_fini = gfx_v11_0_sw_fini,
6778 .hw_init = gfx_v11_0_hw_init,
6779 .hw_fini = gfx_v11_0_hw_fini,
6780 .suspend = gfx_v11_0_suspend,
6781 .resume = gfx_v11_0_resume,
6782 .is_idle = gfx_v11_0_is_idle,
6783 .wait_for_idle = gfx_v11_0_wait_for_idle,
6784 .soft_reset = gfx_v11_0_soft_reset,
6785 .check_soft_reset = gfx_v11_0_check_soft_reset,
6786 .post_soft_reset = gfx_v11_0_post_soft_reset,
6787 .set_clockgating_state = gfx_v11_0_set_clockgating_state,
6788 .set_powergating_state = gfx_v11_0_set_powergating_state,
6789 .get_clockgating_state = gfx_v11_0_get_clockgating_state,
6790 .dump_ip_state = gfx_v11_ip_dump,
6791 .print_ip_state = gfx_v11_ip_print,
6792 };
6793
6794 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
6795 .type = AMDGPU_RING_TYPE_GFX,
6796 .align_mask = 0xff,
6797 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6798 .support_64bit_ptrs = true,
6799 .secure_submission_supported = true,
6800 .get_rptr = gfx_v11_0_ring_get_rptr_gfx,
6801 .get_wptr = gfx_v11_0_ring_get_wptr_gfx,
6802 .set_wptr = gfx_v11_0_ring_set_wptr_gfx,
6803 .emit_frame_size = /* totally 247 maximum if 16 IBs */
6804 5 + /* update_spm_vmid */
6805 5 + /* COND_EXEC */
6806 22 + /* SET_Q_PREEMPTION_MODE */
6807 7 + /* PIPELINE_SYNC */
6808 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6809 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6810 4 + /* VM_FLUSH */
6811 8 + /* FENCE for VM_FLUSH */
6812 20 + /* GDS switch */
6813 5 + /* COND_EXEC */
6814 7 + /* HDP_flush */
6815 4 + /* VGT_flush */
6816 31 + /* DE_META */
6817 3 + /* CNTX_CTRL */
6818 5 + /* HDP_INVL */
6819 22 + /* SET_Q_PREEMPTION_MODE */
6820 8 + 8 + /* FENCE x2 */
6821 8, /* gfx_v11_0_emit_mem_sync */
6822 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */
6823 .emit_ib = gfx_v11_0_ring_emit_ib_gfx,
6824 .emit_fence = gfx_v11_0_ring_emit_fence,
6825 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6826 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6827 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6828 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6829 .test_ring = gfx_v11_0_ring_test_ring,
6830 .test_ib = gfx_v11_0_ring_test_ib,
6831 .insert_nop = gfx_v11_ring_insert_nop,
6832 .pad_ib = amdgpu_ring_generic_pad_ib,
6833 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
6834 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
6835 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
6836 .preempt_ib = gfx_v11_0_ring_preempt_ib,
6837 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
6838 .emit_wreg = gfx_v11_0_ring_emit_wreg,
6839 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6840 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6841 .soft_recovery = gfx_v11_0_ring_soft_recovery,
6842 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6843 .reset = gfx_v11_0_reset_kgq,
6844 };
6845
6846 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = {
6847 .type = AMDGPU_RING_TYPE_COMPUTE,
6848 .align_mask = 0xff,
6849 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6850 .support_64bit_ptrs = true,
6851 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6852 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6853 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6854 .emit_frame_size =
6855 5 + /* update_spm_vmid */
6856 20 + /* gfx_v11_0_ring_emit_gds_switch */
6857 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6858 5 + /* hdp invalidate */
6859 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6860 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6861 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6862 2 + /* gfx_v11_0_ring_emit_vm_flush */
6863 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */
6864 8, /* gfx_v11_0_emit_mem_sync */
6865 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6866 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6867 .emit_fence = gfx_v11_0_ring_emit_fence,
6868 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync,
6869 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush,
6870 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch,
6871 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush,
6872 .test_ring = gfx_v11_0_ring_test_ring,
6873 .test_ib = gfx_v11_0_ring_test_ib,
6874 .insert_nop = gfx_v11_ring_insert_nop,
6875 .pad_ib = amdgpu_ring_generic_pad_ib,
6876 .emit_wreg = gfx_v11_0_ring_emit_wreg,
6877 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6878 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6879 .soft_recovery = gfx_v11_0_ring_soft_recovery,
6880 .emit_mem_sync = gfx_v11_0_emit_mem_sync,
6881 .reset = gfx_v11_0_reset_kcq,
6882 };
6883
6884 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = {
6885 .type = AMDGPU_RING_TYPE_KIQ,
6886 .align_mask = 0xff,
6887 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6888 .support_64bit_ptrs = true,
6889 .get_rptr = gfx_v11_0_ring_get_rptr_compute,
6890 .get_wptr = gfx_v11_0_ring_get_wptr_compute,
6891 .set_wptr = gfx_v11_0_ring_set_wptr_compute,
6892 .emit_frame_size =
6893 20 + /* gfx_v11_0_ring_emit_gds_switch */
6894 7 + /* gfx_v11_0_ring_emit_hdp_flush */
6895 5 + /*hdp invalidate */
6896 7 + /* gfx_v11_0_ring_emit_pipeline_sync */
6897 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6898 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6899 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6900 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */
6901 .emit_ib = gfx_v11_0_ring_emit_ib_compute,
6902 .emit_fence = gfx_v11_0_ring_emit_fence_kiq,
6903 .test_ring = gfx_v11_0_ring_test_ring,
6904 .test_ib = gfx_v11_0_ring_test_ib,
6905 .insert_nop = amdgpu_ring_insert_nop,
6906 .pad_ib = amdgpu_ring_generic_pad_ib,
6907 .emit_rreg = gfx_v11_0_ring_emit_rreg,
6908 .emit_wreg = gfx_v11_0_ring_emit_wreg,
6909 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
6910 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait,
6911 };
6912
gfx_v11_0_set_ring_funcs(struct amdgpu_device * adev)6913 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev)
6914 {
6915 int i;
6916
6917 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq;
6918
6919 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6920 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx;
6921
6922 for (i = 0; i < adev->gfx.num_compute_rings; i++)
6923 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute;
6924 }
6925
6926 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = {
6927 .set = gfx_v11_0_set_eop_interrupt_state,
6928 .process = gfx_v11_0_eop_irq,
6929 };
6930
6931 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = {
6932 .set = gfx_v11_0_set_priv_reg_fault_state,
6933 .process = gfx_v11_0_priv_reg_irq,
6934 };
6935
6936 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = {
6937 .set = gfx_v11_0_set_bad_op_fault_state,
6938 .process = gfx_v11_0_bad_op_irq,
6939 };
6940
6941 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = {
6942 .set = gfx_v11_0_set_priv_inst_fault_state,
6943 .process = gfx_v11_0_priv_inst_irq,
6944 };
6945
6946 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = {
6947 .process = gfx_v11_0_rlc_gc_fed_irq,
6948 };
6949
gfx_v11_0_set_irq_funcs(struct amdgpu_device * adev)6950 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev)
6951 {
6952 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6953 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs;
6954
6955 adev->gfx.priv_reg_irq.num_types = 1;
6956 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs;
6957
6958 adev->gfx.bad_op_irq.num_types = 1;
6959 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs;
6960
6961 adev->gfx.priv_inst_irq.num_types = 1;
6962 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs;
6963
6964 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */
6965 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs;
6966
6967 }
6968
gfx_v11_0_set_imu_funcs(struct amdgpu_device * adev)6969 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev)
6970 {
6971 if (adev->flags & AMD_IS_APU)
6972 adev->gfx.imu.mode = MISSION_MODE;
6973 else
6974 adev->gfx.imu.mode = DEBUG_MODE;
6975
6976 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs;
6977 }
6978
gfx_v11_0_set_rlc_funcs(struct amdgpu_device * adev)6979 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev)
6980 {
6981 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs;
6982 }
6983
gfx_v11_0_set_gds_init(struct amdgpu_device * adev)6984 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev)
6985 {
6986 unsigned total_cu = adev->gfx.config.max_cu_per_sh *
6987 adev->gfx.config.max_sh_per_se *
6988 adev->gfx.config.max_shader_engines;
6989
6990 adev->gds.gds_size = 0x1000;
6991 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1;
6992 adev->gds.gws_size = 64;
6993 adev->gds.oa_size = 16;
6994 }
6995
gfx_v11_0_set_mqd_funcs(struct amdgpu_device * adev)6996 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev)
6997 {
6998 /* set gfx eng mqd */
6999 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size =
7000 sizeof(struct v11_gfx_mqd);
7001 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd =
7002 gfx_v11_0_gfx_mqd_init;
7003 /* set compute eng mqd */
7004 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
7005 sizeof(struct v11_compute_mqd);
7006 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
7007 gfx_v11_0_compute_mqd_init;
7008 }
7009
gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device * adev,u32 bitmap)7010 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
7011 u32 bitmap)
7012 {
7013 u32 data;
7014
7015 if (!bitmap)
7016 return;
7017
7018 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7019 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7020
7021 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data);
7022 }
7023
gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device * adev)7024 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
7025 {
7026 u32 data, wgp_bitmask;
7027 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG);
7028 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG);
7029
7030 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
7031 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
7032
7033 wgp_bitmask =
7034 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
7035
7036 return (~data) & wgp_bitmask;
7037 }
7038
gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device * adev)7039 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
7040 {
7041 u32 wgp_idx, wgp_active_bitmap;
7042 u32 cu_bitmap_per_wgp, cu_active_bitmap;
7043
7044 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev);
7045 cu_active_bitmap = 0;
7046
7047 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
7048 /* if there is one WGP enabled, it means 2 CUs will be enabled */
7049 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
7050 if (wgp_active_bitmap & (1 << wgp_idx))
7051 cu_active_bitmap |= cu_bitmap_per_wgp;
7052 }
7053
7054 return cu_active_bitmap;
7055 }
7056
gfx_v11_0_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)7057 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev,
7058 struct amdgpu_cu_info *cu_info)
7059 {
7060 int i, j, k, counter, active_cu_number = 0;
7061 u32 mask, bitmap;
7062 unsigned disable_masks[8 * 2];
7063
7064 if (!adev || !cu_info)
7065 return -EINVAL;
7066
7067 amdgpu_gfx_parse_disable_cu(disable_masks, 8, 2);
7068
7069 mutex_lock(&adev->grbm_idx_mutex);
7070 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7071 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7072 bitmap = i * adev->gfx.config.max_sh_per_se + j;
7073 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1))
7074 continue;
7075 mask = 1;
7076 counter = 0;
7077 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0);
7078 if (i < 8 && j < 2)
7079 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(
7080 adev, disable_masks[i * 2 + j]);
7081 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev);
7082
7083 /**
7084 * GFX11 could support more than 4 SEs, while the bitmap
7085 * in cu_info struct is 4x4 and ioctl interface struct
7086 * drm_amdgpu_info_device should keep stable.
7087 * So we use last two columns of bitmap to store cu mask for
7088 * SEs 4 to 7, the layout of the bitmap is as below:
7089 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]}
7090 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]}
7091 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]}
7092 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]}
7093 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]}
7094 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]}
7095 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]}
7096 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]}
7097 */
7098 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap;
7099
7100 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
7101 if (bitmap & mask)
7102 counter++;
7103
7104 mask <<= 1;
7105 }
7106 active_cu_number += counter;
7107 }
7108 }
7109 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7110 mutex_unlock(&adev->grbm_idx_mutex);
7111
7112 cu_info->number = active_cu_number;
7113 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7114
7115 return 0;
7116 }
7117
7118 const struct amdgpu_ip_block_version gfx_v11_0_ip_block =
7119 {
7120 .type = AMD_IP_BLOCK_TYPE_GFX,
7121 .major = 11,
7122 .minor = 0,
7123 .rev = 0,
7124 .funcs = &gfx_v11_0_ip_funcs,
7125 };
7126