1 /*
2 * Copyright 2022 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/firmware.h>
24
25 #include "amdgpu.h"
26 #include "amdgpu_gfx.h"
27 #include "soc15.h"
28 #include "soc15d.h"
29 #include "soc15_common.h"
30 #include "vega10_enum.h"
31
32 #include "v9_structs.h"
33
34 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
35
36 #include "gc/gc_9_4_3_offset.h"
37 #include "gc/gc_9_4_3_sh_mask.h"
38
39 #include "gfx_v9_4_3.h"
40 #include "gfx_v9_4_3_cleaner_shader.h"
41 #include "amdgpu_xcp.h"
42
43 MODULE_FIRMWARE("amdgpu/gc_9_4_3_mec.bin");
44 MODULE_FIRMWARE("amdgpu/gc_9_4_4_mec.bin");
45 MODULE_FIRMWARE("amdgpu/gc_9_5_0_mec.bin");
46 MODULE_FIRMWARE("amdgpu/gc_9_4_3_rlc.bin");
47 MODULE_FIRMWARE("amdgpu/gc_9_4_4_rlc.bin");
48 MODULE_FIRMWARE("amdgpu/gc_9_5_0_rlc.bin");
49 MODULE_FIRMWARE("amdgpu/gc_9_4_3_sjt_mec.bin");
50 MODULE_FIRMWARE("amdgpu/gc_9_4_4_sjt_mec.bin");
51
52 #define GFX9_MEC_HPD_SIZE 4096
53 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
54
55 #define GOLDEN_GB_ADDR_CONFIG 0x2a114042
56 #define CP_HQD_PERSISTENT_STATE_DEFAULT 0xbe05301
57
58 #define XCC_REG_RANGE_0_LOW 0x2000 /* XCC gfxdec0 lower Bound */
59 #define XCC_REG_RANGE_0_HIGH 0x3400 /* XCC gfxdec0 upper Bound */
60 #define XCC_REG_RANGE_1_LOW 0xA000 /* XCC gfxdec1 lower Bound */
61 #define XCC_REG_RANGE_1_HIGH 0x10000 /* XCC gfxdec1 upper Bound */
62
63 #define NORMALIZE_XCC_REG_OFFSET(offset) \
64 (offset & 0xFFFF)
65
66 static const struct amdgpu_hwip_reg_entry gc_reg_list_9_4_3[] = {
67 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
68 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
69 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
70 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
71 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
72 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
73 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
74 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
75 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
76 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
77 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
78 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
79 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
80 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
81 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
82 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
83 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
84 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
85 SOC15_REG_ENTRY_STR(GC, 0, regSQC_DCACHE_UTCL1_STATUS),
86 SOC15_REG_ENTRY_STR(GC, 0, regSQC_ICACHE_UTCL1_STATUS),
87 SOC15_REG_ENTRY_STR(GC, 0, regSQ_UTCL1_STATUS),
88 SOC15_REG_ENTRY_STR(GC, 0, regTCP_UTCL1_STATUS),
89 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
90 SOC15_REG_ENTRY_STR(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL),
91 SOC15_REG_ENTRY_STR(GC, 0, regVM_L2_PROTECTION_FAULT_STATUS),
92 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
93 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
94 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
95 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC2_INSTR_PNTR),
96 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
97 SOC15_REG_ENTRY_STR(GC, 0, regRLC_STAT),
98 SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_COMMAND),
99 SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_MESSAGE),
100 SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_ARGUMENT_1),
101 SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_ARGUMENT_2),
102 SOC15_REG_ENTRY_STR(GC, 0, regSMU_RLC_RESPONSE),
103 SOC15_REG_ENTRY_STR(GC, 0, regRLC_SAFE_MODE),
104 SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_SAFE_MODE),
105 SOC15_REG_ENTRY_STR(GC, 0, regRLC_INT_STAT),
106 SOC15_REG_ENTRY_STR(GC, 0, regRLC_GPM_GENERAL_6),
107 /* SE status registers */
108 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
109 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
110 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
111 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3)
112 };
113
114 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9_4_3[] = {
115 /* compute queue registers */
116 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
117 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ACTIVE),
118 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
119 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
120 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
121 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
122 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
123 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
124 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
125 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
126 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
127 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
128 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
129 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
130 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
131 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
132 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
133 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
134 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
135 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
136 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
137 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
138 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
139 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
140 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
141 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
142 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
143 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
144 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
145 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
146 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
147 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
148 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
149 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
150 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
151 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
152 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GFX_STATUS),
153 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
154 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
155 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
156 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
157 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
158 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
159 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
160 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
161 };
162
163 struct amdgpu_gfx_ras gfx_v9_4_3_ras;
164
165 static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev);
166 static void gfx_v9_4_3_set_irq_funcs(struct amdgpu_device *adev);
167 static void gfx_v9_4_3_set_gds_init(struct amdgpu_device *adev);
168 static void gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device *adev);
169 static int gfx_v9_4_3_get_cu_info(struct amdgpu_device *adev,
170 struct amdgpu_cu_info *cu_info);
171 static void gfx_v9_4_3_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
172 static void gfx_v9_4_3_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
173
gfx_v9_4_3_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)174 static void gfx_v9_4_3_kiq_set_resources(struct amdgpu_ring *kiq_ring,
175 uint64_t queue_mask)
176 {
177 struct amdgpu_device *adev = kiq_ring->adev;
178 u64 shader_mc_addr;
179
180 /* Cleaner shader MC address */
181 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
182
183 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
184 amdgpu_ring_write(kiq_ring,
185 PACKET3_SET_RESOURCES_VMID_MASK(0) |
186 /* vmid_mask:0* queue_type:0 (KIQ) */
187 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
188 amdgpu_ring_write(kiq_ring,
189 lower_32_bits(queue_mask)); /* queue mask lo */
190 amdgpu_ring_write(kiq_ring,
191 upper_32_bits(queue_mask)); /* queue mask hi */
192 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
193 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
194 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
195 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
196 }
197
gfx_v9_4_3_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)198 static void gfx_v9_4_3_kiq_map_queues(struct amdgpu_ring *kiq_ring,
199 struct amdgpu_ring *ring)
200 {
201 struct amdgpu_device *adev = kiq_ring->adev;
202 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
203 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
204 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
205
206 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
207 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
208 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
209 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
210 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
211 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
212 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
213 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
214 /*queue_type: normal compute queue */
215 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
216 /* alloc format: all_on_one_pipe */
217 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
218 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
219 /* num_queues: must be 1 */
220 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
221 amdgpu_ring_write(kiq_ring,
222 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
223 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
224 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
225 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
226 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
227 }
228
gfx_v9_4_3_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)229 static void gfx_v9_4_3_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
230 struct amdgpu_ring *ring,
231 enum amdgpu_unmap_queues_action action,
232 u64 gpu_addr, u64 seq)
233 {
234 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
235
236 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
237 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
238 PACKET3_UNMAP_QUEUES_ACTION(action) |
239 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
240 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
241 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
242 amdgpu_ring_write(kiq_ring,
243 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
244
245 if (action == PREEMPT_QUEUES_NO_UNMAP) {
246 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
247 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
248 amdgpu_ring_write(kiq_ring, seq);
249 } else {
250 amdgpu_ring_write(kiq_ring, 0);
251 amdgpu_ring_write(kiq_ring, 0);
252 amdgpu_ring_write(kiq_ring, 0);
253 }
254 }
255
gfx_v9_4_3_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)256 static void gfx_v9_4_3_kiq_query_status(struct amdgpu_ring *kiq_ring,
257 struct amdgpu_ring *ring,
258 u64 addr,
259 u64 seq)
260 {
261 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
262
263 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
264 amdgpu_ring_write(kiq_ring,
265 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
266 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
267 PACKET3_QUERY_STATUS_COMMAND(2));
268 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
269 amdgpu_ring_write(kiq_ring,
270 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
271 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
272 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
273 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
274 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
275 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
276 }
277
gfx_v9_4_3_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)278 static void gfx_v9_4_3_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
279 uint16_t pasid, uint32_t flush_type,
280 bool all_hub)
281 {
282 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
283 amdgpu_ring_write(kiq_ring,
284 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
285 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
286 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
287 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
288 }
289
gfx_v9_4_3_kiq_reset_hw_queue(struct amdgpu_ring * kiq_ring,uint32_t queue_type,uint32_t me_id,uint32_t pipe_id,uint32_t queue_id,uint32_t xcc_id,uint32_t vmid)290 static void gfx_v9_4_3_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
291 uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
292 uint32_t xcc_id, uint32_t vmid)
293 {
294 struct amdgpu_device *adev = kiq_ring->adev;
295 unsigned i;
296
297 /* enter save mode */
298 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
299 mutex_lock(&adev->srbm_mutex);
300 soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, xcc_id);
301
302 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
303 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 0x2);
304 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_COMPUTE_QUEUE_RESET, 0x1);
305 /* wait till dequeue take effects */
306 for (i = 0; i < adev->usec_timeout; i++) {
307 if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
308 break;
309 udelay(1);
310 }
311 if (i >= adev->usec_timeout)
312 dev_err(adev->dev, "fail to wait on hqd deactive\n");
313 } else {
314 dev_err(adev->dev, "reset queue_type(%d) not supported\n\n", queue_type);
315 }
316
317 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
318 mutex_unlock(&adev->srbm_mutex);
319 /* exit safe mode */
320 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
321 }
322
323 static const struct kiq_pm4_funcs gfx_v9_4_3_kiq_pm4_funcs = {
324 .kiq_set_resources = gfx_v9_4_3_kiq_set_resources,
325 .kiq_map_queues = gfx_v9_4_3_kiq_map_queues,
326 .kiq_unmap_queues = gfx_v9_4_3_kiq_unmap_queues,
327 .kiq_query_status = gfx_v9_4_3_kiq_query_status,
328 .kiq_invalidate_tlbs = gfx_v9_4_3_kiq_invalidate_tlbs,
329 .kiq_reset_hw_queue = gfx_v9_4_3_kiq_reset_hw_queue,
330 .set_resources_size = 8,
331 .map_queues_size = 7,
332 .unmap_queues_size = 6,
333 .query_status_size = 7,
334 .invalidate_tlbs_size = 2,
335 };
336
gfx_v9_4_3_set_kiq_pm4_funcs(struct amdgpu_device * adev)337 static void gfx_v9_4_3_set_kiq_pm4_funcs(struct amdgpu_device *adev)
338 {
339 int i, num_xcc;
340
341 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
342 for (i = 0; i < num_xcc; i++)
343 adev->gfx.kiq[i].pmf = &gfx_v9_4_3_kiq_pm4_funcs;
344 }
345
gfx_v9_4_3_init_golden_registers(struct amdgpu_device * adev)346 static void gfx_v9_4_3_init_golden_registers(struct amdgpu_device *adev)
347 {
348 int i, num_xcc, dev_inst;
349
350 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
351 for (i = 0; i < num_xcc; i++) {
352 dev_inst = GET_INST(GC, i);
353
354 WREG32_SOC15(GC, dev_inst, regGB_ADDR_CONFIG,
355 GOLDEN_GB_ADDR_CONFIG);
356 WREG32_FIELD15_PREREG(GC, dev_inst, TCP_UTCL1_CNTL2, SPARE, 0x1);
357 }
358 }
359
gfx_v9_4_3_normalize_xcc_reg_offset(uint32_t reg)360 static uint32_t gfx_v9_4_3_normalize_xcc_reg_offset(uint32_t reg)
361 {
362 uint32_t normalized_reg = NORMALIZE_XCC_REG_OFFSET(reg);
363
364 /* If it is an XCC reg, normalize the reg to keep
365 lower 16 bits in local xcc */
366
367 if (((normalized_reg >= XCC_REG_RANGE_0_LOW) && (normalized_reg < XCC_REG_RANGE_0_HIGH)) ||
368 ((normalized_reg >= XCC_REG_RANGE_1_LOW) && (normalized_reg < XCC_REG_RANGE_1_HIGH)))
369 return normalized_reg;
370 else
371 return reg;
372 }
373
gfx_v9_4_3_write_data_to_reg(struct amdgpu_ring * ring,int eng_sel,bool wc,uint32_t reg,uint32_t val)374 static void gfx_v9_4_3_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
375 bool wc, uint32_t reg, uint32_t val)
376 {
377 reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
378 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
379 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
380 WRITE_DATA_DST_SEL(0) |
381 (wc ? WR_CONFIRM : 0));
382 amdgpu_ring_write(ring, reg);
383 amdgpu_ring_write(ring, 0);
384 amdgpu_ring_write(ring, val);
385 }
386
gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)387 static void gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
388 int mem_space, int opt, uint32_t addr0,
389 uint32_t addr1, uint32_t ref, uint32_t mask,
390 uint32_t inv)
391 {
392 /* Only do the normalization on regspace */
393 if (mem_space == 0) {
394 addr0 = gfx_v9_4_3_normalize_xcc_reg_offset(addr0);
395 addr1 = gfx_v9_4_3_normalize_xcc_reg_offset(addr1);
396 }
397
398 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
399 amdgpu_ring_write(ring,
400 /* memory (1) or register (0) */
401 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
402 WAIT_REG_MEM_OPERATION(opt) | /* wait */
403 WAIT_REG_MEM_FUNCTION(3) | /* equal */
404 WAIT_REG_MEM_ENGINE(eng_sel)));
405
406 if (mem_space)
407 WARN_ON(addr0 & 0x3); /* Dword align */
408 amdgpu_ring_write(ring, addr0);
409 amdgpu_ring_write(ring, addr1);
410 amdgpu_ring_write(ring, ref);
411 amdgpu_ring_write(ring, mask);
412 amdgpu_ring_write(ring, inv); /* poll interval */
413 }
414
gfx_v9_4_3_ring_test_ring(struct amdgpu_ring * ring)415 static int gfx_v9_4_3_ring_test_ring(struct amdgpu_ring *ring)
416 {
417 uint32_t scratch_reg0_offset, xcc_offset;
418 struct amdgpu_device *adev = ring->adev;
419 uint32_t tmp = 0;
420 unsigned i;
421 int r;
422
423 /* Use register offset which is local to XCC in the packet */
424 xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
425 scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0);
426 WREG32(scratch_reg0_offset, 0xCAFEDEAD);
427 tmp = RREG32(scratch_reg0_offset);
428
429 r = amdgpu_ring_alloc(ring, 3);
430 if (r)
431 return r;
432
433 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
434 amdgpu_ring_write(ring, xcc_offset - PACKET3_SET_UCONFIG_REG_START);
435 amdgpu_ring_write(ring, 0xDEADBEEF);
436 amdgpu_ring_commit(ring);
437
438 for (i = 0; i < adev->usec_timeout; i++) {
439 tmp = RREG32(scratch_reg0_offset);
440 if (tmp == 0xDEADBEEF)
441 break;
442 udelay(1);
443 }
444
445 if (i >= adev->usec_timeout)
446 r = -ETIMEDOUT;
447 return r;
448 }
449
gfx_v9_4_3_ring_test_ib(struct amdgpu_ring * ring,long timeout)450 static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
451 {
452 struct amdgpu_device *adev = ring->adev;
453 struct amdgpu_ib ib;
454 struct dma_fence *f = NULL;
455
456 unsigned index;
457 uint64_t gpu_addr;
458 uint32_t tmp;
459 long r;
460
461 r = amdgpu_wb_get(adev, &index);
462 if (r)
463 return r;
464
465 gpu_addr = adev->wb.gpu_addr + (index * 4);
466 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
467 memset(&ib, 0, sizeof(ib));
468
469 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
470 if (r)
471 goto err1;
472
473 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
474 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
475 ib.ptr[2] = lower_32_bits(gpu_addr);
476 ib.ptr[3] = upper_32_bits(gpu_addr);
477 ib.ptr[4] = 0xDEADBEEF;
478 ib.length_dw = 5;
479
480 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
481 if (r)
482 goto err2;
483
484 r = dma_fence_wait_timeout(f, false, timeout);
485 if (r == 0) {
486 r = -ETIMEDOUT;
487 goto err2;
488 } else if (r < 0) {
489 goto err2;
490 }
491
492 tmp = adev->wb.wb[index];
493 if (tmp == 0xDEADBEEF)
494 r = 0;
495 else
496 r = -EINVAL;
497
498 err2:
499 amdgpu_ib_free(&ib, NULL);
500 dma_fence_put(f);
501 err1:
502 amdgpu_wb_free(adev, index);
503 return r;
504 }
505
506
507 /* This value might differs per partition */
gfx_v9_4_3_get_gpu_clock_counter(struct amdgpu_device * adev)508 static uint64_t gfx_v9_4_3_get_gpu_clock_counter(struct amdgpu_device *adev)
509 {
510 uint64_t clock;
511
512 mutex_lock(&adev->gfx.gpu_clock_mutex);
513 WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
514 clock = (uint64_t)RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_GPU_CLOCK_COUNT_LSB) |
515 ((uint64_t)RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
516 mutex_unlock(&adev->gfx.gpu_clock_mutex);
517
518 return clock;
519 }
520
gfx_v9_4_3_free_microcode(struct amdgpu_device * adev)521 static void gfx_v9_4_3_free_microcode(struct amdgpu_device *adev)
522 {
523 amdgpu_ucode_release(&adev->gfx.pfp_fw);
524 amdgpu_ucode_release(&adev->gfx.me_fw);
525 amdgpu_ucode_release(&adev->gfx.ce_fw);
526 amdgpu_ucode_release(&adev->gfx.rlc_fw);
527 amdgpu_ucode_release(&adev->gfx.mec_fw);
528 amdgpu_ucode_release(&adev->gfx.mec2_fw);
529
530 kfree(adev->gfx.rlc.register_list_format);
531 }
532
gfx_v9_4_3_init_rlc_microcode(struct amdgpu_device * adev,const char * chip_name)533 static int gfx_v9_4_3_init_rlc_microcode(struct amdgpu_device *adev,
534 const char *chip_name)
535 {
536 int err;
537 const struct rlc_firmware_header_v2_0 *rlc_hdr;
538 uint16_t version_major;
539 uint16_t version_minor;
540
541
542 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
543 AMDGPU_UCODE_REQUIRED,
544 "amdgpu/%s_rlc.bin", chip_name);
545 if (err)
546 goto out;
547 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
548
549 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
550 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
551 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
552 out:
553 if (err)
554 amdgpu_ucode_release(&adev->gfx.rlc_fw);
555
556 return err;
557 }
558
gfx_v9_4_3_init_cp_compute_microcode(struct amdgpu_device * adev,const char * chip_name)559 static int gfx_v9_4_3_init_cp_compute_microcode(struct amdgpu_device *adev,
560 const char *chip_name)
561 {
562 int err;
563
564 if (amdgpu_sriov_vf(adev)) {
565 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
566 AMDGPU_UCODE_REQUIRED,
567 "amdgpu/%s_sjt_mec.bin", chip_name);
568
569 if (err)
570 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
571 AMDGPU_UCODE_REQUIRED,
572 "amdgpu/%s_mec.bin", chip_name);
573 } else
574 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
575 AMDGPU_UCODE_REQUIRED,
576 "amdgpu/%s_mec.bin", chip_name);
577 if (err)
578 goto out;
579 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
580 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
581
582 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
583 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
584
585 out:
586 if (err)
587 amdgpu_ucode_release(&adev->gfx.mec_fw);
588 return err;
589 }
590
gfx_v9_4_3_init_microcode(struct amdgpu_device * adev)591 static int gfx_v9_4_3_init_microcode(struct amdgpu_device *adev)
592 {
593 char ucode_prefix[15];
594 int r;
595
596 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
597
598 r = gfx_v9_4_3_init_rlc_microcode(adev, ucode_prefix);
599 if (r)
600 return r;
601
602 r = gfx_v9_4_3_init_cp_compute_microcode(adev, ucode_prefix);
603 if (r)
604 return r;
605
606 return r;
607 }
608
gfx_v9_4_3_mec_fini(struct amdgpu_device * adev)609 static void gfx_v9_4_3_mec_fini(struct amdgpu_device *adev)
610 {
611 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
612 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
613 }
614
gfx_v9_4_3_mec_init(struct amdgpu_device * adev)615 static int gfx_v9_4_3_mec_init(struct amdgpu_device *adev)
616 {
617 int r, i, num_xcc;
618 u32 *hpd;
619 const __le32 *fw_data;
620 unsigned fw_size;
621 u32 *fw;
622 size_t mec_hpd_size;
623
624 const struct gfx_firmware_header_v1_0 *mec_hdr;
625
626 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
627 for (i = 0; i < num_xcc; i++)
628 bitmap_zero(adev->gfx.mec_bitmap[i].queue_bitmap,
629 AMDGPU_MAX_COMPUTE_QUEUES);
630
631 /* take ownership of the relevant compute queues */
632 amdgpu_gfx_compute_queue_acquire(adev);
633 mec_hpd_size =
634 adev->gfx.num_compute_rings * num_xcc * GFX9_MEC_HPD_SIZE;
635 if (mec_hpd_size) {
636 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
637 AMDGPU_GEM_DOMAIN_VRAM |
638 AMDGPU_GEM_DOMAIN_GTT,
639 &adev->gfx.mec.hpd_eop_obj,
640 &adev->gfx.mec.hpd_eop_gpu_addr,
641 (void **)&hpd);
642 if (r) {
643 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
644 gfx_v9_4_3_mec_fini(adev);
645 return r;
646 }
647
648 if (amdgpu_emu_mode == 1) {
649 for (i = 0; i < mec_hpd_size / 4; i++) {
650 memset((void *)(hpd + i), 0, 4);
651 if (i % 50 == 0)
652 msleep(1);
653 }
654 } else {
655 memset(hpd, 0, mec_hpd_size);
656 }
657
658 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
659 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
660 }
661
662 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
663
664 fw_data = (const __le32 *)
665 (adev->gfx.mec_fw->data +
666 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
667 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
668
669 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
670 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
671 &adev->gfx.mec.mec_fw_obj,
672 &adev->gfx.mec.mec_fw_gpu_addr,
673 (void **)&fw);
674 if (r) {
675 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
676 gfx_v9_4_3_mec_fini(adev);
677 return r;
678 }
679
680 memcpy(fw, fw_data, fw_size);
681
682 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
683 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
684
685 return 0;
686 }
687
gfx_v9_4_3_xcc_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)688 static void gfx_v9_4_3_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
689 u32 sh_num, u32 instance, int xcc_id)
690 {
691 u32 data;
692
693 if (instance == 0xffffffff)
694 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
695 INSTANCE_BROADCAST_WRITES, 1);
696 else
697 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
698 INSTANCE_INDEX, instance);
699
700 if (se_num == 0xffffffff)
701 data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
702 SE_BROADCAST_WRITES, 1);
703 else
704 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
705
706 if (sh_num == 0xffffffff)
707 data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
708 SH_BROADCAST_WRITES, 1);
709 else
710 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
711
712 WREG32_SOC15_RLC_SHADOW_EX(reg, GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX, data);
713 }
714
wave_read_ind(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t address)715 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t address)
716 {
717 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
718 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
719 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
720 (address << SQ_IND_INDEX__INDEX__SHIFT) |
721 (SQ_IND_INDEX__FORCE_READ_MASK));
722 return RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
723 }
724
wave_read_regs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)725 static void wave_read_regs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
726 uint32_t wave, uint32_t thread,
727 uint32_t regno, uint32_t num, uint32_t *out)
728 {
729 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
730 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
731 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
732 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
733 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
734 (SQ_IND_INDEX__FORCE_READ_MASK) |
735 (SQ_IND_INDEX__AUTO_INCR_MASK));
736 while (num--)
737 *(out++) = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
738 }
739
gfx_v9_4_3_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)740 static void gfx_v9_4_3_read_wave_data(struct amdgpu_device *adev,
741 uint32_t xcc_id, uint32_t simd, uint32_t wave,
742 uint32_t *dst, int *no_fields)
743 {
744 /* type 1 wave data */
745 dst[(*no_fields)++] = 1;
746 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_STATUS);
747 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_PC_LO);
748 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_PC_HI);
749 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_EXEC_LO);
750 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_EXEC_HI);
751 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_HW_ID);
752 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_INST_DW0);
753 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_INST_DW1);
754 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_GPR_ALLOC);
755 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_LDS_ALLOC);
756 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_TRAPSTS);
757 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_IB_STS);
758 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_IB_DBG0);
759 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_M0);
760 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_MODE);
761 }
762
gfx_v9_4_3_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)763 static void gfx_v9_4_3_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
764 uint32_t wave, uint32_t start,
765 uint32_t size, uint32_t *dst)
766 {
767 wave_read_regs(adev, xcc_id, simd, wave, 0,
768 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
769 }
770
gfx_v9_4_3_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)771 static void gfx_v9_4_3_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
772 uint32_t wave, uint32_t thread,
773 uint32_t start, uint32_t size,
774 uint32_t *dst)
775 {
776 wave_read_regs(adev, xcc_id, simd, wave, thread,
777 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
778 }
779
gfx_v9_4_3_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)780 static void gfx_v9_4_3_select_me_pipe_q(struct amdgpu_device *adev,
781 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
782 {
783 soc15_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
784 }
785
gfx_v9_4_3_get_xccs_per_xcp(struct amdgpu_device * adev)786 static int gfx_v9_4_3_get_xccs_per_xcp(struct amdgpu_device *adev)
787 {
788 u32 xcp_ctl;
789
790 /* Value is expected to be the same on all, fetch from first instance */
791 xcp_ctl = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HYP_XCP_CTL);
792
793 return REG_GET_FIELD(xcp_ctl, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP);
794 }
795
gfx_v9_4_3_switch_compute_partition(struct amdgpu_device * adev,int num_xccs_per_xcp)796 static int gfx_v9_4_3_switch_compute_partition(struct amdgpu_device *adev,
797 int num_xccs_per_xcp)
798 {
799 int ret, i, num_xcc;
800 u32 tmp = 0;
801
802 if (adev->psp.funcs) {
803 ret = psp_spatial_partition(&adev->psp,
804 NUM_XCC(adev->gfx.xcc_mask) /
805 num_xccs_per_xcp);
806 if (ret)
807 return ret;
808 } else {
809 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
810
811 for (i = 0; i < num_xcc; i++) {
812 tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP,
813 num_xccs_per_xcp);
814 tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, VIRTUAL_XCC_ID,
815 i % num_xccs_per_xcp);
816 WREG32_SOC15(GC, GET_INST(GC, i), regCP_HYP_XCP_CTL,
817 tmp);
818 }
819 ret = 0;
820 }
821
822 adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;
823
824 return ret;
825 }
826
gfx_v9_4_3_ih_to_xcc_inst(struct amdgpu_device * adev,int ih_node)827 static int gfx_v9_4_3_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
828 {
829 int xcc;
830
831 xcc = hweight8(adev->gfx.xcc_mask & GENMASK(ih_node / 2, 0));
832 if (!xcc) {
833 dev_err(adev->dev, "Couldn't find xcc mapping from IH node");
834 return -EINVAL;
835 }
836
837 return xcc - 1;
838 }
839
840 static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
841 .get_gpu_clock_counter = &gfx_v9_4_3_get_gpu_clock_counter,
842 .select_se_sh = &gfx_v9_4_3_xcc_select_se_sh,
843 .read_wave_data = &gfx_v9_4_3_read_wave_data,
844 .read_wave_sgprs = &gfx_v9_4_3_read_wave_sgprs,
845 .read_wave_vgprs = &gfx_v9_4_3_read_wave_vgprs,
846 .select_me_pipe_q = &gfx_v9_4_3_select_me_pipe_q,
847 .switch_partition_mode = &gfx_v9_4_3_switch_compute_partition,
848 .ih_node_to_logical_xcc = &gfx_v9_4_3_ih_to_xcc_inst,
849 .get_xccs_per_xcp = &gfx_v9_4_3_get_xccs_per_xcp,
850 .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
851 };
852
gfx_v9_4_3_gpu_early_init(struct amdgpu_device * adev)853 static int gfx_v9_4_3_gpu_early_init(struct amdgpu_device *adev)
854 {
855 adev->gfx.funcs = &gfx_v9_4_3_gfx_funcs;
856 adev->gfx.ras = &gfx_v9_4_3_ras;
857
858 adev->gfx.config.max_hw_contexts = 8;
859 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
860 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
861 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
862 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
863 adev->gfx.config.gb_addr_config = GOLDEN_GB_ADDR_CONFIG;
864
865 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
866 REG_GET_FIELD(
867 adev->gfx.config.gb_addr_config,
868 GB_ADDR_CONFIG,
869 NUM_PIPES);
870
871 adev->gfx.config.max_tile_pipes =
872 adev->gfx.config.gb_addr_config_fields.num_pipes;
873
874 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
875 REG_GET_FIELD(
876 adev->gfx.config.gb_addr_config,
877 GB_ADDR_CONFIG,
878 NUM_BANKS);
879 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
880 REG_GET_FIELD(
881 adev->gfx.config.gb_addr_config,
882 GB_ADDR_CONFIG,
883 MAX_COMPRESSED_FRAGS);
884 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
885 REG_GET_FIELD(
886 adev->gfx.config.gb_addr_config,
887 GB_ADDR_CONFIG,
888 NUM_RB_PER_SE);
889 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
890 REG_GET_FIELD(
891 adev->gfx.config.gb_addr_config,
892 GB_ADDR_CONFIG,
893 NUM_SHADER_ENGINES);
894 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
895 REG_GET_FIELD(
896 adev->gfx.config.gb_addr_config,
897 GB_ADDR_CONFIG,
898 PIPE_INTERLEAVE_SIZE));
899
900 return 0;
901 }
902
gfx_v9_4_3_compute_ring_init(struct amdgpu_device * adev,int ring_id,int xcc_id,int mec,int pipe,int queue)903 static int gfx_v9_4_3_compute_ring_init(struct amdgpu_device *adev, int ring_id,
904 int xcc_id, int mec, int pipe, int queue)
905 {
906 unsigned irq_type;
907 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
908 unsigned int hw_prio;
909 uint32_t xcc_doorbell_start;
910
911 ring = &adev->gfx.compute_ring[xcc_id * adev->gfx.num_compute_rings +
912 ring_id];
913
914 /* mec0 is me1 */
915 ring->xcc_id = xcc_id;
916 ring->me = mec + 1;
917 ring->pipe = pipe;
918 ring->queue = queue;
919
920 ring->ring_obj = NULL;
921 ring->use_doorbell = true;
922 xcc_doorbell_start = adev->doorbell_index.mec_ring0 +
923 xcc_id * adev->doorbell_index.xcc_doorbell_range;
924 ring->doorbell_index = (xcc_doorbell_start + ring_id) << 1;
925 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
926 (ring_id + xcc_id * adev->gfx.num_compute_rings) *
927 GFX9_MEC_HPD_SIZE;
928 ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
929 sprintf(ring->name, "comp_%d.%d.%d.%d",
930 ring->xcc_id, ring->me, ring->pipe, ring->queue);
931
932 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
933 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
934 + ring->pipe;
935 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
936 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
937 /* type-2 packets are deprecated on MEC, use type-3 instead */
938 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
939 hw_prio, NULL);
940 }
941
gfx_v9_4_3_alloc_ip_dump(struct amdgpu_device * adev)942 static void gfx_v9_4_3_alloc_ip_dump(struct amdgpu_device *adev)
943 {
944 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
945 uint32_t *ptr, num_xcc, inst;
946
947 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
948
949 ptr = kcalloc(reg_count * num_xcc, sizeof(uint32_t), GFP_KERNEL);
950 if (!ptr) {
951 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
952 adev->gfx.ip_dump_core = NULL;
953 } else {
954 adev->gfx.ip_dump_core = ptr;
955 }
956
957 /* Allocate memory for compute queue registers for all the instances */
958 reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
959 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
960 adev->gfx.mec.num_queue_per_pipe;
961
962 ptr = kcalloc(reg_count * inst * num_xcc, sizeof(uint32_t), GFP_KERNEL);
963 if (!ptr) {
964 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
965 adev->gfx.ip_dump_compute_queues = NULL;
966 } else {
967 adev->gfx.ip_dump_compute_queues = ptr;
968 }
969 }
970
gfx_v9_4_3_sw_init(struct amdgpu_ip_block * ip_block)971 static int gfx_v9_4_3_sw_init(struct amdgpu_ip_block *ip_block)
972 {
973 int i, j, k, r, ring_id, xcc_id, num_xcc;
974 struct amdgpu_device *adev = ip_block->adev;
975
976 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
977 case IP_VERSION(9, 4, 3):
978 case IP_VERSION(9, 4, 4):
979 adev->gfx.cleaner_shader_ptr = gfx_9_4_3_cleaner_shader_hex;
980 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_3_cleaner_shader_hex);
981 if (adev->gfx.mec_fw_version >= 153) {
982 adev->gfx.enable_cleaner_shader = true;
983 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size);
984 if (r) {
985 adev->gfx.enable_cleaner_shader = false;
986 dev_err(adev->dev, "Failed to initialize cleaner shader\n");
987 }
988 }
989 break;
990 default:
991 adev->gfx.enable_cleaner_shader = false;
992 break;
993 }
994
995 adev->gfx.mec.num_mec = 2;
996 adev->gfx.mec.num_pipe_per_mec = 4;
997 adev->gfx.mec.num_queue_per_pipe = 8;
998
999 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1000
1001 /* EOP Event */
1002 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
1003 if (r)
1004 return r;
1005
1006 /* Bad opcode Event */
1007 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
1008 GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
1009 &adev->gfx.bad_op_irq);
1010 if (r)
1011 return r;
1012
1013 /* Privileged reg */
1014 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
1015 &adev->gfx.priv_reg_irq);
1016 if (r)
1017 return r;
1018
1019 /* Privileged inst */
1020 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
1021 &adev->gfx.priv_inst_irq);
1022 if (r)
1023 return r;
1024
1025 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1026
1027 r = adev->gfx.rlc.funcs->init(adev);
1028 if (r) {
1029 DRM_ERROR("Failed to init rlc BOs!\n");
1030 return r;
1031 }
1032
1033 r = gfx_v9_4_3_mec_init(adev);
1034 if (r) {
1035 DRM_ERROR("Failed to init MEC BOs!\n");
1036 return r;
1037 }
1038
1039 /* set up the compute queues - allocate horizontally across pipes */
1040 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1041 ring_id = 0;
1042 if (!adev->gfx.disable_kq) {
1043 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1044 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1045 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec;
1046 k++) {
1047 if (!amdgpu_gfx_is_mec_queue_enabled(
1048 adev, xcc_id, i, k, j))
1049 continue;
1050
1051 r = gfx_v9_4_3_compute_ring_init(adev,
1052 ring_id,
1053 xcc_id,
1054 i, k, j);
1055 if (r)
1056 return r;
1057
1058 ring_id++;
1059 }
1060 }
1061 }
1062 }
1063
1064 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, xcc_id);
1065 if (r) {
1066 DRM_ERROR("Failed to init KIQ BOs!\n");
1067 return r;
1068 }
1069
1070 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1071 if (r)
1072 return r;
1073
1074 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
1075 r = amdgpu_gfx_mqd_sw_init(adev,
1076 sizeof(struct v9_mqd_allocation), xcc_id);
1077 if (r)
1078 return r;
1079 }
1080
1081 adev->gfx.compute_supported_reset =
1082 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
1083 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1084 case IP_VERSION(9, 4, 3):
1085 case IP_VERSION(9, 4, 4):
1086 if ((adev->gfx.mec_fw_version >= 155) &&
1087 !amdgpu_sriov_vf(adev) &&
1088 !adev->debug_disable_gpu_ring_reset) {
1089 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1090 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE;
1091 }
1092 break;
1093 case IP_VERSION(9, 5, 0):
1094 if ((adev->gfx.mec_fw_version >= 21) &&
1095 !amdgpu_sriov_vf(adev) &&
1096 !adev->debug_disable_gpu_ring_reset) {
1097 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1098 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE;
1099 }
1100 break;
1101 default:
1102 break;
1103 }
1104 r = gfx_v9_4_3_gpu_early_init(adev);
1105 if (r)
1106 return r;
1107
1108 r = amdgpu_gfx_ras_sw_init(adev);
1109 if (r)
1110 return r;
1111
1112 r = amdgpu_gfx_sysfs_init(adev);
1113 if (r)
1114 return r;
1115
1116 gfx_v9_4_3_alloc_ip_dump(adev);
1117
1118 return 0;
1119 }
1120
gfx_v9_4_3_sw_fini(struct amdgpu_ip_block * ip_block)1121 static int gfx_v9_4_3_sw_fini(struct amdgpu_ip_block *ip_block)
1122 {
1123 int i, num_xcc;
1124 struct amdgpu_device *adev = ip_block->adev;
1125
1126 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1127 for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)
1128 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1129
1130 for (i = 0; i < num_xcc; i++) {
1131 amdgpu_gfx_mqd_sw_fini(adev, i);
1132 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[i].ring);
1133 amdgpu_gfx_kiq_fini(adev, i);
1134 }
1135
1136 amdgpu_gfx_cleaner_shader_sw_fini(adev);
1137
1138 gfx_v9_4_3_mec_fini(adev);
1139 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
1140 gfx_v9_4_3_free_microcode(adev);
1141 amdgpu_gfx_sysfs_fini(adev);
1142
1143 kfree(adev->gfx.ip_dump_core);
1144 kfree(adev->gfx.ip_dump_compute_queues);
1145
1146 return 0;
1147 }
1148
1149 #define DEFAULT_SH_MEM_BASES (0x6000)
gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device * adev,int xcc_id)1150 static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev,
1151 int xcc_id)
1152 {
1153 int i;
1154 uint32_t sh_mem_config;
1155 uint32_t sh_mem_bases;
1156 uint32_t data;
1157
1158 /*
1159 * Configure apertures:
1160 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
1161 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
1162 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
1163 */
1164 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1165
1166 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
1167 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
1168 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
1169
1170 mutex_lock(&adev->srbm_mutex);
1171 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1172 soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
1173 /* CP and shaders */
1174 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, sh_mem_config);
1175 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, sh_mem_bases);
1176
1177 /* Enable trap for each kfd vmid. */
1178 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL);
1179 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1180 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL, data);
1181 }
1182 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1183 mutex_unlock(&adev->srbm_mutex);
1184
1185 /*
1186 * Initialize all compute VMIDs to have no GDS, GWS, or OA
1187 * access. These should be enabled by FW for target VMIDs.
1188 */
1189 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1190 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * i, 0);
1191 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * i, 0);
1192 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_GWS_VMID0, i, 0);
1193 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_OA_VMID0, i, 0);
1194 }
1195 }
1196
gfx_v9_4_3_xcc_init_gds_vmid(struct amdgpu_device * adev,int xcc_id)1197 static void gfx_v9_4_3_xcc_init_gds_vmid(struct amdgpu_device *adev, int xcc_id)
1198 {
1199 int vmid;
1200
1201 /*
1202 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1203 * access. Compute VMIDs should be enabled by FW for target VMIDs,
1204 * the driver can enable them for graphics. VMID0 should maintain
1205 * access so that HWS firmware can save/restore entries.
1206 */
1207 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
1208 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * vmid, 0);
1209 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * vmid, 0);
1210 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_GWS_VMID0, vmid, 0);
1211 WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_OA_VMID0, vmid, 0);
1212 }
1213 }
1214
1215 /* For ASICs that needs xnack chain and MEC version supports, set SG_CONFIG1
1216 * DISABLE_XNACK_CHECK_IN_RETRY_DISABLE bit and inform KFD to set xnack_chain
1217 * bit in SET_RESOURCES
1218 */
gfx_v9_4_3_xcc_init_sq(struct amdgpu_device * adev,int xcc_id)1219 static void gfx_v9_4_3_xcc_init_sq(struct amdgpu_device *adev, int xcc_id)
1220 {
1221 uint32_t data;
1222
1223 if (!(adev->gmc.xnack_flags & AMDGPU_GMC_XNACK_FLAG_CHAIN))
1224 return;
1225
1226 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_CONFIG1);
1227 data = REG_SET_FIELD(data, SQ_CONFIG1, DISABLE_XNACK_CHECK_IN_RETRY_DISABLE, 1);
1228 WREG32_SOC15(GC, xcc_id, regSQ_CONFIG1, data);
1229 }
1230
gfx_v9_4_3_xcc_constants_init(struct amdgpu_device * adev,int xcc_id)1231 static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev,
1232 int xcc_id)
1233 {
1234 u32 tmp;
1235 int i;
1236
1237 /* XXX SH_MEM regs */
1238 /* where to put LDS, scratch, GPUVM in FSA64 space */
1239 mutex_lock(&adev->srbm_mutex);
1240 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1241 soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
1242 /* CP and shaders */
1243 if (i == 0) {
1244 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1245 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1246 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
1247 !!adev->gmc.noretry);
1248 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1249 regSH_MEM_CONFIG, tmp);
1250 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1251 regSH_MEM_BASES, 0);
1252 } else {
1253 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1254 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1255 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
1256 !!adev->gmc.noretry);
1257 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1258 regSH_MEM_CONFIG, tmp);
1259 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1260 (adev->gmc.private_aperture_start >>
1261 48));
1262 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1263 (adev->gmc.shared_aperture_start >>
1264 48));
1265 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1266 regSH_MEM_BASES, tmp);
1267 }
1268 }
1269 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, 0));
1270
1271 mutex_unlock(&adev->srbm_mutex);
1272
1273 gfx_v9_4_3_xcc_init_compute_vmid(adev, xcc_id);
1274 gfx_v9_4_3_xcc_init_gds_vmid(adev, xcc_id);
1275 gfx_v9_4_3_xcc_init_sq(adev, xcc_id);
1276 }
1277
gfx_v9_4_3_constants_init(struct amdgpu_device * adev)1278 static void gfx_v9_4_3_constants_init(struct amdgpu_device *adev)
1279 {
1280 int i, num_xcc;
1281
1282 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1283
1284 gfx_v9_4_3_get_cu_info(adev, &adev->gfx.cu_info);
1285 adev->gfx.config.db_debug2 =
1286 RREG32_SOC15(GC, GET_INST(GC, 0), regDB_DEBUG2);
1287
1288 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1289 /* ToDo: GC 9.4.4 */
1290 case IP_VERSION(9, 4, 3):
1291 if (adev->gfx.mec_fw_version >= 184 &&
1292 (amdgpu_sriov_reg_access_sq_config(adev) ||
1293 !amdgpu_sriov_vf(adev)))
1294 adev->gmc.xnack_flags |= AMDGPU_GMC_XNACK_FLAG_CHAIN;
1295 break;
1296 case IP_VERSION(9, 5, 0):
1297 if (adev->gfx.mec_fw_version >= 23)
1298 adev->gmc.xnack_flags |= AMDGPU_GMC_XNACK_FLAG_CHAIN;
1299 break;
1300 default:
1301 break;
1302 }
1303
1304 for (i = 0; i < num_xcc; i++)
1305 gfx_v9_4_3_xcc_constants_init(adev, i);
1306 }
1307
1308 static void
gfx_v9_4_3_xcc_enable_save_restore_machine(struct amdgpu_device * adev,int xcc_id)1309 gfx_v9_4_3_xcc_enable_save_restore_machine(struct amdgpu_device *adev,
1310 int xcc_id)
1311 {
1312 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_SRM_CNTL, SRM_ENABLE, 1);
1313 }
1314
gfx_v9_4_3_xcc_init_pg(struct amdgpu_device * adev,int xcc_id)1315 static void gfx_v9_4_3_xcc_init_pg(struct amdgpu_device *adev, int xcc_id)
1316 {
1317 /*
1318 * Rlc save restore list is workable since v2_1.
1319 */
1320 gfx_v9_4_3_xcc_enable_save_restore_machine(adev, xcc_id);
1321 }
1322
gfx_v9_4_3_xcc_disable_gpa_mode(struct amdgpu_device * adev,int xcc_id)1323 static void gfx_v9_4_3_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id)
1324 {
1325 uint32_t data;
1326
1327 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG);
1328 data |= CPC_PSP_DEBUG__UTCL2IUGPAOVERRIDE_MASK;
1329 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG, data);
1330 }
1331
gfx_v9_4_3_is_rlc_enabled(struct amdgpu_device * adev)1332 static bool gfx_v9_4_3_is_rlc_enabled(struct amdgpu_device *adev)
1333 {
1334 uint32_t rlc_setting;
1335
1336 /* if RLC is not enabled, do nothing */
1337 rlc_setting = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CNTL);
1338 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
1339 return false;
1340
1341 return true;
1342 }
1343
gfx_v9_4_3_xcc_set_safe_mode(struct amdgpu_device * adev,int xcc_id)1344 static void gfx_v9_4_3_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
1345 {
1346 uint32_t data;
1347 unsigned i;
1348
1349 data = RLC_SAFE_MODE__CMD_MASK;
1350 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
1351 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data);
1352
1353 /* wait for RLC_SAFE_MODE */
1354 for (i = 0; i < adev->usec_timeout; i++) {
1355 if (!REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
1356 break;
1357 udelay(1);
1358 }
1359 }
1360
gfx_v9_4_3_xcc_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)1361 static void gfx_v9_4_3_xcc_unset_safe_mode(struct amdgpu_device *adev,
1362 int xcc_id)
1363 {
1364 uint32_t data;
1365
1366 data = RLC_SAFE_MODE__CMD_MASK;
1367 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data);
1368 }
1369
gfx_v9_4_3_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)1370 static void gfx_v9_4_3_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1371 {
1372 int xcc_id, num_xcc;
1373 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1374
1375 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1376 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1377 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[GET_INST(GC, xcc_id)];
1378 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG0);
1379 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG1);
1380 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG2);
1381 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG3);
1382 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_CNTL);
1383 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX);
1384 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPARE_INT);
1385 }
1386 adev->gfx.rlc.rlcg_reg_access_supported = true;
1387 }
1388
gfx_v9_4_3_rlc_init(struct amdgpu_device * adev)1389 static int gfx_v9_4_3_rlc_init(struct amdgpu_device *adev)
1390 {
1391 /* init spm vmid with 0xf */
1392 if (adev->gfx.rlc.funcs->update_spm_vmid)
1393 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
1394
1395 return 0;
1396 }
1397
gfx_v9_4_3_xcc_wait_for_rlc_serdes(struct amdgpu_device * adev,int xcc_id)1398 static void gfx_v9_4_3_xcc_wait_for_rlc_serdes(struct amdgpu_device *adev,
1399 int xcc_id)
1400 {
1401 u32 i, j, k;
1402 u32 mask;
1403
1404 mutex_lock(&adev->grbm_idx_mutex);
1405 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1406 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1407 gfx_v9_4_3_xcc_select_se_sh(adev, i, j, 0xffffffff,
1408 xcc_id);
1409 for (k = 0; k < adev->usec_timeout; k++) {
1410 if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SERDES_CU_MASTER_BUSY) == 0)
1411 break;
1412 udelay(1);
1413 }
1414 if (k == adev->usec_timeout) {
1415 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff,
1416 0xffffffff,
1417 0xffffffff, xcc_id);
1418 mutex_unlock(&adev->grbm_idx_mutex);
1419 drm_info(adev_to_drm(adev), "Timeout wait for RLC serdes %u,%u\n",
1420 i, j);
1421 return;
1422 }
1423 }
1424 }
1425 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
1426 xcc_id);
1427 mutex_unlock(&adev->grbm_idx_mutex);
1428
1429 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
1430 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
1431 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
1432 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
1433 for (k = 0; k < adev->usec_timeout; k++) {
1434 if ((RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
1435 break;
1436 udelay(1);
1437 }
1438 }
1439
gfx_v9_4_3_xcc_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable,int xcc_id)1440 static void gfx_v9_4_3_xcc_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1441 bool enable, int xcc_id)
1442 {
1443 u32 tmp;
1444
1445 /* These interrupts should be enabled to drive DS clock */
1446
1447 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0);
1448
1449 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
1450 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
1451 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
1452
1453 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0, tmp);
1454 }
1455
gfx_v9_4_3_xcc_rlc_stop(struct amdgpu_device * adev,int xcc_id)1456 static void gfx_v9_4_3_xcc_rlc_stop(struct amdgpu_device *adev, int xcc_id)
1457 {
1458 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL,
1459 RLC_ENABLE_F32, 0);
1460 gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
1461 gfx_v9_4_3_xcc_wait_for_rlc_serdes(adev, xcc_id);
1462 }
1463
gfx_v9_4_3_rlc_stop(struct amdgpu_device * adev)1464 static void gfx_v9_4_3_rlc_stop(struct amdgpu_device *adev)
1465 {
1466 int i, num_xcc;
1467
1468 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1469 for (i = 0; i < num_xcc; i++)
1470 gfx_v9_4_3_xcc_rlc_stop(adev, i);
1471 }
1472
gfx_v9_4_3_xcc_rlc_reset(struct amdgpu_device * adev,int xcc_id)1473 static void gfx_v9_4_3_xcc_rlc_reset(struct amdgpu_device *adev, int xcc_id)
1474 {
1475 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), GRBM_SOFT_RESET,
1476 SOFT_RESET_RLC, 1);
1477 udelay(50);
1478 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), GRBM_SOFT_RESET,
1479 SOFT_RESET_RLC, 0);
1480 udelay(50);
1481 }
1482
gfx_v9_4_3_rlc_reset(struct amdgpu_device * adev)1483 static void gfx_v9_4_3_rlc_reset(struct amdgpu_device *adev)
1484 {
1485 int i, num_xcc;
1486
1487 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1488 for (i = 0; i < num_xcc; i++)
1489 gfx_v9_4_3_xcc_rlc_reset(adev, i);
1490 }
1491
gfx_v9_4_3_xcc_rlc_start(struct amdgpu_device * adev,int xcc_id)1492 static void gfx_v9_4_3_xcc_rlc_start(struct amdgpu_device *adev, int xcc_id)
1493 {
1494 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL,
1495 RLC_ENABLE_F32, 1);
1496 udelay(50);
1497
1498 /* carrizo do enable cp interrupt after cp inited */
1499 if (!(adev->flags & AMD_IS_APU)) {
1500 gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, true, xcc_id);
1501 udelay(50);
1502 }
1503 }
1504
gfx_v9_4_3_rlc_start(struct amdgpu_device * adev)1505 static void gfx_v9_4_3_rlc_start(struct amdgpu_device *adev)
1506 {
1507 #ifdef AMDGPU_RLC_DEBUG_RETRY
1508 u32 rlc_ucode_ver;
1509 #endif
1510 int i, num_xcc;
1511
1512 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1513 for (i = 0; i < num_xcc; i++) {
1514 gfx_v9_4_3_xcc_rlc_start(adev, i);
1515 #ifdef AMDGPU_RLC_DEBUG_RETRY
1516 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
1517 rlc_ucode_ver = RREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_GENERAL_6);
1518 if (rlc_ucode_ver == 0x108) {
1519 dev_info(adev->dev,
1520 "Using rlc debug ucode. regRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i\n",
1521 rlc_ucode_ver, adev->gfx.rlc_fw_version);
1522 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
1523 * default is 0x9C4 to create a 100us interval */
1524 WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_TIMER_INT_3, 0x9C4);
1525 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
1526 * to disable the page fault retry interrupts, default is
1527 * 0x100 (256) */
1528 WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_GENERAL_12, 0x100);
1529 }
1530 #endif
1531 }
1532 }
1533
gfx_v9_4_3_xcc_rlc_load_microcode(struct amdgpu_device * adev,int xcc_id)1534 static int gfx_v9_4_3_xcc_rlc_load_microcode(struct amdgpu_device *adev,
1535 int xcc_id)
1536 {
1537 const struct rlc_firmware_header_v2_0 *hdr;
1538 const __le32 *fw_data;
1539 unsigned i, fw_size;
1540
1541 if (!adev->gfx.rlc_fw)
1542 return -EINVAL;
1543
1544 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1545 amdgpu_ucode_print_rlc_hdr(&hdr->header);
1546
1547 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1548 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1549 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1550
1551 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR,
1552 RLCG_UCODE_LOADING_START_ADDRESS);
1553 for (i = 0; i < fw_size; i++) {
1554 if (amdgpu_emu_mode == 1 && i % 100 == 0) {
1555 dev_info(adev->dev, "Write RLC ucode data %u DWs\n", i);
1556 msleep(1);
1557 }
1558 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
1559 }
1560 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1561
1562 return 0;
1563 }
1564
gfx_v9_4_3_xcc_rlc_resume(struct amdgpu_device * adev,int xcc_id)1565 static int gfx_v9_4_3_xcc_rlc_resume(struct amdgpu_device *adev, int xcc_id)
1566 {
1567 int r;
1568
1569 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
1570 gfx_v9_4_3_xcc_rlc_stop(adev, xcc_id);
1571 /* legacy rlc firmware loading */
1572 r = gfx_v9_4_3_xcc_rlc_load_microcode(adev, xcc_id);
1573 if (r)
1574 return r;
1575 gfx_v9_4_3_xcc_rlc_start(adev, xcc_id);
1576 }
1577
1578 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1579 /* disable CG */
1580 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, 0);
1581 gfx_v9_4_3_xcc_init_pg(adev, xcc_id);
1582 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1583
1584 return 0;
1585 }
1586
gfx_v9_4_3_rlc_resume(struct amdgpu_device * adev)1587 static int gfx_v9_4_3_rlc_resume(struct amdgpu_device *adev)
1588 {
1589 int r, i, num_xcc;
1590
1591 if (amdgpu_sriov_vf(adev))
1592 return 0;
1593
1594 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1595 for (i = 0; i < num_xcc; i++) {
1596 r = gfx_v9_4_3_xcc_rlc_resume(adev, i);
1597 if (r)
1598 return r;
1599 }
1600
1601 return 0;
1602 }
1603
gfx_v9_4_3_update_spm_vmid(struct amdgpu_device * adev,int inst,struct amdgpu_ring * ring,unsigned int vmid)1604 static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev,
1605 int inst, struct amdgpu_ring *ring, unsigned int vmid)
1606 {
1607 u32 reg, pre_data, data;
1608
1609 reg = SOC15_REG_OFFSET(GC, GET_INST(GC, inst), regRLC_SPM_MC_CNTL);
1610 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
1611 pre_data = RREG32_NO_KIQ(reg);
1612 else
1613 pre_data = RREG32(reg);
1614
1615 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
1616 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
1617
1618 if (pre_data != data) {
1619 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
1620 WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, inst), regRLC_SPM_MC_CNTL, data);
1621 } else
1622 WREG32_SOC15(GC, GET_INST(GC, inst), regRLC_SPM_MC_CNTL, data);
1623 }
1624 }
1625
1626 static const struct soc15_reg_rlcg rlcg_access_gc_9_4_3[] = {
1627 {SOC15_REG_ENTRY(GC, 0, regGRBM_GFX_INDEX)},
1628 {SOC15_REG_ENTRY(GC, 0, regSQ_IND_INDEX)},
1629 };
1630
gfx_v9_4_3_check_rlcg_range(struct amdgpu_device * adev,uint32_t offset,struct soc15_reg_rlcg * entries,int arr_size)1631 static bool gfx_v9_4_3_check_rlcg_range(struct amdgpu_device *adev,
1632 uint32_t offset,
1633 struct soc15_reg_rlcg *entries, int arr_size)
1634 {
1635 int i, inst;
1636 uint32_t reg;
1637
1638 if (!entries)
1639 return false;
1640
1641 for (i = 0; i < arr_size; i++) {
1642 const struct soc15_reg_rlcg *entry;
1643
1644 entry = &entries[i];
1645 inst = adev->ip_map.logical_to_dev_inst ?
1646 adev->ip_map.logical_to_dev_inst(
1647 adev, entry->hwip, entry->instance) :
1648 entry->instance;
1649 reg = adev->reg_offset[entry->hwip][inst][entry->segment] +
1650 entry->reg;
1651 if (offset == reg)
1652 return true;
1653 }
1654
1655 return false;
1656 }
1657
gfx_v9_4_3_is_rlcg_access_range(struct amdgpu_device * adev,u32 offset)1658 static bool gfx_v9_4_3_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
1659 {
1660 return gfx_v9_4_3_check_rlcg_range(adev, offset,
1661 (void *)rlcg_access_gc_9_4_3,
1662 ARRAY_SIZE(rlcg_access_gc_9_4_3));
1663 }
1664
gfx_v9_4_3_xcc_cp_compute_enable(struct amdgpu_device * adev,bool enable,int xcc_id)1665 static void gfx_v9_4_3_xcc_cp_compute_enable(struct amdgpu_device *adev,
1666 bool enable, int xcc_id)
1667 {
1668 if (enable) {
1669 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL, 0);
1670 } else {
1671 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL,
1672 (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
1673 CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
1674 CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
1675 CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
1676 CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
1677 CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
1678 CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
1679 CP_MEC_CNTL__MEC_ME1_HALT_MASK |
1680 CP_MEC_CNTL__MEC_ME2_HALT_MASK));
1681 adev->gfx.kiq[xcc_id].ring.sched.ready = false;
1682 }
1683 udelay(50);
1684 }
1685
gfx_v9_4_3_xcc_cp_compute_load_microcode(struct amdgpu_device * adev,int xcc_id)1686 static int gfx_v9_4_3_xcc_cp_compute_load_microcode(struct amdgpu_device *adev,
1687 int xcc_id)
1688 {
1689 const struct gfx_firmware_header_v1_0 *mec_hdr;
1690 const __le32 *fw_data;
1691 unsigned i;
1692 u32 tmp;
1693 u32 mec_ucode_addr_offset;
1694 u32 mec_ucode_data_offset;
1695
1696 if (!adev->gfx.mec_fw)
1697 return -EINVAL;
1698
1699 gfx_v9_4_3_xcc_cp_compute_enable(adev, false, xcc_id);
1700
1701 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1702 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
1703
1704 fw_data = (const __le32 *)
1705 (adev->gfx.mec_fw->data +
1706 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1707 tmp = 0;
1708 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
1709 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
1710 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL, tmp);
1711
1712 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_LO,
1713 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
1714 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_HI,
1715 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
1716
1717 mec_ucode_addr_offset =
1718 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_MEC_ME1_UCODE_ADDR);
1719 mec_ucode_data_offset =
1720 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_MEC_ME1_UCODE_DATA);
1721
1722 /* MEC1 */
1723 WREG32(mec_ucode_addr_offset, mec_hdr->jt_offset);
1724 for (i = 0; i < mec_hdr->jt_size; i++)
1725 WREG32(mec_ucode_data_offset,
1726 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
1727
1728 WREG32(mec_ucode_addr_offset, adev->gfx.mec_fw_version);
1729 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
1730
1731 return 0;
1732 }
1733
1734 /* KIQ functions */
gfx_v9_4_3_xcc_kiq_setting(struct amdgpu_ring * ring,int xcc_id)1735 static void gfx_v9_4_3_xcc_kiq_setting(struct amdgpu_ring *ring, int xcc_id)
1736 {
1737 uint32_t tmp;
1738 struct amdgpu_device *adev = ring->adev;
1739
1740 /* tell RLC which is KIQ queue */
1741 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
1742 tmp &= 0xffffff00;
1743 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1744 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp | 0x80);
1745 }
1746
gfx_v9_4_3_mqd_set_priority(struct amdgpu_ring * ring,struct v9_mqd * mqd)1747 static void gfx_v9_4_3_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
1748 {
1749 struct amdgpu_device *adev = ring->adev;
1750
1751 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
1752 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
1753 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
1754 mqd->cp_hqd_queue_priority =
1755 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
1756 }
1757 }
1758 }
1759
gfx_v9_4_3_xcc_mqd_init(struct amdgpu_ring * ring,int xcc_id)1760 static int gfx_v9_4_3_xcc_mqd_init(struct amdgpu_ring *ring, int xcc_id)
1761 {
1762 struct amdgpu_device *adev = ring->adev;
1763 struct v9_mqd *mqd = ring->mqd_ptr;
1764 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1765 uint32_t tmp;
1766
1767 mqd->header = 0xC0310800;
1768 mqd->compute_pipelinestat_enable = 0x00000001;
1769 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1770 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1771 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1772 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1773 mqd->compute_misc_reserved = 0x00000003;
1774
1775 mqd->dynamic_cu_mask_addr_lo =
1776 lower_32_bits(ring->mqd_gpu_addr
1777 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
1778 mqd->dynamic_cu_mask_addr_hi =
1779 upper_32_bits(ring->mqd_gpu_addr
1780 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
1781
1782 eop_base_addr = ring->eop_gpu_addr >> 8;
1783 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
1784 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1785
1786 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1787 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL);
1788 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1789 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
1790
1791 mqd->cp_hqd_eop_control = tmp;
1792
1793 /* enable doorbell? */
1794 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL);
1795
1796 if (ring->use_doorbell) {
1797 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1798 DOORBELL_OFFSET, ring->doorbell_index);
1799 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1800 DOORBELL_EN, 1);
1801 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1802 DOORBELL_SOURCE, 0);
1803 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1804 DOORBELL_HIT, 0);
1805 if (amdgpu_sriov_multi_vf_mode(adev))
1806 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1807 DOORBELL_MODE, 1);
1808 } else {
1809 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1810 DOORBELL_EN, 0);
1811 }
1812
1813 mqd->cp_hqd_pq_doorbell_control = tmp;
1814
1815 /* disable the queue if it's active */
1816 ring->wptr = 0;
1817 mqd->cp_hqd_dequeue_request = 0;
1818 mqd->cp_hqd_pq_rptr = 0;
1819 mqd->cp_hqd_pq_wptr_lo = 0;
1820 mqd->cp_hqd_pq_wptr_hi = 0;
1821
1822 /* set the pointer to the MQD */
1823 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1824 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1825
1826 /* set MQD vmid to 0 */
1827 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL);
1828 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1829 mqd->cp_mqd_control = tmp;
1830
1831 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1832 hqd_gpu_addr = ring->gpu_addr >> 8;
1833 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
1834 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1835
1836 /* set up the HQD, this is similar to CP_RB0_CNTL */
1837 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL);
1838 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1839 (order_base_2(ring->ring_size / 4) - 1));
1840 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1841 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1842 #ifdef __BIG_ENDIAN
1843 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
1844 #endif
1845 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
1846 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
1847 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1848 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1849 mqd->cp_hqd_pq_control = tmp;
1850
1851 /* set the wb address whether it's enabled or not */
1852 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
1853 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1854 mqd->cp_hqd_pq_rptr_report_addr_hi =
1855 upper_32_bits(wb_gpu_addr) & 0xffff;
1856
1857 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1858 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
1859 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
1860 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1861
1862 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
1863 ring->wptr = 0;
1864 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR);
1865
1866 /* set the vmid for the queue */
1867 mqd->cp_hqd_vmid = 0;
1868
1869 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE);
1870 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
1871 mqd->cp_hqd_persistent_state = tmp;
1872
1873 /* set MIN_IB_AVAIL_SIZE */
1874 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_IB_CONTROL);
1875 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
1876 mqd->cp_hqd_ib_control = tmp;
1877
1878 /* set static priority for a queue/ring */
1879 gfx_v9_4_3_mqd_set_priority(ring, mqd);
1880 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM);
1881 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
1882 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
1883 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
1884 mqd->cp_hqd_quantum = tmp;
1885
1886 /* map_queues packet doesn't need activate the queue,
1887 * so only kiq need set this field.
1888 */
1889 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
1890 mqd->cp_hqd_active = 1;
1891
1892 return 0;
1893 }
1894
gfx_v9_4_3_xcc_kiq_init_register(struct amdgpu_ring * ring,int xcc_id)1895 static int gfx_v9_4_3_xcc_kiq_init_register(struct amdgpu_ring *ring,
1896 int xcc_id)
1897 {
1898 struct amdgpu_device *adev = ring->adev;
1899 struct v9_mqd *mqd = ring->mqd_ptr;
1900 int j;
1901
1902 /* disable wptr polling */
1903 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
1904
1905 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR,
1906 mqd->cp_hqd_eop_base_addr_lo);
1907 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR_HI,
1908 mqd->cp_hqd_eop_base_addr_hi);
1909
1910 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1911 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL,
1912 mqd->cp_hqd_eop_control);
1913
1914 /* enable doorbell? */
1915 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
1916 mqd->cp_hqd_pq_doorbell_control);
1917
1918 /* disable the queue if it's active */
1919 if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
1920 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
1921 for (j = 0; j < adev->usec_timeout; j++) {
1922 if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
1923 break;
1924 udelay(1);
1925 }
1926 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
1927 mqd->cp_hqd_dequeue_request);
1928 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR,
1929 mqd->cp_hqd_pq_rptr);
1930 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
1931 mqd->cp_hqd_pq_wptr_lo);
1932 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
1933 mqd->cp_hqd_pq_wptr_hi);
1934 }
1935
1936 /* set the pointer to the MQD */
1937 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR,
1938 mqd->cp_mqd_base_addr_lo);
1939 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI,
1940 mqd->cp_mqd_base_addr_hi);
1941
1942 /* set MQD vmid to 0 */
1943 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL,
1944 mqd->cp_mqd_control);
1945
1946 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1947 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE,
1948 mqd->cp_hqd_pq_base_lo);
1949 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI,
1950 mqd->cp_hqd_pq_base_hi);
1951
1952 /* set up the HQD, this is similar to CP_RB0_CNTL */
1953 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL,
1954 mqd->cp_hqd_pq_control);
1955
1956 /* set the wb address whether it's enabled or not */
1957 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
1958 mqd->cp_hqd_pq_rptr_report_addr_lo);
1959 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
1960 mqd->cp_hqd_pq_rptr_report_addr_hi);
1961
1962 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1963 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
1964 mqd->cp_hqd_pq_wptr_poll_addr_lo);
1965 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
1966 mqd->cp_hqd_pq_wptr_poll_addr_hi);
1967
1968 /* enable the doorbell if requested */
1969 if (ring->use_doorbell) {
1970 WREG32_SOC15(
1971 GC, GET_INST(GC, xcc_id),
1972 regCP_MEC_DOORBELL_RANGE_LOWER,
1973 ((adev->doorbell_index.kiq +
1974 xcc_id * adev->doorbell_index.xcc_doorbell_range) *
1975 2) << 2);
1976 WREG32_SOC15(
1977 GC, GET_INST(GC, xcc_id),
1978 regCP_MEC_DOORBELL_RANGE_UPPER,
1979 ((adev->doorbell_index.userqueue_end +
1980 xcc_id * adev->doorbell_index.xcc_doorbell_range) *
1981 2) << 2);
1982 }
1983
1984 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
1985 mqd->cp_hqd_pq_doorbell_control);
1986
1987 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
1988 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
1989 mqd->cp_hqd_pq_wptr_lo);
1990 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
1991 mqd->cp_hqd_pq_wptr_hi);
1992
1993 /* set the vmid for the queue */
1994 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, mqd->cp_hqd_vmid);
1995
1996 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE,
1997 mqd->cp_hqd_persistent_state);
1998
1999 /* activate the queue */
2000 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE,
2001 mqd->cp_hqd_active);
2002
2003 if (ring->use_doorbell)
2004 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2005
2006 return 0;
2007 }
2008
gfx_v9_4_3_xcc_q_fini_register(struct amdgpu_ring * ring,int xcc_id)2009 static int gfx_v9_4_3_xcc_q_fini_register(struct amdgpu_ring *ring,
2010 int xcc_id)
2011 {
2012 struct amdgpu_device *adev = ring->adev;
2013 int j;
2014
2015 /* disable the queue if it's active */
2016 if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
2017
2018 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
2019
2020 for (j = 0; j < adev->usec_timeout; j++) {
2021 if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
2022 break;
2023 udelay(1);
2024 }
2025
2026 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
2027 DRM_DEBUG("%s dequeue request failed.\n", ring->name);
2028
2029 /* Manual disable if dequeue request times out */
2030 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 0);
2031 }
2032
2033 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
2034 0);
2035 }
2036
2037 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_IQ_TIMER, 0);
2038 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_IB_CONTROL, 0);
2039 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE, CP_HQD_PERSISTENT_STATE_DEFAULT);
2040 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
2041 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 0);
2042 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR, 0);
2043 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI, 0);
2044 WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO, 0);
2045
2046 return 0;
2047 }
2048
gfx_v9_4_3_xcc_kiq_init_queue(struct amdgpu_ring * ring,int xcc_id)2049 static int gfx_v9_4_3_xcc_kiq_init_queue(struct amdgpu_ring *ring, int xcc_id)
2050 {
2051 struct amdgpu_device *adev = ring->adev;
2052 struct v9_mqd *mqd = ring->mqd_ptr;
2053 struct v9_mqd *tmp_mqd;
2054
2055 gfx_v9_4_3_xcc_kiq_setting(ring, xcc_id);
2056
2057 /* GPU could be in bad state during probe, driver trigger the reset
2058 * after load the SMU, in this case , the mqd is not be initialized.
2059 * driver need to re-init the mqd.
2060 * check mqd->cp_hqd_pq_control since this value should not be 0
2061 */
2062 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[xcc_id].mqd_backup;
2063 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control) {
2064 /* for GPU_RESET case , reset MQD to a clean status */
2065 if (adev->gfx.kiq[xcc_id].mqd_backup)
2066 memcpy(mqd, adev->gfx.kiq[xcc_id].mqd_backup, sizeof(struct v9_mqd_allocation));
2067
2068 /* reset ring buffer */
2069 ring->wptr = 0;
2070 amdgpu_ring_clear_ring(ring);
2071 mutex_lock(&adev->srbm_mutex);
2072 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2073 gfx_v9_4_3_xcc_kiq_init_register(ring, xcc_id);
2074 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2075 mutex_unlock(&adev->srbm_mutex);
2076 } else {
2077 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
2078 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
2079 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
2080 mutex_lock(&adev->srbm_mutex);
2081 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
2082 amdgpu_ring_clear_ring(ring);
2083 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2084 gfx_v9_4_3_xcc_mqd_init(ring, xcc_id);
2085 gfx_v9_4_3_xcc_kiq_init_register(ring, xcc_id);
2086 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2087 mutex_unlock(&adev->srbm_mutex);
2088
2089 if (adev->gfx.kiq[xcc_id].mqd_backup)
2090 memcpy(adev->gfx.kiq[xcc_id].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
2091 }
2092
2093 return 0;
2094 }
2095
gfx_v9_4_3_xcc_kcq_init_queue(struct amdgpu_ring * ring,int xcc_id,bool restore)2096 static void gfx_v9_4_3_xcc_kcq_init_queue(struct amdgpu_ring *ring, int xcc_id,
2097 bool restore)
2098 {
2099 struct amdgpu_device *adev = ring->adev;
2100 struct v9_mqd *mqd = ring->mqd_ptr;
2101 int mqd_idx = ring - &adev->gfx.compute_ring[0];
2102 struct v9_mqd *tmp_mqd;
2103
2104 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
2105 * is not be initialized before
2106 */
2107 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
2108
2109 if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
2110 (!amdgpu_in_reset(adev) && !adev->in_suspend))) {
2111 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
2112 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
2113 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
2114 mutex_lock(&adev->srbm_mutex);
2115 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2116 gfx_v9_4_3_xcc_mqd_init(ring, xcc_id);
2117 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2118 mutex_unlock(&adev->srbm_mutex);
2119
2120 if (adev->gfx.mec.mqd_backup[mqd_idx])
2121 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
2122 } else {
2123 /* restore MQD to a clean status */
2124 if (adev->gfx.mec.mqd_backup[mqd_idx])
2125 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
2126 /* reset ring buffer */
2127 ring->wptr = 0;
2128 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
2129 amdgpu_ring_clear_ring(ring);
2130 }
2131 }
2132
gfx_v9_4_3_xcc_kcq_fini_register(struct amdgpu_device * adev,int xcc_id)2133 static int gfx_v9_4_3_xcc_kcq_fini_register(struct amdgpu_device *adev, int xcc_id)
2134 {
2135 struct amdgpu_ring *ring;
2136 int j;
2137
2138 for (j = 0; j < adev->gfx.num_compute_rings; j++) {
2139 ring = &adev->gfx.compute_ring[j + xcc_id * adev->gfx.num_compute_rings];
2140 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2141 mutex_lock(&adev->srbm_mutex);
2142 soc15_grbm_select(adev, ring->me,
2143 ring->pipe,
2144 ring->queue, 0, GET_INST(GC, xcc_id));
2145 gfx_v9_4_3_xcc_q_fini_register(ring, xcc_id);
2146 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2147 mutex_unlock(&adev->srbm_mutex);
2148 }
2149 }
2150
2151 return 0;
2152 }
2153
gfx_v9_4_3_xcc_kiq_resume(struct amdgpu_device * adev,int xcc_id)2154 static int gfx_v9_4_3_xcc_kiq_resume(struct amdgpu_device *adev, int xcc_id)
2155 {
2156 gfx_v9_4_3_xcc_kiq_init_queue(&adev->gfx.kiq[xcc_id].ring, xcc_id);
2157 return 0;
2158 }
2159
gfx_v9_4_3_xcc_kcq_resume(struct amdgpu_device * adev,int xcc_id)2160 static int gfx_v9_4_3_xcc_kcq_resume(struct amdgpu_device *adev, int xcc_id)
2161 {
2162 struct amdgpu_ring *ring;
2163 int i;
2164
2165 gfx_v9_4_3_xcc_cp_compute_enable(adev, true, xcc_id);
2166
2167 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2168 ring = &adev->gfx.compute_ring[i + xcc_id *
2169 adev->gfx.num_compute_rings];
2170
2171 gfx_v9_4_3_xcc_kcq_init_queue(ring, xcc_id, false);
2172 }
2173
2174 return amdgpu_gfx_enable_kcq(adev, xcc_id);
2175 }
2176
gfx_v9_4_3_xcc_cp_resume(struct amdgpu_device * adev,int xcc_id)2177 static int gfx_v9_4_3_xcc_cp_resume(struct amdgpu_device *adev, int xcc_id)
2178 {
2179 struct amdgpu_ring *ring;
2180 int r, j;
2181
2182 gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
2183
2184 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2185 gfx_v9_4_3_xcc_disable_gpa_mode(adev, xcc_id);
2186
2187 r = gfx_v9_4_3_xcc_cp_compute_load_microcode(adev, xcc_id);
2188 if (r)
2189 return r;
2190 } else {
2191 gfx_v9_4_3_xcc_cp_compute_enable(adev, false, xcc_id);
2192 }
2193
2194 r = gfx_v9_4_3_xcc_kiq_resume(adev, xcc_id);
2195 if (r)
2196 return r;
2197
2198 r = gfx_v9_4_3_xcc_kcq_resume(adev, xcc_id);
2199 if (r)
2200 return r;
2201
2202 for (j = 0; j < adev->gfx.num_compute_rings; j++) {
2203 ring = &adev->gfx.compute_ring
2204 [j + xcc_id * adev->gfx.num_compute_rings];
2205 r = amdgpu_ring_test_helper(ring);
2206 if (r)
2207 return r;
2208 }
2209
2210 gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, true, xcc_id);
2211
2212 return 0;
2213 }
2214
gfx_v9_4_3_cp_resume(struct amdgpu_device * adev)2215 static int gfx_v9_4_3_cp_resume(struct amdgpu_device *adev)
2216 {
2217 int r = 0, i, num_xcc, num_xcp, num_xcc_per_xcp;
2218
2219 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2220 if (amdgpu_sriov_vf(adev)) {
2221 enum amdgpu_gfx_partition mode;
2222
2223 mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
2224 AMDGPU_XCP_FL_NONE);
2225 if (mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2226 return -EINVAL;
2227 num_xcc_per_xcp = gfx_v9_4_3_get_xccs_per_xcp(adev);
2228 adev->gfx.num_xcc_per_xcp = num_xcc_per_xcp;
2229 num_xcp = num_xcc / num_xcc_per_xcp;
2230 r = amdgpu_xcp_init(adev->xcp_mgr, num_xcp, mode);
2231
2232 } else {
2233 if (adev->in_suspend)
2234 amdgpu_xcp_restore_partition_mode(adev->xcp_mgr);
2235 else if (amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
2236 AMDGPU_XCP_FL_NONE) ==
2237 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2238 r = amdgpu_xcp_switch_partition_mode(
2239 adev->xcp_mgr, amdgpu_user_partt_mode);
2240 }
2241 if (r)
2242 return r;
2243
2244 for (i = 0; i < num_xcc; i++) {
2245 r = gfx_v9_4_3_xcc_cp_resume(adev, i);
2246 if (r)
2247 return r;
2248 }
2249
2250 return 0;
2251 }
2252
gfx_v9_4_3_xcc_fini(struct amdgpu_device * adev,int xcc_id)2253 static void gfx_v9_4_3_xcc_fini(struct amdgpu_device *adev, int xcc_id)
2254 {
2255 if (amdgpu_gfx_disable_kcq(adev, xcc_id))
2256 DRM_ERROR("XCD %d KCQ disable failed\n", xcc_id);
2257
2258 if (amdgpu_sriov_vf(adev)) {
2259 /* must disable polling for SRIOV when hw finished, otherwise
2260 * CPC engine may still keep fetching WB address which is already
2261 * invalid after sw finished and trigger DMAR reading error in
2262 * hypervisor side.
2263 */
2264 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
2265 return;
2266 }
2267
2268 /* Use deinitialize sequence from CAIL when unbinding device
2269 * from driver, otherwise KIQ is hanging when binding back
2270 */
2271 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2272 mutex_lock(&adev->srbm_mutex);
2273 soc15_grbm_select(adev, adev->gfx.kiq[xcc_id].ring.me,
2274 adev->gfx.kiq[xcc_id].ring.pipe,
2275 adev->gfx.kiq[xcc_id].ring.queue, 0,
2276 GET_INST(GC, xcc_id));
2277 gfx_v9_4_3_xcc_q_fini_register(&adev->gfx.kiq[xcc_id].ring,
2278 xcc_id);
2279 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2280 mutex_unlock(&adev->srbm_mutex);
2281 }
2282
2283 gfx_v9_4_3_xcc_kcq_fini_register(adev, xcc_id);
2284 gfx_v9_4_3_xcc_cp_compute_enable(adev, false, xcc_id);
2285 }
2286
gfx_v9_4_3_set_userq_eop_interrupts(struct amdgpu_device * adev,bool enable)2287 static int gfx_v9_4_3_set_userq_eop_interrupts(struct amdgpu_device *adev,
2288 bool enable)
2289 {
2290 int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2291 unsigned int irq_type;
2292 int m, p, xcc_id, r;
2293
2294 if (adev->gfx.disable_kq) {
2295 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
2296 for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
2297 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
2298 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2299 + (m * adev->gfx.mec.num_pipe_per_mec)
2300 + p;
2301
2302 if (enable)
2303 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
2304 irq_type);
2305 else
2306 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
2307 irq_type);
2308 if (r) {
2309 if (!enable)
2310 return r;
2311 goto err_compute;
2312 }
2313 }
2314 }
2315 }
2316 }
2317
2318 return 0;
2319
2320 err_compute:
2321 for (p--; p >= 0; p--) {
2322 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2323 + (m * adev->gfx.mec.num_pipe_per_mec) + p;
2324 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
2325 }
2326 for (m--; m >= 0; m--) {
2327 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
2328 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2329 + (m * adev->gfx.mec.num_pipe_per_mec) + p;
2330 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
2331 }
2332 }
2333 for (xcc_id--; xcc_id >= 0; xcc_id--) {
2334 for (m = adev->gfx.mec.num_mec - 1; m >= 0; m--) {
2335 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
2336 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2337 + (m * adev->gfx.mec.num_pipe_per_mec) + p;
2338 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
2339 }
2340 }
2341 }
2342
2343 return r;
2344 }
2345
gfx_v9_4_3_hw_init(struct amdgpu_ip_block * ip_block)2346 static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block)
2347 {
2348 int r;
2349 struct amdgpu_device *adev = ip_block->adev;
2350
2351 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size,
2352 adev->gfx.cleaner_shader_ptr);
2353
2354 if (!amdgpu_sriov_vf(adev))
2355 gfx_v9_4_3_init_golden_registers(adev);
2356
2357 gfx_v9_4_3_constants_init(adev);
2358
2359 r = adev->gfx.rlc.funcs->resume(adev);
2360 if (r)
2361 return r;
2362
2363 r = gfx_v9_4_3_cp_resume(adev);
2364 if (r)
2365 return r;
2366
2367 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
2368 if (r)
2369 return r;
2370
2371 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
2372 if (r)
2373 goto err_priv_inst;
2374
2375 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
2376 if (r)
2377 goto err_bad_op;
2378 r = gfx_v9_4_3_set_userq_eop_interrupts(adev, true);
2379 if (r)
2380 goto err_bad_eop;
2381
2382 return 0;
2383
2384 err_bad_eop:
2385 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
2386 err_bad_op:
2387 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
2388 err_priv_inst:
2389 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
2390 return r;
2391 }
2392
gfx_v9_4_3_perf_monitor_ptl_init(struct amdgpu_device * adev,bool enable)2393 static int gfx_v9_4_3_perf_monitor_ptl_init(struct amdgpu_device *adev, bool enable)
2394 {
2395 struct amdgpu_ptl *ptl = &adev->psp.ptl;
2396 uint32_t ptl_state = enable ? 1 : 0;
2397 uint32_t fmt1, fmt2;
2398 int r;
2399
2400 if (!adev->psp.funcs)
2401 return -EOPNOTSUPP;
2402
2403 if (ptl->hw_supported_state == AMDGPU_PTL_HW_NOT_SUPPORTED)
2404 return -EOPNOTSUPP;
2405
2406 if (ptl->hw_supported_state == AMDGPU_PTL_HW_UNINIT) {
2407 fmt1 = GFX_FTYPE_VECTOR;
2408 fmt2 = GFX_FTYPE_F8;
2409 } else {
2410 fmt1 = ptl->fmt1;
2411 fmt2 = ptl->fmt2;
2412 }
2413
2414 /* initialize PTL with default formats: GFX_FTYPE_VECTOR & GFX_FTYPE_F8 */
2415 r = amdgpu_ptl_perf_monitor_ctrl(adev, PSP_PTL_PERF_MON_SET, &ptl_state,
2416 &fmt1, &fmt2);
2417 if (r) {
2418 if (ptl->hw_supported_state == AMDGPU_PTL_HW_UNINIT)
2419 ptl->hw_supported_state = AMDGPU_PTL_HW_NOT_SUPPORTED;
2420
2421 if (r != -EOPNOTSUPP)
2422 dev_err(adev->dev, "PTL initialization failed (%d)\n", r);
2423
2424 return r;
2425 }
2426
2427 ptl->hw_supported_state = AMDGPU_PTL_HW_SUPPORTED;
2428
2429 atomic_set(&ptl->disable_ref, 0);
2430 if (!enable && !amdgpu_in_reset(adev) && !adev->in_suspend) {
2431 dev_dbg(adev->dev,
2432 "PTL disabled (amdgpu.ptl=%d)\
2433 To enable, set amdgpu.ptl=1 via module param or kernel cmdline\n",
2434 amdgpu_ptl);
2435 set_bit(AMDGPU_PTL_DISABLE_SYSFS, ptl->disable_bitmap);
2436 }
2437
2438 return 0;
2439 }
2440
gfx_v9_4_3_ptl_hw_init(struct amdgpu_device * adev)2441 static int gfx_v9_4_3_ptl_hw_init(struct amdgpu_device *adev)
2442 {
2443 struct amdgpu_ptl *ptl = &adev->psp.ptl;
2444 bool enable;
2445
2446 switch (amdgpu_ptl) {
2447 case 1:
2448 enable = true;
2449 break;
2450 case 2:
2451 /* Permanently disabled - cannot be re-enabled */
2452 enable = false;
2453 ptl->permanently_disabled = true;
2454 break;
2455 case -1:
2456 case 0:
2457 default:
2458 enable = false;
2459 break;
2460 }
2461
2462 gfx_v9_4_3_perf_monitor_ptl_init(adev, enable ? 1 : 0);
2463
2464 return 0;
2465 }
2466
gfx_v9_4_3_hw_fini(struct amdgpu_ip_block * ip_block)2467 static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block)
2468 {
2469 struct amdgpu_device *adev = ip_block->adev;
2470 int i, num_xcc;
2471
2472 if (adev->psp.ptl.hw_supported_state == AMDGPU_PTL_HW_SUPPORTED &&
2473 !amdgpu_in_reset(adev))
2474 gfx_v9_4_3_perf_monitor_ptl_init(adev, false);
2475
2476 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
2477 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
2478 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
2479 gfx_v9_4_3_set_userq_eop_interrupts(adev, false);
2480
2481 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2482 for (i = 0; i < num_xcc; i++) {
2483 gfx_v9_4_3_xcc_fini(adev, i);
2484 }
2485
2486 return 0;
2487 }
2488
gfx_v9_4_3_suspend(struct amdgpu_ip_block * ip_block)2489 static int gfx_v9_4_3_suspend(struct amdgpu_ip_block *ip_block)
2490 {
2491 return gfx_v9_4_3_hw_fini(ip_block);
2492 }
2493
gfx_v9_4_3_resume(struct amdgpu_ip_block * ip_block)2494 static int gfx_v9_4_3_resume(struct amdgpu_ip_block *ip_block)
2495 {
2496 return gfx_v9_4_3_hw_init(ip_block);
2497 }
2498
gfx_v9_4_3_is_idle(struct amdgpu_ip_block * ip_block)2499 static bool gfx_v9_4_3_is_idle(struct amdgpu_ip_block *ip_block)
2500 {
2501 struct amdgpu_device *adev = ip_block->adev;
2502 int i, num_xcc;
2503 u32 gc_ip_version;
2504
2505 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2506 gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
2507
2508 for (i = 0; i < num_xcc; i++) {
2509 if (gc_ip_version == IP_VERSION(9, 4, 4)) {
2510 if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS),
2511 GRBM_STATUS, SPI_BUSY))
2512 return false;
2513 } else {
2514 if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS),
2515 GRBM_STATUS, GUI_ACTIVE))
2516 return false;
2517 }
2518 }
2519 return true;
2520 }
2521
gfx_v9_4_3_wait_for_idle(struct amdgpu_ip_block * ip_block)2522 static int gfx_v9_4_3_wait_for_idle(struct amdgpu_ip_block *ip_block)
2523 {
2524 unsigned i;
2525 struct amdgpu_device *adev = ip_block->adev;
2526
2527 for (i = 0; i < adev->usec_timeout; i++) {
2528 if (gfx_v9_4_3_is_idle(ip_block))
2529 return 0;
2530 udelay(1);
2531 }
2532 return -ETIMEDOUT;
2533 }
2534
gfx_v9_4_3_soft_reset(struct amdgpu_ip_block * ip_block)2535 static int gfx_v9_4_3_soft_reset(struct amdgpu_ip_block *ip_block)
2536 {
2537 u32 grbm_soft_reset = 0;
2538 u32 tmp;
2539 struct amdgpu_device *adev = ip_block->adev;
2540
2541 /* GRBM_STATUS */
2542 tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_STATUS);
2543 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
2544 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
2545 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
2546 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
2547 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
2548 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
2549 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2550 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
2551 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2552 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
2553 }
2554
2555 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
2556 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2557 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
2558 }
2559
2560 /* GRBM_STATUS2 */
2561 tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_STATUS2);
2562 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
2563 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2564 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2565
2566
2567 if (grbm_soft_reset) {
2568 /* stop the rlc */
2569 adev->gfx.rlc.funcs->stop(adev);
2570
2571 /* Disable MEC parsing/prefetching */
2572 gfx_v9_4_3_xcc_cp_compute_enable(adev, false, 0);
2573
2574 tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2575 tmp |= grbm_soft_reset;
2576 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2577 WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);
2578 tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2579
2580 udelay(50);
2581
2582 tmp &= ~grbm_soft_reset;
2583 WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);
2584 tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2585
2586 /* Wait a little for things to settle down */
2587 udelay(50);
2588 }
2589 return 0;
2590 }
2591
gfx_v9_4_3_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)2592 static void gfx_v9_4_3_ring_emit_gds_switch(struct amdgpu_ring *ring,
2593 uint32_t vmid,
2594 uint32_t gds_base, uint32_t gds_size,
2595 uint32_t gws_base, uint32_t gws_size,
2596 uint32_t oa_base, uint32_t oa_size)
2597 {
2598 struct amdgpu_device *adev = ring->adev;
2599
2600 /* GDS Base */
2601 gfx_v9_4_3_write_data_to_reg(ring, 0, false,
2602 SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_VMID0_BASE) + 2 * vmid,
2603 gds_base);
2604
2605 /* GDS Size */
2606 gfx_v9_4_3_write_data_to_reg(ring, 0, false,
2607 SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_VMID0_SIZE) + 2 * vmid,
2608 gds_size);
2609
2610 /* GWS */
2611 gfx_v9_4_3_write_data_to_reg(ring, 0, false,
2612 SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_GWS_VMID0) + vmid,
2613 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
2614
2615 /* OA */
2616 gfx_v9_4_3_write_data_to_reg(ring, 0, false,
2617 SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_OA_VMID0) + vmid,
2618 (1 << (oa_size + oa_base)) - (1 << oa_base));
2619 }
2620
gfx_v9_4_3_early_init(struct amdgpu_ip_block * ip_block)2621 static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block)
2622 {
2623 struct amdgpu_device *adev = ip_block->adev;
2624
2625 switch (amdgpu_user_queue) {
2626 case -1:
2627 case 0:
2628 default:
2629 adev->gfx.disable_kq = false;
2630 adev->gfx.disable_uq = true;
2631 break;
2632 case 2:
2633 adev->gfx.disable_kq = true;
2634 adev->gfx.disable_uq = true;
2635 break;
2636 }
2637
2638 if (adev->gfx.disable_kq)
2639 adev->gfx.num_compute_rings = 0;
2640 else
2641 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
2642 AMDGPU_MAX_COMPUTE_RINGS);
2643 gfx_v9_4_3_set_kiq_pm4_funcs(adev);
2644 gfx_v9_4_3_set_ring_funcs(adev);
2645 gfx_v9_4_3_set_irq_funcs(adev);
2646 gfx_v9_4_3_set_gds_init(adev);
2647 gfx_v9_4_3_set_rlc_funcs(adev);
2648
2649 /* init rlcg reg access ctrl */
2650 gfx_v9_4_3_init_rlcg_reg_access_ctrl(adev);
2651
2652 amdgpu_init_rlc_reg_funcs(adev);
2653
2654 return gfx_v9_4_3_init_microcode(adev);
2655 }
2656
gfx_v9_4_3_late_init(struct amdgpu_ip_block * ip_block)2657 static int gfx_v9_4_3_late_init(struct amdgpu_ip_block *ip_block)
2658 {
2659 struct amdgpu_device *adev = ip_block->adev;
2660
2661 if (adev->gfx.ras &&
2662 adev->gfx.ras->enable_watchdog_timer)
2663 adev->gfx.ras->enable_watchdog_timer(adev);
2664
2665 gfx_v9_4_3_ptl_hw_init(adev);
2666
2667 return 0;
2668 }
2669
gfx_v9_4_3_xcc_update_sram_fgcg(struct amdgpu_device * adev,bool enable,int xcc_id)2670 static void gfx_v9_4_3_xcc_update_sram_fgcg(struct amdgpu_device *adev,
2671 bool enable, int xcc_id)
2672 {
2673 uint32_t def, data;
2674
2675 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
2676 return;
2677
2678 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2679 regRLC_CGTT_MGCG_OVERRIDE);
2680
2681 if (enable)
2682 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
2683 else
2684 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
2685
2686 if (def != data)
2687 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2688 regRLC_CGTT_MGCG_OVERRIDE, data);
2689
2690 }
2691
gfx_v9_4_3_xcc_update_repeater_fgcg(struct amdgpu_device * adev,bool enable,int xcc_id)2692 static void gfx_v9_4_3_xcc_update_repeater_fgcg(struct amdgpu_device *adev,
2693 bool enable, int xcc_id)
2694 {
2695 uint32_t def, data;
2696
2697 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
2698 return;
2699
2700 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2701 regRLC_CGTT_MGCG_OVERRIDE);
2702
2703 if (enable)
2704 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REP_FGCG_OVERRIDE_MASK;
2705 else
2706 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REP_FGCG_OVERRIDE_MASK;
2707
2708 if (def != data)
2709 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2710 regRLC_CGTT_MGCG_OVERRIDE, data);
2711 }
2712
2713 static void
gfx_v9_4_3_xcc_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable,int xcc_id)2714 gfx_v9_4_3_xcc_update_medium_grain_clock_gating(struct amdgpu_device *adev,
2715 bool enable, int xcc_id)
2716 {
2717 uint32_t data, def;
2718
2719 /* It is disabled by HW by default */
2720 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
2721 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
2722 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2723
2724 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2725 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
2726 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2727 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
2728
2729 if (def != data)
2730 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2731
2732 /* MGLS is a global flag to control all MGLS in GFX */
2733 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
2734 /* 2 - RLC memory Light sleep */
2735 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
2736 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL);
2737 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2738 if (def != data)
2739 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL, data);
2740 }
2741 /* 3 - CP memory Light sleep */
2742 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
2743 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL);
2744 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2745 if (def != data)
2746 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL, data);
2747 }
2748 }
2749 } else {
2750 /* 1 - MGCG_OVERRIDE */
2751 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2752
2753 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2754 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2755 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
2756 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
2757
2758 if (def != data)
2759 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2760
2761 /* 2 - disable MGLS in RLC */
2762 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL);
2763 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
2764 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2765 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL, data);
2766 }
2767
2768 /* 3 - disable MGLS in CP */
2769 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL);
2770 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
2771 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2772 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL, data);
2773 }
2774 }
2775
2776 }
2777
2778 static void
gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable,int xcc_id)2779 gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
2780 bool enable, int xcc_id)
2781 {
2782 uint32_t def, data;
2783
2784 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
2785
2786 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2787 /* unset CGCG override */
2788 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
2789 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2790 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2791 else
2792 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2793 /* update CGCG and CGLS override bits */
2794 if (def != data)
2795 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2796
2797 /* CGCG Hysteresis: 400us */
2798 def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
2799
2800 data = (0x2710
2801 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2802 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
2803 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2804 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
2805 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
2806 if (def != data)
2807 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
2808
2809 /* set IDLE_POLL_COUNT(0x33450100)*/
2810 def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);
2811 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2812 (0x3345 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2813 if (def != data)
2814 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);
2815 } else {
2816 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
2817 /* reset CGCG/CGLS bits */
2818 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
2819 /* disable cgcg and cgls in FSM */
2820 if (def != data)
2821 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
2822 }
2823
2824 }
2825
gfx_v9_4_3_xcc_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable,int xcc_id)2826 static int gfx_v9_4_3_xcc_update_gfx_clock_gating(struct amdgpu_device *adev,
2827 bool enable, int xcc_id)
2828 {
2829 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
2830
2831 if (enable) {
2832 /* FGCG */
2833 gfx_v9_4_3_xcc_update_sram_fgcg(adev, enable, xcc_id);
2834 gfx_v9_4_3_xcc_update_repeater_fgcg(adev, enable, xcc_id);
2835
2836 /* CGCG/CGLS should be enabled after MGCG/MGLS
2837 * === MGCG + MGLS ===
2838 */
2839 gfx_v9_4_3_xcc_update_medium_grain_clock_gating(adev, enable,
2840 xcc_id);
2841 /* === CGCG + CGLS === */
2842 gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(adev, enable,
2843 xcc_id);
2844 } else {
2845 /* CGCG/CGLS should be disabled before MGCG/MGLS
2846 * === CGCG + CGLS ===
2847 */
2848 gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(adev, enable,
2849 xcc_id);
2850 /* === MGCG + MGLS === */
2851 gfx_v9_4_3_xcc_update_medium_grain_clock_gating(adev, enable,
2852 xcc_id);
2853
2854 /* FGCG */
2855 gfx_v9_4_3_xcc_update_sram_fgcg(adev, enable, xcc_id);
2856 gfx_v9_4_3_xcc_update_repeater_fgcg(adev, enable, xcc_id);
2857 }
2858
2859 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
2860
2861 return 0;
2862 }
2863
2864 static const struct amdgpu_rlc_funcs gfx_v9_4_3_rlc_funcs = {
2865 .is_rlc_enabled = gfx_v9_4_3_is_rlc_enabled,
2866 .set_safe_mode = gfx_v9_4_3_xcc_set_safe_mode,
2867 .unset_safe_mode = gfx_v9_4_3_xcc_unset_safe_mode,
2868 .init = gfx_v9_4_3_rlc_init,
2869 .resume = gfx_v9_4_3_rlc_resume,
2870 .stop = gfx_v9_4_3_rlc_stop,
2871 .reset = gfx_v9_4_3_rlc_reset,
2872 .start = gfx_v9_4_3_rlc_start,
2873 .update_spm_vmid = gfx_v9_4_3_update_spm_vmid,
2874 .is_rlcg_access_range = gfx_v9_4_3_is_rlcg_access_range,
2875 };
2876
gfx_v9_4_3_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)2877 static int gfx_v9_4_3_set_powergating_state(struct amdgpu_ip_block *ip_block,
2878 enum amd_powergating_state state)
2879 {
2880 return 0;
2881 }
2882
gfx_v9_4_3_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)2883 static int gfx_v9_4_3_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2884 enum amd_clockgating_state state)
2885 {
2886 struct amdgpu_device *adev = ip_block->adev;
2887 int i, num_xcc;
2888
2889 if (amdgpu_sriov_vf(adev))
2890 return 0;
2891
2892 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2893 for (i = 0; i < num_xcc; i++)
2894 gfx_v9_4_3_xcc_update_gfx_clock_gating(
2895 adev, state == AMD_CG_STATE_GATE, i);
2896
2897 return 0;
2898 }
2899
gfx_v9_4_3_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)2900 static void gfx_v9_4_3_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
2901 {
2902 struct amdgpu_device *adev = ip_block->adev;
2903 int data;
2904
2905 if (amdgpu_sriov_vf(adev))
2906 *flags = 0;
2907
2908 /* AMD_CG_SUPPORT_GFX_MGCG */
2909 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_CGTT_MGCG_OVERRIDE));
2910 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
2911 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
2912
2913 /* AMD_CG_SUPPORT_GFX_CGCG */
2914 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_CGCG_CGLS_CTRL));
2915 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
2916 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
2917
2918 /* AMD_CG_SUPPORT_GFX_CGLS */
2919 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
2920 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
2921
2922 /* AMD_CG_SUPPORT_GFX_RLC_LS */
2923 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_MEM_SLP_CNTL));
2924 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
2925 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
2926
2927 /* AMD_CG_SUPPORT_GFX_CP_LS */
2928 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCP_MEM_SLP_CNTL));
2929 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
2930 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
2931 }
2932
gfx_v9_4_3_ring_emit_hdp_flush(struct amdgpu_ring * ring)2933 static void gfx_v9_4_3_ring_emit_hdp_flush(struct amdgpu_ring *ring)
2934 {
2935 struct amdgpu_device *adev = ring->adev;
2936 u32 ref_and_mask, reg_mem_engine;
2937
2938 if (!adev->gfx.funcs->get_hdp_flush_mask) {
2939 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
2940 return;
2941 }
2942
2943 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine);
2944 gfx_v9_4_3_wait_reg_mem(ring, reg_mem_engine, 0, 1,
2945 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
2946 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
2947 ref_and_mask, ref_and_mask, 0x20);
2948 }
2949
gfx_v9_4_3_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)2950 static void gfx_v9_4_3_ring_emit_ib_compute(struct amdgpu_ring *ring,
2951 struct amdgpu_job *job,
2952 struct amdgpu_ib *ib,
2953 uint32_t flags)
2954 {
2955 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
2956 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
2957
2958 /* Currently, there is a high possibility to get wave ID mismatch
2959 * between ME and GDS, leading to a hw deadlock, because ME generates
2960 * different wave IDs than the GDS expects. This situation happens
2961 * randomly when at least 5 compute pipes use GDS ordered append.
2962 * The wave IDs generated by ME are also wrong after suspend/resume.
2963 * Those are probably bugs somewhere else in the kernel driver.
2964 *
2965 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
2966 * GDS to 0 for this ring (me/pipe).
2967 */
2968 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
2969 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2970 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
2971 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
2972 }
2973
2974 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2975 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
2976 amdgpu_ring_write(ring,
2977 #ifdef __BIG_ENDIAN
2978 (2 << 0) |
2979 #endif
2980 lower_32_bits(ib->gpu_addr));
2981 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
2982 amdgpu_ring_write(ring, control);
2983 }
2984
gfx_v9_4_3_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)2985 static void gfx_v9_4_3_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
2986 u64 seq, unsigned flags)
2987 {
2988 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2989 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2990 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
2991
2992 /* RELEASE_MEM - flush caches, send int */
2993 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
2994 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
2995 EOP_TC_NC_ACTION_EN) :
2996 (EOP_TCL1_ACTION_EN |
2997 EOP_TC_ACTION_EN |
2998 EOP_TC_WB_ACTION_EN |
2999 EOP_TC_MD_ACTION_EN)) |
3000 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3001 EVENT_INDEX(5)));
3002 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
3003
3004 /*
3005 * the address should be Qword aligned if 64bit write, Dword
3006 * aligned if only send 32bit data low (discard data high)
3007 */
3008 if (write64bit)
3009 WARN_ON(addr & 0x7);
3010 else
3011 WARN_ON(addr & 0x3);
3012 amdgpu_ring_write(ring, lower_32_bits(addr));
3013 amdgpu_ring_write(ring, upper_32_bits(addr));
3014 amdgpu_ring_write(ring, lower_32_bits(seq));
3015 amdgpu_ring_write(ring, upper_32_bits(seq));
3016 amdgpu_ring_write(ring, 0);
3017 }
3018
gfx_v9_4_3_ring_emit_pipeline_sync(struct amdgpu_ring * ring)3019 static void gfx_v9_4_3_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3020 {
3021 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3022 uint32_t seq = ring->fence_drv.sync_seq;
3023 uint64_t addr = ring->fence_drv.gpu_addr;
3024
3025 gfx_v9_4_3_wait_reg_mem(ring, usepfp, 1, 0,
3026 lower_32_bits(addr), upper_32_bits(addr),
3027 seq, 0xffffffff, 4);
3028 }
3029
gfx_v9_4_3_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)3030 static void gfx_v9_4_3_ring_emit_vm_flush(struct amdgpu_ring *ring,
3031 unsigned vmid, uint64_t pd_addr)
3032 {
3033 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
3034 }
3035
gfx_v9_4_3_ring_get_rptr_compute(struct amdgpu_ring * ring)3036 static u64 gfx_v9_4_3_ring_get_rptr_compute(struct amdgpu_ring *ring)
3037 {
3038 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
3039 }
3040
gfx_v9_4_3_ring_get_wptr_compute(struct amdgpu_ring * ring)3041 static u64 gfx_v9_4_3_ring_get_wptr_compute(struct amdgpu_ring *ring)
3042 {
3043 struct amdgpu_device *adev = ring->adev;
3044 u64 wptr;
3045
3046 /* XXX check if swapping is necessary on BE */
3047 if (ring->use_doorbell) {
3048 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
3049 } else {
3050 dev_warn_once(adev->dev,
3051 "%s requires doorbell!\n", __func__);
3052 wptr = 0;
3053 }
3054 return wptr;
3055 }
3056
gfx_v9_4_3_ring_set_wptr_compute(struct amdgpu_ring * ring)3057 static void gfx_v9_4_3_ring_set_wptr_compute(struct amdgpu_ring *ring)
3058 {
3059 struct amdgpu_device *adev = ring->adev;
3060
3061 /* XXX check if swapping is necessary on BE */
3062 if (ring->use_doorbell) {
3063 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
3064 WDOORBELL64(ring->doorbell_index, ring->wptr);
3065 } else {
3066 dev_warn_once(adev->dev,
3067 "%s requires doorbell!\n", __func__);
3068 }
3069 }
3070
gfx_v9_4_3_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)3071 static void gfx_v9_4_3_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
3072 u64 seq, unsigned int flags)
3073 {
3074 struct amdgpu_device *adev = ring->adev;
3075
3076 /* write fence seq to the "addr" */
3077 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3078 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3079 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
3080 amdgpu_ring_write(ring, lower_32_bits(addr));
3081 amdgpu_ring_write(ring, upper_32_bits(addr));
3082 amdgpu_ring_write(ring, lower_32_bits(seq));
3083
3084 if (flags & AMDGPU_FENCE_FLAG_INT) {
3085 /* set register to trigger INT */
3086 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3087 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3088 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
3089 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
3090 amdgpu_ring_write(ring, 0);
3091 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
3092 }
3093 }
3094
gfx_v9_4_3_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)3095 static void gfx_v9_4_3_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
3096 uint32_t reg_val_offs)
3097 {
3098 struct amdgpu_device *adev = ring->adev;
3099
3100 reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
3101
3102 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
3103 amdgpu_ring_write(ring, 0 | /* src: register*/
3104 (5 << 8) | /* dst: memory */
3105 (1 << 20)); /* write confirm */
3106 amdgpu_ring_write(ring, reg);
3107 amdgpu_ring_write(ring, 0);
3108 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3109 reg_val_offs * 4));
3110 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3111 reg_val_offs * 4));
3112 }
3113
gfx_v9_4_3_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)3114 static void gfx_v9_4_3_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
3115 uint32_t val)
3116 {
3117 uint32_t cmd = 0;
3118
3119 reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
3120
3121 switch (ring->funcs->type) {
3122 case AMDGPU_RING_TYPE_GFX:
3123 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
3124 break;
3125 case AMDGPU_RING_TYPE_KIQ:
3126 cmd = (1 << 16); /* no inc addr */
3127 break;
3128 default:
3129 cmd = WR_CONFIRM;
3130 break;
3131 }
3132 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3133 amdgpu_ring_write(ring, cmd);
3134 amdgpu_ring_write(ring, reg);
3135 amdgpu_ring_write(ring, 0);
3136 amdgpu_ring_write(ring, val);
3137 }
3138
gfx_v9_4_3_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)3139 static void gfx_v9_4_3_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
3140 uint32_t val, uint32_t mask)
3141 {
3142 gfx_v9_4_3_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
3143 }
3144
gfx_v9_4_3_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)3145 static void gfx_v9_4_3_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
3146 uint32_t reg0, uint32_t reg1,
3147 uint32_t ref, uint32_t mask)
3148 {
3149 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
3150 ref, mask);
3151 }
3152
gfx_v9_4_3_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)3153 static void gfx_v9_4_3_ring_soft_recovery(struct amdgpu_ring *ring,
3154 unsigned vmid)
3155 {
3156 struct amdgpu_device *adev = ring->adev;
3157 uint32_t value = 0;
3158
3159 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
3160 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
3161 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
3162 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
3163 amdgpu_gfx_rlc_enter_safe_mode(adev, ring->xcc_id);
3164 WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regSQ_CMD, value);
3165 amdgpu_gfx_rlc_exit_safe_mode(adev, ring->xcc_id);
3166 }
3167
gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state,int xcc_id)3168 static void gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3169 struct amdgpu_device *adev, int me, int pipe,
3170 enum amdgpu_interrupt_state state, int xcc_id)
3171 {
3172 u32 mec_int_cntl, mec_int_cntl_reg;
3173
3174 /*
3175 * amdgpu controls only the first MEC. That's why this function only
3176 * handles the setting of interrupts for this specific MEC. All other
3177 * pipes' interrupts are set by amdkfd.
3178 */
3179
3180 if (me == 1) {
3181 switch (pipe) {
3182 case 0:
3183 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE0_INT_CNTL);
3184 break;
3185 case 1:
3186 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE1_INT_CNTL);
3187 break;
3188 case 2:
3189 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE2_INT_CNTL);
3190 break;
3191 case 3:
3192 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE3_INT_CNTL);
3193 break;
3194 default:
3195 DRM_DEBUG("invalid pipe %d\n", pipe);
3196 return;
3197 }
3198 } else {
3199 DRM_DEBUG("invalid me %d\n", me);
3200 return;
3201 }
3202
3203 switch (state) {
3204 case AMDGPU_IRQ_STATE_DISABLE:
3205 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3206 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3207 TIME_STAMP_INT_ENABLE, 0);
3208 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3209 break;
3210 case AMDGPU_IRQ_STATE_ENABLE:
3211 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3212 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3213 TIME_STAMP_INT_ENABLE, 1);
3214 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3215 break;
3216 default:
3217 break;
3218 }
3219 }
3220
gfx_v9_4_3_get_cpc_int_cntl(struct amdgpu_device * adev,int xcc_id,int me,int pipe)3221 static u32 gfx_v9_4_3_get_cpc_int_cntl(struct amdgpu_device *adev,
3222 int xcc_id, int me, int pipe)
3223 {
3224 /*
3225 * amdgpu controls only the first MEC. That's why this function only
3226 * handles the setting of interrupts for this specific MEC. All other
3227 * pipes' interrupts are set by amdkfd.
3228 */
3229 if (me != 1)
3230 return 0;
3231
3232 switch (pipe) {
3233 case 0:
3234 return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE0_INT_CNTL);
3235 case 1:
3236 return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE1_INT_CNTL);
3237 case 2:
3238 return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE2_INT_CNTL);
3239 case 3:
3240 return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE3_INT_CNTL);
3241 default:
3242 return 0;
3243 }
3244 }
3245
gfx_v9_4_3_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)3246 static int gfx_v9_4_3_set_priv_reg_fault_state(struct amdgpu_device *adev,
3247 struct amdgpu_irq_src *source,
3248 unsigned type,
3249 enum amdgpu_interrupt_state state)
3250 {
3251 u32 mec_int_cntl_reg, mec_int_cntl;
3252 int i, j, k, num_xcc;
3253
3254 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3255 switch (state) {
3256 case AMDGPU_IRQ_STATE_DISABLE:
3257 case AMDGPU_IRQ_STATE_ENABLE:
3258 for (i = 0; i < num_xcc; i++) {
3259 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3260 PRIV_REG_INT_ENABLE,
3261 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3262 for (j = 0; j < adev->gfx.mec.num_mec; j++) {
3263 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
3264 /* MECs start at 1 */
3265 mec_int_cntl_reg = gfx_v9_4_3_get_cpc_int_cntl(adev, i, j + 1, k);
3266
3267 if (mec_int_cntl_reg) {
3268 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, i);
3269 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3270 PRIV_REG_INT_ENABLE,
3271 state == AMDGPU_IRQ_STATE_ENABLE ?
3272 1 : 0);
3273 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, i);
3274 }
3275 }
3276 }
3277 }
3278 break;
3279 default:
3280 break;
3281 }
3282
3283 return 0;
3284 }
3285
gfx_v9_4_3_set_bad_op_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)3286 static int gfx_v9_4_3_set_bad_op_fault_state(struct amdgpu_device *adev,
3287 struct amdgpu_irq_src *source,
3288 unsigned type,
3289 enum amdgpu_interrupt_state state)
3290 {
3291 u32 mec_int_cntl_reg, mec_int_cntl;
3292 int i, j, k, num_xcc;
3293
3294 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3295 switch (state) {
3296 case AMDGPU_IRQ_STATE_DISABLE:
3297 case AMDGPU_IRQ_STATE_ENABLE:
3298 for (i = 0; i < num_xcc; i++) {
3299 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3300 OPCODE_ERROR_INT_ENABLE,
3301 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3302 for (j = 0; j < adev->gfx.mec.num_mec; j++) {
3303 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
3304 /* MECs start at 1 */
3305 mec_int_cntl_reg = gfx_v9_4_3_get_cpc_int_cntl(adev, i, j + 1, k);
3306
3307 if (mec_int_cntl_reg) {
3308 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, i);
3309 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3310 OPCODE_ERROR_INT_ENABLE,
3311 state == AMDGPU_IRQ_STATE_ENABLE ?
3312 1 : 0);
3313 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, i);
3314 }
3315 }
3316 }
3317 }
3318 break;
3319 default:
3320 break;
3321 }
3322
3323 return 0;
3324 }
3325
gfx_v9_4_3_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)3326 static int gfx_v9_4_3_set_priv_inst_fault_state(struct amdgpu_device *adev,
3327 struct amdgpu_irq_src *source,
3328 unsigned type,
3329 enum amdgpu_interrupt_state state)
3330 {
3331 int i, num_xcc;
3332
3333 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3334 switch (state) {
3335 case AMDGPU_IRQ_STATE_DISABLE:
3336 case AMDGPU_IRQ_STATE_ENABLE:
3337 for (i = 0; i < num_xcc; i++)
3338 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3339 PRIV_INSTR_INT_ENABLE,
3340 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3341 break;
3342 default:
3343 break;
3344 }
3345
3346 return 0;
3347 }
3348
gfx_v9_4_3_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)3349 static int gfx_v9_4_3_set_eop_interrupt_state(struct amdgpu_device *adev,
3350 struct amdgpu_irq_src *src,
3351 unsigned type,
3352 enum amdgpu_interrupt_state state)
3353 {
3354 int i, num_xcc;
3355
3356 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3357 for (i = 0; i < num_xcc; i++) {
3358 switch (type) {
3359 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
3360 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3361 adev, 1, 0, state, i);
3362 break;
3363 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
3364 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3365 adev, 1, 1, state, i);
3366 break;
3367 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
3368 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3369 adev, 1, 2, state, i);
3370 break;
3371 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
3372 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3373 adev, 1, 3, state, i);
3374 break;
3375 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
3376 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3377 adev, 2, 0, state, i);
3378 break;
3379 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
3380 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3381 adev, 2, 1, state, i);
3382 break;
3383 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
3384 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3385 adev, 2, 2, state, i);
3386 break;
3387 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
3388 gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3389 adev, 2, 3, state, i);
3390 break;
3391 default:
3392 break;
3393 }
3394 }
3395
3396 return 0;
3397 }
3398
gfx_v9_4_3_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)3399 static int gfx_v9_4_3_eop_irq(struct amdgpu_device *adev,
3400 struct amdgpu_irq_src *source,
3401 struct amdgpu_iv_entry *entry)
3402 {
3403 int i, xcc_id;
3404 u8 me_id, pipe_id, queue_id;
3405 struct amdgpu_ring *ring;
3406
3407 DRM_DEBUG("IH: CP EOP\n");
3408 me_id = (entry->ring_id & 0x0c) >> 2;
3409 pipe_id = (entry->ring_id & 0x03) >> 0;
3410 queue_id = (entry->ring_id & 0x70) >> 4;
3411
3412 xcc_id = gfx_v9_4_3_ih_to_xcc_inst(adev, entry->node_id);
3413
3414 if (xcc_id == -EINVAL)
3415 return -EINVAL;
3416
3417 switch (me_id) {
3418 case 0:
3419 case 1:
3420 case 2:
3421 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3422 ring = &adev->gfx.compute_ring
3423 [i +
3424 xcc_id * adev->gfx.num_compute_rings];
3425 /* Per-queue interrupt is supported for MEC starting from VI.
3426 * The interrupt can only be enabled/disabled per pipe instead of per queue.
3427 */
3428
3429 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
3430 amdgpu_fence_process(ring);
3431 }
3432 break;
3433 }
3434 return 0;
3435 }
3436
gfx_v9_4_3_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)3437 static void gfx_v9_4_3_fault(struct amdgpu_device *adev,
3438 struct amdgpu_iv_entry *entry)
3439 {
3440 u8 me_id, pipe_id, queue_id;
3441 struct amdgpu_ring *ring;
3442 int i, xcc_id;
3443
3444 me_id = (entry->ring_id & 0x0c) >> 2;
3445 pipe_id = (entry->ring_id & 0x03) >> 0;
3446 queue_id = (entry->ring_id & 0x70) >> 4;
3447
3448 xcc_id = gfx_v9_4_3_ih_to_xcc_inst(adev, entry->node_id);
3449
3450 if (xcc_id == -EINVAL)
3451 return;
3452
3453 switch (me_id) {
3454 case 0:
3455 case 1:
3456 case 2:
3457 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3458 ring = &adev->gfx.compute_ring
3459 [i +
3460 xcc_id * adev->gfx.num_compute_rings];
3461 if (ring->me == me_id && ring->pipe == pipe_id &&
3462 ring->queue == queue_id)
3463 drm_sched_fault(&ring->sched);
3464 }
3465 break;
3466 }
3467 }
3468
gfx_v9_4_3_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)3469 static int gfx_v9_4_3_priv_reg_irq(struct amdgpu_device *adev,
3470 struct amdgpu_irq_src *source,
3471 struct amdgpu_iv_entry *entry)
3472 {
3473 DRM_ERROR("Illegal register access in command stream\n");
3474 gfx_v9_4_3_fault(adev, entry);
3475 return 0;
3476 }
3477
gfx_v9_4_3_bad_op_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)3478 static int gfx_v9_4_3_bad_op_irq(struct amdgpu_device *adev,
3479 struct amdgpu_irq_src *source,
3480 struct amdgpu_iv_entry *entry)
3481 {
3482 DRM_ERROR("Illegal opcode in command stream\n");
3483 gfx_v9_4_3_fault(adev, entry);
3484 return 0;
3485 }
3486
gfx_v9_4_3_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)3487 static int gfx_v9_4_3_priv_inst_irq(struct amdgpu_device *adev,
3488 struct amdgpu_irq_src *source,
3489 struct amdgpu_iv_entry *entry)
3490 {
3491 DRM_ERROR("Illegal instruction in command stream\n");
3492 gfx_v9_4_3_fault(adev, entry);
3493 return 0;
3494 }
3495
gfx_v9_4_3_emit_mem_sync(struct amdgpu_ring * ring)3496 static void gfx_v9_4_3_emit_mem_sync(struct amdgpu_ring *ring)
3497 {
3498 const unsigned int cp_coher_cntl =
3499 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
3500 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
3501 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
3502 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
3503 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
3504
3505 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
3506 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
3507 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */
3508 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
3509 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
3510 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
3511 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
3512 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
3513 }
3514
gfx_v9_4_3_emit_wave_limit_cs(struct amdgpu_ring * ring,uint32_t pipe,bool enable)3515 static void gfx_v9_4_3_emit_wave_limit_cs(struct amdgpu_ring *ring,
3516 uint32_t pipe, bool enable)
3517 {
3518 struct amdgpu_device *adev = ring->adev;
3519 uint32_t val;
3520 uint32_t wcl_cs_reg;
3521
3522 /* regSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
3523 val = enable ? 0x1 : 0x7f;
3524
3525 switch (pipe) {
3526 case 0:
3527 wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS0);
3528 break;
3529 case 1:
3530 wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS1);
3531 break;
3532 case 2:
3533 wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS2);
3534 break;
3535 case 3:
3536 wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS3);
3537 break;
3538 default:
3539 DRM_DEBUG("invalid pipe %d\n", pipe);
3540 return;
3541 }
3542
3543 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
3544
3545 }
gfx_v9_4_3_emit_wave_limit(struct amdgpu_ring * ring,bool enable)3546 static void gfx_v9_4_3_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
3547 {
3548 struct amdgpu_device *adev = ring->adev;
3549 uint32_t val;
3550 int i;
3551
3552 /* regSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
3553 * number of gfx waves. Setting 5 bit will make sure gfx only gets
3554 * around 25% of gpu resources.
3555 */
3556 val = enable ? 0x1f : 0x07ffffff;
3557 amdgpu_ring_emit_wreg(ring,
3558 SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_GFX),
3559 val);
3560
3561 /* Restrict waves for normal/low priority compute queues as well
3562 * to get best QoS for high priority compute jobs.
3563 *
3564 * amdgpu controls only 1st ME(0-3 CS pipes).
3565 */
3566 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3567 if (i != ring->pipe)
3568 gfx_v9_4_3_emit_wave_limit_cs(ring, i, enable);
3569
3570 }
3571 }
3572
gfx_v9_4_3_unmap_done(struct amdgpu_device * adev,uint32_t me,uint32_t pipe,uint32_t queue,uint32_t xcc_id)3573 static int gfx_v9_4_3_unmap_done(struct amdgpu_device *adev, uint32_t me,
3574 uint32_t pipe, uint32_t queue,
3575 uint32_t xcc_id)
3576 {
3577 int i, r;
3578 /* make sure dequeue is complete*/
3579 gfx_v9_4_3_xcc_set_safe_mode(adev, xcc_id);
3580 mutex_lock(&adev->srbm_mutex);
3581 soc15_grbm_select(adev, me, pipe, queue, 0, GET_INST(GC, xcc_id));
3582 for (i = 0; i < adev->usec_timeout; i++) {
3583 if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
3584 break;
3585 udelay(1);
3586 }
3587 if (i >= adev->usec_timeout)
3588 r = -ETIMEDOUT;
3589 else
3590 r = 0;
3591 soc15_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
3592 mutex_unlock(&adev->srbm_mutex);
3593 gfx_v9_4_3_xcc_unset_safe_mode(adev, xcc_id);
3594
3595 return r;
3596
3597 }
3598
gfx_v9_4_3_pipe_reset_support(struct amdgpu_device * adev)3599 static bool gfx_v9_4_3_pipe_reset_support(struct amdgpu_device *adev)
3600 {
3601 if (!!(adev->gfx.compute_supported_reset & AMDGPU_RESET_TYPE_PER_PIPE))
3602 return true;
3603 else
3604 dev_warn_once(adev->dev, "Please use the latest MEC version to see whether support pipe reset\n");
3605
3606 return false;
3607 }
3608
gfx_v9_4_3_reset_hw_pipe(struct amdgpu_ring * ring)3609 static int gfx_v9_4_3_reset_hw_pipe(struct amdgpu_ring *ring)
3610 {
3611 struct amdgpu_device *adev = ring->adev;
3612 uint32_t reset_pipe, clean_pipe;
3613 int r;
3614
3615 if (!gfx_v9_4_3_pipe_reset_support(adev))
3616 return -EINVAL;
3617
3618 gfx_v9_4_3_xcc_set_safe_mode(adev, ring->xcc_id);
3619 mutex_lock(&adev->srbm_mutex);
3620
3621 reset_pipe = RREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL);
3622 clean_pipe = reset_pipe;
3623
3624 if (ring->me == 1) {
3625 switch (ring->pipe) {
3626 case 0:
3627 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3628 MEC_ME1_PIPE0_RESET, 1);
3629 break;
3630 case 1:
3631 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3632 MEC_ME1_PIPE1_RESET, 1);
3633 break;
3634 case 2:
3635 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3636 MEC_ME1_PIPE2_RESET, 1);
3637 break;
3638 case 3:
3639 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3640 MEC_ME1_PIPE3_RESET, 1);
3641 break;
3642 default:
3643 break;
3644 }
3645 } else {
3646 if (ring->pipe)
3647 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3648 MEC_ME2_PIPE1_RESET, 1);
3649 else
3650 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3651 MEC_ME2_PIPE0_RESET, 1);
3652 }
3653
3654 WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL, reset_pipe);
3655 WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL, clean_pipe);
3656 mutex_unlock(&adev->srbm_mutex);
3657 gfx_v9_4_3_xcc_unset_safe_mode(adev, ring->xcc_id);
3658
3659 r = gfx_v9_4_3_unmap_done(adev, ring->me, ring->pipe, ring->queue, ring->xcc_id);
3660 return r;
3661 }
3662
gfx_v9_4_3_reset_kcq(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)3663 static int gfx_v9_4_3_reset_kcq(struct amdgpu_ring *ring,
3664 unsigned int vmid,
3665 struct amdgpu_fence *timedout_fence)
3666 {
3667 struct amdgpu_device *adev = ring->adev;
3668 struct amdgpu_kiq *kiq = &adev->gfx.kiq[ring->xcc_id];
3669 struct amdgpu_ring *kiq_ring = &kiq->ring;
3670 int reset_mode = AMDGPU_RESET_TYPE_PER_QUEUE;
3671 unsigned long flags;
3672 int r;
3673
3674 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
3675 return -EINVAL;
3676
3677 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
3678
3679 spin_lock_irqsave(&kiq->ring_lock, flags);
3680
3681 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
3682 spin_unlock_irqrestore(&kiq->ring_lock, flags);
3683 return -ENOMEM;
3684 }
3685
3686 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
3687 0, 0);
3688 amdgpu_ring_commit(kiq_ring);
3689
3690 spin_unlock_irqrestore(&kiq->ring_lock, flags);
3691
3692 r = amdgpu_ring_test_ring(kiq_ring);
3693 if (r) {
3694 dev_err(adev->dev, "kiq ring test failed after ring: %s queue reset\n",
3695 ring->name);
3696 goto pipe_reset;
3697 }
3698
3699 r = gfx_v9_4_3_unmap_done(adev, ring->me, ring->pipe, ring->queue, ring->xcc_id);
3700 if (r)
3701 dev_err(adev->dev, "fail to wait on hqd deactive and will try pipe reset\n");
3702
3703 pipe_reset:
3704 if (r) {
3705 if (!(adev->gfx.compute_supported_reset & AMDGPU_RESET_TYPE_PER_PIPE))
3706 return -EOPNOTSUPP;
3707 r = gfx_v9_4_3_reset_hw_pipe(ring);
3708 reset_mode = AMDGPU_RESET_TYPE_PER_PIPE;
3709 dev_info(adev->dev, "ring: %s pipe reset :%s\n", ring->name,
3710 r ? "failed" : "successfully");
3711 if (r)
3712 return r;
3713 }
3714
3715 gfx_v9_4_3_xcc_kcq_init_queue(ring, ring->xcc_id, true);
3716
3717 spin_lock_irqsave(&kiq->ring_lock, flags);
3718 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
3719 if (r) {
3720 spin_unlock_irqrestore(&kiq->ring_lock, flags);
3721 return -ENOMEM;
3722 }
3723 kiq->pmf->kiq_map_queues(kiq_ring, ring);
3724 amdgpu_ring_commit(kiq_ring);
3725 r = amdgpu_ring_test_ring(kiq_ring);
3726 spin_unlock_irqrestore(&kiq->ring_lock, flags);
3727 if (r) {
3728 if (reset_mode == AMDGPU_RESET_TYPE_PER_QUEUE)
3729 goto pipe_reset;
3730
3731 dev_err(adev->dev, "fail to remap queue\n");
3732 return r;
3733 }
3734
3735 if (reset_mode == AMDGPU_RESET_TYPE_PER_QUEUE) {
3736 r = amdgpu_ring_test_ring(ring);
3737 if (r)
3738 goto pipe_reset;
3739 }
3740
3741
3742 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
3743 }
3744
gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device * adev,void * ras_error_status,int xcc_id)3745 static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev,
3746 void *ras_error_status, int xcc_id)
3747 {
3748 uint32_t i;
3749 uint32_t data;
3750
3751 if (amdgpu_sriov_vf(adev))
3752 return;
3753
3754 data = RREG32_SOC15(GC, GET_INST(GC, 0), regSQ_TIMEOUT_CONFIG);
3755 data = REG_SET_FIELD(data, SQ_TIMEOUT_CONFIG, TIMEOUT_FATAL_DISABLE,
3756 amdgpu_watchdog_timer.timeout_fatal_disable ? 1 : 0);
3757
3758 if (amdgpu_watchdog_timer.timeout_fatal_disable &&
3759 (amdgpu_watchdog_timer.period < 1 ||
3760 amdgpu_watchdog_timer.period > 0x23)) {
3761 dev_warn(adev->dev, "Watchdog period range is 1 to 0x23\n");
3762 amdgpu_watchdog_timer.period = 0x23;
3763 }
3764 data = REG_SET_FIELD(data, SQ_TIMEOUT_CONFIG, PERIOD_SEL,
3765 amdgpu_watchdog_timer.period);
3766
3767 mutex_lock(&adev->grbm_idx_mutex);
3768 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3769 gfx_v9_4_3_xcc_select_se_sh(adev, i, 0xffffffff, 0xffffffff, xcc_id);
3770 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_TIMEOUT_CONFIG, data);
3771 }
3772 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
3773 xcc_id);
3774 mutex_unlock(&adev->grbm_idx_mutex);
3775 }
3776
gfx_v9_4_3_enable_watchdog_timer(struct amdgpu_device * adev)3777 static void gfx_v9_4_3_enable_watchdog_timer(struct amdgpu_device *adev)
3778 {
3779 amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_enable_watchdog_timer);
3780 }
3781
gfx_v9_4_3_ring_insert_nop(struct amdgpu_ring * ring,uint32_t num_nop)3782 static void gfx_v9_4_3_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
3783 {
3784 /* Header itself is a NOP packet */
3785 if (num_nop == 1) {
3786 amdgpu_ring_write(ring, ring->funcs->nop);
3787 return;
3788 }
3789
3790 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
3791 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
3792
3793 /* Header is at index 0, followed by num_nops - 1 NOP packet's */
3794 amdgpu_ring_insert_nop(ring, num_nop - 1);
3795 }
3796
gfx_v9_4_3_ip_print(struct amdgpu_ip_block * ip_block,struct drm_printer * p)3797 static void gfx_v9_4_3_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
3798 {
3799 struct amdgpu_device *adev = ip_block->adev;
3800 uint32_t i, j, k;
3801 uint32_t xcc_id, xcc_offset, inst_offset;
3802 uint32_t num_xcc, reg, num_inst;
3803 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
3804
3805 if (!adev->gfx.ip_dump_core)
3806 return;
3807
3808 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3809 drm_printf(p, "Number of Instances:%d\n", num_xcc);
3810 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
3811 xcc_offset = xcc_id * reg_count;
3812 drm_printf(p, "\nInstance id:%d\n", xcc_id);
3813 for (i = 0; i < reg_count; i++)
3814 drm_printf(p, "%-50s \t 0x%08x\n",
3815 gc_reg_list_9_4_3[i].reg_name,
3816 adev->gfx.ip_dump_core[xcc_offset + i]);
3817 }
3818
3819 /* print compute queue registers for all instances */
3820 if (!adev->gfx.ip_dump_compute_queues)
3821 return;
3822
3823 num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
3824 adev->gfx.mec.num_queue_per_pipe;
3825
3826 reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
3827 drm_printf(p, "\nnum_xcc: %d num_mec: %d num_pipe: %d num_queue: %d\n",
3828 num_xcc,
3829 adev->gfx.mec.num_mec,
3830 adev->gfx.mec.num_pipe_per_mec,
3831 adev->gfx.mec.num_queue_per_pipe);
3832
3833 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
3834 xcc_offset = xcc_id * reg_count * num_inst;
3835 inst_offset = 0;
3836 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
3837 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
3838 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
3839 drm_printf(p,
3840 "\nxcc:%d mec:%d, pipe:%d, queue:%d\n",
3841 xcc_id, i, j, k);
3842 for (reg = 0; reg < reg_count; reg++) {
3843 if (i && gc_cp_reg_list_9_4_3[reg].reg_offset ==
3844 regCP_MEC_ME1_HEADER_DUMP)
3845 drm_printf(p,
3846 "%-50s \t 0x%08x\n",
3847 "regCP_MEC_ME2_HEADER_DUMP",
3848 adev->gfx.ip_dump_compute_queues
3849 [xcc_offset + inst_offset +
3850 reg]);
3851 else
3852 drm_printf(p,
3853 "%-50s \t 0x%08x\n",
3854 gc_cp_reg_list_9_4_3[reg].reg_name,
3855 adev->gfx.ip_dump_compute_queues
3856 [xcc_offset + inst_offset +
3857 reg]);
3858 }
3859 inst_offset += reg_count;
3860 }
3861 }
3862 }
3863 }
3864 }
3865
gfx_v9_4_3_ip_dump(struct amdgpu_ip_block * ip_block)3866 static void gfx_v9_4_3_ip_dump(struct amdgpu_ip_block *ip_block)
3867 {
3868 struct amdgpu_device *adev = ip_block->adev;
3869 uint32_t i, j, k;
3870 uint32_t num_xcc, reg, num_inst;
3871 uint32_t xcc_id, xcc_offset, inst_offset;
3872 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
3873
3874 if (!adev->gfx.ip_dump_core)
3875 return;
3876
3877 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3878
3879 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
3880 xcc_offset = xcc_id * reg_count;
3881 for (i = 0; i < reg_count; i++)
3882 adev->gfx.ip_dump_core[xcc_offset + i] =
3883 RREG32(SOC15_REG_ENTRY_OFFSET_INST(gc_reg_list_9_4_3[i],
3884 GET_INST(GC, xcc_id)));
3885 }
3886
3887 /* dump compute queue registers for all instances */
3888 if (!adev->gfx.ip_dump_compute_queues)
3889 return;
3890
3891 num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
3892 adev->gfx.mec.num_queue_per_pipe;
3893 reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
3894 mutex_lock(&adev->srbm_mutex);
3895 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
3896 xcc_offset = xcc_id * reg_count * num_inst;
3897 inst_offset = 0;
3898 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
3899 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
3900 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
3901 /* ME0 is for GFX so start from 1 for CP */
3902 soc15_grbm_select(adev, 1 + i, j, k, 0,
3903 GET_INST(GC, xcc_id));
3904
3905 for (reg = 0; reg < reg_count; reg++) {
3906 if (i && gc_cp_reg_list_9_4_3[reg].reg_offset ==
3907 regCP_MEC_ME1_HEADER_DUMP)
3908 adev->gfx.ip_dump_compute_queues
3909 [xcc_offset +
3910 inst_offset + reg] =
3911 RREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id),
3912 regCP_MEC_ME2_HEADER_DUMP));
3913 else
3914 adev->gfx.ip_dump_compute_queues
3915 [xcc_offset +
3916 inst_offset + reg] =
3917 RREG32(SOC15_REG_ENTRY_OFFSET_INST(
3918 gc_cp_reg_list_9_4_3[reg],
3919 GET_INST(GC, xcc_id)));
3920 }
3921 inst_offset += reg_count;
3922 }
3923 }
3924 }
3925 }
3926 soc15_grbm_select(adev, 0, 0, 0, 0, 0);
3927 mutex_unlock(&adev->srbm_mutex);
3928 }
3929
gfx_v9_4_3_ring_emit_cleaner_shader(struct amdgpu_ring * ring)3930 static void gfx_v9_4_3_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
3931 {
3932 /* Emit the cleaner shader */
3933 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
3934 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */
3935 }
3936
3937 static const struct amd_ip_funcs gfx_v9_4_3_ip_funcs = {
3938 .name = "gfx_v9_4_3",
3939 .early_init = gfx_v9_4_3_early_init,
3940 .late_init = gfx_v9_4_3_late_init,
3941 .sw_init = gfx_v9_4_3_sw_init,
3942 .sw_fini = gfx_v9_4_3_sw_fini,
3943 .hw_init = gfx_v9_4_3_hw_init,
3944 .hw_fini = gfx_v9_4_3_hw_fini,
3945 .suspend = gfx_v9_4_3_suspend,
3946 .resume = gfx_v9_4_3_resume,
3947 .is_idle = gfx_v9_4_3_is_idle,
3948 .wait_for_idle = gfx_v9_4_3_wait_for_idle,
3949 .soft_reset = gfx_v9_4_3_soft_reset,
3950 .set_clockgating_state = gfx_v9_4_3_set_clockgating_state,
3951 .set_powergating_state = gfx_v9_4_3_set_powergating_state,
3952 .get_clockgating_state = gfx_v9_4_3_get_clockgating_state,
3953 .dump_ip_state = gfx_v9_4_3_ip_dump,
3954 .print_ip_state = gfx_v9_4_3_ip_print,
3955 };
3956
3957 static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_compute = {
3958 .type = AMDGPU_RING_TYPE_COMPUTE,
3959 .align_mask = 0xff,
3960 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
3961 .support_64bit_ptrs = true,
3962 .get_rptr = gfx_v9_4_3_ring_get_rptr_compute,
3963 .get_wptr = gfx_v9_4_3_ring_get_wptr_compute,
3964 .set_wptr = gfx_v9_4_3_ring_set_wptr_compute,
3965 .emit_frame_size =
3966 20 + /* gfx_v9_4_3_ring_emit_gds_switch */
3967 7 + /* gfx_v9_4_3_ring_emit_hdp_flush */
3968 5 + /* hdp invalidate */
3969 7 + /* gfx_v9_4_3_ring_emit_pipeline_sync */
3970 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
3971 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
3972 2 + /* gfx_v9_4_3_ring_emit_vm_flush */
3973 8 + 8 + 8 + /* gfx_v9_4_3_ring_emit_fence x3 for user fence, vm fence */
3974 7 + /* gfx_v9_4_3_emit_mem_sync */
3975 5 + /* gfx_v9_4_3_emit_wave_limit for updating regSPI_WCL_PIPE_PERCENT_GFX register */
3976 15 + /* for updating 3 regSPI_WCL_PIPE_PERCENT_CS registers */
3977 2, /* gfx_v9_4_3_ring_emit_cleaner_shader */
3978 .emit_ib_size = 7, /* gfx_v9_4_3_ring_emit_ib_compute */
3979 .emit_ib = gfx_v9_4_3_ring_emit_ib_compute,
3980 .emit_fence = gfx_v9_4_3_ring_emit_fence,
3981 .emit_pipeline_sync = gfx_v9_4_3_ring_emit_pipeline_sync,
3982 .emit_vm_flush = gfx_v9_4_3_ring_emit_vm_flush,
3983 .emit_gds_switch = gfx_v9_4_3_ring_emit_gds_switch,
3984 .emit_hdp_flush = gfx_v9_4_3_ring_emit_hdp_flush,
3985 .test_ring = gfx_v9_4_3_ring_test_ring,
3986 .test_ib = gfx_v9_4_3_ring_test_ib,
3987 .insert_nop = gfx_v9_4_3_ring_insert_nop,
3988 .pad_ib = amdgpu_ring_generic_pad_ib,
3989 .emit_wreg = gfx_v9_4_3_ring_emit_wreg,
3990 .emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,
3991 .emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,
3992 .soft_recovery = gfx_v9_4_3_ring_soft_recovery,
3993 .emit_mem_sync = gfx_v9_4_3_emit_mem_sync,
3994 .emit_wave_limit = gfx_v9_4_3_emit_wave_limit,
3995 .reset = gfx_v9_4_3_reset_kcq,
3996 .emit_cleaner_shader = gfx_v9_4_3_ring_emit_cleaner_shader,
3997 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
3998 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
3999 };
4000
4001 static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_kiq = {
4002 .type = AMDGPU_RING_TYPE_KIQ,
4003 .align_mask = 0xff,
4004 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4005 .support_64bit_ptrs = true,
4006 .get_rptr = gfx_v9_4_3_ring_get_rptr_compute,
4007 .get_wptr = gfx_v9_4_3_ring_get_wptr_compute,
4008 .set_wptr = gfx_v9_4_3_ring_set_wptr_compute,
4009 .emit_frame_size =
4010 20 + /* gfx_v9_4_3_ring_emit_gds_switch */
4011 7 + /* gfx_v9_4_3_ring_emit_hdp_flush */
4012 5 + /* hdp invalidate */
4013 7 + /* gfx_v9_4_3_ring_emit_pipeline_sync */
4014 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4015 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4016 2 + /* gfx_v9_4_3_ring_emit_vm_flush */
4017 8 + 8 + 8, /* gfx_v9_4_3_ring_emit_fence_kiq x3 for user fence, vm fence */
4018 .emit_ib_size = 7, /* gfx_v9_4_3_ring_emit_ib_compute */
4019 .emit_fence = gfx_v9_4_3_ring_emit_fence_kiq,
4020 .test_ring = gfx_v9_4_3_ring_test_ring,
4021 .insert_nop = amdgpu_ring_insert_nop,
4022 .pad_ib = amdgpu_ring_generic_pad_ib,
4023 .emit_rreg = gfx_v9_4_3_ring_emit_rreg,
4024 .emit_wreg = gfx_v9_4_3_ring_emit_wreg,
4025 .emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,
4026 .emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,
4027 .emit_hdp_flush = gfx_v9_4_3_ring_emit_hdp_flush,
4028 };
4029
gfx_v9_4_3_set_ring_funcs(struct amdgpu_device * adev)4030 static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev)
4031 {
4032 int i, j, num_xcc;
4033
4034 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4035 for (i = 0; i < num_xcc; i++) {
4036 adev->gfx.kiq[i].ring.funcs = &gfx_v9_4_3_ring_funcs_kiq;
4037
4038 for (j = 0; j < adev->gfx.num_compute_rings; j++)
4039 adev->gfx.compute_ring[j + i * adev->gfx.num_compute_rings].funcs
4040 = &gfx_v9_4_3_ring_funcs_compute;
4041 }
4042 }
4043
4044 static const struct amdgpu_irq_src_funcs gfx_v9_4_3_eop_irq_funcs = {
4045 .set = gfx_v9_4_3_set_eop_interrupt_state,
4046 .process = gfx_v9_4_3_eop_irq,
4047 };
4048
4049 static const struct amdgpu_irq_src_funcs gfx_v9_4_3_priv_reg_irq_funcs = {
4050 .set = gfx_v9_4_3_set_priv_reg_fault_state,
4051 .process = gfx_v9_4_3_priv_reg_irq,
4052 };
4053
4054 static const struct amdgpu_irq_src_funcs gfx_v9_4_3_bad_op_irq_funcs = {
4055 .set = gfx_v9_4_3_set_bad_op_fault_state,
4056 .process = gfx_v9_4_3_bad_op_irq,
4057 };
4058
4059 static const struct amdgpu_irq_src_funcs gfx_v9_4_3_priv_inst_irq_funcs = {
4060 .set = gfx_v9_4_3_set_priv_inst_fault_state,
4061 .process = gfx_v9_4_3_priv_inst_irq,
4062 };
4063
gfx_v9_4_3_set_irq_funcs(struct amdgpu_device * adev)4064 static void gfx_v9_4_3_set_irq_funcs(struct amdgpu_device *adev)
4065 {
4066 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
4067 adev->gfx.eop_irq.funcs = &gfx_v9_4_3_eop_irq_funcs;
4068
4069 adev->gfx.priv_reg_irq.num_types = 1;
4070 adev->gfx.priv_reg_irq.funcs = &gfx_v9_4_3_priv_reg_irq_funcs;
4071
4072 adev->gfx.bad_op_irq.num_types = 1;
4073 adev->gfx.bad_op_irq.funcs = &gfx_v9_4_3_bad_op_irq_funcs;
4074
4075 adev->gfx.priv_inst_irq.num_types = 1;
4076 adev->gfx.priv_inst_irq.funcs = &gfx_v9_4_3_priv_inst_irq_funcs;
4077 }
4078
gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device * adev)4079 static void gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device *adev)
4080 {
4081 adev->gfx.rlc.funcs = &gfx_v9_4_3_rlc_funcs;
4082 }
4083
4084
gfx_v9_4_3_set_gds_init(struct amdgpu_device * adev)4085 static void gfx_v9_4_3_set_gds_init(struct amdgpu_device *adev)
4086 {
4087 /* 9.4.3 variants removed all the GDS internal memory,
4088 * only support GWS opcode in kernel, like barrier
4089 * semaphore.etc */
4090
4091 /* init asic gds info */
4092 adev->gds.gds_size = 0;
4093 adev->gds.gds_compute_max_wave_id = 0;
4094 adev->gds.gws_size = 64;
4095 adev->gds.oa_size = 16;
4096 }
4097
gfx_v9_4_3_set_user_cu_inactive_bitmap(struct amdgpu_device * adev,u32 bitmap,int xcc_id)4098 static void gfx_v9_4_3_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
4099 u32 bitmap, int xcc_id)
4100 {
4101 u32 data;
4102
4103 if (!bitmap)
4104 return;
4105
4106 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
4107 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
4108
4109 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG, data);
4110 }
4111
gfx_v9_4_3_get_cu_active_bitmap(struct amdgpu_device * adev,int xcc_id)4112 static u32 gfx_v9_4_3_get_cu_active_bitmap(struct amdgpu_device *adev, int xcc_id)
4113 {
4114 u32 data, mask;
4115
4116 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SHADER_ARRAY_CONFIG);
4117 data |= RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG);
4118
4119 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
4120 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
4121
4122 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
4123
4124 return (~data) & mask;
4125 }
4126
gfx_v9_4_3_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)4127 static int gfx_v9_4_3_get_cu_info(struct amdgpu_device *adev,
4128 struct amdgpu_cu_info *cu_info)
4129 {
4130 int i, j, k, prev_counter, counter, xcc_id, active_cu_number = 0;
4131 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0, tmp;
4132 unsigned disable_masks[4 * 4];
4133 bool is_symmetric_cus;
4134
4135 if (!adev || !cu_info)
4136 return -EINVAL;
4137
4138 /*
4139 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
4140 */
4141 if (adev->gfx.config.max_shader_engines *
4142 adev->gfx.config.max_sh_per_se > 16)
4143 return -EINVAL;
4144
4145 amdgpu_gfx_parse_disable_cu(adev, disable_masks,
4146 adev->gfx.config.max_shader_engines,
4147 adev->gfx.config.max_sh_per_se);
4148
4149 mutex_lock(&adev->grbm_idx_mutex);
4150 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
4151 is_symmetric_cus = true;
4152 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
4153 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
4154 mask = 1;
4155 ao_bitmap = 0;
4156 counter = 0;
4157 gfx_v9_4_3_xcc_select_se_sh(adev, i, j, 0xffffffff, xcc_id);
4158 gfx_v9_4_3_set_user_cu_inactive_bitmap(
4159 adev,
4160 disable_masks[i * adev->gfx.config.max_sh_per_se + j],
4161 xcc_id);
4162 bitmap = gfx_v9_4_3_get_cu_active_bitmap(adev, xcc_id);
4163
4164 cu_info->bitmap[xcc_id][i][j] = bitmap;
4165
4166 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
4167 if (bitmap & mask) {
4168 if (counter < adev->gfx.config.max_cu_per_sh)
4169 ao_bitmap |= mask;
4170 counter++;
4171 }
4172 mask <<= 1;
4173 }
4174 active_cu_number += counter;
4175 if (i < 2 && j < 2)
4176 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
4177 cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
4178 }
4179 if (i && is_symmetric_cus && prev_counter != counter)
4180 is_symmetric_cus = false;
4181 prev_counter = counter;
4182 }
4183 if (is_symmetric_cus) {
4184 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_DEBUG);
4185 tmp = REG_SET_FIELD(tmp, CP_CPC_DEBUG, CPC_HARVESTING_RELAUNCH_DISABLE, 1);
4186 tmp = REG_SET_FIELD(tmp, CP_CPC_DEBUG, CPC_HARVESTING_DISPATCH_DISABLE, 1);
4187 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_DEBUG, tmp);
4188 }
4189 gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
4190 xcc_id);
4191 }
4192 mutex_unlock(&adev->grbm_idx_mutex);
4193
4194 cu_info->number = active_cu_number;
4195 cu_info->ao_cu_mask = ao_cu_mask;
4196 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
4197
4198 return 0;
4199 }
4200
4201 const struct amdgpu_ip_block_version gfx_v9_4_3_ip_block = {
4202 .type = AMD_IP_BLOCK_TYPE_GFX,
4203 .major = 9,
4204 .minor = 4,
4205 .rev = 3,
4206 .funcs = &gfx_v9_4_3_ip_funcs,
4207 };
4208
gfx_v9_4_3_xcp_resume(void * handle,uint32_t inst_mask)4209 static int gfx_v9_4_3_xcp_resume(void *handle, uint32_t inst_mask)
4210 {
4211 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4212 uint32_t tmp_mask;
4213 int i, r;
4214
4215 /* TODO : Initialize golden regs */
4216 /* gfx_v9_4_3_init_golden_registers(adev); */
4217
4218 tmp_mask = inst_mask;
4219 for_each_inst(i, tmp_mask)
4220 gfx_v9_4_3_xcc_constants_init(adev, i);
4221
4222 if (!amdgpu_sriov_vf(adev)) {
4223 tmp_mask = inst_mask;
4224 for_each_inst(i, tmp_mask) {
4225 r = gfx_v9_4_3_xcc_rlc_resume(adev, i);
4226 if (r)
4227 return r;
4228 }
4229 }
4230
4231 tmp_mask = inst_mask;
4232 for_each_inst(i, tmp_mask) {
4233 r = gfx_v9_4_3_xcc_cp_resume(adev, i);
4234 if (r)
4235 return r;
4236 }
4237
4238 return 0;
4239 }
4240
gfx_v9_4_3_xcp_suspend(void * handle,uint32_t inst_mask)4241 static int gfx_v9_4_3_xcp_suspend(void *handle, uint32_t inst_mask)
4242 {
4243 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4244 int i;
4245
4246 for_each_inst(i, inst_mask)
4247 gfx_v9_4_3_xcc_fini(adev, i);
4248
4249 return 0;
4250 }
4251
4252 struct amdgpu_xcp_ip_funcs gfx_v9_4_3_xcp_funcs = {
4253 .suspend = &gfx_v9_4_3_xcp_suspend,
4254 .resume = &gfx_v9_4_3_xcp_resume
4255 };
4256
4257 struct amdgpu_gfx_ras gfx_v9_4_3_ras = {
4258 .ras_block = {
4259 .hw_ops = NULL,
4260 },
4261 .enable_watchdog_timer = &gfx_v9_4_3_enable_watchdog_timer,
4262 };
4263