1 /*
2 * Copyright 2025 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/delay.h>
24 #include <linux/kernel.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 #include "amdgpu.h"
29 #include "amdgpu_gfx.h"
30 #include "amdgpu_psp.h"
31 #include "amdgpu_smu.h"
32 #include "amdgpu_atomfirmware.h"
33 #include "amdgpu_userq_fence.h"
34 #include "imu_v12_1.h"
35 #include "soc_v1_0.h"
36 #include "gfx_v12_1_pkt.h"
37
38 #include "gc/gc_12_1_0_offset.h"
39 #include "gc/gc_12_1_0_sh_mask.h"
40 #include "soc24_enum.h"
41 #include "ivsrcid/gfx/irqsrcs_gfx_12_1_0.h"
42
43 #include "soc15.h"
44 #include "clearstate_gfx12.h"
45 #include "v12_structs.h"
46 #include "gfx_v12_1.h"
47 #include "mes_v12_1.h"
48 #include "amdgpu_ras_mgr.h"
49
50 #define GFX12_MEC_HPD_SIZE 2048
51 #define NUM_SIMD_PER_CU_GFX12_1 4
52
53 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
54
55 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000000
56 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000
57 #define regCP_MQD_CONTROL_DEFAULT 0x00000100
58 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509
59 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000
60 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0ae06301
61 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00100000
62
63 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin");
64 MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc_1.bin");
65
66 #define SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 0x00000001
67 #define DEFAULT_SH_MEM_CONFIG \
68 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
69 (SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
70 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
71
72
73 static const struct amdgpu_hwip_reg_entry gc_reg_list_12_1[] = {
74 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
75 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
76 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
77 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
78 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
79 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
80 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
81 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
82 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
83 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
84 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
85 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
86 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
87 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
88 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
89 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
90 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
91 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
92 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
93 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
94 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
95 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
96 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
97 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
98 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
99 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
100 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
101 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
102 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
103 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
104 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
105 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
106 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
107 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
108 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
109 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
110 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
111 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
112 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
113 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
114 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2),
115 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_LO32),
116 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_HI32),
117 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
118 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
119 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
120 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
121 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
122 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
123 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
124 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR0),
125 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR1),
126 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_RS64_INSTR_PNTR),
127 /* cp header registers */
128 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
129 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
130 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
131 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
132 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
133 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
134 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
135 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
136 /* SE status registers */
137 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
138 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
139 };
140
141 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_12_1[] = {
142 /* compute registers */
143 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
144 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
145 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
146 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
147 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
148 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
149 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
150 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
151 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
152 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
153 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
154 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
155 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
156 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
157 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
158 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
159 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
160 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
161 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
162 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
163 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
164 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
165 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
166 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
167 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
168 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
169 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
170 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
171 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
172 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
173 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
174 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
175 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
176 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
177 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
178 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
179 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
180 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
181 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS),
182 /* cp header registers */
183 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
184 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
185 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
186 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
187 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
188 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
189 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
190 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
191 };
192
193 static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id);
194 static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev);
195 static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev);
196 static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev);
197 static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev);
198 static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev);
199 static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev,
200 struct amdgpu_cu_info *cu_info);
201 static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev);
202 static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
203 u32 sh_num, u32 instance, int xcc_id);
204 static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
205 uint32_t val);
206 static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
207 static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
208 uint16_t pasid, uint32_t flush_type,
209 bool all_hub, uint8_t dst_sel);
210 static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
211 static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
212 static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
213 bool enable);
214 static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev,
215 bool enable, int xcc_id);
216 static int gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device *adev);
217
gfx_v12_1_kiq_set_resources(struct amdgpu_ring * kiq_ring,uint64_t queue_mask)218 static void gfx_v12_1_kiq_set_resources(struct amdgpu_ring *kiq_ring,
219 uint64_t queue_mask)
220 {
221 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
222 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
223 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */
224 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
225 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
226 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
227 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
228 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
229 amdgpu_ring_write(kiq_ring, 0);
230 }
231
gfx_v12_1_kiq_map_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring)232 static void gfx_v12_1_kiq_map_queues(struct amdgpu_ring *kiq_ring,
233 struct amdgpu_ring *ring)
234 {
235 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
236 uint64_t wptr_addr = ring->wptr_gpu_addr;
237 uint32_t me = 0, eng_sel = 0;
238
239 switch (ring->funcs->type) {
240 case AMDGPU_RING_TYPE_COMPUTE:
241 me = 1;
242 eng_sel = 0;
243 break;
244 case AMDGPU_RING_TYPE_MES:
245 me = 2;
246 eng_sel = 5;
247 break;
248 default:
249 WARN_ON(1);
250 }
251
252 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
253 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
254 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
255 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
256 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
257 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
258 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
259 PACKET3_MAP_QUEUES_ME((me)) |
260 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
261 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
262 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
263 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
264 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
265 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
266 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
267 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
268 }
269
gfx_v12_1_kiq_unmap_queues(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,enum amdgpu_unmap_queues_action action,u64 gpu_addr,u64 seq)270 static void gfx_v12_1_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
271 struct amdgpu_ring *ring,
272 enum amdgpu_unmap_queues_action action,
273 u64 gpu_addr, u64 seq)
274 {
275 struct amdgpu_device *adev = kiq_ring->adev;
276 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
277
278 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
279 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr,
280 seq, kiq_ring->xcc_id);
281 return;
282 }
283
284 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
285 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
286 PACKET3_UNMAP_QUEUES_ACTION(action) |
287 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
288 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
289 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
290 amdgpu_ring_write(kiq_ring,
291 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
292
293 if (action == PREEMPT_QUEUES_NO_UNMAP) {
294 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
295 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
296 amdgpu_ring_write(kiq_ring, seq);
297 } else {
298 amdgpu_ring_write(kiq_ring, 0);
299 amdgpu_ring_write(kiq_ring, 0);
300 amdgpu_ring_write(kiq_ring, 0);
301 }
302 }
303
gfx_v12_1_kiq_query_status(struct amdgpu_ring * kiq_ring,struct amdgpu_ring * ring,u64 addr,u64 seq)304 static void gfx_v12_1_kiq_query_status(struct amdgpu_ring *kiq_ring,
305 struct amdgpu_ring *ring,
306 u64 addr, u64 seq)
307 {
308 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
309
310 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
311 amdgpu_ring_write(kiq_ring,
312 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
313 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
314 PACKET3_QUERY_STATUS_COMMAND(2));
315 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
316 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
317 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
318 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
319 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
320 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
321 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
322 }
323
gfx_v12_1_kiq_invalidate_tlbs(struct amdgpu_ring * kiq_ring,uint16_t pasid,uint32_t flush_type,bool all_hub)324 static void gfx_v12_1_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
325 uint16_t pasid,
326 uint32_t flush_type,
327 bool all_hub)
328 {
329 gfx_v12_1_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
330 }
331
332 static const struct kiq_pm4_funcs gfx_v12_1_kiq_pm4_funcs = {
333 .kiq_set_resources = gfx_v12_1_kiq_set_resources,
334 .kiq_map_queues = gfx_v12_1_kiq_map_queues,
335 .kiq_unmap_queues = gfx_v12_1_kiq_unmap_queues,
336 .kiq_query_status = gfx_v12_1_kiq_query_status,
337 .kiq_invalidate_tlbs = gfx_v12_1_kiq_invalidate_tlbs,
338 .set_resources_size = 8,
339 .map_queues_size = 7,
340 .unmap_queues_size = 6,
341 .query_status_size = 7,
342 .invalidate_tlbs_size = 2,
343 };
344
gfx_v12_1_set_kiq_pm4_funcs(struct amdgpu_device * adev)345 static void gfx_v12_1_set_kiq_pm4_funcs(struct amdgpu_device *adev)
346 {
347 int i, num_xcc;
348
349 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
350 for (i =0; i < num_xcc; i++)
351 adev->gfx.kiq[i].pmf = &gfx_v12_1_kiq_pm4_funcs;
352 }
353
gfx_v12_1_wait_reg_mem(struct amdgpu_ring * ring,int eng_sel,int mem_space,int opt,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)354 static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
355 int mem_space, int opt, uint32_t addr0,
356 uint32_t addr1, uint32_t ref,
357 uint32_t mask, uint32_t inv)
358 {
359 if (mem_space == 0) {
360 addr0 = soc_v1_0_normalize_xcc_reg_offset(addr0);
361 addr1 = soc_v1_0_normalize_xcc_reg_offset(addr1);
362 }
363
364 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
365 amdgpu_ring_write(ring,
366 /* memory (1) or register (0) */
367 (PACKET3_WAIT_REG_MEM__MEM_SPACE(mem_space) |
368 PACKET3_WAIT_REG_MEM__OPERATION(opt) | /* wait */
369 PACKET3_WAIT_REG_MEM__FUNCTION(3))); /* equal */
370
371 if (mem_space)
372 WARN_ON(addr0 & 0x3); /* Dword align */
373 amdgpu_ring_write(ring, addr0);
374 amdgpu_ring_write(ring, addr1);
375 amdgpu_ring_write(ring, ref);
376 amdgpu_ring_write(ring, mask);
377 amdgpu_ring_write(ring, inv); /* poll interval */
378 }
379
gfx_v12_1_ring_test_ring(struct amdgpu_ring * ring)380 static int gfx_v12_1_ring_test_ring(struct amdgpu_ring *ring)
381 {
382 struct amdgpu_device *adev = ring->adev;
383 uint32_t scratch_reg0_offset, xcc_offset;
384 uint32_t tmp = 0;
385 unsigned i;
386 int r;
387
388 /* Use register offset which is local to XCC in the packet */
389 xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
390 scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0);
391 WREG32(scratch_reg0_offset, 0xCAFEDEAD);
392 tmp = RREG32(scratch_reg0_offset);
393
394 r = amdgpu_ring_alloc(ring, 5);
395 if (r) {
396 dev_err(adev->dev,
397 "amdgpu: cp failed to lock ring %d (%d).\n",
398 ring->idx, r);
399 return r;
400 }
401
402 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
403 gfx_v12_1_ring_emit_wreg(ring, xcc_offset, 0xDEADBEEF);
404 } else {
405 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
406 amdgpu_ring_write(ring, xcc_offset -
407 PACKET3_SET_UCONFIG_REG_START);
408 amdgpu_ring_write(ring, 0xDEADBEEF);
409 }
410 amdgpu_ring_commit(ring);
411
412 for (i = 0; i < adev->usec_timeout; i++) {
413 tmp = RREG32(scratch_reg0_offset);
414 if (tmp == 0xDEADBEEF)
415 break;
416 if (amdgpu_emu_mode == 1)
417 msleep(1);
418 else
419 udelay(1);
420 }
421
422 if (i >= adev->usec_timeout)
423 r = -ETIMEDOUT;
424 return r;
425 }
426
gfx_v12_1_ring_test_ib(struct amdgpu_ring * ring,long timeout)427 static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
428 {
429 struct amdgpu_device *adev = ring->adev;
430 struct amdgpu_ib ib;
431 struct dma_fence *f = NULL;
432 unsigned index;
433 uint64_t gpu_addr;
434 volatile uint32_t *cpu_ptr;
435 long r;
436
437 /* MES KIQ fw hasn't indirect buffer support for now */
438 if (adev->enable_mes_kiq &&
439 ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
440 return 0;
441
442 memset(&ib, 0, sizeof(ib));
443
444 r = amdgpu_wb_get(adev, &index);
445 if (r)
446 return r;
447
448 gpu_addr = adev->wb.gpu_addr + (index * 4);
449 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
450 cpu_ptr = &adev->wb.wb[index];
451
452 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
453 if (r) {
454 dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r);
455 goto err1;
456 }
457
458 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
459 ib.ptr[1] = PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1);
460 ib.ptr[2] = lower_32_bits(gpu_addr);
461 ib.ptr[3] = upper_32_bits(gpu_addr);
462 ib.ptr[4] = 0xDEADBEEF;
463 ib.length_dw = 5;
464
465 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
466 if (r)
467 goto err2;
468
469 r = dma_fence_wait_timeout(f, false, timeout);
470 if (r == 0) {
471 r = -ETIMEDOUT;
472 goto err2;
473 } else if (r < 0) {
474 goto err2;
475 }
476
477 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF)
478 r = 0;
479 else
480 r = -EINVAL;
481 err2:
482 amdgpu_ib_free(&ib, NULL);
483 dma_fence_put(f);
484 err1:
485 amdgpu_wb_free(adev, index);
486 return r;
487 }
488
gfx_v12_1_free_microcode(struct amdgpu_device * adev)489 static void gfx_v12_1_free_microcode(struct amdgpu_device *adev)
490 {
491 amdgpu_ucode_release(&adev->gfx.rlc_fw);
492 amdgpu_ucode_release(&adev->gfx.mec_fw);
493
494 kfree(adev->gfx.rlc.register_list_format);
495 }
496
gfx_v12_1_init_toc_microcode(struct amdgpu_device * adev,const char * ucode_prefix)497 static int gfx_v12_1_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix)
498 {
499 const struct psp_firmware_header_v1_0 *toc_hdr;
500 int err = 0;
501
502 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw,
503 AMDGPU_UCODE_REQUIRED,
504 "amdgpu/%s_toc.bin", ucode_prefix);
505 if (err)
506 goto out;
507
508 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
509 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
510 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
511 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
512 adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
513 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
514 return 0;
515 out:
516 amdgpu_ucode_release(&adev->psp.toc_fw);
517 return err;
518 }
519
gfx_v12_1_init_microcode(struct amdgpu_device * adev)520 static int gfx_v12_1_init_microcode(struct amdgpu_device *adev)
521 {
522 char ucode_prefix[15];
523 int err;
524 const struct rlc_firmware_header_v2_0 *rlc_hdr;
525 uint16_t version_major;
526 uint16_t version_minor;
527
528 DRM_DEBUG("\n");
529
530 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
531
532 if (!amdgpu_sriov_vf(adev)) {
533 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) &&
534 adev->rev_id == 0)
535 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
536 AMDGPU_UCODE_REQUIRED,
537 "amdgpu/%s_rlc_1.bin", ucode_prefix);
538 else
539 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
540 AMDGPU_UCODE_REQUIRED,
541 "amdgpu/%s_rlc.bin", ucode_prefix);
542 if (err)
543 goto out;
544 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
545 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
546 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
547 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
548 if (err)
549 goto out;
550 }
551
552 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
553 AMDGPU_UCODE_REQUIRED,
554 "amdgpu/%s_mec.bin", ucode_prefix);
555 if (err)
556 goto out;
557 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
558 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
559 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
560 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
561 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
562
563 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
564 err = gfx_v12_1_init_toc_microcode(adev, ucode_prefix);
565
566 /* only one MEC for gfx 12 */
567 adev->gfx.mec2_fw = NULL;
568
569 if (adev->gfx.imu.funcs) {
570 if (adev->gfx.imu.funcs->init_microcode) {
571 err = adev->gfx.imu.funcs->init_microcode(adev);
572 if (err)
573 dev_err(adev->dev, "Failed to load imu firmware!\n");
574 }
575 }
576
577 out:
578 if (err) {
579 amdgpu_ucode_release(&adev->gfx.rlc_fw);
580 amdgpu_ucode_release(&adev->gfx.mec_fw);
581 }
582
583 return err;
584 }
585
gfx_v12_1_get_csb_size(struct amdgpu_device * adev)586 static u32 gfx_v12_1_get_csb_size(struct amdgpu_device *adev)
587 {
588 u32 count = 0;
589 const struct cs_section_def *sect = NULL;
590 const struct cs_extent_def *ext = NULL;
591
592 count += 1;
593
594 for (sect = gfx12_cs_data; sect->section != NULL; ++sect) {
595 if (sect->id == SECT_CONTEXT) {
596 for (ext = sect->section; ext->extent != NULL; ++ext)
597 count += 2 + ext->reg_count;
598 } else
599 return 0;
600 }
601
602 return count;
603 }
604
gfx_v12_1_get_csb_buffer(struct amdgpu_device * adev,u32 * buffer)605 static void gfx_v12_1_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
606 {
607 u32 count = 0, clustercount = 0, i;
608 const struct cs_section_def *sect = NULL;
609 const struct cs_extent_def *ext = NULL;
610
611 if (adev->gfx.rlc.cs_data == NULL)
612 return;
613 if (buffer == NULL)
614 return;
615
616 count += 1;
617
618 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
619 if (sect->id == SECT_CONTEXT) {
620 for (ext = sect->section; ext->extent != NULL; ++ext) {
621 clustercount++;
622 buffer[count++] = ext->reg_count;
623 buffer[count++] = ext->reg_index;
624
625 for (i = 0; i < ext->reg_count; i++)
626 buffer[count++] = cpu_to_le32(ext->extent[i]);
627 }
628 } else
629 return;
630 }
631
632 buffer[0] = clustercount;
633 }
634
gfx_v12_1_rlc_fini(struct amdgpu_device * adev)635 static void gfx_v12_1_rlc_fini(struct amdgpu_device *adev)
636 {
637 /* clear state block */
638 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
639 &adev->gfx.rlc.clear_state_gpu_addr,
640 (void **)&adev->gfx.rlc.cs_ptr);
641
642 /* jump table block */
643 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
644 &adev->gfx.rlc.cp_table_gpu_addr,
645 (void **)&adev->gfx.rlc.cp_table_ptr);
646 }
647
gfx_v12_1_init_rlcg_reg_access_ctrl(struct amdgpu_device * adev)648 static void gfx_v12_1_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
649 {
650 int xcc_id, num_xcc;
651 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
652
653 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
654 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
655 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[GET_INST(GC, xcc_id)];
656
657 reg_access_ctrl->grbm_cntl =
658 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_CNTL);
659 reg_access_ctrl->grbm_idx =
660 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX);
661
662 reg_access_ctrl->vfi_cmd =
663 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_CMD);
664 reg_access_ctrl->vfi_stat =
665 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_STAT);
666 reg_access_ctrl->vfi_addr =
667 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_ADDR);
668 reg_access_ctrl->vfi_data =
669 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_DATA);
670 reg_access_ctrl->vfi_grbm_cntl =
671 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_GRBM_GFX_CNTL);
672 reg_access_ctrl->vfi_grbm_idx =
673 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_GRBM_GFX_INDEX);
674 reg_access_ctrl->vfi_grbm_cntl_data = 0;
675 reg_access_ctrl->vfi_grbm_idx_data = 0;
676 }
677 adev->gfx.rlc.rlcg_reg_access_supported = true;
678 }
679
gfx_v12_1_rlc_init(struct amdgpu_device * adev)680 static int gfx_v12_1_rlc_init(struct amdgpu_device *adev)
681 {
682 const struct cs_section_def *cs_data;
683 int r, i, num_xcc;
684
685 adev->gfx.rlc.cs_data = gfx12_cs_data;
686
687 cs_data = adev->gfx.rlc.cs_data;
688
689 if (cs_data) {
690 /* init clear state block */
691 r = amdgpu_gfx_rlc_init_csb(adev);
692 if (r)
693 return r;
694 }
695
696 /* init spm vmid with 0xf */
697 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
698 for (i = 0; i < num_xcc; i++) {
699 if (adev->gfx.rlc.funcs->update_spm_vmid)
700 adev->gfx.rlc.funcs->update_spm_vmid(adev, i, NULL, 0xf);
701 }
702
703 return 0;
704 }
705
gfx_v12_1_mec_fini(struct amdgpu_device * adev)706 static void gfx_v12_1_mec_fini(struct amdgpu_device *adev)
707 {
708 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
709 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
710 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL);
711 }
712
gfx_v12_1_mec_init(struct amdgpu_device * adev)713 static int gfx_v12_1_mec_init(struct amdgpu_device *adev)
714 {
715 int r, i, num_xcc;
716 u32 *hpd;
717 size_t mec_hpd_size;
718
719 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
720
721 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
722 for (i = 0; i < num_xcc; i++)
723 bitmap_zero(adev->gfx.mec_bitmap[i].queue_bitmap,
724 AMDGPU_MAX_COMPUTE_QUEUES);
725
726 /* take ownership of the relevant compute queues */
727 amdgpu_gfx_compute_queue_acquire(adev);
728 mec_hpd_size = adev->gfx.num_compute_rings *
729 GFX12_MEC_HPD_SIZE * num_xcc;
730
731 if (mec_hpd_size) {
732 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
733 AMDGPU_GEM_DOMAIN_GTT,
734 &adev->gfx.mec.hpd_eop_obj,
735 &adev->gfx.mec.hpd_eop_gpu_addr,
736 (void **)&hpd);
737 if (r) {
738 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
739 gfx_v12_1_mec_fini(adev);
740 return r;
741 }
742
743 memset(hpd, 0, mec_hpd_size);
744
745 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
746 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
747 }
748
749 return 0;
750 }
751
wave_read_ind(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t wave,uint32_t address)752 static uint32_t wave_read_ind(struct amdgpu_device *adev,
753 uint32_t xcc_id, uint32_t wave,
754 uint32_t address)
755 {
756 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
757 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
758 (address << SQ_IND_INDEX__INDEX__SHIFT));
759 return RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
760 }
761
wave_read_regs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)762 static void wave_read_regs(struct amdgpu_device *adev,
763 uint32_t xcc_id, uint32_t wave,
764 uint32_t thread, uint32_t regno,
765 uint32_t num, uint32_t *out)
766 {
767 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
768 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
769 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
770 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
771 (SQ_IND_INDEX__AUTO_INCR_MASK));
772 while (num--)
773 *(out++) = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
774 }
775
gfx_v12_1_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)776 static void gfx_v12_1_read_wave_data(struct amdgpu_device *adev,
777 uint32_t xcc_id,
778 uint32_t simd, uint32_t wave,
779 uint32_t *dst, int *no_fields)
780 {
781 /* in gfx12 the SIMD_ID is specified as part of the INSTANCE
782 * field when performing a select_se_sh so it should be
783 * zero here */
784 WARN_ON(simd != 0);
785
786 /* type 4 wave data */
787 dst[(*no_fields)++] = 4;
788 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATUS);
789 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_LO);
790 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_HI);
791 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_LO);
792 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_HI);
793 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID1);
794 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID2);
795 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_GPR_ALLOC);
796 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_LDS_ALLOC);
797 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS);
798 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS2);
799 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_DBG1);
800 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_M0);
801 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_MODE);
802 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATE_PRIV);
803 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_PRIV);
804 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_USER);
805 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_TRAP_CTRL);
806 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_ACTIVE);
807 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_VALID_AND_IDLE);
808 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_LO);
809 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_HI);
810 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_SCHED_MODE);
811 }
812
gfx_v12_1_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)813 static void gfx_v12_1_read_wave_sgprs(struct amdgpu_device *adev,
814 uint32_t xcc_id, uint32_t simd,
815 uint32_t wave, uint32_t start,
816 uint32_t size, uint32_t *dst)
817 {
818 WARN_ON(simd != 0);
819
820 wave_read_regs(adev, xcc_id, wave, 0,
821 start + SQIND_WAVE_SGPRS_OFFSET,
822 size, dst);
823 }
824
gfx_v12_1_read_wave_vgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t start,uint32_t size,uint32_t * dst)825 static void gfx_v12_1_read_wave_vgprs(struct amdgpu_device *adev,
826 uint32_t xcc_id, uint32_t simd,
827 uint32_t wave, uint32_t thread,
828 uint32_t start, uint32_t size,
829 uint32_t *dst)
830 {
831 wave_read_regs(adev, xcc_id, wave, thread,
832 start + SQIND_WAVE_VGPRS_OFFSET,
833 size, dst);
834 }
835
gfx_v12_1_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)836 static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev,
837 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
838 {
839 soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
840 }
841
842 #define regGFX_IMU_PARTITION_SWITCH 0x5f8c
843 #define regGFX_IMU_PARTITION_SWITCH_BASE_IDX 1
844 #define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP__SHIFT 0x2
845 #define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP_MASK 0x0000003CL
846
gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device * adev)847 static int gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device *adev)
848 {
849 u32 reg_data;
850
851 /* the register data is expected to be the same on all instances */
852 reg_data = RREG32_SOC15(GC, GET_INST(GC, 0), regGFX_IMU_PARTITION_SWITCH);
853
854 return REG_GET_FIELD(reg_data, GFX_IMU_PARTITION_SWITCH, TOTAL_XCCS_IN_XCP);
855 }
856
gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device * adev,int ih_node)857 static int gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
858 {
859 int logic_xcc;
860 int xcc = (ih_node & 0x7) - 2 + (ih_node >> 3) * 4;
861
862 for (logic_xcc = 0; logic_xcc < NUM_XCC(adev->gfx.xcc_mask); logic_xcc++) {
863 if (xcc == GET_INST(GC, logic_xcc))
864 return logic_xcc;
865 }
866
867 dev_err(adev->dev, "Couldn't find xcc mapping from IH node");
868 return -EINVAL;
869 }
870
871 static const struct amdgpu_gfx_funcs gfx_v12_1_gfx_funcs = {
872 .get_gpu_clock_counter = &gfx_v12_1_get_gpu_clock_counter,
873 .select_se_sh = &gfx_v12_1_xcc_select_se_sh,
874 .read_wave_data = &gfx_v12_1_read_wave_data,
875 .read_wave_sgprs = &gfx_v12_1_read_wave_sgprs,
876 .read_wave_vgprs = &gfx_v12_1_read_wave_vgprs,
877 .select_me_pipe_q = &gfx_v12_1_select_me_pipe_q,
878 .update_perfmon_mgcg = &gfx_v12_1_update_perf_clk,
879 .get_xccs_per_xcp = &gfx_v12_1_get_xccs_per_xcp,
880 .ih_node_to_logical_xcc = &gfx_v12_1_ih_to_xcc_inst,
881 };
882
gfx_v12_1_gpu_early_init(struct amdgpu_device * adev)883 static int gfx_v12_1_gpu_early_init(struct amdgpu_device *adev)
884 {
885 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
886 case IP_VERSION(12, 1, 0):
887 adev->gfx.config.max_hw_contexts = 8;
888 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
889 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
890 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
891 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
892 break;
893 default:
894 dev_warn(adev->dev, "Unsupported GC version 0x%08x\n",
895 amdgpu_ip_version(adev, GC_HWIP, 0));
896 return -EINVAL;
897 }
898
899 return 0;
900 }
901
gfx_v12_1_compute_ring_init(struct amdgpu_device * adev,int ring_id,int xcc_id,int mec,int pipe,int queue)902 static int gfx_v12_1_compute_ring_init(struct amdgpu_device *adev, int ring_id,
903 int xcc_id, int mec, int pipe, int queue)
904 {
905 int r;
906 unsigned irq_type;
907 struct amdgpu_ring *ring;
908 unsigned int hw_prio;
909 uint32_t xcc_doorbell_start;
910
911 ring = &adev->gfx.compute_ring[xcc_id * adev->gfx.num_compute_rings +
912 ring_id];
913
914 /* mec0 is me1 */
915 ring->xcc_id = xcc_id;
916 ring->me = mec + 1;
917 ring->pipe = pipe;
918 ring->queue = queue;
919
920 ring->ring_obj = NULL;
921 ring->use_doorbell = true;
922 xcc_doorbell_start = adev->doorbell_index.mec_ring0 +
923 xcc_id * adev->doorbell_index.xcc_doorbell_range;
924 ring->doorbell_index = (xcc_doorbell_start + ring_id) << 1;
925 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
926 (ring_id + xcc_id * adev->gfx.num_compute_rings) *
927 GFX12_MEC_HPD_SIZE;
928 ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
929 sprintf(ring->name, "comp_%d.%d.%d.%d",
930 ring->xcc_id, ring->me, ring->pipe, ring->queue);
931
932 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
933 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
934 + ring->pipe;
935 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
936 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
937 /* type-2 packets are deprecated on MEC, use type-3 instead */
938 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
939 hw_prio, NULL);
940 if (r)
941 return r;
942
943 return 0;
944 }
945
946 static struct {
947 SOC24_FIRMWARE_ID id;
948 unsigned int offset;
949 unsigned int size;
950 unsigned int size_x16;
951 unsigned int num_inst;
952 } rlc_autoload_info[SOC24_FIRMWARE_ID_MAX];
953
954 #define RLC_TOC_OFFSET_DWUNIT 8
955 #define RLC_SIZE_MULTIPLE 1024
956 #define RLC_TOC_UMF_SIZE_inM 23ULL
957 #define RLC_TOC_FORMAT_API 165ULL
958
959 #define RLC_NUM_INS_CODE0 1
960 #define RLC_NUM_INS_CODE1 8
961 #define RLC_NUM_INS_CODE2 2
962 #define RLC_NUM_INS_CODE3 16
963
gfx_v12_1_parse_rlc_toc(struct amdgpu_device * adev,void * rlc_toc)964 static void gfx_v12_1_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc)
965 {
966 RLC_TABLE_OF_CONTENT_V2 *ucode = rlc_toc;
967
968 while (ucode && (ucode->id > SOC24_FIRMWARE_ID_INVALID)) {
969 rlc_autoload_info[ucode->id].id = ucode->id;
970 rlc_autoload_info[ucode->id].offset =
971 ucode->offset * RLC_TOC_OFFSET_DWUNIT * 4;
972 rlc_autoload_info[ucode->id].size =
973 ucode->size_x16 ? ucode->size * RLC_SIZE_MULTIPLE * 4 :
974 ucode->size * 4;
975 switch (ucode->vfflr_image_code) {
976 case 0:
977 rlc_autoload_info[ucode->id].num_inst =
978 RLC_NUM_INS_CODE0;
979 break;
980 case 1:
981 rlc_autoload_info[ucode->id].num_inst =
982 RLC_NUM_INS_CODE1;
983 break;
984 case 2:
985 rlc_autoload_info[ucode->id].num_inst =
986 RLC_NUM_INS_CODE2;
987 break;
988 case 3:
989 rlc_autoload_info[ucode->id].num_inst =
990 RLC_NUM_INS_CODE3;
991 break;
992 default:
993 dev_err(adev->dev,
994 "Invalid Instance number detected\n");
995 break;
996 }
997 ucode++;
998 }
999 }
1000
gfx_v12_1_calc_toc_total_size(struct amdgpu_device * adev)1001 static uint32_t gfx_v12_1_calc_toc_total_size(struct amdgpu_device *adev)
1002 {
1003 uint32_t total_size = 0;
1004 SOC24_FIRMWARE_ID id;
1005
1006 gfx_v12_1_parse_rlc_toc(adev, adev->psp.toc.start_addr);
1007
1008 for (id = SOC24_FIRMWARE_ID_RLC_G_UCODE; id < SOC24_FIRMWARE_ID_MAX; id++)
1009 total_size += rlc_autoload_info[id].size;
1010
1011 /* In case the offset in rlc toc ucode is aligned */
1012 if (total_size < rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset)
1013 total_size = rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset +
1014 rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].size;
1015 if (total_size < (RLC_TOC_UMF_SIZE_inM << 20))
1016 total_size = RLC_TOC_UMF_SIZE_inM << 20;
1017
1018 return total_size;
1019 }
1020
gfx_v12_1_rlc_autoload_buffer_init(struct amdgpu_device * adev)1021 static int gfx_v12_1_rlc_autoload_buffer_init(struct amdgpu_device *adev)
1022 {
1023 int r;
1024 uint32_t total_size;
1025
1026 total_size = gfx_v12_1_calc_toc_total_size(adev);
1027
1028 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024,
1029 AMDGPU_GEM_DOMAIN_VRAM,
1030 &adev->gfx.rlc.rlc_autoload_bo,
1031 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1032 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1033
1034 if (r) {
1035 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
1036 return r;
1037 }
1038
1039 return 0;
1040 }
1041
gfx_v12_1_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device * adev,SOC24_FIRMWARE_ID id,const void * fw_data,uint32_t fw_size)1042 static void gfx_v12_1_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
1043 SOC24_FIRMWARE_ID id,
1044 const void *fw_data,
1045 uint32_t fw_size)
1046 {
1047 uint32_t toc_offset;
1048 uint32_t toc_fw_size, toc_fw_inst_size;
1049 char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
1050 int i, num_inst;
1051
1052 if (id <= SOC24_FIRMWARE_ID_INVALID || id >= SOC24_FIRMWARE_ID_MAX)
1053 return;
1054
1055 toc_offset = rlc_autoload_info[id].offset;
1056 toc_fw_size = rlc_autoload_info[id].size;
1057 num_inst = rlc_autoload_info[id].num_inst;
1058 toc_fw_inst_size = toc_fw_size / num_inst;
1059
1060 if (fw_size == 0)
1061 fw_size = toc_fw_inst_size;
1062
1063 if (fw_size > toc_fw_inst_size)
1064 fw_size = toc_fw_inst_size;
1065
1066 for (i = 0; i < num_inst; i++) {
1067 if ((num_inst == RLC_NUM_INS_CODE0) ||
1068 ((1 << (i / 2)) & adev->gfx.xcc_mask)) {
1069 memcpy(ptr + toc_offset + i * toc_fw_inst_size, fw_data, fw_size);
1070
1071 if (fw_size < toc_fw_inst_size)
1072 memset(ptr + toc_offset + fw_size + i * toc_fw_inst_size,
1073 0, toc_fw_inst_size - fw_size);
1074 }
1075 }
1076 }
1077
1078 static void
gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device * adev)1079 gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
1080 {
1081 void *data;
1082 uint32_t size;
1083 uint32_t *toc_ptr;
1084
1085 data = adev->psp.toc.start_addr;
1086 size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_TOC].size;
1087
1088 toc_ptr = (uint32_t *)data + size / 4 - 2;
1089 *toc_ptr = (RLC_TOC_FORMAT_API << 24) | 0x1;
1090
1091 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_TOC,
1092 data, size);
1093 }
1094
1095 static void
gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device * adev)1096 gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
1097 {
1098 const __le32 *fw_data;
1099 uint32_t fw_size;
1100 const struct gfx_firmware_header_v2_0 *cpv2_hdr;
1101 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1102 const struct rlc_firmware_header_v2_1 *rlcv21_hdr;
1103 const struct rlc_firmware_header_v2_2 *rlcv22_hdr;
1104 uint16_t version_major, version_minor;
1105
1106 /* mec ucode */
1107 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *)
1108 adev->gfx.mec_fw->data;
1109 /* instruction */
1110 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1111 le32_to_cpu(cpv2_hdr->ucode_offset_bytes));
1112 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes);
1113 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC,
1114 fw_data, fw_size);
1115 /* data */
1116 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1117 le32_to_cpu(cpv2_hdr->data_offset_bytes));
1118 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes);
1119 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P0_STACK,
1120 fw_data, fw_size);
1121 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P1_STACK,
1122 fw_data, fw_size);
1123 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P2_STACK,
1124 fw_data, fw_size);
1125 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P3_STACK,
1126 fw_data, fw_size);
1127
1128 /* rlc ucode */
1129 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
1130 adev->gfx.rlc_fw->data;
1131 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1132 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
1133 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
1134 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_G_UCODE,
1135 fw_data, fw_size);
1136
1137 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1138 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1139 if (version_major == 2) {
1140 if (version_minor >= 1) {
1141 rlcv21_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1142
1143 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1144 le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_offset_bytes));
1145 fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_size_bytes);
1146 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLCG_SCRATCH,
1147 fw_data, fw_size);
1148
1149 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1150 le32_to_cpu(rlcv21_hdr->save_restore_list_srm_offset_bytes));
1151 fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_srm_size_bytes);
1152 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_SRM_ARAM,
1153 fw_data, fw_size);
1154 }
1155 if (version_minor >= 2) {
1156 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1157
1158 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1159 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes));
1160 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes);
1161 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_UCODE,
1162 fw_data, fw_size);
1163
1164 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1165 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes));
1166 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes);
1167 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_DRAM_BOOT,
1168 fw_data, fw_size);
1169 }
1170 }
1171 }
1172
1173 static void
gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device * adev)1174 gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
1175 {
1176 const __le32 *fw_data;
1177 uint32_t fw_size;
1178 const struct sdma_firmware_header_v3_0 *sdma_hdr;
1179
1180 if (adev->sdma.instance[0].fw) {
1181 sdma_hdr = (const struct sdma_firmware_header_v3_0 *)
1182 adev->sdma.instance[0].fw->data;
1183 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data +
1184 le32_to_cpu(sdma_hdr->ucode_offset_bytes));
1185 fw_size = le32_to_cpu(sdma_hdr->ucode_size_bytes);
1186
1187 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_SDMA_UCODE_TH0,
1188 fw_data, fw_size);
1189 }
1190 }
1191
1192 static void
gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device * adev)1193 gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev)
1194 {
1195 const __le32 *fw_data;
1196 unsigned fw_size;
1197 const struct mes_firmware_header_v1_0 *mes_hdr;
1198 int pipe, ucode_id, data_id;
1199
1200 for (pipe = 0; pipe < 2; pipe++) {
1201 if (pipe == 0) {
1202 ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P0;
1203 data_id = SOC24_FIRMWARE_ID_RS64_MES_P0_STACK;
1204 } else {
1205 ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P1;
1206 data_id = SOC24_FIRMWARE_ID_RS64_MES_P1_STACK;
1207 }
1208
1209 mes_hdr = (const struct mes_firmware_header_v1_0 *)
1210 adev->mes.fw[pipe]->data;
1211
1212 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1213 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
1214 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
1215
1216 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, ucode_id, fw_data, fw_size);
1217
1218 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
1219 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
1220 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
1221
1222 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, data_id, fw_data, fw_size);
1223 }
1224 }
1225
gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device * adev)1226 static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
1227 {
1228 uint32_t rlc_g_offset, rlc_g_size;
1229 uint64_t gpu_addr;
1230 uint32_t data;
1231 int i, num_xcc;
1232
1233 /* RLC autoload sequence 2: copy ucode */
1234 gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(adev);
1235 gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(adev);
1236 gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(adev);
1237 gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(adev);
1238
1239 rlc_g_offset = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].offset;
1240 rlc_g_size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].size;
1241 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset - adev->gmc.vram_start;
1242
1243 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1244 for (i = 0; i < num_xcc; i++) {
1245 WREG32_SOC15(GC, GET_INST(GC, i),
1246 regGFX_IMU_RLC_BOOTLOADER_ADDR_HI,
1247 upper_32_bits(gpu_addr));
1248 WREG32_SOC15(GC, GET_INST(GC, i),
1249 regGFX_IMU_RLC_BOOTLOADER_ADDR_LO,
1250 lower_32_bits(gpu_addr));
1251 WREG32_SOC15(GC, GET_INST(GC, i),
1252 regGFX_IMU_RLC_BOOTLOADER_SIZE,
1253 rlc_g_size);
1254 }
1255
1256 if (adev->gfx.imu.funcs) {
1257 /* RLC autoload sequence 3: load IMU fw */
1258 if (adev->gfx.imu.funcs->load_microcode)
1259 adev->gfx.imu.funcs->load_microcode(adev);
1260 }
1261
1262 /* unhalt rlc to start autoload */
1263 for (i = 0; i < num_xcc; i++) {
1264 data = RREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE);
1265 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD0_ENABLE, 1);
1266 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1);
1267 WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE, data);
1268 WREG32_SOC15(GC, GET_INST(GC, i), regRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
1269 }
1270
1271 return 0;
1272 }
1273
gfx_v12_1_alloc_ip_dump(struct amdgpu_device * adev)1274 static void gfx_v12_1_alloc_ip_dump(struct amdgpu_device *adev)
1275 {
1276 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1);
1277 uint32_t *ptr, inst, num_xcc;
1278
1279 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1280
1281 ptr = kcalloc(reg_count * num_xcc, sizeof(uint32_t), GFP_KERNEL);
1282 if (!ptr) {
1283 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1284 adev->gfx.ip_dump_core = NULL;
1285 } else {
1286 adev->gfx.ip_dump_core = ptr;
1287 }
1288
1289 /* Allocate memory for compute queue registers for all the instances */
1290 reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1);
1291 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1292 adev->gfx.mec.num_queue_per_pipe;
1293
1294 ptr = kcalloc(reg_count * inst * num_xcc, sizeof(uint32_t), GFP_KERNEL);
1295 if (!ptr) {
1296 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1297 adev->gfx.ip_dump_compute_queues = NULL;
1298 } else {
1299 adev->gfx.ip_dump_compute_queues = ptr;
1300 }
1301 }
1302
gfx_v12_1_ip_print(struct amdgpu_ip_block * ip_block,struct drm_printer * p)1303 static void gfx_v12_1_ip_print(struct amdgpu_ip_block *ip_block,
1304 struct drm_printer *p)
1305 {
1306 struct amdgpu_device *adev = ip_block->adev;
1307 uint32_t i, j, k;
1308 uint32_t xcc_id, xcc_offset, inst_offset;
1309 uint32_t num_xcc, reg, num_inst;
1310 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1);
1311
1312 if (!adev->gfx.ip_dump_core)
1313 return;
1314
1315 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1316 drm_printf(p, "Number of Instances:%d\n", num_xcc);
1317 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1318 xcc_offset = xcc_id * reg_count;
1319 drm_printf(p, "\nInstance id:%d\n", xcc_id);
1320 for (i = 0; i < reg_count; i++)
1321 drm_printf(p, "%-50s \t 0x%08x\n",
1322 gc_reg_list_12_1[i].reg_name,
1323 adev->gfx.ip_dump_core[xcc_offset + i]);
1324 }
1325
1326 /* print compute queue registers for all instances */
1327 if (!adev->gfx.ip_dump_compute_queues)
1328 return;
1329
1330 reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1);
1331 drm_printf(p, "\nnum_xcc: %d num_mec: %d num_pipe: %d num_queue: %d\n",
1332 num_xcc,
1333 adev->gfx.mec.num_mec,
1334 adev->gfx.mec.num_pipe_per_mec,
1335 adev->gfx.mec.num_queue_per_pipe);
1336
1337 num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1338 adev->gfx.mec.num_queue_per_pipe;
1339 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1340 xcc_offset = xcc_id * reg_count * num_inst;
1341 inst_offset = 0;
1342 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
1343 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
1344 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
1345 drm_printf(p,
1346 "\nxcc:%d mec:%d, pipe:%d, queue:%d\n",
1347 xcc_id, i, j, k);
1348 for (reg = 0; reg < reg_count; reg++) {
1349 drm_printf(p,
1350 "%-50s \t 0x%08x\n",
1351 gc_cp_reg_list_12_1[reg].reg_name,
1352 adev->gfx.ip_dump_compute_queues
1353 [xcc_offset + inst_offset +
1354 reg]);
1355 }
1356 inst_offset += reg_count;
1357 }
1358 }
1359 }
1360 }
1361 }
1362
gfx_v12_1_ip_dump(struct amdgpu_ip_block * ip_block)1363 static void gfx_v12_1_ip_dump(struct amdgpu_ip_block *ip_block)
1364 {
1365 struct amdgpu_device *adev = ip_block->adev;
1366 uint32_t i, j, k;
1367 uint32_t num_xcc, reg, num_inst;
1368 uint32_t xcc_id, xcc_offset, inst_offset;
1369 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1);
1370
1371 if (!adev->gfx.ip_dump_core)
1372 return;
1373
1374 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1375
1376 amdgpu_gfx_off_ctrl(adev, false);
1377 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1378 xcc_offset = xcc_id * reg_count;
1379 for (i = 0; i < reg_count; i++)
1380 adev->gfx.ip_dump_core[xcc_offset + i] =
1381 RREG32(SOC15_REG_ENTRY_OFFSET_INST(gc_reg_list_12_1[i],
1382 GET_INST(GC, xcc_id)));
1383 }
1384 amdgpu_gfx_off_ctrl(adev, true);
1385
1386 /* dump compute queue registers for all instances */
1387 if (!adev->gfx.ip_dump_compute_queues)
1388 return;
1389
1390 num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1391 adev->gfx.mec.num_queue_per_pipe;
1392 reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1);
1393 amdgpu_gfx_off_ctrl(adev, false);
1394 mutex_lock(&adev->srbm_mutex);
1395 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1396 xcc_offset = xcc_id * reg_count * num_inst;
1397 inst_offset = 0;
1398 for (i = 0; i < adev->gfx.mec.num_mec; i++) {
1399 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
1400 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
1401 /* ME0 is for GFX so start from 1 for CP */
1402 soc_v1_0_grbm_select(adev, 1 + i, j, k, 0,
1403 GET_INST(GC, xcc_id));
1404
1405 for (reg = 0; reg < reg_count; reg++) {
1406 adev->gfx.ip_dump_compute_queues
1407 [xcc_offset +
1408 inst_offset + reg] =
1409 RREG32(SOC15_REG_ENTRY_OFFSET_INST(
1410 gc_cp_reg_list_12_1[reg],
1411 GET_INST(GC, xcc_id)));
1412 }
1413 inst_offset += reg_count;
1414 }
1415 }
1416 }
1417 }
1418 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, 0);
1419 mutex_unlock(&adev->srbm_mutex);
1420 amdgpu_gfx_off_ctrl(adev, true);
1421 }
1422
gfx_v12_1_sw_init(struct amdgpu_ip_block * ip_block)1423 static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
1424 {
1425 uint16_t major_ver, minor_ver;
1426 int i, j, k, r, ring_id = 0;
1427 unsigned num_compute_rings;
1428 int xcc_id, num_xcc;
1429 struct amdgpu_device *adev = ip_block->adev;
1430
1431 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1432 case IP_VERSION(12, 1, 0):
1433 adev->gfx.mec.num_mec = 1;
1434 adev->gfx.mec.num_pipe_per_mec = 4;
1435 adev->gfx.mec.num_queue_per_pipe = 8;
1436
1437 if (!amdgpu_discovery_get_gc_major_minor_version(
1438 adev, &major_ver, &minor_ver)) {
1439 if (major_ver == 1 && minor_ver == 3) {
1440 adev->gfx.config.max_cu_per_sh /= 2;
1441 dev_dbg(adev->dev, "Halving max_cu_per_sh for GC Discovery table v1:3 %d\n",
1442 adev->gfx.config.max_cu_per_sh);
1443 }
1444 }
1445 break;
1446 default:
1447 adev->gfx.mec.num_mec = 2;
1448 adev->gfx.mec.num_pipe_per_mec = 2;
1449 adev->gfx.mec.num_queue_per_pipe = 4;
1450 break;
1451 }
1452
1453 if (adev->gfx.num_compute_rings) {
1454 /* recalculate compute rings to use based on hardware configuration */
1455 num_compute_rings = (adev->gfx.mec.num_pipe_per_mec *
1456 adev->gfx.mec.num_queue_per_pipe) / 2;
1457 adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings,
1458 num_compute_rings);
1459 }
1460
1461 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1462
1463 /* EOP Event */
1464 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP,
1465 GFX_12_1_0__SRCID__CP_EOP_INTERRUPT,
1466 &adev->gfx.eop_irq);
1467 if (r)
1468 return r;
1469
1470 /* Privileged reg */
1471 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP,
1472 GFX_12_1_0__SRCID__CP_PRIV_REG_FAULT,
1473 &adev->gfx.priv_reg_irq);
1474 if (r)
1475 return r;
1476
1477 /* Privileged inst */
1478 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP,
1479 GFX_12_1_0__SRCID__CP_PRIV_INSTR_FAULT,
1480 &adev->gfx.priv_inst_irq);
1481 if (r)
1482 return r;
1483
1484 /* RLC POISON Error */
1485 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_RLC,
1486 GFX_12_1_0__SRCID__RLC_POISON_INTERRUPT,
1487 &adev->gfx.rlc_poison_irq);
1488 if (r)
1489 return r;
1490
1491 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1492
1493 r = gfx_v12_1_rlc_init(adev);
1494 if (r) {
1495 dev_err(adev->dev, "Failed to init rlc BOs!\n");
1496 return r;
1497 }
1498
1499 r = gfx_v12_1_mec_init(adev);
1500 if (r) {
1501 dev_err(adev->dev, "Failed to init MEC BOs!\n");
1502 return r;
1503 }
1504
1505 /* set up the compute queues - allocate horizontally across pipes */
1506 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1507 ring_id = 0;
1508 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1509 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1510 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1511 if (!amdgpu_gfx_is_mec_queue_enabled(adev,
1512 xcc_id, i, k, j))
1513 continue;
1514
1515 r = gfx_v12_1_compute_ring_init(adev, ring_id,
1516 xcc_id, i, k, j);
1517 if (r)
1518 return r;
1519
1520 ring_id++;
1521 }
1522 }
1523 }
1524
1525 if (!adev->enable_mes_kiq) {
1526 r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, xcc_id);
1527 if (r) {
1528 dev_err(adev->dev, "Failed to init KIQ BOs!\n");
1529 return r;
1530 }
1531
1532 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1533 if (r)
1534 return r;
1535 }
1536
1537 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_1_compute_mqd), xcc_id);
1538 if (r)
1539 return r;
1540 }
1541
1542 /* allocate visible FB for rlc auto-loading fw */
1543 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1544 r = gfx_v12_1_rlc_autoload_buffer_init(adev);
1545 if (r)
1546 return r;
1547 } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1548 r = gfx_v12_1_init_cp_compute_microcode_bo(adev);
1549 if (r)
1550 return r;
1551 }
1552
1553 r = gfx_v12_1_gpu_early_init(adev);
1554 if (r)
1555 return r;
1556
1557 r = amdgpu_gfx_sysfs_init(adev);
1558 if (r)
1559 return r;
1560
1561 gfx_v12_1_alloc_ip_dump(adev);
1562
1563 mutex_init(&adev->gfx.mec.reset_mutex);
1564
1565 return 0;
1566 }
1567
gfx_v12_1_rlc_autoload_buffer_fini(struct amdgpu_device * adev)1568 static void gfx_v12_1_rlc_autoload_buffer_fini(struct amdgpu_device *adev)
1569 {
1570 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
1571 &adev->gfx.rlc.rlc_autoload_gpu_addr,
1572 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
1573 }
1574
gfx_v12_1_sw_fini(struct amdgpu_ip_block * ip_block)1575 static int gfx_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
1576 {
1577 int i, num_xcc;
1578 struct amdgpu_device *adev = ip_block->adev;
1579
1580 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1581 for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)
1582 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1583
1584 for (i = 0; i < num_xcc; i++) {
1585 amdgpu_gfx_mqd_sw_fini(adev, i);
1586
1587 if (!adev->enable_mes_kiq) {
1588 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[i].ring);
1589 amdgpu_gfx_kiq_fini(adev, i);
1590 }
1591 }
1592
1593 gfx_v12_1_rlc_fini(adev);
1594 gfx_v12_1_mec_fini(adev);
1595
1596 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1597 gfx_v12_1_rlc_autoload_buffer_fini(adev);
1598
1599 gfx_v12_1_free_microcode(adev);
1600 amdgpu_gfx_sysfs_fini(adev);
1601
1602 kfree(adev->gfx.ip_dump_core);
1603 kfree(adev->gfx.ip_dump_compute_queues);
1604
1605 return 0;
1606 }
1607
gfx_v12_1_xcc_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)1608 static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1609 u32 sh_num, u32 instance, int xcc_id)
1610 {
1611 u32 data;
1612
1613 if (instance == 0xffffffff)
1614 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1615 INSTANCE_BROADCAST_WRITES, 1);
1616 else
1617 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1618 instance);
1619
1620 if (se_num == 0xffffffff)
1621 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1622 1);
1623 else
1624 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1625
1626 if (sh_num == 0xffffffff)
1627 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1628 1);
1629 else
1630 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1631
1632 WREG32_SOC15_RLC_SHADOW_EX(reg, GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX, data);
1633 }
1634
gfx_v12_1_get_sa_active_bitmap(struct amdgpu_device * adev,int xcc_id)1635 static u32 gfx_v12_1_get_sa_active_bitmap(struct amdgpu_device *adev,
1636 int xcc_id)
1637 {
1638 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask;
1639
1640 gc_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SA_UNIT_DISABLE);
1641 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask,
1642 CC_GC_SA_UNIT_DISABLE,
1643 SA_DISABLE);
1644 gc_user_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SA_UNIT_DISABLE);
1645 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask,
1646 GC_USER_SA_UNIT_DISABLE,
1647 SA_DISABLE);
1648 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se *
1649 adev->gfx.config.max_shader_engines);
1650
1651 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
1652 }
1653
gfx_v12_1_get_rb_active_bitmap(struct amdgpu_device * adev,int xcc_id)1654 static u32 gfx_v12_1_get_rb_active_bitmap(struct amdgpu_device *adev,
1655 int xcc_id)
1656 {
1657 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
1658 u32 rb_mask;
1659
1660 gc_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
1661 regCC_RB_BACKEND_DISABLE);
1662 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
1663 CC_RB_BACKEND_DISABLE,
1664 BACKEND_DISABLE);
1665 gc_user_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
1666 regGC_USER_RB_BACKEND_DISABLE);
1667 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
1668 GC_USER_RB_BACKEND_DISABLE,
1669 BACKEND_DISABLE);
1670 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
1671 adev->gfx.config.max_shader_engines);
1672
1673 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
1674 }
1675
gfx_v12_1_setup_rb(struct amdgpu_device * adev)1676 static void gfx_v12_1_setup_rb(struct amdgpu_device *adev)
1677 {
1678 u32 rb_bitmap_width_per_sa;
1679 u32 max_sa;
1680 u32 active_sa_bitmap;
1681 u32 global_active_rb_bitmap;
1682 u32 active_rb_bitmap = 0;
1683 u32 i;
1684 int xcc_id;
1685
1686 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
1687 /* query sa bitmap from SA_UNIT_DISABLE registers */
1688 active_sa_bitmap = gfx_v12_1_get_sa_active_bitmap(adev, xcc_id);
1689 /* query rb bitmap from RB_BACKEND_DISABLE registers */
1690 global_active_rb_bitmap = gfx_v12_1_get_rb_active_bitmap(adev, xcc_id);
1691
1692 /* generate active rb bitmap according to active sa bitmap */
1693 max_sa = adev->gfx.config.max_shader_engines *
1694 adev->gfx.config.max_sh_per_se;
1695 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
1696 adev->gfx.config.max_sh_per_se;
1697 for (i = 0; i < max_sa; i++) {
1698 if (active_sa_bitmap & (1 << i))
1699 active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
1700 }
1701
1702 active_rb_bitmap |= global_active_rb_bitmap;
1703 }
1704
1705 adev->gfx.config.backend_enable_mask = active_rb_bitmap;
1706 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
1707 }
1708
gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device * adev,int xcc_id)1709 static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev,
1710 int xcc_id)
1711 {
1712 int i;
1713 uint32_t sh_mem_bases;
1714 uint32_t data;
1715
1716 /*
1717 * Configure apertures:
1718 * LDS: 0x20000000'00000000 - 0x20000001'00000000 (4GB)
1719 * Scratch: 0x10000000'00000000 - 0x11ffffff'ffffffff (128PB 57-bit)
1720 */
1721 sh_mem_bases = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1722 (adev->gmc.private_aperture_start >> 58));
1723 sh_mem_bases = REG_SET_FIELD(sh_mem_bases, SH_MEM_BASES, SHARED_BASE,
1724 (adev->gmc.shared_aperture_start >> 48));
1725
1726 mutex_lock(&adev->srbm_mutex);
1727 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1728 soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
1729 /* CP and shaders */
1730 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1731 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, sh_mem_bases);
1732
1733 /* Enable trap for each kfd vmid. */
1734 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL);
1735 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1736 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL, data);
1737
1738 /* Disable VGPR deallocation instruction for each KFD vmid. */
1739 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_DEBUG);
1740 data = REG_SET_FIELD(data, SQ_DEBUG, DISABLE_VGPR_DEALLOC, 1);
1741 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_DEBUG, data);
1742 }
1743 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1744 mutex_unlock(&adev->srbm_mutex);
1745 }
1746
gfx_v12_1_tcp_harvest(struct amdgpu_device * adev)1747 static void gfx_v12_1_tcp_harvest(struct amdgpu_device *adev)
1748 {
1749 /* TODO: harvest feature to be added later. */
1750 }
1751
gfx_v12_1_get_tcc_info(struct amdgpu_device * adev)1752 static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev)
1753 {
1754 }
1755
gfx_v12_1_xcc_xnack_set_chicken_bits(struct amdgpu_device * adev,int xcc_id)1756 static void gfx_v12_1_xcc_xnack_set_chicken_bits(struct amdgpu_device *adev, int xcc_id)
1757 {
1758 /* NOTE: COMPRESSION_ENABLE is used a chicken bit to enable/disable xcc xnack */
1759 mutex_lock(&adev->srbm_mutex);
1760 if (!adev->gmc.noretry) {
1761 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
1762 TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x1);
1763 } else
1764 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
1765 TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x0);
1766 mutex_unlock(&adev->srbm_mutex);
1767 }
1768
gfx_v12_1_xcc_constants_init(struct amdgpu_device * adev,int xcc_id)1769 static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
1770 int xcc_id)
1771 {
1772 u32 tmp;
1773 int i;
1774
1775 /* XXX SH_MEM regs */
1776 /* where to put LDS, scratch, GPUVM in FSA64 space */
1777 mutex_lock(&adev->srbm_mutex);
1778 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1779 soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
1780 /* CP and shaders */
1781 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1782 regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1783 if (i != 0) {
1784 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1785 (adev->gmc.private_aperture_start >> 58));
1786 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1787 (adev->gmc.shared_aperture_start >> 48));
1788 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, tmp);
1789 }
1790 }
1791 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
1792
1793 mutex_unlock(&adev->srbm_mutex);
1794
1795 gfx_v12_1_xcc_init_compute_vmid(adev, xcc_id);
1796 gfx_v12_1_xcc_xnack_set_chicken_bits(adev, xcc_id);
1797
1798 }
1799
gfx_v12_1_constants_init(struct amdgpu_device * adev)1800 static void gfx_v12_1_constants_init(struct amdgpu_device *adev)
1801 {
1802 int i, num_xcc;
1803
1804 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1805
1806 gfx_v12_1_setup_rb(adev);
1807 gfx_v12_1_get_cu_info(adev, &adev->gfx.cu_info);
1808 gfx_v12_1_get_tcc_info(adev);
1809 adev->gfx.config.pa_sc_tile_steering_override = 0;
1810
1811 for (i = 0; i < num_xcc; i++)
1812 gfx_v12_1_xcc_constants_init(adev, i);
1813 }
1814
gfx_v12_1_xcc_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable,int xcc_id)1815 static void gfx_v12_1_xcc_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1816 bool enable, int xcc_id)
1817 {
1818 u32 tmp;
1819
1820 if (amdgpu_sriov_vf(adev))
1821 return;
1822
1823 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0);
1824
1825 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1826 enable ? 1 : 0);
1827 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1828 enable ? 1 : 0);
1829 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1830 enable ? 1 : 0);
1831 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1832 enable ? 1 : 0);
1833
1834 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0, tmp);
1835 }
1836
gfx_v12_1_xcc_init_csb(struct amdgpu_device * adev,int xcc_id)1837 static int gfx_v12_1_xcc_init_csb(struct amdgpu_device *adev,
1838 int xcc_id)
1839 {
1840 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
1841
1842 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_HI,
1843 adev->gfx.rlc.clear_state_gpu_addr >> 32);
1844 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_LO,
1845 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1846 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1847 regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1848
1849 return 0;
1850 }
1851
gfx_v12_1_xcc_rlc_stop(struct amdgpu_device * adev,int xcc_id)1852 static void gfx_v12_1_xcc_rlc_stop(struct amdgpu_device *adev,
1853 int xcc_id)
1854 {
1855 u32 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL);
1856
1857 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1858 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL, tmp);
1859 }
1860
gfx_v12_1_rlc_stop(struct amdgpu_device * adev)1861 static void gfx_v12_1_rlc_stop(struct amdgpu_device *adev)
1862 {
1863 int i, num_xcc;
1864
1865 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1866 for (i = 0; i < num_xcc; i++)
1867 gfx_v12_1_xcc_rlc_stop(adev, i);
1868 }
1869
gfx_v12_1_xcc_rlc_reset(struct amdgpu_device * adev,int xcc_id)1870 static void gfx_v12_1_xcc_rlc_reset(struct amdgpu_device *adev,
1871 int xcc_id)
1872 {
1873 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
1874 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1875 udelay(50);
1876 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
1877 GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1878 udelay(50);
1879 }
1880
gfx_v12_1_rlc_reset(struct amdgpu_device * adev)1881 static void gfx_v12_1_rlc_reset(struct amdgpu_device *adev)
1882 {
1883 int i, num_xcc;
1884
1885 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1886 for (i = 0; i < num_xcc; i++)
1887 gfx_v12_1_xcc_rlc_reset(adev, i);
1888 }
1889
gfx_v12_1_xcc_rlc_smu_handshake_cntl(struct amdgpu_device * adev,bool enable,int xcc_id)1890 static void gfx_v12_1_xcc_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1891 bool enable, int xcc_id)
1892 {
1893 uint32_t rlc_pg_cntl;
1894
1895 rlc_pg_cntl = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL);
1896
1897 if (!enable) {
1898 /* RLC_PG_CNTL[23] = 0 (default)
1899 * RLC will wait for handshake acks with SMU
1900 * GFXOFF will be enabled
1901 * RLC_PG_CNTL[23] = 1
1902 * RLC will not issue any message to SMU
1903 * hence no handshake between SMU & RLC
1904 * GFXOFF will be disabled
1905 */
1906 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1907 } else
1908 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK;
1909 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, rlc_pg_cntl);
1910 }
1911
gfx_v12_1_xcc_rlc_start(struct amdgpu_device * adev,int xcc_id)1912 static void gfx_v12_1_xcc_rlc_start(struct amdgpu_device *adev,
1913 int xcc_id)
1914 {
1915 /* TODO: enable rlc & smu handshake until smu
1916 * and gfxoff feature works as expected */
1917 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
1918 gfx_v12_1_xcc_rlc_smu_handshake_cntl(adev, false, xcc_id);
1919
1920 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL, RLC_ENABLE_F32, 1);
1921 udelay(50);
1922 }
1923
gfx_v12_1_rlc_start(struct amdgpu_device * adev)1924 static void gfx_v12_1_rlc_start(struct amdgpu_device *adev)
1925 {
1926 int i, num_xcc;
1927
1928 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1929 for (i = 0; i < num_xcc; i++) {
1930 gfx_v12_1_xcc_rlc_start(adev, i);
1931 }
1932 }
1933
gfx_v12_1_xcc_rlc_enable_srm(struct amdgpu_device * adev,int xcc_id)1934 static void gfx_v12_1_xcc_rlc_enable_srm(struct amdgpu_device *adev,
1935 int xcc_id)
1936 {
1937 uint32_t tmp;
1938
1939 /* enable Save Restore Machine */
1940 tmp = RREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL));
1941 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1942 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
1943 WREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL), tmp);
1944 }
1945
gfx_v12_1_xcc_load_rlcg_microcode(struct amdgpu_device * adev,int xcc_id)1946 static void gfx_v12_1_xcc_load_rlcg_microcode(struct amdgpu_device *adev,
1947 int xcc_id)
1948 {
1949 const struct rlc_firmware_header_v2_0 *hdr;
1950 const __le32 *fw_data;
1951 unsigned i, fw_size;
1952
1953 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1954 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1955 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1956 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1957
1958 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR,
1959 RLCG_UCODE_LOADING_START_ADDRESS);
1960
1961 for (i = 0; i < fw_size; i++)
1962 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1963 regRLC_GPM_UCODE_DATA,
1964 le32_to_cpup(fw_data++));
1965
1966 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1967 regRLC_GPM_UCODE_ADDR,
1968 adev->gfx.rlc_fw_version);
1969 }
1970
gfx_v12_1_xcc_load_rlc_iram_dram_microcode(struct amdgpu_device * adev,int xcc_id)1971 static void gfx_v12_1_xcc_load_rlc_iram_dram_microcode(struct amdgpu_device *adev,
1972 int xcc_id)
1973 {
1974 const struct rlc_firmware_header_v2_2 *hdr;
1975 const __le32 *fw_data;
1976 unsigned i, fw_size;
1977 u32 tmp;
1978
1979 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
1980
1981 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1982 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes));
1983 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4;
1984
1985 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_IRAM_ADDR, 0);
1986
1987 for (i = 0; i < fw_size; i++) {
1988 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
1989 msleep(1);
1990 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1991 regRLC_LX6_IRAM_DATA,
1992 le32_to_cpup(fw_data++));
1993 }
1994
1995 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
1996 regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
1997
1998 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1999 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes));
2000 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4;
2001
2002 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2003 regRLC_LX6_DRAM_ADDR, 0);
2004 for (i = 0; i < fw_size; i++) {
2005 if ((amdgpu_emu_mode == 1) && (i % 100 == 99))
2006 msleep(1);
2007 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2008 regRLC_LX6_DRAM_DATA,
2009 le32_to_cpup(fw_data++));
2010 }
2011
2012 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2013 regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version);
2014
2015 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL);
2016 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1);
2017 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0);
2018 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL, tmp);
2019 }
2020
gfx_v12_1_xcc_rlc_load_microcode(struct amdgpu_device * adev,int xcc_id)2021 static int gfx_v12_1_xcc_rlc_load_microcode(struct amdgpu_device *adev,
2022 int xcc_id)
2023 {
2024 const struct rlc_firmware_header_v2_0 *hdr;
2025 uint16_t version_major;
2026 uint16_t version_minor;
2027
2028 if (!adev->gfx.rlc_fw)
2029 return -EINVAL;
2030
2031 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2032 amdgpu_ucode_print_rlc_hdr(&hdr->header);
2033
2034 version_major = le16_to_cpu(hdr->header.header_version_major);
2035 version_minor = le16_to_cpu(hdr->header.header_version_minor);
2036
2037 if (version_major == 2) {
2038 gfx_v12_1_xcc_load_rlcg_microcode(adev, xcc_id);
2039 if (amdgpu_dpm == 1) {
2040 if (version_minor >= 2)
2041 gfx_v12_1_xcc_load_rlc_iram_dram_microcode(adev, xcc_id);
2042 }
2043
2044 return 0;
2045 }
2046
2047 return -EINVAL;
2048 }
2049
gfx_v12_1_xcc_rlc_resume(struct amdgpu_device * adev,int xcc_id)2050 static int gfx_v12_1_xcc_rlc_resume(struct amdgpu_device *adev,
2051 int xcc_id)
2052 {
2053 int r;
2054
2055 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
2056 gfx_v12_1_xcc_init_csb(adev, xcc_id);
2057
2058 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
2059 gfx_v12_1_xcc_rlc_enable_srm(adev, xcc_id);
2060 } else {
2061 if (amdgpu_sriov_vf(adev)) {
2062 gfx_v12_1_xcc_init_csb(adev, xcc_id);
2063 return 0;
2064 }
2065
2066 gfx_v12_1_xcc_rlc_stop(adev, xcc_id);
2067
2068 /* disable CG */
2069 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, 0);
2070
2071 /* disable PG */
2072 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, 0);
2073
2074 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2075 /* legacy rlc firmware loading */
2076 r = gfx_v12_1_xcc_rlc_load_microcode(adev, xcc_id);
2077 if (r)
2078 return r;
2079 }
2080
2081 gfx_v12_1_xcc_init_csb(adev, xcc_id);
2082
2083 gfx_v12_1_xcc_rlc_start(adev, xcc_id);
2084 }
2085
2086 return 0;
2087 }
2088
gfx_v12_1_rlc_resume(struct amdgpu_device * adev)2089 static int gfx_v12_1_rlc_resume(struct amdgpu_device *adev)
2090 {
2091 int r, i, num_xcc;
2092
2093 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2094 for (i = 0; i < num_xcc; i++) {
2095 r = gfx_v12_1_xcc_rlc_resume(adev, i);
2096 if (r)
2097 return r;
2098 }
2099
2100 return 0;
2101 }
2102
gfx_v12_1_xcc_config_gfx_rs64(struct amdgpu_device * adev,int xcc_id)2103 static void gfx_v12_1_xcc_config_gfx_rs64(struct amdgpu_device *adev,
2104 int xcc_id)
2105 {
2106 const struct gfx_firmware_header_v2_0 *mec_hdr;
2107 uint32_t pipe_id, tmp;
2108
2109 mec_hdr = (const struct gfx_firmware_header_v2_0 *)
2110 adev->gfx.mec_fw->data;
2111
2112 /* config mec program start addr */
2113 for (pipe_id = 0; pipe_id < 4; pipe_id++) {
2114 soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id));
2115 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START,
2116 mec_hdr->ucode_start_addr_lo >> 2 |
2117 mec_hdr->ucode_start_addr_hi << 30);
2118 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2119 mec_hdr->ucode_start_addr_hi >> 2);
2120 }
2121 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2122
2123 /* reset mec pipe */
2124 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL);
2125 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
2126 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
2127 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
2128 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
2129 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp);
2130
2131 /* clear mec pipe reset */
2132 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
2133 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
2134 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
2135 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
2136 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp);
2137 }
2138
gfx_v12_1_config_gfx_rs64(struct amdgpu_device * adev)2139 static void gfx_v12_1_config_gfx_rs64(struct amdgpu_device *adev)
2140 {
2141 int i, num_xcc;
2142
2143 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2144
2145 for (i = 0; i < num_xcc; i++)
2146 gfx_v12_1_xcc_config_gfx_rs64(adev, i);
2147 }
2148
gfx_v12_1_xcc_set_mec_ucode_start_addr(struct amdgpu_device * adev,int xcc_id)2149 static void gfx_v12_1_xcc_set_mec_ucode_start_addr(struct amdgpu_device *adev,
2150 int xcc_id)
2151 {
2152 const struct gfx_firmware_header_v2_0 *cp_hdr;
2153 unsigned pipe_id;
2154
2155 cp_hdr = (const struct gfx_firmware_header_v2_0 *)
2156 adev->gfx.mec_fw->data;
2157 mutex_lock(&adev->srbm_mutex);
2158 for (pipe_id = 0; pipe_id < adev->gfx.mec.num_pipe_per_mec; pipe_id++) {
2159 soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id));
2160 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START,
2161 cp_hdr->ucode_start_addr_lo >> 2 |
2162 cp_hdr->ucode_start_addr_hi << 30);
2163 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI,
2164 cp_hdr->ucode_start_addr_hi >> 2);
2165 }
2166 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2167 mutex_unlock(&adev->srbm_mutex);
2168 }
2169
gfx_v12_1_xcc_wait_for_rlc_autoload_complete(struct amdgpu_device * adev,int xcc_id)2170 static int gfx_v12_1_xcc_wait_for_rlc_autoload_complete(struct amdgpu_device *adev,
2171 int xcc_id)
2172 {
2173 uint32_t cp_status;
2174 uint32_t bootload_status;
2175 int i;
2176
2177 for (i = 0; i < adev->usec_timeout; i++) {
2178 cp_status = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_STAT);
2179 bootload_status = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2180 regRLC_RLCS_BOOTLOAD_STATUS);
2181
2182 if ((cp_status == 0) &&
2183 (REG_GET_FIELD(bootload_status,
2184 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2185 break;
2186 }
2187 udelay(1);
2188 if (amdgpu_emu_mode)
2189 msleep(10);
2190 }
2191
2192 if (i >= adev->usec_timeout) {
2193 dev_err(adev->dev,
2194 "rlc autoload: xcc%d gc ucode autoload timeout\n", xcc_id);
2195 return -ETIMEDOUT;
2196 }
2197
2198 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2199 gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id);
2200 }
2201
2202 return 0;
2203 }
2204
gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device * adev)2205 static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2206 {
2207 int xcc_id;
2208
2209 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++)
2210 gfx_v12_1_xcc_wait_for_rlc_autoload_complete(adev, xcc_id);
2211
2212 return 0;
2213 }
2214
gfx_v12_1_xcc_cp_compute_enable(struct amdgpu_device * adev,bool enable,int xcc_id)2215 static void gfx_v12_1_xcc_cp_compute_enable(struct amdgpu_device *adev,
2216 bool enable, int xcc_id)
2217 {
2218 u32 data;
2219
2220 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL);
2221 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE,
2222 enable ? 0 : 1);
2223 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET,
2224 enable ? 0 : 1);
2225 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET,
2226 enable ? 0 : 1);
2227 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET,
2228 enable ? 0 : 1);
2229 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET,
2230 enable ? 0 : 1);
2231 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE,
2232 enable ? 1 : 0);
2233 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE,
2234 enable ? 1 : 0);
2235 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE,
2236 enable ? 1 : 0);
2237 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE,
2238 enable ? 1 : 0);
2239 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT,
2240 enable ? 0 : 1);
2241 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, data);
2242
2243 adev->gfx.kiq[xcc_id].ring.sched.ready = enable;
2244
2245 udelay(50);
2246 }
2247
gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device * adev)2248 static int gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device *adev)
2249 {
2250 const struct gfx_firmware_header_v2_0 *mec_hdr;
2251 const __le32 *fw_ucode, *fw_data;
2252 u32 fw_ucode_size, fw_data_size;
2253 u32 *fw_ucode_ptr, *fw_data_ptr;
2254 int i, r, xcc_id;
2255
2256 if (!adev->gfx.mec_fw)
2257 return -EINVAL;
2258
2259 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
2260 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2261
2262 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data +
2263 le32_to_cpu(mec_hdr->ucode_offset_bytes));
2264 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes);
2265
2266 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
2267 le32_to_cpu(mec_hdr->data_offset_bytes));
2268 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
2269
2270 if (adev->gfx.mec.mec_fw_obj == NULL) {
2271 r = amdgpu_bo_create_reserved(adev, fw_ucode_size,
2272 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2273 &adev->gfx.mec.mec_fw_obj,
2274 &adev->gfx.mec.mec_fw_gpu_addr,
2275 (void **)&fw_ucode_ptr);
2276 if (r) {
2277 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r);
2278 gfx_v12_1_mec_fini(adev);
2279 return r;
2280 }
2281
2282 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size);
2283
2284 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
2285 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
2286 }
2287
2288 if (adev->gfx.mec.mec_fw_data_obj == NULL) {
2289 r = amdgpu_bo_create_reserved(adev,
2290 ALIGN(fw_data_size, 64 * 1024) *
2291 adev->gfx.mec.num_pipe_per_mec * NUM_XCC(adev->gfx.xcc_mask),
2292 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
2293 &adev->gfx.mec.mec_fw_data_obj,
2294 &adev->gfx.mec.mec_fw_data_gpu_addr,
2295 (void **)&fw_data_ptr);
2296 if (r) {
2297 dev_err(adev->dev, "(%d) failed to create mec fw data bo\n", r);
2298 gfx_v12_1_mec_fini(adev);
2299 return r;
2300 }
2301
2302 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
2303 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2304 u32 offset = (xcc_id * adev->gfx.mec.num_pipe_per_mec + i) *
2305 ALIGN(fw_data_size, 64 * 1024) / 4;
2306 memcpy(fw_data_ptr + offset, fw_data, fw_data_size);
2307 }
2308 }
2309
2310 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj);
2311 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj);
2312 }
2313
2314 return 0;
2315 }
2316
gfx_v12_1_xcc_cp_compute_load_microcode_rs64(struct amdgpu_device * adev,int xcc_id)2317 static int gfx_v12_1_xcc_cp_compute_load_microcode_rs64(struct amdgpu_device *adev,
2318 int xcc_id)
2319 {
2320 const struct gfx_firmware_header_v2_0 *mec_hdr;
2321 u32 fw_data_size;
2322 u32 tmp, i, usec_timeout = 50000; /* Wait for 50 ms */
2323
2324 if (!adev->gfx.mec_fw)
2325 return -EINVAL;
2326
2327 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
2328 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes);
2329
2330 gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id);
2331
2332 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL);
2333 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
2334 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2335 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2336 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL, tmp);
2337
2338 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL);
2339 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0);
2340 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0);
2341 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL, tmp);
2342
2343 mutex_lock(&adev->srbm_mutex);
2344 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
2345 soc_v1_0_grbm_select(adev, 1, i, 0, 0, GET_INST(GC, xcc_id));
2346
2347 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_LO,
2348 lower_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2349 (xcc_id * adev->gfx.mec.num_pipe_per_mec + i) *
2350 ALIGN(fw_data_size, 64 * 1024)));
2351 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_HI,
2352 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr +
2353 (xcc_id * adev->gfx.mec.num_pipe_per_mec + i) *
2354 ALIGN(fw_data_size, 64 * 1024)));
2355
2356 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_LO,
2357 lower_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2358 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_HI,
2359 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2360 }
2361 mutex_unlock(&adev->srbm_mutex);
2362 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2363
2364 /* Trigger an invalidation of the L1 instruction caches */
2365 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL);
2366 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1);
2367 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL, tmp);
2368
2369 /* Wait for invalidation complete */
2370 for (i = 0; i < usec_timeout; i++) {
2371 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL);
2372 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL,
2373 INVALIDATE_DCACHE_COMPLETE))
2374 break;
2375 udelay(1);
2376 }
2377
2378 if (i >= usec_timeout) {
2379 dev_err(adev->dev, "failed to invalidate data cache\n");
2380 return -EINVAL;
2381 }
2382
2383 /* Trigger an invalidation of the L1 instruction caches */
2384 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL);
2385 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2386 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL, tmp);
2387
2388 /* Wait for invalidation complete */
2389 for (i = 0; i < usec_timeout; i++) {
2390 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL);
2391 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2392 INVALIDATE_CACHE_COMPLETE))
2393 break;
2394 udelay(1);
2395 }
2396
2397 if (i >= usec_timeout) {
2398 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2399 return -EINVAL;
2400 }
2401
2402 gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id);
2403
2404 return 0;
2405 }
2406
gfx_v12_1_xcc_kiq_setting(struct amdgpu_ring * ring,int xcc_id)2407 static void gfx_v12_1_xcc_kiq_setting(struct amdgpu_ring *ring,
2408 int xcc_id)
2409 {
2410 uint32_t tmp;
2411 struct amdgpu_device *adev = ring->adev;
2412
2413 /* tell RLC which is KIQ queue */
2414 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
2415 tmp &= 0xffffff00;
2416 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
2417 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
2418 tmp |= 0x80;
2419 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
2420 }
2421
gfx_v12_1_xcc_cp_set_doorbell_range(struct amdgpu_device * adev,int xcc_id)2422 static void gfx_v12_1_xcc_cp_set_doorbell_range(struct amdgpu_device *adev,
2423 int xcc_id)
2424 {
2425 /* disable gfx engine doorbell range */
2426 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_LOWER, 0);
2427 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_UPPER, 0);
2428
2429 /* set compute engine doorbell range */
2430 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_LOWER,
2431 ((adev->doorbell_index.kiq +
2432 xcc_id * adev->doorbell_index.xcc_doorbell_range) *
2433 2) << 2);
2434 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_UPPER,
2435 ((adev->doorbell_index.userqueue_end +
2436 xcc_id * adev->doorbell_index.xcc_doorbell_range) *
2437 2) << 2);
2438 }
2439
gfx_v12_1_compute_mqd_init(struct amdgpu_device * adev,void * m,struct amdgpu_mqd_prop * prop)2440 static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m,
2441 struct amdgpu_mqd_prop *prop)
2442 {
2443 struct v12_1_compute_mqd *mqd = m;
2444 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
2445 uint32_t tmp;
2446
2447 mqd->header = 0xC0310800;
2448 mqd->compute_pipelinestat_enable = 0x00000001;
2449 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
2450 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
2451 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
2452 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
2453 mqd->compute_misc_reserved = 0x00000007;
2454
2455 eop_base_addr = prop->eop_gpu_addr >> 8;
2456 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
2457 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
2458
2459 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2460 tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
2461 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
2462 (order_base_2(GFX12_MEC_HPD_SIZE / 4) - 1));
2463
2464 mqd->cp_hqd_eop_control = tmp;
2465
2466 /* enable doorbell? */
2467 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
2468
2469 if (prop->use_doorbell) {
2470 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2471 DOORBELL_OFFSET, prop->doorbell_index);
2472 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2473 DOORBELL_EN, 1);
2474 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2475 DOORBELL_SOURCE, 0);
2476 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2477 DOORBELL_HIT, 0);
2478 } else {
2479 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2480 DOORBELL_EN, 0);
2481 }
2482
2483 mqd->cp_hqd_pq_doorbell_control = tmp;
2484
2485 /* disable the queue if it's active */
2486 mqd->cp_hqd_dequeue_request = 0;
2487 mqd->cp_hqd_pq_rptr = 0;
2488 mqd->cp_hqd_pq_wptr_lo = 0;
2489 mqd->cp_hqd_pq_wptr_hi = 0;
2490
2491 /* set the pointer to the MQD */
2492 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc;
2493 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr);
2494
2495 /* set MQD vmid to 0 */
2496 tmp = regCP_MQD_CONTROL_DEFAULT;
2497 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
2498 mqd->cp_mqd_control = tmp;
2499
2500 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2501 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8;
2502 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
2503 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
2504
2505 /* set up the HQD, this is similar to CP_RB0_CNTL */
2506 tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
2507 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
2508 (order_base_2(prop->queue_size / 4) - 1));
2509 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
2510 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
2511 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
2512 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
2513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
2514 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
2515 mqd->cp_hqd_pq_control = tmp;
2516
2517 /* set the wb address whether it's enabled or not */
2518 wb_gpu_addr = prop->rptr_gpu_addr;
2519 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
2520 mqd->cp_hqd_pq_rptr_report_addr_hi =
2521 upper_32_bits(wb_gpu_addr) & 0xffff;
2522
2523 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2524 wb_gpu_addr = prop->wptr_gpu_addr;
2525 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2526 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2527
2528 tmp = 0;
2529 /* enable the doorbell if requested */
2530 if (prop->use_doorbell) {
2531 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT;
2532 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2533 DOORBELL_OFFSET, prop->doorbell_index);
2534
2535 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2536 DOORBELL_EN, 1);
2537 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2538 DOORBELL_SOURCE, 0);
2539 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
2540 DOORBELL_HIT, 0);
2541 }
2542
2543 mqd->cp_hqd_pq_doorbell_control = tmp;
2544
2545 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2546 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT;
2547
2548 /* set the vmid for the queue */
2549 mqd->cp_hqd_vmid = 0;
2550
2551 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
2552 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x63);
2553 mqd->cp_hqd_persistent_state = tmp;
2554
2555 /* set MIN_IB_AVAIL_SIZE */
2556 tmp = regCP_HQD_IB_CONTROL_DEFAULT;
2557 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 1);
2558 mqd->cp_hqd_ib_control = tmp;
2559
2560 /* set static priority for a compute queue/ring */
2561 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
2562 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
2563
2564 mqd->cp_mqd_stride_size = prop->mqd_stride_size ? prop->mqd_stride_size :
2565 AMDGPU_MQD_SIZE_ALIGN(adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size);
2566
2567 tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
2568 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
2569 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
2570 mqd->cp_hqd_quantum = tmp;
2571
2572 mqd->cp_hqd_active = prop->hqd_active;
2573
2574 return 0;
2575 }
2576
gfx_v12_1_xcc_kiq_init_register(struct amdgpu_ring * ring,int xcc_id)2577 static int gfx_v12_1_xcc_kiq_init_register(struct amdgpu_ring *ring,
2578 int xcc_id)
2579 {
2580 struct amdgpu_device *adev = ring->adev;
2581 struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
2582 int j;
2583
2584 /* inactivate the queue */
2585 if (amdgpu_sriov_vf(adev))
2586 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 0);
2587
2588 /* disable wptr polling */
2589 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
2590
2591 /* write the EOP addr */
2592 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR,
2593 mqd->cp_hqd_eop_base_addr_lo);
2594 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR_HI,
2595 mqd->cp_hqd_eop_base_addr_hi);
2596
2597 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2598 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL,
2599 mqd->cp_hqd_eop_control);
2600
2601 /* enable doorbell? */
2602 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
2603 mqd->cp_hqd_pq_doorbell_control);
2604
2605 /* disable the queue if it's active */
2606 if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
2607 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
2608 for (j = 0; j < adev->usec_timeout; j++) {
2609 if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
2610 break;
2611 udelay(1);
2612 }
2613 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
2614 mqd->cp_hqd_dequeue_request);
2615 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR,
2616 mqd->cp_hqd_pq_rptr);
2617 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
2618 mqd->cp_hqd_pq_wptr_lo);
2619 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
2620 mqd->cp_hqd_pq_wptr_hi);
2621 }
2622
2623 /* set the pointer to the MQD */
2624 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR,
2625 mqd->cp_mqd_base_addr_lo);
2626 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI,
2627 mqd->cp_mqd_base_addr_hi);
2628
2629 /* set MQD vmid to 0 */
2630 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL,
2631 mqd->cp_mqd_control);
2632
2633 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2634 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE,
2635 mqd->cp_hqd_pq_base_lo);
2636 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI,
2637 mqd->cp_hqd_pq_base_hi);
2638
2639 /* set up the HQD, this is similar to CP_RB0_CNTL */
2640 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL,
2641 mqd->cp_hqd_pq_control);
2642
2643 /* set the wb address whether it's enabled or not */
2644 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
2645 mqd->cp_hqd_pq_rptr_report_addr_lo);
2646 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
2647 mqd->cp_hqd_pq_rptr_report_addr_hi);
2648
2649 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2650 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
2651 mqd->cp_hqd_pq_wptr_poll_addr_lo);
2652 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
2653 mqd->cp_hqd_pq_wptr_poll_addr_hi);
2654
2655 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
2656 mqd->cp_hqd_pq_doorbell_control);
2657
2658 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2659 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
2660 mqd->cp_hqd_pq_wptr_lo);
2661 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
2662 mqd->cp_hqd_pq_wptr_hi);
2663
2664 /* set the vmid for the queue */
2665 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, mqd->cp_hqd_vmid);
2666
2667 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE,
2668 mqd->cp_hqd_persistent_state);
2669
2670 /* activate the queue */
2671 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE,
2672 mqd->cp_hqd_active);
2673
2674 if (ring->use_doorbell)
2675 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2676
2677 return 0;
2678 }
2679
gfx_v12_1_xcc_kiq_init_queue(struct amdgpu_ring * ring,int xcc_id)2680 static int gfx_v12_1_xcc_kiq_init_queue(struct amdgpu_ring *ring,
2681 int xcc_id)
2682 {
2683 struct amdgpu_device *adev = ring->adev;
2684 struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
2685
2686 gfx_v12_1_xcc_kiq_setting(ring, xcc_id);
2687
2688 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
2689 /* reset MQD to a clean status */
2690 if (adev->gfx.kiq[xcc_id].mqd_backup)
2691 memcpy(mqd, adev->gfx.kiq[xcc_id].mqd_backup, sizeof(*mqd));
2692
2693 /* reset ring buffer */
2694 ring->wptr = 0;
2695 amdgpu_ring_clear_ring(ring);
2696
2697 mutex_lock(&adev->srbm_mutex);
2698 soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2699 gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
2700 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2701 mutex_unlock(&adev->srbm_mutex);
2702 } else {
2703 memset((void *)mqd, 0, sizeof(*mqd));
2704 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
2705 amdgpu_ring_clear_ring(ring);
2706 mutex_lock(&adev->srbm_mutex);
2707 soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2708 amdgpu_ring_init_mqd(ring);
2709 gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
2710 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2711 mutex_unlock(&adev->srbm_mutex);
2712
2713 if (adev->gfx.kiq[xcc_id].mqd_backup)
2714 memcpy(adev->gfx.kiq[xcc_id].mqd_backup, mqd, sizeof(*mqd));
2715 }
2716
2717 return 0;
2718 }
2719
gfx_v12_1_xcc_kcq_init_queue(struct amdgpu_ring * ring,int xcc_id)2720 static int gfx_v12_1_xcc_kcq_init_queue(struct amdgpu_ring *ring,
2721 int xcc_id)
2722 {
2723 struct amdgpu_device *adev = ring->adev;
2724 struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
2725 int mqd_idx = ring - &adev->gfx.compute_ring[0];
2726
2727 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2728 memset((void *)mqd, 0, sizeof(*mqd));
2729 mutex_lock(&adev->srbm_mutex);
2730 soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
2731 amdgpu_ring_init_mqd(ring);
2732 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
2733 mutex_unlock(&adev->srbm_mutex);
2734
2735 if (adev->gfx.mec.mqd_backup[mqd_idx])
2736 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
2737 } else {
2738 /* restore MQD to a clean status */
2739 if (adev->gfx.mec.mqd_backup[mqd_idx])
2740 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
2741 /* reset ring buffer */
2742 ring->wptr = 0;
2743 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
2744 amdgpu_ring_clear_ring(ring);
2745 }
2746
2747 return 0;
2748 }
2749
gfx_v12_1_xcc_kiq_resume(struct amdgpu_device * adev,int xcc_id)2750 static int gfx_v12_1_xcc_kiq_resume(struct amdgpu_device *adev,
2751 int xcc_id)
2752 {
2753 struct amdgpu_ring *ring;
2754 int r;
2755
2756 ring = &adev->gfx.kiq[xcc_id].ring;
2757
2758 r = amdgpu_bo_reserve(ring->mqd_obj, false);
2759 if (unlikely(r != 0))
2760 return r;
2761
2762 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2763 if (unlikely(r != 0)) {
2764 amdgpu_bo_unreserve(ring->mqd_obj);
2765 return r;
2766 }
2767
2768 gfx_v12_1_xcc_kiq_init_queue(ring, xcc_id);
2769 amdgpu_bo_kunmap(ring->mqd_obj);
2770 ring->mqd_ptr = NULL;
2771 amdgpu_bo_unreserve(ring->mqd_obj);
2772 ring->sched.ready = true;
2773 return 0;
2774 }
2775
gfx_v12_1_xcc_kcq_resume(struct amdgpu_device * adev,int xcc_id)2776 static int gfx_v12_1_xcc_kcq_resume(struct amdgpu_device *adev,
2777 int xcc_id)
2778 {
2779 struct amdgpu_ring *ring = NULL;
2780 int r = 0, i;
2781
2782 if (!amdgpu_async_gfx_ring)
2783 gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id);
2784
2785 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2786 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
2787
2788 r = amdgpu_bo_reserve(ring->mqd_obj, false);
2789 if (unlikely(r != 0))
2790 goto done;
2791 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
2792 if (!r) {
2793 r = gfx_v12_1_xcc_kcq_init_queue(ring, xcc_id);
2794 amdgpu_bo_kunmap(ring->mqd_obj);
2795 ring->mqd_ptr = NULL;
2796 }
2797 amdgpu_bo_unreserve(ring->mqd_obj);
2798 if (r)
2799 goto done;
2800 }
2801
2802 r = amdgpu_gfx_enable_kcq(adev, xcc_id);
2803 done:
2804 return r;
2805 }
2806
gfx_v12_1_xcc_cp_resume(struct amdgpu_device * adev,uint16_t xcc_mask)2807 static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask)
2808 {
2809 int r, i, xcc_id;
2810 struct amdgpu_ring *ring;
2811
2812 for_each_inst(xcc_id, xcc_mask) {
2813 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
2814 /* legacy firmware loading */
2815 r = gfx_v12_1_xcc_cp_compute_load_microcode_rs64(adev, xcc_id);
2816 if (r)
2817 return r;
2818 }
2819
2820 /* GFX CGCG and LS is set by default */
2821 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
2822 gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, true, xcc_id);
2823
2824 gfx_v12_1_xcc_cp_set_doorbell_range(adev, xcc_id);
2825
2826 gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id);
2827
2828 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) {
2829 r = amdgpu_mes_kiq_hw_init(adev, xcc_id);
2830 /*
2831 * With MES, GFX KIQ ring is owned by the MES and is never
2832 * initialized/used directly by the driver, so it must
2833 * not be left flagged as ready. mes_v12_0_hw_init() clears
2834 * but clear here if MES init fails
2835 */
2836 if (r)
2837 adev->gfx.kiq[xcc_id].ring.sched.ready = false;
2838 } else {
2839 r = gfx_v12_1_xcc_kiq_resume(adev, xcc_id);
2840 }
2841 if (r)
2842 return r;
2843
2844 r = gfx_v12_1_xcc_kcq_resume(adev, xcc_id);
2845 if (r)
2846 return r;
2847
2848 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2849 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
2850 r = amdgpu_ring_test_helper(ring);
2851 if (r)
2852 return r;
2853 }
2854 }
2855
2856 return 0;
2857 }
2858
gfx_v12_1_cp_resume(struct amdgpu_device * adev)2859 static int gfx_v12_1_cp_resume(struct amdgpu_device *adev)
2860 {
2861 int num_xcc, num_xcp, num_xcc_per_xcp;
2862 uint16_t xcc_mask;
2863 int r = 0;
2864
2865 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2866 if (amdgpu_sriov_vf(adev)) {
2867 enum amdgpu_gfx_partition mode;
2868
2869 mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
2870 AMDGPU_XCP_FL_NONE);
2871 if (mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2872 return -EINVAL;
2873 if (adev->gfx.funcs &&
2874 adev->gfx.funcs->get_xccs_per_xcp) {
2875 num_xcc_per_xcp = adev->gfx.funcs->get_xccs_per_xcp(adev);
2876 adev->gfx.num_xcc_per_xcp = num_xcc_per_xcp;
2877 num_xcp = num_xcc / num_xcc_per_xcp;
2878 } else {
2879 return -EINVAL;
2880 }
2881 r = amdgpu_xcp_init(adev->xcp_mgr, num_xcp, mode);
2882
2883 } else {
2884 if (amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
2885 AMDGPU_XCP_FL_NONE) ==
2886 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2887 r = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr,
2888 amdgpu_user_partt_mode);
2889 }
2890
2891 if (r)
2892 return r;
2893
2894 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0);
2895
2896 return gfx_v12_1_xcc_cp_resume(adev, xcc_mask);
2897 }
2898
gfx_v12_1_gfxhub_enable(struct amdgpu_device * adev)2899 static int gfx_v12_1_gfxhub_enable(struct amdgpu_device *adev)
2900 {
2901 int r, i;
2902 bool value;
2903
2904 r = adev->gfxhub.funcs->gart_enable(adev);
2905 if (r)
2906 return r;
2907
2908 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
2909 false : true;
2910
2911 adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2912 /* TODO investigate why TLB flush is needed,
2913 * are we missing a flush somewhere else? */
2914 for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
2915 if (AMDGPU_IS_GFXHUB(i))
2916 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(i), 0);
2917 }
2918
2919 return 0;
2920 }
2921
get_gb_addr_config(struct amdgpu_device * adev)2922 static int get_gb_addr_config(struct amdgpu_device *adev)
2923 {
2924 u32 gb_addr_config;
2925
2926 gb_addr_config = RREG32_SOC15(GC, GET_INST(GC, 0), regGB_ADDR_CONFIG_READ);
2927 if (gb_addr_config == 0)
2928 return -EINVAL;
2929
2930 adev->gfx.config.gb_addr_config_fields.num_pkrs =
2931 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG_READ, NUM_PKRS);
2932
2933 adev->gfx.config.gb_addr_config = gb_addr_config;
2934
2935 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2936 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2937 GB_ADDR_CONFIG_READ, NUM_PIPES);
2938
2939 adev->gfx.config.max_tile_pipes =
2940 adev->gfx.config.gb_addr_config_fields.num_pipes;
2941
2942 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2943 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2944 GB_ADDR_CONFIG_READ, MAX_COMPRESSED_FRAGS);
2945 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2946 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2947 GB_ADDR_CONFIG_READ, NUM_RB_PER_SE);
2948 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2949 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2950 GB_ADDR_CONFIG_READ, NUM_SHADER_ENGINES);
2951 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2952 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
2953 GB_ADDR_CONFIG_READ, PIPE_INTERLEAVE_SIZE));
2954
2955 return 0;
2956 }
2957
gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device * adev,int xcc_id)2958 static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev,
2959 int xcc_id)
2960 {
2961 uint32_t data;
2962
2963 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG);
2964 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK;
2965 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG, data);
2966
2967 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG);
2968 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK;
2969 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG, data);
2970 }
2971
gfx_v12_1_xcc_enable_atomics(struct amdgpu_device * adev,int xcc_id)2972 static void gfx_v12_1_xcc_enable_atomics(struct amdgpu_device *adev,
2973 int xcc_id)
2974 {
2975 uint32_t data;
2976
2977 /* Set the TCP UTCL0 register to enable atomics */
2978 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_UTCL0_CNTL1);
2979 data = REG_SET_FIELD(data, TCP_UTCL0_CNTL1, ATOMIC_REQUESTER_EN, 0x1);
2980
2981 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_UTCL0_CNTL1, data);
2982 }
2983
gfx_v12_1_xcc_disable_burst(struct amdgpu_device * adev,int xcc_id)2984 static void gfx_v12_1_xcc_disable_burst(struct amdgpu_device *adev,
2985 int xcc_id)
2986 {
2987 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGL1_DRAM_BURST_CTRL, 0xf);
2988 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGLARB_DRAM_BURST_CTRL, 0xf);
2989 }
2990
gfx_v12_1_xcc_disable_early_write_ack(struct amdgpu_device * adev,int xcc_id)2991 static void gfx_v12_1_xcc_disable_early_write_ack(struct amdgpu_device *adev,
2992 int xcc_id)
2993 {
2994 uint32_t data;
2995
2996 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL3);
2997 data = REG_SET_FIELD(data, TCP_CNTL3, DISABLE_EARLY_WRITE_ACK, 0x1);
2998
2999 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL3, data);
3000 }
3001
gfx_v12_1_xcc_disable_tcp_spill_cache(struct amdgpu_device * adev,int xcc_id)3002 static void gfx_v12_1_xcc_disable_tcp_spill_cache(struct amdgpu_device *adev,
3003 int xcc_id)
3004 {
3005 uint32_t data;
3006
3007 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL);
3008 data = REG_SET_FIELD(data, TCP_CNTL, TCP_SPILL_CACHE_DISABLE, 0x1);
3009
3010 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL, data);
3011 }
3012
gfx_v12_1_init_golden_registers(struct amdgpu_device * adev)3013 static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev)
3014 {
3015 int i;
3016
3017 for (i = 0; i < NUM_XCC(adev->gfx.xcc_mask); i++) {
3018 gfx_v12_1_xcc_disable_burst(adev, i);
3019 gfx_v12_1_xcc_enable_atomics(adev, i);
3020 gfx_v12_1_xcc_disable_early_write_ack(adev, i);
3021 gfx_v12_1_xcc_disable_tcp_spill_cache(adev, i);
3022 }
3023 }
3024
gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device * adev,bool enable)3025 static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev,
3026 bool enable)
3027 {
3028 unsigned int irq_type;
3029 int m, p, r;
3030
3031 if (!adev->gfx.disable_kq)
3032 return 0;
3033
3034 for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
3035 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
3036 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
3037 + (m * adev->gfx.mec.num_pipe_per_mec)
3038 + p;
3039 if (enable)
3040 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
3041 else
3042 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
3043 if (r) {
3044 if (!enable)
3045 return r;
3046 goto err_unwind;
3047 }
3048 }
3049 }
3050
3051 return 0;
3052
3053 err_unwind:
3054 for (p--; p >= 0; p--) {
3055 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
3056 + (m * adev->gfx.mec.num_pipe_per_mec) + p;
3057 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
3058 }
3059 for (m--; m >= 0; m--) {
3060 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
3061 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
3062 + (m * adev->gfx.mec.num_pipe_per_mec) + p;
3063 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
3064 }
3065 }
3066 return r;
3067 }
3068
gfx_v12_1_hw_init(struct amdgpu_ip_block * ip_block)3069 static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
3070 {
3071 int r, i, num_xcc;
3072 struct amdgpu_device *adev = ip_block->adev;
3073
3074 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
3075 /* rlc autoload firmware */
3076 r = gfx_v12_1_rlc_backdoor_autoload_enable(adev);
3077 if (r)
3078 return r;
3079 } else {
3080 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3081 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3082
3083 if (adev->gfx.imu.funcs) {
3084 if (adev->gfx.imu.funcs->load_microcode)
3085 adev->gfx.imu.funcs->load_microcode(adev);
3086 }
3087
3088 for (i = 0; i < num_xcc; i++) {
3089 /* disable gpa mode in backdoor loading */
3090 gfx_v12_1_xcc_disable_gpa_mode(adev, i);
3091 }
3092 }
3093 }
3094
3095 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) ||
3096 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) {
3097 r = gfx_v12_1_wait_for_rlc_autoload_complete(adev);
3098 if (r) {
3099 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r);
3100 return r;
3101 }
3102 }
3103
3104 adev->gfx.is_poweron = true;
3105
3106 if (get_gb_addr_config(adev))
3107 DRM_WARN("Invalid gb_addr_config !\n");
3108
3109 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
3110 gfx_v12_1_config_gfx_rs64(adev);
3111
3112 r = gfx_v12_1_gfxhub_enable(adev);
3113 if (r)
3114 return r;
3115
3116 gfx_v12_1_init_golden_registers(adev);
3117
3118 gfx_v12_1_constants_init(adev);
3119
3120 if (adev->nbio.funcs->gc_doorbell_init)
3121 adev->nbio.funcs->gc_doorbell_init(adev);
3122
3123 r = gfx_v12_1_rlc_resume(adev);
3124 if (r)
3125 return r;
3126
3127 /*
3128 * init golden registers and rlc resume may override some registers,
3129 * reconfig them here
3130 */
3131 gfx_v12_1_tcp_harvest(adev);
3132
3133 r = gfx_v12_1_cp_resume(adev);
3134 if (r)
3135 return r;
3136
3137 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
3138 if (r)
3139 return r;
3140
3141 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
3142 if (r)
3143 goto err_priv_inst;
3144
3145 r = gfx_v12_1_set_userq_eop_interrupts(adev, true);
3146 if (r)
3147 goto err_userq_eop;
3148
3149 return 0;
3150
3151 err_userq_eop:
3152 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3153 err_priv_inst:
3154 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3155 return r;
3156 }
3157
gfx_v12_1_xcc_fini(struct amdgpu_device * adev,int xcc_id)3158 static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev,
3159 int xcc_id)
3160 {
3161 uint32_t tmp;
3162
3163 if (!adev->no_hw_access) {
3164 if (amdgpu_gfx_disable_kcq(adev, xcc_id))
3165 DRM_ERROR("KCQ disable failed\n");
3166
3167 amdgpu_mes_kiq_hw_fini(adev, xcc_id);
3168 }
3169
3170 if (amdgpu_sriov_vf(adev)) {
3171 /* Program KIQ position of RLC_CP_SCHEDULERS during destroy */
3172 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
3173 tmp &= 0xffffff00;
3174 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
3175 }
3176 gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id);
3177 gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
3178 }
3179
gfx_v12_1_hw_fini(struct amdgpu_ip_block * ip_block)3180 static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
3181 {
3182 struct amdgpu_device *adev = ip_block->adev;
3183 int i, num_xcc;
3184
3185 gfx_v12_1_set_userq_eop_interrupts(adev, false);
3186 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3187 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3188
3189 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3190 for (i = 0; i < num_xcc; i++) {
3191 gfx_v12_1_xcc_fini(adev, i);
3192 }
3193
3194 adev->gfxhub.funcs->gart_disable(adev);
3195
3196 adev->gfx.is_poweron = false;
3197
3198 return 0;
3199 }
3200
gfx_v12_1_suspend(struct amdgpu_ip_block * ip_block)3201 static int gfx_v12_1_suspend(struct amdgpu_ip_block *ip_block)
3202 {
3203 return gfx_v12_1_hw_fini(ip_block);
3204 }
3205
gfx_v12_1_resume(struct amdgpu_ip_block * ip_block)3206 static int gfx_v12_1_resume(struct amdgpu_ip_block *ip_block)
3207 {
3208 return gfx_v12_1_hw_init(ip_block);
3209 }
3210
gfx_v12_1_is_idle(struct amdgpu_ip_block * ip_block)3211 static bool gfx_v12_1_is_idle(struct amdgpu_ip_block *ip_block)
3212 {
3213 struct amdgpu_device *adev = ip_block->adev;
3214 int i, num_xcc;
3215
3216 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3217 for (i = 0; i < num_xcc; i++) {
3218 if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i),
3219 regGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE))
3220 return false;
3221 }
3222 return true;
3223 }
3224
gfx_v12_1_wait_for_idle(struct amdgpu_ip_block * ip_block)3225 static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
3226 {
3227 unsigned i;
3228 struct amdgpu_device *adev = ip_block->adev;
3229
3230 for (i = 0; i < adev->usec_timeout; i++) {
3231 if (gfx_v12_1_is_idle(ip_block))
3232 return 0;
3233 udelay(1);
3234 }
3235 return -ETIMEDOUT;
3236 }
3237
gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device * adev)3238 static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev)
3239 {
3240 uint64_t clock = 0;
3241
3242 if (adev->smuio.funcs &&
3243 adev->smuio.funcs->get_gpu_clock_counter)
3244 clock = adev->smuio.funcs->get_gpu_clock_counter(adev);
3245 else
3246 dev_warn(adev->dev, "query gpu clock counter is not supported\n");
3247
3248 return clock;
3249 }
3250
gfx_v12_1_early_init(struct amdgpu_ip_block * ip_block)3251 static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
3252 {
3253 struct amdgpu_device *adev = ip_block->adev;
3254
3255
3256 switch (amdgpu_user_queue) {
3257 case -1:
3258 default:
3259 adev->gfx.disable_kq = true;
3260 adev->gfx.disable_uq = true;
3261 break;
3262 case 0:
3263 adev->gfx.disable_kq = false;
3264 adev->gfx.disable_uq = true;
3265 break;
3266 }
3267
3268 adev->gfx.funcs = &gfx_v12_1_gfx_funcs;
3269
3270 if (adev->gfx.disable_kq)
3271 adev->gfx.num_compute_rings = 0;
3272 else
3273 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
3274 AMDGPU_MAX_COMPUTE_RINGS);
3275
3276 gfx_v12_1_set_kiq_pm4_funcs(adev);
3277 gfx_v12_1_set_ring_funcs(adev);
3278 gfx_v12_1_set_irq_funcs(adev);
3279 gfx_v12_1_set_rlc_funcs(adev);
3280 gfx_v12_1_set_mqd_funcs(adev);
3281 gfx_v12_1_set_imu_funcs(adev);
3282
3283 gfx_v12_1_init_rlcg_reg_access_ctrl(adev);
3284
3285 amdgpu_init_rlc_reg_funcs(adev);
3286
3287 return gfx_v12_1_init_microcode(adev);
3288 }
3289
gfx_v12_1_is_rlc_enabled(struct amdgpu_device * adev)3290 static bool gfx_v12_1_is_rlc_enabled(struct amdgpu_device *adev)
3291 {
3292 uint32_t rlc_cntl;
3293
3294 /* if RLC is not enabled, do nothing */
3295 rlc_cntl = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CNTL);
3296 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
3297 }
3298
gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device * adev,int xcc_id)3299 static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev,
3300 int xcc_id)
3301 {
3302 uint32_t data;
3303 unsigned i;
3304
3305 data = RLC_SAFE_MODE__CMD_MASK;
3306 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
3307
3308 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data);
3309
3310 /* wait for RLC_SAFE_MODE */
3311 for (i = 0; i < adev->usec_timeout; i++) {
3312 if (!REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, xcc_id),
3313 regRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
3314 break;
3315 udelay(1);
3316 }
3317 }
3318
gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)3319 static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev,
3320 int xcc_id)
3321 {
3322 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
3323 regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK);
3324 }
3325
gfx_v12_1_update_perf_clk(struct amdgpu_device * adev,bool enable)3326 static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
3327 bool enable)
3328 {
3329 int i, num_xcc;
3330
3331 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3332 for (i = 0; i < num_xcc; i++)
3333 gfx_v12_1_xcc_update_perf_clk(adev, enable, i);
3334 }
3335
gfx_v12_1_update_spm_vmid(struct amdgpu_device * adev,int xcc_id,struct amdgpu_ring * ring,unsigned vmid)3336 static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev,
3337 int xcc_id,
3338 struct amdgpu_ring *ring,
3339 unsigned vmid)
3340 {
3341 u32 reg, data;
3342
3343 reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL);
3344 if (amdgpu_sriov_is_pp_one_vf(adev))
3345 data = RREG32_NO_KIQ(reg);
3346 else
3347 data = RREG32(reg);
3348
3349 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
3350 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
3351
3352 if (amdgpu_sriov_is_pp_one_vf(adev))
3353 WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
3354 else
3355 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data);
3356
3357 if (ring
3358 && amdgpu_sriov_is_pp_one_vf(adev)
3359 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX)
3360 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) {
3361 uint32_t reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL);
3362 amdgpu_ring_emit_wreg(ring, reg, data);
3363 }
3364 }
3365
3366 static const struct amdgpu_rlc_funcs gfx_v12_1_rlc_funcs = {
3367 .is_rlc_enabled = gfx_v12_1_is_rlc_enabled,
3368 .set_safe_mode = gfx_v12_1_xcc_set_safe_mode,
3369 .unset_safe_mode = gfx_v12_1_xcc_unset_safe_mode,
3370 .init = gfx_v12_1_rlc_init,
3371 .get_csb_size = gfx_v12_1_get_csb_size,
3372 .get_csb_buffer = gfx_v12_1_get_csb_buffer,
3373 .resume = gfx_v12_1_rlc_resume,
3374 .stop = gfx_v12_1_rlc_stop,
3375 .reset = gfx_v12_1_rlc_reset,
3376 .start = gfx_v12_1_rlc_start,
3377 .update_spm_vmid = gfx_v12_1_update_spm_vmid,
3378 };
3379
3380 #if 0
3381 static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable)
3382 {
3383 /* TODO */
3384 }
3385
3386 static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable)
3387 {
3388 /* TODO */
3389 }
3390 #endif
3391
gfx_v12_1_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)3392 static int gfx_v12_1_set_powergating_state(struct amdgpu_ip_block *ip_block,
3393 enum amd_powergating_state state)
3394 {
3395 struct amdgpu_device *adev = ip_block->adev;
3396 bool enable = (state == AMD_PG_STATE_GATE);
3397
3398 if (amdgpu_sriov_vf(adev))
3399 return 0;
3400
3401 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
3402 case IP_VERSION(12, 1, 0):
3403 amdgpu_gfx_off_ctrl(adev, enable);
3404 break;
3405 default:
3406 break;
3407 }
3408
3409 return 0;
3410 }
3411
gfx_v12_1_xcc_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable,int xcc_id)3412 static void gfx_v12_1_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
3413 bool enable, int xcc_id)
3414 {
3415 uint32_t def, data;
3416
3417 if (!(adev->cg_flags &
3418 (AMD_CG_SUPPORT_GFX_CGCG |
3419 AMD_CG_SUPPORT_GFX_CGLS |
3420 AMD_CG_SUPPORT_GFX_3D_CGCG |
3421 AMD_CG_SUPPORT_GFX_3D_CGLS)))
3422 return;
3423
3424 if (enable) {
3425 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
3426 regRLC_CGTT_MGCG_OVERRIDE);
3427
3428 /* unset CGCG override */
3429 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3430 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
3431 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3432 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
3433 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG ||
3434 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
3435 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
3436
3437 /* update CGCG override bits */
3438 if (def != data)
3439 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
3440 regRLC_CGTT_MGCG_OVERRIDE, data);
3441
3442 /* enable cgcg FSM(0x0000363F) */
3443 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
3444
3445 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
3446 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK;
3447 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
3448 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
3449 }
3450
3451 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
3452 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK;
3453 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
3454 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3455 }
3456
3457 if (def != data)
3458 WREG32_SOC15(GC, GET_INST(GC, xcc_id),
3459 regRLC_CGCG_CGLS_CTRL, data);
3460
3461 /* set IDLE_POLL_COUNT(0x00900100) */
3462 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);
3463
3464 data &= ~CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK;
3465 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
3466 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
3467 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
3468
3469 if (def != data)
3470 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);
3471
3472 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL);
3473 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1);
3474 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1);
3475 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1);
3476 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1);
3477 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL, data);
3478 } else {
3479 /* Program RLC_CGCG_CGLS_CTRL */
3480 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
3481
3482 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)
3483 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
3484
3485 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
3486 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3487
3488 if (def != data)
3489 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
3490 }
3491 }
3492
gfx_v12_1_xcc_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable,int xcc_id)3493 static void gfx_v12_1_xcc_update_medium_grain_clock_gating(struct amdgpu_device *adev,
3494 bool enable, int xcc_id)
3495 {
3496 uint32_t data, def;
3497 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
3498 return;
3499
3500 /* It is disabled by HW by default */
3501 if (enable) {
3502 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
3503 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
3504 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
3505
3506 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3507 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
3508 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
3509
3510 if (def != data)
3511 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
3512 }
3513 } else {
3514 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
3515 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
3516
3517 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
3518 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
3519 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
3520
3521 if (def != data)
3522 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
3523 }
3524 }
3525 }
3526
gfx_v12_1_xcc_update_repeater_fgcg(struct amdgpu_device * adev,bool enable,int xcc_id)3527 static void gfx_v12_1_xcc_update_repeater_fgcg(struct amdgpu_device *adev,
3528 bool enable, int xcc_id)
3529 {
3530 uint32_t def, data;
3531
3532 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
3533 return;
3534
3535 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
3536
3537 if (enable)
3538 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
3539 RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK);
3540 else
3541 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK |
3542 RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK;
3543
3544 if (def != data)
3545 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
3546 }
3547
gfx_v12_1_xcc_update_sram_fgcg(struct amdgpu_device * adev,bool enable,int xcc_id)3548 static void gfx_v12_1_xcc_update_sram_fgcg(struct amdgpu_device *adev,
3549 bool enable, int xcc_id)
3550 {
3551 uint32_t def, data;
3552
3553 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
3554 return;
3555
3556 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
3557
3558 if (enable)
3559 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
3560 else
3561 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
3562
3563 if (def != data)
3564 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
3565 }
3566
gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device * adev,bool enable,int xcc_id)3567 static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev,
3568 bool enable, int xcc_id)
3569 {
3570 uint32_t def, data;
3571
3572 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK))
3573 return;
3574
3575 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
3576
3577 if (enable)
3578 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3579 else
3580 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK;
3581
3582 if (def != data)
3583 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
3584 }
3585
gfx_v12_1_xcc_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable,int xcc_id)3586 static int gfx_v12_1_xcc_update_gfx_clock_gating(struct amdgpu_device *adev,
3587 bool enable, int xcc_id)
3588 {
3589 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
3590
3591 gfx_v12_1_xcc_update_coarse_grain_clock_gating(adev, enable, xcc_id);
3592
3593 gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, enable, xcc_id);
3594
3595 gfx_v12_1_xcc_update_repeater_fgcg(adev, enable, xcc_id);
3596
3597 gfx_v12_1_xcc_update_sram_fgcg(adev, enable, xcc_id);
3598
3599 gfx_v12_1_xcc_update_perf_clk(adev, enable, xcc_id);
3600
3601 if (adev->cg_flags &
3602 (AMD_CG_SUPPORT_GFX_MGCG |
3603 AMD_CG_SUPPORT_GFX_CGLS |
3604 AMD_CG_SUPPORT_GFX_CGCG |
3605 AMD_CG_SUPPORT_GFX_3D_CGCG |
3606 AMD_CG_SUPPORT_GFX_3D_CGLS))
3607 gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, enable, xcc_id);
3608
3609 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
3610
3611 return 0;
3612 }
3613
gfx_v12_1_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)3614 static int gfx_v12_1_set_clockgating_state(struct amdgpu_ip_block *ip_block,
3615 enum amd_clockgating_state state)
3616 {
3617 struct amdgpu_device *adev = ip_block->adev;
3618 int i, num_xcc;
3619
3620 if (amdgpu_sriov_vf(adev))
3621 return 0;
3622
3623 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3624 switch (adev->ip_versions[GC_HWIP][0]) {
3625 case IP_VERSION(12, 1, 0):
3626 for (i = 0; i < num_xcc; i++)
3627 gfx_v12_1_xcc_update_gfx_clock_gating(adev,
3628 state == AMD_CG_STATE_GATE, i);
3629 break;
3630 default:
3631 break;
3632 }
3633
3634 return 0;
3635 }
3636
gfx_v12_1_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)3637 static void gfx_v12_1_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
3638 {
3639 struct amdgpu_device *adev = ip_block->adev;
3640 int data;
3641
3642 /* AMD_CG_SUPPORT_GFX_MGCG */
3643 data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGTT_MGCG_OVERRIDE);
3644 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
3645 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
3646
3647 /* AMD_CG_SUPPORT_REPEATER_FGCG */
3648 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK))
3649 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG;
3650
3651 /* AMD_CG_SUPPORT_GFX_FGCG */
3652 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK))
3653 *flags |= AMD_CG_SUPPORT_GFX_FGCG;
3654
3655 /* AMD_CG_SUPPORT_GFX_PERF_CLK */
3656 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK))
3657 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK;
3658
3659 /* AMD_CG_SUPPORT_GFX_CGCG */
3660 data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGCG_CGLS_CTRL);
3661 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
3662 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
3663
3664 /* AMD_CG_SUPPORT_GFX_CGLS */
3665 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
3666 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
3667 }
3668
gfx_v12_1_ring_get_rptr_compute(struct amdgpu_ring * ring)3669 static u64 gfx_v12_1_ring_get_rptr_compute(struct amdgpu_ring *ring)
3670 {
3671 /* gfx12 hardware is 32bit rptr */
3672 return *(uint32_t *)ring->rptr_cpu_addr;
3673 }
3674
gfx_v12_1_ring_get_wptr_compute(struct amdgpu_ring * ring)3675 static u64 gfx_v12_1_ring_get_wptr_compute(struct amdgpu_ring *ring)
3676 {
3677 struct amdgpu_device *adev = ring->adev;
3678 u64 wptr;
3679
3680 /* XXX check if swapping is necessary on BE */
3681 if (ring->use_doorbell) {
3682 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
3683 } else {
3684 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
3685 wptr = 0;
3686 }
3687 return wptr;
3688 }
3689
gfx_v12_1_ring_set_wptr_compute(struct amdgpu_ring * ring)3690 static void gfx_v12_1_ring_set_wptr_compute(struct amdgpu_ring *ring)
3691 {
3692 struct amdgpu_device *adev = ring->adev;
3693
3694 /* XXX check if swapping is necessary on BE */
3695 if (ring->use_doorbell) {
3696 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
3697 ring->wptr);
3698 WDOORBELL64(ring->doorbell_index, ring->wptr);
3699 } else {
3700 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__);
3701 }
3702 }
3703
gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)3704 static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring,
3705 struct amdgpu_job *job,
3706 struct amdgpu_ib *ib,
3707 uint32_t flags)
3708 {
3709 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
3710 u32 control = PACKET3_INDIRECT_BUFFER__VALID(1) | ib->length_dw | (vmid << 24);
3711
3712 /* Currently, there is a high possibility to get wave ID mismatch
3713 * between ME and GDS, leading to a hw deadlock, because ME generates
3714 * different wave IDs than the GDS expects. This situation happens
3715 * randomly when at least 5 compute pipes use GDS ordered append.
3716 * The wave IDs generated by ME are also wrong after suspend/resume.
3717 * Those are probably bugs somewhere else in the kernel driver.
3718 *
3719 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
3720 * GDS to 0 for this ring (me/pipe).
3721 */
3722 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
3723 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3724 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
3725 }
3726
3727 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3728 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */
3729 amdgpu_ring_write(ring,
3730 #ifdef __BIG_ENDIAN
3731 (2 << 0) |
3732 #endif
3733 lower_32_bits(ib->gpu_addr));
3734 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
3735 amdgpu_ring_write(ring, control);
3736 }
3737
gfx_v12_1_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)3738 static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
3739 u64 seq, unsigned flags)
3740 {
3741 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
3742 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
3743
3744 /* RELEASE_MEM - flush caches, send int */
3745 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
3746 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__GCR_SEQ(1) |
3747 PACKET3_RELEASE_MEM__GCR_GLV_WB |
3748 PACKET3_RELEASE_MEM__GCR_GL2_WB |
3749 PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(2) |
3750 PACKET3_RELEASE_MEM__TEMPORAL(3) |
3751 PACKET3_RELEASE_MEM__EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3752 PACKET3_RELEASE_MEM__EVENT_INDEX(5)));
3753 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__DATA_SEL(write64bit ? 2 : 1) |
3754 PACKET3_RELEASE_MEM__INT_SEL(int_sel ? 2 : 0)));
3755
3756 /*
3757 * the address should be Qword aligned if 64bit write, Dword
3758 * aligned if only send 32bit data low (discard data high)
3759 */
3760 if (write64bit)
3761 WARN_ON(addr & 0x7);
3762 else
3763 WARN_ON(addr & 0x3);
3764 amdgpu_ring_write(ring, lower_32_bits(addr));
3765 amdgpu_ring_write(ring, upper_32_bits(addr));
3766 amdgpu_ring_write(ring, lower_32_bits(seq));
3767 amdgpu_ring_write(ring, upper_32_bits(seq));
3768 amdgpu_ring_write(ring, 0);
3769 }
3770
gfx_v12_1_ring_emit_pipeline_sync(struct amdgpu_ring * ring)3771 static void gfx_v12_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3772 {
3773 uint32_t seq = ring->fence_drv.sync_seq;
3774 uint64_t addr = ring->fence_drv.gpu_addr;
3775
3776 gfx_v12_1_wait_reg_mem(ring, 0, 1, 0, lower_32_bits(addr),
3777 upper_32_bits(addr), seq, 0xffffffff, 4);
3778 }
3779
gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring * ring,uint16_t pasid,uint32_t flush_type,bool all_hub,uint8_t dst_sel)3780 static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
3781 uint16_t pasid, uint32_t flush_type,
3782 bool all_hub, uint8_t dst_sel)
3783 {
3784 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
3785 amdgpu_ring_write(ring,
3786 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
3787 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
3788 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
3789 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
3790 }
3791
gfx_v12_1_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)3792 static void gfx_v12_1_ring_emit_vm_flush(struct amdgpu_ring *ring,
3793 unsigned vmid, uint64_t pd_addr)
3794 {
3795 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
3796
3797 /* compute doesn't have PFP */
3798 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
3799 /* sync PFP to ME, otherwise we might get invalid PFP reads */
3800 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3801 amdgpu_ring_write(ring, 0x0);
3802 }
3803 }
3804
gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)3805 static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
3806 u64 seq, unsigned int flags)
3807 {
3808 struct amdgpu_device *adev = ring->adev;
3809
3810 /* write fence seq to the "addr" */
3811 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3812 amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
3813 amdgpu_ring_write(ring, lower_32_bits(addr));
3814 amdgpu_ring_write(ring, upper_32_bits(addr));
3815 amdgpu_ring_write(ring, lower_32_bits(seq));
3816
3817 if (flags & AMDGPU_FENCE_FLAG_INT) {
3818 /* set register to trigger INT */
3819 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3820 amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(0) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
3821 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
3822 amdgpu_ring_write(ring, 0);
3823 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
3824 }
3825 }
3826
gfx_v12_1_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)3827 static void gfx_v12_1_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
3828 uint32_t reg_val_offs)
3829 {
3830 struct amdgpu_device *adev = ring->adev;
3831
3832 reg = soc_v1_0_normalize_xcc_reg_offset(reg);
3833
3834 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
3835 amdgpu_ring_write(ring, 0 | /* src: register*/
3836 (5 << 8) | /* dst: memory */
3837 (1 << 20)); /* write confirm */
3838 amdgpu_ring_write(ring, reg);
3839 amdgpu_ring_write(ring, 0);
3840 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3841 reg_val_offs * 4));
3842 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3843 reg_val_offs * 4));
3844 }
3845
gfx_v12_1_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)3846 static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring,
3847 uint32_t reg,
3848 uint32_t val)
3849 {
3850 uint32_t cmd = 0;
3851
3852 reg = soc_v1_0_normalize_xcc_reg_offset(reg);
3853
3854 switch (ring->funcs->type) {
3855 case AMDGPU_RING_TYPE_KIQ:
3856 cmd = (1 << 16); /* no inc addr */
3857 break;
3858 default:
3859 cmd = PACKET3_WRITE_DATA__WR_CONFIRM(1);
3860 break;
3861 }
3862 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3863 amdgpu_ring_write(ring, cmd);
3864 amdgpu_ring_write(ring, reg);
3865 amdgpu_ring_write(ring, 0);
3866 amdgpu_ring_write(ring, val);
3867 }
3868
gfx_v12_1_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)3869 static void gfx_v12_1_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
3870 uint32_t val, uint32_t mask)
3871 {
3872 gfx_v12_1_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
3873 }
3874
gfx_v12_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)3875 static void gfx_v12_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
3876 uint32_t reg0, uint32_t reg1,
3877 uint32_t ref, uint32_t mask)
3878 {
3879 gfx_v12_1_wait_reg_mem(ring, 0, 0, 1, reg0, reg1,
3880 ref, mask, 0x20);
3881 }
3882
gfx_v12_1_xcc_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state,int xcc_id)3883 static void gfx_v12_1_xcc_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
3884 int me, int pipe,
3885 enum amdgpu_interrupt_state state,
3886 int xcc_id)
3887 {
3888 u32 mec_int_cntl, mec_int_cntl_reg;
3889
3890 /*
3891 * amdgpu controls only the first MEC. That's why this function only
3892 * handles the setting of interrupts for this specific MEC. All other
3893 * pipes' interrupts are set by amdkfd.
3894 */
3895
3896 if (me == 1) {
3897 switch (pipe) {
3898 case 0:
3899 mec_int_cntl_reg = SOC15_REG_OFFSET(
3900 GC, GET_INST(GC, xcc_id),
3901 regCP_ME1_PIPE0_INT_CNTL);
3902 break;
3903 case 1:
3904 mec_int_cntl_reg = SOC15_REG_OFFSET(
3905 GC, GET_INST(GC, xcc_id),
3906 regCP_ME1_PIPE1_INT_CNTL);
3907 break;
3908 case 2:
3909 mec_int_cntl_reg = SOC15_REG_OFFSET(
3910 GC, GET_INST(GC, xcc_id),
3911 regCP_ME1_PIPE2_INT_CNTL);
3912 break;
3913 case 3:
3914 mec_int_cntl_reg = SOC15_REG_OFFSET(
3915 GC, GET_INST(GC, xcc_id),
3916 regCP_ME1_PIPE3_INT_CNTL);
3917 break;
3918 default:
3919 DRM_DEBUG("invalid pipe %d\n", pipe);
3920 return;
3921 }
3922 } else {
3923 DRM_DEBUG("invalid me %d\n", me);
3924 return;
3925 }
3926
3927 switch (state) {
3928 case AMDGPU_IRQ_STATE_DISABLE:
3929 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3930 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3931 TIME_STAMP_INT_ENABLE, 0);
3932 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3933 GENERIC0_INT_ENABLE, 0);
3934 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3935 break;
3936 case AMDGPU_IRQ_STATE_ENABLE:
3937 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3938 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3939 TIME_STAMP_INT_ENABLE, 1);
3940 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3941 GENERIC0_INT_ENABLE, 1);
3942 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3943 break;
3944 default:
3945 break;
3946 }
3947 }
3948
gfx_v12_1_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)3949 static int gfx_v12_1_set_eop_interrupt_state(struct amdgpu_device *adev,
3950 struct amdgpu_irq_src *src,
3951 unsigned type,
3952 enum amdgpu_interrupt_state state)
3953 {
3954 int i, num_xcc;
3955
3956 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3957 for (i = 0; i < num_xcc; i++) {
3958 switch (type) {
3959 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
3960 gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3961 adev, 1, 0, state, i);
3962 break;
3963 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
3964 gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3965 adev, 1, 1, state, i);
3966 break;
3967 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
3968 gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3969 adev, 1, 2, state, i);
3970 break;
3971 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
3972 gfx_v12_1_xcc_set_compute_eop_interrupt_state(
3973 adev, 1, 3, state, i);
3974 break;
3975 default:
3976 break;
3977 }
3978 }
3979
3980 return 0;
3981 }
3982
gfx_v12_1_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)3983 static int gfx_v12_1_eop_irq(struct amdgpu_device *adev,
3984 struct amdgpu_irq_src *source,
3985 struct amdgpu_iv_entry *entry)
3986 {
3987 u32 doorbell_offset = entry->src_data[0];
3988 u8 me_id, pipe_id, queue_id;
3989 struct amdgpu_ring *ring;
3990 int i, xcc_id;
3991
3992 DRM_DEBUG("IH: CP EOP\n");
3993
3994 if (adev->enable_mes && doorbell_offset) {
3995 amdgpu_userq_process_fence_irq(adev, doorbell_offset);
3996 } else {
3997 me_id = (entry->ring_id & 0x0c) >> 2;
3998 pipe_id = (entry->ring_id & 0x03) >> 0;
3999 queue_id = (entry->ring_id & 0x70) >> 4;
4000 xcc_id = gfx_v12_1_ih_to_xcc_inst(adev, entry->node_id);
4001
4002 if (xcc_id == -EINVAL)
4003 return -EINVAL;
4004
4005 switch (me_id) {
4006 case 1:
4007 case 2:
4008 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4009 ring = &adev->gfx.compute_ring
4010 [i +
4011 xcc_id * adev->gfx.num_compute_rings];
4012 /* Per-queue interrupt is supported for MEC starting from VI.
4013 * The interrupt can only be enabled/disabled per pipe instead
4014 * of per queue.
4015 */
4016 if ((ring->me == me_id) &&
4017 (ring->pipe == pipe_id) &&
4018 (ring->queue == queue_id))
4019 amdgpu_fence_process(ring);
4020 }
4021 break;
4022 default:
4023 dev_dbg(adev->dev, "Unexpected me %d in eop_irq\n", me_id);
4024 break;
4025 }
4026 }
4027
4028 return 0;
4029 }
4030
gfx_v12_1_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)4031 static int gfx_v12_1_set_priv_reg_fault_state(struct amdgpu_device *adev,
4032 struct amdgpu_irq_src *source,
4033 unsigned type,
4034 enum amdgpu_interrupt_state state)
4035 {
4036 int i, num_xcc;
4037
4038 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4039 switch (state) {
4040 case AMDGPU_IRQ_STATE_DISABLE:
4041 case AMDGPU_IRQ_STATE_ENABLE:
4042 for (i = 0; i < num_xcc; i++)
4043 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
4044 PRIV_REG_INT_ENABLE,
4045 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4046 break;
4047 default:
4048 break;
4049 }
4050
4051 return 0;
4052 }
4053
gfx_v12_1_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)4054 static int gfx_v12_1_set_priv_inst_fault_state(struct amdgpu_device *adev,
4055 struct amdgpu_irq_src *source,
4056 unsigned type,
4057 enum amdgpu_interrupt_state state)
4058 {
4059 int i, num_xcc;
4060
4061 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4062 switch (state) {
4063 case AMDGPU_IRQ_STATE_DISABLE:
4064 case AMDGPU_IRQ_STATE_ENABLE:
4065 for (i = 0; i < num_xcc; i++)
4066 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
4067 PRIV_INSTR_INT_ENABLE,
4068 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
4069 break;
4070 default:
4071 break;
4072 }
4073
4074 return 0;
4075 }
4076
gfx_v12_1_handle_priv_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)4077 static void gfx_v12_1_handle_priv_fault(struct amdgpu_device *adev,
4078 struct amdgpu_iv_entry *entry)
4079 {
4080 u8 me_id, pipe_id, queue_id;
4081 struct amdgpu_ring *ring;
4082 int i, xcc_id;
4083
4084 me_id = (entry->ring_id & 0x0c) >> 2;
4085 pipe_id = (entry->ring_id & 0x03) >> 0;
4086 queue_id = (entry->ring_id & 0x70) >> 4;
4087 xcc_id = gfx_v12_1_ih_to_xcc_inst(adev, entry->node_id);
4088
4089 if (xcc_id == -EINVAL)
4090 return;
4091
4092 if (!adev->gfx.disable_kq) {
4093 switch (me_id) {
4094 case 1:
4095 case 2:
4096 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4097 ring = &adev->gfx.compute_ring
4098 [i +
4099 xcc_id * adev->gfx.num_compute_rings];
4100 if (ring->me == me_id && ring->pipe == pipe_id &&
4101 ring->queue == queue_id)
4102 drm_sched_fault(&ring->sched);
4103 }
4104 break;
4105 default:
4106 dev_dbg(adev->dev, "Unexpected me %d in priv_fault\n", me_id);
4107 break;
4108 }
4109 }
4110 }
4111
gfx_v12_1_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)4112 static int gfx_v12_1_priv_reg_irq(struct amdgpu_device *adev,
4113 struct amdgpu_irq_src *source,
4114 struct amdgpu_iv_entry *entry)
4115 {
4116 DRM_ERROR("Illegal register access in command stream\n");
4117 gfx_v12_1_handle_priv_fault(adev, entry);
4118 return 0;
4119 }
4120
gfx_v12_1_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)4121 static int gfx_v12_1_priv_inst_irq(struct amdgpu_device *adev,
4122 struct amdgpu_irq_src *source,
4123 struct amdgpu_iv_entry *entry)
4124 {
4125 DRM_ERROR("Illegal instruction in command stream\n");
4126 gfx_v12_1_handle_priv_fault(adev, entry);
4127 return 0;
4128 }
4129
gfx_v12_1_rlc_poison_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)4130 static int gfx_v12_1_rlc_poison_irq(struct amdgpu_device *adev,
4131 struct amdgpu_irq_src *source,
4132 struct amdgpu_iv_entry *entry)
4133 {
4134 uint32_t rlc_fed_status = 0;
4135 uint32_t ras_blk = RAS_BLOCK_ID__GFX;
4136 struct ras_ih_info ih_info = {0};
4137 int i, num_xcc;
4138
4139 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4140 for (i = 0; i < num_xcc; i++)
4141 rlc_fed_status |= RREG32(SOC15_REG_OFFSET(GC,
4142 GET_INST(GC, i), regRLC_RLCS_FED_STATUS));
4143
4144 if (!rlc_fed_status)
4145 return 0;
4146
4147 if (REG_GET_FIELD(rlc_fed_status, RLC_RLCS_FED_STATUS, SDMA0_FED_ERR) ||
4148 REG_GET_FIELD(rlc_fed_status, RLC_RLCS_FED_STATUS, SDMA1_FED_ERR))
4149 ras_blk = RAS_BLOCK_ID__SDMA;
4150
4151 dev_warn(adev->dev, "RLC %d FED IRQ\n", ras_blk);
4152
4153 ih_info.block = ras_blk;
4154 ih_info.reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
4155 amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info);
4156 return 0;
4157 }
4158
gfx_v12_1_emit_mem_sync(struct amdgpu_ring * ring)4159 static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
4160 {
4161 const unsigned int gcr_cntl =
4162 PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(1) |
4163 PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(1) |
4164 PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(1) |
4165 PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(1) |
4166 PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(1) |
4167 PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(2);
4168
4169 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
4170 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
4171 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */
4172 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
4173 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */
4174 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
4175 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
4176 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */
4177 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
4178 }
4179
4180 static const struct amd_ip_funcs gfx_v12_1_ip_funcs = {
4181 .name = "gfx_v12_1",
4182 .early_init = gfx_v12_1_early_init,
4183 .sw_init = gfx_v12_1_sw_init,
4184 .sw_fini = gfx_v12_1_sw_fini,
4185 .hw_init = gfx_v12_1_hw_init,
4186 .hw_fini = gfx_v12_1_hw_fini,
4187 .suspend = gfx_v12_1_suspend,
4188 .resume = gfx_v12_1_resume,
4189 .is_idle = gfx_v12_1_is_idle,
4190 .wait_for_idle = gfx_v12_1_wait_for_idle,
4191 .set_clockgating_state = gfx_v12_1_set_clockgating_state,
4192 .set_powergating_state = gfx_v12_1_set_powergating_state,
4193 .get_clockgating_state = gfx_v12_1_get_clockgating_state,
4194 .dump_ip_state = gfx_v12_1_ip_dump,
4195 .print_ip_state = gfx_v12_1_ip_print,
4196 };
4197
4198 static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_compute = {
4199 .type = AMDGPU_RING_TYPE_COMPUTE,
4200 .align_mask = 0xff,
4201 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4202 .support_64bit_ptrs = true,
4203 .get_rptr = gfx_v12_1_ring_get_rptr_compute,
4204 .get_wptr = gfx_v12_1_ring_get_wptr_compute,
4205 .set_wptr = gfx_v12_1_ring_set_wptr_compute,
4206 .emit_frame_size =
4207 7 + /* gfx_v12_1_ring_emit_pipeline_sync */
4208 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4209 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4210 2 + /* gfx_v12_1_ring_emit_vm_flush */
4211 8 + 8 + 8 + /* gfx_v12_1_ring_emit_fence x3 for user fence, vm fence */
4212 8, /* gfx_v12_1_emit_mem_sync */
4213 .emit_ib_size = 7, /* gfx_v12_1_ring_emit_ib_compute */
4214 .emit_ib = gfx_v12_1_ring_emit_ib_compute,
4215 .emit_fence = gfx_v12_1_ring_emit_fence,
4216 .emit_pipeline_sync = gfx_v12_1_ring_emit_pipeline_sync,
4217 .emit_vm_flush = gfx_v12_1_ring_emit_vm_flush,
4218 .test_ring = gfx_v12_1_ring_test_ring,
4219 .test_ib = gfx_v12_1_ring_test_ib,
4220 .insert_nop = amdgpu_ring_insert_nop,
4221 .pad_ib = amdgpu_ring_generic_pad_ib,
4222 .emit_wreg = gfx_v12_1_ring_emit_wreg,
4223 .emit_reg_wait = gfx_v12_1_ring_emit_reg_wait,
4224 .emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait,
4225 .emit_mem_sync = gfx_v12_1_emit_mem_sync,
4226 };
4227
4228 static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_kiq = {
4229 .type = AMDGPU_RING_TYPE_KIQ,
4230 .align_mask = 0xff,
4231 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4232 .support_64bit_ptrs = true,
4233 .get_rptr = gfx_v12_1_ring_get_rptr_compute,
4234 .get_wptr = gfx_v12_1_ring_get_wptr_compute,
4235 .set_wptr = gfx_v12_1_ring_set_wptr_compute,
4236 .emit_frame_size =
4237 7 + /* gfx_v12_1_ring_emit_pipeline_sync */
4238 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4239 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4240 2 + /* gfx_v12_1_ring_emit_vm_flush */
4241 8 + 8 + 8, /* gfx_v12_1_ring_emit_fence_kiq x3 for user fence, vm fence */
4242 .emit_ib_size = 7, /* gfx_v12_1_ring_emit_ib_compute */
4243 .emit_ib = gfx_v12_1_ring_emit_ib_compute,
4244 .emit_fence = gfx_v12_1_ring_emit_fence_kiq,
4245 .test_ring = gfx_v12_1_ring_test_ring,
4246 .test_ib = gfx_v12_1_ring_test_ib,
4247 .insert_nop = amdgpu_ring_insert_nop,
4248 .pad_ib = amdgpu_ring_generic_pad_ib,
4249 .emit_rreg = gfx_v12_1_ring_emit_rreg,
4250 .emit_wreg = gfx_v12_1_ring_emit_wreg,
4251 .emit_reg_wait = gfx_v12_1_ring_emit_reg_wait,
4252 .emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait,
4253 };
4254
gfx_v12_1_set_ring_funcs(struct amdgpu_device * adev)4255 static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev)
4256 {
4257 int i, j, num_xcc;
4258
4259 num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4260 for (i = 0; i < num_xcc; i++) {
4261 adev->gfx.kiq[i].ring.funcs = &gfx_v12_1_ring_funcs_kiq;
4262
4263 for (j = 0; j < adev->gfx.num_compute_rings; j++)
4264 adev->gfx.compute_ring[j + i * adev->gfx.num_compute_rings].funcs =
4265 &gfx_v12_1_ring_funcs_compute;
4266 }
4267 }
4268
4269 static const struct amdgpu_irq_src_funcs gfx_v12_1_eop_irq_funcs = {
4270 .set = gfx_v12_1_set_eop_interrupt_state,
4271 .process = gfx_v12_1_eop_irq,
4272 };
4273
4274 static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_reg_irq_funcs = {
4275 .set = gfx_v12_1_set_priv_reg_fault_state,
4276 .process = gfx_v12_1_priv_reg_irq,
4277 };
4278
4279 static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_inst_irq_funcs = {
4280 .set = gfx_v12_1_set_priv_inst_fault_state,
4281 .process = gfx_v12_1_priv_inst_irq,
4282 };
4283
4284 static const struct amdgpu_irq_src_funcs gfx_v12_1_rlc_poison_irq_funcs = {
4285 .process = gfx_v12_1_rlc_poison_irq,
4286 };
4287
gfx_v12_1_set_irq_funcs(struct amdgpu_device * adev)4288 static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev)
4289 {
4290 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
4291 adev->gfx.eop_irq.funcs = &gfx_v12_1_eop_irq_funcs;
4292
4293 adev->gfx.priv_reg_irq.num_types = 1;
4294 adev->gfx.priv_reg_irq.funcs = &gfx_v12_1_priv_reg_irq_funcs;
4295
4296 adev->gfx.priv_inst_irq.num_types = 1;
4297 adev->gfx.priv_inst_irq.funcs = &gfx_v12_1_priv_inst_irq_funcs;
4298
4299 adev->gfx.rlc_poison_irq.num_types = 1;
4300 adev->gfx.rlc_poison_irq.funcs = &gfx_v12_1_rlc_poison_irq_funcs;
4301 }
4302
gfx_v12_1_set_imu_funcs(struct amdgpu_device * adev)4303 static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev)
4304 {
4305 if (adev->flags & AMD_IS_APU)
4306 adev->gfx.imu.mode = MISSION_MODE;
4307 else
4308 adev->gfx.imu.mode = DEBUG_MODE;
4309 if (!amdgpu_sriov_vf(adev))
4310 adev->gfx.imu.funcs = &gfx_v12_1_imu_funcs;
4311 }
4312
gfx_v12_1_set_rlc_funcs(struct amdgpu_device * adev)4313 static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev)
4314 {
4315 adev->gfx.rlc.funcs = &gfx_v12_1_rlc_funcs;
4316 }
4317
gfx_v12_1_set_mqd_funcs(struct amdgpu_device * adev)4318 static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev)
4319 {
4320 /* set compute eng mqd */
4321 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size =
4322 sizeof(struct v12_1_compute_mqd);
4323 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd =
4324 gfx_v12_1_compute_mqd_init;
4325 }
4326
gfx_v12_1_set_user_cu_inactive_bitmap_per_sh(struct amdgpu_device * adev,u32 bitmap,int xcc_id)4327 static void gfx_v12_1_set_user_cu_inactive_bitmap_per_sh(struct amdgpu_device *adev,
4328 u32 bitmap, int xcc_id)
4329 {
4330 u32 data;
4331
4332 if (!bitmap)
4333 return;
4334
4335 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
4336 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
4337
4338 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG, data);
4339 }
4340
gfx_v12_1_get_cu_active_bitmap_per_sh(struct amdgpu_device * adev,int xcc_id)4341 static u32 gfx_v12_1_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev,
4342 int xcc_id)
4343 {
4344 u32 data, mask;
4345
4346 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SHADER_ARRAY_CONFIG);
4347 data |= RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG);
4348
4349 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
4350 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
4351
4352 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
4353
4354 return (~data) & mask;
4355 }
4356
gfx_v12_1_get_cu_info(struct amdgpu_device * adev,struct amdgpu_cu_info * cu_info)4357 static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev,
4358 struct amdgpu_cu_info *cu_info)
4359 {
4360 int i, j, k, counter, xcc_id, active_cu_number = 0;
4361 u32 mask, bitmap;
4362 unsigned int disable_masks[2 * 2];
4363
4364 if (!adev || !cu_info)
4365 return -EINVAL;
4366
4367 if (adev->gfx.config.max_shader_engines > 2 ||
4368 adev->gfx.config.max_sh_per_se > 2) {
4369 dev_err(adev->dev,
4370 "Max SE (%d) and Max SA per SE (%d) is greater than expected\n",
4371 adev->gfx.config.max_shader_engines,
4372 adev->gfx.config.max_sh_per_se);
4373 return -EINVAL;
4374 }
4375
4376 amdgpu_gfx_parse_disable_cu(adev, disable_masks,
4377 adev->gfx.config.max_shader_engines,
4378 adev->gfx.config.max_sh_per_se);
4379
4380 mutex_lock(&adev->grbm_idx_mutex);
4381 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
4382 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
4383 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
4384 bitmap = i * adev->gfx.config.max_sh_per_se + j;
4385 if (!((gfx_v12_1_get_sa_active_bitmap(adev, xcc_id) >> bitmap) & 1))
4386 continue;
4387 mask = 1;
4388 counter = 0;
4389 gfx_v12_1_xcc_select_se_sh(adev, i, j, 0xffffffff, xcc_id);
4390 gfx_v12_1_set_user_cu_inactive_bitmap_per_sh(
4391 adev,
4392 disable_masks[i * adev->gfx.config.max_sh_per_se + j],
4393 xcc_id);
4394 bitmap = gfx_v12_1_get_cu_active_bitmap_per_sh(adev, xcc_id);
4395
4396 cu_info->bitmap[xcc_id][i][j] = bitmap;
4397
4398 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
4399 if (bitmap & mask)
4400 counter++;
4401
4402 mask <<= 1;
4403 }
4404 active_cu_number += counter;
4405 }
4406 }
4407 gfx_v12_1_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, xcc_id);
4408 }
4409 mutex_unlock(&adev->grbm_idx_mutex);
4410
4411 cu_info->number = active_cu_number;
4412 cu_info->simd_per_cu = NUM_SIMD_PER_CU_GFX12_1;
4413 cu_info->lds_size = 320;
4414
4415 return 0;
4416 }
4417
4418 const struct amdgpu_ip_block_version gfx_v12_1_ip_block = {
4419 .type = AMD_IP_BLOCK_TYPE_GFX,
4420 .major = 12,
4421 .minor = 1,
4422 .rev = 0,
4423 .funcs = &gfx_v12_1_ip_funcs,
4424 };
4425
gfx_v12_1_xcp_resume(void * handle,uint32_t inst_mask)4426 static int gfx_v12_1_xcp_resume(void *handle, uint32_t inst_mask)
4427 {
4428 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4429 uint32_t tmp_mask;
4430 int i, r;
4431
4432 /* TODO : Initialize golden regs */
4433 /* gfx_v12_1_init_golden_registers(adev); */
4434
4435 tmp_mask = inst_mask;
4436 for_each_inst(i, tmp_mask)
4437 gfx_v12_1_xcc_constants_init(adev, i);
4438
4439 if (!amdgpu_sriov_vf(adev)) {
4440 tmp_mask = inst_mask;
4441 for_each_inst(i, tmp_mask) {
4442 r = gfx_v12_1_xcc_rlc_resume(adev, i);
4443 if (r)
4444 return r;
4445 }
4446 }
4447
4448 r = gfx_v12_1_xcc_cp_resume(adev, inst_mask);
4449
4450 return r;
4451 }
4452
gfx_v12_1_xcp_suspend(void * handle,uint32_t inst_mask)4453 static int gfx_v12_1_xcp_suspend(void *handle, uint32_t inst_mask)
4454 {
4455 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4456 int i;
4457
4458 for_each_inst(i, inst_mask)
4459 gfx_v12_1_xcc_fini(adev, i);
4460
4461 return 0;
4462 }
4463
4464 struct amdgpu_xcp_ip_funcs gfx_v12_1_xcp_funcs = {
4465 .suspend = &gfx_v12_1_xcp_suspend,
4466 .resume = &gfx_v12_1_xcp_resume
4467 };
4468