1 /*
2 * Copyright 2022 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28
29 #include "amdgpu.h"
30 #include "amdgpu_xcp.h"
31 #include "amdgpu_ucode.h"
32 #include "amdgpu_trace.h"
33
34 #include "sdma/sdma_4_4_2_offset.h"
35 #include "sdma/sdma_4_4_2_sh_mask.h"
36
37 #include "soc15_common.h"
38 #include "soc15.h"
39 #include "vega10_sdma_pkt_open.h"
40
41 #include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
42 #include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
43
44 #include "amdgpu_ras.h"
45
46 MODULE_FIRMWARE("amdgpu/sdma_4_4_2.bin");
47 MODULE_FIRMWARE("amdgpu/sdma_4_4_5.bin");
48
49 static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_4_2[] = {
50 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS_REG),
51 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS1_REG),
52 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS2_REG),
53 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS3_REG),
54 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UCODE_CHECKSUM),
55 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RB_RPTR_FETCH_HI),
56 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RB_RPTR_FETCH),
57 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_STATUS),
58 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_STATUS),
59 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_XNACK0),
60 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_XNACK1),
61 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_XNACK0),
62 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_XNACK1),
63 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_CNTL),
64 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_RPTR),
65 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_RPTR_HI),
66 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_WPTR),
67 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_WPTR_HI),
68 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_OFFSET),
69 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_BASE_LO),
70 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_BASE_HI),
71 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_CNTL),
72 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_RPTR),
73 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_SUB_REMAIN),
74 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_DUMMY_REG),
75 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_CNTL),
76 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_RPTR),
77 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_RPTR_HI),
78 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_WPTR),
79 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_WPTR_HI),
80 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_OFFSET),
81 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_BASE_LO),
82 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_BASE_HI),
83 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_DUMMY_REG),
84 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_CNTL),
85 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_RPTR),
86 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_RPTR_HI),
87 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_WPTR),
88 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_WPTR_HI),
89 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_OFFSET),
90 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_BASE_LO),
91 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_BASE_HI),
92 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_DUMMY_REG),
93 SOC15_REG_ENTRY_STR(GC, 0, regSDMA_VM_CNTL)
94 };
95
96 #define mmSMNAID_AID0_MCA_SMU 0x03b30400
97
98 #define WREG32_SDMA(instance, offset, value) \
99 WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value)
100 #define RREG32_SDMA(instance, offset) \
101 RREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)))
102
103 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev);
104 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev);
105 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev);
106 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev);
107 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev);
108
sdma_v4_4_2_get_reg_offset(struct amdgpu_device * adev,u32 instance,u32 offset)109 static u32 sdma_v4_4_2_get_reg_offset(struct amdgpu_device *adev,
110 u32 instance, u32 offset)
111 {
112 u32 dev_inst = GET_INST(SDMA0, instance);
113
114 return (adev->reg_offset[SDMA0_HWIP][dev_inst][0] + offset);
115 }
116
sdma_v4_4_2_seq_to_irq_id(int seq_num)117 static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
118 {
119 switch (seq_num) {
120 case 0:
121 return SOC15_IH_CLIENTID_SDMA0;
122 case 1:
123 return SOC15_IH_CLIENTID_SDMA1;
124 case 2:
125 return SOC15_IH_CLIENTID_SDMA2;
126 case 3:
127 return SOC15_IH_CLIENTID_SDMA3;
128 default:
129 return -EINVAL;
130 }
131 }
132
sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device * adev,unsigned client_id)133 static int sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device *adev, unsigned client_id)
134 {
135 switch (client_id) {
136 case SOC15_IH_CLIENTID_SDMA0:
137 return 0;
138 case SOC15_IH_CLIENTID_SDMA1:
139 return 1;
140 case SOC15_IH_CLIENTID_SDMA2:
141 if (amdgpu_sriov_vf(adev) && (adev->gfx.xcc_mask == 0x1))
142 return 0;
143 else
144 return 2;
145 case SOC15_IH_CLIENTID_SDMA3:
146 if (amdgpu_sriov_vf(adev) && (adev->gfx.xcc_mask == 0x1))
147 return 1;
148 else
149 return 3;
150 default:
151 return -EINVAL;
152 }
153 }
154
sdma_v4_4_2_inst_init_golden_registers(struct amdgpu_device * adev,uint32_t inst_mask)155 static void sdma_v4_4_2_inst_init_golden_registers(struct amdgpu_device *adev,
156 uint32_t inst_mask)
157 {
158 u32 val;
159 int i;
160
161 for (i = 0; i < adev->sdma.num_instances; i++) {
162 val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG);
163 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG, NUM_BANKS, 4);
164 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG,
165 PIPE_INTERLEAVE_SIZE, 0);
166 WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG, val);
167
168 val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ);
169 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ, NUM_BANKS,
170 4);
171 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ,
172 PIPE_INTERLEAVE_SIZE, 0);
173 WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ, val);
174 }
175 }
176
177 /**
178 * sdma_v4_4_2_init_microcode - load ucode images from disk
179 *
180 * @adev: amdgpu_device pointer
181 *
182 * Use the firmware interface to load the ucode images into
183 * the driver (not loaded into hw).
184 * Returns 0 on success, error on failure.
185 */
sdma_v4_4_2_init_microcode(struct amdgpu_device * adev)186 static int sdma_v4_4_2_init_microcode(struct amdgpu_device *adev)
187 {
188 int ret, i;
189
190 for (i = 0; i < adev->sdma.num_instances; i++) {
191 if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 2) ||
192 amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 4) ||
193 amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 5)) {
194 ret = amdgpu_sdma_init_microcode(adev, 0, true);
195 break;
196 } else {
197 ret = amdgpu_sdma_init_microcode(adev, i, false);
198 if (ret)
199 return ret;
200 }
201 }
202
203 return ret;
204 }
205
206 /**
207 * sdma_v4_4_2_ring_get_rptr - get the current read pointer
208 *
209 * @ring: amdgpu ring pointer
210 *
211 * Get the current rptr from the hardware.
212 */
sdma_v4_4_2_ring_get_rptr(struct amdgpu_ring * ring)213 static uint64_t sdma_v4_4_2_ring_get_rptr(struct amdgpu_ring *ring)
214 {
215 u64 rptr;
216
217 /* XXX check if swapping is necessary on BE */
218 rptr = READ_ONCE(*((u64 *)&ring->adev->wb.wb[ring->rptr_offs]));
219
220 DRM_DEBUG("rptr before shift == 0x%016llx\n", rptr);
221 return rptr >> 2;
222 }
223
224 /**
225 * sdma_v4_4_2_ring_get_wptr - get the current write pointer
226 *
227 * @ring: amdgpu ring pointer
228 *
229 * Get the current wptr from the hardware.
230 */
sdma_v4_4_2_ring_get_wptr(struct amdgpu_ring * ring)231 static uint64_t sdma_v4_4_2_ring_get_wptr(struct amdgpu_ring *ring)
232 {
233 struct amdgpu_device *adev = ring->adev;
234 u64 wptr;
235
236 if (ring->use_doorbell) {
237 /* XXX check if swapping is necessary on BE */
238 wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
239 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
240 } else {
241 wptr = RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI);
242 wptr = wptr << 32;
243 wptr |= RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR);
244 DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",
245 ring->me, wptr);
246 }
247
248 return wptr >> 2;
249 }
250
251 /**
252 * sdma_v4_4_2_ring_set_wptr - commit the write pointer
253 *
254 * @ring: amdgpu ring pointer
255 *
256 * Write the wptr back to the hardware.
257 */
sdma_v4_4_2_ring_set_wptr(struct amdgpu_ring * ring)258 static void sdma_v4_4_2_ring_set_wptr(struct amdgpu_ring *ring)
259 {
260 struct amdgpu_device *adev = ring->adev;
261
262 DRM_DEBUG("Setting write pointer\n");
263 if (ring->use_doorbell) {
264 u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
265
266 DRM_DEBUG("Using doorbell -- "
267 "wptr_offs == 0x%08x "
268 "lower_32_bits(ring->wptr) << 2 == 0x%08x "
269 "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
270 ring->wptr_offs,
271 lower_32_bits(ring->wptr << 2),
272 upper_32_bits(ring->wptr << 2));
273 /* XXX check if swapping is necessary on BE */
274 WRITE_ONCE(*wb, (ring->wptr << 2));
275 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
276 ring->doorbell_index, ring->wptr << 2);
277 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
278 } else {
279 DRM_DEBUG("Not using doorbell -- "
280 "regSDMA%i_GFX_RB_WPTR == 0x%08x "
281 "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
282 ring->me,
283 lower_32_bits(ring->wptr << 2),
284 ring->me,
285 upper_32_bits(ring->wptr << 2));
286 WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR,
287 lower_32_bits(ring->wptr << 2));
288 WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI,
289 upper_32_bits(ring->wptr << 2));
290 }
291 }
292
293 /**
294 * sdma_v4_4_2_page_ring_get_wptr - get the current write pointer
295 *
296 * @ring: amdgpu ring pointer
297 *
298 * Get the current wptr from the hardware.
299 */
sdma_v4_4_2_page_ring_get_wptr(struct amdgpu_ring * ring)300 static uint64_t sdma_v4_4_2_page_ring_get_wptr(struct amdgpu_ring *ring)
301 {
302 struct amdgpu_device *adev = ring->adev;
303 u64 wptr;
304
305 if (ring->use_doorbell) {
306 /* XXX check if swapping is necessary on BE */
307 wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
308 } else {
309 wptr = RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI);
310 wptr = wptr << 32;
311 wptr |= RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR);
312 }
313
314 return wptr >> 2;
315 }
316
317 /**
318 * sdma_v4_4_2_page_ring_set_wptr - commit the write pointer
319 *
320 * @ring: amdgpu ring pointer
321 *
322 * Write the wptr back to the hardware.
323 */
sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring * ring)324 static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring)
325 {
326 struct amdgpu_device *adev = ring->adev;
327
328 if (ring->use_doorbell) {
329 u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
330
331 /* XXX check if swapping is necessary on BE */
332 WRITE_ONCE(*wb, (ring->wptr << 2));
333 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
334 } else {
335 uint64_t wptr = ring->wptr << 2;
336
337 WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR,
338 lower_32_bits(wptr));
339 WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI,
340 upper_32_bits(wptr));
341 }
342 }
343
sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring * ring,uint32_t count)344 static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
345 {
346 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
347 int i;
348
349 for (i = 0; i < count; i++)
350 if (sdma && sdma->burst_nop && (i == 0))
351 amdgpu_ring_write(ring, ring->funcs->nop |
352 SDMA_PKT_NOP_HEADER_COUNT(count - 1));
353 else
354 amdgpu_ring_write(ring, ring->funcs->nop);
355 }
356
357 /**
358 * sdma_v4_4_2_ring_emit_ib - Schedule an IB on the DMA engine
359 *
360 * @ring: amdgpu ring pointer
361 * @job: job to retrieve vmid from
362 * @ib: IB object to schedule
363 * @flags: unused
364 *
365 * Schedule an IB in the DMA ring.
366 */
sdma_v4_4_2_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)367 static void sdma_v4_4_2_ring_emit_ib(struct amdgpu_ring *ring,
368 struct amdgpu_job *job,
369 struct amdgpu_ib *ib,
370 uint32_t flags)
371 {
372 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
373
374 /* IB packet must end on a 8 DW boundary */
375 sdma_v4_4_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
376
377 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
378 SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
379 /* base must be 32 byte aligned */
380 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
381 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
382 amdgpu_ring_write(ring, ib->length_dw);
383 amdgpu_ring_write(ring, 0);
384 amdgpu_ring_write(ring, 0);
385
386 }
387
sdma_v4_4_2_wait_reg_mem(struct amdgpu_ring * ring,int mem_space,int hdp,uint32_t addr0,uint32_t addr1,uint32_t ref,uint32_t mask,uint32_t inv)388 static void sdma_v4_4_2_wait_reg_mem(struct amdgpu_ring *ring,
389 int mem_space, int hdp,
390 uint32_t addr0, uint32_t addr1,
391 uint32_t ref, uint32_t mask,
392 uint32_t inv)
393 {
394 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
395 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
396 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
397 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
398 if (mem_space) {
399 /* memory */
400 amdgpu_ring_write(ring, addr0);
401 amdgpu_ring_write(ring, addr1);
402 } else {
403 /* registers */
404 amdgpu_ring_write(ring, addr0 << 2);
405 amdgpu_ring_write(ring, addr1 << 2);
406 }
407 amdgpu_ring_write(ring, ref); /* reference */
408 amdgpu_ring_write(ring, mask); /* mask */
409 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
410 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
411 }
412
413 /**
414 * sdma_v4_4_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
415 *
416 * @ring: amdgpu ring pointer
417 *
418 * Emit an hdp flush packet on the requested DMA ring.
419 */
sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring * ring)420 static void sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
421 {
422 struct amdgpu_device *adev = ring->adev;
423 u32 ref_and_mask = 0;
424 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
425
426 ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0
427 << (ring->me % adev->sdma.num_inst_per_aid);
428
429 sdma_v4_4_2_wait_reg_mem(ring, 0, 1,
430 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
431 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
432 ref_and_mask, ref_and_mask, 10);
433 }
434
435 /**
436 * sdma_v4_4_2_ring_emit_fence - emit a fence on the DMA ring
437 *
438 * @ring: amdgpu ring pointer
439 * @addr: address
440 * @seq: sequence number
441 * @flags: fence related flags
442 *
443 * Add a DMA fence packet to the ring to write
444 * the fence seq number and DMA trap packet to generate
445 * an interrupt if needed.
446 */
sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)447 static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
448 unsigned flags)
449 {
450 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
451 /* write the fence */
452 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
453 /* zero in first two bits */
454 BUG_ON(addr & 0x3);
455 amdgpu_ring_write(ring, lower_32_bits(addr));
456 amdgpu_ring_write(ring, upper_32_bits(addr));
457 amdgpu_ring_write(ring, lower_32_bits(seq));
458
459 /* optionally write high bits as well */
460 if (write64bit) {
461 addr += 4;
462 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
463 /* zero in first two bits */
464 BUG_ON(addr & 0x3);
465 amdgpu_ring_write(ring, lower_32_bits(addr));
466 amdgpu_ring_write(ring, upper_32_bits(addr));
467 amdgpu_ring_write(ring, upper_32_bits(seq));
468 }
469
470 /* generate an interrupt */
471 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
472 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
473 }
474
475
476 /**
477 * sdma_v4_4_2_inst_gfx_stop - stop the gfx async dma engines
478 *
479 * @adev: amdgpu_device pointer
480 * @inst_mask: mask of dma engine instances to be disabled
481 *
482 * Stop the gfx async dma ring buffers.
483 */
sdma_v4_4_2_inst_gfx_stop(struct amdgpu_device * adev,uint32_t inst_mask)484 static void sdma_v4_4_2_inst_gfx_stop(struct amdgpu_device *adev,
485 uint32_t inst_mask)
486 {
487 struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
488 u32 doorbell_offset, doorbell;
489 u32 rb_cntl, ib_cntl;
490 int i;
491
492 for_each_inst(i, inst_mask) {
493 sdma[i] = &adev->sdma.instance[i].ring;
494
495 rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
496 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 0);
497 WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
498 ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
499 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 0);
500 WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
501
502 if (sdma[i]->use_doorbell) {
503 doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
504 doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
505
506 doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE, 0);
507 doorbell_offset = REG_SET_FIELD(doorbell_offset,
508 SDMA_GFX_DOORBELL_OFFSET,
509 OFFSET, 0);
510 WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
511 WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
512 }
513 }
514 }
515
516 /**
517 * sdma_v4_4_2_inst_rlc_stop - stop the compute async dma engines
518 *
519 * @adev: amdgpu_device pointer
520 * @inst_mask: mask of dma engine instances to be disabled
521 *
522 * Stop the compute async dma queues.
523 */
sdma_v4_4_2_inst_rlc_stop(struct amdgpu_device * adev,uint32_t inst_mask)524 static void sdma_v4_4_2_inst_rlc_stop(struct amdgpu_device *adev,
525 uint32_t inst_mask)
526 {
527 /* XXX todo */
528 }
529
530 /**
531 * sdma_v4_4_2_inst_page_stop - stop the page async dma engines
532 *
533 * @adev: amdgpu_device pointer
534 * @inst_mask: mask of dma engine instances to be disabled
535 *
536 * Stop the page async dma ring buffers.
537 */
sdma_v4_4_2_inst_page_stop(struct amdgpu_device * adev,uint32_t inst_mask)538 static void sdma_v4_4_2_inst_page_stop(struct amdgpu_device *adev,
539 uint32_t inst_mask)
540 {
541 u32 rb_cntl, ib_cntl;
542 int i;
543
544 for_each_inst(i, inst_mask) {
545 rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
546 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
547 RB_ENABLE, 0);
548 WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
549 ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
550 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL,
551 IB_ENABLE, 0);
552 WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
553 }
554 }
555
556 /**
557 * sdma_v4_4_2_inst_ctx_switch_enable - stop the async dma engines context switch
558 *
559 * @adev: amdgpu_device pointer
560 * @enable: enable/disable the DMA MEs context switch.
561 * @inst_mask: mask of dma engine instances to be enabled
562 *
563 * Halt or unhalt the async dma engines context switch.
564 */
sdma_v4_4_2_inst_ctx_switch_enable(struct amdgpu_device * adev,bool enable,uint32_t inst_mask)565 static void sdma_v4_4_2_inst_ctx_switch_enable(struct amdgpu_device *adev,
566 bool enable, uint32_t inst_mask)
567 {
568 u32 f32_cntl, phase_quantum = 0;
569 int i;
570
571 if (amdgpu_sdma_phase_quantum) {
572 unsigned value = amdgpu_sdma_phase_quantum;
573 unsigned unit = 0;
574
575 while (value > (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
576 SDMA_PHASE0_QUANTUM__VALUE__SHIFT)) {
577 value = (value + 1) >> 1;
578 unit++;
579 }
580 if (unit > (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
581 SDMA_PHASE0_QUANTUM__UNIT__SHIFT)) {
582 value = (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
583 SDMA_PHASE0_QUANTUM__VALUE__SHIFT);
584 unit = (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
585 SDMA_PHASE0_QUANTUM__UNIT__SHIFT);
586 WARN_ONCE(1,
587 "clamping sdma_phase_quantum to %uK clock cycles\n",
588 value << unit);
589 }
590 phase_quantum =
591 value << SDMA_PHASE0_QUANTUM__VALUE__SHIFT |
592 unit << SDMA_PHASE0_QUANTUM__UNIT__SHIFT;
593 }
594
595 for_each_inst(i, inst_mask) {
596 f32_cntl = RREG32_SDMA(i, regSDMA_CNTL);
597 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_CNTL,
598 AUTO_CTXSW_ENABLE, enable ? 1 : 0);
599 if (enable && amdgpu_sdma_phase_quantum) {
600 WREG32_SDMA(i, regSDMA_PHASE0_QUANTUM, phase_quantum);
601 WREG32_SDMA(i, regSDMA_PHASE1_QUANTUM, phase_quantum);
602 WREG32_SDMA(i, regSDMA_PHASE2_QUANTUM, phase_quantum);
603 }
604 WREG32_SDMA(i, regSDMA_CNTL, f32_cntl);
605
606 /* Extend page fault timeout to avoid interrupt storm */
607 WREG32_SDMA(i, regSDMA_UTCL1_TIMEOUT, 0x00800080);
608 }
609 }
610
611 /**
612 * sdma_v4_4_2_inst_enable - stop the async dma engines
613 *
614 * @adev: amdgpu_device pointer
615 * @enable: enable/disable the DMA MEs.
616 * @inst_mask: mask of dma engine instances to be enabled
617 *
618 * Halt or unhalt the async dma engines.
619 */
sdma_v4_4_2_inst_enable(struct amdgpu_device * adev,bool enable,uint32_t inst_mask)620 static void sdma_v4_4_2_inst_enable(struct amdgpu_device *adev, bool enable,
621 uint32_t inst_mask)
622 {
623 u32 f32_cntl;
624 int i;
625
626 if (!enable) {
627 sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
628 sdma_v4_4_2_inst_rlc_stop(adev, inst_mask);
629 if (adev->sdma.has_page_queue)
630 sdma_v4_4_2_inst_page_stop(adev, inst_mask);
631
632 /* SDMA FW needs to respond to FREEZE requests during reset.
633 * Keep it running during reset */
634 if (!amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
635 return;
636 }
637
638 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
639 return;
640
641 for_each_inst(i, inst_mask) {
642 f32_cntl = RREG32_SDMA(i, regSDMA_F32_CNTL);
643 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_F32_CNTL, HALT, enable ? 0 : 1);
644 WREG32_SDMA(i, regSDMA_F32_CNTL, f32_cntl);
645 }
646 }
647
648 /*
649 * sdma_v4_4_2_rb_cntl - get parameters for rb_cntl
650 */
sdma_v4_4_2_rb_cntl(struct amdgpu_ring * ring,uint32_t rb_cntl)651 static uint32_t sdma_v4_4_2_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
652 {
653 /* Set ring buffer size in dwords */
654 uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);
655
656 barrier(); /* work around https://llvm.org/pr42576 */
657 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
658 #ifdef __BIG_ENDIAN
659 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
660 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
661 RPTR_WRITEBACK_SWAP_ENABLE, 1);
662 #endif
663 return rb_cntl;
664 }
665
666 /**
667 * sdma_v4_4_2_gfx_resume - setup and start the async dma engines
668 *
669 * @adev: amdgpu_device pointer
670 * @i: instance to resume
671 * @restore: used to restore wptr when restart
672 *
673 * Set up the gfx DMA ring buffers and enable them.
674 * Returns 0 for success, error for failure.
675 */
sdma_v4_4_2_gfx_resume(struct amdgpu_device * adev,unsigned int i,bool restore)676 static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
677 {
678 struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
679 u32 rb_cntl, ib_cntl, wptr_poll_cntl;
680 u32 wb_offset;
681 u32 doorbell;
682 u32 doorbell_offset;
683 u64 wptr_gpu_addr;
684
685 wb_offset = (ring->rptr_offs * 4);
686
687 rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
688 rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
689 WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
690
691 /* set the wb address whether it's enabled or not */
692 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_HI,
693 upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
694 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_LO,
695 lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
696
697 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
698 RPTR_WRITEBACK_ENABLE, 1);
699
700 WREG32_SDMA(i, regSDMA_GFX_RB_BASE, ring->gpu_addr >> 8);
701 WREG32_SDMA(i, regSDMA_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
702
703 if (!restore)
704 ring->wptr = 0;
705
706 /* before programing wptr to a less value, need set minor_ptr_update first */
707 WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 1);
708
709 /* Initialize the ring buffer's read and write pointers */
710 if (restore) {
711 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, lower_32_bits(ring->wptr << 2));
712 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, upper_32_bits(ring->wptr << 2));
713 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, lower_32_bits(ring->wptr << 2));
714 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, upper_32_bits(ring->wptr << 2));
715 } else {
716 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
717 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
718 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
719 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
720 }
721
722 doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
723 doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
724
725 doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE,
726 ring->use_doorbell);
727 doorbell_offset = REG_SET_FIELD(doorbell_offset,
728 SDMA_GFX_DOORBELL_OFFSET,
729 OFFSET, ring->doorbell_index);
730 WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
731 WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
732
733 sdma_v4_4_2_ring_set_wptr(ring);
734
735 /* set minor_ptr_update to 0 after wptr programed */
736 WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 0);
737
738 /* setup the wptr shadow polling */
739 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
740 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_LO,
741 lower_32_bits(wptr_gpu_addr));
742 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_HI,
743 upper_32_bits(wptr_gpu_addr));
744 wptr_poll_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL);
745 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
746 SDMA_GFX_RB_WPTR_POLL_CNTL,
747 F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
748 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
749
750 /* enable DMA RB */
751 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 1);
752 WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
753
754 ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
755 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 1);
756 #ifdef __BIG_ENDIAN
757 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
758 #endif
759 /* enable DMA IBs */
760 WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
761 }
762
763 /**
764 * sdma_v4_4_2_page_resume - setup and start the async dma engines
765 *
766 * @adev: amdgpu_device pointer
767 * @i: instance to resume
768 * @restore: boolean to say restore needed or not
769 *
770 * Set up the page DMA ring buffers and enable them.
771 * Returns 0 for success, error for failure.
772 */
sdma_v4_4_2_page_resume(struct amdgpu_device * adev,unsigned int i,bool restore)773 static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
774 {
775 struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
776 u32 rb_cntl, ib_cntl, wptr_poll_cntl;
777 u32 wb_offset;
778 u32 doorbell;
779 u32 doorbell_offset;
780 u64 wptr_gpu_addr;
781
782 wb_offset = (ring->rptr_offs * 4);
783
784 rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
785 rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
786 WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
787
788 /* Initialize the ring buffer's read and write pointers */
789 if (restore) {
790 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, lower_32_bits(ring->wptr << 2));
791 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, upper_32_bits(ring->wptr << 2));
792 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, lower_32_bits(ring->wptr << 2));
793 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, upper_32_bits(ring->wptr << 2));
794 } else {
795 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
796 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
797 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
798 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
799 }
800
801 /* set the wb address whether it's enabled or not */
802 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_HI,
803 upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
804 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_LO,
805 lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
806
807 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
808 RPTR_WRITEBACK_ENABLE, 1);
809
810 WREG32_SDMA(i, regSDMA_PAGE_RB_BASE, ring->gpu_addr >> 8);
811 WREG32_SDMA(i, regSDMA_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
812
813 if (!restore)
814 ring->wptr = 0;
815
816 /* before programing wptr to a less value, need set minor_ptr_update first */
817 WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 1);
818
819 doorbell = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL);
820 doorbell_offset = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET);
821
822 doorbell = REG_SET_FIELD(doorbell, SDMA_PAGE_DOORBELL, ENABLE,
823 ring->use_doorbell);
824 doorbell_offset = REG_SET_FIELD(doorbell_offset,
825 SDMA_PAGE_DOORBELL_OFFSET,
826 OFFSET, ring->doorbell_index);
827 WREG32_SDMA(i, regSDMA_PAGE_DOORBELL, doorbell);
828 WREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET, doorbell_offset);
829
830 /* paging queue doorbell range is setup at sdma_v4_4_2_gfx_resume */
831 sdma_v4_4_2_page_ring_set_wptr(ring);
832
833 /* set minor_ptr_update to 0 after wptr programed */
834 WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 0);
835
836 /* setup the wptr shadow polling */
837 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
838 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO,
839 lower_32_bits(wptr_gpu_addr));
840 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI,
841 upper_32_bits(wptr_gpu_addr));
842 wptr_poll_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL);
843 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
844 SDMA_PAGE_RB_WPTR_POLL_CNTL,
845 F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
846 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
847
848 /* enable DMA RB */
849 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL, RB_ENABLE, 1);
850 WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
851
852 ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
853 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_ENABLE, 1);
854 #ifdef __BIG_ENDIAN
855 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);
856 #endif
857 /* enable DMA IBs */
858 WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
859 }
860
sdma_v4_4_2_init_pg(struct amdgpu_device * adev)861 static void sdma_v4_4_2_init_pg(struct amdgpu_device *adev)
862 {
863
864 }
865
866 /**
867 * sdma_v4_4_2_inst_rlc_resume - setup and start the async dma engines
868 *
869 * @adev: amdgpu_device pointer
870 * @inst_mask: mask of dma engine instances to be enabled
871 *
872 * Set up the compute DMA queues and enable them.
873 * Returns 0 for success, error for failure.
874 */
sdma_v4_4_2_inst_rlc_resume(struct amdgpu_device * adev,uint32_t inst_mask)875 static int sdma_v4_4_2_inst_rlc_resume(struct amdgpu_device *adev,
876 uint32_t inst_mask)
877 {
878 sdma_v4_4_2_init_pg(adev);
879
880 return 0;
881 }
882
883 /**
884 * sdma_v4_4_2_inst_load_microcode - load the sDMA ME ucode
885 *
886 * @adev: amdgpu_device pointer
887 * @inst_mask: mask of dma engine instances to be enabled
888 *
889 * Loads the sDMA0/1 ucode.
890 * Returns 0 for success, -EINVAL if the ucode is not available.
891 */
sdma_v4_4_2_inst_load_microcode(struct amdgpu_device * adev,uint32_t inst_mask)892 static int sdma_v4_4_2_inst_load_microcode(struct amdgpu_device *adev,
893 uint32_t inst_mask)
894 {
895 const struct sdma_firmware_header_v1_0 *hdr;
896 const __le32 *fw_data;
897 u32 fw_size;
898 int i, j;
899
900 /* halt the MEs */
901 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
902
903 for_each_inst(i, inst_mask) {
904 if (!adev->sdma.instance[i].fw)
905 return -EINVAL;
906
907 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
908 amdgpu_ucode_print_sdma_hdr(&hdr->header);
909 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
910
911 fw_data = (const __le32 *)
912 (adev->sdma.instance[i].fw->data +
913 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
914
915 WREG32_SDMA(i, regSDMA_UCODE_ADDR, 0);
916
917 for (j = 0; j < fw_size; j++)
918 WREG32_SDMA(i, regSDMA_UCODE_DATA,
919 le32_to_cpup(fw_data++));
920
921 WREG32_SDMA(i, regSDMA_UCODE_ADDR,
922 adev->sdma.instance[i].fw_version);
923 }
924
925 return 0;
926 }
927
928 /**
929 * sdma_v4_4_2_inst_start - setup and start the async dma engines
930 *
931 * @adev: amdgpu_device pointer
932 * @inst_mask: mask of dma engine instances to be enabled
933 * @restore: boolean to say restore needed or not
934 *
935 * Set up the DMA engines and enable them.
936 * Returns 0 for success, error for failure.
937 */
sdma_v4_4_2_inst_start(struct amdgpu_device * adev,uint32_t inst_mask,bool restore)938 static int sdma_v4_4_2_inst_start(struct amdgpu_device *adev,
939 uint32_t inst_mask, bool restore)
940 {
941 struct amdgpu_ring *ring;
942 uint32_t tmp_mask;
943 int i, r = 0;
944
945 if (amdgpu_sriov_vf(adev)) {
946 sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
947 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
948 } else {
949 /* bypass sdma microcode loading on Gopher */
950 if (!restore && adev->firmware.load_type != AMDGPU_FW_LOAD_PSP &&
951 adev->sdma.instance[0].fw) {
952 r = sdma_v4_4_2_inst_load_microcode(adev, inst_mask);
953 if (r)
954 return r;
955 }
956
957 /* unhalt the MEs */
958 sdma_v4_4_2_inst_enable(adev, true, inst_mask);
959 /* enable sdma ring preemption */
960 sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
961 }
962
963 /* start the gfx rings and rlc compute queues */
964 tmp_mask = inst_mask;
965 for_each_inst(i, tmp_mask) {
966 uint32_t temp;
967
968 WREG32_SDMA(i, regSDMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
969 sdma_v4_4_2_gfx_resume(adev, i, restore);
970 if (adev->sdma.has_page_queue)
971 sdma_v4_4_2_page_resume(adev, i, restore);
972
973 /* set utc l1 enable flag always to 1 */
974 temp = RREG32_SDMA(i, regSDMA_CNTL);
975 temp = REG_SET_FIELD(temp, SDMA_CNTL, UTC_L1_ENABLE, 1);
976
977 if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) < IP_VERSION(4, 4, 5)) {
978 /* enable context empty interrupt during initialization */
979 temp = REG_SET_FIELD(temp, SDMA_CNTL, CTXEMPTY_INT_ENABLE, 1);
980 WREG32_SDMA(i, regSDMA_CNTL, temp);
981 }
982 if (!amdgpu_sriov_vf(adev)) {
983 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
984 /* unhalt engine */
985 temp = RREG32_SDMA(i, regSDMA_F32_CNTL);
986 temp = REG_SET_FIELD(temp, SDMA_F32_CNTL, HALT, 0);
987 WREG32_SDMA(i, regSDMA_F32_CNTL, temp);
988 }
989 }
990 }
991
992 if (amdgpu_sriov_vf(adev)) {
993 sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
994 sdma_v4_4_2_inst_enable(adev, true, inst_mask);
995 } else {
996 r = sdma_v4_4_2_inst_rlc_resume(adev, inst_mask);
997 if (r)
998 return r;
999 }
1000
1001 tmp_mask = inst_mask;
1002 for_each_inst(i, tmp_mask) {
1003 ring = &adev->sdma.instance[i].ring;
1004
1005 r = amdgpu_ring_test_helper(ring);
1006 if (r)
1007 return r;
1008
1009 if (adev->sdma.has_page_queue) {
1010 struct amdgpu_ring *page = &adev->sdma.instance[i].page;
1011
1012 r = amdgpu_ring_test_helper(page);
1013 if (r)
1014 return r;
1015 }
1016 }
1017
1018 return r;
1019 }
1020
1021 /**
1022 * sdma_v4_4_2_ring_test_ring - simple async dma engine test
1023 *
1024 * @ring: amdgpu_ring structure holding ring information
1025 *
1026 * Test the DMA engine by writing using it to write an
1027 * value to memory.
1028 * Returns 0 for success, error for failure.
1029 */
sdma_v4_4_2_ring_test_ring(struct amdgpu_ring * ring)1030 static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
1031 {
1032 struct amdgpu_device *adev = ring->adev;
1033 unsigned i;
1034 unsigned index;
1035 int r;
1036 u32 tmp;
1037 u64 gpu_addr;
1038
1039 r = amdgpu_device_wb_get(adev, &index);
1040 if (r)
1041 return r;
1042
1043 gpu_addr = adev->wb.gpu_addr + (index * 4);
1044 tmp = 0xCAFEDEAD;
1045 adev->wb.wb[index] = cpu_to_le32(tmp);
1046
1047 r = amdgpu_ring_alloc(ring, 5);
1048 if (r)
1049 goto error_free_wb;
1050
1051 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1052 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
1053 amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
1054 amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
1055 amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
1056 amdgpu_ring_write(ring, 0xDEADBEEF);
1057 amdgpu_ring_commit(ring);
1058
1059 for (i = 0; i < adev->usec_timeout; i++) {
1060 tmp = le32_to_cpu(adev->wb.wb[index]);
1061 if (tmp == 0xDEADBEEF)
1062 break;
1063 udelay(1);
1064 }
1065
1066 if (i >= adev->usec_timeout)
1067 r = -ETIMEDOUT;
1068
1069 error_free_wb:
1070 amdgpu_device_wb_free(adev, index);
1071 return r;
1072 }
1073
1074 /**
1075 * sdma_v4_4_2_ring_test_ib - test an IB on the DMA engine
1076 *
1077 * @ring: amdgpu_ring structure holding ring information
1078 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1079 *
1080 * Test a simple IB in the DMA ring.
1081 * Returns 0 on success, error on failure.
1082 */
sdma_v4_4_2_ring_test_ib(struct amdgpu_ring * ring,long timeout)1083 static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1084 {
1085 struct amdgpu_device *adev = ring->adev;
1086 struct amdgpu_ib ib;
1087 struct dma_fence *f = NULL;
1088 unsigned index;
1089 long r;
1090 u32 tmp = 0;
1091 u64 gpu_addr;
1092
1093 r = amdgpu_device_wb_get(adev, &index);
1094 if (r)
1095 return r;
1096
1097 gpu_addr = adev->wb.gpu_addr + (index * 4);
1098 tmp = 0xCAFEDEAD;
1099 adev->wb.wb[index] = cpu_to_le32(tmp);
1100 memset(&ib, 0, sizeof(ib));
1101 r = amdgpu_ib_get(adev, NULL, 256,
1102 AMDGPU_IB_POOL_DIRECT, &ib);
1103 if (r)
1104 goto err0;
1105
1106 ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1107 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1108 ib.ptr[1] = lower_32_bits(gpu_addr);
1109 ib.ptr[2] = upper_32_bits(gpu_addr);
1110 ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1111 ib.ptr[4] = 0xDEADBEEF;
1112 ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1113 ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1114 ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1115 ib.length_dw = 8;
1116
1117 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1118 if (r)
1119 goto err1;
1120
1121 r = dma_fence_wait_timeout(f, false, timeout);
1122 if (r == 0) {
1123 r = -ETIMEDOUT;
1124 goto err1;
1125 } else if (r < 0) {
1126 goto err1;
1127 }
1128 tmp = le32_to_cpu(adev->wb.wb[index]);
1129 if (tmp == 0xDEADBEEF)
1130 r = 0;
1131 else
1132 r = -EINVAL;
1133
1134 err1:
1135 amdgpu_ib_free(&ib, NULL);
1136 dma_fence_put(f);
1137 err0:
1138 amdgpu_device_wb_free(adev, index);
1139 return r;
1140 }
1141
1142
1143 /**
1144 * sdma_v4_4_2_vm_copy_pte - update PTEs by copying them from the GART
1145 *
1146 * @ib: indirect buffer to fill with commands
1147 * @pe: addr of the page entry
1148 * @src: src addr to copy from
1149 * @count: number of page entries to update
1150 *
1151 * Update PTEs by copying them from the GART using sDMA.
1152 */
sdma_v4_4_2_vm_copy_pte(struct amdgpu_ib * ib,uint64_t pe,uint64_t src,unsigned count)1153 static void sdma_v4_4_2_vm_copy_pte(struct amdgpu_ib *ib,
1154 uint64_t pe, uint64_t src,
1155 unsigned count)
1156 {
1157 unsigned bytes = count * 8;
1158
1159 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1160 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1161 ib->ptr[ib->length_dw++] = bytes - 1;
1162 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1163 ib->ptr[ib->length_dw++] = lower_32_bits(src);
1164 ib->ptr[ib->length_dw++] = upper_32_bits(src);
1165 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1166 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1167
1168 }
1169
1170 /**
1171 * sdma_v4_4_2_vm_write_pte - update PTEs by writing them manually
1172 *
1173 * @ib: indirect buffer to fill with commands
1174 * @pe: addr of the page entry
1175 * @value: dst addr to write into pe
1176 * @count: number of page entries to update
1177 * @incr: increase next addr by incr bytes
1178 *
1179 * Update PTEs by writing them manually using sDMA.
1180 */
sdma_v4_4_2_vm_write_pte(struct amdgpu_ib * ib,uint64_t pe,uint64_t value,unsigned count,uint32_t incr)1181 static void sdma_v4_4_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1182 uint64_t value, unsigned count,
1183 uint32_t incr)
1184 {
1185 unsigned ndw = count * 2;
1186
1187 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1188 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1189 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1190 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1191 ib->ptr[ib->length_dw++] = ndw - 1;
1192 for (; ndw > 0; ndw -= 2) {
1193 ib->ptr[ib->length_dw++] = lower_32_bits(value);
1194 ib->ptr[ib->length_dw++] = upper_32_bits(value);
1195 value += incr;
1196 }
1197 }
1198
1199 /**
1200 * sdma_v4_4_2_vm_set_pte_pde - update the page tables using sDMA
1201 *
1202 * @ib: indirect buffer to fill with commands
1203 * @pe: addr of the page entry
1204 * @addr: dst addr to write into pe
1205 * @count: number of page entries to update
1206 * @incr: increase next addr by incr bytes
1207 * @flags: access flags
1208 *
1209 * Update the page tables using sDMA.
1210 */
sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib * ib,uint64_t pe,uint64_t addr,unsigned count,uint32_t incr,uint64_t flags)1211 static void sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1212 uint64_t pe,
1213 uint64_t addr, unsigned count,
1214 uint32_t incr, uint64_t flags)
1215 {
1216 /* for physically contiguous pages (vram) */
1217 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1218 ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1219 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1220 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1221 ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1222 ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1223 ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1224 ib->ptr[ib->length_dw++] = incr; /* increment size */
1225 ib->ptr[ib->length_dw++] = 0;
1226 ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1227 }
1228
1229 /**
1230 * sdma_v4_4_2_ring_pad_ib - pad the IB to the required number of dw
1231 *
1232 * @ring: amdgpu_ring structure holding ring information
1233 * @ib: indirect buffer to fill with padding
1234 */
sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring * ring,struct amdgpu_ib * ib)1235 static void sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1236 {
1237 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1238 u32 pad_count;
1239 int i;
1240
1241 pad_count = (-ib->length_dw) & 7;
1242 for (i = 0; i < pad_count; i++)
1243 if (sdma && sdma->burst_nop && (i == 0))
1244 ib->ptr[ib->length_dw++] =
1245 SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1246 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1247 else
1248 ib->ptr[ib->length_dw++] =
1249 SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1250 }
1251
1252
1253 /**
1254 * sdma_v4_4_2_ring_emit_pipeline_sync - sync the pipeline
1255 *
1256 * @ring: amdgpu_ring pointer
1257 *
1258 * Make sure all previous operations are completed (CIK).
1259 */
sdma_v4_4_2_ring_emit_pipeline_sync(struct amdgpu_ring * ring)1260 static void sdma_v4_4_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1261 {
1262 uint32_t seq = ring->fence_drv.sync_seq;
1263 uint64_t addr = ring->fence_drv.gpu_addr;
1264
1265 /* wait for idle */
1266 sdma_v4_4_2_wait_reg_mem(ring, 1, 0,
1267 addr & 0xfffffffc,
1268 upper_32_bits(addr) & 0xffffffff,
1269 seq, 0xffffffff, 4);
1270 }
1271
1272
1273 /**
1274 * sdma_v4_4_2_ring_emit_vm_flush - vm flush using sDMA
1275 *
1276 * @ring: amdgpu_ring pointer
1277 * @vmid: vmid number to use
1278 * @pd_addr: address
1279 *
1280 * Update the page table base and flush the VM TLB
1281 * using sDMA.
1282 */
sdma_v4_4_2_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)1283 static void sdma_v4_4_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1284 unsigned vmid, uint64_t pd_addr)
1285 {
1286 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1287 }
1288
sdma_v4_4_2_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)1289 static void sdma_v4_4_2_ring_emit_wreg(struct amdgpu_ring *ring,
1290 uint32_t reg, uint32_t val)
1291 {
1292 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1293 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1294 amdgpu_ring_write(ring, reg);
1295 amdgpu_ring_write(ring, val);
1296 }
1297
sdma_v4_4_2_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)1298 static void sdma_v4_4_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1299 uint32_t val, uint32_t mask)
1300 {
1301 sdma_v4_4_2_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);
1302 }
1303
sdma_v4_4_2_fw_support_paging_queue(struct amdgpu_device * adev)1304 static bool sdma_v4_4_2_fw_support_paging_queue(struct amdgpu_device *adev)
1305 {
1306 switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1307 case IP_VERSION(4, 4, 2):
1308 case IP_VERSION(4, 4, 5):
1309 return false;
1310 default:
1311 return false;
1312 }
1313 }
1314
sdma_v4_4_2_early_init(struct amdgpu_ip_block * ip_block)1315 static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
1316 {
1317 struct amdgpu_device *adev = ip_block->adev;
1318 int r;
1319
1320 r = sdma_v4_4_2_init_microcode(adev);
1321 if (r)
1322 return r;
1323
1324 /* TODO: Page queue breaks driver reload under SRIOV */
1325 if (sdma_v4_4_2_fw_support_paging_queue(adev))
1326 adev->sdma.has_page_queue = true;
1327
1328 sdma_v4_4_2_set_ring_funcs(adev);
1329 sdma_v4_4_2_set_buffer_funcs(adev);
1330 sdma_v4_4_2_set_vm_pte_funcs(adev);
1331 sdma_v4_4_2_set_irq_funcs(adev);
1332 sdma_v4_4_2_set_ras_funcs(adev);
1333
1334 return 0;
1335 }
1336
1337 #if 0
1338 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1339 void *err_data,
1340 struct amdgpu_iv_entry *entry);
1341 #endif
1342
sdma_v4_4_2_late_init(struct amdgpu_ip_block * ip_block)1343 static int sdma_v4_4_2_late_init(struct amdgpu_ip_block *ip_block)
1344 {
1345 struct amdgpu_device *adev = ip_block->adev;
1346 #if 0
1347 struct ras_ih_if ih_info = {
1348 .cb = sdma_v4_4_2_process_ras_data_cb,
1349 };
1350 #endif
1351 if (!amdgpu_persistent_edc_harvesting_supported(adev))
1352 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__SDMA);
1353
1354 return 0;
1355 }
1356
sdma_v4_4_2_sw_init(struct amdgpu_ip_block * ip_block)1357 static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block)
1358 {
1359 struct amdgpu_ring *ring;
1360 int r, i;
1361 struct amdgpu_device *adev = ip_block->adev;
1362 u32 aid_id;
1363 uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
1364 uint32_t *ptr;
1365
1366 /* SDMA trap event */
1367 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1368 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1369 SDMA0_4_0__SRCID__SDMA_TRAP,
1370 &adev->sdma.trap_irq);
1371 if (r)
1372 return r;
1373 }
1374
1375 /* SDMA SRAM ECC event */
1376 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1377 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1378 SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
1379 &adev->sdma.ecc_irq);
1380 if (r)
1381 return r;
1382 }
1383
1384 /* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/
1385 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1386 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1387 SDMA0_4_0__SRCID__SDMA_VM_HOLE,
1388 &adev->sdma.vm_hole_irq);
1389 if (r)
1390 return r;
1391
1392 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1393 SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,
1394 &adev->sdma.doorbell_invalid_irq);
1395 if (r)
1396 return r;
1397
1398 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1399 SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,
1400 &adev->sdma.pool_timeout_irq);
1401 if (r)
1402 return r;
1403
1404 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1405 SDMA0_4_0__SRCID__SDMA_SRBMWRITE,
1406 &adev->sdma.srbm_write_irq);
1407 if (r)
1408 return r;
1409
1410 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1411 SDMA0_4_0__SRCID__SDMA_CTXEMPTY,
1412 &adev->sdma.ctxt_empty_irq);
1413 if (r)
1414 return r;
1415 }
1416
1417 for (i = 0; i < adev->sdma.num_instances; i++) {
1418 ring = &adev->sdma.instance[i].ring;
1419 ring->ring_obj = NULL;
1420 ring->use_doorbell = true;
1421 aid_id = adev->sdma.instance[i].aid_id;
1422
1423 DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1424 ring->use_doorbell?"true":"false");
1425
1426 /* doorbell size is 2 dwords, get DWORD offset */
1427 ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
1428 ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1429
1430 sprintf(ring->name, "sdma%d.%d", aid_id,
1431 i % adev->sdma.num_inst_per_aid);
1432 r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1433 AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1434 AMDGPU_RING_PRIO_DEFAULT, NULL);
1435 if (r)
1436 return r;
1437
1438 if (adev->sdma.has_page_queue) {
1439 ring = &adev->sdma.instance[i].page;
1440 ring->ring_obj = NULL;
1441 ring->use_doorbell = true;
1442
1443 /* doorbell index of page queue is assigned right after
1444 * gfx queue on the same instance
1445 */
1446 ring->doorbell_index =
1447 (adev->doorbell_index.sdma_engine[i] + 1) << 1;
1448 ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1449
1450 sprintf(ring->name, "page%d.%d", aid_id,
1451 i % adev->sdma.num_inst_per_aid);
1452 r = amdgpu_ring_init(adev, ring, 1024,
1453 &adev->sdma.trap_irq,
1454 AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1455 AMDGPU_RING_PRIO_DEFAULT, NULL);
1456 if (r)
1457 return r;
1458 }
1459 }
1460
1461 /* TODO: Add queue reset mask when FW fully supports it */
1462 adev->sdma.supported_reset =
1463 amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
1464
1465 if (amdgpu_sdma_ras_sw_init(adev)) {
1466 dev_err(adev->dev, "fail to initialize sdma ras block\n");
1467 return -EINVAL;
1468 }
1469
1470 /* Allocate memory for SDMA IP Dump buffer */
1471 ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
1472 if (ptr)
1473 adev->sdma.ip_dump = ptr;
1474 else
1475 DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1476
1477 r = amdgpu_sdma_sysfs_reset_mask_init(adev);
1478 if (r)
1479 return r;
1480
1481 return r;
1482 }
1483
sdma_v4_4_2_sw_fini(struct amdgpu_ip_block * ip_block)1484 static int sdma_v4_4_2_sw_fini(struct amdgpu_ip_block *ip_block)
1485 {
1486 struct amdgpu_device *adev = ip_block->adev;
1487 int i;
1488
1489 for (i = 0; i < adev->sdma.num_instances; i++) {
1490 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1491 if (adev->sdma.has_page_queue)
1492 amdgpu_ring_fini(&adev->sdma.instance[i].page);
1493 }
1494
1495 amdgpu_sdma_sysfs_reset_mask_fini(adev);
1496 if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 2) ||
1497 amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 4) ||
1498 amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 5))
1499 amdgpu_sdma_destroy_inst_ctx(adev, true);
1500 else
1501 amdgpu_sdma_destroy_inst_ctx(adev, false);
1502
1503 kfree(adev->sdma.ip_dump);
1504
1505 return 0;
1506 }
1507
sdma_v4_4_2_hw_init(struct amdgpu_ip_block * ip_block)1508 static int sdma_v4_4_2_hw_init(struct amdgpu_ip_block *ip_block)
1509 {
1510 int r;
1511 struct amdgpu_device *adev = ip_block->adev;
1512 uint32_t inst_mask;
1513
1514 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1515 if (!amdgpu_sriov_vf(adev))
1516 sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
1517
1518 r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
1519
1520 return r;
1521 }
1522
sdma_v4_4_2_hw_fini(struct amdgpu_ip_block * ip_block)1523 static int sdma_v4_4_2_hw_fini(struct amdgpu_ip_block *ip_block)
1524 {
1525 struct amdgpu_device *adev = ip_block->adev;
1526 uint32_t inst_mask;
1527 int i;
1528
1529 if (amdgpu_sriov_vf(adev))
1530 return 0;
1531
1532 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1533 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
1534 for (i = 0; i < adev->sdma.num_instances; i++) {
1535 amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
1536 AMDGPU_SDMA_IRQ_INSTANCE0 + i);
1537 }
1538 }
1539
1540 sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
1541 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
1542
1543 return 0;
1544 }
1545
1546 static int sdma_v4_4_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1547 enum amd_clockgating_state state);
1548
sdma_v4_4_2_suspend(struct amdgpu_ip_block * ip_block)1549 static int sdma_v4_4_2_suspend(struct amdgpu_ip_block *ip_block)
1550 {
1551 struct amdgpu_device *adev = ip_block->adev;
1552
1553 if (amdgpu_in_reset(adev))
1554 sdma_v4_4_2_set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE);
1555
1556 return sdma_v4_4_2_hw_fini(ip_block);
1557 }
1558
sdma_v4_4_2_resume(struct amdgpu_ip_block * ip_block)1559 static int sdma_v4_4_2_resume(struct amdgpu_ip_block *ip_block)
1560 {
1561 return sdma_v4_4_2_hw_init(ip_block);
1562 }
1563
sdma_v4_4_2_is_idle(void * handle)1564 static bool sdma_v4_4_2_is_idle(void *handle)
1565 {
1566 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1567 u32 i;
1568
1569 for (i = 0; i < adev->sdma.num_instances; i++) {
1570 u32 tmp = RREG32_SDMA(i, regSDMA_STATUS_REG);
1571
1572 if (!(tmp & SDMA_STATUS_REG__IDLE_MASK))
1573 return false;
1574 }
1575
1576 return true;
1577 }
1578
sdma_v4_4_2_wait_for_idle(struct amdgpu_ip_block * ip_block)1579 static int sdma_v4_4_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
1580 {
1581 unsigned i, j;
1582 u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
1583 struct amdgpu_device *adev = ip_block->adev;
1584
1585 for (i = 0; i < adev->usec_timeout; i++) {
1586 for (j = 0; j < adev->sdma.num_instances; j++) {
1587 sdma[j] = RREG32_SDMA(j, regSDMA_STATUS_REG);
1588 if (!(sdma[j] & SDMA_STATUS_REG__IDLE_MASK))
1589 break;
1590 }
1591 if (j == adev->sdma.num_instances)
1592 return 0;
1593 udelay(1);
1594 }
1595 return -ETIMEDOUT;
1596 }
1597
sdma_v4_4_2_soft_reset(struct amdgpu_ip_block * ip_block)1598 static int sdma_v4_4_2_soft_reset(struct amdgpu_ip_block *ip_block)
1599 {
1600 /* todo */
1601
1602 return 0;
1603 }
1604
sdma_v4_4_2_reset_queue(struct amdgpu_ring * ring,unsigned int vmid)1605 static int sdma_v4_4_2_reset_queue(struct amdgpu_ring *ring, unsigned int vmid)
1606 {
1607 struct amdgpu_device *adev = ring->adev;
1608 int i, r;
1609 u32 inst_mask;
1610
1611 if (amdgpu_sriov_vf(adev))
1612 return -EINVAL;
1613
1614 /* stop queue */
1615 inst_mask = 1 << ring->me;
1616 sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
1617 if (adev->sdma.has_page_queue)
1618 sdma_v4_4_2_inst_page_stop(adev, inst_mask);
1619
1620 r = amdgpu_dpm_reset_sdma(adev, 1 << GET_INST(SDMA0, ring->me));
1621 if (r)
1622 return r;
1623
1624 udelay(50);
1625
1626 for (i = 0; i < adev->usec_timeout; i++) {
1627 if (!REG_GET_FIELD(RREG32_SDMA(ring->me, regSDMA_F32_CNTL), SDMA_F32_CNTL, HALT))
1628 break;
1629 udelay(1);
1630 }
1631
1632 if (i == adev->usec_timeout) {
1633 dev_err(adev->dev, "timed out waiting for SDMA%d unhalt after reset\n",
1634 ring->me);
1635 return -ETIMEDOUT;
1636 }
1637
1638 return sdma_v4_4_2_inst_start(adev, inst_mask, true);
1639 }
1640
sdma_v4_4_2_set_trap_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)1641 static int sdma_v4_4_2_set_trap_irq_state(struct amdgpu_device *adev,
1642 struct amdgpu_irq_src *source,
1643 unsigned type,
1644 enum amdgpu_interrupt_state state)
1645 {
1646 u32 sdma_cntl;
1647
1648 sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1649 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, TRAP_ENABLE,
1650 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1651 WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1652
1653 return 0;
1654 }
1655
sdma_v4_4_2_process_trap_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1656 static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
1657 struct amdgpu_irq_src *source,
1658 struct amdgpu_iv_entry *entry)
1659 {
1660 uint32_t instance, i;
1661
1662 DRM_DEBUG("IH: SDMA trap\n");
1663 instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1664
1665 /* Client id gives the SDMA instance in AID. To know the exact SDMA
1666 * instance, interrupt entry gives the node id which corresponds to the AID instance.
1667 * Match node id with the AID id associated with the SDMA instance. */
1668 for (i = instance; i < adev->sdma.num_instances;
1669 i += adev->sdma.num_inst_per_aid) {
1670 if (adev->sdma.instance[i].aid_id ==
1671 node_id_to_phys_map[entry->node_id])
1672 break;
1673 }
1674
1675 if (i >= adev->sdma.num_instances) {
1676 dev_WARN_ONCE(
1677 adev->dev, 1,
1678 "Couldn't find the right sdma instance in trap handler");
1679 return 0;
1680 }
1681
1682 switch (entry->ring_id) {
1683 case 0:
1684 amdgpu_fence_process(&adev->sdma.instance[i].ring);
1685 break;
1686 default:
1687 break;
1688 }
1689 return 0;
1690 }
1691
1692 #if 0
1693 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1694 void *err_data,
1695 struct amdgpu_iv_entry *entry)
1696 {
1697 int instance;
1698
1699 /* When “Full RAS” is enabled, the per-IP interrupt sources should
1700 * be disabled and the driver should only look for the aggregated
1701 * interrupt via sync flood
1702 */
1703 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
1704 goto out;
1705
1706 instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1707 if (instance < 0)
1708 goto out;
1709
1710 amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);
1711
1712 out:
1713 return AMDGPU_RAS_SUCCESS;
1714 }
1715 #endif
1716
sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1717 static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1718 struct amdgpu_irq_src *source,
1719 struct amdgpu_iv_entry *entry)
1720 {
1721 int instance;
1722
1723 DRM_ERROR("Illegal instruction in SDMA command stream\n");
1724
1725 instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1726 if (instance < 0)
1727 return 0;
1728
1729 switch (entry->ring_id) {
1730 case 0:
1731 drm_sched_fault(&adev->sdma.instance[instance].ring.sched);
1732 break;
1733 }
1734 return 0;
1735 }
1736
sdma_v4_4_2_set_ecc_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)1737 static int sdma_v4_4_2_set_ecc_irq_state(struct amdgpu_device *adev,
1738 struct amdgpu_irq_src *source,
1739 unsigned type,
1740 enum amdgpu_interrupt_state state)
1741 {
1742 u32 sdma_cntl;
1743
1744 sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1745 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, DRAM_ECC_INT_ENABLE,
1746 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1747 WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1748
1749 return 0;
1750 }
1751
sdma_v4_4_2_print_iv_entry(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)1752 static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
1753 struct amdgpu_iv_entry *entry)
1754 {
1755 int instance;
1756 struct amdgpu_task_info *task_info;
1757 u64 addr;
1758
1759 instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1760 if (instance < 0 || instance >= adev->sdma.num_instances) {
1761 dev_err(adev->dev, "sdma instance invalid %d\n", instance);
1762 return -EINVAL;
1763 }
1764
1765 addr = (u64)entry->src_data[0] << 12;
1766 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
1767
1768 dev_dbg_ratelimited(adev->dev,
1769 "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u pasid:%u\n",
1770 instance, addr, entry->src_id, entry->ring_id, entry->vmid,
1771 entry->pasid);
1772
1773 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
1774 if (task_info) {
1775 dev_dbg_ratelimited(adev->dev, " for process %s pid %d thread %s pid %d\n",
1776 task_info->process_name, task_info->tgid,
1777 task_info->task_name, task_info->pid);
1778 amdgpu_vm_put_task_info(task_info);
1779 }
1780
1781 return 0;
1782 }
1783
sdma_v4_4_2_process_vm_hole_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1784 static int sdma_v4_4_2_process_vm_hole_irq(struct amdgpu_device *adev,
1785 struct amdgpu_irq_src *source,
1786 struct amdgpu_iv_entry *entry)
1787 {
1788 dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
1789 sdma_v4_4_2_print_iv_entry(adev, entry);
1790 return 0;
1791 }
1792
sdma_v4_4_2_process_doorbell_invalid_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1793 static int sdma_v4_4_2_process_doorbell_invalid_irq(struct amdgpu_device *adev,
1794 struct amdgpu_irq_src *source,
1795 struct amdgpu_iv_entry *entry)
1796 {
1797
1798 dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
1799 sdma_v4_4_2_print_iv_entry(adev, entry);
1800 return 0;
1801 }
1802
sdma_v4_4_2_process_pool_timeout_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1803 static int sdma_v4_4_2_process_pool_timeout_irq(struct amdgpu_device *adev,
1804 struct amdgpu_irq_src *source,
1805 struct amdgpu_iv_entry *entry)
1806 {
1807 dev_dbg_ratelimited(adev->dev,
1808 "Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
1809 sdma_v4_4_2_print_iv_entry(adev, entry);
1810 return 0;
1811 }
1812
sdma_v4_4_2_process_srbm_write_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1813 static int sdma_v4_4_2_process_srbm_write_irq(struct amdgpu_device *adev,
1814 struct amdgpu_irq_src *source,
1815 struct amdgpu_iv_entry *entry)
1816 {
1817 dev_dbg_ratelimited(adev->dev,
1818 "SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
1819 sdma_v4_4_2_print_iv_entry(adev, entry);
1820 return 0;
1821 }
1822
sdma_v4_4_2_process_ctxt_empty_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1823 static int sdma_v4_4_2_process_ctxt_empty_irq(struct amdgpu_device *adev,
1824 struct amdgpu_irq_src *source,
1825 struct amdgpu_iv_entry *entry)
1826 {
1827 /* There is nothing useful to be done here, only kept for debug */
1828 dev_dbg_ratelimited(adev->dev, "SDMA context empty interrupt");
1829 sdma_v4_4_2_print_iv_entry(adev, entry);
1830 return 0;
1831 }
1832
sdma_v4_4_2_inst_update_medium_grain_light_sleep(struct amdgpu_device * adev,bool enable,uint32_t inst_mask)1833 static void sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1834 struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1835 {
1836 uint32_t data, def;
1837 int i;
1838
1839 /* leave as default if it is not driver controlled */
1840 if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS))
1841 return;
1842
1843 if (enable) {
1844 for_each_inst(i, inst_mask) {
1845 /* 1-not override: enable sdma mem light sleep */
1846 def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1847 data |= SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1848 if (def != data)
1849 WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1850 }
1851 } else {
1852 for_each_inst(i, inst_mask) {
1853 /* 0-override:disable sdma mem light sleep */
1854 def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1855 data &= ~SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1856 if (def != data)
1857 WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1858 }
1859 }
1860 }
1861
sdma_v4_4_2_inst_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable,uint32_t inst_mask)1862 static void sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1863 struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1864 {
1865 uint32_t data, def;
1866 int i;
1867
1868 /* leave as default if it is not driver controlled */
1869 if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG))
1870 return;
1871
1872 if (enable) {
1873 for_each_inst(i, inst_mask) {
1874 def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1875 data &= ~(SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1876 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1877 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1878 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1879 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1880 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1881 if (def != data)
1882 WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1883 }
1884 } else {
1885 for_each_inst(i, inst_mask) {
1886 def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1887 data |= (SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1888 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1889 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1890 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1891 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1892 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1893 if (def != data)
1894 WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1895 }
1896 }
1897 }
1898
sdma_v4_4_2_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)1899 static int sdma_v4_4_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1900 enum amd_clockgating_state state)
1901 {
1902 struct amdgpu_device *adev = ip_block->adev;
1903 uint32_t inst_mask;
1904
1905 if (amdgpu_sriov_vf(adev))
1906 return 0;
1907
1908 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1909
1910 sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1911 adev, state == AMD_CG_STATE_GATE, inst_mask);
1912 sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1913 adev, state == AMD_CG_STATE_GATE, inst_mask);
1914 return 0;
1915 }
1916
sdma_v4_4_2_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)1917 static int sdma_v4_4_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
1918 enum amd_powergating_state state)
1919 {
1920 return 0;
1921 }
1922
sdma_v4_4_2_get_clockgating_state(void * handle,u64 * flags)1923 static void sdma_v4_4_2_get_clockgating_state(void *handle, u64 *flags)
1924 {
1925 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1926 int data;
1927
1928 if (amdgpu_sriov_vf(adev))
1929 *flags = 0;
1930
1931 /* AMD_CG_SUPPORT_SDMA_MGCG */
1932 data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_CLK_CTRL));
1933 if (!(data & SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK))
1934 *flags |= AMD_CG_SUPPORT_SDMA_MGCG;
1935
1936 /* AMD_CG_SUPPORT_SDMA_LS */
1937 data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_POWER_CNTL));
1938 if (data & SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
1939 *flags |= AMD_CG_SUPPORT_SDMA_LS;
1940 }
1941
sdma_v4_4_2_print_ip_state(struct amdgpu_ip_block * ip_block,struct drm_printer * p)1942 static void sdma_v4_4_2_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
1943 {
1944 struct amdgpu_device *adev = ip_block->adev;
1945 int i, j;
1946 uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
1947 uint32_t instance_offset;
1948
1949 if (!adev->sdma.ip_dump)
1950 return;
1951
1952 drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
1953 for (i = 0; i < adev->sdma.num_instances; i++) {
1954 instance_offset = i * reg_count;
1955 drm_printf(p, "\nInstance:%d\n", i);
1956
1957 for (j = 0; j < reg_count; j++)
1958 drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_4_4_2[j].reg_name,
1959 adev->sdma.ip_dump[instance_offset + j]);
1960 }
1961 }
1962
sdma_v4_4_2_dump_ip_state(struct amdgpu_ip_block * ip_block)1963 static void sdma_v4_4_2_dump_ip_state(struct amdgpu_ip_block *ip_block)
1964 {
1965 struct amdgpu_device *adev = ip_block->adev;
1966 int i, j;
1967 uint32_t instance_offset;
1968 uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
1969
1970 if (!adev->sdma.ip_dump)
1971 return;
1972
1973 for (i = 0; i < adev->sdma.num_instances; i++) {
1974 instance_offset = i * reg_count;
1975 for (j = 0; j < reg_count; j++)
1976 adev->sdma.ip_dump[instance_offset + j] =
1977 RREG32(sdma_v4_4_2_get_reg_offset(adev, i,
1978 sdma_reg_list_4_4_2[j].reg_offset));
1979 }
1980 }
1981
1982 const struct amd_ip_funcs sdma_v4_4_2_ip_funcs = {
1983 .name = "sdma_v4_4_2",
1984 .early_init = sdma_v4_4_2_early_init,
1985 .late_init = sdma_v4_4_2_late_init,
1986 .sw_init = sdma_v4_4_2_sw_init,
1987 .sw_fini = sdma_v4_4_2_sw_fini,
1988 .hw_init = sdma_v4_4_2_hw_init,
1989 .hw_fini = sdma_v4_4_2_hw_fini,
1990 .suspend = sdma_v4_4_2_suspend,
1991 .resume = sdma_v4_4_2_resume,
1992 .is_idle = sdma_v4_4_2_is_idle,
1993 .wait_for_idle = sdma_v4_4_2_wait_for_idle,
1994 .soft_reset = sdma_v4_4_2_soft_reset,
1995 .set_clockgating_state = sdma_v4_4_2_set_clockgating_state,
1996 .set_powergating_state = sdma_v4_4_2_set_powergating_state,
1997 .get_clockgating_state = sdma_v4_4_2_get_clockgating_state,
1998 .dump_ip_state = sdma_v4_4_2_dump_ip_state,
1999 .print_ip_state = sdma_v4_4_2_print_ip_state,
2000 };
2001
2002 static const struct amdgpu_ring_funcs sdma_v4_4_2_ring_funcs = {
2003 .type = AMDGPU_RING_TYPE_SDMA,
2004 .align_mask = 0xff,
2005 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
2006 .support_64bit_ptrs = true,
2007 .get_rptr = sdma_v4_4_2_ring_get_rptr,
2008 .get_wptr = sdma_v4_4_2_ring_get_wptr,
2009 .set_wptr = sdma_v4_4_2_ring_set_wptr,
2010 .emit_frame_size =
2011 6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
2012 3 + /* hdp invalidate */
2013 6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
2014 /* sdma_v4_4_2_ring_emit_vm_flush */
2015 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2016 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
2017 10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
2018 .emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
2019 .emit_ib = sdma_v4_4_2_ring_emit_ib,
2020 .emit_fence = sdma_v4_4_2_ring_emit_fence,
2021 .emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
2022 .emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
2023 .emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
2024 .test_ring = sdma_v4_4_2_ring_test_ring,
2025 .test_ib = sdma_v4_4_2_ring_test_ib,
2026 .insert_nop = sdma_v4_4_2_ring_insert_nop,
2027 .pad_ib = sdma_v4_4_2_ring_pad_ib,
2028 .emit_wreg = sdma_v4_4_2_ring_emit_wreg,
2029 .emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
2030 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2031 .reset = sdma_v4_4_2_reset_queue,
2032 };
2033
2034 static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
2035 .type = AMDGPU_RING_TYPE_SDMA,
2036 .align_mask = 0xff,
2037 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
2038 .support_64bit_ptrs = true,
2039 .get_rptr = sdma_v4_4_2_ring_get_rptr,
2040 .get_wptr = sdma_v4_4_2_page_ring_get_wptr,
2041 .set_wptr = sdma_v4_4_2_page_ring_set_wptr,
2042 .emit_frame_size =
2043 6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
2044 3 + /* hdp invalidate */
2045 6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
2046 /* sdma_v4_4_2_ring_emit_vm_flush */
2047 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2048 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
2049 10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
2050 .emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
2051 .emit_ib = sdma_v4_4_2_ring_emit_ib,
2052 .emit_fence = sdma_v4_4_2_ring_emit_fence,
2053 .emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
2054 .emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
2055 .emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
2056 .test_ring = sdma_v4_4_2_ring_test_ring,
2057 .test_ib = sdma_v4_4_2_ring_test_ib,
2058 .insert_nop = sdma_v4_4_2_ring_insert_nop,
2059 .pad_ib = sdma_v4_4_2_ring_pad_ib,
2060 .emit_wreg = sdma_v4_4_2_ring_emit_wreg,
2061 .emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
2062 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2063 };
2064
sdma_v4_4_2_set_ring_funcs(struct amdgpu_device * adev)2065 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev)
2066 {
2067 int i, dev_inst;
2068
2069 for (i = 0; i < adev->sdma.num_instances; i++) {
2070 adev->sdma.instance[i].ring.funcs = &sdma_v4_4_2_ring_funcs;
2071 adev->sdma.instance[i].ring.me = i;
2072 if (adev->sdma.has_page_queue) {
2073 adev->sdma.instance[i].page.funcs =
2074 &sdma_v4_4_2_page_ring_funcs;
2075 adev->sdma.instance[i].page.me = i;
2076 }
2077
2078 dev_inst = GET_INST(SDMA0, i);
2079 /* AID to which SDMA belongs depends on physical instance */
2080 adev->sdma.instance[i].aid_id =
2081 dev_inst / adev->sdma.num_inst_per_aid;
2082 }
2083 }
2084
2085 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_trap_irq_funcs = {
2086 .set = sdma_v4_4_2_set_trap_irq_state,
2087 .process = sdma_v4_4_2_process_trap_irq,
2088 };
2089
2090 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_illegal_inst_irq_funcs = {
2091 .process = sdma_v4_4_2_process_illegal_inst_irq,
2092 };
2093
2094 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ecc_irq_funcs = {
2095 .set = sdma_v4_4_2_set_ecc_irq_state,
2096 .process = amdgpu_sdma_process_ecc_irq,
2097 };
2098
2099 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_vm_hole_irq_funcs = {
2100 .process = sdma_v4_4_2_process_vm_hole_irq,
2101 };
2102
2103 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_doorbell_invalid_irq_funcs = {
2104 .process = sdma_v4_4_2_process_doorbell_invalid_irq,
2105 };
2106
2107 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_pool_timeout_irq_funcs = {
2108 .process = sdma_v4_4_2_process_pool_timeout_irq,
2109 };
2110
2111 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_srbm_write_irq_funcs = {
2112 .process = sdma_v4_4_2_process_srbm_write_irq,
2113 };
2114
2115 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ctxt_empty_irq_funcs = {
2116 .process = sdma_v4_4_2_process_ctxt_empty_irq,
2117 };
2118
sdma_v4_4_2_set_irq_funcs(struct amdgpu_device * adev)2119 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev)
2120 {
2121 adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
2122 adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
2123 adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
2124 adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
2125 adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
2126 adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
2127 adev->sdma.ctxt_empty_irq.num_types = adev->sdma.num_instances;
2128
2129 adev->sdma.trap_irq.funcs = &sdma_v4_4_2_trap_irq_funcs;
2130 adev->sdma.illegal_inst_irq.funcs = &sdma_v4_4_2_illegal_inst_irq_funcs;
2131 adev->sdma.ecc_irq.funcs = &sdma_v4_4_2_ecc_irq_funcs;
2132 adev->sdma.vm_hole_irq.funcs = &sdma_v4_4_2_vm_hole_irq_funcs;
2133 adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_4_2_doorbell_invalid_irq_funcs;
2134 adev->sdma.pool_timeout_irq.funcs = &sdma_v4_4_2_pool_timeout_irq_funcs;
2135 adev->sdma.srbm_write_irq.funcs = &sdma_v4_4_2_srbm_write_irq_funcs;
2136 adev->sdma.ctxt_empty_irq.funcs = &sdma_v4_4_2_ctxt_empty_irq_funcs;
2137 }
2138
2139 /**
2140 * sdma_v4_4_2_emit_copy_buffer - copy buffer using the sDMA engine
2141 *
2142 * @ib: indirect buffer to copy to
2143 * @src_offset: src GPU address
2144 * @dst_offset: dst GPU address
2145 * @byte_count: number of bytes to xfer
2146 * @copy_flags: copy flags for the buffers
2147 *
2148 * Copy GPU buffers using the DMA engine.
2149 * Used by the amdgpu ttm implementation to move pages if
2150 * registered as the asic copy callback.
2151 */
sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib * ib,uint64_t src_offset,uint64_t dst_offset,uint32_t byte_count,uint32_t copy_flags)2152 static void sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib *ib,
2153 uint64_t src_offset,
2154 uint64_t dst_offset,
2155 uint32_t byte_count,
2156 uint32_t copy_flags)
2157 {
2158 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
2159 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
2160 SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
2161 ib->ptr[ib->length_dw++] = byte_count - 1;
2162 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
2163 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
2164 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
2165 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2166 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2167 }
2168
2169 /**
2170 * sdma_v4_4_2_emit_fill_buffer - fill buffer using the sDMA engine
2171 *
2172 * @ib: indirect buffer to copy to
2173 * @src_data: value to write to buffer
2174 * @dst_offset: dst GPU address
2175 * @byte_count: number of bytes to xfer
2176 *
2177 * Fill GPU buffers using the DMA engine.
2178 */
sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib * ib,uint32_t src_data,uint64_t dst_offset,uint32_t byte_count)2179 static void sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib *ib,
2180 uint32_t src_data,
2181 uint64_t dst_offset,
2182 uint32_t byte_count)
2183 {
2184 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
2185 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2186 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2187 ib->ptr[ib->length_dw++] = src_data;
2188 ib->ptr[ib->length_dw++] = byte_count - 1;
2189 }
2190
2191 static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
2192 .copy_max_bytes = 0x400000,
2193 .copy_num_dw = 7,
2194 .emit_copy_buffer = sdma_v4_4_2_emit_copy_buffer,
2195
2196 .fill_max_bytes = 0x400000,
2197 .fill_num_dw = 5,
2198 .emit_fill_buffer = sdma_v4_4_2_emit_fill_buffer,
2199 };
2200
sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device * adev)2201 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
2202 {
2203 adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
2204 if (adev->sdma.has_page_queue)
2205 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
2206 else
2207 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
2208 }
2209
2210 static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
2211 .copy_pte_num_dw = 7,
2212 .copy_pte = sdma_v4_4_2_vm_copy_pte,
2213
2214 .write_pte = sdma_v4_4_2_vm_write_pte,
2215 .set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
2216 };
2217
sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device * adev)2218 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev)
2219 {
2220 struct drm_gpu_scheduler *sched;
2221 unsigned i;
2222
2223 adev->vm_manager.vm_pte_funcs = &sdma_v4_4_2_vm_pte_funcs;
2224 for (i = 0; i < adev->sdma.num_instances; i++) {
2225 if (adev->sdma.has_page_queue)
2226 sched = &adev->sdma.instance[i].page.sched;
2227 else
2228 sched = &adev->sdma.instance[i].ring.sched;
2229 adev->vm_manager.vm_pte_scheds[i] = sched;
2230 }
2231 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
2232 }
2233
2234 const struct amdgpu_ip_block_version sdma_v4_4_2_ip_block = {
2235 .type = AMD_IP_BLOCK_TYPE_SDMA,
2236 .major = 4,
2237 .minor = 4,
2238 .rev = 2,
2239 .funcs = &sdma_v4_4_2_ip_funcs,
2240 };
2241
sdma_v4_4_2_xcp_resume(void * handle,uint32_t inst_mask)2242 static int sdma_v4_4_2_xcp_resume(void *handle, uint32_t inst_mask)
2243 {
2244 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2245 int r;
2246
2247 if (!amdgpu_sriov_vf(adev))
2248 sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
2249
2250 r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
2251
2252 return r;
2253 }
2254
sdma_v4_4_2_xcp_suspend(void * handle,uint32_t inst_mask)2255 static int sdma_v4_4_2_xcp_suspend(void *handle, uint32_t inst_mask)
2256 {
2257 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2258 uint32_t tmp_mask = inst_mask;
2259 int i;
2260
2261 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2262 for_each_inst(i, tmp_mask) {
2263 amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
2264 AMDGPU_SDMA_IRQ_INSTANCE0 + i);
2265 }
2266 }
2267
2268 sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
2269 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
2270
2271 return 0;
2272 }
2273
2274 struct amdgpu_xcp_ip_funcs sdma_v4_4_2_xcp_funcs = {
2275 .suspend = &sdma_v4_4_2_xcp_suspend,
2276 .resume = &sdma_v4_4_2_xcp_resume
2277 };
2278
2279 static const struct amdgpu_ras_err_status_reg_entry sdma_v4_2_2_ue_reg_list[] = {
2280 {AMDGPU_RAS_REG_ENTRY(SDMA0, 0, regSDMA_UE_ERR_STATUS_LO, regSDMA_UE_ERR_STATUS_HI),
2281 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"},
2282 };
2283
2284 static const struct amdgpu_ras_memory_id_entry sdma_v4_4_2_ras_memory_list[] = {
2285 {AMDGPU_SDMA_MBANK_DATA_BUF0, "SDMA_MBANK_DATA_BUF0"},
2286 {AMDGPU_SDMA_MBANK_DATA_BUF1, "SDMA_MBANK_DATA_BUF1"},
2287 {AMDGPU_SDMA_MBANK_DATA_BUF2, "SDMA_MBANK_DATA_BUF2"},
2288 {AMDGPU_SDMA_MBANK_DATA_BUF3, "SDMA_MBANK_DATA_BUF3"},
2289 {AMDGPU_SDMA_MBANK_DATA_BUF4, "SDMA_MBANK_DATA_BUF4"},
2290 {AMDGPU_SDMA_MBANK_DATA_BUF5, "SDMA_MBANK_DATA_BUF5"},
2291 {AMDGPU_SDMA_MBANK_DATA_BUF6, "SDMA_MBANK_DATA_BUF6"},
2292 {AMDGPU_SDMA_MBANK_DATA_BUF7, "SDMA_MBANK_DATA_BUF7"},
2293 {AMDGPU_SDMA_MBANK_DATA_BUF8, "SDMA_MBANK_DATA_BUF8"},
2294 {AMDGPU_SDMA_MBANK_DATA_BUF9, "SDMA_MBANK_DATA_BUF9"},
2295 {AMDGPU_SDMA_MBANK_DATA_BUF10, "SDMA_MBANK_DATA_BUF10"},
2296 {AMDGPU_SDMA_MBANK_DATA_BUF11, "SDMA_MBANK_DATA_BUF11"},
2297 {AMDGPU_SDMA_MBANK_DATA_BUF12, "SDMA_MBANK_DATA_BUF12"},
2298 {AMDGPU_SDMA_MBANK_DATA_BUF13, "SDMA_MBANK_DATA_BUF13"},
2299 {AMDGPU_SDMA_MBANK_DATA_BUF14, "SDMA_MBANK_DATA_BUF14"},
2300 {AMDGPU_SDMA_MBANK_DATA_BUF15, "SDMA_MBANK_DATA_BUF15"},
2301 {AMDGPU_SDMA_UCODE_BUF, "SDMA_UCODE_BUF"},
2302 {AMDGPU_SDMA_RB_CMD_BUF, "SDMA_RB_CMD_BUF"},
2303 {AMDGPU_SDMA_IB_CMD_BUF, "SDMA_IB_CMD_BUF"},
2304 {AMDGPU_SDMA_UTCL1_RD_FIFO, "SDMA_UTCL1_RD_FIFO"},
2305 {AMDGPU_SDMA_UTCL1_RDBST_FIFO, "SDMA_UTCL1_RDBST_FIFO"},
2306 {AMDGPU_SDMA_UTCL1_WR_FIFO, "SDMA_UTCL1_WR_FIFO"},
2307 {AMDGPU_SDMA_DATA_LUT_FIFO, "SDMA_DATA_LUT_FIFO"},
2308 {AMDGPU_SDMA_SPLIT_DAT_BUF, "SDMA_SPLIT_DAT_BUF"},
2309 };
2310
sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device * adev,uint32_t sdma_inst,void * ras_err_status)2311 static void sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device *adev,
2312 uint32_t sdma_inst,
2313 void *ras_err_status)
2314 {
2315 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
2316 uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2317 unsigned long ue_count = 0;
2318 struct amdgpu_smuio_mcm_config_info mcm_info = {
2319 .socket_id = adev->smuio.funcs->get_socket_id(adev),
2320 .die_id = adev->sdma.instance[sdma_inst].aid_id,
2321 };
2322
2323 /* sdma v4_4_2 doesn't support query ce counts */
2324 amdgpu_ras_inst_query_ras_error_count(adev,
2325 sdma_v4_2_2_ue_reg_list,
2326 ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2327 sdma_v4_4_2_ras_memory_list,
2328 ARRAY_SIZE(sdma_v4_4_2_ras_memory_list),
2329 sdma_dev_inst,
2330 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
2331 &ue_count);
2332
2333 amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
2334 }
2335
sdma_v4_4_2_query_ras_error_count(struct amdgpu_device * adev,void * ras_err_status)2336 static void sdma_v4_4_2_query_ras_error_count(struct amdgpu_device *adev,
2337 void *ras_err_status)
2338 {
2339 uint32_t inst_mask;
2340 int i = 0;
2341
2342 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2343 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2344 for_each_inst(i, inst_mask)
2345 sdma_v4_4_2_inst_query_ras_error_count(adev, i, ras_err_status);
2346 } else {
2347 dev_warn(adev->dev, "SDMA RAS is not supported\n");
2348 }
2349 }
2350
sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device * adev,uint32_t sdma_inst)2351 static void sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device *adev,
2352 uint32_t sdma_inst)
2353 {
2354 uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2355
2356 amdgpu_ras_inst_reset_ras_error_count(adev,
2357 sdma_v4_2_2_ue_reg_list,
2358 ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2359 sdma_dev_inst);
2360 }
2361
sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device * adev)2362 static void sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device *adev)
2363 {
2364 uint32_t inst_mask;
2365 int i = 0;
2366
2367 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2368 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2369 for_each_inst(i, inst_mask)
2370 sdma_v4_4_2_inst_reset_ras_error_count(adev, i);
2371 } else {
2372 dev_warn(adev->dev, "SDMA RAS is not supported\n");
2373 }
2374 }
2375
2376 static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = {
2377 .query_ras_error_count = sdma_v4_4_2_query_ras_error_count,
2378 .reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count,
2379 };
2380
sdma_v4_4_2_aca_bank_parser(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)2381 static int sdma_v4_4_2_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
2382 enum aca_smu_type type, void *data)
2383 {
2384 struct aca_bank_info info;
2385 u64 misc0;
2386 int ret;
2387
2388 ret = aca_bank_info_decode(bank, &info);
2389 if (ret)
2390 return ret;
2391
2392 misc0 = bank->regs[ACA_REG_IDX_MISC0];
2393 switch (type) {
2394 case ACA_SMU_TYPE_UE:
2395 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
2396 1ULL);
2397 break;
2398 case ACA_SMU_TYPE_CE:
2399 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE,
2400 ACA_REG__MISC0__ERRCNT(misc0));
2401 break;
2402 default:
2403 return -EINVAL;
2404 }
2405
2406 return ret;
2407 }
2408
2409 /* CODE_SDMA0 - CODE_SDMA4, reference to smu driver if header file */
2410 static int sdma_v4_4_2_err_codes[] = { 33, 34, 35, 36 };
2411
sdma_v4_4_2_aca_bank_is_valid(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)2412 static bool sdma_v4_4_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
2413 enum aca_smu_type type, void *data)
2414 {
2415 u32 instlo;
2416
2417 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
2418 instlo &= GENMASK(31, 1);
2419
2420 if (instlo != mmSMNAID_AID0_MCA_SMU)
2421 return false;
2422
2423 if (aca_bank_check_error_codes(handle->adev, bank,
2424 sdma_v4_4_2_err_codes,
2425 ARRAY_SIZE(sdma_v4_4_2_err_codes)))
2426 return false;
2427
2428 return true;
2429 }
2430
2431 static const struct aca_bank_ops sdma_v4_4_2_aca_bank_ops = {
2432 .aca_bank_parser = sdma_v4_4_2_aca_bank_parser,
2433 .aca_bank_is_valid = sdma_v4_4_2_aca_bank_is_valid,
2434 };
2435
2436 static const struct aca_info sdma_v4_4_2_aca_info = {
2437 .hwip = ACA_HWIP_TYPE_SMU,
2438 .mask = ACA_ERROR_UE_MASK,
2439 .bank_ops = &sdma_v4_4_2_aca_bank_ops,
2440 };
2441
sdma_v4_4_2_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)2442 static int sdma_v4_4_2_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
2443 {
2444 int r;
2445
2446 r = amdgpu_sdma_ras_late_init(adev, ras_block);
2447 if (r)
2448 return r;
2449
2450 return amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__SDMA,
2451 &sdma_v4_4_2_aca_info, NULL);
2452 }
2453
2454 static struct amdgpu_sdma_ras sdma_v4_4_2_ras = {
2455 .ras_block = {
2456 .hw_ops = &sdma_v4_4_2_ras_hw_ops,
2457 .ras_late_init = sdma_v4_4_2_ras_late_init,
2458 },
2459 };
2460
sdma_v4_4_2_set_ras_funcs(struct amdgpu_device * adev)2461 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev)
2462 {
2463 adev->sdma.ras = &sdma_v4_4_2_ras;
2464 }
2465