xref: /linux/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c (revision be4e3509314af751f08677f428f93c306aaa2f8e)
1 /*
2  * Copyright 2022 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include "amdgpu.h"
30 #include "amdgpu_xcp.h"
31 #include "amdgpu_ucode.h"
32 #include "amdgpu_trace.h"
33 
34 #include "sdma/sdma_4_4_2_offset.h"
35 #include "sdma/sdma_4_4_2_sh_mask.h"
36 
37 #include "soc15_common.h"
38 #include "soc15.h"
39 #include "vega10_sdma_pkt_open.h"
40 
41 #include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
42 #include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
43 
44 #include "amdgpu_ras.h"
45 
46 MODULE_FIRMWARE("amdgpu/sdma_4_4_2.bin");
47 MODULE_FIRMWARE("amdgpu/sdma_4_4_5.bin");
48 
49 static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_4_2[] = {
50 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS_REG),
51 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS1_REG),
52 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS2_REG),
53 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_STATUS3_REG),
54 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UCODE_CHECKSUM),
55 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RB_RPTR_FETCH_HI),
56 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RB_RPTR_FETCH),
57 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_STATUS),
58 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_STATUS),
59 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_XNACK0),
60 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_RD_XNACK1),
61 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_XNACK0),
62 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_UTCL1_WR_XNACK1),
63 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_CNTL),
64 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_RPTR),
65 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_RPTR_HI),
66 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_WPTR),
67 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_RB_WPTR_HI),
68 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_OFFSET),
69 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_BASE_LO),
70 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_BASE_HI),
71 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_CNTL),
72 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_RPTR),
73 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_IB_SUB_REMAIN),
74 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_GFX_DUMMY_REG),
75 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_CNTL),
76 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_RPTR),
77 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_RPTR_HI),
78 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_WPTR),
79 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_RB_WPTR_HI),
80 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_OFFSET),
81 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_BASE_LO),
82 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_IB_BASE_HI),
83 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_PAGE_DUMMY_REG),
84 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_CNTL),
85 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_RPTR),
86 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_RPTR_HI),
87 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_WPTR),
88 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_RB_WPTR_HI),
89 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_OFFSET),
90 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_BASE_LO),
91 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_IB_BASE_HI),
92 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_RLC0_DUMMY_REG),
93 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_VM_CNTL)
94 };
95 
96 #define mmSMNAID_AID0_MCA_SMU 0x03b30400
97 
98 #define WREG32_SDMA(instance, offset, value) \
99 	WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value)
100 #define RREG32_SDMA(instance, offset) \
101 	RREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)))
102 
103 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev);
104 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev);
105 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev);
106 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev);
107 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev);
108 
109 static u32 sdma_v4_4_2_get_reg_offset(struct amdgpu_device *adev,
110 		u32 instance, u32 offset)
111 {
112 	u32 dev_inst = GET_INST(SDMA0, instance);
113 
114 	return (adev->reg_offset[SDMA0_HWIP][dev_inst][0] + offset);
115 }
116 
117 static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
118 {
119 	switch (seq_num) {
120 	case 0:
121 		return SOC15_IH_CLIENTID_SDMA0;
122 	case 1:
123 		return SOC15_IH_CLIENTID_SDMA1;
124 	case 2:
125 		return SOC15_IH_CLIENTID_SDMA2;
126 	case 3:
127 		return SOC15_IH_CLIENTID_SDMA3;
128 	default:
129 		return -EINVAL;
130 	}
131 }
132 
133 static int sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device *adev, unsigned client_id)
134 {
135 	switch (client_id) {
136 	case SOC15_IH_CLIENTID_SDMA0:
137 		return 0;
138 	case SOC15_IH_CLIENTID_SDMA1:
139 		return 1;
140 	case SOC15_IH_CLIENTID_SDMA2:
141 		if (amdgpu_sriov_vf(adev) && (adev->gfx.xcc_mask == 0x1))
142 			return 0;
143 		else
144 			return 2;
145 	case SOC15_IH_CLIENTID_SDMA3:
146 		if (amdgpu_sriov_vf(adev) && (adev->gfx.xcc_mask == 0x1))
147 			return 1;
148 		else
149 			return 3;
150 	default:
151 		return -EINVAL;
152 	}
153 }
154 
155 static void sdma_v4_4_2_inst_init_golden_registers(struct amdgpu_device *adev,
156 						   uint32_t inst_mask)
157 {
158 	u32 val;
159 	int i;
160 
161 	for (i = 0; i < adev->sdma.num_instances; i++) {
162 		val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG);
163 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG, NUM_BANKS, 4);
164 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG,
165 				    PIPE_INTERLEAVE_SIZE, 0);
166 		WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG, val);
167 
168 		val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ);
169 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ, NUM_BANKS,
170 				    4);
171 		val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ,
172 				    PIPE_INTERLEAVE_SIZE, 0);
173 		WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ, val);
174 	}
175 }
176 
177 /**
178  * sdma_v4_4_2_init_microcode - load ucode images from disk
179  *
180  * @adev: amdgpu_device pointer
181  *
182  * Use the firmware interface to load the ucode images into
183  * the driver (not loaded into hw).
184  * Returns 0 on success, error on failure.
185  */
186 static int sdma_v4_4_2_init_microcode(struct amdgpu_device *adev)
187 {
188 	int ret, i;
189 
190 	for (i = 0; i < adev->sdma.num_instances; i++) {
191 		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 2) ||
192 		    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 4) ||
193 		    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 5)) {
194 			ret = amdgpu_sdma_init_microcode(adev, 0, true);
195 			break;
196 		} else {
197 			ret = amdgpu_sdma_init_microcode(adev, i, false);
198 			if (ret)
199 				return ret;
200 		}
201 	}
202 
203 	return ret;
204 }
205 
206 /**
207  * sdma_v4_4_2_ring_get_rptr - get the current read pointer
208  *
209  * @ring: amdgpu ring pointer
210  *
211  * Get the current rptr from the hardware.
212  */
213 static uint64_t sdma_v4_4_2_ring_get_rptr(struct amdgpu_ring *ring)
214 {
215 	u64 rptr;
216 
217 	/* XXX check if swapping is necessary on BE */
218 	rptr = READ_ONCE(*((u64 *)&ring->adev->wb.wb[ring->rptr_offs]));
219 
220 	DRM_DEBUG("rptr before shift == 0x%016llx\n", rptr);
221 	return rptr >> 2;
222 }
223 
224 /**
225  * sdma_v4_4_2_ring_get_wptr - get the current write pointer
226  *
227  * @ring: amdgpu ring pointer
228  *
229  * Get the current wptr from the hardware.
230  */
231 static uint64_t sdma_v4_4_2_ring_get_wptr(struct amdgpu_ring *ring)
232 {
233 	struct amdgpu_device *adev = ring->adev;
234 	u64 wptr;
235 
236 	if (ring->use_doorbell) {
237 		/* XXX check if swapping is necessary on BE */
238 		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
239 		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
240 	} else {
241 		wptr = RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI);
242 		wptr = wptr << 32;
243 		wptr |= RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR);
244 		DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",
245 				ring->me, wptr);
246 	}
247 
248 	return wptr >> 2;
249 }
250 
251 /**
252  * sdma_v4_4_2_ring_set_wptr - commit the write pointer
253  *
254  * @ring: amdgpu ring pointer
255  *
256  * Write the wptr back to the hardware.
257  */
258 static void sdma_v4_4_2_ring_set_wptr(struct amdgpu_ring *ring)
259 {
260 	struct amdgpu_device *adev = ring->adev;
261 
262 	DRM_DEBUG("Setting write pointer\n");
263 	if (ring->use_doorbell) {
264 		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
265 
266 		DRM_DEBUG("Using doorbell -- "
267 				"wptr_offs == 0x%08x "
268 				"lower_32_bits(ring->wptr) << 2 == 0x%08x "
269 				"upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
270 				ring->wptr_offs,
271 				lower_32_bits(ring->wptr << 2),
272 				upper_32_bits(ring->wptr << 2));
273 		/* XXX check if swapping is necessary on BE */
274 		WRITE_ONCE(*wb, (ring->wptr << 2));
275 		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
276 				ring->doorbell_index, ring->wptr << 2);
277 		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
278 	} else {
279 		DRM_DEBUG("Not using doorbell -- "
280 				"regSDMA%i_GFX_RB_WPTR == 0x%08x "
281 				"regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
282 				ring->me,
283 				lower_32_bits(ring->wptr << 2),
284 				ring->me,
285 				upper_32_bits(ring->wptr << 2));
286 		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR,
287 			    lower_32_bits(ring->wptr << 2));
288 		WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI,
289 			    upper_32_bits(ring->wptr << 2));
290 	}
291 }
292 
293 /**
294  * sdma_v4_4_2_page_ring_get_wptr - get the current write pointer
295  *
296  * @ring: amdgpu ring pointer
297  *
298  * Get the current wptr from the hardware.
299  */
300 static uint64_t sdma_v4_4_2_page_ring_get_wptr(struct amdgpu_ring *ring)
301 {
302 	struct amdgpu_device *adev = ring->adev;
303 	u64 wptr;
304 
305 	if (ring->use_doorbell) {
306 		/* XXX check if swapping is necessary on BE */
307 		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
308 	} else {
309 		wptr = RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI);
310 		wptr = wptr << 32;
311 		wptr |= RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR);
312 	}
313 
314 	return wptr >> 2;
315 }
316 
317 /**
318  * sdma_v4_4_2_page_ring_set_wptr - commit the write pointer
319  *
320  * @ring: amdgpu ring pointer
321  *
322  * Write the wptr back to the hardware.
323  */
324 static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring)
325 {
326 	struct amdgpu_device *adev = ring->adev;
327 
328 	if (ring->use_doorbell) {
329 		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
330 
331 		/* XXX check if swapping is necessary on BE */
332 		WRITE_ONCE(*wb, (ring->wptr << 2));
333 		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
334 	} else {
335 		uint64_t wptr = ring->wptr << 2;
336 
337 		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR,
338 			    lower_32_bits(wptr));
339 		WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI,
340 			    upper_32_bits(wptr));
341 	}
342 }
343 
344 static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
345 {
346 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
347 	int i;
348 
349 	for (i = 0; i < count; i++)
350 		if (sdma && sdma->burst_nop && (i == 0))
351 			amdgpu_ring_write(ring, ring->funcs->nop |
352 				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
353 		else
354 			amdgpu_ring_write(ring, ring->funcs->nop);
355 }
356 
357 /**
358  * sdma_v4_4_2_ring_emit_ib - Schedule an IB on the DMA engine
359  *
360  * @ring: amdgpu ring pointer
361  * @job: job to retrieve vmid from
362  * @ib: IB object to schedule
363  * @flags: unused
364  *
365  * Schedule an IB in the DMA ring.
366  */
367 static void sdma_v4_4_2_ring_emit_ib(struct amdgpu_ring *ring,
368 				   struct amdgpu_job *job,
369 				   struct amdgpu_ib *ib,
370 				   uint32_t flags)
371 {
372 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
373 
374 	/* IB packet must end on a 8 DW boundary */
375 	sdma_v4_4_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
376 
377 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
378 			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
379 	/* base must be 32 byte aligned */
380 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
381 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
382 	amdgpu_ring_write(ring, ib->length_dw);
383 	amdgpu_ring_write(ring, 0);
384 	amdgpu_ring_write(ring, 0);
385 
386 }
387 
388 static void sdma_v4_4_2_wait_reg_mem(struct amdgpu_ring *ring,
389 				   int mem_space, int hdp,
390 				   uint32_t addr0, uint32_t addr1,
391 				   uint32_t ref, uint32_t mask,
392 				   uint32_t inv)
393 {
394 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
395 			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
396 			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
397 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
398 	if (mem_space) {
399 		/* memory */
400 		amdgpu_ring_write(ring, addr0);
401 		amdgpu_ring_write(ring, addr1);
402 	} else {
403 		/* registers */
404 		amdgpu_ring_write(ring, addr0 << 2);
405 		amdgpu_ring_write(ring, addr1 << 2);
406 	}
407 	amdgpu_ring_write(ring, ref); /* reference */
408 	amdgpu_ring_write(ring, mask); /* mask */
409 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
410 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
411 }
412 
413 /**
414  * sdma_v4_4_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
415  *
416  * @ring: amdgpu ring pointer
417  *
418  * Emit an hdp flush packet on the requested DMA ring.
419  */
420 static void sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
421 {
422 	struct amdgpu_device *adev = ring->adev;
423 	u32 ref_and_mask = 0;
424 	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
425 
426 	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0
427 		       << (ring->me % adev->sdma.num_inst_per_aid);
428 
429 	sdma_v4_4_2_wait_reg_mem(ring, 0, 1,
430 			       adev->nbio.funcs->get_hdp_flush_done_offset(adev),
431 			       adev->nbio.funcs->get_hdp_flush_req_offset(adev),
432 			       ref_and_mask, ref_and_mask, 10);
433 }
434 
435 /**
436  * sdma_v4_4_2_ring_emit_fence - emit a fence on the DMA ring
437  *
438  * @ring: amdgpu ring pointer
439  * @addr: address
440  * @seq: sequence number
441  * @flags: fence related flags
442  *
443  * Add a DMA fence packet to the ring to write
444  * the fence seq number and DMA trap packet to generate
445  * an interrupt if needed.
446  */
447 static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
448 				      unsigned flags)
449 {
450 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
451 	/* write the fence */
452 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
453 	/* zero in first two bits */
454 	BUG_ON(addr & 0x3);
455 	amdgpu_ring_write(ring, lower_32_bits(addr));
456 	amdgpu_ring_write(ring, upper_32_bits(addr));
457 	amdgpu_ring_write(ring, lower_32_bits(seq));
458 
459 	/* optionally write high bits as well */
460 	if (write64bit) {
461 		addr += 4;
462 		amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
463 		/* zero in first two bits */
464 		BUG_ON(addr & 0x3);
465 		amdgpu_ring_write(ring, lower_32_bits(addr));
466 		amdgpu_ring_write(ring, upper_32_bits(addr));
467 		amdgpu_ring_write(ring, upper_32_bits(seq));
468 	}
469 
470 	/* generate an interrupt */
471 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
472 	amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
473 }
474 
475 
476 /**
477  * sdma_v4_4_2_inst_gfx_stop - stop the gfx async dma engines
478  *
479  * @adev: amdgpu_device pointer
480  * @inst_mask: mask of dma engine instances to be disabled
481  *
482  * Stop the gfx async dma ring buffers.
483  */
484 static void sdma_v4_4_2_inst_gfx_stop(struct amdgpu_device *adev,
485 				      uint32_t inst_mask)
486 {
487 	struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
488 	u32 doorbell_offset, doorbell;
489 	u32 rb_cntl, ib_cntl;
490 	int i;
491 
492 	for_each_inst(i, inst_mask) {
493 		sdma[i] = &adev->sdma.instance[i].ring;
494 
495 		rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
496 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 0);
497 		WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
498 		ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
499 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 0);
500 		WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
501 
502 		if (sdma[i]->use_doorbell) {
503 			doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
504 			doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
505 
506 			doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE, 0);
507 			doorbell_offset = REG_SET_FIELD(doorbell_offset,
508 					SDMA_GFX_DOORBELL_OFFSET,
509 					OFFSET, 0);
510 			WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
511 			WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
512 		}
513 	}
514 }
515 
516 /**
517  * sdma_v4_4_2_inst_rlc_stop - stop the compute async dma engines
518  *
519  * @adev: amdgpu_device pointer
520  * @inst_mask: mask of dma engine instances to be disabled
521  *
522  * Stop the compute async dma queues.
523  */
524 static void sdma_v4_4_2_inst_rlc_stop(struct amdgpu_device *adev,
525 				      uint32_t inst_mask)
526 {
527 	/* XXX todo */
528 }
529 
530 /**
531  * sdma_v4_4_2_inst_page_stop - stop the page async dma engines
532  *
533  * @adev: amdgpu_device pointer
534  * @inst_mask: mask of dma engine instances to be disabled
535  *
536  * Stop the page async dma ring buffers.
537  */
538 static void sdma_v4_4_2_inst_page_stop(struct amdgpu_device *adev,
539 				       uint32_t inst_mask)
540 {
541 	u32 rb_cntl, ib_cntl;
542 	int i;
543 
544 	for_each_inst(i, inst_mask) {
545 		rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
546 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
547 					RB_ENABLE, 0);
548 		WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
549 		ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
550 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL,
551 					IB_ENABLE, 0);
552 		WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
553 	}
554 }
555 
556 /**
557  * sdma_v4_4_2_inst_ctx_switch_enable - stop the async dma engines context switch
558  *
559  * @adev: amdgpu_device pointer
560  * @enable: enable/disable the DMA MEs context switch.
561  * @inst_mask: mask of dma engine instances to be enabled
562  *
563  * Halt or unhalt the async dma engines context switch.
564  */
565 static void sdma_v4_4_2_inst_ctx_switch_enable(struct amdgpu_device *adev,
566 					       bool enable, uint32_t inst_mask)
567 {
568 	u32 f32_cntl, phase_quantum = 0;
569 	int i;
570 
571 	if (amdgpu_sdma_phase_quantum) {
572 		unsigned value = amdgpu_sdma_phase_quantum;
573 		unsigned unit = 0;
574 
575 		while (value > (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
576 				SDMA_PHASE0_QUANTUM__VALUE__SHIFT)) {
577 			value = (value + 1) >> 1;
578 			unit++;
579 		}
580 		if (unit > (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
581 			    SDMA_PHASE0_QUANTUM__UNIT__SHIFT)) {
582 			value = (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
583 				 SDMA_PHASE0_QUANTUM__VALUE__SHIFT);
584 			unit = (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
585 				SDMA_PHASE0_QUANTUM__UNIT__SHIFT);
586 			WARN_ONCE(1,
587 			"clamping sdma_phase_quantum to %uK clock cycles\n",
588 				  value << unit);
589 		}
590 		phase_quantum =
591 			value << SDMA_PHASE0_QUANTUM__VALUE__SHIFT |
592 			unit  << SDMA_PHASE0_QUANTUM__UNIT__SHIFT;
593 	}
594 
595 	for_each_inst(i, inst_mask) {
596 		f32_cntl = RREG32_SDMA(i, regSDMA_CNTL);
597 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_CNTL,
598 				AUTO_CTXSW_ENABLE, enable ? 1 : 0);
599 		if (enable && amdgpu_sdma_phase_quantum) {
600 			WREG32_SDMA(i, regSDMA_PHASE0_QUANTUM, phase_quantum);
601 			WREG32_SDMA(i, regSDMA_PHASE1_QUANTUM, phase_quantum);
602 			WREG32_SDMA(i, regSDMA_PHASE2_QUANTUM, phase_quantum);
603 		}
604 		WREG32_SDMA(i, regSDMA_CNTL, f32_cntl);
605 
606 		/* Extend page fault timeout to avoid interrupt storm */
607 		WREG32_SDMA(i, regSDMA_UTCL1_TIMEOUT, 0x00800080);
608 	}
609 }
610 
611 /**
612  * sdma_v4_4_2_inst_enable - stop the async dma engines
613  *
614  * @adev: amdgpu_device pointer
615  * @enable: enable/disable the DMA MEs.
616  * @inst_mask: mask of dma engine instances to be enabled
617  *
618  * Halt or unhalt the async dma engines.
619  */
620 static void sdma_v4_4_2_inst_enable(struct amdgpu_device *adev, bool enable,
621 				    uint32_t inst_mask)
622 {
623 	u32 f32_cntl;
624 	int i;
625 
626 	if (!enable) {
627 		sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
628 		sdma_v4_4_2_inst_rlc_stop(adev, inst_mask);
629 		if (adev->sdma.has_page_queue)
630 			sdma_v4_4_2_inst_page_stop(adev, inst_mask);
631 
632 		/* SDMA FW needs to respond to FREEZE requests during reset.
633 		 * Keep it running during reset */
634 		if (!amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
635 			return;
636 	}
637 
638 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
639 		return;
640 
641 	for_each_inst(i, inst_mask) {
642 		f32_cntl = RREG32_SDMA(i, regSDMA_F32_CNTL);
643 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_F32_CNTL, HALT, enable ? 0 : 1);
644 		WREG32_SDMA(i, regSDMA_F32_CNTL, f32_cntl);
645 	}
646 }
647 
648 /*
649  * sdma_v4_4_2_rb_cntl - get parameters for rb_cntl
650  */
651 static uint32_t sdma_v4_4_2_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
652 {
653 	/* Set ring buffer size in dwords */
654 	uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);
655 
656 	barrier(); /* work around https://llvm.org/pr42576 */
657 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
658 #ifdef __BIG_ENDIAN
659 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
660 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
661 				RPTR_WRITEBACK_SWAP_ENABLE, 1);
662 #endif
663 	return rb_cntl;
664 }
665 
666 /**
667  * sdma_v4_4_2_gfx_resume - setup and start the async dma engines
668  *
669  * @adev: amdgpu_device pointer
670  * @i: instance to resume
671  * @restore: used to restore wptr when restart
672  *
673  * Set up the gfx DMA ring buffers and enable them.
674  * Returns 0 for success, error for failure.
675  */
676 static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
677 {
678 	struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
679 	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
680 	u32 wb_offset;
681 	u32 doorbell;
682 	u32 doorbell_offset;
683 	u64 wptr_gpu_addr;
684 
685 	wb_offset = (ring->rptr_offs * 4);
686 
687 	rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
688 	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
689 	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
690 
691 	/* set the wb address whether it's enabled or not */
692 	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_HI,
693 	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
694 	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_LO,
695 	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
696 
697 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
698 				RPTR_WRITEBACK_ENABLE, 1);
699 
700 	WREG32_SDMA(i, regSDMA_GFX_RB_BASE, ring->gpu_addr >> 8);
701 	WREG32_SDMA(i, regSDMA_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
702 
703 	if (!restore)
704 		ring->wptr = 0;
705 
706 	/* before programing wptr to a less value, need set minor_ptr_update first */
707 	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 1);
708 
709 	/* Initialize the ring buffer's read and write pointers */
710 	if (restore) {
711 		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, lower_32_bits(ring->wptr << 2));
712 		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, upper_32_bits(ring->wptr << 2));
713 		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, lower_32_bits(ring->wptr << 2));
714 		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, upper_32_bits(ring->wptr << 2));
715 	} else {
716 		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
717 		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
718 		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
719 		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
720 	}
721 
722 	doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
723 	doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
724 
725 	doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE,
726 				 ring->use_doorbell);
727 	doorbell_offset = REG_SET_FIELD(doorbell_offset,
728 					SDMA_GFX_DOORBELL_OFFSET,
729 					OFFSET, ring->doorbell_index);
730 	WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
731 	WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
732 
733 	sdma_v4_4_2_ring_set_wptr(ring);
734 
735 	/* set minor_ptr_update to 0 after wptr programed */
736 	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 0);
737 
738 	/* setup the wptr shadow polling */
739 	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
740 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_LO,
741 		    lower_32_bits(wptr_gpu_addr));
742 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_HI,
743 		    upper_32_bits(wptr_gpu_addr));
744 	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL);
745 	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
746 				       SDMA_GFX_RB_WPTR_POLL_CNTL,
747 				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
748 	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
749 
750 	/* enable DMA RB */
751 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 1);
752 	WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
753 
754 	ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
755 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 1);
756 #ifdef __BIG_ENDIAN
757 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
758 #endif
759 	/* enable DMA IBs */
760 	WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
761 }
762 
763 /**
764  * sdma_v4_4_2_page_resume - setup and start the async dma engines
765  *
766  * @adev: amdgpu_device pointer
767  * @i: instance to resume
768  * @restore: boolean to say restore needed or not
769  *
770  * Set up the page DMA ring buffers and enable them.
771  * Returns 0 for success, error for failure.
772  */
773 static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
774 {
775 	struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
776 	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
777 	u32 wb_offset;
778 	u32 doorbell;
779 	u32 doorbell_offset;
780 	u64 wptr_gpu_addr;
781 
782 	wb_offset = (ring->rptr_offs * 4);
783 
784 	rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
785 	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
786 	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
787 
788 	/* Initialize the ring buffer's read and write pointers */
789 	if (restore) {
790 		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, lower_32_bits(ring->wptr << 2));
791 		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, upper_32_bits(ring->wptr << 2));
792 		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, lower_32_bits(ring->wptr << 2));
793 		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, upper_32_bits(ring->wptr << 2));
794 	} else {
795 		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
796 		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
797 		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
798 		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
799 	}
800 
801 	/* set the wb address whether it's enabled or not */
802 	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_HI,
803 	       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
804 	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_LO,
805 	       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
806 
807 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
808 				RPTR_WRITEBACK_ENABLE, 1);
809 
810 	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE, ring->gpu_addr >> 8);
811 	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
812 
813 	if (!restore)
814 		ring->wptr = 0;
815 
816 	/* before programing wptr to a less value, need set minor_ptr_update first */
817 	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 1);
818 
819 	doorbell = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL);
820 	doorbell_offset = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET);
821 
822 	doorbell = REG_SET_FIELD(doorbell, SDMA_PAGE_DOORBELL, ENABLE,
823 				 ring->use_doorbell);
824 	doorbell_offset = REG_SET_FIELD(doorbell_offset,
825 					SDMA_PAGE_DOORBELL_OFFSET,
826 					OFFSET, ring->doorbell_index);
827 	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL, doorbell);
828 	WREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET, doorbell_offset);
829 
830 	/* paging queue doorbell range is setup at sdma_v4_4_2_gfx_resume */
831 	sdma_v4_4_2_page_ring_set_wptr(ring);
832 
833 	/* set minor_ptr_update to 0 after wptr programed */
834 	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 0);
835 
836 	/* setup the wptr shadow polling */
837 	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
838 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO,
839 		    lower_32_bits(wptr_gpu_addr));
840 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI,
841 		    upper_32_bits(wptr_gpu_addr));
842 	wptr_poll_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL);
843 	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
844 				       SDMA_PAGE_RB_WPTR_POLL_CNTL,
845 				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
846 	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
847 
848 	/* enable DMA RB */
849 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL, RB_ENABLE, 1);
850 	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
851 
852 	ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
853 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_ENABLE, 1);
854 #ifdef __BIG_ENDIAN
855 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);
856 #endif
857 	/* enable DMA IBs */
858 	WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
859 }
860 
861 static void sdma_v4_4_2_init_pg(struct amdgpu_device *adev)
862 {
863 
864 }
865 
866 /**
867  * sdma_v4_4_2_inst_rlc_resume - setup and start the async dma engines
868  *
869  * @adev: amdgpu_device pointer
870  * @inst_mask: mask of dma engine instances to be enabled
871  *
872  * Set up the compute DMA queues and enable them.
873  * Returns 0 for success, error for failure.
874  */
875 static int sdma_v4_4_2_inst_rlc_resume(struct amdgpu_device *adev,
876 				       uint32_t inst_mask)
877 {
878 	sdma_v4_4_2_init_pg(adev);
879 
880 	return 0;
881 }
882 
883 /**
884  * sdma_v4_4_2_inst_load_microcode - load the sDMA ME ucode
885  *
886  * @adev: amdgpu_device pointer
887  * @inst_mask: mask of dma engine instances to be enabled
888  *
889  * Loads the sDMA0/1 ucode.
890  * Returns 0 for success, -EINVAL if the ucode is not available.
891  */
892 static int sdma_v4_4_2_inst_load_microcode(struct amdgpu_device *adev,
893 					   uint32_t inst_mask)
894 {
895 	const struct sdma_firmware_header_v1_0 *hdr;
896 	const __le32 *fw_data;
897 	u32 fw_size;
898 	int i, j;
899 
900 	/* halt the MEs */
901 	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
902 
903 	for_each_inst(i, inst_mask) {
904 		if (!adev->sdma.instance[i].fw)
905 			return -EINVAL;
906 
907 		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
908 		amdgpu_ucode_print_sdma_hdr(&hdr->header);
909 		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
910 
911 		fw_data = (const __le32 *)
912 			(adev->sdma.instance[i].fw->data +
913 				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
914 
915 		WREG32_SDMA(i, regSDMA_UCODE_ADDR, 0);
916 
917 		for (j = 0; j < fw_size; j++)
918 			WREG32_SDMA(i, regSDMA_UCODE_DATA,
919 				    le32_to_cpup(fw_data++));
920 
921 		WREG32_SDMA(i, regSDMA_UCODE_ADDR,
922 			    adev->sdma.instance[i].fw_version);
923 	}
924 
925 	return 0;
926 }
927 
928 /**
929  * sdma_v4_4_2_inst_start - setup and start the async dma engines
930  *
931  * @adev: amdgpu_device pointer
932  * @inst_mask: mask of dma engine instances to be enabled
933  * @restore: boolean to say restore needed or not
934  *
935  * Set up the DMA engines and enable them.
936  * Returns 0 for success, error for failure.
937  */
938 static int sdma_v4_4_2_inst_start(struct amdgpu_device *adev,
939 				  uint32_t inst_mask, bool restore)
940 {
941 	struct amdgpu_ring *ring;
942 	uint32_t tmp_mask;
943 	int i, r = 0;
944 
945 	if (amdgpu_sriov_vf(adev)) {
946 		sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
947 		sdma_v4_4_2_inst_enable(adev, false, inst_mask);
948 	} else {
949 		/* bypass sdma microcode loading on Gopher */
950 		if (!restore && adev->firmware.load_type != AMDGPU_FW_LOAD_PSP &&
951 		    adev->sdma.instance[0].fw) {
952 			r = sdma_v4_4_2_inst_load_microcode(adev, inst_mask);
953 			if (r)
954 				return r;
955 		}
956 
957 		/* unhalt the MEs */
958 		sdma_v4_4_2_inst_enable(adev, true, inst_mask);
959 		/* enable sdma ring preemption */
960 		sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
961 	}
962 
963 	/* start the gfx rings and rlc compute queues */
964 	tmp_mask = inst_mask;
965 	for_each_inst(i, tmp_mask) {
966 		uint32_t temp;
967 
968 		WREG32_SDMA(i, regSDMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
969 		sdma_v4_4_2_gfx_resume(adev, i, restore);
970 		if (adev->sdma.has_page_queue)
971 			sdma_v4_4_2_page_resume(adev, i, restore);
972 
973 		/* set utc l1 enable flag always to 1 */
974 		temp = RREG32_SDMA(i, regSDMA_CNTL);
975 		temp = REG_SET_FIELD(temp, SDMA_CNTL, UTC_L1_ENABLE, 1);
976 		/* enable context empty interrupt during initialization */
977 		temp = REG_SET_FIELD(temp, SDMA_CNTL, CTXEMPTY_INT_ENABLE, 1);
978 		WREG32_SDMA(i, regSDMA_CNTL, temp);
979 
980 		if (!amdgpu_sriov_vf(adev)) {
981 			if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
982 				/* unhalt engine */
983 				temp = RREG32_SDMA(i, regSDMA_F32_CNTL);
984 				temp = REG_SET_FIELD(temp, SDMA_F32_CNTL, HALT, 0);
985 				WREG32_SDMA(i, regSDMA_F32_CNTL, temp);
986 			}
987 		}
988 	}
989 
990 	if (amdgpu_sriov_vf(adev)) {
991 		sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
992 		sdma_v4_4_2_inst_enable(adev, true, inst_mask);
993 	} else {
994 		r = sdma_v4_4_2_inst_rlc_resume(adev, inst_mask);
995 		if (r)
996 			return r;
997 	}
998 
999 	tmp_mask = inst_mask;
1000 	for_each_inst(i, tmp_mask) {
1001 		ring = &adev->sdma.instance[i].ring;
1002 
1003 		r = amdgpu_ring_test_helper(ring);
1004 		if (r)
1005 			return r;
1006 
1007 		if (adev->sdma.has_page_queue) {
1008 			struct amdgpu_ring *page = &adev->sdma.instance[i].page;
1009 
1010 			r = amdgpu_ring_test_helper(page);
1011 			if (r)
1012 				return r;
1013 		}
1014 	}
1015 
1016 	return r;
1017 }
1018 
1019 /**
1020  * sdma_v4_4_2_ring_test_ring - simple async dma engine test
1021  *
1022  * @ring: amdgpu_ring structure holding ring information
1023  *
1024  * Test the DMA engine by writing using it to write an
1025  * value to memory.
1026  * Returns 0 for success, error for failure.
1027  */
1028 static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
1029 {
1030 	struct amdgpu_device *adev = ring->adev;
1031 	unsigned i;
1032 	unsigned index;
1033 	int r;
1034 	u32 tmp;
1035 	u64 gpu_addr;
1036 
1037 	r = amdgpu_device_wb_get(adev, &index);
1038 	if (r)
1039 		return r;
1040 
1041 	gpu_addr = adev->wb.gpu_addr + (index * 4);
1042 	tmp = 0xCAFEDEAD;
1043 	adev->wb.wb[index] = cpu_to_le32(tmp);
1044 
1045 	r = amdgpu_ring_alloc(ring, 5);
1046 	if (r)
1047 		goto error_free_wb;
1048 
1049 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1050 			  SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
1051 	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
1052 	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
1053 	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
1054 	amdgpu_ring_write(ring, 0xDEADBEEF);
1055 	amdgpu_ring_commit(ring);
1056 
1057 	for (i = 0; i < adev->usec_timeout; i++) {
1058 		tmp = le32_to_cpu(adev->wb.wb[index]);
1059 		if (tmp == 0xDEADBEEF)
1060 			break;
1061 		udelay(1);
1062 	}
1063 
1064 	if (i >= adev->usec_timeout)
1065 		r = -ETIMEDOUT;
1066 
1067 error_free_wb:
1068 	amdgpu_device_wb_free(adev, index);
1069 	return r;
1070 }
1071 
1072 /**
1073  * sdma_v4_4_2_ring_test_ib - test an IB on the DMA engine
1074  *
1075  * @ring: amdgpu_ring structure holding ring information
1076  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1077  *
1078  * Test a simple IB in the DMA ring.
1079  * Returns 0 on success, error on failure.
1080  */
1081 static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1082 {
1083 	struct amdgpu_device *adev = ring->adev;
1084 	struct amdgpu_ib ib;
1085 	struct dma_fence *f = NULL;
1086 	unsigned index;
1087 	long r;
1088 	u32 tmp = 0;
1089 	u64 gpu_addr;
1090 
1091 	r = amdgpu_device_wb_get(adev, &index);
1092 	if (r)
1093 		return r;
1094 
1095 	gpu_addr = adev->wb.gpu_addr + (index * 4);
1096 	tmp = 0xCAFEDEAD;
1097 	adev->wb.wb[index] = cpu_to_le32(tmp);
1098 	memset(&ib, 0, sizeof(ib));
1099 	r = amdgpu_ib_get(adev, NULL, 256,
1100 					AMDGPU_IB_POOL_DIRECT, &ib);
1101 	if (r)
1102 		goto err0;
1103 
1104 	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1105 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1106 	ib.ptr[1] = lower_32_bits(gpu_addr);
1107 	ib.ptr[2] = upper_32_bits(gpu_addr);
1108 	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1109 	ib.ptr[4] = 0xDEADBEEF;
1110 	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1111 	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1112 	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1113 	ib.length_dw = 8;
1114 
1115 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1116 	if (r)
1117 		goto err1;
1118 
1119 	r = dma_fence_wait_timeout(f, false, timeout);
1120 	if (r == 0) {
1121 		r = -ETIMEDOUT;
1122 		goto err1;
1123 	} else if (r < 0) {
1124 		goto err1;
1125 	}
1126 	tmp = le32_to_cpu(adev->wb.wb[index]);
1127 	if (tmp == 0xDEADBEEF)
1128 		r = 0;
1129 	else
1130 		r = -EINVAL;
1131 
1132 err1:
1133 	amdgpu_ib_free(&ib, NULL);
1134 	dma_fence_put(f);
1135 err0:
1136 	amdgpu_device_wb_free(adev, index);
1137 	return r;
1138 }
1139 
1140 
1141 /**
1142  * sdma_v4_4_2_vm_copy_pte - update PTEs by copying them from the GART
1143  *
1144  * @ib: indirect buffer to fill with commands
1145  * @pe: addr of the page entry
1146  * @src: src addr to copy from
1147  * @count: number of page entries to update
1148  *
1149  * Update PTEs by copying them from the GART using sDMA.
1150  */
1151 static void sdma_v4_4_2_vm_copy_pte(struct amdgpu_ib *ib,
1152 				  uint64_t pe, uint64_t src,
1153 				  unsigned count)
1154 {
1155 	unsigned bytes = count * 8;
1156 
1157 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1158 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1159 	ib->ptr[ib->length_dw++] = bytes - 1;
1160 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1161 	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1162 	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1163 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1164 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1165 
1166 }
1167 
1168 /**
1169  * sdma_v4_4_2_vm_write_pte - update PTEs by writing them manually
1170  *
1171  * @ib: indirect buffer to fill with commands
1172  * @pe: addr of the page entry
1173  * @value: dst addr to write into pe
1174  * @count: number of page entries to update
1175  * @incr: increase next addr by incr bytes
1176  *
1177  * Update PTEs by writing them manually using sDMA.
1178  */
1179 static void sdma_v4_4_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1180 				   uint64_t value, unsigned count,
1181 				   uint32_t incr)
1182 {
1183 	unsigned ndw = count * 2;
1184 
1185 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1186 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1187 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1188 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1189 	ib->ptr[ib->length_dw++] = ndw - 1;
1190 	for (; ndw > 0; ndw -= 2) {
1191 		ib->ptr[ib->length_dw++] = lower_32_bits(value);
1192 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
1193 		value += incr;
1194 	}
1195 }
1196 
1197 /**
1198  * sdma_v4_4_2_vm_set_pte_pde - update the page tables using sDMA
1199  *
1200  * @ib: indirect buffer to fill with commands
1201  * @pe: addr of the page entry
1202  * @addr: dst addr to write into pe
1203  * @count: number of page entries to update
1204  * @incr: increase next addr by incr bytes
1205  * @flags: access flags
1206  *
1207  * Update the page tables using sDMA.
1208  */
1209 static void sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1210 				     uint64_t pe,
1211 				     uint64_t addr, unsigned count,
1212 				     uint32_t incr, uint64_t flags)
1213 {
1214 	/* for physically contiguous pages (vram) */
1215 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1216 	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1217 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1218 	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1219 	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1220 	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1221 	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1222 	ib->ptr[ib->length_dw++] = incr; /* increment size */
1223 	ib->ptr[ib->length_dw++] = 0;
1224 	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1225 }
1226 
1227 /**
1228  * sdma_v4_4_2_ring_pad_ib - pad the IB to the required number of dw
1229  *
1230  * @ring: amdgpu_ring structure holding ring information
1231  * @ib: indirect buffer to fill with padding
1232  */
1233 static void sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1234 {
1235 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1236 	u32 pad_count;
1237 	int i;
1238 
1239 	pad_count = (-ib->length_dw) & 7;
1240 	for (i = 0; i < pad_count; i++)
1241 		if (sdma && sdma->burst_nop && (i == 0))
1242 			ib->ptr[ib->length_dw++] =
1243 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1244 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1245 		else
1246 			ib->ptr[ib->length_dw++] =
1247 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1248 }
1249 
1250 
1251 /**
1252  * sdma_v4_4_2_ring_emit_pipeline_sync - sync the pipeline
1253  *
1254  * @ring: amdgpu_ring pointer
1255  *
1256  * Make sure all previous operations are completed (CIK).
1257  */
1258 static void sdma_v4_4_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1259 {
1260 	uint32_t seq = ring->fence_drv.sync_seq;
1261 	uint64_t addr = ring->fence_drv.gpu_addr;
1262 
1263 	/* wait for idle */
1264 	sdma_v4_4_2_wait_reg_mem(ring, 1, 0,
1265 			       addr & 0xfffffffc,
1266 			       upper_32_bits(addr) & 0xffffffff,
1267 			       seq, 0xffffffff, 4);
1268 }
1269 
1270 
1271 /**
1272  * sdma_v4_4_2_ring_emit_vm_flush - vm flush using sDMA
1273  *
1274  * @ring: amdgpu_ring pointer
1275  * @vmid: vmid number to use
1276  * @pd_addr: address
1277  *
1278  * Update the page table base and flush the VM TLB
1279  * using sDMA.
1280  */
1281 static void sdma_v4_4_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1282 					 unsigned vmid, uint64_t pd_addr)
1283 {
1284 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1285 }
1286 
1287 static void sdma_v4_4_2_ring_emit_wreg(struct amdgpu_ring *ring,
1288 				     uint32_t reg, uint32_t val)
1289 {
1290 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1291 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1292 	amdgpu_ring_write(ring, reg);
1293 	amdgpu_ring_write(ring, val);
1294 }
1295 
1296 static void sdma_v4_4_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1297 					 uint32_t val, uint32_t mask)
1298 {
1299 	sdma_v4_4_2_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);
1300 }
1301 
1302 static bool sdma_v4_4_2_fw_support_paging_queue(struct amdgpu_device *adev)
1303 {
1304 	switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1305 	case IP_VERSION(4, 4, 2):
1306 	case IP_VERSION(4, 4, 5):
1307 		return false;
1308 	default:
1309 		return false;
1310 	}
1311 }
1312 
1313 static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
1314 {
1315 	struct amdgpu_device *adev = ip_block->adev;
1316 	int r;
1317 
1318 	r = sdma_v4_4_2_init_microcode(adev);
1319 	if (r)
1320 		return r;
1321 
1322 	/* TODO: Page queue breaks driver reload under SRIOV */
1323 	if (sdma_v4_4_2_fw_support_paging_queue(adev))
1324 		adev->sdma.has_page_queue = true;
1325 
1326 	sdma_v4_4_2_set_ring_funcs(adev);
1327 	sdma_v4_4_2_set_buffer_funcs(adev);
1328 	sdma_v4_4_2_set_vm_pte_funcs(adev);
1329 	sdma_v4_4_2_set_irq_funcs(adev);
1330 	sdma_v4_4_2_set_ras_funcs(adev);
1331 
1332 	return 0;
1333 }
1334 
1335 #if 0
1336 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1337 		void *err_data,
1338 		struct amdgpu_iv_entry *entry);
1339 #endif
1340 
1341 static int sdma_v4_4_2_late_init(struct amdgpu_ip_block *ip_block)
1342 {
1343 	struct amdgpu_device *adev = ip_block->adev;
1344 #if 0
1345 	struct ras_ih_if ih_info = {
1346 		.cb = sdma_v4_4_2_process_ras_data_cb,
1347 	};
1348 #endif
1349 	if (!amdgpu_persistent_edc_harvesting_supported(adev))
1350 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__SDMA);
1351 
1352 	return 0;
1353 }
1354 
1355 static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block)
1356 {
1357 	struct amdgpu_ring *ring;
1358 	int r, i;
1359 	struct amdgpu_device *adev = ip_block->adev;
1360 	u32 aid_id;
1361 	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
1362 	uint32_t *ptr;
1363 
1364 	/* SDMA trap event */
1365 	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1366 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1367 				      SDMA0_4_0__SRCID__SDMA_TRAP,
1368 				      &adev->sdma.trap_irq);
1369 		if (r)
1370 			return r;
1371 	}
1372 
1373 	/* SDMA SRAM ECC event */
1374 	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1375 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1376 				      SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
1377 				      &adev->sdma.ecc_irq);
1378 		if (r)
1379 			return r;
1380 	}
1381 
1382 	/* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/
1383 	for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1384 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1385 				      SDMA0_4_0__SRCID__SDMA_VM_HOLE,
1386 				      &adev->sdma.vm_hole_irq);
1387 		if (r)
1388 			return r;
1389 
1390 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1391 				      SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,
1392 				      &adev->sdma.doorbell_invalid_irq);
1393 		if (r)
1394 			return r;
1395 
1396 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1397 				      SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,
1398 				      &adev->sdma.pool_timeout_irq);
1399 		if (r)
1400 			return r;
1401 
1402 		r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1403 				      SDMA0_4_0__SRCID__SDMA_SRBMWRITE,
1404 				      &adev->sdma.srbm_write_irq);
1405 		if (r)
1406 			return r;
1407 	}
1408 
1409 	for (i = 0; i < adev->sdma.num_instances; i++) {
1410 		ring = &adev->sdma.instance[i].ring;
1411 		ring->ring_obj = NULL;
1412 		ring->use_doorbell = true;
1413 		aid_id = adev->sdma.instance[i].aid_id;
1414 
1415 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1416 				ring->use_doorbell?"true":"false");
1417 
1418 		/* doorbell size is 2 dwords, get DWORD offset */
1419 		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
1420 		ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1421 
1422 		sprintf(ring->name, "sdma%d.%d", aid_id,
1423 				i % adev->sdma.num_inst_per_aid);
1424 		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1425 				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1426 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
1427 		if (r)
1428 			return r;
1429 
1430 		if (adev->sdma.has_page_queue) {
1431 			ring = &adev->sdma.instance[i].page;
1432 			ring->ring_obj = NULL;
1433 			ring->use_doorbell = true;
1434 
1435 			/* doorbell index of page queue is assigned right after
1436 			 * gfx queue on the same instance
1437 			 */
1438 			ring->doorbell_index =
1439 				(adev->doorbell_index.sdma_engine[i] + 1) << 1;
1440 			ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1441 
1442 			sprintf(ring->name, "page%d.%d", aid_id,
1443 					i % adev->sdma.num_inst_per_aid);
1444 			r = amdgpu_ring_init(adev, ring, 1024,
1445 					     &adev->sdma.trap_irq,
1446 					     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1447 					     AMDGPU_RING_PRIO_DEFAULT, NULL);
1448 			if (r)
1449 				return r;
1450 		}
1451 	}
1452 
1453 	/* TODO: Add queue reset mask when FW fully supports it */
1454 	adev->sdma.supported_reset =
1455 		amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
1456 
1457 	if (amdgpu_sdma_ras_sw_init(adev)) {
1458 		dev_err(adev->dev, "fail to initialize sdma ras block\n");
1459 		return -EINVAL;
1460 	}
1461 
1462 	/* Allocate memory for SDMA IP Dump buffer */
1463 	ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
1464 	if (ptr)
1465 		adev->sdma.ip_dump = ptr;
1466 	else
1467 		DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1468 
1469 	r = amdgpu_sdma_sysfs_reset_mask_init(adev);
1470 	if (r)
1471 		return r;
1472 
1473 	return r;
1474 }
1475 
1476 static int sdma_v4_4_2_sw_fini(struct amdgpu_ip_block *ip_block)
1477 {
1478 	struct amdgpu_device *adev = ip_block->adev;
1479 	int i;
1480 
1481 	for (i = 0; i < adev->sdma.num_instances; i++) {
1482 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1483 		if (adev->sdma.has_page_queue)
1484 			amdgpu_ring_fini(&adev->sdma.instance[i].page);
1485 	}
1486 
1487 	amdgpu_sdma_sysfs_reset_mask_fini(adev);
1488 	if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 2) ||
1489 	    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 4) ||
1490 	    amdgpu_ip_version(adev, SDMA0_HWIP, 0) == IP_VERSION(4, 4, 5))
1491 		amdgpu_sdma_destroy_inst_ctx(adev, true);
1492 	else
1493 		amdgpu_sdma_destroy_inst_ctx(adev, false);
1494 
1495 	kfree(adev->sdma.ip_dump);
1496 
1497 	return 0;
1498 }
1499 
1500 static int sdma_v4_4_2_hw_init(struct amdgpu_ip_block *ip_block)
1501 {
1502 	int r;
1503 	struct amdgpu_device *adev = ip_block->adev;
1504 	uint32_t inst_mask;
1505 
1506 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1507 	if (!amdgpu_sriov_vf(adev))
1508 		sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
1509 
1510 	r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
1511 
1512 	return r;
1513 }
1514 
1515 static int sdma_v4_4_2_hw_fini(struct amdgpu_ip_block *ip_block)
1516 {
1517 	struct amdgpu_device *adev = ip_block->adev;
1518 	uint32_t inst_mask;
1519 	int i;
1520 
1521 	if (amdgpu_sriov_vf(adev))
1522 		return 0;
1523 
1524 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1525 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
1526 		for (i = 0; i < adev->sdma.num_instances; i++) {
1527 			amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
1528 				       AMDGPU_SDMA_IRQ_INSTANCE0 + i);
1529 		}
1530 	}
1531 
1532 	sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
1533 	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
1534 
1535 	return 0;
1536 }
1537 
1538 static int sdma_v4_4_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1539 					     enum amd_clockgating_state state);
1540 
1541 static int sdma_v4_4_2_suspend(struct amdgpu_ip_block *ip_block)
1542 {
1543 	struct amdgpu_device *adev = ip_block->adev;
1544 
1545 	if (amdgpu_in_reset(adev))
1546 		sdma_v4_4_2_set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE);
1547 
1548 	return sdma_v4_4_2_hw_fini(ip_block);
1549 }
1550 
1551 static int sdma_v4_4_2_resume(struct amdgpu_ip_block *ip_block)
1552 {
1553 	return sdma_v4_4_2_hw_init(ip_block);
1554 }
1555 
1556 static bool sdma_v4_4_2_is_idle(void *handle)
1557 {
1558 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1559 	u32 i;
1560 
1561 	for (i = 0; i < adev->sdma.num_instances; i++) {
1562 		u32 tmp = RREG32_SDMA(i, regSDMA_STATUS_REG);
1563 
1564 		if (!(tmp & SDMA_STATUS_REG__IDLE_MASK))
1565 			return false;
1566 	}
1567 
1568 	return true;
1569 }
1570 
1571 static int sdma_v4_4_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
1572 {
1573 	unsigned i, j;
1574 	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
1575 	struct amdgpu_device *adev = ip_block->adev;
1576 
1577 	for (i = 0; i < adev->usec_timeout; i++) {
1578 		for (j = 0; j < adev->sdma.num_instances; j++) {
1579 			sdma[j] = RREG32_SDMA(j, regSDMA_STATUS_REG);
1580 			if (!(sdma[j] & SDMA_STATUS_REG__IDLE_MASK))
1581 				break;
1582 		}
1583 		if (j == adev->sdma.num_instances)
1584 			return 0;
1585 		udelay(1);
1586 	}
1587 	return -ETIMEDOUT;
1588 }
1589 
1590 static int sdma_v4_4_2_soft_reset(struct amdgpu_ip_block *ip_block)
1591 {
1592 	/* todo */
1593 
1594 	return 0;
1595 }
1596 
1597 static int sdma_v4_4_2_reset_queue(struct amdgpu_ring *ring, unsigned int vmid)
1598 {
1599 	struct amdgpu_device *adev = ring->adev;
1600 	int i, r;
1601 	u32 inst_mask;
1602 
1603 	if ((adev->flags & AMD_IS_APU) || amdgpu_sriov_vf(adev))
1604 		return -EINVAL;
1605 
1606 	/* stop queue */
1607 	inst_mask = 1 << ring->me;
1608 	sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
1609 	if (adev->sdma.has_page_queue)
1610 		sdma_v4_4_2_inst_page_stop(adev, inst_mask);
1611 
1612 	r = amdgpu_dpm_reset_sdma(adev, 1 << GET_INST(SDMA0, ring->me));
1613 	if (r)
1614 		return r;
1615 
1616 	udelay(50);
1617 
1618 	for (i = 0; i < adev->usec_timeout; i++) {
1619 		if (!REG_GET_FIELD(RREG32_SDMA(ring->me, regSDMA_F32_CNTL), SDMA_F32_CNTL, HALT))
1620 			break;
1621 		udelay(1);
1622 	}
1623 
1624 	if (i == adev->usec_timeout) {
1625 		dev_err(adev->dev, "timed out waiting for SDMA%d unhalt after reset\n",
1626 			ring->me);
1627 		return -ETIMEDOUT;
1628 	}
1629 
1630 	return sdma_v4_4_2_inst_start(adev, inst_mask, true);
1631 }
1632 
1633 static int sdma_v4_4_2_set_trap_irq_state(struct amdgpu_device *adev,
1634 					struct amdgpu_irq_src *source,
1635 					unsigned type,
1636 					enum amdgpu_interrupt_state state)
1637 {
1638 	u32 sdma_cntl;
1639 
1640 	sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1641 	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, TRAP_ENABLE,
1642 		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1643 	WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1644 
1645 	return 0;
1646 }
1647 
1648 static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
1649 				      struct amdgpu_irq_src *source,
1650 				      struct amdgpu_iv_entry *entry)
1651 {
1652 	uint32_t instance, i;
1653 
1654 	DRM_DEBUG("IH: SDMA trap\n");
1655 	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1656 
1657 	/* Client id gives the SDMA instance in AID. To know the exact SDMA
1658 	 * instance, interrupt entry gives the node id which corresponds to the AID instance.
1659 	 * Match node id with the AID id associated with the SDMA instance. */
1660 	for (i = instance; i < adev->sdma.num_instances;
1661 	     i += adev->sdma.num_inst_per_aid) {
1662 		if (adev->sdma.instance[i].aid_id ==
1663 		    node_id_to_phys_map[entry->node_id])
1664 			break;
1665 	}
1666 
1667 	if (i >= adev->sdma.num_instances) {
1668 		dev_WARN_ONCE(
1669 			adev->dev, 1,
1670 			"Couldn't find the right sdma instance in trap handler");
1671 		return 0;
1672 	}
1673 
1674 	switch (entry->ring_id) {
1675 	case 0:
1676 		amdgpu_fence_process(&adev->sdma.instance[i].ring);
1677 		break;
1678 	default:
1679 		break;
1680 	}
1681 	return 0;
1682 }
1683 
1684 #if 0
1685 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1686 		void *err_data,
1687 		struct amdgpu_iv_entry *entry)
1688 {
1689 	int instance;
1690 
1691 	/* When “Full RAS” is enabled, the per-IP interrupt sources should
1692 	 * be disabled and the driver should only look for the aggregated
1693 	 * interrupt via sync flood
1694 	 */
1695 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
1696 		goto out;
1697 
1698 	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1699 	if (instance < 0)
1700 		goto out;
1701 
1702 	amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);
1703 
1704 out:
1705 	return AMDGPU_RAS_SUCCESS;
1706 }
1707 #endif
1708 
1709 static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1710 					      struct amdgpu_irq_src *source,
1711 					      struct amdgpu_iv_entry *entry)
1712 {
1713 	int instance;
1714 
1715 	DRM_ERROR("Illegal instruction in SDMA command stream\n");
1716 
1717 	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1718 	if (instance < 0)
1719 		return 0;
1720 
1721 	switch (entry->ring_id) {
1722 	case 0:
1723 		drm_sched_fault(&adev->sdma.instance[instance].ring.sched);
1724 		break;
1725 	}
1726 	return 0;
1727 }
1728 
1729 static int sdma_v4_4_2_set_ecc_irq_state(struct amdgpu_device *adev,
1730 					struct amdgpu_irq_src *source,
1731 					unsigned type,
1732 					enum amdgpu_interrupt_state state)
1733 {
1734 	u32 sdma_cntl;
1735 
1736 	sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1737 	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, DRAM_ECC_INT_ENABLE,
1738 					state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1739 	WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1740 
1741 	return 0;
1742 }
1743 
1744 static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
1745 					      struct amdgpu_iv_entry *entry)
1746 {
1747 	int instance;
1748 	struct amdgpu_task_info *task_info;
1749 	u64 addr;
1750 
1751 	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
1752 	if (instance < 0 || instance >= adev->sdma.num_instances) {
1753 		dev_err(adev->dev, "sdma instance invalid %d\n", instance);
1754 		return -EINVAL;
1755 	}
1756 
1757 	addr = (u64)entry->src_data[0] << 12;
1758 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
1759 
1760 	dev_dbg_ratelimited(adev->dev,
1761 			    "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u pasid:%u\n",
1762 			    instance, addr, entry->src_id, entry->ring_id, entry->vmid,
1763 			    entry->pasid);
1764 
1765 	task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
1766 	if (task_info) {
1767 		dev_dbg_ratelimited(adev->dev, " for process %s pid %d thread %s pid %d\n",
1768 				    task_info->process_name, task_info->tgid,
1769 				    task_info->task_name, task_info->pid);
1770 		amdgpu_vm_put_task_info(task_info);
1771 	}
1772 
1773 	return 0;
1774 }
1775 
1776 static int sdma_v4_4_2_process_vm_hole_irq(struct amdgpu_device *adev,
1777 					      struct amdgpu_irq_src *source,
1778 					      struct amdgpu_iv_entry *entry)
1779 {
1780 	dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
1781 	sdma_v4_4_2_print_iv_entry(adev, entry);
1782 	return 0;
1783 }
1784 
1785 static int sdma_v4_4_2_process_doorbell_invalid_irq(struct amdgpu_device *adev,
1786 					      struct amdgpu_irq_src *source,
1787 					      struct amdgpu_iv_entry *entry)
1788 {
1789 
1790 	dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
1791 	sdma_v4_4_2_print_iv_entry(adev, entry);
1792 	return 0;
1793 }
1794 
1795 static int sdma_v4_4_2_process_pool_timeout_irq(struct amdgpu_device *adev,
1796 					      struct amdgpu_irq_src *source,
1797 					      struct amdgpu_iv_entry *entry)
1798 {
1799 	dev_dbg_ratelimited(adev->dev,
1800 		"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
1801 	sdma_v4_4_2_print_iv_entry(adev, entry);
1802 	return 0;
1803 }
1804 
1805 static int sdma_v4_4_2_process_srbm_write_irq(struct amdgpu_device *adev,
1806 					      struct amdgpu_irq_src *source,
1807 					      struct amdgpu_iv_entry *entry)
1808 {
1809 	dev_dbg_ratelimited(adev->dev,
1810 		"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
1811 	sdma_v4_4_2_print_iv_entry(adev, entry);
1812 	return 0;
1813 }
1814 
1815 static void sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1816 	struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1817 {
1818 	uint32_t data, def;
1819 	int i;
1820 
1821 	/* leave as default if it is not driver controlled */
1822 	if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS))
1823 		return;
1824 
1825 	if (enable) {
1826 		for_each_inst(i, inst_mask) {
1827 			/* 1-not override: enable sdma mem light sleep */
1828 			def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1829 			data |= SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1830 			if (def != data)
1831 				WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1832 		}
1833 	} else {
1834 		for_each_inst(i, inst_mask) {
1835 			/* 0-override:disable sdma mem light sleep */
1836 			def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1837 			data &= ~SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1838 			if (def != data)
1839 				WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1840 		}
1841 	}
1842 }
1843 
1844 static void sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1845 	struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1846 {
1847 	uint32_t data, def;
1848 	int i;
1849 
1850 	/* leave as default if it is not driver controlled */
1851 	if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG))
1852 		return;
1853 
1854 	if (enable) {
1855 		for_each_inst(i, inst_mask) {
1856 			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1857 			data &= ~(SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1858 				  SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1859 				  SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1860 				  SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1861 				  SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1862 				  SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1863 			if (def != data)
1864 				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1865 		}
1866 	} else {
1867 		for_each_inst(i, inst_mask) {
1868 			def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1869 			data |= (SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1870 				 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1871 				 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1872 				 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1873 				 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1874 				 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1875 			if (def != data)
1876 				WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1877 		}
1878 	}
1879 }
1880 
1881 static int sdma_v4_4_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1882 					  enum amd_clockgating_state state)
1883 {
1884 	struct amdgpu_device *adev = ip_block->adev;
1885 	uint32_t inst_mask;
1886 
1887 	if (amdgpu_sriov_vf(adev))
1888 		return 0;
1889 
1890 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1891 
1892 	sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1893 		adev, state == AMD_CG_STATE_GATE, inst_mask);
1894 	sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1895 		adev, state == AMD_CG_STATE_GATE, inst_mask);
1896 	return 0;
1897 }
1898 
1899 static int sdma_v4_4_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
1900 					  enum amd_powergating_state state)
1901 {
1902 	return 0;
1903 }
1904 
1905 static void sdma_v4_4_2_get_clockgating_state(void *handle, u64 *flags)
1906 {
1907 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1908 	int data;
1909 
1910 	if (amdgpu_sriov_vf(adev))
1911 		*flags = 0;
1912 
1913 	/* AMD_CG_SUPPORT_SDMA_MGCG */
1914 	data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_CLK_CTRL));
1915 	if (!(data & SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK))
1916 		*flags |= AMD_CG_SUPPORT_SDMA_MGCG;
1917 
1918 	/* AMD_CG_SUPPORT_SDMA_LS */
1919 	data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_POWER_CNTL));
1920 	if (data & SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
1921 		*flags |= AMD_CG_SUPPORT_SDMA_LS;
1922 }
1923 
1924 static void sdma_v4_4_2_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
1925 {
1926 	struct amdgpu_device *adev = ip_block->adev;
1927 	int i, j;
1928 	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
1929 	uint32_t instance_offset;
1930 
1931 	if (!adev->sdma.ip_dump)
1932 		return;
1933 
1934 	drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
1935 	for (i = 0; i < adev->sdma.num_instances; i++) {
1936 		instance_offset = i * reg_count;
1937 		drm_printf(p, "\nInstance:%d\n", i);
1938 
1939 		for (j = 0; j < reg_count; j++)
1940 			drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_4_4_2[j].reg_name,
1941 				   adev->sdma.ip_dump[instance_offset + j]);
1942 	}
1943 }
1944 
1945 static void sdma_v4_4_2_dump_ip_state(struct amdgpu_ip_block *ip_block)
1946 {
1947 	struct amdgpu_device *adev = ip_block->adev;
1948 	int i, j;
1949 	uint32_t instance_offset;
1950 	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_4_2);
1951 
1952 	if (!adev->sdma.ip_dump)
1953 		return;
1954 
1955 	for (i = 0; i < adev->sdma.num_instances; i++) {
1956 		instance_offset = i * reg_count;
1957 		for (j = 0; j < reg_count; j++)
1958 			adev->sdma.ip_dump[instance_offset + j] =
1959 				RREG32(sdma_v4_4_2_get_reg_offset(adev, i,
1960 				       sdma_reg_list_4_4_2[j].reg_offset));
1961 	}
1962 }
1963 
1964 const struct amd_ip_funcs sdma_v4_4_2_ip_funcs = {
1965 	.name = "sdma_v4_4_2",
1966 	.early_init = sdma_v4_4_2_early_init,
1967 	.late_init = sdma_v4_4_2_late_init,
1968 	.sw_init = sdma_v4_4_2_sw_init,
1969 	.sw_fini = sdma_v4_4_2_sw_fini,
1970 	.hw_init = sdma_v4_4_2_hw_init,
1971 	.hw_fini = sdma_v4_4_2_hw_fini,
1972 	.suspend = sdma_v4_4_2_suspend,
1973 	.resume = sdma_v4_4_2_resume,
1974 	.is_idle = sdma_v4_4_2_is_idle,
1975 	.wait_for_idle = sdma_v4_4_2_wait_for_idle,
1976 	.soft_reset = sdma_v4_4_2_soft_reset,
1977 	.set_clockgating_state = sdma_v4_4_2_set_clockgating_state,
1978 	.set_powergating_state = sdma_v4_4_2_set_powergating_state,
1979 	.get_clockgating_state = sdma_v4_4_2_get_clockgating_state,
1980 	.dump_ip_state = sdma_v4_4_2_dump_ip_state,
1981 	.print_ip_state = sdma_v4_4_2_print_ip_state,
1982 };
1983 
1984 static const struct amdgpu_ring_funcs sdma_v4_4_2_ring_funcs = {
1985 	.type = AMDGPU_RING_TYPE_SDMA,
1986 	.align_mask = 0xff,
1987 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1988 	.support_64bit_ptrs = true,
1989 	.get_rptr = sdma_v4_4_2_ring_get_rptr,
1990 	.get_wptr = sdma_v4_4_2_ring_get_wptr,
1991 	.set_wptr = sdma_v4_4_2_ring_set_wptr,
1992 	.emit_frame_size =
1993 		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
1994 		3 + /* hdp invalidate */
1995 		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
1996 		/* sdma_v4_4_2_ring_emit_vm_flush */
1997 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1998 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1999 		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
2000 	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
2001 	.emit_ib = sdma_v4_4_2_ring_emit_ib,
2002 	.emit_fence = sdma_v4_4_2_ring_emit_fence,
2003 	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
2004 	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
2005 	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
2006 	.test_ring = sdma_v4_4_2_ring_test_ring,
2007 	.test_ib = sdma_v4_4_2_ring_test_ib,
2008 	.insert_nop = sdma_v4_4_2_ring_insert_nop,
2009 	.pad_ib = sdma_v4_4_2_ring_pad_ib,
2010 	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
2011 	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
2012 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2013 	.reset = sdma_v4_4_2_reset_queue,
2014 };
2015 
2016 static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
2017 	.type = AMDGPU_RING_TYPE_SDMA,
2018 	.align_mask = 0xff,
2019 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
2020 	.support_64bit_ptrs = true,
2021 	.get_rptr = sdma_v4_4_2_ring_get_rptr,
2022 	.get_wptr = sdma_v4_4_2_page_ring_get_wptr,
2023 	.set_wptr = sdma_v4_4_2_page_ring_set_wptr,
2024 	.emit_frame_size =
2025 		6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
2026 		3 + /* hdp invalidate */
2027 		6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
2028 		/* sdma_v4_4_2_ring_emit_vm_flush */
2029 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2030 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
2031 		10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
2032 	.emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
2033 	.emit_ib = sdma_v4_4_2_ring_emit_ib,
2034 	.emit_fence = sdma_v4_4_2_ring_emit_fence,
2035 	.emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
2036 	.emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
2037 	.emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
2038 	.test_ring = sdma_v4_4_2_ring_test_ring,
2039 	.test_ib = sdma_v4_4_2_ring_test_ib,
2040 	.insert_nop = sdma_v4_4_2_ring_insert_nop,
2041 	.pad_ib = sdma_v4_4_2_ring_pad_ib,
2042 	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
2043 	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
2044 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2045 };
2046 
2047 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev)
2048 {
2049 	int i, dev_inst;
2050 
2051 	for (i = 0; i < adev->sdma.num_instances; i++) {
2052 		adev->sdma.instance[i].ring.funcs = &sdma_v4_4_2_ring_funcs;
2053 		adev->sdma.instance[i].ring.me = i;
2054 		if (adev->sdma.has_page_queue) {
2055 			adev->sdma.instance[i].page.funcs =
2056 				&sdma_v4_4_2_page_ring_funcs;
2057 			adev->sdma.instance[i].page.me = i;
2058 		}
2059 
2060 		dev_inst = GET_INST(SDMA0, i);
2061 		/* AID to which SDMA belongs depends on physical instance */
2062 		adev->sdma.instance[i].aid_id =
2063 			dev_inst / adev->sdma.num_inst_per_aid;
2064 	}
2065 }
2066 
2067 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_trap_irq_funcs = {
2068 	.set = sdma_v4_4_2_set_trap_irq_state,
2069 	.process = sdma_v4_4_2_process_trap_irq,
2070 };
2071 
2072 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_illegal_inst_irq_funcs = {
2073 	.process = sdma_v4_4_2_process_illegal_inst_irq,
2074 };
2075 
2076 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ecc_irq_funcs = {
2077 	.set = sdma_v4_4_2_set_ecc_irq_state,
2078 	.process = amdgpu_sdma_process_ecc_irq,
2079 };
2080 
2081 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_vm_hole_irq_funcs = {
2082 	.process = sdma_v4_4_2_process_vm_hole_irq,
2083 };
2084 
2085 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_doorbell_invalid_irq_funcs = {
2086 	.process = sdma_v4_4_2_process_doorbell_invalid_irq,
2087 };
2088 
2089 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_pool_timeout_irq_funcs = {
2090 	.process = sdma_v4_4_2_process_pool_timeout_irq,
2091 };
2092 
2093 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_srbm_write_irq_funcs = {
2094 	.process = sdma_v4_4_2_process_srbm_write_irq,
2095 };
2096 
2097 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev)
2098 {
2099 	adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
2100 	adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
2101 	adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
2102 	adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
2103 	adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
2104 	adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
2105 
2106 	adev->sdma.trap_irq.funcs = &sdma_v4_4_2_trap_irq_funcs;
2107 	adev->sdma.illegal_inst_irq.funcs = &sdma_v4_4_2_illegal_inst_irq_funcs;
2108 	adev->sdma.ecc_irq.funcs = &sdma_v4_4_2_ecc_irq_funcs;
2109 	adev->sdma.vm_hole_irq.funcs = &sdma_v4_4_2_vm_hole_irq_funcs;
2110 	adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_4_2_doorbell_invalid_irq_funcs;
2111 	adev->sdma.pool_timeout_irq.funcs = &sdma_v4_4_2_pool_timeout_irq_funcs;
2112 	adev->sdma.srbm_write_irq.funcs = &sdma_v4_4_2_srbm_write_irq_funcs;
2113 }
2114 
2115 /**
2116  * sdma_v4_4_2_emit_copy_buffer - copy buffer using the sDMA engine
2117  *
2118  * @ib: indirect buffer to copy to
2119  * @src_offset: src GPU address
2120  * @dst_offset: dst GPU address
2121  * @byte_count: number of bytes to xfer
2122  * @copy_flags: copy flags for the buffers
2123  *
2124  * Copy GPU buffers using the DMA engine.
2125  * Used by the amdgpu ttm implementation to move pages if
2126  * registered as the asic copy callback.
2127  */
2128 static void sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib *ib,
2129 				       uint64_t src_offset,
2130 				       uint64_t dst_offset,
2131 				       uint32_t byte_count,
2132 				       uint32_t copy_flags)
2133 {
2134 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
2135 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
2136 		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
2137 	ib->ptr[ib->length_dw++] = byte_count - 1;
2138 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
2139 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
2140 	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
2141 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2142 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2143 }
2144 
2145 /**
2146  * sdma_v4_4_2_emit_fill_buffer - fill buffer using the sDMA engine
2147  *
2148  * @ib: indirect buffer to copy to
2149  * @src_data: value to write to buffer
2150  * @dst_offset: dst GPU address
2151  * @byte_count: number of bytes to xfer
2152  *
2153  * Fill GPU buffers using the DMA engine.
2154  */
2155 static void sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib *ib,
2156 				       uint32_t src_data,
2157 				       uint64_t dst_offset,
2158 				       uint32_t byte_count)
2159 {
2160 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
2161 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2162 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2163 	ib->ptr[ib->length_dw++] = src_data;
2164 	ib->ptr[ib->length_dw++] = byte_count - 1;
2165 }
2166 
2167 static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
2168 	.copy_max_bytes = 0x400000,
2169 	.copy_num_dw = 7,
2170 	.emit_copy_buffer = sdma_v4_4_2_emit_copy_buffer,
2171 
2172 	.fill_max_bytes = 0x400000,
2173 	.fill_num_dw = 5,
2174 	.emit_fill_buffer = sdma_v4_4_2_emit_fill_buffer,
2175 };
2176 
2177 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
2178 {
2179 	adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
2180 	if (adev->sdma.has_page_queue)
2181 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
2182 	else
2183 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
2184 }
2185 
2186 static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
2187 	.copy_pte_num_dw = 7,
2188 	.copy_pte = sdma_v4_4_2_vm_copy_pte,
2189 
2190 	.write_pte = sdma_v4_4_2_vm_write_pte,
2191 	.set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
2192 };
2193 
2194 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev)
2195 {
2196 	struct drm_gpu_scheduler *sched;
2197 	unsigned i;
2198 
2199 	adev->vm_manager.vm_pte_funcs = &sdma_v4_4_2_vm_pte_funcs;
2200 	for (i = 0; i < adev->sdma.num_instances; i++) {
2201 		if (adev->sdma.has_page_queue)
2202 			sched = &adev->sdma.instance[i].page.sched;
2203 		else
2204 			sched = &adev->sdma.instance[i].ring.sched;
2205 		adev->vm_manager.vm_pte_scheds[i] = sched;
2206 	}
2207 	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
2208 }
2209 
2210 const struct amdgpu_ip_block_version sdma_v4_4_2_ip_block = {
2211 	.type = AMD_IP_BLOCK_TYPE_SDMA,
2212 	.major = 4,
2213 	.minor = 4,
2214 	.rev = 2,
2215 	.funcs = &sdma_v4_4_2_ip_funcs,
2216 };
2217 
2218 static int sdma_v4_4_2_xcp_resume(void *handle, uint32_t inst_mask)
2219 {
2220 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2221 	int r;
2222 
2223 	if (!amdgpu_sriov_vf(adev))
2224 		sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
2225 
2226 	r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
2227 
2228 	return r;
2229 }
2230 
2231 static int sdma_v4_4_2_xcp_suspend(void *handle, uint32_t inst_mask)
2232 {
2233 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2234 	uint32_t tmp_mask = inst_mask;
2235 	int i;
2236 
2237 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2238 		for_each_inst(i, tmp_mask) {
2239 			amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
2240 				       AMDGPU_SDMA_IRQ_INSTANCE0 + i);
2241 		}
2242 	}
2243 
2244 	sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
2245 	sdma_v4_4_2_inst_enable(adev, false, inst_mask);
2246 
2247 	return 0;
2248 }
2249 
2250 struct amdgpu_xcp_ip_funcs sdma_v4_4_2_xcp_funcs = {
2251 	.suspend = &sdma_v4_4_2_xcp_suspend,
2252 	.resume = &sdma_v4_4_2_xcp_resume
2253 };
2254 
2255 static const struct amdgpu_ras_err_status_reg_entry sdma_v4_2_2_ue_reg_list[] = {
2256 	{AMDGPU_RAS_REG_ENTRY(SDMA0, 0, regSDMA_UE_ERR_STATUS_LO, regSDMA_UE_ERR_STATUS_HI),
2257 	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"},
2258 };
2259 
2260 static const struct amdgpu_ras_memory_id_entry sdma_v4_4_2_ras_memory_list[] = {
2261 	{AMDGPU_SDMA_MBANK_DATA_BUF0, "SDMA_MBANK_DATA_BUF0"},
2262 	{AMDGPU_SDMA_MBANK_DATA_BUF1, "SDMA_MBANK_DATA_BUF1"},
2263 	{AMDGPU_SDMA_MBANK_DATA_BUF2, "SDMA_MBANK_DATA_BUF2"},
2264 	{AMDGPU_SDMA_MBANK_DATA_BUF3, "SDMA_MBANK_DATA_BUF3"},
2265 	{AMDGPU_SDMA_MBANK_DATA_BUF4, "SDMA_MBANK_DATA_BUF4"},
2266 	{AMDGPU_SDMA_MBANK_DATA_BUF5, "SDMA_MBANK_DATA_BUF5"},
2267 	{AMDGPU_SDMA_MBANK_DATA_BUF6, "SDMA_MBANK_DATA_BUF6"},
2268 	{AMDGPU_SDMA_MBANK_DATA_BUF7, "SDMA_MBANK_DATA_BUF7"},
2269 	{AMDGPU_SDMA_MBANK_DATA_BUF8, "SDMA_MBANK_DATA_BUF8"},
2270 	{AMDGPU_SDMA_MBANK_DATA_BUF9, "SDMA_MBANK_DATA_BUF9"},
2271 	{AMDGPU_SDMA_MBANK_DATA_BUF10, "SDMA_MBANK_DATA_BUF10"},
2272 	{AMDGPU_SDMA_MBANK_DATA_BUF11, "SDMA_MBANK_DATA_BUF11"},
2273 	{AMDGPU_SDMA_MBANK_DATA_BUF12, "SDMA_MBANK_DATA_BUF12"},
2274 	{AMDGPU_SDMA_MBANK_DATA_BUF13, "SDMA_MBANK_DATA_BUF13"},
2275 	{AMDGPU_SDMA_MBANK_DATA_BUF14, "SDMA_MBANK_DATA_BUF14"},
2276 	{AMDGPU_SDMA_MBANK_DATA_BUF15, "SDMA_MBANK_DATA_BUF15"},
2277 	{AMDGPU_SDMA_UCODE_BUF, "SDMA_UCODE_BUF"},
2278 	{AMDGPU_SDMA_RB_CMD_BUF, "SDMA_RB_CMD_BUF"},
2279 	{AMDGPU_SDMA_IB_CMD_BUF, "SDMA_IB_CMD_BUF"},
2280 	{AMDGPU_SDMA_UTCL1_RD_FIFO, "SDMA_UTCL1_RD_FIFO"},
2281 	{AMDGPU_SDMA_UTCL1_RDBST_FIFO, "SDMA_UTCL1_RDBST_FIFO"},
2282 	{AMDGPU_SDMA_UTCL1_WR_FIFO, "SDMA_UTCL1_WR_FIFO"},
2283 	{AMDGPU_SDMA_DATA_LUT_FIFO, "SDMA_DATA_LUT_FIFO"},
2284 	{AMDGPU_SDMA_SPLIT_DAT_BUF, "SDMA_SPLIT_DAT_BUF"},
2285 };
2286 
2287 static void sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device *adev,
2288 						   uint32_t sdma_inst,
2289 						   void *ras_err_status)
2290 {
2291 	struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
2292 	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2293 	unsigned long ue_count = 0;
2294 	struct amdgpu_smuio_mcm_config_info mcm_info = {
2295 		.socket_id = adev->smuio.funcs->get_socket_id(adev),
2296 		.die_id = adev->sdma.instance[sdma_inst].aid_id,
2297 	};
2298 
2299 	/* sdma v4_4_2 doesn't support query ce counts */
2300 	amdgpu_ras_inst_query_ras_error_count(adev,
2301 					sdma_v4_2_2_ue_reg_list,
2302 					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2303 					sdma_v4_4_2_ras_memory_list,
2304 					ARRAY_SIZE(sdma_v4_4_2_ras_memory_list),
2305 					sdma_dev_inst,
2306 					AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
2307 					&ue_count);
2308 
2309 	amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
2310 }
2311 
2312 static void sdma_v4_4_2_query_ras_error_count(struct amdgpu_device *adev,
2313 					      void *ras_err_status)
2314 {
2315 	uint32_t inst_mask;
2316 	int i = 0;
2317 
2318 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2319 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2320 		for_each_inst(i, inst_mask)
2321 			sdma_v4_4_2_inst_query_ras_error_count(adev, i, ras_err_status);
2322 	} else {
2323 		dev_warn(adev->dev, "SDMA RAS is not supported\n");
2324 	}
2325 }
2326 
2327 static void sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device *adev,
2328 						   uint32_t sdma_inst)
2329 {
2330 	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2331 
2332 	amdgpu_ras_inst_reset_ras_error_count(adev,
2333 					sdma_v4_2_2_ue_reg_list,
2334 					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2335 					sdma_dev_inst);
2336 }
2337 
2338 static void sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device *adev)
2339 {
2340 	uint32_t inst_mask;
2341 	int i = 0;
2342 
2343 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2344 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2345 		for_each_inst(i, inst_mask)
2346 			sdma_v4_4_2_inst_reset_ras_error_count(adev, i);
2347 	} else {
2348 		dev_warn(adev->dev, "SDMA RAS is not supported\n");
2349 	}
2350 }
2351 
2352 static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = {
2353 	.query_ras_error_count = sdma_v4_4_2_query_ras_error_count,
2354 	.reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count,
2355 };
2356 
2357 static int sdma_v4_4_2_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
2358 				       enum aca_smu_type type, void *data)
2359 {
2360 	struct aca_bank_info info;
2361 	u64 misc0;
2362 	int ret;
2363 
2364 	ret = aca_bank_info_decode(bank, &info);
2365 	if (ret)
2366 		return ret;
2367 
2368 	misc0 = bank->regs[ACA_REG_IDX_MISC0];
2369 	switch (type) {
2370 	case ACA_SMU_TYPE_UE:
2371 		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
2372 						     1ULL);
2373 		break;
2374 	case ACA_SMU_TYPE_CE:
2375 		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE,
2376 						     ACA_REG__MISC0__ERRCNT(misc0));
2377 		break;
2378 	default:
2379 		return -EINVAL;
2380 	}
2381 
2382 	return ret;
2383 }
2384 
2385 /* CODE_SDMA0 - CODE_SDMA4, reference to smu driver if header file */
2386 static int sdma_v4_4_2_err_codes[] = { 33, 34, 35, 36 };
2387 
2388 static bool sdma_v4_4_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
2389 					  enum aca_smu_type type, void *data)
2390 {
2391 	u32 instlo;
2392 
2393 	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
2394 	instlo &= GENMASK(31, 1);
2395 
2396 	if (instlo != mmSMNAID_AID0_MCA_SMU)
2397 		return false;
2398 
2399 	if (aca_bank_check_error_codes(handle->adev, bank,
2400 				       sdma_v4_4_2_err_codes,
2401 				       ARRAY_SIZE(sdma_v4_4_2_err_codes)))
2402 		return false;
2403 
2404 	return true;
2405 }
2406 
2407 static const struct aca_bank_ops sdma_v4_4_2_aca_bank_ops = {
2408 	.aca_bank_parser = sdma_v4_4_2_aca_bank_parser,
2409 	.aca_bank_is_valid = sdma_v4_4_2_aca_bank_is_valid,
2410 };
2411 
2412 static const struct aca_info sdma_v4_4_2_aca_info = {
2413 	.hwip = ACA_HWIP_TYPE_SMU,
2414 	.mask = ACA_ERROR_UE_MASK,
2415 	.bank_ops = &sdma_v4_4_2_aca_bank_ops,
2416 };
2417 
2418 static int sdma_v4_4_2_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
2419 {
2420 	int r;
2421 
2422 	r = amdgpu_sdma_ras_late_init(adev, ras_block);
2423 	if (r)
2424 		return r;
2425 
2426 	return amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__SDMA,
2427 				   &sdma_v4_4_2_aca_info, NULL);
2428 }
2429 
2430 static struct amdgpu_sdma_ras sdma_v4_4_2_ras = {
2431 	.ras_block = {
2432 		.hw_ops = &sdma_v4_4_2_ras_hw_ops,
2433 		.ras_late_init = sdma_v4_4_2_ras_late_init,
2434 	},
2435 };
2436 
2437 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev)
2438 {
2439 	adev->sdma.ras = &sdma_v4_4_2_ras;
2440 }
2441