xref: /linux/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 /*
2  * Copyright 2025 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include "amdgpu.h"
30 #include "amdgpu_ucode.h"
31 #include "amdgpu_sdma.h"
32 #include "amdgpu_trace.h"
33 
34 #include "gc/gc_12_1_0_offset.h"
35 #include "gc/gc_12_1_0_sh_mask.h"
36 #include "ivsrcid/gfx/irqsrcs_gfx_12_1_0.h"
37 
38 #include "soc15_common.h"
39 #include "soc15.h"
40 #include "sdma_v7_1_0_pkt_open.h"
41 #include "nbio_v4_3.h"
42 #include "sdma_common.h"
43 #include "sdma_v7_1.h"
44 #include "v12_structs.h"
45 #include "mes_userqueue.h"
46 #include "soc_v1_0.h"
47 
48 MODULE_FIRMWARE("amdgpu/sdma_7_1_0.bin");
49 
50 #define SDMA1_REG_OFFSET 0x600
51 #define SDMA0_SDMA_IDX_0_END 0x450
52 #define SDMA1_HYP_DEC_REG_OFFSET 0x30
53 
54 static const struct amdgpu_hwip_reg_entry sdma_reg_list_7_1[] = {
55 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS_REG),
56 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS1_REG),
57 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS2_REG),
58 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS3_REG),
59 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS4_REG),
60 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS5_REG),
61 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_STATUS6_REG),
62 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UCODE_REV),
63 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_RB_RPTR_FETCH_HI),
64 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_RB_RPTR_FETCH),
65 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_RD_STATUS),
66 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_WR_STATUS),
67 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_RD_XNACK0),
68 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_RD_XNACK1),
69 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_WR_XNACK0),
70 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_UTCL1_WR_XNACK1),
71 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_CNTL),
72 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_RPTR),
73 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_RPTR_HI),
74 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_WPTR),
75 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI),
76 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_OFFSET),
77 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_BASE_LO),
78 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_BASE_HI),
79 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_CNTL),
80 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_RPTR),
81 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_IB_SUB_REMAIN),
82 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE0_DUMMY_REG),
83 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE_STATUS0),
84 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_CNTL),
85 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_RPTR),
86 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_RPTR_HI),
87 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_WPTR),
88 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_RB_WPTR_HI),
89 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_OFFSET),
90 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_BASE_LO),
91 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_BASE_HI),
92 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_RPTR),
93 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_IB_SUB_REMAIN),
94 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE1_DUMMY_REG),
95 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_CNTL),
96 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_RPTR),
97 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_RPTR_HI),
98 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_WPTR),
99 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_RB_WPTR_HI),
100 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_OFFSET),
101 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_BASE_LO),
102 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_BASE_HI),
103 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_RPTR),
104 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_IB_SUB_REMAIN),
105 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_QUEUE2_DUMMY_REG),
106 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_INT_STATUS),
107 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_VM_CNTL),
108 	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
109 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_SDMA_CHICKEN_BITS),
110 };
111 
112 static void sdma_v7_1_set_ring_funcs(struct amdgpu_device *adev);
113 static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev);
114 static void sdma_v7_1_set_irq_funcs(struct amdgpu_device *adev);
115 static int sdma_v7_1_inst_start(struct amdgpu_device *adev,
116 				uint32_t inst_mask);
117 
118 static u32 sdma_v7_1_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
119 {
120 	u32 base;
121 	u32 dev_inst = GET_INST(SDMA0, instance);
122 	int xcc_id = adev->sdma.instance[instance].xcc_id;
123 	int xcc_inst = dev_inst % adev->sdma.num_inst_per_xcc;
124 
125 	if (internal_offset >= SDMA0_SDMA_IDX_0_END) {
126 		base = adev->reg_offset[GC_HWIP][xcc_id][1];
127 		if (xcc_inst != 0)
128 			internal_offset += SDMA1_HYP_DEC_REG_OFFSET * xcc_inst;
129 	} else {
130 		base = adev->reg_offset[GC_HWIP][xcc_id][0];
131 		if (xcc_inst != 0)
132 			internal_offset += SDMA1_REG_OFFSET * xcc_inst;
133 	}
134 
135 	return base + internal_offset;
136 }
137 
138 static unsigned sdma_v7_1_ring_init_cond_exec(struct amdgpu_ring *ring,
139 					      uint64_t addr)
140 {
141 	unsigned ret;
142 
143 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COND_EXE));
144 	amdgpu_ring_write(ring, lower_32_bits(addr));
145 	amdgpu_ring_write(ring, upper_32_bits(addr));
146 	amdgpu_ring_write(ring, 1);
147 	/* this is the offset we need patch later */
148 	ret = ring->wptr & ring->buf_mask;
149 	/* insert dummy here and patch it later */
150 	amdgpu_ring_write(ring, 0);
151 
152 	return ret;
153 }
154 
155 /**
156  * sdma_v7_1_ring_get_rptr - get the current read pointer
157  *
158  * @ring: amdgpu ring pointer
159  *
160  * Get the current rptr from the hardware.
161  */
162 static uint64_t sdma_v7_1_ring_get_rptr(struct amdgpu_ring *ring)
163 {
164 	u64 *rptr;
165 
166 	/* XXX check if swapping is necessary on BE */
167 	rptr = (u64 *)ring->rptr_cpu_addr;
168 
169 	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
170 	return ((*rptr) >> 2);
171 }
172 
173 /**
174  * sdma_v7_1_ring_get_wptr - get the current write pointer
175  *
176  * @ring: amdgpu ring pointer
177  *
178  * Get the current wptr from the hardware.
179  */
180 static uint64_t sdma_v7_1_ring_get_wptr(struct amdgpu_ring *ring)
181 {
182 	u64 wptr = 0;
183 
184 	if (ring->use_doorbell) {
185 		/* XXX check if swapping is necessary on BE */
186 		wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
187 		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
188 	}
189 
190 	return wptr >> 2;
191 }
192 
193 /**
194  * sdma_v7_1_ring_set_wptr - commit the write pointer
195  *
196  * @ring: amdgpu ring pointer
197  *
198  * Write the wptr back to the hardware.
199  */
200 static void sdma_v7_1_ring_set_wptr(struct amdgpu_ring *ring)
201 {
202 	struct amdgpu_device *adev = ring->adev;
203 
204 	DRM_DEBUG("Setting write pointer\n");
205 
206 	if (ring->use_doorbell) {
207 		DRM_DEBUG("Using doorbell -- "
208 			  "wptr_offs == 0x%08x "
209 			  "lower_32_bits(ring->wptr) << 2 == 0x%08x "
210 			  "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
211 			  ring->wptr_offs,
212 			  lower_32_bits(ring->wptr << 2),
213 			  upper_32_bits(ring->wptr << 2));
214 		/* XXX check if swapping is necessary on BE */
215 		atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
216 			     ring->wptr << 2);
217 		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
218 			  ring->doorbell_index, ring->wptr << 2);
219 		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
220 	} else {
221 		DRM_DEBUG("Not using doorbell -- "
222 			  "regSDMA%i_GFX_RB_WPTR == 0x%08x "
223 			  "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
224 			  ring->me,
225 			  lower_32_bits(ring->wptr << 2),
226 			  ring->me,
227 			  upper_32_bits(ring->wptr << 2));
228 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev,
229 							     ring->me,
230 							     regSDMA0_SDMA_QUEUE0_RB_WPTR),
231 				lower_32_bits(ring->wptr << 2));
232 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev,
233 							     ring->me,
234 							     regSDMA0_SDMA_QUEUE0_RB_WPTR_HI),
235 				upper_32_bits(ring->wptr << 2));
236 	}
237 }
238 
239 static void sdma_v7_1_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
240 {
241 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
242 	const u32 nop = ring->funcs->nop;
243 
244 	if (count && sdma->burst_nop) {
245 		--count;
246 		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
247 	}
248 
249 	amdgpu_ring_fill(ring, nop, count);
250 }
251 
252 /**
253  * sdma_v7_1_ring_emit_ib - Schedule an IB on the DMA engine
254  *
255  * @ring: amdgpu ring pointer
256  * @job: job to retrieve vmid from
257  * @ib: IB object to schedule
258  * @flags: unused
259  *
260  * Schedule an IB in the DMA ring.
261  */
262 static void sdma_v7_1_ring_emit_ib(struct amdgpu_ring *ring,
263 				   struct amdgpu_job *job,
264 				   struct amdgpu_ib *ib,
265 				   uint32_t flags)
266 {
267 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
268 	uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
269 
270 	/* An IB packet must end on a 8 DW boundary--the next dword
271 	 * must be on a 8-dword boundary. Our IB packet below is 6
272 	 * dwords long, thus add x number of NOPs, such that, in
273 	 * modular arithmetic,
274 	 * wptr + 6 + x = 8k, k >= 0, which in C is,
275 	 * (wptr + 6 + x) % 8 = 0.
276 	 * The expression below, is a solution of x.
277 	 */
278 	sdma_v7_1_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
279 
280 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_INDIRECT) |
281 			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
282 	/* base must be 32 byte aligned */
283 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
284 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
285 	amdgpu_ring_write(ring, ib->length_dw);
286 	amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
287 	amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
288 }
289 
290 /**
291  * sdma_v7_1_ring_emit_mem_sync - flush the IB by graphics cache rinse
292  *
293  * @ring: amdgpu ring pointer
294  *
295  * flush the IB by graphics cache rinse.
296  */
297 static void sdma_v7_1_ring_emit_mem_sync(struct amdgpu_ring *ring)
298 {
299 	uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | SDMA_GCR_GLM_INV |
300 		SDMA_GCR_GL1_INV | SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
301 		SDMA_GCR_GLI_INV(1);
302 
303 	/* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
304 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_GCR_REQ));
305 	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
306 	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_56_32(0));
307 	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_0(gcr_cntl) |
308 			  SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_15_7(0));
309 	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_16(0));
310 	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD5_LIMIT_VA_56_48(0) |
311 			  SDMA_PKT_GCR_REQ_PAYLOAD5_VMID(0));
312 }
313 
314 
315 /**
316  * sdma_v7_1_ring_emit_fence - emit a fence on the DMA ring
317  *
318  * @ring: amdgpu ring pointer
319  * @addr: address
320  * @seq: fence seq number
321  * @flags: fence flags
322  *
323  * Add a DMA fence packet to the ring to write
324  * the fence seq number and DMA trap packet to generate
325  * an interrupt if needed.
326  */
327 static void sdma_v7_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
328 				      unsigned flags)
329 {
330 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
331 	/* write the fence */
332 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
333 			  SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
334 	/* zero in first two bits */
335 	WARN_ON(addr & 0x3);
336 	amdgpu_ring_write(ring, lower_32_bits(addr));
337 	amdgpu_ring_write(ring, upper_32_bits(addr));
338 	amdgpu_ring_write(ring, lower_32_bits(seq));
339 
340 	/* optionally write high bits as well */
341 	if (write64bit) {
342 		addr += 4;
343 		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
344 				  SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
345 		/* zero in first two bits */
346 		WARN_ON(addr & 0x3);
347 		amdgpu_ring_write(ring, lower_32_bits(addr));
348 		amdgpu_ring_write(ring, upper_32_bits(addr));
349 		amdgpu_ring_write(ring, upper_32_bits(seq));
350 	}
351 
352 	if (flags & AMDGPU_FENCE_FLAG_INT) {
353 		/* generate an interrupt */
354 		amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_TRAP));
355 		amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
356 	}
357 }
358 
359 /**
360  * sdma_v7_1_inst_gfx_stop - stop the gfx async dma engines
361  *
362  * @adev: amdgpu_device pointer
363  * @inst_mask: mask of dma engine instances to be disabled
364  *
365  * Stop the gfx async dma ring buffers.
366  */
367 static void sdma_v7_1_inst_gfx_stop(struct amdgpu_device *adev,
368 				    uint32_t inst_mask)
369 {
370 	u32 rb_cntl, ib_cntl;
371 	int i;
372 
373 	for_each_inst(i, inst_mask) {
374 		rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL));
375 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_ENABLE, 0);
376 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL), rb_cntl);
377 		ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL));
378 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_SDMA_QUEUE0_IB_CNTL, IB_ENABLE, 0);
379 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL), ib_cntl);
380 	}
381 }
382 
383 /**
384  * sdma_v7_1_inst_rlc_stop - stop the compute async dma engines
385  *
386  * @adev: amdgpu_device pointer
387  * @inst_mask: mask of dma engine instances to be disabled
388  *
389  * Stop the compute async dma queues.
390  */
391 static void sdma_v7_1_inst_rlc_stop(struct amdgpu_device *adev,
392 				    uint32_t inst_mask)
393 {
394 	/* XXX todo */
395 }
396 
397 /**
398  * sdma_v7_1_inst_ctx_switch_enable - stop the async dma engines context switch
399  *
400  * @adev: amdgpu_device pointer
401  * @enable: enable/disable the DMA MEs context switch.
402  * @inst_mask: mask of dma engine instances to be enabled
403  *
404  * Halt or unhalt the async dma engines context switch.
405  */
406 static void sdma_v7_1_inst_ctx_switch_enable(struct amdgpu_device *adev,
407 					     bool enable, uint32_t inst_mask)
408 {
409 	int i;
410 
411 	for_each_inst(i, inst_mask) {
412 		WREG32_SOC15_IP(GC,
413 			sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_TIMEOUT), 0x80);
414 	}
415 }
416 
417 /**
418  * sdma_v7_1_inst_enable - stop the async dma engines
419  *
420  * @adev: amdgpu_device pointer
421  * @enable: enable/disable the DMA MEs.
422  * @inst_mask: mask of dma engine instances to be enabled
423  *
424  * Halt or unhalt the async dma engines.
425  */
426 static void sdma_v7_1_inst_enable(struct amdgpu_device *adev,
427 				  bool enable, uint32_t inst_mask)
428 {
429 	u32 mcu_cntl;
430 	int i;
431 
432 	if (!enable) {
433 		sdma_v7_1_inst_gfx_stop(adev, inst_mask);
434 		sdma_v7_1_inst_rlc_stop(adev, inst_mask);
435 	}
436 
437 	if (amdgpu_sriov_vf(adev))
438 		return;
439 
440 	for_each_inst(i, inst_mask) {
441 		mcu_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL));
442 		mcu_cntl = REG_SET_FIELD(mcu_cntl, SDMA0_SDMA_MCU_CNTL, HALT, enable ? 0 : 1);
443 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL), mcu_cntl);
444 	}
445 }
446 
447 /**
448  * sdma_v7_1_gfx_resume_instance - start/restart a certain sdma engine
449  *
450  * @adev: amdgpu_device pointer
451  * @i: instance
452  * @restore: used to restore wptr when restart
453  *
454  * Set up the gfx DMA ring buffers and enable them. On restart, we will restore wptr and rptr.
455  * Return 0 for success.
456  */
457 static int sdma_v7_1_gfx_resume_instance(struct amdgpu_device *adev, int i, bool restore)
458 {
459 	struct amdgpu_ring *ring;
460 	u32 rb_cntl, ib_cntl;
461 	u32 rb_bufsz;
462 	u32 doorbell;
463 	u32 doorbell_offset;
464 	u32 temp;
465 	u64 wptr_gpu_addr;
466 	int r;
467 
468 	ring = &adev->sdma.instance[i].ring;
469 
470 	/* Set ring buffer size in dwords */
471 	rb_bufsz = order_base_2(ring->ring_size / 4);
472 	rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL));
473 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz);
474 #ifdef __BIG_ENDIAN
475 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_SWAP_ENABLE, 1);
476 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL,
477 				RPTR_WRITEBACK_SWAP_ENABLE, 1);
478 #endif
479 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_PRIV, 1);
480 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL), rb_cntl);
481 
482 	/* Initialize the ring buffer's read and write pointers */
483 	if (restore) {
484 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR), lower_32_bits(ring->wptr << 2));
485 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_HI), upper_32_bits(ring->wptr << 2));
486 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr << 2));
487 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
488 	} else {
489 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR), 0);
490 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_HI), 0);
491 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR), 0);
492 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI), 0);
493 	}
494 	/* setup the wptr shadow polling */
495 	wptr_gpu_addr = ring->wptr_gpu_addr;
496 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_LO),
497 	       lower_32_bits(wptr_gpu_addr));
498 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_POLL_ADDR_HI),
499 	       upper_32_bits(wptr_gpu_addr));
500 
501 	/* set the wb address whether it's enabled or not */
502 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_HI),
503 	       upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
504 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_RPTR_ADDR_LO),
505 	       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
506 
507 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
508 	if (amdgpu_sriov_vf(adev))
509 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 1);
510 	else
511 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
512 
513 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, MCU_WPTR_POLL_ENABLE, 1);
514 
515 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
516 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40);
517 
518 	if (!restore)
519 		ring->wptr = 0;
520 
521 	/* before programing wptr to a less value, need set minor_ptr_update first */
522 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE), 1);
523 
524 	if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
525 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2);
526 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
527 	}
528 
529 	doorbell = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL));
530 	doorbell_offset = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL_OFFSET));
531 
532 	if (ring->use_doorbell) {
533 		doorbell = REG_SET_FIELD(doorbell, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 1);
534 		doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET,
535 				OFFSET, ring->doorbell_index);
536 	} else {
537 		doorbell = REG_SET_FIELD(doorbell, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 0);
538 	}
539 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL), doorbell);
540 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_DOORBELL_OFFSET), doorbell_offset);
541 
542 	if (i == 0)
543 		adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
544 					      ring->doorbell_index,
545 					      adev->doorbell_index.sdma_doorbell_range * adev->sdma.num_instances);
546 
547 	if (amdgpu_sriov_vf(adev))
548 		sdma_v7_1_ring_set_wptr(ring);
549 
550 	/* set minor_ptr_update to 0 after wptr programed */
551 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_MINOR_PTR_UPDATE), 0);
552 
553 	/* Set up sdma hang watchdog */
554 	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL));
555 	/* Disable sdma hang watchdog, need revist this when issue resolved.
556 	Once issue resoved, the QUEUE_HANG_COUNT should be set to max(adev->usec_timeout/100000,1)
557 	*/
558 	temp = REG_SET_FIELD(temp, SDMA0_SDMA_WATCHDOG_CNTL, QUEUE_HANG_COUNT, 0);
559 	temp = REG_SET_FIELD(temp, SDMA0_SDMA_WATCHDOG_CNTL, CMD_TIMEOUT_COUNT, 0);
560 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL), temp);
561 
562 	dev_dbg(adev->dev, "Disable SDMA Hang WatchDog, regSDMA0_SDMA_WATCHDOG_CNTL = %d\n",
563 		RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL)));
564 
565 	/* Set up RESP_MODE to non-copy addresses */
566 	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_CNTL));
567 	temp = REG_SET_FIELD(temp, SDMA0_SDMA_UTCL1_CNTL, RESP_MODE, 3);
568 	temp = REG_SET_FIELD(temp, SDMA0_SDMA_UTCL1_CNTL, REDO_DELAY, 9);
569 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_CNTL), temp);
570 
571 	/* program default cache read and write policy */
572 	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_PAGE));
573 	/* clean read policy and write policy bits */
574 	temp &= 0xFF0FFF;
575 	temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
576 		 (CACHE_WRITE_POLICY_L2__DEFAULT << 14));
577 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_PAGE), temp);
578 
579 	if (!amdgpu_sriov_vf(adev)) {
580 		/* unhalt engine */
581 		temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL));
582 		temp = REG_SET_FIELD(temp, SDMA0_SDMA_MCU_CNTL, HALT, 0);
583 		temp = REG_SET_FIELD(temp, SDMA0_SDMA_MCU_CNTL, RESET, 0);
584 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL), temp);
585 	}
586 
587 	/* enable DMA RB */
588 	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_SDMA_QUEUE0_RB_CNTL, RB_ENABLE, 1);
589 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_RB_CNTL), rb_cntl);
590 
591 	ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL));
592 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_SDMA_QUEUE0_IB_CNTL, IB_ENABLE, 1);
593 #ifdef __BIG_ENDIAN
594 	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_SDMA_QUEUE0_IB_CNTL, IB_SWAP_ENABLE, 1);
595 #endif
596 	/* enable DMA IBs */
597 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_IB_CNTL), ib_cntl);
598 	ring->sched.ready = true;
599 
600 	if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
601 		sdma_v7_1_inst_ctx_switch_enable(adev, true, i);
602 		sdma_v7_1_inst_enable(adev, true, i);
603 	}
604 
605 	r = amdgpu_ring_test_helper(ring);
606 	if (r)
607 		ring->sched.ready = false;
608 
609 	return r;
610 }
611 
612 /**
613  * sdma_v7_1_inst_gfx_resume - setup and start the async dma engines
614  *
615  * @adev: amdgpu_device pointer
616  * @inst_mask: mask of dma engine instances to be enabled
617  *
618  * Set up the gfx DMA ring buffers and enable them.
619  * Returns 0 for success, error for failure.
620  */
621 static int sdma_v7_1_inst_gfx_resume(struct amdgpu_device *adev,
622 				     uint32_t inst_mask)
623 {
624 	int i, r;
625 
626 	for_each_inst(i, inst_mask) {
627 		r = sdma_v7_1_gfx_resume_instance(adev, i, false);
628 		if (r)
629 			return r;
630 	}
631 
632 	return 0;
633 
634 }
635 
636 /**
637  * sdma_v7_1_inst_rlc_resume - setup and start the async dma engines
638  *
639  * @adev: amdgpu_device pointer
640  * @inst_mask: mask of dma engine instances to be enabled
641  *
642  * Set up the compute DMA queues and enable them.
643  * Returns 0 for success, error for failure.
644  */
645 static int sdma_v7_1_inst_rlc_resume(struct amdgpu_device *adev,
646 				     uint32_t inst_mask)
647 {
648 	return 0;
649 }
650 
651 static void sdma_v7_1_inst_free_ucode_buffer(struct amdgpu_device *adev,
652 					     uint32_t inst_mask)
653 {
654 	int i;
655 
656 	for_each_inst(i, inst_mask) {
657 		amdgpu_bo_free_kernel(&adev->sdma.instance[i].sdma_fw_obj,
658 				      &adev->sdma.instance[i].sdma_fw_gpu_addr,
659 				      (void **)&adev->sdma.instance[i].sdma_fw_ptr);
660 	}
661 }
662 
663 /**
664  * sdma_v7_1_inst_load_microcode - load the sDMA ME ucode
665  *
666  * @adev: amdgpu_device pointer
667  * @inst_mask: mask of dma engine instances to be enabled
668  *
669  * Loads the sDMA0/1 ucode.
670  * Returns 0 for success, -EINVAL if the ucode is not available.
671  */
672 static int sdma_v7_1_inst_load_microcode(struct amdgpu_device *adev,
673 					 uint32_t inst_mask)
674 {
675 	const struct sdma_firmware_header_v3_0 *hdr;
676 	const __le32 *fw_data;
677 	u32 fw_size;
678 	uint32_t tmp, sdma_status, ic_op_cntl;
679 	int i, r, j;
680 
681 	/* halt the MEs */
682 	sdma_v7_1_inst_enable(adev, false, inst_mask);
683 
684 	if (!adev->sdma.instance[0].fw)
685 		return -EINVAL;
686 
687 	hdr = (const struct sdma_firmware_header_v3_0 *)
688 		adev->sdma.instance[0].fw->data;
689 	amdgpu_ucode_print_sdma_hdr(&hdr->header);
690 
691 	fw_data = (const __le32 *)(adev->sdma.instance[0].fw->data +
692 			le32_to_cpu(hdr->ucode_offset_bytes));
693 	fw_size = le32_to_cpu(hdr->ucode_size_bytes);
694 
695 	for_each_inst(i, inst_mask) {
696 		r = amdgpu_bo_create_reserved(adev, fw_size,
697 					      PAGE_SIZE,
698 					      AMDGPU_GEM_DOMAIN_VRAM,
699 					      &adev->sdma.instance[i].sdma_fw_obj,
700 					      &adev->sdma.instance[i].sdma_fw_gpu_addr,
701 					      (void **)&adev->sdma.instance[i].sdma_fw_ptr);
702 		if (r) {
703 			dev_err(adev->dev, "(%d) failed to create sdma ucode bo\n", r);
704 			return r;
705 		}
706 
707 		memcpy(adev->sdma.instance[i].sdma_fw_ptr, fw_data, fw_size);
708 
709 		amdgpu_bo_kunmap(adev->sdma.instance[i].sdma_fw_obj);
710 		amdgpu_bo_unreserve(adev->sdma.instance[i].sdma_fw_obj);
711 
712 		tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_CNTL));
713 		tmp = REG_SET_FIELD(tmp, SDMA0_SDMA_IC_CNTL, GPA, 0);
714 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_CNTL), tmp);
715 
716 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_BASE_LO),
717 			lower_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
718 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_BASE_HI),
719 			upper_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
720 
721 		tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_OP_CNTL));
722 		tmp = REG_SET_FIELD(tmp, SDMA0_SDMA_IC_OP_CNTL, PRIME_ICACHE, 1);
723 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_OP_CNTL), tmp);
724 
725 		/* Wait for sdma ucode init complete */
726 		for (j = 0; j < adev->usec_timeout; j++) {
727 			ic_op_cntl = RREG32_SOC15_IP(GC,
728 					sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_IC_OP_CNTL));
729 			sdma_status = RREG32_SOC15_IP(GC,
730 					sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_STATUS_REG));
731 			if ((REG_GET_FIELD(ic_op_cntl, SDMA0_SDMA_IC_OP_CNTL, ICACHE_PRIMED) == 1) &&
732 			    (REG_GET_FIELD(sdma_status, SDMA0_SDMA_STATUS_REG, UCODE_INIT_DONE) == 1))
733 				break;
734 			udelay(1);
735 		}
736 
737 		if (j >= adev->usec_timeout) {
738 			dev_err(adev->dev, "failed to init sdma ucode\n");
739 			return -EINVAL;
740 		}
741 	}
742 
743 	return 0;
744 }
745 
746 static int sdma_v7_1_soft_reset(struct amdgpu_ip_block *ip_block)
747 {
748 	struct amdgpu_device *adev = ip_block->adev;
749 	uint32_t inst_mask;
750 	u32 tmp;
751 	int i;
752 
753 	inst_mask = GENMASK(NUM_XCC(adev->sdma.sdma_mask) - 1, 0);
754 	sdma_v7_1_inst_gfx_stop(adev, inst_mask);
755 
756 	for_each_inst(i, inst_mask) {
757 		//tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FREEZE));
758 		//tmp |= SDMA0_SDMA_FREEZE__FREEZE_MASK;
759 		//WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FREEZE), tmp);
760 		tmp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL));
761 		tmp |= SDMA0_SDMA_MCU_CNTL__HALT_MASK;
762 		tmp |= SDMA0_SDMA_MCU_CNTL__RESET_MASK;
763 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_MCU_CNTL), tmp);
764 
765 		WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_QUEUE0_PREEMPT), 0);
766 
767 		udelay(100);
768 
769 		tmp = GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK << i;
770 		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
771 		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
772 
773 		udelay(100);
774 
775 		WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, 0);
776 		tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
777 
778 		udelay(100);
779 	}
780 
781 	return sdma_v7_1_inst_start(adev, inst_mask);
782 }
783 
784 static int sdma_v7_1_reset_queue(struct amdgpu_ring *ring,
785 				 unsigned int vmid,
786 				 struct amdgpu_fence *timedout_fence)
787 {
788 	struct amdgpu_device *adev = ring->adev;
789 	int r;
790 
791 	if (ring->me >= adev->sdma.num_instances) {
792 		dev_err(adev->dev, "sdma instance not found\n");
793 		return -EINVAL;
794 	}
795 
796 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
797 
798 	r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true, 0);
799 	if (r)
800 		return r;
801 
802 	r = sdma_v7_1_gfx_resume_instance(adev, ring->me, true);
803 	if (r)
804 		return r;
805 
806 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
807 }
808 
809 /**
810  * sdma_v7_1_inst_start - setup and start the async dma engines
811  *
812  * @adev: amdgpu_device pointer
813  * @inst_mask: mask of dma engine instances to be enabled
814  *
815  * Set up the DMA engines and enable them.
816  * Returns 0 for success, error for failure.
817  */
818 static int sdma_v7_1_inst_start(struct amdgpu_device *adev,
819 				uint32_t inst_mask)
820 {
821 	int r = 0;
822 
823 	if (amdgpu_sriov_vf(adev)) {
824 		sdma_v7_1_inst_ctx_switch_enable(adev, false, inst_mask);
825 		sdma_v7_1_inst_enable(adev, false, inst_mask);
826 
827 		/* set RB registers */
828 		r = sdma_v7_1_inst_gfx_resume(adev, inst_mask);
829 		return r;
830 	}
831 
832 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
833 		r = sdma_v7_1_inst_load_microcode(adev, inst_mask);
834 		if (r) {
835 			sdma_v7_1_inst_free_ucode_buffer(adev, inst_mask);
836 			return r;
837 		}
838 
839 		if (amdgpu_emu_mode == 1)
840 			msleep(1000);
841 	}
842 
843 	/* unhalt the MEs */
844 	sdma_v7_1_inst_enable(adev, true, inst_mask);
845 	/* enable sdma ring preemption */
846 	sdma_v7_1_inst_ctx_switch_enable(adev, true, inst_mask);
847 
848 	/* start the gfx rings and rlc compute queues */
849 	r = sdma_v7_1_inst_gfx_resume(adev, inst_mask);
850 	if (r)
851 		return r;
852 	r = sdma_v7_1_inst_rlc_resume(adev, inst_mask);
853 
854 	return r;
855 }
856 
857 static int sdma_v7_1_mqd_init(struct amdgpu_device *adev, void *mqd,
858 			      struct amdgpu_mqd_prop *prop)
859 {
860 	struct v12_sdma_mqd *m = mqd;
861 	uint64_t wb_gpu_addr;
862 
863 	m->sdmax_rlcx_rb_cntl =
864 		order_base_2(prop->queue_size / 4) << SDMA0_SDMA_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
865 		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
866 		4 << SDMA0_SDMA_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
867 		1 << SDMA0_SDMA_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT;
868 
869 	m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
870 	m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
871 
872 	wb_gpu_addr = prop->wptr_gpu_addr;
873 	m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
874 	m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
875 
876 	wb_gpu_addr = prop->rptr_gpu_addr;
877 	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
878 	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
879 
880 	m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, 0,
881 							regSDMA0_SDMA_QUEUE0_IB_CNTL));
882 
883 	m->sdmax_rlcx_doorbell_offset =
884 		prop->doorbell_index << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
885 
886 	m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_SDMA_QUEUE0_DOORBELL, ENABLE, 1);
887 
888 	m->sdmax_rlcx_doorbell_log = 0;
889 	m->sdmax_rlcx_rb_aql_cntl = 0x4000;	//regSDMA0_SDMA_QUEUE0_RB_AQL_CNTL_DEFAULT;
890 	m->sdmax_rlcx_dummy_reg = 0xf;	//regSDMA0_SDMA_QUEUE0_DUMMY_REG_DEFAULT;
891 
892 	m->sdmax_rlcx_csa_addr_lo = lower_32_bits(prop->csa_addr);
893 	m->sdmax_rlcx_csa_addr_hi = upper_32_bits(prop->csa_addr);
894 
895 	return 0;
896 }
897 
898 static void sdma_v7_1_set_mqd_funcs(struct amdgpu_device *adev)
899 {
900 	adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v12_sdma_mqd);
901 	adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v7_1_mqd_init;
902 }
903 
904 /**
905  * sdma_v7_1_ring_test_ring - simple async dma engine test
906  *
907  * @ring: amdgpu_ring structure holding ring information
908  *
909  * Test the DMA engine by writing using it to write an
910  * value to memory.
911  * Returns 0 for success, error for failure.
912  */
913 static int sdma_v7_1_ring_test_ring(struct amdgpu_ring *ring)
914 {
915 	struct amdgpu_device *adev = ring->adev;
916 	unsigned i;
917 	unsigned index;
918 	int r;
919 	u32 tmp;
920 	u64 gpu_addr;
921 
922 	tmp = 0xCAFEDEAD;
923 
924 	r = amdgpu_wb_get(adev, &index);
925 	if (r) {
926 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
927 		return r;
928 	}
929 
930 	gpu_addr = adev->wb.gpu_addr + (index * 4);
931 	adev->wb.wb[index] = cpu_to_le32(tmp);
932 
933 	r = amdgpu_ring_alloc(ring, 5);
934 	if (r) {
935 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
936 		amdgpu_wb_free(adev, index);
937 		return r;
938 	}
939 
940 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
941 			  SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
942 	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
943 	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
944 	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
945 	amdgpu_ring_write(ring, 0xDEADBEEF);
946 	amdgpu_ring_commit(ring);
947 
948 	for (i = 0; i < adev->usec_timeout; i++) {
949 		tmp = le32_to_cpu(adev->wb.wb[index]);
950 		if (tmp == 0xDEADBEEF)
951 			break;
952 		if (amdgpu_emu_mode == 1)
953 			msleep(1);
954 		else
955 			udelay(1);
956 	}
957 
958 	if (i >= adev->usec_timeout)
959 		r = -ETIMEDOUT;
960 
961 	amdgpu_wb_free(adev, index);
962 
963 	return r;
964 }
965 
966 /**
967  * sdma_v7_1_ring_test_ib - test an IB on the DMA engine
968  *
969  * @ring: amdgpu_ring structure holding ring information
970  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
971  *
972  * Test a simple IB in the DMA ring.
973  * Returns 0 on success, error on failure.
974  */
975 static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
976 {
977 	struct amdgpu_device *adev = ring->adev;
978 	struct amdgpu_job *job;
979 	struct amdgpu_ib *ib;
980 	struct dma_fence *f = NULL;
981 	unsigned index;
982 	long r;
983 	u32 tmp = 0;
984 	u64 gpu_addr;
985 
986 	tmp = 0xCAFEDEAD;
987 
988 	r = amdgpu_wb_get(adev, &index);
989 	if (r) {
990 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
991 		return r;
992 	}
993 
994 	gpu_addr = adev->wb.gpu_addr + (index * 4);
995 	adev->wb.wb[index] = cpu_to_le32(tmp);
996 
997 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
998 				     AMDGPU_IB_POOL_DIRECT,
999 				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
1000 				     &job);
1001 	if (r) {
1002 		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
1003 		goto err0;
1004 	}
1005 
1006 	ib = &job->ibs[0];
1007 	ib->ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1008 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1009 	ib->ptr[1] = lower_32_bits(gpu_addr);
1010 	ib->ptr[2] = upper_32_bits(gpu_addr);
1011 	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1012 	ib->ptr[4] = 0xDEADBEEF;
1013 	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1014 	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1015 	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1016 	ib->length_dw = 8;
1017 
1018 	r = amdgpu_job_submit_direct(job, ring, &f);
1019 	if (r) {
1020 		amdgpu_job_free(job);
1021 		goto err1;
1022 	}
1023 
1024 	r = dma_fence_wait_timeout(f, false, timeout);
1025 	if (r == 0) {
1026 		DRM_ERROR("amdgpu: IB test timed out\n");
1027 		r = -ETIMEDOUT;
1028 		goto err1;
1029 	} else if (r < 0) {
1030 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
1031 		goto err1;
1032 	}
1033 
1034 	tmp = le32_to_cpu(adev->wb.wb[index]);
1035 
1036 	if (tmp == 0xDEADBEEF)
1037 		r = 0;
1038 	else
1039 		r = -EINVAL;
1040 
1041 err1:
1042 	dma_fence_put(f);
1043 err0:
1044 	amdgpu_wb_free(adev, index);
1045 	return r;
1046 }
1047 
1048 
1049 /**
1050  * sdma_v7_1_vm_copy_pte - update PTEs by copying them from the GART
1051  *
1052  * @ib: indirect buffer to fill with commands
1053  * @pe: addr of the page entry
1054  * @src: src addr to copy from
1055  * @count: number of page entries to update
1056  *
1057  * Update PTEs by copying them from the GART using sDMA.
1058  */
1059 static void sdma_v7_1_vm_copy_pte(struct amdgpu_ib *ib,
1060 				  uint64_t pe, uint64_t src,
1061 				  unsigned count)
1062 {
1063 	unsigned bytes = count * 8;
1064 
1065 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1066 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1067 
1068 	ib->ptr[ib->length_dw++] = bytes - 1;
1069 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1070 	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1071 	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1072 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1073 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1074 
1075 }
1076 
1077 /**
1078  * sdma_v7_1_vm_write_pte - update PTEs by writing them manually
1079  *
1080  * @ib: indirect buffer to fill with commands
1081  * @pe: addr of the page entry
1082  * @value: dst addr to write into pe
1083  * @count: number of page entries to update
1084  * @incr: increase next addr by incr bytes
1085  *
1086  * Update PTEs by writing them manually using sDMA.
1087  */
1088 static void sdma_v7_1_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1089 				   uint64_t value, unsigned count,
1090 				   uint32_t incr)
1091 {
1092 	unsigned ndw = count * 2;
1093 
1094 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1095 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1096 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1097 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1098 	ib->ptr[ib->length_dw++] = ndw - 1;
1099 	for (; ndw > 0; ndw -= 2) {
1100 		ib->ptr[ib->length_dw++] = lower_32_bits(value);
1101 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
1102 		value += incr;
1103 	}
1104 }
1105 
1106 /**
1107  * sdma_v7_1_vm_set_pte_pde - update the page tables using sDMA
1108  *
1109  * @ib: indirect buffer to fill with commands
1110  * @pe: addr of the page entry
1111  * @addr: dst addr to write into pe
1112  * @count: number of page entries to update
1113  * @incr: increase next addr by incr bytes
1114  * @flags: access flags
1115  *
1116  * Update the page tables using sDMA.
1117  */
1118 static void sdma_v7_1_vm_set_pte_pde(struct amdgpu_ib *ib,
1119 				     uint64_t pe,
1120 				     uint64_t addr, unsigned count,
1121 				     uint32_t incr, uint64_t flags)
1122 {
1123 	/* for physically contiguous pages (vram) */
1124 	u32 header = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_PTEPDE);
1125 
1126 	/* TODO:
1127 	 * When VM_L2_CNTL5.WALKER_FETCH_PDE_MTYPE_ENABLE is enabled, change below MTYPE
1128 	 * to RW for AID A1 and UC for AID A0. NC needs additional GCR flush and need not
1129 	 * be supported. Also, honour amdgpu_mtype_local override. RW would additionally
1130 	 * require setting SCOPE bits in the header.
1131 	 *
1132 	 * header |= (SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x2:RW) |
1133 	 *           SDMA_PKT_PTEPDE_COPY_HEADER_SNOOP(0x1) |
1134 	 *           SDMA_PKT_PTEPDE_COPY_HEADER_SCOPE(0x3:SYS_SCOPE));
1135 	 */
1136 
1137 	/* VM_L2_CNTL5.WALKER_FETCH_PDE_MTYPE_ENABLE is 0 which defaults to UC. So,
1138 	 * use MTYPE_UC (0x3). For ref. MTYPE_RW=0x2 MTYPE_NC=0x0
1139 	 */
1140 	header |= SDMA_PKT_PTEPDE_COPY_HEADER_MTYPE(0x3) | SDMA_PKT_PTEPDE_COPY_HEADER_SNOOP(0x1);
1141 
1142 	ib->ptr[ib->length_dw++] = header;
1143 	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1144 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1145 	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1146 	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1147 	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1148 	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1149 	ib->ptr[ib->length_dw++] = incr; /* increment size */
1150 	ib->ptr[ib->length_dw++] = 0;
1151 	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1152 }
1153 
1154 /**
1155  * sdma_v7_1_ring_pad_ib - pad the IB
1156  *
1157  * @ring: amdgpu ring pointer
1158  * @ib: indirect buffer to fill with padding
1159  *
1160  * Pad the IB with NOPs to a boundary multiple of 8.
1161  */
1162 static void sdma_v7_1_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1163 {
1164 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1165 	const bool burst_nop = sdma->burst_nop;
1166 	u32 pad_count;
1167 	int i;
1168 
1169 	pad_count = (-ib->length_dw) & 0x7;
1170 	for (i = 0; i < pad_count; i++)
1171 		if (i == 0 && burst_nop)
1172 			ib->ptr[ib->length_dw++] =
1173 				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
1174 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1175 		else
1176 			ib->ptr[ib->length_dw++] =
1177 				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP);
1178 }
1179 
1180 /**
1181  * sdma_v7_1_ring_emit_pipeline_sync - sync the pipeline
1182  *
1183  * @ring: amdgpu_ring pointer
1184  *
1185  * Make sure all previous operations are completed (CIK).
1186  */
1187 static void sdma_v7_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1188 {
1189 	uint32_t seq = ring->fence_drv.sync_seq;
1190 	uint64_t addr = ring->fence_drv.gpu_addr;
1191 
1192 	/* wait for idle */
1193 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1194 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1195 			  SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1196 	amdgpu_ring_write(ring, addr & 0xfffffffc);
1197 	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1198 	amdgpu_ring_write(ring, seq); /* reference */
1199 	amdgpu_ring_write(ring, 0xffffffff); /* mask */
1200 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1201 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1202 }
1203 
1204 /**
1205  * sdma_v7_1_ring_emit_vm_flush - vm flush using sDMA
1206  *
1207  * @ring: amdgpu_ring pointer
1208  * @vmid: vmid number to use
1209  * @pd_addr: address
1210  *
1211  * Update the page table base and flush the VM TLB
1212  * using sDMA.
1213  */
1214 static void sdma_v7_1_ring_emit_vm_flush(struct amdgpu_ring *ring,
1215 					 unsigned vmid, uint64_t pd_addr)
1216 {
1217 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1218 }
1219 
1220 static void sdma_v7_1_ring_emit_wreg(struct amdgpu_ring *ring,
1221 				     uint32_t reg, uint32_t val)
1222 {
1223 	/* SRBM WRITE command will not support on sdma v7.
1224 	 * Use Register WRITE command instead, which OPCODE is same as SRBM WRITE
1225 	 */
1226 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_SRBM_WRITE));
1227 	amdgpu_ring_write(ring, soc_v1_0_normalize_xcc_reg_offset(reg) << 2);
1228 	amdgpu_ring_write(ring, val);
1229 }
1230 
1231 static void sdma_v7_1_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1232 					 uint32_t val, uint32_t mask)
1233 {
1234 	amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1235 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1236 	amdgpu_ring_write(ring, soc_v1_0_normalize_xcc_reg_offset(reg) << 2);
1237 	amdgpu_ring_write(ring, 0);
1238 	amdgpu_ring_write(ring, val); /* reference */
1239 	amdgpu_ring_write(ring, mask); /* mask */
1240 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1241 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1242 }
1243 
1244 static void sdma_v7_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1245 						   uint32_t reg0, uint32_t reg1,
1246 						   uint32_t ref, uint32_t mask)
1247 {
1248 	amdgpu_ring_emit_wreg(ring, reg0, ref);
1249 	/* wait for a cycle to reset vm_inv_eng*_ack */
1250 	amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1251 	amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1252 }
1253 
1254 static const struct amdgpu_vm_pte_funcs sdma_v7_1_vm_pte_funcs = {
1255 	.copy_pte_num_dw = 8,
1256 	.copy_pte = sdma_v7_1_vm_copy_pte,
1257 	.write_pte = sdma_v7_1_vm_write_pte,
1258 	.set_pte_pde = sdma_v7_1_vm_set_pte_pde,
1259 };
1260 
1261 static int sdma_v7_1_early_init(struct amdgpu_ip_block *ip_block)
1262 {
1263 	struct amdgpu_device *adev = ip_block->adev;
1264 	int r;
1265 
1266 	switch (amdgpu_user_queue) {
1267 	case -1:
1268 	default:
1269 		adev->sdma.no_user_submission = true;
1270 		adev->sdma.disable_uq = true;
1271 		break;
1272 	case 0:
1273 		adev->sdma.no_user_submission = false;
1274 		adev->sdma.disable_uq = true;
1275 		break;
1276 	}
1277 
1278 	r = amdgpu_sdma_init_microcode(adev, 0, true);
1279 	if (r) {
1280 		DRM_ERROR("Failed to init sdma firmware!\n");
1281 		return r;
1282 	}
1283 
1284 	sdma_v7_1_set_ring_funcs(adev);
1285 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_1_vm_pte_funcs);
1286 	sdma_v7_1_set_irq_funcs(adev);
1287 	sdma_v7_1_set_mqd_funcs(adev);
1288 
1289 	return 0;
1290 }
1291 
1292 static int sdma_v7_1_sw_init(struct amdgpu_ip_block *ip_block)
1293 {
1294 	struct amdgpu_ring *ring;
1295 	int r, i;
1296 	struct amdgpu_device *adev = ip_block->adev;
1297 	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_1);
1298 	uint32_t *ptr;
1299 	u32 xcc_id;
1300 
1301 	/* SDMA trap event */
1302 	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GFX,
1303 			      GFX_12_1_0__SRCID__SDMA_TRAP,
1304 			      &adev->sdma.trap_irq);
1305 	if (r)
1306 		return r;
1307 
1308 	for (i = 0; i < adev->sdma.num_instances; i++) {
1309 		ring = &adev->sdma.instance[i].ring;
1310 		ring->use_doorbell = true;
1311 		ring->me = i;
1312 		ring->no_user_submission = adev->sdma.no_user_submission;
1313 
1314 		for (xcc_id = 0; xcc_id < fls(adev->gfx.xcc_mask); xcc_id++) {
1315 			if (adev->sdma.instance[i].xcc_id == GET_INST(GC, xcc_id))
1316 				break;
1317 		}
1318 
1319 		DRM_DEBUG("SDMA%d.%d use_doorbell being set to: [%s]\n",
1320 				xcc_id, GET_INST(SDMA0, i) % adev->sdma.num_inst_per_xcc,
1321 				ring->use_doorbell?"true":"false");
1322 
1323 		ring->doorbell_index =
1324 			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
1325 
1326 		ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
1327 		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d.%d", xcc_id,
1328 					  GET_INST(SDMA0, i) %
1329 					  adev->sdma.num_inst_per_xcc);
1330 		if (r)
1331 			return r;
1332 	}
1333 
1334 	adev->sdma.supported_reset =
1335 		amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
1336 	if (!amdgpu_sriov_vf(adev) &&
1337 	    !adev->debug_disable_gpu_ring_reset)
1338 		adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1339 
1340 	r = amdgpu_sdma_sysfs_reset_mask_init(adev);
1341 	if (r)
1342 		return r;
1343 	/* Allocate memory for SDMA IP Dump buffer */
1344 	ptr = kzalloc_objs(*ptr, adev->sdma.num_instances * reg_count);
1345 	if (ptr)
1346 		adev->sdma.ip_dump = ptr;
1347 	else
1348 		DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1349 
1350 	return r;
1351 }
1352 
1353 static int sdma_v7_1_sw_fini(struct amdgpu_ip_block *ip_block)
1354 {
1355 	struct amdgpu_device *adev = ip_block->adev;
1356 	int i;
1357 
1358 	for (i = 0; i < adev->sdma.num_instances; i++)
1359 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1360 
1361 	amdgpu_sdma_sysfs_reset_mask_fini(adev);
1362 	amdgpu_sdma_destroy_inst_ctx(adev, true);
1363 
1364 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)
1365 		sdma_v7_1_inst_free_ucode_buffer(adev, adev->sdma.sdma_mask);
1366 
1367 	kfree(adev->sdma.ip_dump);
1368 
1369 	return 0;
1370 }
1371 
1372 #define regSDMA0_SDMA_FE_CNTL0			0x10
1373 #define regSDMA0_SDMA_FE_CNTL0_BASE_IDX		0
1374 static void sdma_v7_1_rb_cmd_switch(struct amdgpu_device *adev,
1375 				    uint32_t inst_mask)
1376 {
1377 	int i, fe_cntl;
1378 
1379 	if (adev->sdma.sdma_debug) {
1380 		for_each_inst(i, inst_mask) {
1381 			fe_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FE_CNTL0));
1382 			fe_cntl &= 0x7fffffff;
1383 			WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FE_CNTL0), fe_cntl);
1384 		}
1385 	}
1386 }
1387 
1388 static int sdma_v7_1_hw_init(struct amdgpu_ip_block *ip_block)
1389 {
1390 	struct amdgpu_device *adev = ip_block->adev;
1391 	uint32_t inst_mask;
1392 	int r;
1393 
1394 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1395 
1396 	if (!amdgpu_sriov_vf(adev))
1397 		sdma_v7_1_rb_cmd_switch(adev, inst_mask);
1398 
1399 	r = sdma_v7_1_inst_start(adev, inst_mask);
1400 	if (r)
1401 		return r;
1402 	sdma_v7_1_set_buffer_funcs(adev);
1403 
1404 	return 0;
1405 }
1406 
1407 static int sdma_v7_1_hw_fini(struct amdgpu_ip_block *ip_block)
1408 {
1409 	struct amdgpu_device *adev = ip_block->adev;
1410 
1411 	if (amdgpu_sriov_vf(adev))
1412 		return 0;
1413 
1414 	sdma_v7_1_inst_ctx_switch_enable(adev, false, adev->sdma.sdma_mask);
1415 	sdma_v7_1_inst_enable(adev, false, adev->sdma.sdma_mask);
1416 
1417 	return 0;
1418 }
1419 
1420 static int sdma_v7_1_suspend(struct amdgpu_ip_block *ip_block)
1421 {
1422 	return sdma_v7_1_hw_fini(ip_block);
1423 }
1424 
1425 static int sdma_v7_1_resume(struct amdgpu_ip_block *ip_block)
1426 {
1427 	return sdma_v7_1_hw_init(ip_block);
1428 }
1429 
1430 static int sdma_v7_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
1431 {
1432 	unsigned i, j;
1433 	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
1434 	struct amdgpu_device *adev = ip_block->adev;
1435 
1436 	for (i = 0; i < adev->usec_timeout; i++) {
1437 		for (j = 0; j < adev->sdma.num_instances; j++) {
1438 			sdma[j] = RREG32(sdma_v7_1_get_reg_offset(adev,
1439 						j, regSDMA0_SDMA_STATUS_REG));
1440 			if (!(sdma[j] & SDMA0_SDMA_STATUS_REG__IDLE_MASK))
1441 				break;
1442 		}
1443 		if (j == adev->sdma.num_instances)
1444 			return 0;
1445 		udelay(1);
1446 	}
1447 	return -ETIMEDOUT;
1448 }
1449 
1450 static int sdma_v7_1_ring_preempt_ib(struct amdgpu_ring *ring)
1451 {
1452 	int i, r = 0;
1453 	struct amdgpu_device *adev = ring->adev;
1454 	u64 sdma_gfx_preempt;
1455 
1456 	sdma_gfx_preempt = sdma_v7_1_get_reg_offset(adev, ring->me,
1457 						    regSDMA0_SDMA_QUEUE0_PREEMPT);
1458 
1459 	/* assert preemption condition */
1460 	amdgpu_ring_set_preempt_cond_exec(ring, false);
1461 
1462 	/* emit the trailing fence */
1463 	ring->trail_seq += 1;
1464 	r = amdgpu_ring_alloc(ring, 10);
1465 	if (r) {
1466 		DRM_ERROR("ring %d failed to be allocated \n", ring->idx);
1467 		return r;
1468 	}
1469 	sdma_v7_1_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1470 				  ring->trail_seq, 0);
1471 	amdgpu_ring_commit(ring);
1472 
1473 	/* assert IB preemption */
1474 	WREG32(sdma_gfx_preempt, 1);
1475 
1476 	/* poll the trailing fence */
1477 	for (i = 0; i < adev->usec_timeout; i++) {
1478 		if (ring->trail_seq ==
1479 		    le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1480 			break;
1481 		udelay(1);
1482 	}
1483 
1484 	if (i >= adev->usec_timeout) {
1485 		r = -EINVAL;
1486 		DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1487 	}
1488 
1489 	/* deassert IB preemption */
1490 	WREG32(sdma_gfx_preempt, 0);
1491 
1492 	/* deassert the preemption condition */
1493 	amdgpu_ring_set_preempt_cond_exec(ring, true);
1494 	return r;
1495 }
1496 
1497 static int sdma_v7_1_set_trap_irq_state(struct amdgpu_device *adev,
1498 					struct amdgpu_irq_src *source,
1499 					unsigned type,
1500 					enum amdgpu_interrupt_state state)
1501 {
1502 	u32 sdma_cntl;
1503 
1504 	u32 reg_offset = sdma_v7_1_get_reg_offset(adev, type, regSDMA0_SDMA_CNTL);
1505 
1506 	sdma_cntl = RREG32(reg_offset);
1507 	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_SDMA_CNTL, TRAP_ENABLE,
1508 		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1509 	WREG32(reg_offset, sdma_cntl);
1510 
1511 	return 0;
1512 }
1513 
1514 static int sdma_v7_1_process_trap_irq(struct amdgpu_device *adev,
1515 				      struct amdgpu_irq_src *source,
1516 				      struct amdgpu_iv_entry *entry)
1517 {
1518 	int inst, instances, queue, xcc_id = 0;
1519 
1520 	DRM_DEBUG("IH: SDMA trap\n");
1521 
1522 	if (drm_WARN_ON_ONCE(&adev->ddev,
1523 			     adev->enable_mes &&
1524 			     (entry->src_data[0] & AMDGPU_FENCE_MES_QUEUE_FLAG)))
1525 		return 0;
1526 
1527 	queue = entry->ring_id & 0xf;
1528 	if (adev->gfx.funcs && adev->gfx.funcs->ih_node_to_logical_xcc)
1529 		xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, entry->node_id);
1530 	else
1531 		dev_warn(adev->dev, "IH: SDMA may get wrong xcc id as gfx function not available\n");
1532 	inst = ((entry->ring_id & 0xf0) >> 4) +
1533 		GET_INST(GC, xcc_id) * adev->sdma.num_inst_per_xcc;
1534 	for (instances = 0; instances < adev->sdma.num_instances; instances++) {
1535 		if (inst == GET_INST(SDMA0, instances))
1536 			break;
1537 	}
1538 	if (instances > adev->sdma.num_instances - 1) {
1539 		DRM_ERROR("IH: wrong ring_ID detected, as wrong sdma instance\n");
1540 		return -EINVAL;
1541 	}
1542 
1543 	switch (entry->client_id) {
1544 	case SOC_V1_0_IH_CLIENTID_GFX:
1545 		switch (queue) {
1546 		case 0:
1547 			amdgpu_fence_process(&adev->sdma.instance[instances].ring);
1548 			break;
1549 		default:
1550 			break;
1551 		}
1552 		break;
1553 	}
1554 	return 0;
1555 }
1556 
1557 static int sdma_v7_1_process_illegal_inst_irq(struct amdgpu_device *adev,
1558 					      struct amdgpu_irq_src *source,
1559 					      struct amdgpu_iv_entry *entry)
1560 {
1561 	return 0;
1562 }
1563 
1564 static int sdma_v7_1_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1565 					   enum amd_clockgating_state state)
1566 {
1567 	return 0;
1568 }
1569 
1570 static int sdma_v7_1_set_powergating_state(struct amdgpu_ip_block *ip_block,
1571 					  enum amd_powergating_state state)
1572 {
1573 	return 0;
1574 }
1575 
1576 static void sdma_v7_1_get_clockgating_state(struct amdgpu_ip_block *ip_block,
1577 					    u64 *flags)
1578 {
1579 }
1580 
1581 static void sdma_v7_1_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
1582 {
1583 	struct amdgpu_device *adev = ip_block->adev;
1584 	int i, j;
1585 	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_1);
1586 	uint32_t instance_offset;
1587 
1588 	if (!adev->sdma.ip_dump)
1589 		return;
1590 
1591 	drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
1592 	for (i = 0; i < adev->sdma.num_instances; i++) {
1593 		instance_offset = i * reg_count;
1594 		drm_printf(p, "\nInstance:%d\n", i);
1595 
1596 		for (j = 0; j < reg_count; j++)
1597 			drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_7_1[j].reg_name,
1598 				   adev->sdma.ip_dump[instance_offset + j]);
1599 	}
1600 }
1601 
1602 static void sdma_v7_1_dump_ip_state(struct amdgpu_ip_block *ip_block)
1603 {
1604 	struct amdgpu_device *adev = ip_block->adev;
1605 	int i, j;
1606 	uint32_t instance_offset;
1607 	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_1);
1608 
1609 	if (!adev->sdma.ip_dump)
1610 		return;
1611 
1612 	amdgpu_gfx_off_ctrl(adev, false);
1613 	for (i = 0; i < adev->sdma.num_instances; i++) {
1614 		instance_offset = i * reg_count;
1615 		for (j = 0; j < reg_count; j++)
1616 			adev->sdma.ip_dump[instance_offset + j] =
1617 				RREG32(sdma_v7_1_get_reg_offset(adev, i,
1618 				       sdma_reg_list_7_1[j].reg_offset));
1619 	}
1620 	amdgpu_gfx_off_ctrl(adev, true);
1621 }
1622 
1623 const struct amd_ip_funcs sdma_v7_1_ip_funcs = {
1624 	.name = "sdma_v7_1",
1625 	.early_init = sdma_v7_1_early_init,
1626 	.late_init = NULL,
1627 	.sw_init = sdma_v7_1_sw_init,
1628 	.sw_fini = sdma_v7_1_sw_fini,
1629 	.hw_init = sdma_v7_1_hw_init,
1630 	.hw_fini = sdma_v7_1_hw_fini,
1631 	.suspend = sdma_v7_1_suspend,
1632 	.resume = sdma_v7_1_resume,
1633 	.wait_for_idle = sdma_v7_1_wait_for_idle,
1634 	.soft_reset = sdma_v7_1_soft_reset,
1635 	.set_clockgating_state = sdma_v7_1_set_clockgating_state,
1636 	.set_powergating_state = sdma_v7_1_set_powergating_state,
1637 	.get_clockgating_state = sdma_v7_1_get_clockgating_state,
1638 	.dump_ip_state = sdma_v7_1_dump_ip_state,
1639 	.print_ip_state = sdma_v7_1_print_ip_state,
1640 };
1641 
1642 static const struct amdgpu_ring_funcs sdma_v7_1_ring_funcs = {
1643 	.type = AMDGPU_RING_TYPE_SDMA,
1644 	.align_mask = 0xf,
1645 	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1646 	.support_64bit_ptrs = true,
1647 	.secure_submission_supported = true,
1648 	.get_rptr = sdma_v7_1_ring_get_rptr,
1649 	.get_wptr = sdma_v7_1_ring_get_wptr,
1650 	.set_wptr = sdma_v7_1_ring_set_wptr,
1651 	.emit_frame_size =
1652 		5 + /* sdma_v7_1_ring_init_cond_exec */
1653 		6 + /* sdma_v7_1_ring_emit_pipeline_sync */
1654 		/* sdma_v7_1_ring_emit_vm_flush */
1655 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1656 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1657 		10 + 10 + 10, /* sdma_v7_1_ring_emit_fence x3 for user fence, vm fence */
1658 	.emit_ib_size = 5 + 7 + 6, /* sdma_v7_1_ring_emit_ib */
1659 	.emit_ib = sdma_v7_1_ring_emit_ib,
1660 	.emit_mem_sync = sdma_v7_1_ring_emit_mem_sync,
1661 	.emit_fence = sdma_v7_1_ring_emit_fence,
1662 	.emit_pipeline_sync = sdma_v7_1_ring_emit_pipeline_sync,
1663 	.emit_vm_flush = sdma_v7_1_ring_emit_vm_flush,
1664 	.test_ring = sdma_v7_1_ring_test_ring,
1665 	.test_ib = sdma_v7_1_ring_test_ib,
1666 	.insert_nop = sdma_v7_1_ring_insert_nop,
1667 	.pad_ib = sdma_v7_1_ring_pad_ib,
1668 	.emit_wreg = sdma_v7_1_ring_emit_wreg,
1669 	.emit_reg_wait = sdma_v7_1_ring_emit_reg_wait,
1670 	.emit_reg_write_reg_wait = sdma_v7_1_ring_emit_reg_write_reg_wait,
1671 	.init_cond_exec = sdma_v7_1_ring_init_cond_exec,
1672 	.preempt_ib = sdma_v7_1_ring_preempt_ib,
1673 	.reset = sdma_v7_1_reset_queue,
1674 };
1675 
1676 static void sdma_v7_1_set_ring_funcs(struct amdgpu_device *adev)
1677 {
1678 	int i, dev_inst;
1679 
1680 	for (i = 0; i < adev->sdma.num_instances; i++) {
1681 		adev->sdma.instance[i].ring.funcs = &sdma_v7_1_ring_funcs;
1682 		adev->sdma.instance[i].ring.me = i;
1683 
1684 		dev_inst = GET_INST(SDMA0, i);
1685 		/* XCC to which SDMA belongs depends on physical instance */
1686 		adev->sdma.instance[i].xcc_id =
1687 			dev_inst / adev->sdma.num_inst_per_xcc;
1688 	}
1689 }
1690 
1691 static const struct amdgpu_irq_src_funcs sdma_v7_1_trap_irq_funcs = {
1692 	.set = sdma_v7_1_set_trap_irq_state,
1693 	.process = sdma_v7_1_process_trap_irq,
1694 };
1695 
1696 static const struct amdgpu_irq_src_funcs sdma_v7_1_illegal_inst_irq_funcs = {
1697 	.process = sdma_v7_1_process_illegal_inst_irq,
1698 };
1699 
1700 static void sdma_v7_1_set_irq_funcs(struct amdgpu_device *adev)
1701 {
1702 	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1703 					adev->sdma.num_instances;
1704 	adev->sdma.trap_irq.funcs = &sdma_v7_1_trap_irq_funcs;
1705 	adev->sdma.illegal_inst_irq.funcs = &sdma_v7_1_illegal_inst_irq_funcs;
1706 }
1707 
1708 /**
1709  * sdma_v7_1_emit_copy_buffer - copy buffer using the sDMA engine
1710  *
1711  * @ib: indirect buffer to fill with commands
1712  * @src_offset: src GPU address
1713  * @dst_offset: dst GPU address
1714  * @byte_count: number of bytes to xfer
1715  * @copy_flags: copy flags for the buffers
1716  *
1717  * Copy GPU buffers using the DMA engine.
1718  * Used by the amdgpu ttm implementation to move pages if
1719  * registered as the asic copy callback.
1720  */
1721 static void sdma_v7_1_emit_copy_buffer(struct amdgpu_ib *ib,
1722 				       uint64_t src_offset,
1723 				       uint64_t dst_offset,
1724 				       uint32_t byte_count,
1725 				       uint32_t copy_flags)
1726 {
1727 	ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1728 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1729 		SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
1730 
1731 	ib->ptr[ib->length_dw++] = byte_count - 1;
1732 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1733 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1734 	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1735 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1736 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1737 }
1738 
1739 /**
1740  * sdma_v7_1_emit_fill_buffer - fill buffer using the sDMA engine
1741  *
1742  * @ib: indirect buffer to fill
1743  * @src_data: value to write to buffer
1744  * @dst_offset: dst GPU address
1745  * @byte_count: number of bytes to xfer
1746  *
1747  * Fill GPU buffers using the DMA engine.
1748  */
1749 static void sdma_v7_1_emit_fill_buffer(struct amdgpu_ib *ib,
1750 				       uint32_t src_data,
1751 				       uint64_t dst_offset,
1752 				       uint32_t byte_count)
1753 {
1754 	ib->ptr[ib->length_dw++] = SDMA_PKT_CONSTANT_FILL_HEADER_OP(SDMA_OP_CONST_FILL);
1755 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1756 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1757 	ib->ptr[ib->length_dw++] = src_data;
1758 	ib->ptr[ib->length_dw++] = byte_count - 1;
1759 }
1760 
1761 static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = {
1762 	.copy_max_bytes = 1 << 30,
1763 	.copy_num_dw = 8,
1764 	.emit_copy_buffer = sdma_v7_1_emit_copy_buffer,
1765 	.fill_max_bytes = 1 << 30,
1766 	.fill_num_dw = 5,
1767 	.emit_fill_buffer = sdma_v7_1_emit_fill_buffer,
1768 };
1769 
1770 static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev)
1771 {
1772 	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_1_buffer_funcs);
1773 }
1774 
1775 const struct amdgpu_ip_block_version sdma_v7_1_ip_block = {
1776 	.type = AMD_IP_BLOCK_TYPE_SDMA,
1777 	.major = 7,
1778 	.minor = 1,
1779 	.rev = 0,
1780 	.funcs = &sdma_v7_1_ip_funcs,
1781 };
1782 
1783 static int sdma_v7_1_xcp_resume(void *handle, uint32_t inst_mask)
1784 {
1785 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1786 	int r;
1787 
1788 	r = sdma_v7_1_inst_start(adev, inst_mask);
1789 
1790 	return r;
1791 }
1792 
1793 static int sdma_v7_1_xcp_suspend(void *handle, uint32_t inst_mask)
1794 {
1795 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1796 
1797 	sdma_v7_1_inst_ctx_switch_enable(adev, false, inst_mask);
1798 	sdma_v7_1_inst_enable(adev, false, inst_mask);
1799 
1800 	return 0;
1801 }
1802 
1803 struct amdgpu_xcp_ip_funcs sdma_v7_1_xcp_funcs = {
1804 	.suspend = &sdma_v7_1_xcp_suspend,
1805 	.resume = &sdma_v7_1_xcp_resume
1806 };
1807