xref: /linux/drivers/gpu/drm/amd/amdgpu/si_dma.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 
25 #include "amdgpu.h"
26 #include "amdgpu_trace.h"
27 #include "si.h"
28 #include "sid.h"
29 
30 #include "oss/oss_1_0_d.h"
31 #include "oss/oss_1_0_sh_mask.h"
32 const u32 sdma_offsets[SDMA_MAX_INSTANCE] =
33 {
34 	DMA0_REGISTER_OFFSET,
35 	DMA1_REGISTER_OFFSET
36 };
37 
38 static void si_dma_set_ring_funcs(struct amdgpu_device *adev);
39 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev);
40 static void si_dma_set_irq_funcs(struct amdgpu_device *adev);
41 
42 /**
43  * si_dma_ring_get_rptr - get the current read pointer
44  *
45  * @ring: amdgpu ring pointer
46  *
47  * Get the current rptr from the hardware (SI).
48  */
49 static uint64_t si_dma_ring_get_rptr(struct amdgpu_ring *ring)
50 {
51 	return *ring->rptr_cpu_addr;
52 }
53 
54 /**
55  * si_dma_ring_get_wptr - get the current write pointer
56  *
57  * @ring: amdgpu ring pointer
58  *
59  * Get the current wptr from the hardware (SI).
60  */
61 static uint64_t si_dma_ring_get_wptr(struct amdgpu_ring *ring)
62 {
63 	struct amdgpu_device *adev = ring->adev;
64 	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
65 
66 	return (RREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[me]) & 0x3fffc) >> 2;
67 }
68 
69 static void si_dma_ring_set_wptr(struct amdgpu_ring *ring)
70 {
71 	struct amdgpu_device *adev = ring->adev;
72 	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
73 
74 	WREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[me], (ring->wptr << 2) & 0x3fffc);
75 }
76 
77 static void si_dma_ring_emit_ib(struct amdgpu_ring *ring,
78 				struct amdgpu_job *job,
79 				struct amdgpu_ib *ib,
80 				uint32_t flags)
81 {
82 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
83 	/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
84 	 * Pad as necessary with NOPs.
85 	 */
86 	while ((lower_32_bits(ring->wptr) & 7) != 5)
87 		amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
88 	amdgpu_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vmid, 0));
89 	amdgpu_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
90 	amdgpu_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
91 
92 }
93 
94 /**
95  * si_dma_ring_emit_fence - emit a fence on the DMA ring
96  *
97  * @ring: amdgpu ring pointer
98  * @addr: address
99  * @seq: sequence number
100  * @flags: fence related flags
101  *
102  * Add a DMA fence packet to the ring to write
103  * the fence seq number and DMA trap packet to generate
104  * an interrupt if needed (VI).
105  */
106 static void si_dma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
107 				      unsigned flags)
108 {
109 
110 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
111 	/* write the fence */
112 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0));
113 	amdgpu_ring_write(ring, addr & 0xfffffffc);
114 	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff));
115 	amdgpu_ring_write(ring, seq);
116 	/* optionally write high bits as well */
117 	if (write64bit) {
118 		addr += 4;
119 		amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0, 0));
120 		amdgpu_ring_write(ring, addr & 0xfffffffc);
121 		amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xff));
122 		amdgpu_ring_write(ring, upper_32_bits(seq));
123 	}
124 	/* generate an interrupt */
125 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0, 0));
126 }
127 
128 static void si_dma_stop(struct amdgpu_device *adev)
129 {
130 	u32 rb_cntl;
131 	unsigned i;
132 
133 	for (i = 0; i < adev->sdma.num_instances; i++) {
134 		/* dma0 */
135 		rb_cntl = RREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i]);
136 		rb_cntl &= ~DMA_GFX_RB_CNTL__RB_ENABLE_MASK;
137 		WREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
138 	}
139 }
140 
141 static int si_dma_start(struct amdgpu_device *adev)
142 {
143 	struct amdgpu_ring *ring;
144 	u32 rb_cntl, dma_cntl, ib_cntl, rb_bufsz;
145 	int i, r;
146 	uint64_t rptr_addr;
147 
148 	for (i = 0; i < adev->sdma.num_instances; i++) {
149 		ring = &adev->sdma.instance[i].ring;
150 
151 		WREG32(mmDMA_SEM_INCOMPLETE_TIMER_CNTL + sdma_offsets[i], 0);
152 		WREG32(mmDMA_SEM_WAIT_FAIL_TIMER_CNTL + sdma_offsets[i], 0);
153 
154 		/* Set ring buffer size in dwords */
155 		rb_bufsz = order_base_2(ring->ring_size / 4);
156 		rb_cntl = rb_bufsz << 1;
157 #ifdef __BIG_ENDIAN
158 		rb_cntl |= DMA_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK | DMA_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK;
159 #endif
160 		WREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i], rb_cntl);
161 
162 		/* Initialize the ring buffer's read and write pointers */
163 		WREG32(mmDMA_GFX_RB_RPTR + sdma_offsets[i], 0);
164 		WREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[i], 0);
165 
166 		rptr_addr = ring->rptr_gpu_addr;
167 
168 		WREG32(mmDMA_GFX_RB_RPTR_ADDR_LO + sdma_offsets[i], lower_32_bits(rptr_addr));
169 		WREG32(mmDMA_GFX_RB_RPTR_ADDR_HI + sdma_offsets[i], upper_32_bits(rptr_addr) & 0xFF);
170 
171 		rb_cntl |= DMA_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK;
172 
173 		WREG32(mmDMA_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8);
174 
175 		/* enable DMA IBs */
176 		ib_cntl = DMA_GFX_IB_CNTL__IB_ENABLE_MASK | DMA_GFX_IB_CNTL__CMD_VMID_FORCE_MASK;
177 #ifdef __BIG_ENDIAN
178 		ib_cntl |= DMA_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK;
179 #endif
180 		WREG32(mmDMA_GFX_IB_CNTL + sdma_offsets[i], ib_cntl);
181 
182 		dma_cntl = RREG32(mmDMA_CNTL + sdma_offsets[i]);
183 		dma_cntl &= ~DMA_CNTL__CTXEMPTY_INT_ENABLE_MASK;
184 		WREG32(mmDMA_CNTL + sdma_offsets[i], dma_cntl);
185 
186 		ring->wptr = 0;
187 		WREG32(mmDMA_GFX_RB_WPTR + sdma_offsets[i], ring->wptr << 2);
188 		WREG32(mmDMA_GFX_RB_CNTL + sdma_offsets[i], rb_cntl | DMA_GFX_RB_CNTL__RB_ENABLE_MASK);
189 
190 		r = amdgpu_ring_test_helper(ring);
191 		if (r)
192 			return r;
193 	}
194 
195 	return 0;
196 }
197 
198 /**
199  * si_dma_ring_test_ring - simple async dma engine test
200  *
201  * @ring: amdgpu_ring structure holding ring information
202  *
203  * Test the DMA engine by writing using it to write an
204  * value to memory. (VI).
205  * Returns 0 for success, error for failure.
206  */
207 static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
208 {
209 	struct amdgpu_device *adev = ring->adev;
210 	unsigned i;
211 	unsigned index;
212 	int r;
213 	u32 tmp;
214 	u64 gpu_addr;
215 
216 	r = amdgpu_wb_get(adev, &index);
217 	if (r)
218 		return r;
219 
220 	gpu_addr = adev->wb.gpu_addr + (index * 4);
221 	tmp = 0xCAFEDEAD;
222 	adev->wb.wb[index] = cpu_to_le32(tmp);
223 
224 	r = amdgpu_ring_alloc(ring, 4);
225 	if (r)
226 		goto error_free_wb;
227 
228 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1));
229 	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
230 	amdgpu_ring_write(ring, upper_32_bits(gpu_addr) & 0xff);
231 	amdgpu_ring_write(ring, 0xDEADBEEF);
232 	amdgpu_ring_commit(ring);
233 
234 	for (i = 0; i < adev->usec_timeout; i++) {
235 		tmp = le32_to_cpu(adev->wb.wb[index]);
236 		if (tmp == 0xDEADBEEF)
237 			break;
238 		udelay(1);
239 	}
240 
241 	if (i >= adev->usec_timeout)
242 		r = -ETIMEDOUT;
243 
244 error_free_wb:
245 	amdgpu_wb_free(adev, index);
246 	return r;
247 }
248 
249 /**
250  * si_dma_ring_test_ib - test an IB on the DMA engine
251  *
252  * @ring: amdgpu_ring structure holding ring information
253  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
254  *
255  * Test a simple IB in the DMA ring (VI).
256  * Returns 0 on success, error on failure.
257  */
258 static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
259 {
260 	struct amdgpu_device *adev = ring->adev;
261 	struct amdgpu_job *job;
262 	struct amdgpu_ib *ib;
263 	struct dma_fence *f = NULL;
264 	unsigned index;
265 	u32 tmp = 0;
266 	u64 gpu_addr;
267 	long r;
268 
269 	r = amdgpu_wb_get(adev, &index);
270 	if (r)
271 		return r;
272 
273 	gpu_addr = adev->wb.gpu_addr + (index * 4);
274 	tmp = 0xCAFEDEAD;
275 	adev->wb.wb[index] = cpu_to_le32(tmp);
276 
277 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
278 				     AMDGPU_IB_POOL_DIRECT,
279 				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
280 				     &job);
281 	if (r)
282 		goto err0;
283 
284 	ib = &job->ibs[0];
285 	ib->ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1);
286 	ib->ptr[1] = lower_32_bits(gpu_addr);
287 	ib->ptr[2] = upper_32_bits(gpu_addr) & 0xff;
288 	ib->ptr[3] = 0xDEADBEEF;
289 	ib->length_dw = 4;
290 
291 	r = amdgpu_job_submit_direct(job, ring, &f);
292 	if (r) {
293 		amdgpu_job_free(job);
294 		goto err1;
295 	}
296 
297 	r = dma_fence_wait_timeout(f, false, timeout);
298 	if (r == 0) {
299 		r = -ETIMEDOUT;
300 		goto err1;
301 	} else if (r < 0) {
302 		goto err1;
303 	}
304 	tmp = le32_to_cpu(adev->wb.wb[index]);
305 	if (tmp == 0xDEADBEEF)
306 		r = 0;
307 	else
308 		r = -EINVAL;
309 
310 err1:
311 	dma_fence_put(f);
312 err0:
313 	amdgpu_wb_free(adev, index);
314 	return r;
315 }
316 
317 /**
318  * si_dma_vm_copy_pte - update PTEs by copying them from the GART
319  *
320  * @ib: indirect buffer to fill with commands
321  * @pe: addr of the page entry
322  * @src: src addr to copy from
323  * @count: number of page entries to update
324  *
325  * Update PTEs by copying them from the GART using DMA (SI).
326  */
327 static void si_dma_vm_copy_pte(struct amdgpu_ib *ib,
328 			       uint64_t pe, uint64_t src,
329 			       unsigned count)
330 {
331 	unsigned bytes = count * 8;
332 
333 	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
334 					      1, 0, 0, bytes);
335 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
336 	ib->ptr[ib->length_dw++] = lower_32_bits(src);
337 	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
338 	ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff;
339 }
340 
341 /**
342  * si_dma_vm_write_pte - update PTEs by writing them manually
343  *
344  * @ib: indirect buffer to fill with commands
345  * @pe: addr of the page entry
346  * @value: dst addr to write into pe
347  * @count: number of page entries to update
348  * @incr: increase next addr by incr bytes
349  *
350  * Update PTEs by writing them manually using DMA (SI).
351  */
352 static void si_dma_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
353 				uint64_t value, unsigned count,
354 				uint32_t incr)
355 {
356 	unsigned ndw = count * 2;
357 
358 	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
359 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
360 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
361 	for (; ndw > 0; ndw -= 2) {
362 		ib->ptr[ib->length_dw++] = lower_32_bits(value);
363 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
364 		value += incr;
365 	}
366 }
367 
368 /**
369  * si_dma_vm_set_pte_pde - update the page tables using sDMA
370  *
371  * @ib: indirect buffer to fill with commands
372  * @pe: addr of the page entry
373  * @addr: dst addr to write into pe
374  * @count: number of page entries to update
375  * @incr: increase next addr by incr bytes
376  * @flags: access flags
377  *
378  * Update the page tables using sDMA (CIK).
379  */
380 static void si_dma_vm_set_pte_pde(struct amdgpu_ib *ib,
381 				     uint64_t pe,
382 				     uint64_t addr, unsigned count,
383 				     uint32_t incr, uint64_t flags)
384 {
385 	uint64_t value;
386 	unsigned ndw;
387 
388 	while (count) {
389 		ndw = count * 2;
390 		if (ndw > 0xFFFFE)
391 			ndw = 0xFFFFE;
392 
393 		if (flags & AMDGPU_PTE_VALID)
394 			value = addr;
395 		else
396 			value = 0;
397 
398 		/* for physically contiguous pages (vram) */
399 		ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
400 		ib->ptr[ib->length_dw++] = pe; /* dst addr */
401 		ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
402 		ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
403 		ib->ptr[ib->length_dw++] = upper_32_bits(flags);
404 		ib->ptr[ib->length_dw++] = value; /* value */
405 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
406 		ib->ptr[ib->length_dw++] = incr; /* increment size */
407 		ib->ptr[ib->length_dw++] = 0;
408 		pe += ndw * 4;
409 		addr += (ndw / 2) * incr;
410 		count -= ndw / 2;
411 	}
412 }
413 
414 /**
415  * si_dma_ring_pad_ib - pad the IB to the required number of dw
416  *
417  * @ring: amdgpu_ring pointer
418  * @ib: indirect buffer to fill with padding
419  *
420  */
421 static void si_dma_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
422 {
423 	while (ib->length_dw & 0x7)
424 		ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
425 }
426 
427 /**
428  * si_dma_ring_emit_pipeline_sync - sync the pipeline
429  *
430  * @ring: amdgpu_ring pointer
431  *
432  * Make sure all previous operations are completed (CIK).
433  */
434 static void si_dma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
435 {
436 	uint32_t seq = ring->fence_drv.sync_seq;
437 	uint64_t addr = ring->fence_drv.gpu_addr;
438 
439 	/* wait for idle */
440 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0) |
441 			  (1 << 27)); /* Poll memory */
442 	amdgpu_ring_write(ring, lower_32_bits(addr));
443 	amdgpu_ring_write(ring, (0xff << 16) | upper_32_bits(addr)); /* retry, addr_hi */
444 	amdgpu_ring_write(ring, 0xffffffff); /* mask */
445 	amdgpu_ring_write(ring, seq); /* value */
446 	amdgpu_ring_write(ring, (3 << 28) | 0x20); /* func(equal) | poll interval */
447 }
448 
449 /**
450  * si_dma_ring_emit_vm_flush - cik vm flush using sDMA
451  *
452  * @ring: amdgpu_ring pointer
453  * @vmid: vmid number to use
454  * @pd_addr: address
455  *
456  * Update the page table base and flush the VM TLB
457  * using sDMA (VI).
458  */
459 static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring,
460 				      unsigned vmid, uint64_t pd_addr)
461 {
462 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
463 
464 	/* wait for invalidate to complete */
465 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
466 	amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST);
467 	amdgpu_ring_write(ring, 0xff << 16); /* retry */
468 	amdgpu_ring_write(ring, 1 << vmid); /* mask */
469 	amdgpu_ring_write(ring, 0); /* value */
470 	amdgpu_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */
471 }
472 
473 static void si_dma_ring_emit_wreg(struct amdgpu_ring *ring,
474 				  uint32_t reg, uint32_t val)
475 {
476 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
477 	amdgpu_ring_write(ring, (0xf << 16) | reg);
478 	amdgpu_ring_write(ring, val);
479 }
480 
481 static const struct amdgpu_vm_pte_funcs si_dma_vm_pte_funcs = {
482 	.copy_pte_num_dw = 5,
483 	.copy_pte = si_dma_vm_copy_pte,
484 
485 	.write_pte = si_dma_vm_write_pte,
486 	.set_pte_pde = si_dma_vm_set_pte_pde,
487 };
488 
489 static int si_dma_early_init(struct amdgpu_ip_block *ip_block)
490 {
491 	struct amdgpu_device *adev = ip_block->adev;
492 
493 	adev->sdma.num_instances = SDMA_MAX_INSTANCE;
494 
495 	si_dma_set_ring_funcs(adev);
496 	amdgpu_sdma_set_vm_pte_scheds(adev, &si_dma_vm_pte_funcs);
497 	si_dma_set_irq_funcs(adev);
498 
499 	return 0;
500 }
501 
502 static int si_dma_sw_init(struct amdgpu_ip_block *ip_block)
503 {
504 	struct amdgpu_ring *ring;
505 	int r, i;
506 	struct amdgpu_device *adev = ip_block->adev;
507 
508 	/* DMA0 trap event */
509 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 224,
510 			      &adev->sdma.trap_irq);
511 	if (r)
512 		return r;
513 
514 	/* DMA1 trap event */
515 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 244,
516 			      &adev->sdma.trap_irq);
517 	if (r)
518 		return r;
519 
520 	for (i = 0; i < adev->sdma.num_instances; i++) {
521 		ring = &adev->sdma.instance[i].ring;
522 		ring->ring_obj = NULL;
523 		ring->use_doorbell = false;
524 		sprintf(ring->name, "sdma%d", i);
525 		r = amdgpu_ring_init(adev, ring, 1024,
526 				     &adev->sdma.trap_irq,
527 				     (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :
528 				     AMDGPU_SDMA_IRQ_INSTANCE1,
529 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
530 		if (r)
531 			return r;
532 	}
533 
534 	return r;
535 }
536 
537 static int si_dma_sw_fini(struct amdgpu_ip_block *ip_block)
538 {
539 	struct amdgpu_device *adev = ip_block->adev;
540 	int i;
541 
542 	for (i = 0; i < adev->sdma.num_instances; i++)
543 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
544 
545 	return 0;
546 }
547 
548 static int si_dma_hw_init(struct amdgpu_ip_block *ip_block)
549 {
550 	struct amdgpu_device *adev = ip_block->adev;
551 	int r;
552 
553 	r = si_dma_start(adev);
554 	if (r)
555 		return r;
556 	si_dma_set_buffer_funcs(adev);
557 
558 	return 0;
559 }
560 
561 static int si_dma_hw_fini(struct amdgpu_ip_block *ip_block)
562 {
563 	si_dma_stop(ip_block->adev);
564 
565 	return 0;
566 }
567 
568 static int si_dma_suspend(struct amdgpu_ip_block *ip_block)
569 {
570 	return si_dma_hw_fini(ip_block);
571 }
572 
573 static int si_dma_resume(struct amdgpu_ip_block *ip_block)
574 {
575 	return si_dma_hw_init(ip_block);
576 }
577 
578 static bool si_dma_is_idle(struct amdgpu_ip_block *ip_block)
579 {
580 	struct amdgpu_device *adev = ip_block->adev;
581 
582 	u32 tmp = RREG32(mmSRBM_STATUS2);
583 
584 	if (tmp & (SRBM_STATUS2__DMA_BUSY_MASK | SRBM_STATUS2__DMA1_BUSY_MASK))
585 	    return false;
586 
587 	return true;
588 }
589 
590 static int si_dma_wait_for_idle(struct amdgpu_ip_block *ip_block)
591 {
592 	unsigned i;
593 	struct amdgpu_device *adev = ip_block->adev;
594 
595 	for (i = 0; i < adev->usec_timeout; i++) {
596 		if (si_dma_is_idle(ip_block))
597 			return 0;
598 		udelay(1);
599 	}
600 	return -ETIMEDOUT;
601 }
602 
603 static int si_dma_set_trap_irq_state(struct amdgpu_device *adev,
604 					struct amdgpu_irq_src *src,
605 					unsigned type,
606 					enum amdgpu_interrupt_state state)
607 {
608 	u32 sdma_cntl;
609 
610 	switch (type) {
611 	case AMDGPU_SDMA_IRQ_INSTANCE0:
612 		switch (state) {
613 		case AMDGPU_IRQ_STATE_DISABLE:
614 			sdma_cntl = RREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET);
615 			sdma_cntl &= ~DMA_CNTL__TRAP_ENABLE_MASK;
616 			WREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl);
617 			break;
618 		case AMDGPU_IRQ_STATE_ENABLE:
619 			sdma_cntl = RREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET);
620 			sdma_cntl |= DMA_CNTL__TRAP_ENABLE_MASK;
621 			WREG32(mmDMA_CNTL + DMA0_REGISTER_OFFSET, sdma_cntl);
622 			break;
623 		default:
624 			break;
625 		}
626 		break;
627 	case AMDGPU_SDMA_IRQ_INSTANCE1:
628 		switch (state) {
629 		case AMDGPU_IRQ_STATE_DISABLE:
630 			sdma_cntl = RREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET);
631 			sdma_cntl &= ~DMA_CNTL__TRAP_ENABLE_MASK;
632 			WREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl);
633 			break;
634 		case AMDGPU_IRQ_STATE_ENABLE:
635 			sdma_cntl = RREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET);
636 			sdma_cntl |= DMA_CNTL__TRAP_ENABLE_MASK;
637 			WREG32(mmDMA_CNTL + DMA1_REGISTER_OFFSET, sdma_cntl);
638 			break;
639 		default:
640 			break;
641 		}
642 		break;
643 	default:
644 		break;
645 	}
646 	return 0;
647 }
648 
649 static int si_dma_process_trap_irq(struct amdgpu_device *adev,
650 				      struct amdgpu_irq_src *source,
651 				      struct amdgpu_iv_entry *entry)
652 {
653 	if (entry->src_id == 224)
654 		amdgpu_fence_process(&adev->sdma.instance[0].ring);
655 	else
656 		amdgpu_fence_process(&adev->sdma.instance[1].ring);
657 	return 0;
658 }
659 
660 static int si_dma_set_clockgating_state(struct amdgpu_ip_block *ip_block,
661 					  enum amd_clockgating_state state)
662 {
663 	u32 orig, data, offset;
664 	int i;
665 	bool enable;
666 	struct amdgpu_device *adev = ip_block->adev;
667 
668 	enable = (state == AMD_CG_STATE_GATE);
669 
670 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
671 		for (i = 0; i < adev->sdma.num_instances; i++) {
672 			if (i == 0)
673 				offset = DMA0_REGISTER_OFFSET;
674 			else
675 				offset = DMA1_REGISTER_OFFSET;
676 			orig = data = RREG32(mmDMA_POWER_CNTL + offset);
677 			data &= ~DMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
678 			if (data != orig)
679 				WREG32(mmDMA_POWER_CNTL + offset, data);
680 			WREG32(mmDMA_CLK_CTRL + offset, 0x00000100);
681 		}
682 	} else {
683 		for (i = 0; i < adev->sdma.num_instances; i++) {
684 			if (i == 0)
685 				offset = DMA0_REGISTER_OFFSET;
686 			else
687 				offset = DMA1_REGISTER_OFFSET;
688 			orig = data = RREG32(mmDMA_POWER_CNTL + offset);
689 			data |= DMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
690 			if (data != orig)
691 				WREG32(mmDMA_POWER_CNTL + offset, data);
692 
693 			orig = data = RREG32(mmDMA_CLK_CTRL + offset);
694 			data = 0xff000000;
695 			if (data != orig)
696 				WREG32(mmDMA_CLK_CTRL + offset, data);
697 		}
698 	}
699 
700 	return 0;
701 }
702 
703 static int si_dma_set_powergating_state(struct amdgpu_ip_block *ip_block,
704 					  enum amd_powergating_state state)
705 {
706 	u32 tmp;
707 
708 	struct amdgpu_device *adev = ip_block->adev;
709 
710 	WREG32(mmDMA_PGFSM_WRITE,  0x00002000);
711 	WREG32(mmDMA_PGFSM_CONFIG, 0x100010ff);
712 
713 	for (tmp = 0; tmp < 5; tmp++)
714 		WREG32(mmDMA_PGFSM_WRITE, 0);
715 
716 	return 0;
717 }
718 
719 static const struct amd_ip_funcs si_dma_ip_funcs = {
720 	.name = "si_dma",
721 	.early_init = si_dma_early_init,
722 	.sw_init = si_dma_sw_init,
723 	.sw_fini = si_dma_sw_fini,
724 	.hw_init = si_dma_hw_init,
725 	.hw_fini = si_dma_hw_fini,
726 	.suspend = si_dma_suspend,
727 	.resume = si_dma_resume,
728 	.wait_for_idle = si_dma_wait_for_idle,
729 	.set_clockgating_state = si_dma_set_clockgating_state,
730 	.set_powergating_state = si_dma_set_powergating_state,
731 };
732 
733 static const struct amdgpu_ring_funcs si_dma_ring_funcs = {
734 	.type = AMDGPU_RING_TYPE_SDMA,
735 	.align_mask = 0xf,
736 	.nop = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0),
737 	.support_64bit_ptrs = false,
738 	.get_rptr = si_dma_ring_get_rptr,
739 	.get_wptr = si_dma_ring_get_wptr,
740 	.set_wptr = si_dma_ring_set_wptr,
741 	.emit_frame_size =
742 		3 + 3 + /* hdp flush / invalidate */
743 		6 + /* si_dma_ring_emit_pipeline_sync */
744 		SI_FLUSH_GPU_TLB_NUM_WREG * 3 + 6 + /* si_dma_ring_emit_vm_flush */
745 		9 + 9 + 9, /* si_dma_ring_emit_fence x3 for user fence, vm fence */
746 	.emit_ib_size = 7 + 3, /* si_dma_ring_emit_ib */
747 	.emit_ib = si_dma_ring_emit_ib,
748 	.emit_fence = si_dma_ring_emit_fence,
749 	.emit_pipeline_sync = si_dma_ring_emit_pipeline_sync,
750 	.emit_vm_flush = si_dma_ring_emit_vm_flush,
751 	.test_ring = si_dma_ring_test_ring,
752 	.test_ib = si_dma_ring_test_ib,
753 	.insert_nop = amdgpu_ring_insert_nop,
754 	.pad_ib = si_dma_ring_pad_ib,
755 	.emit_wreg = si_dma_ring_emit_wreg,
756 };
757 
758 static void si_dma_set_ring_funcs(struct amdgpu_device *adev)
759 {
760 	int i;
761 
762 	for (i = 0; i < adev->sdma.num_instances; i++)
763 		adev->sdma.instance[i].ring.funcs = &si_dma_ring_funcs;
764 }
765 
766 static const struct amdgpu_irq_src_funcs si_dma_trap_irq_funcs = {
767 	.set = si_dma_set_trap_irq_state,
768 	.process = si_dma_process_trap_irq,
769 };
770 
771 static void si_dma_set_irq_funcs(struct amdgpu_device *adev)
772 {
773 	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
774 	adev->sdma.trap_irq.funcs = &si_dma_trap_irq_funcs;
775 }
776 
777 /**
778  * si_dma_emit_copy_buffer - copy buffer using the sDMA engine
779  *
780  * @ib: indirect buffer to copy to
781  * @src_offset: src GPU address
782  * @dst_offset: dst GPU address
783  * @byte_count: number of bytes to xfer
784  * @copy_flags: unused
785  *
786  * Copy GPU buffers using the DMA engine (VI).
787  * Used by the amdgpu ttm implementation to move pages if
788  * registered as the asic copy callback.
789  */
790 static void si_dma_emit_copy_buffer(struct amdgpu_ib *ib,
791 				       uint64_t src_offset,
792 				       uint64_t dst_offset,
793 				       uint32_t byte_count,
794 				       uint32_t copy_flags)
795 {
796 	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
797 					      1, 0, 0, byte_count);
798 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
799 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
800 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) & 0xff;
801 	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset) & 0xff;
802 }
803 
804 /**
805  * si_dma_emit_fill_buffer - fill buffer using the sDMA engine
806  *
807  * @ib: indirect buffer to copy to
808  * @src_data: value to write to buffer
809  * @dst_offset: dst GPU address
810  * @byte_count: number of bytes to xfer
811  *
812  * Fill GPU buffers using the DMA engine (VI).
813  */
814 static void si_dma_emit_fill_buffer(struct amdgpu_ib *ib,
815 				       uint32_t src_data,
816 				       uint64_t dst_offset,
817 				       uint32_t byte_count)
818 {
819 	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_CONSTANT_FILL,
820 					      0, 0, 0, byte_count / 4);
821 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
822 	ib->ptr[ib->length_dw++] = src_data;
823 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset) << 16;
824 }
825 
826 
827 static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = {
828 	.copy_max_bytes = 0xffff8,
829 	.copy_num_dw = 5,
830 	.emit_copy_buffer = si_dma_emit_copy_buffer,
831 
832 	.fill_max_bytes = 0xffff8,
833 	.fill_num_dw = 4,
834 	.emit_fill_buffer = si_dma_emit_fill_buffer,
835 };
836 
837 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
838 {
839 	amdgpu_sdma_set_buffer_funcs_scheds(adev, &si_dma_buffer_funcs);
840 }
841 
842 const struct amdgpu_ip_block_version si_dma_ip_block =
843 {
844 	.type = AMD_IP_BLOCK_TYPE_SDMA,
845 	.major = 1,
846 	.minor = 0,
847 	.rev = 0,
848 	.funcs = &si_dma_ip_funcs,
849 };
850