1 /*
2 * Copyright 2023 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28
29 #include "amdgpu.h"
30 #include "amdgpu_ucode.h"
31 #include "amdgpu_trace.h"
32
33 #include "gc/gc_12_0_0_offset.h"
34 #include "gc/gc_12_0_0_sh_mask.h"
35 #include "hdp/hdp_6_0_0_offset.h"
36 #include "ivsrcid/gfx/irqsrcs_gfx_12_0_0.h"
37
38 #include "soc15_common.h"
39 #include "soc15.h"
40 #include "sdma_v6_0_0_pkt_open.h"
41 #include "nbio_v4_3.h"
42 #include "sdma_common.h"
43 #include "sdma_v7_0.h"
44 #include "v12_structs.h"
45 #include "mes_userqueue.h"
46 #include "amdgpu_userq_fence.h"
47
48 MODULE_FIRMWARE("amdgpu/sdma_7_0_0.bin");
49 MODULE_FIRMWARE("amdgpu/sdma_7_0_1.bin");
50
51 #define SDMA1_REG_OFFSET 0x600
52 #define SDMA0_HYP_DEC_REG_START 0x5880
53 #define SDMA0_HYP_DEC_REG_END 0x589a
54 #define SDMA1_HYP_DEC_REG_OFFSET 0x20
55
56 /*define for compression field for sdma7*/
57 #define SDMA_PKT_CONSTANT_FILL_HEADER_compress_offset 0
58 #define SDMA_PKT_CONSTANT_FILL_HEADER_compress_mask 0x00000001
59 #define SDMA_PKT_CONSTANT_FILL_HEADER_compress_shift 16
60 #define SDMA_PKT_CONSTANT_FILL_HEADER_COMPRESS(x) (((x) & SDMA_PKT_CONSTANT_FILL_HEADER_compress_mask) << SDMA_PKT_CONSTANT_FILL_HEADER_compress_shift)
61
62 static const struct amdgpu_hwip_reg_entry sdma_reg_list_7_0[] = {
63 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS_REG),
64 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS1_REG),
65 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS2_REG),
66 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS3_REG),
67 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS4_REG),
68 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS5_REG),
69 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_STATUS6_REG),
70 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UCODE_REV),
71 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_RB_RPTR_FETCH_HI),
72 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_RB_RPTR_FETCH),
73 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_RD_STATUS),
74 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_WR_STATUS),
75 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_RD_XNACK0),
76 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_RD_XNACK1),
77 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_WR_XNACK0),
78 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_UTCL1_WR_XNACK1),
79 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_CNTL),
80 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_RPTR),
81 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_RPTR_HI),
82 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_WPTR),
83 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_RB_WPTR_HI),
84 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_OFFSET),
85 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_BASE_LO),
86 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_BASE_HI),
87 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_CNTL),
88 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_RPTR),
89 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_IB_SUB_REMAIN),
90 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE0_DUMMY_REG),
91 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE_STATUS0),
92 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_CNTL),
93 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_RPTR),
94 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_RPTR_HI),
95 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_WPTR),
96 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_RB_WPTR_HI),
97 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_OFFSET),
98 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_BASE_LO),
99 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_BASE_HI),
100 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_RPTR),
101 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_IB_SUB_REMAIN),
102 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE1_DUMMY_REG),
103 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_CNTL),
104 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_RPTR),
105 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_RPTR_HI),
106 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_WPTR),
107 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_RB_WPTR_HI),
108 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_OFFSET),
109 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_BASE_LO),
110 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_BASE_HI),
111 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_RPTR),
112 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_IB_SUB_REMAIN),
113 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_QUEUE2_DUMMY_REG),
114 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_INT_STATUS),
115 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_VM_CNTL),
116 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
117 SOC15_REG_ENTRY_STR(GC, 0, regSDMA0_CHICKEN_BITS),
118 };
119
120 static void sdma_v7_0_set_ring_funcs(struct amdgpu_device *adev);
121 static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev);
122 static void sdma_v7_0_set_irq_funcs(struct amdgpu_device *adev);
123 static int sdma_v7_0_start(struct amdgpu_device *adev);
124
sdma_v7_0_get_reg_offset(struct amdgpu_device * adev,u32 instance,u32 internal_offset)125 static u32 sdma_v7_0_get_reg_offset(struct amdgpu_device *adev, u32 instance, u32 internal_offset)
126 {
127 u32 base;
128
129 if (internal_offset >= SDMA0_HYP_DEC_REG_START &&
130 internal_offset <= SDMA0_HYP_DEC_REG_END) {
131 base = adev->reg_offset[GC_HWIP][0][1];
132 if (instance != 0)
133 internal_offset += SDMA1_HYP_DEC_REG_OFFSET * instance;
134 } else {
135 base = adev->reg_offset[GC_HWIP][0][0];
136 if (instance == 1)
137 internal_offset += SDMA1_REG_OFFSET;
138 }
139
140 return base + internal_offset;
141 }
142
sdma_v7_0_ring_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)143 static unsigned sdma_v7_0_ring_init_cond_exec(struct amdgpu_ring *ring,
144 uint64_t addr)
145 {
146 unsigned ret;
147
148 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COND_EXE));
149 amdgpu_ring_write(ring, lower_32_bits(addr));
150 amdgpu_ring_write(ring, upper_32_bits(addr));
151 amdgpu_ring_write(ring, 1);
152 /* this is the offset we need patch later */
153 ret = ring->wptr & ring->buf_mask;
154 /* insert dummy here and patch it later */
155 amdgpu_ring_write(ring, 0);
156
157 return ret;
158 }
159
160 /**
161 * sdma_v7_0_ring_get_rptr - get the current read pointer
162 *
163 * @ring: amdgpu ring pointer
164 *
165 * Get the current rptr from the hardware.
166 */
sdma_v7_0_ring_get_rptr(struct amdgpu_ring * ring)167 static uint64_t sdma_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
168 {
169 u64 *rptr;
170
171 /* XXX check if swapping is necessary on BE */
172 rptr = (u64 *)ring->rptr_cpu_addr;
173
174 DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
175 return ((*rptr) >> 2);
176 }
177
178 /**
179 * sdma_v7_0_ring_get_wptr - get the current write pointer
180 *
181 * @ring: amdgpu ring pointer
182 *
183 * Get the current wptr from the hardware.
184 */
sdma_v7_0_ring_get_wptr(struct amdgpu_ring * ring)185 static uint64_t sdma_v7_0_ring_get_wptr(struct amdgpu_ring *ring)
186 {
187 u64 wptr = 0;
188
189 if (ring->use_doorbell) {
190 /* XXX check if swapping is necessary on BE */
191 wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
192 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
193 }
194
195 return wptr >> 2;
196 }
197
198 /**
199 * sdma_v7_0_ring_set_wptr - commit the write pointer
200 *
201 * @ring: amdgpu ring pointer
202 *
203 * Write the wptr back to the hardware.
204 */
sdma_v7_0_ring_set_wptr(struct amdgpu_ring * ring)205 static void sdma_v7_0_ring_set_wptr(struct amdgpu_ring *ring)
206 {
207 struct amdgpu_device *adev = ring->adev;
208
209 DRM_DEBUG("Setting write pointer\n");
210
211 if (ring->use_doorbell) {
212 DRM_DEBUG("Using doorbell -- "
213 "wptr_offs == 0x%08x "
214 "lower_32_bits(ring->wptr) << 2 == 0x%08x "
215 "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
216 ring->wptr_offs,
217 lower_32_bits(ring->wptr << 2),
218 upper_32_bits(ring->wptr << 2));
219 /* XXX check if swapping is necessary on BE */
220 atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
221 ring->wptr << 2);
222 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
223 ring->doorbell_index, ring->wptr << 2);
224 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
225 } else {
226 DRM_DEBUG("Not using doorbell -- "
227 "regSDMA%i_GFX_RB_WPTR == 0x%08x "
228 "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
229 ring->me,
230 lower_32_bits(ring->wptr << 2),
231 ring->me,
232 upper_32_bits(ring->wptr << 2));
233 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev,
234 ring->me,
235 regSDMA0_QUEUE0_RB_WPTR),
236 lower_32_bits(ring->wptr << 2));
237 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev,
238 ring->me,
239 regSDMA0_QUEUE0_RB_WPTR_HI),
240 upper_32_bits(ring->wptr << 2));
241 }
242 }
243
sdma_v7_0_ring_insert_nop(struct amdgpu_ring * ring,uint32_t count)244 static void sdma_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
245 {
246 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
247 int i;
248
249 for (i = 0; i < count; i++)
250 if (sdma && sdma->burst_nop && (i == 0))
251 amdgpu_ring_write(ring, ring->funcs->nop |
252 SDMA_PKT_NOP_HEADER_COUNT(count - 1));
253 else
254 amdgpu_ring_write(ring, ring->funcs->nop);
255 }
256
257 /**
258 * sdma_v7_0_ring_emit_ib - Schedule an IB on the DMA engine
259 *
260 * @ring: amdgpu ring pointer
261 * @job: job to retrieve vmid from
262 * @ib: IB object to schedule
263 * @flags: unused
264 *
265 * Schedule an IB in the DMA ring.
266 */
sdma_v7_0_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)267 static void sdma_v7_0_ring_emit_ib(struct amdgpu_ring *ring,
268 struct amdgpu_job *job,
269 struct amdgpu_ib *ib,
270 uint32_t flags)
271 {
272 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
273 uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
274
275 /* An IB packet must end on a 8 DW boundary--the next dword
276 * must be on a 8-dword boundary. Our IB packet below is 6
277 * dwords long, thus add x number of NOPs, such that, in
278 * modular arithmetic,
279 * wptr + 6 + x = 8k, k >= 0, which in C is,
280 * (wptr + 6 + x) % 8 = 0.
281 * The expression below, is a solution of x.
282 */
283 sdma_v7_0_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
284
285 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_INDIRECT) |
286 SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
287 /* base must be 32 byte aligned */
288 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
289 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
290 amdgpu_ring_write(ring, ib->length_dw);
291 amdgpu_ring_write(ring, lower_32_bits(csa_mc_addr));
292 amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
293 }
294
295 /**
296 * sdma_v7_0_ring_emit_mem_sync - flush the IB by graphics cache rinse
297 *
298 * @ring: amdgpu ring pointer
299 *
300 * flush the IB by graphics cache rinse.
301 */
sdma_v7_0_ring_emit_mem_sync(struct amdgpu_ring * ring)302 static void sdma_v7_0_ring_emit_mem_sync(struct amdgpu_ring *ring)
303 {
304 uint32_t gcr_cntl = SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | SDMA_GCR_GLM_INV |
305 SDMA_GCR_GL1_INV | SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
306 SDMA_GCR_GLI_INV(1);
307
308 /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
309 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_GCR_REQ));
310 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
311 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
312 SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
313 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
314 SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
315 amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
316 SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
317 }
318
319
320 /**
321 * sdma_v7_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
322 *
323 * @ring: amdgpu ring pointer
324 *
325 * Emit an hdp flush packet on the requested DMA ring.
326 */
sdma_v7_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)327 static void sdma_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
328 {
329 struct amdgpu_device *adev = ring->adev;
330 u32 ref_and_mask = 0;
331 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
332
333 ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
334
335 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
336 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
337 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
338 amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_done_offset(adev)) << 2);
339 amdgpu_ring_write(ring, (adev->nbio.funcs->get_hdp_flush_req_offset(adev)) << 2);
340 amdgpu_ring_write(ring, ref_and_mask); /* reference */
341 amdgpu_ring_write(ring, ref_and_mask); /* mask */
342 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
343 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
344 }
345
346 /**
347 * sdma_v7_0_ring_emit_fence - emit a fence on the DMA ring
348 *
349 * @ring: amdgpu ring pointer
350 * @addr: address
351 * @seq: fence seq number
352 * @flags: fence flags
353 *
354 * Add a DMA fence packet to the ring to write
355 * the fence seq number and DMA trap packet to generate
356 * an interrupt if needed.
357 */
sdma_v7_0_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)358 static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
359 unsigned flags)
360 {
361 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
362 /* write the fence */
363 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
364 SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */
365 /* zero in first two bits */
366 WARN_ON(addr & 0x3);
367 amdgpu_ring_write(ring, lower_32_bits(addr));
368 amdgpu_ring_write(ring, upper_32_bits(addr));
369 amdgpu_ring_write(ring, lower_32_bits(seq));
370
371 /* optionally write high bits as well */
372 if (write64bit) {
373 addr += 4;
374 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) |
375 SDMA_PKT_FENCE_HEADER_MTYPE(0x3));
376 /* zero in first two bits */
377 WARN_ON(addr & 0x3);
378 amdgpu_ring_write(ring, lower_32_bits(addr));
379 amdgpu_ring_write(ring, upper_32_bits(addr));
380 amdgpu_ring_write(ring, upper_32_bits(seq));
381 }
382
383 if (flags & AMDGPU_FENCE_FLAG_INT) {
384 /* generate an interrupt */
385 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_TRAP));
386 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
387 }
388 }
389
390 /**
391 * sdma_v7_0_gfx_stop - stop the gfx async dma engines
392 *
393 * @adev: amdgpu_device pointer
394 *
395 * Stop the gfx async dma ring buffers.
396 */
sdma_v7_0_gfx_stop(struct amdgpu_device * adev)397 static void sdma_v7_0_gfx_stop(struct amdgpu_device *adev)
398 {
399 u32 rb_cntl, ib_cntl;
400 int i;
401
402 for (i = 0; i < adev->sdma.num_instances; i++) {
403 rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
404 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 0);
405 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
406 ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
407 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 0);
408 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
409 }
410 }
411
412 /**
413 * sdma_v7_0_rlc_stop - stop the compute async dma engines
414 *
415 * @adev: amdgpu_device pointer
416 *
417 * Stop the compute async dma queues.
418 */
sdma_v7_0_rlc_stop(struct amdgpu_device * adev)419 static void sdma_v7_0_rlc_stop(struct amdgpu_device *adev)
420 {
421 /* XXX todo */
422 }
423
424 /**
425 * sdma_v7_0_ctx_switch_enable - stop the async dma engines context switch
426 *
427 * @adev: amdgpu_device pointer
428 * @enable: enable/disable the DMA MEs context switch.
429 *
430 * Halt or unhalt the async dma engines context switch.
431 */
sdma_v7_0_ctx_switch_enable(struct amdgpu_device * adev,bool enable)432 static void sdma_v7_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
433 {
434 }
435
436 /**
437 * sdma_v7_0_enable - stop the async dma engines
438 *
439 * @adev: amdgpu_device pointer
440 * @enable: enable/disable the DMA MEs.
441 *
442 * Halt or unhalt the async dma engines.
443 */
sdma_v7_0_enable(struct amdgpu_device * adev,bool enable)444 static void sdma_v7_0_enable(struct amdgpu_device *adev, bool enable)
445 {
446 u32 mcu_cntl;
447 int i;
448
449 if (!enable) {
450 sdma_v7_0_gfx_stop(adev);
451 sdma_v7_0_rlc_stop(adev);
452 }
453
454 if (amdgpu_sriov_vf(adev))
455 return;
456
457 for (i = 0; i < adev->sdma.num_instances; i++) {
458 mcu_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL));
459 mcu_cntl = REG_SET_FIELD(mcu_cntl, SDMA0_MCU_CNTL, HALT, enable ? 0 : 1);
460 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL), mcu_cntl);
461 }
462 }
463
464 /**
465 * sdma_v7_0_gfx_resume_instance - start/restart a certain sdma engine
466 *
467 * @adev: amdgpu_device pointer
468 * @i: instance
469 * @restore: used to restore wptr when restart
470 *
471 * Set up the gfx DMA ring buffers and enable them. On restart, we will restore wptr and rptr.
472 * Return 0 for success.
473 */
sdma_v7_0_gfx_resume_instance(struct amdgpu_device * adev,int i,bool restore)474 static int sdma_v7_0_gfx_resume_instance(struct amdgpu_device *adev, int i, bool restore)
475 {
476 struct amdgpu_ring *ring;
477 u32 rb_cntl, ib_cntl;
478 u32 rb_bufsz;
479 u32 doorbell;
480 u32 doorbell_offset;
481 u32 temp;
482 u64 wptr_gpu_addr;
483 int r;
484
485 ring = &adev->sdma.instance[i].ring;
486
487 /* Set ring buffer size in dwords */
488 rb_bufsz = order_base_2(ring->ring_size / 4);
489 rb_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL));
490 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz);
491 #ifdef __BIG_ENDIAN
492 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_SWAP_ENABLE, 1);
493 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL,
494 RPTR_WRITEBACK_SWAP_ENABLE, 1);
495 #endif
496 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_PRIV, 1);
497 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
498
499 /* Initialize the ring buffer's read and write pointers */
500 if (restore) {
501 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR), lower_32_bits(ring->wptr << 2));
502 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_HI), upper_32_bits(ring->wptr << 2));
503 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr << 2));
504 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
505 } else {
506 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR), 0);
507 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_HI), 0);
508 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), 0);
509 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), 0);
510 }
511 /* setup the wptr shadow polling */
512 wptr_gpu_addr = ring->wptr_gpu_addr;
513 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO),
514 lower_32_bits(wptr_gpu_addr));
515 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI),
516 upper_32_bits(wptr_gpu_addr));
517
518 /* set the wb address whether it's enabled or not */
519 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_HI),
520 upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
521 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_RPTR_ADDR_LO),
522 lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
523
524 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
525 if (amdgpu_sriov_vf(adev))
526 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 1);
527 else
528 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
529
530 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, MCU_WPTR_POLL_ENABLE, 1);
531
532 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
533 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40);
534
535 if (!restore)
536 ring->wptr = 0;
537
538 /* before programing wptr to a less value, need set minor_ptr_update first */
539 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 1);
540
541 if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
542 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2);
543 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
544 }
545
546 doorbell = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL));
547 doorbell_offset = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET));
548
549 if (ring->use_doorbell) {
550 doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
551 doorbell_offset = REG_SET_FIELD(doorbell_offset, SDMA0_QUEUE0_DOORBELL_OFFSET,
552 OFFSET, ring->doorbell_index);
553 } else {
554 doorbell = REG_SET_FIELD(doorbell, SDMA0_QUEUE0_DOORBELL, ENABLE, 0);
555 }
556 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL), doorbell);
557 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_DOORBELL_OFFSET), doorbell_offset);
558
559 if (i == 0)
560 adev->nbio.funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
561 ring->doorbell_index,
562 adev->doorbell_index.sdma_doorbell_range * adev->sdma.num_instances);
563
564 if (amdgpu_sriov_vf(adev))
565 sdma_v7_0_ring_set_wptr(ring);
566
567 /* set minor_ptr_update to 0 after wptr programed */
568 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_MINOR_PTR_UPDATE), 0);
569
570 /* Set up sdma hang watchdog */
571 temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL));
572 /* 100ms per unit */
573 temp = REG_SET_FIELD(temp, SDMA0_WATCHDOG_CNTL, QUEUE_HANG_COUNT,
574 max(adev->usec_timeout/100000, 1));
575 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_WATCHDOG_CNTL), temp);
576
577 /* Set up RESP_MODE to non-copy addresses */
578 temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL));
579 temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, RESP_MODE, 3);
580 temp = REG_SET_FIELD(temp, SDMA0_UTCL1_CNTL, REDO_DELAY, 9);
581 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_CNTL), temp);
582
583 /* program default cache read and write policy */
584 temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE));
585 /* clean read policy and write policy bits */
586 temp &= 0xFF0FFF;
587 temp |= ((CACHE_READ_POLICY_L2__DEFAULT << 12) |
588 (CACHE_WRITE_POLICY_L2__DEFAULT << 14));
589 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_UTCL1_PAGE), temp);
590
591 if (!amdgpu_sriov_vf(adev)) {
592 /* unhalt engine */
593 temp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL));
594 temp = REG_SET_FIELD(temp, SDMA0_MCU_CNTL, HALT, 0);
595 temp = REG_SET_FIELD(temp, SDMA0_MCU_CNTL, RESET, 0);
596 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL), temp);
597 }
598
599 /* enable DMA RB */
600 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RB_ENABLE, 1);
601 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL), rb_cntl);
602
603 ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL));
604 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 1);
605 #ifdef __BIG_ENDIAN
606 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_SWAP_ENABLE, 1);
607 #endif
608 /* enable DMA IBs */
609 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl);
610 ring->sched.ready = true;
611
612 if (amdgpu_sriov_vf(adev)) { /* bare-metal sequence doesn't need below to lines */
613 sdma_v7_0_ctx_switch_enable(adev, true);
614 sdma_v7_0_enable(adev, true);
615 }
616
617 r = amdgpu_ring_test_helper(ring);
618 if (r)
619 ring->sched.ready = false;
620
621 return r;
622 }
623
624 /**
625 * sdma_v7_0_gfx_resume - setup and start the async dma engines
626 *
627 * @adev: amdgpu_device pointer
628 *
629 * Set up the gfx DMA ring buffers and enable them.
630 * Returns 0 for success, error for failure.
631 */
sdma_v7_0_gfx_resume(struct amdgpu_device * adev)632 static int sdma_v7_0_gfx_resume(struct amdgpu_device *adev)
633 {
634 int i, r;
635
636 for (i = 0; i < adev->sdma.num_instances; i++) {
637 r = sdma_v7_0_gfx_resume_instance(adev, i, false);
638 if (r)
639 return r;
640 }
641
642 return 0;
643
644 }
645
646 /**
647 * sdma_v7_0_rlc_resume - setup and start the async dma engines
648 *
649 * @adev: amdgpu_device pointer
650 *
651 * Set up the compute DMA queues and enable them.
652 * Returns 0 for success, error for failure.
653 */
sdma_v7_0_rlc_resume(struct amdgpu_device * adev)654 static int sdma_v7_0_rlc_resume(struct amdgpu_device *adev)
655 {
656 return 0;
657 }
658
sdma_v12_0_free_ucode_buffer(struct amdgpu_device * adev)659 static void sdma_v12_0_free_ucode_buffer(struct amdgpu_device *adev)
660 {
661 int i;
662
663 for (i = 0; i < adev->sdma.num_instances; i++) {
664 amdgpu_bo_free_kernel(&adev->sdma.instance[i].sdma_fw_obj,
665 &adev->sdma.instance[i].sdma_fw_gpu_addr,
666 (void **)&adev->sdma.instance[i].sdma_fw_ptr);
667 }
668 }
669
670 /**
671 * sdma_v7_0_load_microcode - load the sDMA ME ucode
672 *
673 * @adev: amdgpu_device pointer
674 *
675 * Loads the sDMA0/1 ucode.
676 * Returns 0 for success, -EINVAL if the ucode is not available.
677 */
sdma_v7_0_load_microcode(struct amdgpu_device * adev)678 static int sdma_v7_0_load_microcode(struct amdgpu_device *adev)
679 {
680 const struct sdma_firmware_header_v3_0 *hdr;
681 const __le32 *fw_data;
682 u32 fw_size;
683 uint32_t tmp, sdma_status, ic_op_cntl;
684 int i, r, j;
685
686 /* halt the MEs */
687 sdma_v7_0_enable(adev, false);
688
689 if (!adev->sdma.instance[0].fw)
690 return -EINVAL;
691
692 hdr = (const struct sdma_firmware_header_v3_0 *)
693 adev->sdma.instance[0].fw->data;
694 amdgpu_ucode_print_sdma_hdr(&hdr->header);
695
696 fw_data = (const __le32 *)(adev->sdma.instance[0].fw->data +
697 le32_to_cpu(hdr->ucode_offset_bytes));
698 fw_size = le32_to_cpu(hdr->ucode_size_bytes);
699
700 for (i = 0; i < adev->sdma.num_instances; i++) {
701 r = amdgpu_bo_create_reserved(adev, fw_size,
702 PAGE_SIZE,
703 AMDGPU_GEM_DOMAIN_VRAM,
704 &adev->sdma.instance[i].sdma_fw_obj,
705 &adev->sdma.instance[i].sdma_fw_gpu_addr,
706 (void **)&adev->sdma.instance[i].sdma_fw_ptr);
707 if (r) {
708 dev_err(adev->dev, "(%d) failed to create sdma ucode bo\n", r);
709 return r;
710 }
711
712 memcpy(adev->sdma.instance[i].sdma_fw_ptr, fw_data, fw_size);
713
714 amdgpu_bo_kunmap(adev->sdma.instance[i].sdma_fw_obj);
715 amdgpu_bo_unreserve(adev->sdma.instance[i].sdma_fw_obj);
716
717 tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_CNTL));
718 tmp = REG_SET_FIELD(tmp, SDMA0_IC_CNTL, GPA, 0);
719 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_CNTL), tmp);
720
721 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_BASE_LO),
722 lower_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
723 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_BASE_HI),
724 upper_32_bits(adev->sdma.instance[i].sdma_fw_gpu_addr));
725
726 tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_OP_CNTL));
727 tmp = REG_SET_FIELD(tmp, SDMA0_IC_OP_CNTL, PRIME_ICACHE, 1);
728 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_OP_CNTL), tmp);
729
730 /* Wait for sdma ucode init complete */
731 for (j = 0; j < adev->usec_timeout; j++) {
732 ic_op_cntl = RREG32_SOC15_IP(GC,
733 sdma_v7_0_get_reg_offset(adev, i, regSDMA0_IC_OP_CNTL));
734 sdma_status = RREG32_SOC15_IP(GC,
735 sdma_v7_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
736 if ((REG_GET_FIELD(ic_op_cntl, SDMA0_IC_OP_CNTL, ICACHE_PRIMED) == 1) &&
737 (REG_GET_FIELD(sdma_status, SDMA0_STATUS_REG, UCODE_INIT_DONE) == 1))
738 break;
739 udelay(1);
740 }
741
742 if (j >= adev->usec_timeout) {
743 dev_err(adev->dev, "failed to init sdma ucode\n");
744 return -EINVAL;
745 }
746 }
747
748 return 0;
749 }
750
sdma_v7_0_soft_reset(struct amdgpu_ip_block * ip_block)751 static int sdma_v7_0_soft_reset(struct amdgpu_ip_block *ip_block)
752 {
753 struct amdgpu_device *adev = ip_block->adev;
754 u32 tmp;
755 int i;
756
757 sdma_v7_0_gfx_stop(adev);
758
759 for (i = 0; i < adev->sdma.num_instances; i++) {
760 //tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_FREEZE));
761 //tmp |= SDMA0_FREEZE__FREEZE_MASK;
762 //WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_FREEZE), tmp);
763 tmp = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL));
764 tmp |= SDMA0_MCU_CNTL__HALT_MASK;
765 tmp |= SDMA0_MCU_CNTL__RESET_MASK;
766 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_MCU_CNTL), tmp);
767
768 WREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_PREEMPT), 0);
769
770 udelay(100);
771
772 tmp = GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK << i;
773 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, tmp);
774 tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
775
776 udelay(100);
777
778 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, 0);
779 tmp = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET);
780
781 udelay(100);
782 }
783
784 return sdma_v7_0_start(adev);
785 }
786
sdma_v7_0_reset_queue(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)787 static int sdma_v7_0_reset_queue(struct amdgpu_ring *ring,
788 unsigned int vmid,
789 struct amdgpu_fence *timedout_fence)
790 {
791 struct amdgpu_device *adev = ring->adev;
792 int r;
793
794 if (ring->me >= adev->sdma.num_instances) {
795 dev_err(adev->dev, "sdma instance not found\n");
796 return -EINVAL;
797 }
798
799 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
800
801 r = amdgpu_mes_reset_legacy_queue(adev, ring, vmid, true, 0);
802 if (r)
803 return r;
804
805 r = sdma_v7_0_gfx_resume_instance(adev, ring->me, true);
806 if (r)
807 return r;
808
809 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
810 }
811
812 /**
813 * sdma_v7_0_start - setup and start the async dma engines
814 *
815 * @adev: amdgpu_device pointer
816 *
817 * Set up the DMA engines and enable them.
818 * Returns 0 for success, error for failure.
819 */
sdma_v7_0_start(struct amdgpu_device * adev)820 static int sdma_v7_0_start(struct amdgpu_device *adev)
821 {
822 int r = 0;
823
824 if (amdgpu_sriov_vf(adev)) {
825 sdma_v7_0_ctx_switch_enable(adev, false);
826 sdma_v7_0_enable(adev, false);
827
828 /* set RB registers */
829 r = sdma_v7_0_gfx_resume(adev);
830 return r;
831 }
832
833 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
834 r = sdma_v7_0_load_microcode(adev);
835 if (r) {
836 sdma_v12_0_free_ucode_buffer(adev);
837 return r;
838 }
839
840 if (amdgpu_emu_mode == 1)
841 msleep(1000);
842 }
843
844 /* unhalt the MEs */
845 sdma_v7_0_enable(adev, true);
846 /* enable sdma ring preemption */
847 sdma_v7_0_ctx_switch_enable(adev, true);
848
849 /* start the gfx rings and rlc compute queues */
850 r = sdma_v7_0_gfx_resume(adev);
851 if (r)
852 return r;
853 r = sdma_v7_0_rlc_resume(adev);
854
855 return r;
856 }
857
sdma_v7_0_mqd_init(struct amdgpu_device * adev,void * mqd,struct amdgpu_mqd_prop * prop)858 static int sdma_v7_0_mqd_init(struct amdgpu_device *adev, void *mqd,
859 struct amdgpu_mqd_prop *prop)
860 {
861 struct v12_sdma_mqd *m = mqd;
862 uint64_t wb_gpu_addr;
863
864 m->sdmax_rlcx_rb_cntl =
865 order_base_2(prop->queue_size / 4) << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT |
866 1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
867 4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT |
868 1 << SDMA0_QUEUE0_RB_CNTL__MCU_WPTR_POLL_ENABLE__SHIFT;
869
870 m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8);
871 m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8);
872
873 wb_gpu_addr = prop->wptr_gpu_addr;
874 m->sdmax_rlcx_rb_wptr_poll_addr_lo = lower_32_bits(wb_gpu_addr);
875 m->sdmax_rlcx_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr);
876
877 wb_gpu_addr = prop->rptr_gpu_addr;
878 m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits(wb_gpu_addr);
879 m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits(wb_gpu_addr);
880
881 m->sdmax_rlcx_ib_cntl = RREG32_SOC15_IP(GC, sdma_v7_0_get_reg_offset(adev, 0,
882 regSDMA0_QUEUE0_IB_CNTL));
883
884 m->sdmax_rlcx_doorbell_offset =
885 prop->doorbell_index << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
886
887 m->sdmax_rlcx_doorbell = REG_SET_FIELD(0, SDMA0_QUEUE0_DOORBELL, ENABLE, 1);
888
889 m->sdmax_rlcx_doorbell_log = 0;
890 m->sdmax_rlcx_rb_aql_cntl = 0x4000; //regSDMA0_QUEUE0_RB_AQL_CNTL_DEFAULT;
891 m->sdmax_rlcx_dummy_reg = 0xf; //regSDMA0_QUEUE0_DUMMY_REG_DEFAULT;
892
893 m->sdmax_rlcx_csa_addr_lo = lower_32_bits(prop->csa_addr);
894 m->sdmax_rlcx_csa_addr_hi = upper_32_bits(prop->csa_addr);
895
896 m->sdmax_rlcx_mcu_dbg0 = lower_32_bits(prop->fence_address);
897 m->sdmax_rlcx_mcu_dbg1 = upper_32_bits(prop->fence_address);
898
899 return 0;
900 }
901
sdma_v7_0_set_mqd_funcs(struct amdgpu_device * adev)902 static void sdma_v7_0_set_mqd_funcs(struct amdgpu_device *adev)
903 {
904 adev->mqds[AMDGPU_HW_IP_DMA].mqd_size = sizeof(struct v12_sdma_mqd);
905 adev->mqds[AMDGPU_HW_IP_DMA].init_mqd = sdma_v7_0_mqd_init;
906 }
907
908 /**
909 * sdma_v7_0_ring_test_ring - simple async dma engine test
910 *
911 * @ring: amdgpu_ring structure holding ring information
912 *
913 * Test the DMA engine by writing using it to write an
914 * value to memory.
915 * Returns 0 for success, error for failure.
916 */
sdma_v7_0_ring_test_ring(struct amdgpu_ring * ring)917 static int sdma_v7_0_ring_test_ring(struct amdgpu_ring *ring)
918 {
919 struct amdgpu_device *adev = ring->adev;
920 unsigned i;
921 unsigned index;
922 int r;
923 u32 tmp;
924 u64 gpu_addr;
925
926 tmp = 0xCAFEDEAD;
927
928 r = amdgpu_wb_get(adev, &index);
929 if (r) {
930 dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
931 return r;
932 }
933
934 gpu_addr = adev->wb.gpu_addr + (index * 4);
935 adev->wb.wb[index] = cpu_to_le32(tmp);
936
937 r = amdgpu_ring_alloc(ring, 5);
938 if (r) {
939 drm_err(adev_to_drm(adev), "dma failed to lock ring %d (%d).\n", ring->idx, r);
940 amdgpu_wb_free(adev, index);
941 return r;
942 }
943
944 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
945 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
946 amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
947 amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
948 amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
949 amdgpu_ring_write(ring, 0xDEADBEEF);
950 amdgpu_ring_commit(ring);
951
952 for (i = 0; i < adev->usec_timeout; i++) {
953 tmp = le32_to_cpu(adev->wb.wb[index]);
954 if (tmp == 0xDEADBEEF)
955 break;
956 if (amdgpu_emu_mode == 1)
957 msleep(1);
958 else
959 udelay(1);
960 }
961
962 if (i >= adev->usec_timeout)
963 r = -ETIMEDOUT;
964
965 amdgpu_wb_free(adev, index);
966
967 return r;
968 }
969
970 /**
971 * sdma_v7_0_ring_test_ib - test an IB on the DMA engine
972 *
973 * @ring: amdgpu_ring structure holding ring information
974 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
975 *
976 * Test a simple IB in the DMA ring.
977 * Returns 0 on success, error on failure.
978 */
sdma_v7_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)979 static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
980 {
981 struct amdgpu_device *adev = ring->adev;
982 struct amdgpu_ib ib;
983 struct dma_fence *f = NULL;
984 unsigned index;
985 long r;
986 u32 tmp = 0;
987 u64 gpu_addr;
988
989 tmp = 0xCAFEDEAD;
990 memset(&ib, 0, sizeof(ib));
991
992 r = amdgpu_wb_get(adev, &index);
993 if (r) {
994 dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
995 return r;
996 }
997
998 gpu_addr = adev->wb.gpu_addr + (index * 4);
999 adev->wb.wb[index] = cpu_to_le32(tmp);
1000
1001 r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
1002 if (r) {
1003 drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
1004 goto err0;
1005 }
1006
1007 ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1008 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1009 ib.ptr[1] = lower_32_bits(gpu_addr);
1010 ib.ptr[2] = upper_32_bits(gpu_addr);
1011 ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1012 ib.ptr[4] = 0xDEADBEEF;
1013 ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1014 ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1015 ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1016 ib.length_dw = 8;
1017
1018 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1019 if (r)
1020 goto err1;
1021
1022 r = dma_fence_wait_timeout(f, false, timeout);
1023 if (r == 0) {
1024 drm_err(adev_to_drm(adev), "IB test timed out\n");
1025 r = -ETIMEDOUT;
1026 goto err1;
1027 } else if (r < 0) {
1028 drm_err(adev_to_drm(adev), "fence wait failed (%ld).\n", r);
1029 goto err1;
1030 }
1031
1032 tmp = le32_to_cpu(adev->wb.wb[index]);
1033
1034 if (tmp == 0xDEADBEEF)
1035 r = 0;
1036 else
1037 r = -EINVAL;
1038
1039 err1:
1040 amdgpu_ib_free(&ib, NULL);
1041 dma_fence_put(f);
1042 err0:
1043 amdgpu_wb_free(adev, index);
1044 return r;
1045 }
1046
1047
1048 /**
1049 * sdma_v7_0_vm_copy_pte - update PTEs by copying them from the GART
1050 *
1051 * @ib: indirect buffer to fill with commands
1052 * @pe: addr of the page entry
1053 * @src: src addr to copy from
1054 * @count: number of page entries to update
1055 *
1056 * Update PTEs by copying them from the GART using sDMA.
1057 */
sdma_v7_0_vm_copy_pte(struct amdgpu_ib * ib,uint64_t pe,uint64_t src,unsigned count)1058 static void sdma_v7_0_vm_copy_pte(struct amdgpu_ib *ib,
1059 uint64_t pe, uint64_t src,
1060 unsigned count)
1061 {
1062 unsigned bytes = count * 8;
1063
1064 ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1065 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1066 SDMA_PKT_COPY_LINEAR_HEADER_CPV(1);
1067
1068 ib->ptr[ib->length_dw++] = bytes - 1;
1069 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1070 ib->ptr[ib->length_dw++] = lower_32_bits(src);
1071 ib->ptr[ib->length_dw++] = upper_32_bits(src);
1072 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1073 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1074 ib->ptr[ib->length_dw++] = 0;
1075
1076 }
1077
1078 /**
1079 * sdma_v7_0_vm_write_pte - update PTEs by writing them manually
1080 *
1081 * @ib: indirect buffer to fill with commands
1082 * @pe: addr of the page entry
1083 * @value: dst addr to write into pe
1084 * @count: number of page entries to update
1085 * @incr: increase next addr by incr bytes
1086 *
1087 * Update PTEs by writing them manually using sDMA.
1088 */
sdma_v7_0_vm_write_pte(struct amdgpu_ib * ib,uint64_t pe,uint64_t value,unsigned count,uint32_t incr)1089 static void sdma_v7_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1090 uint64_t value, unsigned count,
1091 uint32_t incr)
1092 {
1093 unsigned ndw = count * 2;
1094
1095 ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
1096 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1097 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1098 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1099 ib->ptr[ib->length_dw++] = ndw - 1;
1100 for (; ndw > 0; ndw -= 2) {
1101 ib->ptr[ib->length_dw++] = lower_32_bits(value);
1102 ib->ptr[ib->length_dw++] = upper_32_bits(value);
1103 value += incr;
1104 }
1105 }
1106
1107 /**
1108 * sdma_v7_0_vm_set_pte_pde - update the page tables using sDMA
1109 *
1110 * @ib: indirect buffer to fill with commands
1111 * @pe: addr of the page entry
1112 * @addr: dst addr to write into pe
1113 * @count: number of page entries to update
1114 * @incr: increase next addr by incr bytes
1115 * @flags: access flags
1116 *
1117 * Update the page tables using sDMA.
1118 */
sdma_v7_0_vm_set_pte_pde(struct amdgpu_ib * ib,uint64_t pe,uint64_t addr,unsigned count,uint32_t incr,uint64_t flags)1119 static void sdma_v7_0_vm_set_pte_pde(struct amdgpu_ib *ib,
1120 uint64_t pe,
1121 uint64_t addr, unsigned count,
1122 uint32_t incr, uint64_t flags)
1123 {
1124 /* for physically contiguous pages (vram) */
1125 ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_PTEPDE);
1126 ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1127 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1128 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1129 ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1130 ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1131 ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1132 ib->ptr[ib->length_dw++] = incr; /* increment size */
1133 ib->ptr[ib->length_dw++] = 0;
1134 ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1135 }
1136
1137 /**
1138 * sdma_v7_0_ring_pad_ib - pad the IB
1139 *
1140 * @ring: amdgpu ring pointer
1141 * @ib: indirect buffer to fill with padding
1142 *
1143 * Pad the IB with NOPs to a boundary multiple of 8.
1144 */
sdma_v7_0_ring_pad_ib(struct amdgpu_ring * ring,struct amdgpu_ib * ib)1145 static void sdma_v7_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1146 {
1147 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1148 u32 pad_count;
1149 int i;
1150
1151 pad_count = (-ib->length_dw) & 0x7;
1152 for (i = 0; i < pad_count; i++)
1153 if (sdma && sdma->burst_nop && (i == 0))
1154 ib->ptr[ib->length_dw++] =
1155 SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
1156 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1157 else
1158 ib->ptr[ib->length_dw++] =
1159 SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP);
1160 }
1161
1162 /**
1163 * sdma_v7_0_ring_emit_pipeline_sync - sync the pipeline
1164 *
1165 * @ring: amdgpu_ring pointer
1166 *
1167 * Make sure all previous operations are completed (CIK).
1168 */
sdma_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)1169 static void sdma_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1170 {
1171 uint32_t seq = ring->fence_drv.sync_seq;
1172 uint64_t addr = ring->fence_drv.gpu_addr;
1173
1174 /* wait for idle */
1175 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1176 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1177 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3) | /* equal */
1178 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(1));
1179 amdgpu_ring_write(ring, addr & 0xfffffffc);
1180 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
1181 amdgpu_ring_write(ring, seq); /* reference */
1182 amdgpu_ring_write(ring, 0xffffffff); /* mask */
1183 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1184 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(4)); /* retry count, poll interval */
1185 }
1186
1187 /**
1188 * sdma_v7_0_ring_emit_vm_flush - vm flush using sDMA
1189 *
1190 * @ring: amdgpu_ring pointer
1191 * @vmid: vmid number to use
1192 * @pd_addr: address
1193 *
1194 * Update the page table base and flush the VM TLB
1195 * using sDMA.
1196 */
sdma_v7_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)1197 static void sdma_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1198 unsigned vmid, uint64_t pd_addr)
1199 {
1200 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1201 }
1202
sdma_v7_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)1203 static void sdma_v7_0_ring_emit_wreg(struct amdgpu_ring *ring,
1204 uint32_t reg, uint32_t val)
1205 {
1206 /* SRBM WRITE command will not support on sdma v7.
1207 * Use Register WRITE command instead, which OPCODE is same as SRBM WRITE
1208 */
1209 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_SRBM_WRITE));
1210 amdgpu_ring_write(ring, reg << 2);
1211 amdgpu_ring_write(ring, val);
1212 }
1213
sdma_v7_0_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)1214 static void sdma_v7_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1215 uint32_t val, uint32_t mask)
1216 {
1217 amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_POLL_REGMEM) |
1218 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(0) |
1219 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
1220 amdgpu_ring_write(ring, reg << 2);
1221 amdgpu_ring_write(ring, 0);
1222 amdgpu_ring_write(ring, val); /* reference */
1223 amdgpu_ring_write(ring, mask); /* mask */
1224 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
1225 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
1226 }
1227
sdma_v7_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring * ring,uint32_t reg0,uint32_t reg1,uint32_t ref,uint32_t mask)1228 static void sdma_v7_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
1229 uint32_t reg0, uint32_t reg1,
1230 uint32_t ref, uint32_t mask)
1231 {
1232 amdgpu_ring_emit_wreg(ring, reg0, ref);
1233 /* wait for a cycle to reset vm_inv_eng*_ack */
1234 amdgpu_ring_emit_reg_wait(ring, reg0, 0, 0);
1235 amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
1236 }
1237
1238 /* all sizes are in bytes */
1239 #define SDMA7_CSA_SIZE 32
1240 #define SDMA7_CSA_ALIGNMENT 4
1241
sdma_v7_0_get_csa_info(struct amdgpu_device * adev,struct amdgpu_sdma_csa_info * csa_info)1242 static void sdma_v7_0_get_csa_info(struct amdgpu_device *adev,
1243 struct amdgpu_sdma_csa_info *csa_info)
1244 {
1245 csa_info->size = SDMA7_CSA_SIZE;
1246 csa_info->alignment = SDMA7_CSA_ALIGNMENT;
1247 }
1248
1249 static const struct amdgpu_vm_pte_funcs sdma_v7_0_vm_pte_funcs = {
1250 .copy_pte_num_dw = 8,
1251 .copy_pte = sdma_v7_0_vm_copy_pte,
1252 .write_pte = sdma_v7_0_vm_write_pte,
1253 .set_pte_pde = sdma_v7_0_vm_set_pte_pde,
1254 };
1255
sdma_v7_0_early_init(struct amdgpu_ip_block * ip_block)1256 static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
1257 {
1258 struct amdgpu_device *adev = ip_block->adev;
1259 int r;
1260
1261 switch (amdgpu_user_queue) {
1262 case -1:
1263 case 0:
1264 default:
1265 adev->sdma.no_user_submission = false;
1266 adev->sdma.disable_uq = true;
1267 break;
1268 case 1:
1269 adev->sdma.no_user_submission = false;
1270 adev->sdma.disable_uq = false;
1271 break;
1272 case 2:
1273 adev->sdma.no_user_submission = true;
1274 adev->sdma.disable_uq = false;
1275 break;
1276 }
1277
1278 r = amdgpu_sdma_init_microcode(adev, 0, true);
1279 if (r) {
1280 DRM_ERROR("Failed to init sdma firmware!\n");
1281 return r;
1282 }
1283
1284 sdma_v7_0_set_ring_funcs(adev);
1285 amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_0_vm_pte_funcs);
1286 sdma_v7_0_set_irq_funcs(adev);
1287 sdma_v7_0_set_mqd_funcs(adev);
1288 adev->sdma.get_csa_info = &sdma_v7_0_get_csa_info;
1289
1290 return 0;
1291 }
1292
sdma_v7_0_sw_init(struct amdgpu_ip_block * ip_block)1293 static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
1294 {
1295 struct amdgpu_ring *ring;
1296 int r, i;
1297 struct amdgpu_device *adev = ip_block->adev;
1298 uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_0);
1299 uint32_t *ptr;
1300
1301 /* SDMA trap event */
1302 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1303 GFX_12_0_0__SRCID__SDMA_TRAP,
1304 &adev->sdma.trap_irq);
1305 if (r)
1306 return r;
1307
1308 /* SDMA user fence event */
1309 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
1310 GFX_12_0_0__SRCID__SDMA_FENCE,
1311 &adev->sdma.fence_irq);
1312 if (r)
1313 return r;
1314
1315 for (i = 0; i < adev->sdma.num_instances; i++) {
1316 ring = &adev->sdma.instance[i].ring;
1317 ring->ring_obj = NULL;
1318 ring->use_doorbell = true;
1319 ring->me = i;
1320 ring->no_user_submission = adev->sdma.no_user_submission;
1321
1322 DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1323 ring->use_doorbell?"true":"false");
1324
1325 ring->doorbell_index =
1326 (adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
1327
1328 ring->vm_hub = AMDGPU_GFXHUB(0);
1329 sprintf(ring->name, "sdma%d", i);
1330 r = amdgpu_ring_init(adev, ring, 1024,
1331 &adev->sdma.trap_irq,
1332 AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1333 AMDGPU_RING_PRIO_DEFAULT, NULL);
1334 if (r)
1335 return r;
1336 }
1337
1338 adev->sdma.supported_reset =
1339 amdgpu_get_soft_full_reset_mask(&adev->sdma.instance[0].ring);
1340 if (!amdgpu_sriov_vf(adev) &&
1341 !adev->debug_disable_gpu_ring_reset)
1342 adev->sdma.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1343
1344 r = amdgpu_sdma_sysfs_reset_mask_init(adev);
1345 if (r)
1346 return r;
1347 /* Allocate memory for SDMA IP Dump buffer */
1348 ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
1349 if (ptr)
1350 adev->sdma.ip_dump = ptr;
1351 else
1352 DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1353
1354 switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) {
1355 case IP_VERSION(7, 0, 0):
1356 case IP_VERSION(7, 0, 1):
1357 if ((adev->sdma.instance[0].fw_version >= 7966358) && !adev->sdma.disable_uq)
1358 adev->userq_funcs[AMDGPU_HW_IP_DMA] = &userq_mes_funcs;
1359 break;
1360 default:
1361 break;
1362 }
1363
1364 return r;
1365 }
1366
sdma_v7_0_sw_fini(struct amdgpu_ip_block * ip_block)1367 static int sdma_v7_0_sw_fini(struct amdgpu_ip_block *ip_block)
1368 {
1369 struct amdgpu_device *adev = ip_block->adev;
1370 int i;
1371
1372 for (i = 0; i < adev->sdma.num_instances; i++)
1373 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1374
1375 amdgpu_sdma_sysfs_reset_mask_fini(adev);
1376 amdgpu_sdma_destroy_inst_ctx(adev, true);
1377
1378 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)
1379 sdma_v12_0_free_ucode_buffer(adev);
1380
1381 kfree(adev->sdma.ip_dump);
1382
1383 return 0;
1384 }
1385
sdma_v7_0_set_userq_trap_interrupts(struct amdgpu_device * adev,bool enable)1386 static int sdma_v7_0_set_userq_trap_interrupts(struct amdgpu_device *adev,
1387 bool enable)
1388 {
1389 unsigned int irq_type;
1390 int i, r;
1391
1392 if (adev->userq_funcs[AMDGPU_HW_IP_DMA]) {
1393 for (i = 0; i < adev->sdma.num_instances; i++) {
1394 irq_type = AMDGPU_SDMA_IRQ_INSTANCE0 + i;
1395 if (enable)
1396 r = amdgpu_irq_get(adev, &adev->sdma.trap_irq,
1397 irq_type);
1398 else
1399 r = amdgpu_irq_put(adev, &adev->sdma.trap_irq,
1400 irq_type);
1401 if (r)
1402 return r;
1403 }
1404 }
1405
1406 return 0;
1407 }
1408
sdma_v7_0_hw_init(struct amdgpu_ip_block * ip_block)1409 static int sdma_v7_0_hw_init(struct amdgpu_ip_block *ip_block)
1410 {
1411 struct amdgpu_device *adev = ip_block->adev;
1412 int r;
1413
1414 r = sdma_v7_0_start(adev);
1415 if (r)
1416 return r;
1417 sdma_v7_0_set_buffer_funcs(adev);
1418
1419 return sdma_v7_0_set_userq_trap_interrupts(adev, true);
1420 }
1421
sdma_v7_0_hw_fini(struct amdgpu_ip_block * ip_block)1422 static int sdma_v7_0_hw_fini(struct amdgpu_ip_block *ip_block)
1423 {
1424 struct amdgpu_device *adev = ip_block->adev;
1425
1426 if (amdgpu_sriov_vf(adev))
1427 return 0;
1428
1429 sdma_v7_0_ctx_switch_enable(adev, false);
1430 sdma_v7_0_enable(adev, false);
1431 sdma_v7_0_set_userq_trap_interrupts(adev, false);
1432
1433 return 0;
1434 }
1435
sdma_v7_0_suspend(struct amdgpu_ip_block * ip_block)1436 static int sdma_v7_0_suspend(struct amdgpu_ip_block *ip_block)
1437 {
1438 return sdma_v7_0_hw_fini(ip_block);
1439 }
1440
sdma_v7_0_resume(struct amdgpu_ip_block * ip_block)1441 static int sdma_v7_0_resume(struct amdgpu_ip_block *ip_block)
1442 {
1443 return sdma_v7_0_hw_init(ip_block);
1444 }
1445
sdma_v7_0_is_idle(struct amdgpu_ip_block * ip_block)1446 static bool sdma_v7_0_is_idle(struct amdgpu_ip_block *ip_block)
1447 {
1448 struct amdgpu_device *adev = ip_block->adev;
1449 u32 i;
1450
1451 for (i = 0; i < adev->sdma.num_instances; i++) {
1452 u32 tmp = RREG32(sdma_v7_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
1453
1454 if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1455 return false;
1456 }
1457
1458 return true;
1459 }
1460
sdma_v7_0_wait_for_idle(struct amdgpu_ip_block * ip_block)1461 static int sdma_v7_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
1462 {
1463 unsigned i;
1464 u32 sdma0, sdma1;
1465 struct amdgpu_device *adev = ip_block->adev;
1466
1467 for (i = 0; i < adev->usec_timeout; i++) {
1468 sdma0 = RREG32(sdma_v7_0_get_reg_offset(adev, 0, regSDMA0_STATUS_REG));
1469 sdma1 = RREG32(sdma_v7_0_get_reg_offset(adev, 1, regSDMA0_STATUS_REG));
1470
1471 if (sdma0 & sdma1 & SDMA0_STATUS_REG__IDLE_MASK)
1472 return 0;
1473 udelay(1);
1474 }
1475 return -ETIMEDOUT;
1476 }
1477
sdma_v7_0_ring_preempt_ib(struct amdgpu_ring * ring)1478 static int sdma_v7_0_ring_preempt_ib(struct amdgpu_ring *ring)
1479 {
1480 int i, r = 0;
1481 struct amdgpu_device *adev = ring->adev;
1482 u32 index = 0;
1483 u64 sdma_gfx_preempt;
1484
1485 amdgpu_sdma_get_index_from_ring(ring, &index);
1486 sdma_gfx_preempt =
1487 sdma_v7_0_get_reg_offset(adev, index, regSDMA0_QUEUE0_PREEMPT);
1488
1489 /* assert preemption condition */
1490 amdgpu_ring_set_preempt_cond_exec(ring, false);
1491
1492 /* emit the trailing fence */
1493 ring->trail_seq += 1;
1494 r = amdgpu_ring_alloc(ring, 10);
1495 if (r) {
1496 DRM_ERROR("ring %d failed to be allocated\n", ring->idx);
1497 return r;
1498 }
1499 sdma_v7_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr,
1500 ring->trail_seq, 0);
1501 amdgpu_ring_commit(ring);
1502
1503 /* assert IB preemption */
1504 WREG32(sdma_gfx_preempt, 1);
1505
1506 /* poll the trailing fence */
1507 for (i = 0; i < adev->usec_timeout; i++) {
1508 if (ring->trail_seq ==
1509 le32_to_cpu(*(ring->trail_fence_cpu_addr)))
1510 break;
1511 udelay(1);
1512 }
1513
1514 if (i >= adev->usec_timeout) {
1515 r = -EINVAL;
1516 DRM_ERROR("ring %d failed to be preempted\n", ring->idx);
1517 }
1518
1519 /* deassert IB preemption */
1520 WREG32(sdma_gfx_preempt, 0);
1521
1522 /* deassert the preemption condition */
1523 amdgpu_ring_set_preempt_cond_exec(ring, true);
1524 return r;
1525 }
1526
sdma_v7_0_set_trap_irq_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)1527 static int sdma_v7_0_set_trap_irq_state(struct amdgpu_device *adev,
1528 struct amdgpu_irq_src *source,
1529 unsigned type,
1530 enum amdgpu_interrupt_state state)
1531 {
1532 u32 sdma_cntl;
1533
1534 u32 reg_offset = sdma_v7_0_get_reg_offset(adev, type, regSDMA0_CNTL);
1535
1536 sdma_cntl = RREG32(reg_offset);
1537 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
1538 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1539 WREG32(reg_offset, sdma_cntl);
1540
1541 return 0;
1542 }
1543
sdma_v7_0_process_trap_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1544 static int sdma_v7_0_process_trap_irq(struct amdgpu_device *adev,
1545 struct amdgpu_irq_src *source,
1546 struct amdgpu_iv_entry *entry)
1547 {
1548 int instances, queue;
1549
1550 DRM_DEBUG("IH: SDMA trap\n");
1551
1552 queue = entry->ring_id & 0xf;
1553 instances = (entry->ring_id & 0xf0) >> 4;
1554 if (instances > 1) {
1555 DRM_ERROR("IH: wrong ring_ID detected, as wrong sdma instance\n");
1556 return -EINVAL;
1557 }
1558
1559 switch (entry->client_id) {
1560 case SOC21_IH_CLIENTID_GFX:
1561 switch (queue) {
1562 case 0:
1563 amdgpu_fence_process(&adev->sdma.instance[instances].ring);
1564 break;
1565 default:
1566 break;
1567 }
1568 break;
1569 }
1570 return 0;
1571 }
1572
sdma_v7_0_process_fence_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1573 static int sdma_v7_0_process_fence_irq(struct amdgpu_device *adev,
1574 struct amdgpu_irq_src *source,
1575 struct amdgpu_iv_entry *entry)
1576 {
1577 u32 doorbell_offset = entry->src_data[0];
1578
1579 if (adev->enable_mes && doorbell_offset) {
1580 doorbell_offset >>= SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
1581 amdgpu_userq_process_fence_irq(adev, doorbell_offset);
1582 }
1583
1584 return 0;
1585 }
1586
sdma_v7_0_process_illegal_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1587 static int sdma_v7_0_process_illegal_inst_irq(struct amdgpu_device *adev,
1588 struct amdgpu_irq_src *source,
1589 struct amdgpu_iv_entry *entry)
1590 {
1591 return 0;
1592 }
1593
sdma_v7_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)1594 static int sdma_v7_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1595 enum amd_clockgating_state state)
1596 {
1597 return 0;
1598 }
1599
sdma_v7_0_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)1600 static int sdma_v7_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
1601 enum amd_powergating_state state)
1602 {
1603 return 0;
1604 }
1605
sdma_v7_0_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)1606 static void sdma_v7_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
1607 {
1608 }
1609
sdma_v7_0_print_ip_state(struct amdgpu_ip_block * ip_block,struct drm_printer * p)1610 static void sdma_v7_0_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
1611 {
1612 struct amdgpu_device *adev = ip_block->adev;
1613 int i, j;
1614 uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_0);
1615 uint32_t instance_offset;
1616
1617 if (!adev->sdma.ip_dump)
1618 return;
1619
1620 drm_printf(p, "num_instances:%d\n", adev->sdma.num_instances);
1621 for (i = 0; i < adev->sdma.num_instances; i++) {
1622 instance_offset = i * reg_count;
1623 drm_printf(p, "\nInstance:%d\n", i);
1624
1625 for (j = 0; j < reg_count; j++)
1626 drm_printf(p, "%-50s \t 0x%08x\n", sdma_reg_list_7_0[j].reg_name,
1627 adev->sdma.ip_dump[instance_offset + j]);
1628 }
1629 }
1630
sdma_v7_0_dump_ip_state(struct amdgpu_ip_block * ip_block)1631 static void sdma_v7_0_dump_ip_state(struct amdgpu_ip_block *ip_block)
1632 {
1633 struct amdgpu_device *adev = ip_block->adev;
1634 int i, j;
1635 uint32_t instance_offset;
1636 uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_7_0);
1637
1638 if (!adev->sdma.ip_dump)
1639 return;
1640
1641 amdgpu_gfx_off_ctrl(adev, false);
1642 for (i = 0; i < adev->sdma.num_instances; i++) {
1643 instance_offset = i * reg_count;
1644 for (j = 0; j < reg_count; j++)
1645 adev->sdma.ip_dump[instance_offset + j] =
1646 RREG32(sdma_v7_0_get_reg_offset(adev, i,
1647 sdma_reg_list_7_0[j].reg_offset));
1648 }
1649 amdgpu_gfx_off_ctrl(adev, true);
1650 }
1651
1652 const struct amd_ip_funcs sdma_v7_0_ip_funcs = {
1653 .name = "sdma_v7_0",
1654 .early_init = sdma_v7_0_early_init,
1655 .late_init = NULL,
1656 .sw_init = sdma_v7_0_sw_init,
1657 .sw_fini = sdma_v7_0_sw_fini,
1658 .hw_init = sdma_v7_0_hw_init,
1659 .hw_fini = sdma_v7_0_hw_fini,
1660 .suspend = sdma_v7_0_suspend,
1661 .resume = sdma_v7_0_resume,
1662 .is_idle = sdma_v7_0_is_idle,
1663 .wait_for_idle = sdma_v7_0_wait_for_idle,
1664 .soft_reset = sdma_v7_0_soft_reset,
1665 .set_clockgating_state = sdma_v7_0_set_clockgating_state,
1666 .set_powergating_state = sdma_v7_0_set_powergating_state,
1667 .get_clockgating_state = sdma_v7_0_get_clockgating_state,
1668 .dump_ip_state = sdma_v7_0_dump_ip_state,
1669 .print_ip_state = sdma_v7_0_print_ip_state,
1670 };
1671
1672 static const struct amdgpu_ring_funcs sdma_v7_0_ring_funcs = {
1673 .type = AMDGPU_RING_TYPE_SDMA,
1674 .align_mask = 0xf,
1675 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1676 .support_64bit_ptrs = true,
1677 .secure_submission_supported = true,
1678 .get_rptr = sdma_v7_0_ring_get_rptr,
1679 .get_wptr = sdma_v7_0_ring_get_wptr,
1680 .set_wptr = sdma_v7_0_ring_set_wptr,
1681 .emit_frame_size =
1682 5 + /* sdma_v7_0_ring_init_cond_exec */
1683 6 + /* sdma_v7_0_ring_emit_hdp_flush */
1684 6 + /* sdma_v7_0_ring_emit_pipeline_sync */
1685 /* sdma_v7_0_ring_emit_vm_flush */
1686 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1687 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1688 10 + 10 + 10, /* sdma_v7_0_ring_emit_fence x3 for user fence, vm fence */
1689 .emit_ib_size = 5 + 7 + 6, /* sdma_v7_0_ring_emit_ib */
1690 .emit_ib = sdma_v7_0_ring_emit_ib,
1691 .emit_mem_sync = sdma_v7_0_ring_emit_mem_sync,
1692 .emit_fence = sdma_v7_0_ring_emit_fence,
1693 .emit_pipeline_sync = sdma_v7_0_ring_emit_pipeline_sync,
1694 .emit_vm_flush = sdma_v7_0_ring_emit_vm_flush,
1695 .emit_hdp_flush = sdma_v7_0_ring_emit_hdp_flush,
1696 .test_ring = sdma_v7_0_ring_test_ring,
1697 .test_ib = sdma_v7_0_ring_test_ib,
1698 .insert_nop = sdma_v7_0_ring_insert_nop,
1699 .pad_ib = sdma_v7_0_ring_pad_ib,
1700 .emit_wreg = sdma_v7_0_ring_emit_wreg,
1701 .emit_reg_wait = sdma_v7_0_ring_emit_reg_wait,
1702 .emit_reg_write_reg_wait = sdma_v7_0_ring_emit_reg_write_reg_wait,
1703 .init_cond_exec = sdma_v7_0_ring_init_cond_exec,
1704 .preempt_ib = sdma_v7_0_ring_preempt_ib,
1705 .reset = sdma_v7_0_reset_queue,
1706 };
1707
sdma_v7_0_set_ring_funcs(struct amdgpu_device * adev)1708 static void sdma_v7_0_set_ring_funcs(struct amdgpu_device *adev)
1709 {
1710 int i;
1711
1712 for (i = 0; i < adev->sdma.num_instances; i++) {
1713 adev->sdma.instance[i].ring.funcs = &sdma_v7_0_ring_funcs;
1714 adev->sdma.instance[i].ring.me = i;
1715 }
1716 }
1717
1718 static const struct amdgpu_irq_src_funcs sdma_v7_0_trap_irq_funcs = {
1719 .set = sdma_v7_0_set_trap_irq_state,
1720 .process = sdma_v7_0_process_trap_irq,
1721 };
1722
1723 static const struct amdgpu_irq_src_funcs sdma_v7_0_fence_irq_funcs = {
1724 .process = sdma_v7_0_process_fence_irq,
1725 };
1726
1727 static const struct amdgpu_irq_src_funcs sdma_v7_0_illegal_inst_irq_funcs = {
1728 .process = sdma_v7_0_process_illegal_inst_irq,
1729 };
1730
sdma_v7_0_set_irq_funcs(struct amdgpu_device * adev)1731 static void sdma_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1732 {
1733 adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE0 +
1734 adev->sdma.num_instances;
1735 adev->sdma.trap_irq.funcs = &sdma_v7_0_trap_irq_funcs;
1736 adev->sdma.fence_irq.funcs = &sdma_v7_0_fence_irq_funcs;
1737 adev->sdma.illegal_inst_irq.funcs = &sdma_v7_0_illegal_inst_irq_funcs;
1738 }
1739
1740 /**
1741 * sdma_v7_0_emit_copy_buffer - copy buffer using the sDMA engine
1742 *
1743 * @ib: indirect buffer to fill with commands
1744 * @src_offset: src GPU address
1745 * @dst_offset: dst GPU address
1746 * @byte_count: number of bytes to xfer
1747 * @copy_flags: copy flags for the buffers
1748 *
1749 * Copy GPU buffers using the DMA engine.
1750 * Used by the amdgpu ttm implementation to move pages if
1751 * registered as the asic copy callback.
1752 */
sdma_v7_0_emit_copy_buffer(struct amdgpu_ib * ib,uint64_t src_offset,uint64_t dst_offset,uint32_t byte_count,uint32_t copy_flags)1753 static void sdma_v7_0_emit_copy_buffer(struct amdgpu_ib *ib,
1754 uint64_t src_offset,
1755 uint64_t dst_offset,
1756 uint32_t byte_count,
1757 uint32_t copy_flags)
1758 {
1759 uint32_t num_type, data_format, max_com, write_cm;
1760
1761 max_com = AMDGPU_COPY_FLAGS_GET(copy_flags, MAX_COMPRESSED);
1762 data_format = AMDGPU_COPY_FLAGS_GET(copy_flags, DATA_FORMAT);
1763 num_type = AMDGPU_COPY_FLAGS_GET(copy_flags, NUMBER_TYPE);
1764 write_cm = AMDGPU_COPY_FLAGS_GET(copy_flags, WRITE_COMPRESS_DISABLE) ? 2 : 1;
1765
1766 ib->ptr[ib->length_dw++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_COPY) |
1767 SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1768 SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0) |
1769 SDMA_PKT_COPY_LINEAR_HEADER_CPV(1);
1770
1771 ib->ptr[ib->length_dw++] = byte_count - 1;
1772 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1773 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1774 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1775 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1776 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1777
1778 if ((copy_flags & (AMDGPU_COPY_FLAGS_READ_DECOMPRESSED | AMDGPU_COPY_FLAGS_WRITE_COMPRESSED)))
1779 ib->ptr[ib->length_dw++] = SDMA_DCC_DATA_FORMAT(data_format) | SDMA_DCC_NUM_TYPE(num_type) |
1780 ((copy_flags & AMDGPU_COPY_FLAGS_READ_DECOMPRESSED) ? SDMA_DCC_READ_CM(2) : 0) |
1781 ((copy_flags & AMDGPU_COPY_FLAGS_WRITE_COMPRESSED) ? SDMA_DCC_WRITE_CM(write_cm) : 0) |
1782 SDMA_DCC_MAX_COM(max_com) | SDMA_DCC_MAX_UCOM(1);
1783 else
1784 ib->ptr[ib->length_dw++] = 0;
1785 }
1786
1787 /**
1788 * sdma_v7_0_emit_fill_buffer - fill buffer using the sDMA engine
1789 *
1790 * @ib: indirect buffer to fill
1791 * @src_data: value to write to buffer
1792 * @dst_offset: dst GPU address
1793 * @byte_count: number of bytes to xfer
1794 *
1795 * Fill GPU buffers using the DMA engine.
1796 */
sdma_v7_0_emit_fill_buffer(struct amdgpu_ib * ib,uint32_t src_data,uint64_t dst_offset,uint32_t byte_count)1797 static void sdma_v7_0_emit_fill_buffer(struct amdgpu_ib *ib,
1798 uint32_t src_data,
1799 uint64_t dst_offset,
1800 uint32_t byte_count)
1801 {
1802 ib->ptr[ib->length_dw++] = SDMA_PKT_CONSTANT_FILL_HEADER_OP(SDMA_OP_CONST_FILL) |
1803 SDMA_PKT_CONSTANT_FILL_HEADER_COMPRESS(1);
1804 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1805 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1806 ib->ptr[ib->length_dw++] = src_data;
1807 ib->ptr[ib->length_dw++] = byte_count - 1;
1808 }
1809
1810 static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = {
1811 .copy_max_bytes = 1 << 30,
1812 .copy_num_dw = 8,
1813 .emit_copy_buffer = sdma_v7_0_emit_copy_buffer,
1814 .fill_max_bytes = 1 << 30,
1815 .fill_num_dw = 5,
1816 .emit_fill_buffer = sdma_v7_0_emit_fill_buffer,
1817 };
1818
sdma_v7_0_set_buffer_funcs(struct amdgpu_device * adev)1819 static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev)
1820 {
1821 amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_0_buffer_funcs);
1822 }
1823
1824 const struct amdgpu_ip_block_version sdma_v7_0_ip_block = {
1825 .type = AMD_IP_BLOCK_TYPE_SDMA,
1826 .major = 7,
1827 .minor = 0,
1828 .rev = 0,
1829 .funcs = &sdma_v7_0_ip_funcs,
1830 };
1831