1 /*
2 * Copyright 2024 Advanced Micro Devices, Inc. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_vcn.h"
27 #include "amdgpu_pm.h"
28 #include "soc15.h"
29 #include "soc15d.h"
30 #include "soc15_hw_ip.h"
31 #include "vcn_v2_0.h"
32 #include "vcn_v4_0_3.h"
33 #include "mmsch_v5_0.h"
34
35 #include "vcn/vcn_5_0_0_offset.h"
36 #include "vcn/vcn_5_0_0_sh_mask.h"
37 #include "ivsrcid/vcn/irqsrcs_vcn_5_0.h"
38 #include "vcn_v5_0_0.h"
39 #include "vcn_v5_0_1.h"
40
41 #include <drm/drm_drv.h>
42
43 static const struct amdgpu_hwip_reg_entry vcn_reg_list_5_0_1[] = {
44 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS),
45 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS),
46 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID),
47 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2),
48 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0),
49 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1),
50 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD),
51 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI),
52 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO),
53 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2),
54 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2),
55 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3),
56 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3),
57 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4),
58 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4),
59 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR),
60 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR),
61 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2),
62 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2),
63 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3),
64 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3),
65 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4),
66 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4),
67 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE),
68 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2),
69 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3),
70 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4),
71 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL),
72 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA),
73 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK),
74 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE)
75 };
76
77 static int vcn_v5_0_1_start_sriov(struct amdgpu_device *adev);
78 static void vcn_v5_0_1_set_unified_ring_funcs(struct amdgpu_device *adev);
79 static void vcn_v5_0_1_set_irq_funcs(struct amdgpu_device *adev);
80 static int vcn_v5_0_1_set_pg_state(struct amdgpu_vcn_inst *vinst,
81 enum amd_powergating_state state);
82 static void vcn_v5_0_1_unified_ring_set_wptr(struct amdgpu_ring *ring);
83 static void vcn_v5_0_1_set_ras_funcs(struct amdgpu_device *adev);
84 /**
85 * vcn_v5_0_1_early_init - set function pointers and load microcode
86 *
87 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
88 *
89 * Set ring and irq function pointers
90 * Load microcode from filesystem
91 */
vcn_v5_0_1_early_init(struct amdgpu_ip_block * ip_block)92 static int vcn_v5_0_1_early_init(struct amdgpu_ip_block *ip_block)
93 {
94 struct amdgpu_device *adev = ip_block->adev;
95 int i, r;
96
97 for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
98 /* re-use enc ring as unified ring */
99 adev->vcn.inst[i].num_enc_rings = 1;
100
101 vcn_v5_0_1_set_unified_ring_funcs(adev);
102 vcn_v5_0_1_set_irq_funcs(adev);
103 vcn_v5_0_1_set_ras_funcs(adev);
104
105 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
106 adev->vcn.inst[i].set_pg_state = vcn_v5_0_1_set_pg_state;
107
108 r = amdgpu_vcn_early_init(adev, i);
109 if (r)
110 return r;
111 }
112
113 return 0;
114 }
115
vcn_v5_0_1_late_init(struct amdgpu_ip_block * ip_block)116 static int vcn_v5_0_1_late_init(struct amdgpu_ip_block *ip_block)
117 {
118 struct amdgpu_device *adev = ip_block->adev;
119
120 adev->vcn.supported_reset =
121 amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]);
122
123 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
124 case IP_VERSION(13, 0, 12):
125 if ((adev->psp.sos.fw_version >= 0x00450025) &&
126 amdgpu_dpm_reset_vcn_is_supported(adev) &&
127 !amdgpu_sriov_vf(adev))
128 adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
129 break;
130 default:
131 break;
132 }
133
134 return 0;
135 }
136
vcn_v5_0_1_fw_shared_init(struct amdgpu_device * adev,int inst_idx)137 static void vcn_v5_0_1_fw_shared_init(struct amdgpu_device *adev, int inst_idx)
138 {
139 struct amdgpu_vcn5_fw_shared *fw_shared;
140
141 fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
142
143 if (fw_shared->sq.is_enabled)
144 return;
145 fw_shared->present_flag_0 =
146 cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
147 fw_shared->sq.is_enabled = 1;
148
149 if (amdgpu_vcnfw_log)
150 amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]);
151 }
152
153 /**
154 * vcn_v5_0_1_sw_init - sw init for VCN block
155 *
156 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
157 *
158 * Load firmware and sw initialization
159 */
vcn_v5_0_1_sw_init(struct amdgpu_ip_block * ip_block)160 static int vcn_v5_0_1_sw_init(struct amdgpu_ip_block *ip_block)
161 {
162 struct amdgpu_device *adev = ip_block->adev;
163 struct amdgpu_ring *ring;
164 int i, r, vcn_inst;
165
166 /* VCN UNIFIED TRAP */
167 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
168 VCN_5_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq);
169 if (r)
170 return r;
171
172 /* VCN POISON TRAP */
173 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
174 VCN_5_0__SRCID_UVD_POISON, &adev->vcn.inst->ras_poison_irq);
175
176 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
177 vcn_inst = GET_INST(VCN, i);
178
179 r = amdgpu_vcn_sw_init(adev, i);
180 if (r)
181 return r;
182
183 amdgpu_vcn_setup_ucode(adev, i);
184
185 r = amdgpu_vcn_resume(adev, i);
186 if (r)
187 return r;
188
189 ring = &adev->vcn.inst[i].ring_enc[0];
190 ring->use_doorbell = true;
191 if (!amdgpu_sriov_vf(adev))
192 ring->doorbell_index =
193 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
194 11 * vcn_inst;
195 else
196 ring->doorbell_index =
197 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
198 32 * vcn_inst;
199
200 ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id);
201 sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id);
202
203 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
204 AMDGPU_RING_PRIO_DEFAULT, &adev->vcn.inst[i].sched_score);
205 if (r)
206 return r;
207
208 vcn_v5_0_1_fw_shared_init(adev, i);
209 }
210
211 if (amdgpu_sriov_vf(adev)) {
212 r = amdgpu_virt_alloc_mm_table(adev);
213 if (r)
214 return r;
215 }
216
217 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) {
218 r = amdgpu_vcn_ras_sw_init(adev);
219 if (r) {
220 dev_err(adev->dev, "Failed to initialize vcn ras block!\n");
221 return r;
222 }
223 }
224
225 r = amdgpu_vcn_reg_dump_init(adev, vcn_reg_list_5_0_1, ARRAY_SIZE(vcn_reg_list_5_0_1));
226 if (r)
227 return r;
228
229 return amdgpu_vcn_sysfs_reset_mask_init(adev);
230 }
231
232 /**
233 * vcn_v5_0_1_sw_fini - sw fini for VCN block
234 *
235 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
236 *
237 * VCN suspend and free up sw allocation
238 */
vcn_v5_0_1_sw_fini(struct amdgpu_ip_block * ip_block)239 static int vcn_v5_0_1_sw_fini(struct amdgpu_ip_block *ip_block)
240 {
241 struct amdgpu_device *adev = ip_block->adev;
242 int i, r, idx;
243
244 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
245 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
246 struct amdgpu_vcn5_fw_shared *fw_shared;
247
248 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
249 fw_shared->present_flag_0 = 0;
250 fw_shared->sq.is_enabled = 0;
251 }
252
253 drm_dev_exit(idx);
254 }
255
256 if (amdgpu_sriov_vf(adev))
257 amdgpu_virt_free_mm_table(adev);
258
259 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
260 r = amdgpu_vcn_suspend(adev, i);
261 if (r)
262 return r;
263 }
264
265 amdgpu_vcn_sysfs_reset_mask_fini(adev);
266
267 for (i = 0; i < adev->vcn.num_vcn_inst; i++)
268 amdgpu_vcn_sw_fini(adev, i);
269
270 return 0;
271 }
272
vcn_v5_0_1_hw_init_inst(struct amdgpu_device * adev,int i)273 static int vcn_v5_0_1_hw_init_inst(struct amdgpu_device *adev, int i)
274 {
275 struct amdgpu_ring *ring;
276 int vcn_inst;
277
278 vcn_inst = GET_INST(VCN, i);
279 ring = &adev->vcn.inst[i].ring_enc[0];
280
281 if (ring->use_doorbell)
282 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
283 ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
284 11 * vcn_inst),
285 adev->vcn.inst[i].aid_id);
286
287 return 0;
288 }
289
290 /**
291 * vcn_v5_0_1_hw_init - start and test VCN block
292 *
293 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
294 *
295 * Initialize the hardware, boot up the VCPU and do some testing
296 */
vcn_v5_0_1_hw_init(struct amdgpu_ip_block * ip_block)297 static int vcn_v5_0_1_hw_init(struct amdgpu_ip_block *ip_block)
298 {
299 struct amdgpu_device *adev = ip_block->adev;
300 struct amdgpu_ring *ring;
301 int i, r;
302
303 if (amdgpu_sriov_vf(adev)) {
304 r = vcn_v5_0_1_start_sriov(adev);
305 if (r)
306 return r;
307
308 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
309 ring = &adev->vcn.inst[i].ring_enc[0];
310 ring->wptr = 0;
311 ring->wptr_old = 0;
312 vcn_v5_0_1_unified_ring_set_wptr(ring);
313 ring->sched.ready = true;
314 }
315 } else {
316 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x100)
317 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED);
318 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
319 ring = &adev->vcn.inst[i].ring_enc[0];
320 vcn_v5_0_1_hw_init_inst(adev, i);
321
322 /* Re-init fw_shared, if required */
323 vcn_v5_0_1_fw_shared_init(adev, i);
324
325 r = amdgpu_ring_test_helper(ring);
326 if (r)
327 return r;
328 }
329 }
330
331 return 0;
332 }
333
334 /**
335 * vcn_v5_0_1_hw_fini - stop the hardware block
336 *
337 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
338 *
339 * Stop the VCN block, mark ring as not ready any more
340 */
vcn_v5_0_1_hw_fini(struct amdgpu_ip_block * ip_block)341 static int vcn_v5_0_1_hw_fini(struct amdgpu_ip_block *ip_block)
342 {
343 struct amdgpu_device *adev = ip_block->adev;
344 int i;
345
346 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
347 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
348
349 cancel_delayed_work_sync(&adev->vcn.inst[i].idle_work);
350 if (vinst->cur_state != AMD_PG_STATE_GATE)
351 vinst->set_pg_state(vinst, AMD_PG_STATE_GATE);
352 }
353
354 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN) && !amdgpu_sriov_vf(adev))
355 amdgpu_irq_put(adev, &adev->vcn.inst->ras_poison_irq, 0);
356
357 return 0;
358 }
359
360 /**
361 * vcn_v5_0_1_suspend - suspend VCN block
362 *
363 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
364 *
365 * HW fini and suspend VCN block
366 */
vcn_v5_0_1_suspend(struct amdgpu_ip_block * ip_block)367 static int vcn_v5_0_1_suspend(struct amdgpu_ip_block *ip_block)
368 {
369 struct amdgpu_device *adev = ip_block->adev;
370 int r, i;
371
372 r = vcn_v5_0_1_hw_fini(ip_block);
373 if (r)
374 return r;
375
376 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
377 r = amdgpu_vcn_suspend(ip_block->adev, i);
378 if (r)
379 return r;
380 }
381
382 return r;
383 }
384
385 /**
386 * vcn_v5_0_1_resume - resume VCN block
387 *
388 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
389 *
390 * Resume firmware and hw init VCN block
391 */
vcn_v5_0_1_resume(struct amdgpu_ip_block * ip_block)392 static int vcn_v5_0_1_resume(struct amdgpu_ip_block *ip_block)
393 {
394 struct amdgpu_device *adev = ip_block->adev;
395 int r, i;
396
397 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
398 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
399
400 if (amdgpu_in_reset(adev))
401 vinst->cur_state = AMD_PG_STATE_GATE;
402
403 r = amdgpu_vcn_resume(ip_block->adev, i);
404 if (r)
405 return r;
406 }
407
408 r = vcn_v5_0_1_hw_init(ip_block);
409
410 return r;
411 }
412
413 /**
414 * vcn_v5_0_1_mc_resume - memory controller programming
415 *
416 * @vinst: VCN instance
417 *
418 * Let the VCN memory controller know it's offsets
419 */
vcn_v5_0_1_mc_resume(struct amdgpu_vcn_inst * vinst)420 static void vcn_v5_0_1_mc_resume(struct amdgpu_vcn_inst *vinst)
421 {
422 struct amdgpu_device *adev = vinst->adev;
423 int inst = vinst->inst;
424 uint32_t offset, size, vcn_inst;
425 const struct common_firmware_header *hdr;
426
427 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst].fw->data;
428 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
429
430 vcn_inst = GET_INST(VCN, inst);
431 /* cache window 0: fw */
432 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
433 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
434 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
435 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
436 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
437 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0);
438 offset = 0;
439 } else {
440 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
441 lower_32_bits(adev->vcn.inst[inst].gpu_addr));
442 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
443 upper_32_bits(adev->vcn.inst[inst].gpu_addr));
444 offset = size;
445 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0,
446 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
447 }
448 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size);
449
450 /* cache window 1: stack */
451 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
452 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
453 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
454 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
455 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0);
456 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
457
458 /* cache window 2: context */
459 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
460 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
461 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
462 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
463 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0);
464 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
465
466 /* non-cache window */
467 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
468 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
469 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
470 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
471 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
472 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0,
473 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)));
474 }
475
476 /**
477 * vcn_v5_0_1_mc_resume_dpg_mode - memory controller programming for dpg mode
478 *
479 * @vinst: VCN instance
480 * @indirect: indirectly write sram
481 *
482 * Let the VCN memory controller know it's offsets with dpg mode
483 */
vcn_v5_0_1_mc_resume_dpg_mode(struct amdgpu_vcn_inst * vinst,bool indirect)484 static void vcn_v5_0_1_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst,
485 bool indirect)
486 {
487 struct amdgpu_device *adev = vinst->adev;
488 int inst_idx = vinst->inst;
489 uint32_t offset, size;
490 const struct common_firmware_header *hdr;
491
492 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data;
493 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
494
495 /* cache window 0: fw */
496 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
497 if (!indirect) {
498 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
499 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
500 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN +
501 inst_idx].tmr_mc_addr_lo), 0, indirect);
502 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
503 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
504 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN +
505 inst_idx].tmr_mc_addr_hi), 0, indirect);
506 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
507 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
508 } else {
509 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
510 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
511 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
512 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
513 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
514 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
515 }
516 offset = 0;
517 } else {
518 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
519 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
520 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
521 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
522 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
523 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
524 offset = size;
525 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
526 VCN, 0, regUVD_VCPU_CACHE_OFFSET0),
527 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
528 }
529
530 if (!indirect)
531 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
532 VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
533 else
534 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
535 VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
536
537 /* cache window 1: stack */
538 if (!indirect) {
539 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
540 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
541 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
542 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
543 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
544 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
545 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
546 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
547 } else {
548 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
549 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
550 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
551 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
552 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
553 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
554 }
555 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
556 VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
557
558 /* cache window 2: context */
559 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
560 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
561 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset +
562 AMDGPU_VCN_STACK_SIZE), 0, indirect);
563 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
564 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
565 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset +
566 AMDGPU_VCN_STACK_SIZE), 0, indirect);
567 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
568 VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
569 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
570 VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
571
572 /* non-cache window */
573 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
574 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
575 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
576 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
577 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
578 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
579 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
580 VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
581 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
582 VCN, 0, regUVD_VCPU_NONCACHE_SIZE0),
583 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared)), 0, indirect);
584
585 /* VCN global tiling registers */
586 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
587 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
588 }
589
590 /**
591 * vcn_v5_0_1_disable_clock_gating - disable VCN clock gating
592 *
593 * @vinst: VCN instance
594 *
595 * Disable clock gating for VCN block
596 */
vcn_v5_0_1_disable_clock_gating(struct amdgpu_vcn_inst * vinst)597 static void vcn_v5_0_1_disable_clock_gating(struct amdgpu_vcn_inst *vinst)
598 {
599 }
600
601 /**
602 * vcn_v5_0_1_enable_clock_gating - enable VCN clock gating
603 *
604 * @vinst: VCN instance
605 *
606 * Enable clock gating for VCN block
607 */
vcn_v5_0_1_enable_clock_gating(struct amdgpu_vcn_inst * vinst)608 static void vcn_v5_0_1_enable_clock_gating(struct amdgpu_vcn_inst *vinst)
609 {
610 }
611
612 /**
613 * vcn_v5_0_1_pause_dpg_mode - VCN pause with dpg mode
614 *
615 * @vinst: VCN instance
616 * @new_state: pause state
617 *
618 * Pause dpg mode for VCN block
619 */
vcn_v5_0_1_pause_dpg_mode(struct amdgpu_vcn_inst * vinst,struct dpg_pause_state * new_state)620 static int vcn_v5_0_1_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
621 struct dpg_pause_state *new_state)
622 {
623 struct amdgpu_device *adev = vinst->adev;
624 uint32_t reg_data = 0;
625 int vcn_inst;
626
627 vcn_inst = GET_INST(VCN, vinst->inst);
628
629 /* pause/unpause if state is changed */
630 if (vinst->pause_state.fw_based != new_state->fw_based) {
631 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d %s\n",
632 vinst->pause_state.fw_based, new_state->fw_based,
633 new_state->fw_based ? "VCN_DPG_STATE__PAUSE" : "VCN_DPG_STATE__UNPAUSE");
634 reg_data = RREG32_SOC15(VCN, vcn_inst, regUVD_DPG_PAUSE) &
635 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
636
637 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
638 /* pause DPG */
639 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
640 WREG32_SOC15(VCN, vcn_inst, regUVD_DPG_PAUSE, reg_data);
641
642 /* wait for ACK */
643 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_DPG_PAUSE,
644 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
645 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
646 } else {
647 /* unpause DPG, no need to wait */
648 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
649 WREG32_SOC15(VCN, vcn_inst, regUVD_DPG_PAUSE, reg_data);
650 }
651 vinst->pause_state.fw_based = new_state->fw_based;
652 }
653
654 return 0;
655 }
656
657
658 /**
659 * vcn_v5_0_1_start_dpg_mode - VCN start with dpg mode
660 *
661 * @vinst: VCN instance
662 * @indirect: indirectly write sram
663 *
664 * Start VCN block with dpg mode
665 */
vcn_v5_0_1_start_dpg_mode(struct amdgpu_vcn_inst * vinst,bool indirect)666 static int vcn_v5_0_1_start_dpg_mode(struct amdgpu_vcn_inst *vinst,
667 bool indirect)
668 {
669 struct amdgpu_device *adev = vinst->adev;
670 int inst_idx = vinst->inst;
671 struct amdgpu_vcn5_fw_shared *fw_shared =
672 adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
673 struct amdgpu_ring *ring;
674 struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__PAUSE};
675 int vcn_inst, ret;
676 uint32_t tmp;
677
678 vcn_inst = GET_INST(VCN, inst_idx);
679
680 /* disable register anti-hang mechanism */
681 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1,
682 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
683
684 /* enable dynamic power gating mode */
685 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS);
686 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
687 WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp);
688
689 if (indirect) {
690 adev->vcn.inst[inst_idx].dpg_sram_curr_addr =
691 (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
692 /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */
693 WREG32_SOC24_DPG_MODE(inst_idx, 0xDEADBEEF,
694 adev->vcn.inst[inst_idx].aid_id, 0, true);
695 }
696
697 /* enable VCPU clock */
698 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
699 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
700 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
701 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect);
702
703 /* disable master interrupt */
704 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
705 VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect);
706
707 /* setup regUVD_LMI_CTRL */
708 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
709 UVD_LMI_CTRL__REQ_MODE_MASK |
710 UVD_LMI_CTRL__CRC_RESET_MASK |
711 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
712 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
713 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
714 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
715 0x00100000L);
716 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
717 VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect);
718
719 vcn_v5_0_1_mc_resume_dpg_mode(vinst, indirect);
720
721 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
722 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
723 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
724 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect);
725
726 /* enable LMI MC and UMC channels */
727 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
728 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
729 VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect);
730
731 /* enable master interrupt */
732 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
733 VCN, 0, regUVD_MASTINT_EN),
734 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
735
736 if (indirect) {
737 ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM);
738 if (ret) {
739 dev_err(adev->dev, "vcn sram load failed %d\n", ret);
740 return ret;
741 }
742 }
743
744 /* resetting ring, fw should not check RB ring */
745 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
746
747 /* Pause dpg */
748 vcn_v5_0_1_pause_dpg_mode(vinst, &state);
749
750 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
751
752 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, lower_32_bits(ring->gpu_addr));
753 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
754 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, ring->ring_size / sizeof(uint32_t));
755
756 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
757 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
758 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
759
760 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0);
761 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0);
762
763 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR);
764 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, tmp);
765 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR);
766
767 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
768 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
769 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
770 /* resetting done, fw can check RB ring */
771 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
772
773 WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL,
774 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
775 VCN_RB1_DB_CTRL__EN_MASK);
776 /* Read DB_CTRL to flush the write DB_CTRL command. */
777 RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL);
778
779 return 0;
780 }
781
vcn_v5_0_1_start_sriov(struct amdgpu_device * adev)782 static int vcn_v5_0_1_start_sriov(struct amdgpu_device *adev)
783 {
784 int i, vcn_inst;
785 struct amdgpu_ring *ring_enc;
786 uint64_t cache_addr;
787 uint64_t rb_enc_addr;
788 uint64_t ctx_addr;
789 uint32_t param, resp, expected;
790 uint32_t offset, cache_size;
791 uint32_t tmp, timeout;
792
793 struct amdgpu_mm_table *table = &adev->virt.mm_table;
794 uint32_t *table_loc;
795 uint32_t table_size;
796 uint32_t size, size_dw;
797 uint32_t init_status;
798 uint32_t enabled_vcn;
799
800 struct mmsch_v5_0_cmd_direct_write
801 direct_wt = { {0} };
802 struct mmsch_v5_0_cmd_direct_read_modify_write
803 direct_rd_mod_wt = { {0} };
804 struct mmsch_v5_0_cmd_end end = { {0} };
805 struct mmsch_v5_0_init_header header;
806
807 struct amdgpu_vcn5_fw_shared *fw_shared;
808 struct amdgpu_fw_shared_rb_setup *rb_setup;
809
810 direct_wt.cmd_header.command_type =
811 MMSCH_COMMAND__DIRECT_REG_WRITE;
812 direct_rd_mod_wt.cmd_header.command_type =
813 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
814 end.cmd_header.command_type = MMSCH_COMMAND__END;
815
816 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
817 vcn_inst = GET_INST(VCN, i);
818
819 vcn_v5_0_1_fw_shared_init(adev, vcn_inst);
820
821 memset(&header, 0, sizeof(struct mmsch_v5_0_init_header));
822 header.version = MMSCH_VERSION;
823 header.total_size = sizeof(struct mmsch_v5_0_init_header) >> 2;
824
825 table_loc = (uint32_t *)table->cpu_addr;
826 table_loc += header.total_size;
827
828 table_size = 0;
829
830 MMSCH_V5_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS),
831 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
832
833 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4);
834
835 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
836 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
837 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
838 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
839
840 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
841 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
842 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
843
844 offset = 0;
845 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
846 regUVD_VCPU_CACHE_OFFSET0), 0);
847 } else {
848 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
849 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
850 lower_32_bits(adev->vcn.inst[i].gpu_addr));
851 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
852 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
853 upper_32_bits(adev->vcn.inst[i].gpu_addr));
854 offset = cache_size;
855 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
856 regUVD_VCPU_CACHE_OFFSET0),
857 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
858 }
859
860 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
861 regUVD_VCPU_CACHE_SIZE0),
862 cache_size);
863
864 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset;
865 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
866 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr));
867 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
868 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr));
869 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
870 regUVD_VCPU_CACHE_OFFSET1), 0);
871 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
872 regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE);
873
874 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset +
875 AMDGPU_VCN_STACK_SIZE;
876
877 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
878 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr));
879
880 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
881 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr));
882
883 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
884 regUVD_VCPU_CACHE_OFFSET2), 0);
885
886 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
887 regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE);
888
889 fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr;
890 rb_setup = &fw_shared->rb_setup;
891
892 ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0];
893 ring_enc->wptr = 0;
894 rb_enc_addr = ring_enc->gpu_addr;
895
896 rb_setup->is_rb_enabled_flags |= RB_ENABLED;
897 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
898 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
899 rb_setup->rb_size = ring_enc->ring_size / 4;
900 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
901
902 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
903 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
904 lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr));
905 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
906 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
907 upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr));
908 MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
909 regUVD_VCPU_NONCACHE_SIZE0),
910 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
911 MMSCH_V5_0_INSERT_END();
912
913 header.vcn0.init_status = 0;
914 header.vcn0.table_offset = header.total_size;
915 header.vcn0.table_size = table_size;
916 header.total_size += table_size;
917
918 /* Send init table to mmsch */
919 size = sizeof(struct mmsch_v5_0_init_header);
920 table_loc = (uint32_t *)table->cpu_addr;
921 memcpy((void *)table_loc, &header, size);
922
923 ctx_addr = table->gpu_addr;
924 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
925 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
926
927 tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID);
928 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
929 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
930 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp);
931
932 size = header.total_size;
933 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size);
934
935 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0);
936
937 param = 0x00000001;
938 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param);
939 tmp = 0;
940 timeout = 1000;
941 resp = 0;
942 expected = MMSCH_VF_MAILBOX_RESP__OK;
943 while (resp != expected) {
944 resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP);
945 if (resp != 0)
946 break;
947
948 udelay(10);
949 tmp = tmp + 10;
950 if (tmp >= timeout) {
951 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
952 " waiting for regMMSCH_VF_MAILBOX_RESP "\
953 "(expected=0x%08x, readback=0x%08x)\n",
954 tmp, expected, resp);
955 return -EBUSY;
956 }
957 }
958
959 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0;
960 init_status = ((struct mmsch_v5_0_init_header *)(table_loc))->vcn0.init_status;
961 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
962 && init_status != MMSCH_VF_ENGINE_STATUS__PASS) {
963 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
964 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);
965 }
966 }
967
968 return 0;
969 }
970
971 /**
972 * vcn_v5_0_1_start - VCN start
973 *
974 * @vinst: VCN instance
975 *
976 * Start VCN block
977 */
vcn_v5_0_1_start(struct amdgpu_vcn_inst * vinst)978 static int vcn_v5_0_1_start(struct amdgpu_vcn_inst *vinst)
979 {
980 struct amdgpu_device *adev = vinst->adev;
981 int i = vinst->inst;
982 struct amdgpu_vcn5_fw_shared *fw_shared;
983 struct amdgpu_ring *ring;
984 uint32_t tmp;
985 int j, k, r, vcn_inst;
986
987 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
988
989 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
990 return vcn_v5_0_1_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram);
991
992 vcn_inst = GET_INST(VCN, i);
993
994 /* set VCN status busy */
995 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
996 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp);
997
998 /* enable VCPU clock */
999 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL),
1000 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1001
1002 /* disable master interrupt */
1003 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0,
1004 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1005
1006 /* enable LMI MC and UMC channels */
1007 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0,
1008 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1009
1010 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET);
1011 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1012 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1013 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp);
1014
1015 /* setup regUVD_LMI_CTRL */
1016 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL);
1017 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, tmp |
1018 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1019 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1020 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1021 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1022
1023 vcn_v5_0_1_mc_resume(vinst);
1024
1025 /* VCN global tiling registers */
1026 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG,
1027 adev->gfx.config.gb_addr_config);
1028
1029 /* unblock VCPU register access */
1030 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0,
1031 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1032
1033 /* release VCPU reset to boot */
1034 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0,
1035 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1036
1037 for (j = 0; j < 10; ++j) {
1038 uint32_t status;
1039
1040 for (k = 0; k < 100; ++k) {
1041 status = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS);
1042 if (status & 2)
1043 break;
1044 mdelay(100);
1045 if (amdgpu_emu_mode == 1)
1046 msleep(20);
1047 }
1048
1049 if (amdgpu_emu_mode == 1) {
1050 r = -1;
1051 if (status & 2) {
1052 r = 0;
1053 break;
1054 }
1055 } else {
1056 r = 0;
1057 if (status & 2)
1058 break;
1059
1060 dev_err(adev->dev,
1061 "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
1062 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL),
1063 UVD_VCPU_CNTL__BLK_RST_MASK,
1064 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1065 mdelay(10);
1066 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0,
1067 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1068
1069 mdelay(10);
1070 r = -1;
1071 }
1072 }
1073
1074 if (r) {
1075 dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
1076 return r;
1077 }
1078
1079 /* enable master interrupt */
1080 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN),
1081 UVD_MASTINT_EN__VCPU_EN_MASK,
1082 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1083
1084 /* clear the busy bit of VCN_STATUS */
1085 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0,
1086 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1087
1088 ring = &adev->vcn.inst[i].ring_enc[0];
1089
1090 WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL,
1091 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1092 VCN_RB1_DB_CTRL__EN_MASK);
1093
1094 /* Read DB_CTRL to flush the write DB_CTRL command. */
1095 RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL);
1096
1097 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, ring->gpu_addr);
1098 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1099 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, ring->ring_size / 4);
1100
1101 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
1102 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1103 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
1104 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1105 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0);
1106 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0);
1107
1108 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR);
1109 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, tmp);
1110 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR);
1111
1112 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE);
1113 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1114 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp);
1115 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1116
1117 /* Keeping one read-back to ensure all register writes are done,
1118 * otherwise it may introduce race conditions.
1119 */
1120 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS);
1121
1122 return 0;
1123 }
1124
1125 /**
1126 * vcn_v5_0_1_stop_dpg_mode - VCN stop with dpg mode
1127 *
1128 * @vinst: VCN instance
1129 *
1130 * Stop VCN block with dpg mode
1131 */
vcn_v5_0_1_stop_dpg_mode(struct amdgpu_vcn_inst * vinst)1132 static void vcn_v5_0_1_stop_dpg_mode(struct amdgpu_vcn_inst *vinst)
1133 {
1134 struct amdgpu_device *adev = vinst->adev;
1135 int inst_idx = vinst->inst;
1136 uint32_t tmp;
1137 int vcn_inst;
1138 struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1139
1140 vcn_inst = GET_INST(VCN, inst_idx);
1141
1142 /* Unpause dpg */
1143 vcn_v5_0_1_pause_dpg_mode(vinst, &state);
1144
1145 /* Wait for power status to be 1 */
1146 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1,
1147 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1148
1149 /* wait for read ptr to be equal to write ptr */
1150 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR);
1151 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1152
1153 /* disable dynamic power gating mode */
1154 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0,
1155 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1156
1157 /* Keeping one read-back to ensure all register writes are done,
1158 * otherwise it may introduce race conditions.
1159 */
1160 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS);
1161 }
1162
1163 /**
1164 * vcn_v5_0_1_stop - VCN stop
1165 *
1166 * @vinst: VCN instance
1167 *
1168 * Stop VCN block
1169 */
vcn_v5_0_1_stop(struct amdgpu_vcn_inst * vinst)1170 static int vcn_v5_0_1_stop(struct amdgpu_vcn_inst *vinst)
1171 {
1172 struct amdgpu_device *adev = vinst->adev;
1173 int i = vinst->inst;
1174 struct amdgpu_vcn5_fw_shared *fw_shared;
1175 uint32_t tmp;
1176 int r = 0, vcn_inst;
1177
1178 vcn_inst = GET_INST(VCN, i);
1179
1180 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1181 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1182
1183 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1184 vcn_v5_0_1_stop_dpg_mode(vinst);
1185 return 0;
1186 }
1187
1188 /* wait for vcn idle */
1189 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1190 if (r)
1191 return r;
1192
1193 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1194 UVD_LMI_STATUS__READ_CLEAN_MASK |
1195 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1196 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1197 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, tmp);
1198 if (r)
1199 return r;
1200
1201 /* disable LMI UMC channel */
1202 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2);
1203 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1204 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp);
1205 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1206 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1207 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, tmp);
1208 if (r)
1209 return r;
1210
1211 /* block VCPU register access */
1212 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL),
1213 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1214 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1215
1216 /* reset VCPU */
1217 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL),
1218 UVD_VCPU_CNTL__BLK_RST_MASK,
1219 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1220
1221 /* disable VCPU clock */
1222 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0,
1223 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1224
1225 /* apply soft reset */
1226 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET);
1227 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1228 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp);
1229 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET);
1230 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1231 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp);
1232
1233 /* clear status */
1234 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0);
1235
1236 /* Keeping one read-back to ensure all register writes are done,
1237 * otherwise it may introduce race conditions.
1238 */
1239 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS);
1240
1241 return 0;
1242 }
1243
1244 /**
1245 * vcn_v5_0_1_unified_ring_get_rptr - get unified read pointer
1246 *
1247 * @ring: amdgpu_ring pointer
1248 *
1249 * Returns the current hardware unified read pointer
1250 */
vcn_v5_0_1_unified_ring_get_rptr(struct amdgpu_ring * ring)1251 static uint64_t vcn_v5_0_1_unified_ring_get_rptr(struct amdgpu_ring *ring)
1252 {
1253 struct amdgpu_device *adev = ring->adev;
1254
1255 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1256 DRM_ERROR("wrong ring id is identified in %s", __func__);
1257
1258 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR);
1259 }
1260
1261 /**
1262 * vcn_v5_0_1_unified_ring_get_wptr - get unified write pointer
1263 *
1264 * @ring: amdgpu_ring pointer
1265 *
1266 * Returns the current hardware unified write pointer
1267 */
vcn_v5_0_1_unified_ring_get_wptr(struct amdgpu_ring * ring)1268 static uint64_t vcn_v5_0_1_unified_ring_get_wptr(struct amdgpu_ring *ring)
1269 {
1270 struct amdgpu_device *adev = ring->adev;
1271
1272 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1273 DRM_ERROR("wrong ring id is identified in %s", __func__);
1274
1275 if (ring->use_doorbell)
1276 return *ring->wptr_cpu_addr;
1277 else
1278 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR);
1279 }
1280
1281 /**
1282 * vcn_v5_0_1_unified_ring_set_wptr - set enc write pointer
1283 *
1284 * @ring: amdgpu_ring pointer
1285 *
1286 * Commits the enc write pointer to the hardware
1287 */
vcn_v5_0_1_unified_ring_set_wptr(struct amdgpu_ring * ring)1288 static void vcn_v5_0_1_unified_ring_set_wptr(struct amdgpu_ring *ring)
1289 {
1290 struct amdgpu_device *adev = ring->adev;
1291
1292 if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1293 DRM_ERROR("wrong ring id is identified in %s", __func__);
1294
1295 if (ring->use_doorbell) {
1296 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1297 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1298 } else {
1299 WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR,
1300 lower_32_bits(ring->wptr));
1301 }
1302 }
1303
vcn_v5_0_1_ring_reset(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)1304 static int vcn_v5_0_1_ring_reset(struct amdgpu_ring *ring,
1305 unsigned int vmid,
1306 struct amdgpu_fence *timedout_fence)
1307 {
1308 int r = 0;
1309 int vcn_inst;
1310 struct amdgpu_device *adev = ring->adev;
1311 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
1312
1313 amdgpu_ring_reset_helper_begin(ring, timedout_fence);
1314
1315 vcn_inst = GET_INST(VCN, ring->me);
1316 r = amdgpu_dpm_reset_vcn(adev, 1 << vcn_inst);
1317
1318 if (r) {
1319 DRM_DEV_ERROR(adev->dev, "VCN reset fail : %d\n", r);
1320 return r;
1321 }
1322
1323 vcn_v5_0_1_hw_init_inst(adev, ring->me);
1324 vcn_v5_0_1_start_dpg_mode(vinst, vinst->indirect_sram);
1325
1326 return amdgpu_ring_reset_helper_end(ring, timedout_fence);
1327 }
1328
1329 static const struct amdgpu_ring_funcs vcn_v5_0_1_unified_ring_vm_funcs = {
1330 .type = AMDGPU_RING_TYPE_VCN_ENC,
1331 .align_mask = 0x3f,
1332 .nop = VCN_ENC_CMD_NO_OP,
1333 .get_rptr = vcn_v5_0_1_unified_ring_get_rptr,
1334 .get_wptr = vcn_v5_0_1_unified_ring_get_wptr,
1335 .set_wptr = vcn_v5_0_1_unified_ring_set_wptr,
1336 .emit_frame_size = SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1337 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1338 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1339 5 +
1340 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1341 1, /* vcn_v2_0_enc_ring_insert_end */
1342 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1343 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1344 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1345 .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush,
1346 .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush,
1347 .test_ring = amdgpu_vcn_enc_ring_test_ring,
1348 .test_ib = amdgpu_vcn_unified_ring_test_ib,
1349 .insert_nop = amdgpu_ring_insert_nop,
1350 .insert_end = vcn_v2_0_enc_ring_insert_end,
1351 .pad_ib = amdgpu_ring_generic_pad_ib,
1352 .begin_use = amdgpu_vcn_ring_begin_use,
1353 .end_use = amdgpu_vcn_ring_end_use,
1354 .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg,
1355 .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait,
1356 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1357 .reset = vcn_v5_0_1_ring_reset,
1358 };
1359
1360 /**
1361 * vcn_v5_0_1_set_unified_ring_funcs - set unified ring functions
1362 *
1363 * @adev: amdgpu_device pointer
1364 *
1365 * Set unified ring functions
1366 */
vcn_v5_0_1_set_unified_ring_funcs(struct amdgpu_device * adev)1367 static void vcn_v5_0_1_set_unified_ring_funcs(struct amdgpu_device *adev)
1368 {
1369 int i, vcn_inst;
1370
1371 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1372 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v5_0_1_unified_ring_vm_funcs;
1373 adev->vcn.inst[i].ring_enc[0].me = i;
1374 vcn_inst = GET_INST(VCN, i);
1375 adev->vcn.inst[i].aid_id = vcn_inst / adev->vcn.num_inst_per_aid;
1376 }
1377 }
1378
1379 /**
1380 * vcn_v5_0_1_is_idle - check VCN block is idle
1381 *
1382 * @ip_block: Pointer to the amdgpu_ip_block structure
1383 *
1384 * Check whether VCN block is idle
1385 */
vcn_v5_0_1_is_idle(struct amdgpu_ip_block * ip_block)1386 static bool vcn_v5_0_1_is_idle(struct amdgpu_ip_block *ip_block)
1387 {
1388 struct amdgpu_device *adev = ip_block->adev;
1389 int i, ret = 1;
1390
1391 for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
1392 ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == UVD_STATUS__IDLE);
1393
1394 return ret;
1395 }
1396
1397 /**
1398 * vcn_v5_0_1_wait_for_idle - wait for VCN block idle
1399 *
1400 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
1401 *
1402 * Wait for VCN block idle
1403 */
vcn_v5_0_1_wait_for_idle(struct amdgpu_ip_block * ip_block)1404 static int vcn_v5_0_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
1405 {
1406 struct amdgpu_device *adev = ip_block->adev;
1407 int i, ret = 0;
1408
1409 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1410 ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, UVD_STATUS__IDLE,
1411 UVD_STATUS__IDLE);
1412 if (ret)
1413 return ret;
1414 }
1415
1416 return ret;
1417 }
1418
1419 /**
1420 * vcn_v5_0_1_set_clockgating_state - set VCN block clockgating state
1421 *
1422 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
1423 * @state: clock gating state
1424 *
1425 * Set VCN block clockgating state
1426 */
vcn_v5_0_1_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)1427 static int vcn_v5_0_1_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1428 enum amd_clockgating_state state)
1429 {
1430 struct amdgpu_device *adev = ip_block->adev;
1431 bool enable = state == AMD_CG_STATE_GATE;
1432 int i;
1433
1434 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1435 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
1436
1437 if (enable) {
1438 if (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) != UVD_STATUS__IDLE)
1439 return -EBUSY;
1440 vcn_v5_0_1_enable_clock_gating(vinst);
1441 } else {
1442 vcn_v5_0_1_disable_clock_gating(vinst);
1443 }
1444 }
1445
1446 return 0;
1447 }
1448
vcn_v5_0_1_set_pg_state(struct amdgpu_vcn_inst * vinst,enum amd_powergating_state state)1449 static int vcn_v5_0_1_set_pg_state(struct amdgpu_vcn_inst *vinst,
1450 enum amd_powergating_state state)
1451 {
1452 struct amdgpu_device *adev = vinst->adev;
1453 int ret = 0;
1454
1455 /* for SRIOV, guest should not control VCN Power-gating
1456 * MMSCH FW should control Power-gating and clock-gating
1457 * guest should avoid touching CGC and PG
1458 */
1459 if (amdgpu_sriov_vf(adev)) {
1460 vinst->cur_state = AMD_PG_STATE_UNGATE;
1461 return 0;
1462 }
1463
1464 if (state == vinst->cur_state)
1465 return 0;
1466
1467 if (state == AMD_PG_STATE_GATE)
1468 ret = vcn_v5_0_1_stop(vinst);
1469 else
1470 ret = vcn_v5_0_1_start(vinst);
1471
1472 if (!ret)
1473 vinst->cur_state = state;
1474
1475 return ret;
1476 }
1477
1478 /**
1479 * vcn_v5_0_1_process_interrupt - process VCN block interrupt
1480 *
1481 * @adev: amdgpu_device pointer
1482 * @source: interrupt sources
1483 * @entry: interrupt entry from clients and sources
1484 *
1485 * Process VCN block interrupt
1486 */
vcn_v5_0_1_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1487 static int vcn_v5_0_1_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1488 struct amdgpu_iv_entry *entry)
1489 {
1490 uint32_t i, inst;
1491
1492 i = node_id_to_phys_map[entry->node_id];
1493
1494 DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n");
1495
1496 for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst)
1497 if (adev->vcn.inst[inst].aid_id == i)
1498 break;
1499 if (inst >= adev->vcn.num_vcn_inst) {
1500 dev_WARN_ONCE(adev->dev, 1,
1501 "Interrupt received for unknown VCN instance %d",
1502 entry->node_id);
1503 return 0;
1504 }
1505
1506 switch (entry->src_id) {
1507 case VCN_5_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1508 amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]);
1509 break;
1510 default:
1511 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
1512 entry->src_id, entry->src_data[0]);
1513 break;
1514 }
1515
1516 return 0;
1517 }
1518
vcn_v5_0_1_set_ras_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)1519 static int vcn_v5_0_1_set_ras_interrupt_state(struct amdgpu_device *adev,
1520 struct amdgpu_irq_src *source,
1521 unsigned int type,
1522 enum amdgpu_interrupt_state state)
1523 {
1524 return 0;
1525 }
1526
1527 static const struct amdgpu_irq_src_funcs vcn_v5_0_1_irq_funcs = {
1528 .process = vcn_v5_0_1_process_interrupt,
1529 };
1530
1531 static const struct amdgpu_irq_src_funcs vcn_v5_0_1_ras_irq_funcs = {
1532 .set = vcn_v5_0_1_set_ras_interrupt_state,
1533 .process = amdgpu_vcn_process_poison_irq,
1534 };
1535
1536
1537 /**
1538 * vcn_v5_0_1_set_irq_funcs - set VCN block interrupt irq functions
1539 *
1540 * @adev: amdgpu_device pointer
1541 *
1542 * Set VCN block interrupt irq functions
1543 */
vcn_v5_0_1_set_irq_funcs(struct amdgpu_device * adev)1544 static void vcn_v5_0_1_set_irq_funcs(struct amdgpu_device *adev)
1545 {
1546 int i;
1547
1548 for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
1549 adev->vcn.inst->irq.num_types++;
1550
1551 adev->vcn.inst->irq.funcs = &vcn_v5_0_1_irq_funcs;
1552
1553 adev->vcn.inst->ras_poison_irq.num_types = 1;
1554 adev->vcn.inst->ras_poison_irq.funcs = &vcn_v5_0_1_ras_irq_funcs;
1555
1556 }
1557
1558 static const struct amd_ip_funcs vcn_v5_0_1_ip_funcs = {
1559 .name = "vcn_v5_0_1",
1560 .early_init = vcn_v5_0_1_early_init,
1561 .late_init = vcn_v5_0_1_late_init,
1562 .sw_init = vcn_v5_0_1_sw_init,
1563 .sw_fini = vcn_v5_0_1_sw_fini,
1564 .hw_init = vcn_v5_0_1_hw_init,
1565 .hw_fini = vcn_v5_0_1_hw_fini,
1566 .suspend = vcn_v5_0_1_suspend,
1567 .resume = vcn_v5_0_1_resume,
1568 .is_idle = vcn_v5_0_1_is_idle,
1569 .wait_for_idle = vcn_v5_0_1_wait_for_idle,
1570 .check_soft_reset = NULL,
1571 .pre_soft_reset = NULL,
1572 .soft_reset = NULL,
1573 .post_soft_reset = NULL,
1574 .set_clockgating_state = vcn_v5_0_1_set_clockgating_state,
1575 .set_powergating_state = vcn_set_powergating_state,
1576 .dump_ip_state = amdgpu_vcn_dump_ip_state,
1577 .print_ip_state = amdgpu_vcn_print_ip_state,
1578 };
1579
1580 const struct amdgpu_ip_block_version vcn_v5_0_1_ip_block = {
1581 .type = AMD_IP_BLOCK_TYPE_VCN,
1582 .major = 5,
1583 .minor = 0,
1584 .rev = 1,
1585 .funcs = &vcn_v5_0_1_ip_funcs,
1586 };
1587
vcn_v5_0_1_query_poison_by_instance(struct amdgpu_device * adev,uint32_t instance,uint32_t sub_block)1588 static uint32_t vcn_v5_0_1_query_poison_by_instance(struct amdgpu_device *adev,
1589 uint32_t instance, uint32_t sub_block)
1590 {
1591 uint32_t poison_stat = 0, reg_value = 0;
1592
1593 switch (sub_block) {
1594 case AMDGPU_VCN_V5_0_1_VCPU_VCODEC:
1595 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS);
1596 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
1597 break;
1598 default:
1599 break;
1600 }
1601
1602 if (poison_stat)
1603 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
1604 instance, sub_block);
1605
1606 return poison_stat;
1607 }
1608
vcn_v5_0_1_query_poison_status(struct amdgpu_device * adev)1609 static bool vcn_v5_0_1_query_poison_status(struct amdgpu_device *adev)
1610 {
1611 uint32_t inst, sub;
1612 uint32_t poison_stat = 0;
1613
1614 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
1615 for (sub = 0; sub < AMDGPU_VCN_V5_0_1_MAX_SUB_BLOCK; sub++)
1616 poison_stat +=
1617 vcn_v5_0_1_query_poison_by_instance(adev, inst, sub);
1618
1619 return !!poison_stat;
1620 }
1621
1622 static const struct amdgpu_ras_block_hw_ops vcn_v5_0_1_ras_hw_ops = {
1623 .query_poison_status = vcn_v5_0_1_query_poison_status,
1624 };
1625
vcn_v5_0_1_aca_bank_parser(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)1626 static int vcn_v5_0_1_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1627 enum aca_smu_type type, void *data)
1628 {
1629 struct aca_bank_info info;
1630 u64 misc0;
1631 int ret;
1632
1633 ret = aca_bank_info_decode(bank, &info);
1634 if (ret)
1635 return ret;
1636
1637 misc0 = bank->regs[ACA_REG_IDX_MISC0];
1638 switch (type) {
1639 case ACA_SMU_TYPE_UE:
1640 bank->aca_err_type = ACA_ERROR_TYPE_UE;
1641 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
1642 1ULL);
1643 break;
1644 case ACA_SMU_TYPE_CE:
1645 bank->aca_err_type = ACA_ERROR_TYPE_CE;
1646 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
1647 ACA_REG__MISC0__ERRCNT(misc0));
1648 break;
1649 default:
1650 return -EINVAL;
1651 }
1652
1653 return ret;
1654 }
1655
1656 /* reference to smu driver if header file */
1657 static int vcn_v5_0_1_err_codes[] = {
1658 14, 15, 47, /* VCN [D|V|S] */
1659 };
1660
vcn_v5_0_1_aca_bank_is_valid(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)1661 static bool vcn_v5_0_1_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
1662 enum aca_smu_type type, void *data)
1663 {
1664 u32 instlo;
1665
1666 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
1667 instlo &= GENMASK(31, 1);
1668
1669 if (instlo != mmSMNAID_AID0_MCA_SMU)
1670 return false;
1671
1672 if (aca_bank_check_error_codes(handle->adev, bank,
1673 vcn_v5_0_1_err_codes,
1674 ARRAY_SIZE(vcn_v5_0_1_err_codes)))
1675 return false;
1676
1677 return true;
1678 }
1679
1680 static const struct aca_bank_ops vcn_v5_0_1_aca_bank_ops = {
1681 .aca_bank_parser = vcn_v5_0_1_aca_bank_parser,
1682 .aca_bank_is_valid = vcn_v5_0_1_aca_bank_is_valid,
1683 };
1684
1685 static const struct aca_info vcn_v5_0_1_aca_info = {
1686 .hwip = ACA_HWIP_TYPE_SMU,
1687 .mask = ACA_ERROR_UE_MASK,
1688 .bank_ops = &vcn_v5_0_1_aca_bank_ops,
1689 };
1690
vcn_v5_0_1_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)1691 static int vcn_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1692 {
1693 int r;
1694
1695 r = amdgpu_ras_block_late_init(adev, ras_block);
1696 if (r)
1697 return r;
1698
1699 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN,
1700 &vcn_v5_0_1_aca_info, NULL);
1701 if (r)
1702 goto late_fini;
1703
1704 if (amdgpu_ras_is_supported(adev, ras_block->block) &&
1705 adev->vcn.inst->ras_poison_irq.funcs) {
1706 r = amdgpu_irq_get(adev, &adev->vcn.inst->ras_poison_irq, 0);
1707 if (r)
1708 goto late_fini;
1709 }
1710
1711 return 0;
1712
1713 late_fini:
1714 amdgpu_ras_block_late_fini(adev, ras_block);
1715
1716 return r;
1717 }
1718
1719 static struct amdgpu_vcn_ras vcn_v5_0_1_ras = {
1720 .ras_block = {
1721 .hw_ops = &vcn_v5_0_1_ras_hw_ops,
1722 .ras_late_init = vcn_v5_0_1_ras_late_init,
1723 },
1724 };
1725
vcn_v5_0_1_set_ras_funcs(struct amdgpu_device * adev)1726 static void vcn_v5_0_1_set_ras_funcs(struct amdgpu_device *adev)
1727 {
1728 adev->vcn.ras = &vcn_v5_0_1_ras;
1729 }
1730