xref: /linux/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c (revision c5288cda69ee2d8607f5026bd599a5cebf0ee783)
1 /*
2  * Copyright 2023 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_vcn.h"
27 #include "amdgpu_pm.h"
28 #include "amdgpu_cs.h"
29 #include "soc15.h"
30 #include "soc15d.h"
31 #include "soc15_hw_ip.h"
32 #include "vcn_v2_0.h"
33 #include "mmsch_v4_0.h"
34 #include "vcn_v4_0_5.h"
35 
36 #include "vcn/vcn_4_0_5_offset.h"
37 #include "vcn/vcn_4_0_5_sh_mask.h"
38 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
39 
40 #include <drm/drm_drv.h>
41 
42 #define mmUVD_DPG_LMA_CTL							regUVD_DPG_LMA_CTL
43 #define mmUVD_DPG_LMA_CTL_BASE_IDX					regUVD_DPG_LMA_CTL_BASE_IDX
44 #define mmUVD_DPG_LMA_DATA							regUVD_DPG_LMA_DATA
45 #define mmUVD_DPG_LMA_DATA_BASE_IDX					regUVD_DPG_LMA_DATA_BASE_IDX
46 
47 #define VCN_VID_SOC_ADDRESS_2_0						0x1fb00
48 #define VCN1_VID_SOC_ADDRESS_3_0					(0x48300 + 0x38000)
49 
50 #define VCN_HARVEST_MMSCH							0
51 
52 #define RDECODE_MSG_CREATE							0x00000000
53 #define RDECODE_MESSAGE_CREATE						0x00000001
54 
55 static int amdgpu_ih_clientid_vcns[] = {
56 	SOC15_IH_CLIENTID_VCN,
57 	SOC15_IH_CLIENTID_VCN1
58 };
59 
60 static void vcn_v4_0_5_set_unified_ring_funcs(struct amdgpu_device *adev);
61 static void vcn_v4_0_5_set_irq_funcs(struct amdgpu_device *adev);
62 static int vcn_v4_0_5_set_powergating_state(void *handle,
63 		enum amd_powergating_state state);
64 static int vcn_v4_0_5_pause_dpg_mode(struct amdgpu_device *adev,
65 		int inst_idx, struct dpg_pause_state *new_state);
66 static void vcn_v4_0_5_unified_ring_set_wptr(struct amdgpu_ring *ring);
67 
68 /**
69  * vcn_v4_0_5_early_init - set function pointers and load microcode
70  *
71  * @handle: amdgpu_device pointer
72  *
73  * Set ring and irq function pointers
74  * Load microcode from filesystem
75  */
76 static int vcn_v4_0_5_early_init(void *handle)
77 {
78 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
79 
80 	/* re-use enc ring as unified ring */
81 	adev->vcn.num_enc_rings = 1;
82 	vcn_v4_0_5_set_unified_ring_funcs(adev);
83 	vcn_v4_0_5_set_irq_funcs(adev);
84 
85 	return amdgpu_vcn_early_init(adev);
86 }
87 
88 /**
89  * vcn_v4_0_5_sw_init - sw init for VCN block
90  *
91  * @handle: amdgpu_device pointer
92  *
93  * Load firmware and sw initialization
94  */
95 static int vcn_v4_0_5_sw_init(void *handle)
96 {
97 	struct amdgpu_ring *ring;
98 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
99 	int i, r;
100 
101 	r = amdgpu_vcn_sw_init(adev);
102 	if (r)
103 		return r;
104 
105 	amdgpu_vcn_setup_ucode(adev);
106 
107 	r = amdgpu_vcn_resume(adev);
108 	if (r)
109 		return r;
110 
111 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
112 		volatile struct amdgpu_vcn4_fw_shared *fw_shared;
113 
114 		if (adev->vcn.harvest_config & (1 << i))
115 			continue;
116 
117 		atomic_set(&adev->vcn.inst[i].sched_score, 0);
118 
119 		/* VCN UNIFIED TRAP */
120 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
121 				VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
122 		if (r)
123 			return r;
124 
125 		/* VCN POISON TRAP */
126 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
127 				VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst[i].irq);
128 		if (r)
129 			return r;
130 
131 		ring = &adev->vcn.inst[i].ring_enc[0];
132 		ring->use_doorbell = true;
133 		if (amdgpu_sriov_vf(adev))
134 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
135 						i * (adev->vcn.num_enc_rings + 1) + 1;
136 		else
137 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
138 						2 + 8 * i;
139 		ring->vm_hub = AMDGPU_MMHUB0(0);
140 		sprintf(ring->name, "vcn_unified_%d", i);
141 
142 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
143 				AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
144 		if (r)
145 			return r;
146 
147 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
148 		fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
149 		fw_shared->sq.is_enabled = 1;
150 
151 		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG);
152 		fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ?
153 			AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU;
154 
155 		if (amdgpu_sriov_vf(adev))
156 			fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
157 
158 		if (amdgpu_vcnfw_log)
159 			amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
160 	}
161 
162 	if (amdgpu_sriov_vf(adev)) {
163 		r = amdgpu_virt_alloc_mm_table(adev);
164 		if (r)
165 			return r;
166 	}
167 
168 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
169 		adev->vcn.pause_dpg_mode = vcn_v4_0_5_pause_dpg_mode;
170 
171 	return 0;
172 }
173 
174 /**
175  * vcn_v4_0_5_sw_fini - sw fini for VCN block
176  *
177  * @handle: amdgpu_device pointer
178  *
179  * VCN suspend and free up sw allocation
180  */
181 static int vcn_v4_0_5_sw_fini(void *handle)
182 {
183 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
184 	int i, r, idx;
185 
186 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
187 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
188 			volatile struct amdgpu_vcn4_fw_shared *fw_shared;
189 
190 			if (adev->vcn.harvest_config & (1 << i))
191 				continue;
192 
193 			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
194 			fw_shared->present_flag_0 = 0;
195 			fw_shared->sq.is_enabled = 0;
196 		}
197 
198 		drm_dev_exit(idx);
199 	}
200 
201 	if (amdgpu_sriov_vf(adev))
202 		amdgpu_virt_free_mm_table(adev);
203 
204 	r = amdgpu_vcn_suspend(adev);
205 	if (r)
206 		return r;
207 
208 	r = amdgpu_vcn_sw_fini(adev);
209 
210 	return r;
211 }
212 
213 /**
214  * vcn_v4_0_5_hw_init - start and test VCN block
215  *
216  * @handle: amdgpu_device pointer
217  *
218  * Initialize the hardware, boot up the VCPU and do some testing
219  */
220 static int vcn_v4_0_5_hw_init(void *handle)
221 {
222 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
223 	struct amdgpu_ring *ring;
224 	int i, r;
225 
226 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
227 		if (adev->vcn.harvest_config & (1 << i))
228 			continue;
229 
230 		ring = &adev->vcn.inst[i].ring_enc[0];
231 
232 		adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
233 				((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
234 
235 		r = amdgpu_ring_test_helper(ring);
236 		if (r)
237 			goto done;
238 	}
239 
240 	return 0;
241 done:
242 	if (!r)
243 		DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
244 			(adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
245 
246 	return r;
247 }
248 
249 /**
250  * vcn_v4_0_5_hw_fini - stop the hardware block
251  *
252  * @handle: amdgpu_device pointer
253  *
254  * Stop the VCN block, mark ring as not ready any more
255  */
256 static int vcn_v4_0_5_hw_fini(void *handle)
257 {
258 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
259 	int i;
260 
261 	cancel_delayed_work_sync(&adev->vcn.idle_work);
262 
263 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
264 		if (adev->vcn.harvest_config & (1 << i))
265 			continue;
266 		if (!amdgpu_sriov_vf(adev)) {
267 			if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
268 				(adev->vcn.cur_state != AMD_PG_STATE_GATE &&
269 				RREG32_SOC15(VCN, i, regUVD_STATUS))) {
270 				vcn_v4_0_5_set_powergating_state(adev, AMD_PG_STATE_GATE);
271 			}
272 		}
273 	}
274 
275 	return 0;
276 }
277 
278 /**
279  * vcn_v4_0_5_suspend - suspend VCN block
280  *
281  * @handle: amdgpu_device pointer
282  *
283  * HW fini and suspend VCN block
284  */
285 static int vcn_v4_0_5_suspend(void *handle)
286 {
287 	int r;
288 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
289 
290 	r = vcn_v4_0_5_hw_fini(adev);
291 	if (r)
292 		return r;
293 
294 	r = amdgpu_vcn_suspend(adev);
295 
296 	return r;
297 }
298 
299 /**
300  * vcn_v4_0_5_resume - resume VCN block
301  *
302  * @handle: amdgpu_device pointer
303  *
304  * Resume firmware and hw init VCN block
305  */
306 static int vcn_v4_0_5_resume(void *handle)
307 {
308 	int r;
309 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
310 
311 	r = amdgpu_vcn_resume(adev);
312 	if (r)
313 		return r;
314 
315 	r = vcn_v4_0_5_hw_init(adev);
316 
317 	return r;
318 }
319 
320 /**
321  * vcn_v4_0_5_mc_resume - memory controller programming
322  *
323  * @adev: amdgpu_device pointer
324  * @inst: instance number
325  *
326  * Let the VCN memory controller know it's offsets
327  */
328 static void vcn_v4_0_5_mc_resume(struct amdgpu_device *adev, int inst)
329 {
330 	uint32_t offset, size;
331 	const struct common_firmware_header *hdr;
332 
333 	hdr = (const struct common_firmware_header *)adev->vcn.fw[inst]->data;
334 	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
335 
336 	/* cache window 0: fw */
337 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
338 		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
339 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
340 		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
341 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
342 		WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
343 		offset = 0;
344 	} else {
345 		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
346 			lower_32_bits(adev->vcn.inst[inst].gpu_addr));
347 		WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
348 			upper_32_bits(adev->vcn.inst[inst].gpu_addr));
349 		offset = size;
350 		WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
351 	}
352 	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
353 
354 	/* cache window 1: stack */
355 	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
356 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
357 	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
358 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
359 	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
360 	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
361 
362 	/* cache window 2: context */
363 	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
364 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
365 	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
366 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
367 	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
368 	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
369 
370 	/* non-cache window */
371 	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
372 		lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
373 	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
374 		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
375 	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
376 	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
377 		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
378 }
379 
380 /**
381  * vcn_v4_0_5_mc_resume_dpg_mode - memory controller programming for dpg mode
382  *
383  * @adev: amdgpu_device pointer
384  * @inst_idx: instance number index
385  * @indirect: indirectly write sram
386  *
387  * Let the VCN memory controller know it's offsets with dpg mode
388  */
389 static void vcn_v4_0_5_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
390 {
391 	uint32_t offset, size;
392 	const struct common_firmware_header *hdr;
393 
394 	hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data;
395 	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
396 
397 	/* cache window 0: fw */
398 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
399 		if (!indirect) {
400 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
401 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
402 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo),
403 			0, indirect);
404 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
405 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
406 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi),
407 			0, indirect);
408 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
409 				VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
410 		} else {
411 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
412 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
413 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
414 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
415 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
416 				VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
417 		}
418 		offset = 0;
419 	} else {
420 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
421 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
422 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
423 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
424 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
425 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
426 		offset = size;
427 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
428 			VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
429 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
430 	}
431 
432 	if (!indirect)
433 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
434 			VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
435 	else
436 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
437 			VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
438 
439 	/* cache window 1: stack */
440 	if (!indirect) {
441 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
442 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
443 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
444 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
445 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
446 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
447 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
448 			VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
449 	} else {
450 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
451 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
452 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
453 			VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
454 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
455 			VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
456 	}
457 
458 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
459 		VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
460 
461 	/* cache window 2: context */
462 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
463 		VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
464 		lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
465 		0, indirect);
466 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
467 		VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
468 		upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
469 		0, indirect);
470 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
471 		VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
472 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
473 		VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
474 
475 	/* non-cache window */
476 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
477 		VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
478 		lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
479 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
480 		VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
481 		upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
482 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
483 		VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
484 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
485 		VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
486 		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
487 
488 	/* VCN global tiling registers */
489 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
490 		VCN, inst_idx, regUVD_GFX10_ADDR_CONFIG),
491 		adev->gfx.config.gb_addr_config, 0, indirect);
492 }
493 
494 /**
495  * vcn_v4_0_5_disable_static_power_gating - disable VCN static power gating
496  *
497  * @adev: amdgpu_device pointer
498  * @inst: instance number
499  *
500  * Disable static power gating for VCN block
501  */
502 static void vcn_v4_0_5_disable_static_power_gating(struct amdgpu_device *adev, int inst)
503 {
504 	uint32_t data = 0;
505 
506 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
507 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
508 					1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT);
509 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
510 					UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
511 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
512 					2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT);
513 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
514 					1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
515 					UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
516 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
517 					2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT);
518 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
519 					1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
520 					UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
521 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
522 					2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT);
523 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
524 					1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
525 					UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
526 	} else {
527 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
528 			1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT);
529 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
530 			0, UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
531 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
532 			1 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT);
533 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
534 			0, UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
535 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
536 			1 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT);
537 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
538 			0, UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
539 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
540 			1 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT);
541 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
542 			0, UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
543 	}
544 
545 	data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
546 	data &= ~0x103;
547 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
548 		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
549 			UVD_POWER_STATUS__UVD_PG_EN_MASK;
550 	WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
551 }
552 
553 /**
554  * vcn_v4_0_5_enable_static_power_gating - enable VCN static power gating
555  *
556  * @adev: amdgpu_device pointer
557  * @inst: instance number
558  *
559  * Enable static power gating for VCN block
560  */
561 static void vcn_v4_0_5_enable_static_power_gating(struct amdgpu_device *adev, int inst)
562 {
563 	uint32_t data;
564 
565 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
566 		/* Before power off, this indicator has to be turned on */
567 		data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
568 		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
569 		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
570 		WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
571 
572 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
573 			2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT);
574 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
575 			1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
576 			UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
577 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
578 			2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT);
579 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
580 			1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
581 			UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
582 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
583 			2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT);
584 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
585 			1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
586 			UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
587 		WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG,
588 			2 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT);
589 		SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
590 			1 << UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS__SHIFT,
591 			UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
592 	}
593 }
594 
595 /**
596  * vcn_v4_0_5_disable_clock_gating - disable VCN clock gating
597  *
598  * @adev: amdgpu_device pointer
599  * @inst: instance number
600  *
601  * Disable clock gating for VCN block
602  */
603 static void vcn_v4_0_5_disable_clock_gating(struct amdgpu_device *adev, int inst)
604 {
605 	uint32_t data;
606 
607 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
608 		return;
609 
610 	/* VCN disable CGC */
611 	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
612 	data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
613 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
614 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
615 	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
616 
617 	data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
618 	data &= ~(UVD_CGC_GATE__SYS_MASK
619 		| UVD_CGC_GATE__UDEC_MASK
620 		| UVD_CGC_GATE__MPEG2_MASK
621 		| UVD_CGC_GATE__REGS_MASK
622 		| UVD_CGC_GATE__RBC_MASK
623 		| UVD_CGC_GATE__LMI_MC_MASK
624 		| UVD_CGC_GATE__LMI_UMC_MASK
625 		| UVD_CGC_GATE__IDCT_MASK
626 		| UVD_CGC_GATE__MPRD_MASK
627 		| UVD_CGC_GATE__MPC_MASK
628 		| UVD_CGC_GATE__LBSI_MASK
629 		| UVD_CGC_GATE__LRBBM_MASK
630 		| UVD_CGC_GATE__UDEC_RE_MASK
631 		| UVD_CGC_GATE__UDEC_CM_MASK
632 		| UVD_CGC_GATE__UDEC_IT_MASK
633 		| UVD_CGC_GATE__UDEC_DB_MASK
634 		| UVD_CGC_GATE__UDEC_MP_MASK
635 		| UVD_CGC_GATE__WCB_MASK
636 		| UVD_CGC_GATE__VCPU_MASK
637 		| UVD_CGC_GATE__MMSCH_MASK);
638 
639 	WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
640 	SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0,  0xFFFFFFFF);
641 
642 	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
643 	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
644 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
645 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
646 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
647 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
648 		| UVD_CGC_CTRL__SYS_MODE_MASK
649 		| UVD_CGC_CTRL__UDEC_MODE_MASK
650 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
651 		| UVD_CGC_CTRL__REGS_MODE_MASK
652 		| UVD_CGC_CTRL__RBC_MODE_MASK
653 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
654 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
655 		| UVD_CGC_CTRL__IDCT_MODE_MASK
656 		| UVD_CGC_CTRL__MPRD_MODE_MASK
657 		| UVD_CGC_CTRL__MPC_MODE_MASK
658 		| UVD_CGC_CTRL__LBSI_MODE_MASK
659 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
660 		| UVD_CGC_CTRL__WCB_MODE_MASK
661 		| UVD_CGC_CTRL__VCPU_MODE_MASK
662 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
663 	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
664 
665 	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
666 	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
667 		| UVD_SUVD_CGC_GATE__SIT_MASK
668 		| UVD_SUVD_CGC_GATE__SMP_MASK
669 		| UVD_SUVD_CGC_GATE__SCM_MASK
670 		| UVD_SUVD_CGC_GATE__SDB_MASK
671 		| UVD_SUVD_CGC_GATE__SRE_H264_MASK
672 		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
673 		| UVD_SUVD_CGC_GATE__SIT_H264_MASK
674 		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
675 		| UVD_SUVD_CGC_GATE__SCM_H264_MASK
676 		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
677 		| UVD_SUVD_CGC_GATE__SDB_H264_MASK
678 		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
679 		| UVD_SUVD_CGC_GATE__SCLR_MASK
680 		| UVD_SUVD_CGC_GATE__UVD_SC_MASK
681 		| UVD_SUVD_CGC_GATE__ENT_MASK
682 		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
683 		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
684 		| UVD_SUVD_CGC_GATE__SITE_MASK
685 		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
686 		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
687 		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
688 		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
689 		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
690 	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
691 
692 	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
693 	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
694 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
695 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
696 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
697 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
698 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
699 		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
700 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
701 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
702 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
703 	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
704 }
705 
706 /**
707  * vcn_v4_0_5_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
708  *
709  * @adev: amdgpu_device pointer
710  * @sram_sel: sram select
711  * @inst_idx: instance number index
712  * @indirect: indirectly write sram
713  *
714  * Disable clock gating for VCN block with dpg mode
715  */
716 static void vcn_v4_0_5_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,
717 		int inst_idx, uint8_t indirect)
718 {
719 	uint32_t reg_data = 0;
720 
721 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
722 		return;
723 
724 	/* enable sw clock gating control */
725 	reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
726 	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
727 	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
728 	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
729 		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
730 		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
731 		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
732 		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
733 		 UVD_CGC_CTRL__SYS_MODE_MASK |
734 		 UVD_CGC_CTRL__UDEC_MODE_MASK |
735 		 UVD_CGC_CTRL__MPEG2_MODE_MASK |
736 		 UVD_CGC_CTRL__REGS_MODE_MASK |
737 		 UVD_CGC_CTRL__RBC_MODE_MASK |
738 		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
739 		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
740 		 UVD_CGC_CTRL__IDCT_MODE_MASK |
741 		 UVD_CGC_CTRL__MPRD_MODE_MASK |
742 		 UVD_CGC_CTRL__MPC_MODE_MASK |
743 		 UVD_CGC_CTRL__LBSI_MODE_MASK |
744 		 UVD_CGC_CTRL__LRBBM_MODE_MASK |
745 		 UVD_CGC_CTRL__WCB_MODE_MASK |
746 		 UVD_CGC_CTRL__VCPU_MODE_MASK);
747 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
748 		VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
749 
750 	/* turn off clock gating */
751 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
752 		VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
753 
754 	/* turn on SUVD clock gating */
755 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
756 		VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
757 
758 	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
759 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
760 		VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
761 }
762 
763 /**
764  * vcn_v4_0_5_enable_clock_gating - enable VCN clock gating
765  *
766  * @adev: amdgpu_device pointer
767  * @inst: instance number
768  *
769  * Enable clock gating for VCN block
770  */
771 static void vcn_v4_0_5_enable_clock_gating(struct amdgpu_device *adev, int inst)
772 {
773 	uint32_t data;
774 
775 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
776 		return;
777 
778 	/* enable VCN CGC */
779 	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
780 	data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
781 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
782 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
783 	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
784 
785 	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
786 	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
787 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
788 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
789 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
790 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
791 		| UVD_CGC_CTRL__SYS_MODE_MASK
792 		| UVD_CGC_CTRL__UDEC_MODE_MASK
793 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
794 		| UVD_CGC_CTRL__REGS_MODE_MASK
795 		| UVD_CGC_CTRL__RBC_MODE_MASK
796 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
797 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
798 		| UVD_CGC_CTRL__IDCT_MODE_MASK
799 		| UVD_CGC_CTRL__MPRD_MODE_MASK
800 		| UVD_CGC_CTRL__MPC_MODE_MASK
801 		| UVD_CGC_CTRL__LBSI_MODE_MASK
802 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
803 		| UVD_CGC_CTRL__WCB_MODE_MASK
804 		| UVD_CGC_CTRL__VCPU_MODE_MASK
805 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
806 	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
807 
808 	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
809 	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
810 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
811 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
812 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
813 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
814 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
815 		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
816 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
817 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
818 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
819 	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
820 }
821 
822 /**
823  * vcn_v4_0_5_start_dpg_mode - VCN start with dpg mode
824  *
825  * @adev: amdgpu_device pointer
826  * @inst_idx: instance number index
827  * @indirect: indirectly write sram
828  *
829  * Start VCN block with dpg mode
830  */
831 static int vcn_v4_0_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
832 {
833 	volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
834 	struct amdgpu_ring *ring;
835 	uint32_t tmp;
836 
837 	/* disable register anti-hang mechanism */
838 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
839 		~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
840 	/* enable dynamic power gating mode */
841 	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
842 	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
843 	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
844 	WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
845 
846 	if (indirect)
847 		adev->vcn.inst[inst_idx].dpg_sram_curr_addr =
848 					(uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
849 
850 	/* enable clock gating */
851 	vcn_v4_0_5_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
852 
853 	/* enable VCPU clock */
854 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
855 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
856 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
857 		VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
858 
859 	/* disable master interrupt */
860 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
861 		VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
862 
863 	/* setup regUVD_LMI_CTRL */
864 	tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
865 		UVD_LMI_CTRL__REQ_MODE_MASK |
866 		UVD_LMI_CTRL__CRC_RESET_MASK |
867 		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
868 		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
869 		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
870 		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
871 		0x00100000L);
872 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
873 		VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
874 
875 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
876 		VCN, inst_idx, regUVD_MPC_CNTL),
877 		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
878 
879 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
880 		VCN, inst_idx, regUVD_MPC_SET_MUXA0),
881 		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
882 		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
883 		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
884 		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
885 
886 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
887 		VCN, inst_idx, regUVD_MPC_SET_MUXB0),
888 		 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
889 		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
890 		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
891 		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
892 
893 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
894 		VCN, inst_idx, regUVD_MPC_SET_MUX),
895 		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
896 		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
897 		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
898 
899 	vcn_v4_0_5_mc_resume_dpg_mode(adev, inst_idx, indirect);
900 
901 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
902 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
903 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
904 		VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
905 
906 	/* enable LMI MC and UMC channels */
907 	tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
908 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
909 		VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
910 
911 	/* enable master interrupt */
912 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
913 		VCN, inst_idx, regUVD_MASTINT_EN),
914 		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
915 
916 	if (indirect)
917 		amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
918 
919 	ring = &adev->vcn.inst[inst_idx].ring_enc[0];
920 
921 	WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
922 	WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
923 	WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
924 
925 	tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
926 	tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
927 	WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
928 	fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
929 	WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
930 	WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
931 
932 	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
933 	WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
934 	ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
935 
936 	tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
937 	tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
938 	WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
939 	fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
940 
941 	WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
942 			ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
943 			VCN_RB1_DB_CTRL__EN_MASK);
944 
945 	return 0;
946 }
947 
948 
949 /**
950  * vcn_v4_0_5_start - VCN start
951  *
952  * @adev: amdgpu_device pointer
953  *
954  * Start VCN block
955  */
956 static int vcn_v4_0_5_start(struct amdgpu_device *adev)
957 {
958 	volatile struct amdgpu_vcn4_fw_shared *fw_shared;
959 	struct amdgpu_ring *ring;
960 	uint32_t tmp;
961 	int i, j, k, r;
962 
963 	if (adev->pm.dpm_enabled)
964 		amdgpu_dpm_enable_uvd(adev, true);
965 
966 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
967 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
968 
969 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
970 			r = vcn_v4_0_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
971 			continue;
972 		}
973 
974 		/* disable VCN power gating */
975 		vcn_v4_0_5_disable_static_power_gating(adev, i);
976 
977 		/* set VCN status busy */
978 		tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
979 		WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
980 
981 		/*SW clock gating */
982 		vcn_v4_0_5_disable_clock_gating(adev, i);
983 
984 		/* enable VCPU clock */
985 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
986 				UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
987 
988 		/* disable master interrupt */
989 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
990 				~UVD_MASTINT_EN__VCPU_EN_MASK);
991 
992 		/* enable LMI MC and UMC channels */
993 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
994 				~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
995 
996 		tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
997 		tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
998 		tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
999 		WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1000 
1001 		/* setup regUVD_LMI_CTRL */
1002 		tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1003 		WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
1004 				UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1005 				UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1006 				UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1007 				UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1008 
1009 		/* setup regUVD_MPC_CNTL */
1010 		tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1011 		tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1012 		tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1013 		WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1014 
1015 		/* setup UVD_MPC_SET_MUXA0 */
1016 		WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
1017 				((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1018 				 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1019 				 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1020 				 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1021 
1022 		/* setup UVD_MPC_SET_MUXB0 */
1023 		WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
1024 				((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1025 				 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1026 				 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1027 				 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1028 
1029 		/* setup UVD_MPC_SET_MUX */
1030 		WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
1031 				((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1032 				 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1033 				 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1034 
1035 		vcn_v4_0_5_mc_resume(adev, i);
1036 
1037 		/* VCN global tiling registers */
1038 		WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
1039 				adev->gfx.config.gb_addr_config);
1040 
1041 		/* unblock VCPU register access */
1042 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
1043 				~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1044 
1045 		/* release VCPU reset to boot */
1046 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1047 				~UVD_VCPU_CNTL__BLK_RST_MASK);
1048 
1049 		for (j = 0; j < 10; ++j) {
1050 			uint32_t status;
1051 
1052 			for (k = 0; k < 100; ++k) {
1053 				status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1054 				if (status & 2)
1055 					break;
1056 				mdelay(10);
1057 				if (amdgpu_emu_mode == 1)
1058 					msleep(1);
1059 			}
1060 
1061 			if (amdgpu_emu_mode == 1) {
1062 				r = -1;
1063 				if (status & 2) {
1064 					r = 0;
1065 					break;
1066 				}
1067 			} else {
1068 				r = 0;
1069 				if (status & 2)
1070 					break;
1071 
1072 				dev_err(adev->dev,
1073 					"VCN[%d] is not responding, trying to reset VCPU!!!\n", i);
1074 				WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1075 							UVD_VCPU_CNTL__BLK_RST_MASK,
1076 							~UVD_VCPU_CNTL__BLK_RST_MASK);
1077 				mdelay(10);
1078 				WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1079 						~UVD_VCPU_CNTL__BLK_RST_MASK);
1080 
1081 				mdelay(10);
1082 				r = -1;
1083 			}
1084 		}
1085 
1086 		if (r) {
1087 			dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
1088 			return r;
1089 		}
1090 
1091 		/* enable master interrupt */
1092 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
1093 				UVD_MASTINT_EN__VCPU_EN_MASK,
1094 				~UVD_MASTINT_EN__VCPU_EN_MASK);
1095 
1096 		/* clear the busy bit of VCN_STATUS */
1097 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
1098 				~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1099 
1100 		ring = &adev->vcn.inst[i].ring_enc[0];
1101 		WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1102 				ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1103 				VCN_RB1_DB_CTRL__EN_MASK);
1104 
1105 		WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1106 		WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1107 		WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
1108 
1109 		tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1110 		tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1111 		WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1112 		fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1113 		WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
1114 		WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
1115 
1116 		tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1117 		WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1118 		ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
1119 
1120 		tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1121 		tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1122 		WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1123 		fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1124 	}
1125 
1126 	return 0;
1127 }
1128 
1129 /**
1130  * vcn_v4_0_5_stop_dpg_mode - VCN stop with dpg mode
1131  *
1132  * @adev: amdgpu_device pointer
1133  * @inst_idx: instance number index
1134  *
1135  * Stop VCN block with dpg mode
1136  */
1137 static void vcn_v4_0_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1138 {
1139 	uint32_t tmp;
1140 
1141 	/* Wait for power status to be 1 */
1142 	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1143 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1144 
1145 	/* wait for read ptr to be equal to write ptr */
1146 	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1147 	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1148 
1149 	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1150 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1151 
1152 	/* disable dynamic power gating mode */
1153 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1154 		~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1155 }
1156 
1157 /**
1158  * vcn_v4_0_5_stop - VCN stop
1159  *
1160  * @adev: amdgpu_device pointer
1161  *
1162  * Stop VCN block
1163  */
1164 static int vcn_v4_0_5_stop(struct amdgpu_device *adev)
1165 {
1166 	volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1167 	uint32_t tmp;
1168 	int i, r = 0;
1169 
1170 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1171 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1172 		fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1173 
1174 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1175 			vcn_v4_0_5_stop_dpg_mode(adev, i);
1176 			continue;
1177 		}
1178 
1179 		/* wait for vcn idle */
1180 		r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1181 		if (r)
1182 			return r;
1183 
1184 		tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1185 			UVD_LMI_STATUS__READ_CLEAN_MASK |
1186 			UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1187 			UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1188 		r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1189 		if (r)
1190 			return r;
1191 
1192 		/* disable LMI UMC channel */
1193 		tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1194 		tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1195 		WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1196 		tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1197 			UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1198 		r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1199 		if (r)
1200 			return r;
1201 
1202 		/* block VCPU register access */
1203 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1204 				UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1205 				~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1206 
1207 		/* reset VCPU */
1208 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1209 				UVD_VCPU_CNTL__BLK_RST_MASK,
1210 				~UVD_VCPU_CNTL__BLK_RST_MASK);
1211 
1212 		/* disable VCPU clock */
1213 		WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1214 				~(UVD_VCPU_CNTL__CLK_EN_MASK));
1215 
1216 		/* apply soft reset */
1217 		tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1218 		tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1219 		WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1220 		tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1221 		tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1222 		WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1223 
1224 		/* clear status */
1225 		WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1226 
1227 		/* apply HW clock gating */
1228 		vcn_v4_0_5_enable_clock_gating(adev, i);
1229 
1230 		/* enable VCN power gating */
1231 		vcn_v4_0_5_enable_static_power_gating(adev, i);
1232 	}
1233 
1234 	if (adev->pm.dpm_enabled)
1235 		amdgpu_dpm_enable_uvd(adev, false);
1236 
1237 	return 0;
1238 }
1239 
1240 /**
1241  * vcn_v4_0_5_pause_dpg_mode - VCN pause with dpg mode
1242  *
1243  * @adev: amdgpu_device pointer
1244  * @inst_idx: instance number index
1245  * @new_state: pause state
1246  *
1247  * Pause dpg mode for VCN block
1248  */
1249 static int vcn_v4_0_5_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,
1250 		struct dpg_pause_state *new_state)
1251 {
1252 	uint32_t reg_data = 0;
1253 	int ret_code;
1254 
1255 	/* pause/unpause if state is changed */
1256 	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1257 		DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1258 			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
1259 		reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1260 			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1261 
1262 		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1263 			ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1264 				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1265 
1266 			if (!ret_code) {
1267 				/* pause DPG */
1268 				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1269 				WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1270 
1271 				/* wait for ACK */
1272 				SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1273 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1274 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1275 
1276 				SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1277 					UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1278 					UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1279 			}
1280 		} else {
1281 			/* unpause dpg, no need to wait */
1282 			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1283 			WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1284 		}
1285 		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1286 	}
1287 
1288 	return 0;
1289 }
1290 
1291 /**
1292  * vcn_v4_0_5_unified_ring_get_rptr - get unified read pointer
1293  *
1294  * @ring: amdgpu_ring pointer
1295  *
1296  * Returns the current hardware unified read pointer
1297  */
1298 static uint64_t vcn_v4_0_5_unified_ring_get_rptr(struct amdgpu_ring *ring)
1299 {
1300 	struct amdgpu_device *adev = ring->adev;
1301 
1302 	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1303 		DRM_ERROR("wrong ring id is identified in %s", __func__);
1304 
1305 	return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1306 }
1307 
1308 /**
1309  * vcn_v4_0_5_unified_ring_get_wptr - get unified write pointer
1310  *
1311  * @ring: amdgpu_ring pointer
1312  *
1313  * Returns the current hardware unified write pointer
1314  */
1315 static uint64_t vcn_v4_0_5_unified_ring_get_wptr(struct amdgpu_ring *ring)
1316 {
1317 	struct amdgpu_device *adev = ring->adev;
1318 
1319 	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1320 		DRM_ERROR("wrong ring id is identified in %s", __func__);
1321 
1322 	if (ring->use_doorbell)
1323 		return *ring->wptr_cpu_addr;
1324 	else
1325 		return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1326 }
1327 
1328 /**
1329  * vcn_v4_0_5_unified_ring_set_wptr - set enc write pointer
1330  *
1331  * @ring: amdgpu_ring pointer
1332  *
1333  * Commits the enc write pointer to the hardware
1334  */
1335 static void vcn_v4_0_5_unified_ring_set_wptr(struct amdgpu_ring *ring)
1336 {
1337 	struct amdgpu_device *adev = ring->adev;
1338 
1339 	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1340 		DRM_ERROR("wrong ring id is identified in %s", __func__);
1341 
1342 	if (ring->use_doorbell) {
1343 		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1344 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1345 	} else {
1346 		WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1347 	}
1348 }
1349 
1350 static int vcn_v4_0_5_limit_sched(struct amdgpu_cs_parser *p,
1351 				struct amdgpu_job *job)
1352 {
1353 	struct drm_gpu_scheduler **scheds;
1354 
1355 	/* The create msg must be in the first IB submitted */
1356 	if (atomic_read(&job->base.entity->fence_seq))
1357 		return -EINVAL;
1358 
1359 	/* if VCN0 is harvested, we can't support AV1 */
1360 	if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)
1361 		return -EINVAL;
1362 
1363 	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]
1364 		[AMDGPU_RING_PRIO_0].sched;
1365 	drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
1366 	return 0;
1367 }
1368 
1369 static int vcn_v4_0_5_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1370 			    uint64_t addr)
1371 {
1372 	struct ttm_operation_ctx ctx = { false, false };
1373 	struct amdgpu_bo_va_mapping *map;
1374 	uint32_t *msg, num_buffers;
1375 	struct amdgpu_bo *bo;
1376 	uint64_t start, end;
1377 	unsigned int i;
1378 	void *ptr;
1379 	int r;
1380 
1381 	addr &= AMDGPU_GMC_HOLE_MASK;
1382 	r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1383 	if (r) {
1384 		DRM_ERROR("Can't find BO for addr 0x%08llx\n", addr);
1385 		return r;
1386 	}
1387 
1388 	start = map->start * AMDGPU_GPU_PAGE_SIZE;
1389 	end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1390 	if (addr & 0x7) {
1391 		DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1392 		return -EINVAL;
1393 	}
1394 
1395 	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1396 	amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1397 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1398 	if (r) {
1399 		DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1400 		return r;
1401 	}
1402 
1403 	r = amdgpu_bo_kmap(bo, &ptr);
1404 	if (r) {
1405 		DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1406 		return r;
1407 	}
1408 
1409 	msg = ptr + addr - start;
1410 
1411 	/* Check length */
1412 	if (msg[1] > end - addr) {
1413 		r = -EINVAL;
1414 		goto out;
1415 	}
1416 
1417 	if (msg[3] != RDECODE_MSG_CREATE)
1418 		goto out;
1419 
1420 	num_buffers = msg[2];
1421 	for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1422 		uint32_t offset, size, *create;
1423 
1424 		if (msg[0] != RDECODE_MESSAGE_CREATE)
1425 			continue;
1426 
1427 		offset = msg[1];
1428 		size = msg[2];
1429 
1430 		if (offset + size > end) {
1431 			r = -EINVAL;
1432 			goto out;
1433 		}
1434 
1435 		create = ptr + addr + offset - start;
1436 
1437 		/* H264, HEVC and VP9 can run on any instance */
1438 		if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1439 			continue;
1440 
1441 		r = vcn_v4_0_5_limit_sched(p, job);
1442 		if (r)
1443 			goto out;
1444 	}
1445 
1446 out:
1447 	amdgpu_bo_kunmap(bo);
1448 	return r;
1449 }
1450 
1451 #define RADEON_VCN_ENGINE_TYPE_ENCODE			(0x00000002)
1452 #define RADEON_VCN_ENGINE_TYPE_DECODE			(0x00000003)
1453 
1454 #define RADEON_VCN_ENGINE_INFO				(0x30000001)
1455 #define RADEON_VCN_ENGINE_INFO_MAX_OFFSET		16
1456 
1457 #define RENCODE_ENCODE_STANDARD_AV1			2
1458 #define RENCODE_IB_PARAM_SESSION_INIT			0x00000003
1459 #define RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET	64
1460 
1461 /* return the offset in ib if id is found, -1 otherwise
1462  * to speed up the searching we only search upto max_offset
1463  */
1464 static int vcn_v4_0_5_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int max_offset)
1465 {
1466 	int i;
1467 
1468 	for (i = 0; i < ib->length_dw && i < max_offset && ib->ptr[i] >= 8; i += ib->ptr[i]/4) {
1469 		if (ib->ptr[i + 1] == id)
1470 			return i;
1471 	}
1472 	return -1;
1473 }
1474 
1475 static int vcn_v4_0_5_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1476 					   struct amdgpu_job *job,
1477 					   struct amdgpu_ib *ib)
1478 {
1479 	struct amdgpu_ring *ring = amdgpu_job_ring(job);
1480 	struct amdgpu_vcn_decode_buffer *decode_buffer;
1481 	uint64_t addr;
1482 	uint32_t val;
1483 	int idx;
1484 
1485 	/* The first instance can decode anything */
1486 	if (!ring->me)
1487 		return 0;
1488 
1489 	/* RADEON_VCN_ENGINE_INFO is at the top of ib block */
1490 	idx = vcn_v4_0_5_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO,
1491 			RADEON_VCN_ENGINE_INFO_MAX_OFFSET);
1492 	if (idx < 0) /* engine info is missing */
1493 		return 0;
1494 
1495 	val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */
1496 	if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
1497 		decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6];
1498 
1499 		if (!(decode_buffer->valid_buf_flag  & 0x1))
1500 			return 0;
1501 
1502 		addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
1503 			decode_buffer->msg_buffer_address_lo;
1504 		return vcn_v4_0_5_dec_msg(p, job, addr);
1505 	} else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) {
1506 		idx = vcn_v4_0_5_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT,
1507 			RENCODE_IB_PARAM_SESSION_INIT_MAX_OFFSET);
1508 		if (idx >= 0 && ib->ptr[idx + 2] == RENCODE_ENCODE_STANDARD_AV1)
1509 			return vcn_v4_0_5_limit_sched(p, job);
1510 	}
1511 	return 0;
1512 }
1513 
1514 static const struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = {
1515 	.type = AMDGPU_RING_TYPE_VCN_ENC,
1516 	.align_mask = 0x3f,
1517 	.nop = VCN_ENC_CMD_NO_OP,
1518 	.get_rptr = vcn_v4_0_5_unified_ring_get_rptr,
1519 	.get_wptr = vcn_v4_0_5_unified_ring_get_wptr,
1520 	.set_wptr = vcn_v4_0_5_unified_ring_set_wptr,
1521 	.patch_cs_in_place = vcn_v4_0_5_ring_patch_cs_in_place,
1522 	.emit_frame_size =
1523 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1524 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1525 		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1526 		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1527 		1, /* vcn_v2_0_enc_ring_insert_end */
1528 	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1529 	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
1530 	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
1531 	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1532 	.test_ring = amdgpu_vcn_enc_ring_test_ring,
1533 	.test_ib = amdgpu_vcn_unified_ring_test_ib,
1534 	.insert_nop = amdgpu_ring_insert_nop,
1535 	.insert_end = vcn_v2_0_enc_ring_insert_end,
1536 	.pad_ib = amdgpu_ring_generic_pad_ib,
1537 	.begin_use = amdgpu_vcn_ring_begin_use,
1538 	.end_use = amdgpu_vcn_ring_end_use,
1539 	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1540 	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1541 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1542 };
1543 
1544 /**
1545  * vcn_v4_0_5_set_unified_ring_funcs - set unified ring functions
1546  *
1547  * @adev: amdgpu_device pointer
1548  *
1549  * Set unified ring functions
1550  */
1551 static void vcn_v4_0_5_set_unified_ring_funcs(struct amdgpu_device *adev)
1552 {
1553 	int i;
1554 
1555 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1556 		if (adev->vcn.harvest_config & (1 << i))
1557 			continue;
1558 
1559 		adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_5_unified_ring_vm_funcs;
1560 		adev->vcn.inst[i].ring_enc[0].me = i;
1561 
1562 		DRM_INFO("VCN(%d) encode/decode are enabled in VM mode\n", i);
1563 	}
1564 }
1565 
1566 /**
1567  * vcn_v4_0_5_is_idle - check VCN block is idle
1568  *
1569  * @handle: amdgpu_device pointer
1570  *
1571  * Check whether VCN block is idle
1572  */
1573 static bool vcn_v4_0_5_is_idle(void *handle)
1574 {
1575 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1576 	int i, ret = 1;
1577 
1578 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1579 		if (adev->vcn.harvest_config & (1 << i))
1580 			continue;
1581 
1582 		ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1583 	}
1584 
1585 	return ret;
1586 }
1587 
1588 /**
1589  * vcn_v4_0_5_wait_for_idle - wait for VCN block idle
1590  *
1591  * @handle: amdgpu_device pointer
1592  *
1593  * Wait for VCN block idle
1594  */
1595 static int vcn_v4_0_5_wait_for_idle(void *handle)
1596 {
1597 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1598 	int i, ret = 0;
1599 
1600 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1601 		if (adev->vcn.harvest_config & (1 << i))
1602 			continue;
1603 
1604 		ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1605 			UVD_STATUS__IDLE);
1606 		if (ret)
1607 			return ret;
1608 	}
1609 
1610 	return ret;
1611 }
1612 
1613 /**
1614  * vcn_v4_0_5_set_clockgating_state - set VCN block clockgating state
1615  *
1616  * @handle: amdgpu_device pointer
1617  * @state: clock gating state
1618  *
1619  * Set VCN block clockgating state
1620  */
1621 static int vcn_v4_0_5_set_clockgating_state(void *handle, enum amd_clockgating_state state)
1622 {
1623 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1624 	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1625 	int i;
1626 
1627 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1628 		if (adev->vcn.harvest_config & (1 << i))
1629 			continue;
1630 
1631 		if (enable) {
1632 			if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1633 				return -EBUSY;
1634 			vcn_v4_0_5_enable_clock_gating(adev, i);
1635 		} else {
1636 			vcn_v4_0_5_disable_clock_gating(adev, i);
1637 		}
1638 	}
1639 
1640 	return 0;
1641 }
1642 
1643 /**
1644  * vcn_v4_0_5_set_powergating_state - set VCN block powergating state
1645  *
1646  * @handle: amdgpu_device pointer
1647  * @state: power gating state
1648  *
1649  * Set VCN block powergating state
1650  */
1651 static int vcn_v4_0_5_set_powergating_state(void *handle, enum amd_powergating_state state)
1652 {
1653 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1654 	int ret;
1655 
1656 	if (state == adev->vcn.cur_state)
1657 		return 0;
1658 
1659 	if (state == AMD_PG_STATE_GATE)
1660 		ret = vcn_v4_0_5_stop(adev);
1661 	else
1662 		ret = vcn_v4_0_5_start(adev);
1663 
1664 	if (!ret)
1665 		adev->vcn.cur_state = state;
1666 
1667 	return ret;
1668 }
1669 
1670 /**
1671  * vcn_v4_0_5_process_interrupt - process VCN block interrupt
1672  *
1673  * @adev: amdgpu_device pointer
1674  * @source: interrupt sources
1675  * @entry: interrupt entry from clients and sources
1676  *
1677  * Process VCN block interrupt
1678  */
1679 static int vcn_v4_0_5_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1680 		struct amdgpu_iv_entry *entry)
1681 {
1682 	uint32_t ip_instance;
1683 
1684 	switch (entry->client_id) {
1685 	case SOC15_IH_CLIENTID_VCN:
1686 		ip_instance = 0;
1687 		break;
1688 	case SOC15_IH_CLIENTID_VCN1:
1689 		ip_instance = 1;
1690 		break;
1691 	default:
1692 		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1693 		return 0;
1694 	}
1695 
1696 	DRM_DEBUG("IH: VCN TRAP\n");
1697 
1698 	switch (entry->src_id) {
1699 	case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1700 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1701 		break;
1702 	case VCN_4_0__SRCID_UVD_POISON:
1703 		amdgpu_vcn_process_poison_irq(adev, source, entry);
1704 		break;
1705 	default:
1706 		DRM_ERROR("Unhandled interrupt: %d %d\n",
1707 			  entry->src_id, entry->src_data[0]);
1708 		break;
1709 	}
1710 
1711 	return 0;
1712 }
1713 
1714 static const struct amdgpu_irq_src_funcs vcn_v4_0_5_irq_funcs = {
1715 	.process = vcn_v4_0_5_process_interrupt,
1716 };
1717 
1718 /**
1719  * vcn_v4_0_5_set_irq_funcs - set VCN block interrupt irq functions
1720  *
1721  * @adev: amdgpu_device pointer
1722  *
1723  * Set VCN block interrupt irq functions
1724  */
1725 static void vcn_v4_0_5_set_irq_funcs(struct amdgpu_device *adev)
1726 {
1727 	int i;
1728 
1729 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1730 		if (adev->vcn.harvest_config & (1 << i))
1731 			continue;
1732 
1733 		adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1734 		adev->vcn.inst[i].irq.funcs = &vcn_v4_0_5_irq_funcs;
1735 	}
1736 }
1737 
1738 static const struct amd_ip_funcs vcn_v4_0_5_ip_funcs = {
1739 	.name = "vcn_v4_0_5",
1740 	.early_init = vcn_v4_0_5_early_init,
1741 	.late_init = NULL,
1742 	.sw_init = vcn_v4_0_5_sw_init,
1743 	.sw_fini = vcn_v4_0_5_sw_fini,
1744 	.hw_init = vcn_v4_0_5_hw_init,
1745 	.hw_fini = vcn_v4_0_5_hw_fini,
1746 	.suspend = vcn_v4_0_5_suspend,
1747 	.resume = vcn_v4_0_5_resume,
1748 	.is_idle = vcn_v4_0_5_is_idle,
1749 	.wait_for_idle = vcn_v4_0_5_wait_for_idle,
1750 	.check_soft_reset = NULL,
1751 	.pre_soft_reset = NULL,
1752 	.soft_reset = NULL,
1753 	.post_soft_reset = NULL,
1754 	.set_clockgating_state = vcn_v4_0_5_set_clockgating_state,
1755 	.set_powergating_state = vcn_v4_0_5_set_powergating_state,
1756 	.dump_ip_state = NULL,
1757 	.print_ip_state = NULL,
1758 };
1759 
1760 const struct amdgpu_ip_block_version vcn_v4_0_5_ip_block = {
1761 	.type = AMD_IP_BLOCK_TYPE_VCN,
1762 	.major = 4,
1763 	.minor = 0,
1764 	.rev = 5,
1765 	.funcs = &vcn_v4_0_5_ip_funcs,
1766 };
1767