xref: /linux/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_ip.h"
27 #include "amdgpu_vcn.h"
28 #include "amdgpu_pm.h"
29 #include "amdgpu_cs.h"
30 #include "soc15.h"
31 #include "soc15d.h"
32 #include "vcn_v2_0.h"
33 #include "mmsch_v3_0.h"
34 #include "vcn_sw_ring.h"
35 
36 #include "vcn/vcn_3_0_0_offset.h"
37 #include "vcn/vcn_3_0_0_sh_mask.h"
38 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
39 
40 #include <drm/drm_drv.h>
41 
42 #define VCN_VID_SOC_ADDRESS_2_0					0x1fa00
43 #define VCN1_VID_SOC_ADDRESS_3_0				0x48200
44 #define VCN1_AON_SOC_ADDRESS_3_0				0x48000
45 
46 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET			0x27
47 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET			0x0f
48 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET			0x10
49 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET			0x11
50 #define mmUVD_NO_OP_INTERNAL_OFFSET				0x29
51 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET			0x66
52 #define mmUVD_SCRATCH9_INTERNAL_OFFSET				0xc01d
53 
54 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET			0x431
55 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET		0x3b4
56 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET		0x3b5
57 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET			0x25c
58 
59 #define VCN_INSTANCES_SIENNA_CICHLID				2
60 #define DEC_SW_RING_ENABLED					FALSE
61 
62 #define RDECODE_MSG_CREATE					0x00000000
63 #define RDECODE_MESSAGE_CREATE					0x00000001
64 
65 static const struct amdgpu_hwip_reg_entry vcn_reg_list_3_0[] = {
66 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_POWER_STATUS),
67 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_STATUS),
68 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID),
69 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID2),
70 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA0),
71 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA1),
72 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_CMD),
73 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI),
74 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO),
75 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI2),
76 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO2),
77 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI3),
78 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO3),
79 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI4),
80 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO4),
81 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR),
82 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR),
83 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR2),
84 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR2),
85 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR3),
86 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR3),
87 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR4),
88 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR4),
89 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE),
90 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE2),
91 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE3),
92 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE4),
93 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_CONFIG),
94 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_STATUS),
95 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_CTL),
96 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_DATA),
97 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_MASK),
98 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_PAUSE)
99 };
100 
101 static int amdgpu_ih_clientid_vcns[] = {
102 	SOC15_IH_CLIENTID_VCN,
103 	SOC15_IH_CLIENTID_VCN1
104 };
105 
106 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
107 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
108 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
109 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
110 static int vcn_v3_0_set_pg_state(struct amdgpu_vcn_inst *vinst,
111 				 enum amd_powergating_state state);
112 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
113 				   struct dpg_pause_state *new_state);
114 static int vcn_v3_0_reset(struct amdgpu_vcn_inst *vinst);
115 
116 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
117 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
118 
119 /**
120  * vcn_v3_0_early_init - set function pointers and load microcode
121  *
122  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
123  *
124  * Set ring and irq function pointers
125  * Load microcode from filesystem
126  */
127 static int vcn_v3_0_early_init(struct amdgpu_ip_block *ip_block)
128 {
129 	struct amdgpu_device *adev = ip_block->adev;
130 	int i, r;
131 
132 	if (amdgpu_sriov_vf(adev)) {
133 		adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
134 		adev->vcn.harvest_config = 0;
135 		for (i = 0; i < adev->vcn.num_vcn_inst; i++)
136 			adev->vcn.inst[i].num_enc_rings = 1;
137 
138 	} else {
139 		if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
140 						 AMDGPU_VCN_HARVEST_VCN1))
141 			/* both instances are harvested, disable the block */
142 			return -ENOENT;
143 
144 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
145 			if (amdgpu_ip_version(adev, UVD_HWIP, 0) ==
146 			    IP_VERSION(3, 0, 33))
147 				adev->vcn.inst[i].num_enc_rings = 0;
148 			else
149 				adev->vcn.inst[i].num_enc_rings = 2;
150 		}
151 	}
152 
153 	vcn_v3_0_set_dec_ring_funcs(adev);
154 	vcn_v3_0_set_enc_ring_funcs(adev);
155 	vcn_v3_0_set_irq_funcs(adev);
156 
157 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
158 		adev->vcn.inst[i].set_pg_state = vcn_v3_0_set_pg_state;
159 
160 		r = amdgpu_vcn_early_init(adev, i);
161 		if (r)
162 			return r;
163 	}
164 	return 0;
165 }
166 
167 /**
168  * vcn_v3_0_sw_init - sw init for VCN block
169  *
170  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
171  *
172  * Load firmware and sw initialization
173  */
174 static int vcn_v3_0_sw_init(struct amdgpu_ip_block *ip_block)
175 {
176 	struct amdgpu_ring *ring;
177 	int i, j, r;
178 	int vcn_doorbell_index = 0;
179 	struct amdgpu_device *adev = ip_block->adev;
180 
181 	/*
182 	 * Note: doorbell assignment is fixed for SRIOV multiple VCN engines
183 	 * Formula:
184 	 *   vcn_db_base  = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
185 	 *   dec_ring_i   = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)
186 	 *   enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j
187 	 */
188 	if (amdgpu_sriov_vf(adev)) {
189 		vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
190 		/* get DWORD offset */
191 		vcn_doorbell_index = vcn_doorbell_index << 1;
192 	}
193 
194 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
195 		struct amdgpu_fw_shared *fw_shared;
196 
197 		if (adev->vcn.harvest_config & (1 << i))
198 			continue;
199 
200 		r = amdgpu_vcn_sw_init(adev, i);
201 		if (r)
202 			return r;
203 
204 		amdgpu_vcn_setup_ucode(adev, i);
205 
206 		r = amdgpu_vcn_resume(adev, i);
207 		if (r)
208 			return r;
209 
210 		adev->vcn.inst[i].internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
211 		adev->vcn.inst[i].internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
212 		adev->vcn.inst[i].internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
213 		adev->vcn.inst[i].internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
214 		adev->vcn.inst[i].internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
215 		adev->vcn.inst[i].internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
216 
217 		adev->vcn.inst[i].internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
218 		adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
219 		adev->vcn.inst[i].internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
220 		adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
221 		adev->vcn.inst[i].internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
222 		adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
223 		adev->vcn.inst[i].internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
224 		adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
225 		adev->vcn.inst[i].internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
226 		adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
227 
228 		/* VCN DEC TRAP */
229 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
230 				VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);
231 		if (r)
232 			return r;
233 
234 		atomic_set(&adev->vcn.inst[i].sched_score, 0);
235 
236 		ring = &adev->vcn.inst[i].ring_dec;
237 		ring->use_doorbell = true;
238 		if (amdgpu_sriov_vf(adev)) {
239 			ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.inst[i].num_enc_rings + 1);
240 		} else {
241 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
242 		}
243 		ring->vm_hub = AMDGPU_MMHUB0(0);
244 		sprintf(ring->name, "vcn_dec_%d", i);
245 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
246 				     AMDGPU_RING_PRIO_DEFAULT,
247 				     &adev->vcn.inst[i].sched_score);
248 		if (r)
249 			return r;
250 
251 		for (j = 0; j < adev->vcn.inst[i].num_enc_rings; ++j) {
252 			enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(j);
253 
254 			/* VCN ENC TRAP */
255 			r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
256 				j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
257 			if (r)
258 				return r;
259 
260 			ring = &adev->vcn.inst[i].ring_enc[j];
261 			ring->use_doorbell = true;
262 			if (amdgpu_sriov_vf(adev)) {
263 				ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.inst[i].num_enc_rings + 1) + 1 + j;
264 			} else {
265 				ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
266 			}
267 			ring->vm_hub = AMDGPU_MMHUB0(0);
268 			sprintf(ring->name, "vcn_enc_%d.%d", i, j);
269 			r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
270 					     hw_prio, &adev->vcn.inst[i].sched_score);
271 			if (r)
272 				return r;
273 		}
274 
275 		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
276 		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
277 					     cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG) |
278 					     cpu_to_le32(AMDGPU_VCN_FW_SHARED_FLAG_0_RB);
279 		fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
280 		fw_shared->present_flag_0 |= AMDGPU_VCN_SMU_VERSION_INFO_FLAG;
281 		if (amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(3, 1, 2))
282 			fw_shared->smu_interface_info.smu_interface_type = 2;
283 		else if (amdgpu_ip_version(adev, UVD_HWIP, 0) ==
284 			 IP_VERSION(3, 1, 1))
285 			fw_shared->smu_interface_info.smu_interface_type = 1;
286 
287 		if (amdgpu_vcnfw_log)
288 			amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
289 
290 		if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
291 			adev->vcn.inst[i].pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
292 		adev->vcn.inst[i].reset = vcn_v3_0_reset;
293 	}
294 
295 	adev->vcn.supported_reset =
296 		amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]);
297 	if (!amdgpu_sriov_vf(adev))
298 		adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
299 
300 	if (amdgpu_sriov_vf(adev)) {
301 		r = amdgpu_virt_alloc_mm_table(adev);
302 		if (r)
303 			return r;
304 	}
305 
306 	r = amdgpu_vcn_reg_dump_init(adev, vcn_reg_list_3_0, ARRAY_SIZE(vcn_reg_list_3_0));
307 	if (r)
308 		return r;
309 
310 	r = amdgpu_vcn_sysfs_reset_mask_init(adev);
311 	if (r)
312 		return r;
313 
314 	return 0;
315 }
316 
317 /**
318  * vcn_v3_0_sw_fini - sw fini for VCN block
319  *
320  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
321  *
322  * VCN suspend and free up sw allocation
323  */
324 static int vcn_v3_0_sw_fini(struct amdgpu_ip_block *ip_block)
325 {
326 	struct amdgpu_device *adev = ip_block->adev;
327 	int i, r, idx;
328 
329 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
330 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
331 			struct amdgpu_fw_shared *fw_shared;
332 
333 			if (adev->vcn.harvest_config & (1 << i))
334 				continue;
335 			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
336 			fw_shared->present_flag_0 = 0;
337 			fw_shared->sw_ring.is_enabled = false;
338 		}
339 
340 		drm_dev_exit(idx);
341 	}
342 
343 	if (amdgpu_sriov_vf(adev))
344 		amdgpu_virt_free_mm_table(adev);
345 
346 	amdgpu_vcn_sysfs_reset_mask_fini(adev);
347 
348 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
349 		r = amdgpu_vcn_suspend(adev, i);
350 		if (r)
351 			return r;
352 
353 		amdgpu_vcn_sw_fini(adev, i);
354 	}
355 
356 	return 0;
357 }
358 
359 /**
360  * vcn_v3_0_hw_init - start and test VCN block
361  *
362  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
363  *
364  * Initialize the hardware, boot up the VCPU and do some testing
365  */
366 static int vcn_v3_0_hw_init(struct amdgpu_ip_block *ip_block)
367 {
368 	struct amdgpu_device *adev = ip_block->adev;
369 	struct amdgpu_ring *ring;
370 	int i, j, r;
371 
372 	if (amdgpu_sriov_vf(adev)) {
373 		r = vcn_v3_0_start_sriov(adev);
374 		if (r)
375 			return r;
376 
377 		/* initialize VCN dec and enc ring buffers */
378 		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
379 			if (adev->vcn.harvest_config & (1 << i))
380 				continue;
381 
382 			ring = &adev->vcn.inst[i].ring_dec;
383 			if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) {
384 				ring->sched.ready = false;
385 				ring->no_scheduler = true;
386 				dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
387 			} else {
388 				ring->wptr = 0;
389 				ring->wptr_old = 0;
390 				vcn_v3_0_dec_ring_set_wptr(ring);
391 				ring->sched.ready = true;
392 			}
393 
394 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; ++j) {
395 				ring = &adev->vcn.inst[i].ring_enc[j];
396 				if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
397 					ring->sched.ready = false;
398 					ring->no_scheduler = true;
399 					dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
400 				} else {
401 					ring->wptr = 0;
402 					ring->wptr_old = 0;
403 					vcn_v3_0_enc_ring_set_wptr(ring);
404 					ring->sched.ready = true;
405 				}
406 			}
407 		}
408 	} else {
409 		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
410 			if (adev->vcn.harvest_config & (1 << i))
411 				continue;
412 
413 			ring = &adev->vcn.inst[i].ring_dec;
414 
415 			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
416 						     ring->doorbell_index, i);
417 
418 			r = amdgpu_ring_test_helper(ring);
419 			if (r)
420 				return r;
421 
422 			for (j = 0; j < adev->vcn.inst[i].num_enc_rings; ++j) {
423 				ring = &adev->vcn.inst[i].ring_enc[j];
424 				r = amdgpu_ring_test_helper(ring);
425 				if (r)
426 					return r;
427 			}
428 		}
429 	}
430 
431 	return 0;
432 }
433 
434 /**
435  * vcn_v3_0_hw_fini - stop the hardware block
436  *
437  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
438  *
439  * Stop the VCN block, mark ring as not ready any more
440  */
441 static int vcn_v3_0_hw_fini(struct amdgpu_ip_block *ip_block)
442 {
443 	struct amdgpu_device *adev = ip_block->adev;
444 	int i;
445 
446 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
447 		struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
448 
449 		if (adev->vcn.harvest_config & (1 << i))
450 			continue;
451 
452 		cancel_delayed_work_sync(&vinst->idle_work);
453 
454 		if (!amdgpu_sriov_vf(adev)) {
455 			if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
456 			    (vinst->cur_state != AMD_PG_STATE_GATE &&
457 			     RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
458 				vinst->set_pg_state(vinst, AMD_PG_STATE_GATE);
459 			}
460 		}
461 	}
462 
463 	return 0;
464 }
465 
466 /**
467  * vcn_v3_0_suspend - suspend VCN block
468  *
469  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
470  *
471  * HW fini and suspend VCN block
472  */
473 static int vcn_v3_0_suspend(struct amdgpu_ip_block *ip_block)
474 {
475 	struct amdgpu_device *adev = ip_block->adev;
476 	int r, i;
477 
478 	r = vcn_v3_0_hw_fini(ip_block);
479 	if (r)
480 		return r;
481 
482 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
483 		r = amdgpu_vcn_suspend(ip_block->adev, i);
484 		if (r)
485 			return r;
486 	}
487 
488 	return 0;
489 }
490 
491 /**
492  * vcn_v3_0_resume - resume VCN block
493  *
494  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
495  *
496  * Resume firmware and hw init VCN block
497  */
498 static int vcn_v3_0_resume(struct amdgpu_ip_block *ip_block)
499 {
500 	struct amdgpu_device *adev = ip_block->adev;
501 	int r, i;
502 
503 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
504 		r = amdgpu_vcn_resume(ip_block->adev, i);
505 		if (r)
506 			return r;
507 	}
508 
509 	r = vcn_v3_0_hw_init(ip_block);
510 
511 	return r;
512 }
513 
514 /**
515  * vcn_v3_0_mc_resume - memory controller programming
516  *
517  * @vinst: VCN instance
518  *
519  * Let the VCN memory controller know it's offsets
520  */
521 static void vcn_v3_0_mc_resume(struct amdgpu_vcn_inst *vinst)
522 {
523 	struct amdgpu_device *adev = vinst->adev;
524 	int inst = vinst->inst;
525 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[inst].fw->size + 4);
526 	uint32_t offset;
527 
528 	/* cache window 0: fw */
529 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
530 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
531 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
532 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
533 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
534 		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
535 		offset = 0;
536 	} else {
537 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
538 			lower_32_bits(adev->vcn.inst[inst].gpu_addr));
539 		WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
540 			upper_32_bits(adev->vcn.inst[inst].gpu_addr));
541 		offset = size;
542 		WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
543 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
544 	}
545 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
546 
547 	/* cache window 1: stack */
548 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
549 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
550 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
551 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
552 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
553 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
554 
555 	/* cache window 2: context */
556 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
557 		lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
558 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
559 		upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
560 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
561 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
562 
563 	/* non-cache window */
564 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
565 		lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
566 	WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
567 		upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
568 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
569 	WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
570 		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
571 }
572 
573 static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst,
574 					bool indirect)
575 {
576 	struct amdgpu_device *adev = vinst->adev;
577 	int inst_idx = vinst->inst;
578 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[inst_idx].fw->size + 4);
579 	uint32_t offset;
580 
581 	/* cache window 0: fw */
582 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
583 		if (!indirect) {
584 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
585 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
586 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
587 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
588 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
589 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
590 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
591 				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
592 		} else {
593 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
594 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
595 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
596 				VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
597 			WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
598 				VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
599 		}
600 		offset = 0;
601 	} else {
602 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
603 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
604 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
605 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
606 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
607 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
608 		offset = size;
609 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
610 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
611 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
612 	}
613 
614 	if (!indirect)
615 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
616 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
617 	else
618 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
619 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
620 
621 	/* cache window 1: stack */
622 	if (!indirect) {
623 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
624 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
625 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
626 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
627 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
628 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
629 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
630 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
631 	} else {
632 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
633 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
634 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
635 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
636 		WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
637 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
638 	}
639 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
640 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
641 
642 	/* cache window 2: context */
643 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
644 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
645 			lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
646 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
647 			VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
648 			upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
649 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
650 			VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
651 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
652 			VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
653 
654 	/* non-cache window */
655 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
656 			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
657 			lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
658 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
659 			VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
660 			upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
661 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
662 			VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
663 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
664 			VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
665 			AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
666 
667 	/* VCN global tiling registers */
668 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
669 		UVD, inst_idx, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
670 }
671 
672 static void vcn_v3_0_disable_static_power_gating(struct amdgpu_vcn_inst *vinst)
673 {
674 	struct amdgpu_device *adev = vinst->adev;
675 	int inst = vinst->inst;
676 	uint32_t data = 0;
677 
678 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
679 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
680 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
681 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
682 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
683 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
684 			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
685 			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
686 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
687 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
688 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
689 			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
690 			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
691 			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
692 			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
693 
694 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
695 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
696 			UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
697 	} else {
698 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
699 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
700 			| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
701 			| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
702 			| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
703 			| 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
704 			| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
705 			| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
706 			| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
707 			| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
708 			| 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
709 			| 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
710 			| 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
711 			| 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
712 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
713 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0,  0x3F3FFFFF);
714 	}
715 
716 	data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
717 	data &= ~0x103;
718 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
719 		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
720 			UVD_POWER_STATUS__UVD_PG_EN_MASK;
721 
722 	WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
723 }
724 
725 static void vcn_v3_0_enable_static_power_gating(struct amdgpu_vcn_inst *vinst)
726 {
727 	struct amdgpu_device *adev = vinst->adev;
728 	int inst = vinst->inst;
729 	uint32_t data;
730 
731 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
732 		/* Before power off, this indicator has to be turned on */
733 		data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
734 		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
735 		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
736 		WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
737 
738 		data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
739 			| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
740 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
741 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
742 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
743 			| 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
744 			| 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
745 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
746 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
747 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
748 			| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
749 			| 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
750 			| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
751 			| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
752 		WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
753 
754 		data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
755 			| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
756 			| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
757 			| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
758 			| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
759 			| 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
760 			| 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
761 			| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
762 			| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
763 			| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
764 			| 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
765 			| 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
766 			| 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
767 			| 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
768 		SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
769 	}
770 }
771 
772 /**
773  * vcn_v3_0_disable_clock_gating - disable VCN clock gating
774  *
775  * @vinst: Pointer to the VCN instance structure
776  *
777  * Disable clock gating for VCN block
778  */
779 static void vcn_v3_0_disable_clock_gating(struct amdgpu_vcn_inst *vinst)
780 {
781 	struct amdgpu_device *adev = vinst->adev;
782 	int inst = vinst->inst;
783 	uint32_t data;
784 
785 	/* VCN disable CGC */
786 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
787 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
788 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
789 	else
790 		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
791 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
792 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
793 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
794 
795 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
796 	data &= ~(UVD_CGC_GATE__SYS_MASK
797 		| UVD_CGC_GATE__UDEC_MASK
798 		| UVD_CGC_GATE__MPEG2_MASK
799 		| UVD_CGC_GATE__REGS_MASK
800 		| UVD_CGC_GATE__RBC_MASK
801 		| UVD_CGC_GATE__LMI_MC_MASK
802 		| UVD_CGC_GATE__LMI_UMC_MASK
803 		| UVD_CGC_GATE__IDCT_MASK
804 		| UVD_CGC_GATE__MPRD_MASK
805 		| UVD_CGC_GATE__MPC_MASK
806 		| UVD_CGC_GATE__LBSI_MASK
807 		| UVD_CGC_GATE__LRBBM_MASK
808 		| UVD_CGC_GATE__UDEC_RE_MASK
809 		| UVD_CGC_GATE__UDEC_CM_MASK
810 		| UVD_CGC_GATE__UDEC_IT_MASK
811 		| UVD_CGC_GATE__UDEC_DB_MASK
812 		| UVD_CGC_GATE__UDEC_MP_MASK
813 		| UVD_CGC_GATE__WCB_MASK
814 		| UVD_CGC_GATE__VCPU_MASK
815 		| UVD_CGC_GATE__MMSCH_MASK);
816 
817 	WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
818 
819 	SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0,  0xFFFFFFFF);
820 
821 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
822 	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
823 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
824 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
825 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
826 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
827 		| UVD_CGC_CTRL__SYS_MODE_MASK
828 		| UVD_CGC_CTRL__UDEC_MODE_MASK
829 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
830 		| UVD_CGC_CTRL__REGS_MODE_MASK
831 		| UVD_CGC_CTRL__RBC_MODE_MASK
832 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
833 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
834 		| UVD_CGC_CTRL__IDCT_MODE_MASK
835 		| UVD_CGC_CTRL__MPRD_MODE_MASK
836 		| UVD_CGC_CTRL__MPC_MODE_MASK
837 		| UVD_CGC_CTRL__LBSI_MODE_MASK
838 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
839 		| UVD_CGC_CTRL__WCB_MODE_MASK
840 		| UVD_CGC_CTRL__VCPU_MODE_MASK
841 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
842 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
843 
844 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
845 	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
846 		| UVD_SUVD_CGC_GATE__SIT_MASK
847 		| UVD_SUVD_CGC_GATE__SMP_MASK
848 		| UVD_SUVD_CGC_GATE__SCM_MASK
849 		| UVD_SUVD_CGC_GATE__SDB_MASK
850 		| UVD_SUVD_CGC_GATE__SRE_H264_MASK
851 		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
852 		| UVD_SUVD_CGC_GATE__SIT_H264_MASK
853 		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
854 		| UVD_SUVD_CGC_GATE__SCM_H264_MASK
855 		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
856 		| UVD_SUVD_CGC_GATE__SDB_H264_MASK
857 		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
858 		| UVD_SUVD_CGC_GATE__SCLR_MASK
859 		| UVD_SUVD_CGC_GATE__ENT_MASK
860 		| UVD_SUVD_CGC_GATE__IME_MASK
861 		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
862 		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
863 		| UVD_SUVD_CGC_GATE__SITE_MASK
864 		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
865 		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
866 		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
867 		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
868 		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK
869 		| UVD_SUVD_CGC_GATE__EFC_MASK
870 		| UVD_SUVD_CGC_GATE__SAOE_MASK
871 		| UVD_SUVD_CGC_GATE__SRE_AV1_MASK
872 		| UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
873 		| UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
874 		| UVD_SUVD_CGC_GATE__SCM_AV1_MASK
875 		| UVD_SUVD_CGC_GATE__SMPA_MASK);
876 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
877 
878 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
879 	data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
880 		| UVD_SUVD_CGC_GATE2__MPBE1_MASK
881 		| UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
882 		| UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
883 		| UVD_SUVD_CGC_GATE2__MPC1_MASK);
884 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
885 
886 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
887 	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
888 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
889 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
890 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
891 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
892 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
893 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
894 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
895 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
896 		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
897 		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
898 		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
899 		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
900 		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
901 		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
902 		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
903 		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
904 		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
905 		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
906 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
907 }
908 
909 static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst,
910 					   uint8_t sram_sel,
911 					   uint8_t indirect)
912 {
913 	struct amdgpu_device *adev = vinst->adev;
914 	int inst_idx = vinst->inst;
915 	uint32_t reg_data = 0;
916 
917 	/* enable sw clock gating control */
918 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
919 		reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
920 	else
921 		reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
922 	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
923 	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
924 	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
925 		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
926 		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
927 		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
928 		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
929 		 UVD_CGC_CTRL__SYS_MODE_MASK |
930 		 UVD_CGC_CTRL__UDEC_MODE_MASK |
931 		 UVD_CGC_CTRL__MPEG2_MODE_MASK |
932 		 UVD_CGC_CTRL__REGS_MODE_MASK |
933 		 UVD_CGC_CTRL__RBC_MODE_MASK |
934 		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
935 		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
936 		 UVD_CGC_CTRL__IDCT_MODE_MASK |
937 		 UVD_CGC_CTRL__MPRD_MODE_MASK |
938 		 UVD_CGC_CTRL__MPC_MODE_MASK |
939 		 UVD_CGC_CTRL__LBSI_MODE_MASK |
940 		 UVD_CGC_CTRL__LRBBM_MODE_MASK |
941 		 UVD_CGC_CTRL__WCB_MODE_MASK |
942 		 UVD_CGC_CTRL__VCPU_MODE_MASK |
943 		 UVD_CGC_CTRL__MMSCH_MODE_MASK);
944 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
945 		VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
946 
947 	/* turn off clock gating */
948 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
949 		VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
950 
951 	/* turn on SUVD clock gating */
952 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
953 		VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
954 
955 	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
956 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
957 		VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
958 }
959 
960 /**
961  * vcn_v3_0_enable_clock_gating - enable VCN clock gating
962  *
963  * @vinst: Pointer to the VCN instance structure
964  *
965  * Enable clock gating for VCN block
966  */
967 static void vcn_v3_0_enable_clock_gating(struct amdgpu_vcn_inst *vinst)
968 {
969 	struct amdgpu_device *adev = vinst->adev;
970 	int inst = vinst->inst;
971 	uint32_t data;
972 
973 	/* enable VCN CGC */
974 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
975 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
976 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
977 	else
978 		data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
979 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
980 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
981 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
982 
983 	data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
984 	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
985 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
986 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
987 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
988 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
989 		| UVD_CGC_CTRL__SYS_MODE_MASK
990 		| UVD_CGC_CTRL__UDEC_MODE_MASK
991 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
992 		| UVD_CGC_CTRL__REGS_MODE_MASK
993 		| UVD_CGC_CTRL__RBC_MODE_MASK
994 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
995 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
996 		| UVD_CGC_CTRL__IDCT_MODE_MASK
997 		| UVD_CGC_CTRL__MPRD_MODE_MASK
998 		| UVD_CGC_CTRL__MPC_MODE_MASK
999 		| UVD_CGC_CTRL__LBSI_MODE_MASK
1000 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
1001 		| UVD_CGC_CTRL__WCB_MODE_MASK
1002 		| UVD_CGC_CTRL__VCPU_MODE_MASK
1003 		| UVD_CGC_CTRL__MMSCH_MODE_MASK);
1004 	WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
1005 
1006 	data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
1007 	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
1008 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
1009 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
1010 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
1011 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
1012 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
1013 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
1014 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
1015 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
1016 		| UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
1017 		| UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
1018 		| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
1019 		| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
1020 		| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
1021 		| UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
1022 		| UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
1023 		| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
1024 		| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
1025 		| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
1026 	WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
1027 }
1028 
1029 static int vcn_v3_0_start_dpg_mode(struct amdgpu_vcn_inst *vinst, bool indirect)
1030 {
1031 	struct amdgpu_device *adev = vinst->adev;
1032 	int inst_idx = vinst->inst;
1033 	struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1034 	struct amdgpu_ring *ring;
1035 	uint32_t rb_bufsz, tmp;
1036 	int ret;
1037 
1038 	/* disable register anti-hang mechanism */
1039 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
1040 		~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1041 	/* enable dynamic power gating mode */
1042 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
1043 	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
1044 	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
1045 	WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
1046 
1047 	if (indirect)
1048 		adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
1049 
1050 	/* enable clock gating */
1051 	vcn_v3_0_clock_gating_dpg_mode(vinst, 0, indirect);
1052 
1053 	/* enable VCPU clock */
1054 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1055 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1056 	tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
1057 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1058 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1059 
1060 	/* disable master interupt */
1061 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1062 		VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
1063 
1064 	/* setup mmUVD_LMI_CTRL */
1065 	tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1066 		UVD_LMI_CTRL__REQ_MODE_MASK |
1067 		UVD_LMI_CTRL__CRC_RESET_MASK |
1068 		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1069 		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1070 		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1071 		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1072 		0x00100000L);
1073 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1074 		VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
1075 
1076 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1077 		VCN, inst_idx, mmUVD_MPC_CNTL),
1078 		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
1079 
1080 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1081 		VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
1082 		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1083 		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1084 		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1085 		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
1086 
1087 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1088 		VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
1089 		 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1090 		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1091 		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1092 		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1093 
1094 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1095 		VCN, inst_idx, mmUVD_MPC_SET_MUX),
1096 		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1097 		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1098 		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1099 
1100 	vcn_v3_0_mc_resume_dpg_mode(vinst, indirect);
1101 
1102 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1103 		VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1104 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1105 		VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1106 
1107 	/* enable LMI MC and UMC channels */
1108 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1109 		VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1110 
1111 	/* unblock VCPU register access */
1112 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1113 		VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1114 
1115 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1116 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1117 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1118 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1119 
1120 	/* enable master interrupt */
1121 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1122 		VCN, inst_idx, mmUVD_MASTINT_EN),
1123 		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1124 
1125 	/* add nop to workaround PSP size check */
1126 	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1127 		VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1128 
1129 	if (indirect) {
1130 		ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
1131 		if (ret) {
1132 			dev_err(adev->dev, "vcn sram load failed %d\n", ret);
1133 			return ret;
1134 		}
1135 	}
1136 
1137 	ring = &adev->vcn.inst[inst_idx].ring_dec;
1138 	/* force RBC into idle state */
1139 	rb_bufsz = order_base_2(ring->ring_size);
1140 	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1141 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1142 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1143 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1144 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1145 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1146 
1147 	/* Stall DPG before WPTR/RPTR reset */
1148 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1149 		UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1150 		~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1151 	fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1152 
1153 	/* set the write pointer delay */
1154 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1155 
1156 	/* set the wb address */
1157 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1158 		(upper_32_bits(ring->gpu_addr) >> 2));
1159 
1160 	/* programm the RB_BASE for ring buffer */
1161 	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1162 		lower_32_bits(ring->gpu_addr));
1163 	WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1164 		upper_32_bits(ring->gpu_addr));
1165 
1166 	/* Initialize the ring buffer's read and write pointers */
1167 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1168 
1169 	WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1170 
1171 	ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1172 	WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1173 		lower_32_bits(ring->wptr));
1174 
1175 	/* Reset FW shared memory RBC WPTR/RPTR */
1176 	fw_shared->rb.rptr = 0;
1177 	fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1178 
1179 	/*resetting done, fw can check RB ring */
1180 	fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1181 
1182 	/* Unstall DPG */
1183 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1184 		0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1185 
1186 	/* Keeping one read-back to ensure all register writes are done,
1187 	 * otherwise it may introduce race conditions.
1188 	 */
1189 	RREG32_SOC15(VCN, inst_idx, mmUVD_STATUS);
1190 
1191 	return 0;
1192 }
1193 
1194 static int vcn_v3_0_start(struct amdgpu_vcn_inst *vinst)
1195 {
1196 	struct amdgpu_device *adev = vinst->adev;
1197 	int i = vinst->inst;
1198 	struct amdgpu_fw_shared *fw_shared;
1199 	struct amdgpu_ring *ring;
1200 	uint32_t rb_bufsz, tmp;
1201 	int j, k, r;
1202 
1203 	if (adev->vcn.harvest_config & (1 << i))
1204 		return 0;
1205 
1206 	if (adev->pm.dpm_enabled)
1207 		amdgpu_dpm_enable_vcn(adev, true, i);
1208 
1209 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1210 		return vcn_v3_0_start_dpg_mode(vinst, vinst->indirect_sram);
1211 
1212 	/* disable VCN power gating */
1213 	vcn_v3_0_disable_static_power_gating(vinst);
1214 
1215 	/* set VCN status busy */
1216 	tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1217 	WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1218 
1219 	/* SW clock gating */
1220 	vcn_v3_0_disable_clock_gating(vinst);
1221 
1222 	/* enable VCPU clock */
1223 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1224 		 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1225 
1226 	/* disable master interrupt */
1227 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1228 		 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1229 
1230 	/* enable LMI MC and UMC channels */
1231 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1232 		 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1233 
1234 	tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1235 	tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1236 	tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1237 	WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1238 
1239 	/* setup mmUVD_LMI_CTRL */
1240 	tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1241 	WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1242 		     UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK	|
1243 		     UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1244 		     UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1245 		     UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1246 
1247 	/* setup mmUVD_MPC_CNTL */
1248 	tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1249 	tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1250 	tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1251 	WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1252 
1253 	/* setup UVD_MPC_SET_MUXA0 */
1254 	WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1255 		     ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1256 		      (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1257 		      (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1258 		      (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1259 
1260 	/* setup UVD_MPC_SET_MUXB0 */
1261 	WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1262 		     ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1263 		      (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1264 		      (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1265 		      (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1266 
1267 	/* setup mmUVD_MPC_SET_MUX */
1268 	WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1269 		     ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1270 		      (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1271 		      (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1272 
1273 	vcn_v3_0_mc_resume(vinst);
1274 
1275 	/* VCN global tiling registers */
1276 	WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1277 		     adev->gfx.config.gb_addr_config);
1278 
1279 	/* unblock VCPU register access */
1280 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1281 		 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1282 
1283 	/* release VCPU reset to boot */
1284 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1285 		 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1286 
1287 	for (j = 0; j < 10; ++j) {
1288 		uint32_t status;
1289 
1290 		for (k = 0; k < 100; ++k) {
1291 			status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1292 			if (status & 2)
1293 				break;
1294 			mdelay(10);
1295 		}
1296 		r = 0;
1297 		if (status & 2)
1298 			break;
1299 
1300 		DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1301 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1302 			 UVD_VCPU_CNTL__BLK_RST_MASK,
1303 			 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1304 		mdelay(10);
1305 		WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1306 			 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1307 
1308 		mdelay(10);
1309 		r = -1;
1310 	}
1311 
1312 	if (r) {
1313 		DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1314 		return r;
1315 	}
1316 
1317 	/* enable master interrupt */
1318 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1319 		 UVD_MASTINT_EN__VCPU_EN_MASK,
1320 		 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1321 
1322 	/* clear the busy bit of VCN_STATUS */
1323 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1324 		 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1325 
1326 	WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1327 
1328 	ring = &adev->vcn.inst[i].ring_dec;
1329 	/* force RBC into idle state */
1330 	rb_bufsz = order_base_2(ring->ring_size);
1331 	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1332 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1333 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1334 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1335 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1336 	WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1337 
1338 	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1339 	fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1340 
1341 	/* programm the RB_BASE for ring buffer */
1342 	WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1343 		     lower_32_bits(ring->gpu_addr));
1344 	WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1345 		     upper_32_bits(ring->gpu_addr));
1346 
1347 	/* Initialize the ring buffer's read and write pointers */
1348 	WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1349 
1350 	WREG32_SOC15(VCN, i, mmUVD_SCRATCH2, 0);
1351 	ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1352 	WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1353 		     lower_32_bits(ring->wptr));
1354 	fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1355 	fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1356 
1357 	if (amdgpu_ip_version(adev, UVD_HWIP, 0) !=
1358 	    IP_VERSION(3, 0, 33)) {
1359 		fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1360 		ring = &adev->vcn.inst[i].ring_enc[0];
1361 		WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1362 		WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1363 		WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1364 		WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1365 		WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1366 		fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1367 
1368 		fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1369 		ring = &adev->vcn.inst[i].ring_enc[1];
1370 		WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1371 		WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1372 		WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1373 		WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1374 		WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1375 		fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1376 	}
1377 
1378 	/* Keeping one read-back to ensure all register writes are done,
1379 	 * otherwise it may introduce race conditions.
1380 	 */
1381 	RREG32_SOC15(VCN, i, mmUVD_STATUS);
1382 
1383 	return 0;
1384 }
1385 
1386 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1387 {
1388 	int i, j;
1389 	struct amdgpu_ring *ring;
1390 	uint64_t cache_addr;
1391 	uint64_t rb_addr;
1392 	uint64_t ctx_addr;
1393 	uint32_t param, resp, expected;
1394 	uint32_t offset, cache_size;
1395 	uint32_t tmp, timeout;
1396 
1397 	struct amdgpu_mm_table *table = &adev->virt.mm_table;
1398 	uint32_t *table_loc;
1399 	uint32_t table_size;
1400 	uint32_t size, size_dw;
1401 
1402 	struct mmsch_v3_0_cmd_direct_write
1403 		direct_wt = { {0} };
1404 	struct mmsch_v3_0_cmd_direct_read_modify_write
1405 		direct_rd_mod_wt = { {0} };
1406 	struct mmsch_v3_0_cmd_end end = { {0} };
1407 	struct mmsch_v3_0_init_header header;
1408 
1409 	direct_wt.cmd_header.command_type =
1410 		MMSCH_COMMAND__DIRECT_REG_WRITE;
1411 	direct_rd_mod_wt.cmd_header.command_type =
1412 		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1413 	end.cmd_header.command_type =
1414 		MMSCH_COMMAND__END;
1415 
1416 	header.version = MMSCH_VERSION;
1417 	header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1418 	for (i = 0; i < MMSCH_V3_0_VCN_INSTANCES; i++) {
1419 		header.inst[i].init_status = 0;
1420 		header.inst[i].table_offset = 0;
1421 		header.inst[i].table_size = 0;
1422 	}
1423 
1424 	table_loc = (uint32_t *)table->cpu_addr;
1425 	table_loc += header.total_size;
1426 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1427 		if (adev->vcn.harvest_config & (1 << i))
1428 			continue;
1429 
1430 		table_size = 0;
1431 
1432 		MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1433 			mmUVD_STATUS),
1434 			~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1435 
1436 		cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4);
1437 
1438 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1439 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1440 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1441 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1442 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1443 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1444 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1445 			offset = 0;
1446 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1447 				mmUVD_VCPU_CACHE_OFFSET0),
1448 				0);
1449 		} else {
1450 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1451 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1452 				lower_32_bits(adev->vcn.inst[i].gpu_addr));
1453 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1454 				mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1455 				upper_32_bits(adev->vcn.inst[i].gpu_addr));
1456 			offset = cache_size;
1457 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1458 				mmUVD_VCPU_CACHE_OFFSET0),
1459 				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1460 		}
1461 
1462 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1463 			mmUVD_VCPU_CACHE_SIZE0),
1464 			cache_size);
1465 
1466 		cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1467 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1468 			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1469 			lower_32_bits(cache_addr));
1470 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1471 			mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1472 			upper_32_bits(cache_addr));
1473 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1474 			mmUVD_VCPU_CACHE_OFFSET1),
1475 			0);
1476 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1477 			mmUVD_VCPU_CACHE_SIZE1),
1478 			AMDGPU_VCN_STACK_SIZE);
1479 
1480 		cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1481 			AMDGPU_VCN_STACK_SIZE;
1482 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1483 			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1484 			lower_32_bits(cache_addr));
1485 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1486 			mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1487 			upper_32_bits(cache_addr));
1488 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1489 			mmUVD_VCPU_CACHE_OFFSET2),
1490 			0);
1491 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1492 			mmUVD_VCPU_CACHE_SIZE2),
1493 			AMDGPU_VCN_CONTEXT_SIZE);
1494 
1495 		for (j = 0; j < adev->vcn.inst[i].num_enc_rings; ++j) {
1496 			ring = &adev->vcn.inst[i].ring_enc[j];
1497 			ring->wptr = 0;
1498 			rb_addr = ring->gpu_addr;
1499 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1500 				mmUVD_RB_BASE_LO),
1501 				lower_32_bits(rb_addr));
1502 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1503 				mmUVD_RB_BASE_HI),
1504 				upper_32_bits(rb_addr));
1505 			MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1506 				mmUVD_RB_SIZE),
1507 				ring->ring_size / 4);
1508 		}
1509 
1510 		ring = &adev->vcn.inst[i].ring_dec;
1511 		ring->wptr = 0;
1512 		rb_addr = ring->gpu_addr;
1513 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1514 			mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1515 			lower_32_bits(rb_addr));
1516 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1517 			mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1518 			upper_32_bits(rb_addr));
1519 		/* force RBC into idle state */
1520 		tmp = order_base_2(ring->ring_size);
1521 		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1522 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1523 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1524 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1525 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1526 		MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1527 			mmUVD_RBC_RB_CNTL),
1528 			tmp);
1529 
1530 		/* add end packet */
1531 		MMSCH_V3_0_INSERT_END();
1532 
1533 		/* refine header */
1534 		header.inst[i].init_status = 0;
1535 		header.inst[i].table_offset = header.total_size;
1536 		header.inst[i].table_size = table_size;
1537 		header.total_size += table_size;
1538 	}
1539 
1540 	/* Update init table header in memory */
1541 	size = sizeof(struct mmsch_v3_0_init_header);
1542 	table_loc = (uint32_t *)table->cpu_addr;
1543 	memcpy((void *)table_loc, &header, size);
1544 
1545 	/* message MMSCH (in VCN[0]) to initialize this client
1546 	 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1547 	 * of memory descriptor location
1548 	 */
1549 	ctx_addr = table->gpu_addr;
1550 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1551 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1552 
1553 	/* 2, update vmid of descriptor */
1554 	tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1555 	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1556 	/* use domain0 for MM scheduler */
1557 	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1558 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1559 
1560 	/* 3, notify mmsch about the size of this descriptor */
1561 	size = header.total_size;
1562 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1563 
1564 	/* 4, set resp to zero */
1565 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1566 
1567 	/* 5, kick off the initialization and wait until
1568 	 * MMSCH_VF_MAILBOX_RESP becomes non-zero
1569 	 */
1570 	param = 0x10000001;
1571 	WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1572 	tmp = 0;
1573 	timeout = 1000;
1574 	resp = 0;
1575 	expected = param + 1;
1576 	while (resp != expected) {
1577 		resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1578 		if (resp == expected)
1579 			break;
1580 
1581 		udelay(10);
1582 		tmp = tmp + 10;
1583 		if (tmp >= timeout) {
1584 			DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1585 				" waiting for mmMMSCH_VF_MAILBOX_RESP "\
1586 				"(expected=0x%08x, readback=0x%08x)\n",
1587 				tmp, expected, resp);
1588 			return -EBUSY;
1589 		}
1590 	}
1591 
1592 	return 0;
1593 }
1594 
1595 static int vcn_v3_0_stop_dpg_mode(struct amdgpu_vcn_inst *vinst)
1596 {
1597 	struct amdgpu_device *adev = vinst->adev;
1598 	int inst_idx = vinst->inst;
1599 	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1600 	uint32_t tmp;
1601 
1602 	vcn_v3_0_pause_dpg_mode(vinst, &state);
1603 
1604 	/* Wait for power status to be 1 */
1605 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1606 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1607 
1608 	/* wait for read ptr to be equal to write ptr */
1609 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1610 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1611 
1612 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1613 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1614 
1615 	tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1616 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1617 
1618 	SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1619 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1620 
1621 	/* disable dynamic power gating mode */
1622 	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1623 		~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1624 
1625 	/* Keeping one read-back to ensure all register writes are done,
1626 	 * otherwise it may introduce race conditions.
1627 	 */
1628 	RREG32_SOC15(VCN, inst_idx, mmUVD_STATUS);
1629 
1630 	return 0;
1631 }
1632 
1633 static int vcn_v3_0_stop(struct amdgpu_vcn_inst *vinst)
1634 {
1635 	struct amdgpu_device *adev = vinst->adev;
1636 	int i = vinst->inst;
1637 	uint32_t tmp;
1638 	int r = 0;
1639 
1640 	if (adev->vcn.harvest_config & (1 << i))
1641 		return 0;
1642 
1643 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1644 		r = vcn_v3_0_stop_dpg_mode(vinst);
1645 		goto done;
1646 	}
1647 
1648 	/* wait for vcn idle */
1649 	r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1650 	if (r)
1651 		goto done;
1652 
1653 	tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1654 		UVD_LMI_STATUS__READ_CLEAN_MASK |
1655 		UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1656 		UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1657 	r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1658 	if (r)
1659 		goto done;
1660 
1661 	/* disable LMI UMC channel */
1662 	tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1663 	tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1664 	WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1665 	tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1666 		UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1667 	r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1668 	if (r)
1669 		goto done;
1670 
1671 	/* block VCPU register access */
1672 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1673 		 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1674 		 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1675 
1676 	/* reset VCPU */
1677 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1678 		 UVD_VCPU_CNTL__BLK_RST_MASK,
1679 		 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1680 
1681 	/* disable VCPU clock */
1682 	WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1683 		 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1684 
1685 	/* apply soft reset */
1686 	tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1687 	tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1688 	WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1689 	tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1690 	tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1691 	WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1692 
1693 	/* clear status */
1694 	WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1695 
1696 	/* apply HW clock gating */
1697 	vcn_v3_0_enable_clock_gating(vinst);
1698 
1699 	/* enable VCN power gating */
1700 	vcn_v3_0_enable_static_power_gating(vinst);
1701 
1702 	/* Keeping one read-back to ensure all register writes are done,
1703 	 * otherwise it may introduce race conditions.
1704 	 */
1705 	RREG32_SOC15(VCN, i, mmUVD_STATUS);
1706 
1707 done:
1708 	if (adev->pm.dpm_enabled)
1709 		amdgpu_dpm_enable_vcn(adev, false, i);
1710 
1711 	return r;
1712 }
1713 
1714 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
1715 				   struct dpg_pause_state *new_state)
1716 {
1717 	struct amdgpu_device *adev = vinst->adev;
1718 	int inst_idx = vinst->inst;
1719 	struct amdgpu_fw_shared *fw_shared;
1720 	struct amdgpu_ring *ring;
1721 	uint32_t reg_data = 0;
1722 	int ret_code;
1723 
1724 	/* pause/unpause if state is changed */
1725 	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1726 		DRM_DEBUG("dpg pause state changed %d -> %d",
1727 			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
1728 		reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1729 			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1730 
1731 		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1732 			ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1733 				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1734 
1735 			if (!ret_code) {
1736 				/* pause DPG */
1737 				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1738 				WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1739 
1740 				/* wait for ACK */
1741 				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1742 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1743 					UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1744 
1745 				/* Stall DPG before WPTR/RPTR reset */
1746 				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1747 					UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1748 					~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1749 
1750 				if (amdgpu_ip_version(adev, UVD_HWIP, 0) !=
1751 				    IP_VERSION(3, 0, 33)) {
1752 					/* Restore */
1753 					fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1754 					fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1755 					ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1756 					ring->wptr = 0;
1757 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1758 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1759 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1760 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1761 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1762 					fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1763 
1764 					fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1765 					ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1766 					ring->wptr = 0;
1767 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1768 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1769 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1770 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1771 					WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1772 					fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1773 
1774 					/* restore wptr/rptr with pointers saved in FW shared memory*/
1775 					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, fw_shared->rb.rptr);
1776 					WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, fw_shared->rb.wptr);
1777 				}
1778 
1779 				/* Unstall DPG */
1780 				WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1781 					0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1782 
1783 				SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1784 					UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1785 			}
1786 		} else {
1787 			/* unpause dpg, no need to wait */
1788 			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1789 			WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1790 		}
1791 		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1792 	}
1793 
1794 	return 0;
1795 }
1796 
1797 /**
1798  * vcn_v3_0_dec_ring_get_rptr - get read pointer
1799  *
1800  * @ring: amdgpu_ring pointer
1801  *
1802  * Returns the current hardware read pointer
1803  */
1804 static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1805 {
1806 	struct amdgpu_device *adev = ring->adev;
1807 
1808 	return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1809 }
1810 
1811 /**
1812  * vcn_v3_0_dec_ring_get_wptr - get write pointer
1813  *
1814  * @ring: amdgpu_ring pointer
1815  *
1816  * Returns the current hardware write pointer
1817  */
1818 static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1819 {
1820 	struct amdgpu_device *adev = ring->adev;
1821 
1822 	if (ring->use_doorbell)
1823 		return *ring->wptr_cpu_addr;
1824 	else
1825 		return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1826 }
1827 
1828 /**
1829  * vcn_v3_0_dec_ring_set_wptr - set write pointer
1830  *
1831  * @ring: amdgpu_ring pointer
1832  *
1833  * Commits the write pointer to the hardware
1834  */
1835 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1836 {
1837 	struct amdgpu_device *adev = ring->adev;
1838 	struct amdgpu_fw_shared *fw_shared;
1839 
1840 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1841 		/*whenever update RBC_RB_WPTR, we save the wptr in shared rb.wptr and scratch2 */
1842 		fw_shared = adev->vcn.inst[ring->me].fw_shared.cpu_addr;
1843 		fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1844 		WREG32_SOC15(VCN, ring->me, mmUVD_SCRATCH2,
1845 			lower_32_bits(ring->wptr));
1846 	}
1847 
1848 	if (ring->use_doorbell) {
1849 		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1850 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1851 	} else {
1852 		WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1853 	}
1854 }
1855 
1856 static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1857 	.type = AMDGPU_RING_TYPE_VCN_DEC,
1858 	.align_mask = 0x3f,
1859 	.nop = VCN_DEC_SW_CMD_NO_OP,
1860 	.no_user_fence = true,
1861 	.secure_submission_supported = true,
1862 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
1863 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
1864 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
1865 	.emit_frame_size =
1866 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1867 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1868 		VCN_SW_RING_EMIT_FRAME_SIZE,
1869 	.emit_ib_size = 5, /* vcn_dec_sw_ring_emit_ib */
1870 	.emit_ib = vcn_dec_sw_ring_emit_ib,
1871 	.emit_fence = vcn_dec_sw_ring_emit_fence,
1872 	.emit_vm_flush = vcn_dec_sw_ring_emit_vm_flush,
1873 	.test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1874 	.test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,
1875 	.insert_nop = amdgpu_ring_insert_nop,
1876 	.insert_end = vcn_dec_sw_ring_insert_end,
1877 	.pad_ib = amdgpu_ring_generic_pad_ib,
1878 	.begin_use = amdgpu_vcn_ring_begin_use,
1879 	.end_use = amdgpu_vcn_ring_end_use,
1880 	.emit_wreg = vcn_dec_sw_ring_emit_wreg,
1881 	.emit_reg_wait = vcn_dec_sw_ring_emit_reg_wait,
1882 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1883 };
1884 
1885 static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p,
1886 				struct amdgpu_job *job)
1887 {
1888 	struct drm_gpu_scheduler **scheds;
1889 	struct dma_fence *fence;
1890 
1891 	/* if VCN0 is harvested, we can't support AV1 */
1892 	if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)
1893 		return -EINVAL;
1894 
1895 	/* wait for all jobs to finish before switching to instance 0 */
1896 	fence = amdgpu_ctx_get_fence(p->ctx, job->base.entity, ~0ull);
1897 	if (fence) {
1898 		dma_fence_wait(fence, false);
1899 		dma_fence_put(fence);
1900 	}
1901 
1902 	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]
1903 		[AMDGPU_RING_PRIO_DEFAULT].sched;
1904 	drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
1905 	return 0;
1906 }
1907 
1908 static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1909 			    uint64_t addr)
1910 {
1911 	struct ttm_operation_ctx ctx = { false, false };
1912 	struct amdgpu_device *adev = p->adev;
1913 	struct amdgpu_bo_va_mapping *map;
1914 	uint32_t *msg, num_buffers, len_dw;
1915 	struct amdgpu_bo *bo;
1916 	uint64_t start, end;
1917 	unsigned int i;
1918 	void *ptr;
1919 	int r;
1920 
1921 	addr &= AMDGPU_GMC_HOLE_MASK;
1922 	r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1923 	if (r) {
1924 		DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
1925 		return r;
1926 	}
1927 
1928 	start = map->start * AMDGPU_GPU_PAGE_SIZE;
1929 	end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1930 	if (addr & 0x7) {
1931 		DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1932 		return -EINVAL;
1933 	}
1934 
1935 	if (end - addr < 16) {
1936 		DRM_ERROR("VCN messages must be at least 4 DWORDs!\n");
1937 		return -EINVAL;
1938 	}
1939 
1940 	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1941 	amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1942 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1943 	if (r) {
1944 		DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1945 		return r;
1946 	}
1947 
1948 	r = amdgpu_bo_kmap(bo, &ptr);
1949 	if (r) {
1950 		DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1951 		return r;
1952 	}
1953 
1954 	msg = ptr + addr - start;
1955 
1956 	if (msg[1] > end - addr) {
1957 		DRM_ERROR("VCN message header does not fit in BO!\n");
1958 		r = -EINVAL;
1959 		goto out;
1960 	}
1961 
1962 	if (msg[3] != RDECODE_MSG_CREATE)
1963 		goto out;
1964 
1965 	len_dw = msg[1] / 4;
1966 	num_buffers = msg[2];
1967 
1968 	/* Verify that all indices fit within the claimed length.
1969 	 * There are 6 dwords in the header before the first buffer.
1970 	 * Each buffer has 4 dwords. Any trailing dwords after the
1971 	 * last buffer are ignored.
1972 	 */
1973 	if (len_dw < 6 || num_buffers > (len_dw - 6) / 4) {
1974 		DRM_ERROR("Invalid VCN message!\n");
1975 		r = -EINVAL;
1976 		goto out;
1977 	}
1978 
1979 	for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1980 		uint32_t offset, size, *create;
1981 		uint64_t buf_end;
1982 
1983 		if (msg[0] != RDECODE_MESSAGE_CREATE)
1984 			continue;
1985 
1986 		offset = msg[1];
1987 		size = msg[2];
1988 
1989 		if (size < 4 || check_add_overflow(offset, size, &buf_end) ||
1990 		    buf_end > end - addr) {
1991 			DRM_ERROR("VCN message buffer exceeds BO bounds!\n");
1992 			r = -EINVAL;
1993 			goto out;
1994 		}
1995 
1996 		create = ptr + addr + offset - start;
1997 
1998 		/* H264, HEVC and VP9 can run on any instance */
1999 		if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
2000 			continue;
2001 
2002 		r = vcn_v3_0_limit_sched(p, job);
2003 		if (r)
2004 			goto out;
2005 	}
2006 
2007 out:
2008 	amdgpu_bo_kunmap(bo);
2009 	return r;
2010 }
2011 
2012 static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
2013 					   struct amdgpu_job *job,
2014 					   struct amdgpu_ib *ib)
2015 {
2016 	struct amdgpu_ring *ring = amdgpu_job_ring(job);
2017 	uint32_t msg_lo = 0, msg_hi = 0;
2018 	unsigned i;
2019 	int r;
2020 
2021 	/* The first instance can decode anything */
2022 	if (!ring->me)
2023 		return 0;
2024 
2025 	for (i = 0; i < ib->length_dw; i += 2) {
2026 		uint32_t reg = amdgpu_ib_get_value(ib, i);
2027 		uint32_t val = amdgpu_ib_get_value(ib, i + 1);
2028 
2029 		if (reg == PACKET0(p->adev->vcn.inst[ring->me].internal.data0, 0)) {
2030 			msg_lo = val;
2031 		} else if (reg == PACKET0(p->adev->vcn.inst[ring->me].internal.data1, 0)) {
2032 			msg_hi = val;
2033 		} else if (reg == PACKET0(p->adev->vcn.inst[ring->me].internal.cmd, 0) &&
2034 			   val == 0) {
2035 			r = vcn_v3_0_dec_msg(p, job,
2036 					     ((u64)msg_hi) << 32 | msg_lo);
2037 			if (r)
2038 				return r;
2039 		}
2040 	}
2041 	return 0;
2042 }
2043 
2044 static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
2045 	.type = AMDGPU_RING_TYPE_VCN_DEC,
2046 	.align_mask = 0xf,
2047 	.no_user_fence = true,
2048 	.secure_submission_supported = true,
2049 	.get_rptr = vcn_v3_0_dec_ring_get_rptr,
2050 	.get_wptr = vcn_v3_0_dec_ring_get_wptr,
2051 	.set_wptr = vcn_v3_0_dec_ring_set_wptr,
2052 	.patch_cs_in_place = vcn_v3_0_ring_patch_cs_in_place,
2053 	.emit_frame_size =
2054 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2055 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2056 		8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
2057 		14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
2058 		6,
2059 	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
2060 	.emit_ib = vcn_v2_0_dec_ring_emit_ib,
2061 	.emit_fence = vcn_v2_0_dec_ring_emit_fence,
2062 	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
2063 	.test_ring = vcn_v2_0_dec_ring_test_ring,
2064 	.test_ib = amdgpu_vcn_dec_ring_test_ib,
2065 	.insert_nop = vcn_v2_0_dec_ring_insert_nop,
2066 	.insert_start = vcn_v2_0_dec_ring_insert_start,
2067 	.insert_end = vcn_v2_0_dec_ring_insert_end,
2068 	.pad_ib = amdgpu_ring_generic_pad_ib,
2069 	.begin_use = amdgpu_vcn_ring_begin_use,
2070 	.end_use = amdgpu_vcn_ring_end_use,
2071 	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
2072 	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
2073 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2074 	.reset = amdgpu_vcn_ring_reset,
2075 };
2076 
2077 /**
2078  * vcn_v3_0_enc_ring_get_rptr - get enc read pointer
2079  *
2080  * @ring: amdgpu_ring pointer
2081  *
2082  * Returns the current hardware enc read pointer
2083  */
2084 static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
2085 {
2086 	struct amdgpu_device *adev = ring->adev;
2087 
2088 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
2089 		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
2090 	else
2091 		return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
2092 }
2093 
2094 /**
2095  * vcn_v3_0_enc_ring_get_wptr - get enc write pointer
2096  *
2097  * @ring: amdgpu_ring pointer
2098  *
2099  * Returns the current hardware enc write pointer
2100  */
2101 static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
2102 {
2103 	struct amdgpu_device *adev = ring->adev;
2104 
2105 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
2106 		if (ring->use_doorbell)
2107 			return *ring->wptr_cpu_addr;
2108 		else
2109 			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
2110 	} else {
2111 		if (ring->use_doorbell)
2112 			return *ring->wptr_cpu_addr;
2113 		else
2114 			return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
2115 	}
2116 }
2117 
2118 /**
2119  * vcn_v3_0_enc_ring_set_wptr - set enc write pointer
2120  *
2121  * @ring: amdgpu_ring pointer
2122  *
2123  * Commits the enc write pointer to the hardware
2124  */
2125 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
2126 {
2127 	struct amdgpu_device *adev = ring->adev;
2128 
2129 	if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
2130 		if (ring->use_doorbell) {
2131 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
2132 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
2133 		} else {
2134 			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
2135 		}
2136 	} else {
2137 		if (ring->use_doorbell) {
2138 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
2139 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
2140 		} else {
2141 			WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
2142 		}
2143 	}
2144 }
2145 
2146 static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
2147 	.type = AMDGPU_RING_TYPE_VCN_ENC,
2148 	.align_mask = 0x3f,
2149 	.nop = VCN_ENC_CMD_NO_OP,
2150 	.no_user_fence = true,
2151 	.get_rptr = vcn_v3_0_enc_ring_get_rptr,
2152 	.get_wptr = vcn_v3_0_enc_ring_get_wptr,
2153 	.set_wptr = vcn_v3_0_enc_ring_set_wptr,
2154 	.emit_frame_size =
2155 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2156 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2157 		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2158 		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2159 		1, /* vcn_v2_0_enc_ring_insert_end */
2160 	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2161 	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
2162 	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
2163 	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2164 	.test_ring = amdgpu_vcn_enc_ring_test_ring,
2165 	.test_ib = amdgpu_vcn_enc_ring_test_ib,
2166 	.insert_nop = amdgpu_ring_insert_nop,
2167 	.insert_end = vcn_v2_0_enc_ring_insert_end,
2168 	.pad_ib = amdgpu_ring_generic_pad_ib,
2169 	.begin_use = amdgpu_vcn_ring_begin_use,
2170 	.end_use = amdgpu_vcn_ring_end_use,
2171 	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2172 	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2173 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2174 	.reset = amdgpu_vcn_ring_reset,
2175 };
2176 
2177 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2178 {
2179 	int i;
2180 
2181 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2182 		if (adev->vcn.harvest_config & (1 << i))
2183 			continue;
2184 
2185 		if (!DEC_SW_RING_ENABLED)
2186 			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
2187 		else
2188 			adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
2189 		adev->vcn.inst[i].ring_dec.me = i;
2190 	}
2191 }
2192 
2193 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2194 {
2195 	int i, j;
2196 
2197 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2198 		if (adev->vcn.harvest_config & (1 << i))
2199 			continue;
2200 
2201 		for (j = 0; j < adev->vcn.inst[i].num_enc_rings; ++j) {
2202 			adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
2203 			adev->vcn.inst[i].ring_enc[j].me = i;
2204 		}
2205 	}
2206 }
2207 
2208 static int vcn_v3_0_reset(struct amdgpu_vcn_inst *vinst)
2209 {
2210 	int r;
2211 
2212 	r = vcn_v3_0_stop(vinst);
2213 	if (r)
2214 		return r;
2215 	vcn_v3_0_enable_clock_gating(vinst);
2216 	vcn_v3_0_enable_static_power_gating(vinst);
2217 	return vcn_v3_0_start(vinst);
2218 }
2219 
2220 static int vcn_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
2221 {
2222 	struct amdgpu_device *adev = ip_block->adev;
2223 	int i, ret = 0;
2224 
2225 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2226 		if (adev->vcn.harvest_config & (1 << i))
2227 			continue;
2228 
2229 		ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2230 			UVD_STATUS__IDLE);
2231 		if (ret)
2232 			return ret;
2233 	}
2234 
2235 	return ret;
2236 }
2237 
2238 static int vcn_v3_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2239 					  enum amd_clockgating_state state)
2240 {
2241 	struct amdgpu_device *adev = ip_block->adev;
2242 	bool enable = state == AMD_CG_STATE_GATE;
2243 	int i;
2244 
2245 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2246 		struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
2247 		if (adev->vcn.harvest_config & (1 << i))
2248 			continue;
2249 
2250 		if (enable) {
2251 			if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2252 				return -EBUSY;
2253 			vcn_v3_0_enable_clock_gating(vinst);
2254 		} else {
2255 			vcn_v3_0_disable_clock_gating(vinst);
2256 		}
2257 	}
2258 
2259 	return 0;
2260 }
2261 
2262 static int vcn_v3_0_set_pg_state(struct amdgpu_vcn_inst *vinst,
2263 				 enum amd_powergating_state state)
2264 {
2265 	struct amdgpu_device *adev = vinst->adev;
2266 	int ret = 0;
2267 
2268 	/* for SRIOV, guest should not control VCN Power-gating
2269 	 * MMSCH FW should control Power-gating and clock-gating
2270 	 * guest should avoid touching CGC and PG
2271 	 */
2272 	if (amdgpu_sriov_vf(adev)) {
2273 		vinst->cur_state = AMD_PG_STATE_UNGATE;
2274 		return 0;
2275 	}
2276 
2277 	if (state == vinst->cur_state)
2278 		return 0;
2279 
2280 	if (state == AMD_PG_STATE_GATE)
2281 		ret = vcn_v3_0_stop(vinst);
2282 	else
2283 		ret = vcn_v3_0_start(vinst);
2284 
2285 	if (!ret)
2286 		vinst->cur_state = state;
2287 
2288 	return ret;
2289 }
2290 
2291 static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2292 					struct amdgpu_irq_src *source,
2293 					unsigned type,
2294 					enum amdgpu_interrupt_state state)
2295 {
2296 	return 0;
2297 }
2298 
2299 static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2300 				      struct amdgpu_irq_src *source,
2301 				      struct amdgpu_iv_entry *entry)
2302 {
2303 	uint32_t ip_instance;
2304 
2305 	switch (entry->client_id) {
2306 	case SOC15_IH_CLIENTID_VCN:
2307 		ip_instance = 0;
2308 		break;
2309 	case SOC15_IH_CLIENTID_VCN1:
2310 		ip_instance = 1;
2311 		break;
2312 	default:
2313 		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2314 		return 0;
2315 	}
2316 
2317 	DRM_DEBUG("IH: VCN TRAP\n");
2318 
2319 	switch (entry->src_id) {
2320 	case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2321 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2322 		break;
2323 	case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2324 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2325 		break;
2326 	case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2327 		amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2328 		break;
2329 	default:
2330 		DRM_ERROR("Unhandled interrupt: %d %d\n",
2331 			  entry->src_id, entry->src_data[0]);
2332 		break;
2333 	}
2334 
2335 	return 0;
2336 }
2337 
2338 static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2339 	.set = vcn_v3_0_set_interrupt_state,
2340 	.process = vcn_v3_0_process_interrupt,
2341 };
2342 
2343 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2344 {
2345 	int i;
2346 
2347 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2348 		if (adev->vcn.harvest_config & (1 << i))
2349 			continue;
2350 
2351 		adev->vcn.inst[i].irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
2352 		adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2353 	}
2354 }
2355 
2356 static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2357 	.name = "vcn_v3_0",
2358 	.early_init = vcn_v3_0_early_init,
2359 	.sw_init = vcn_v3_0_sw_init,
2360 	.sw_fini = vcn_v3_0_sw_fini,
2361 	.hw_init = vcn_v3_0_hw_init,
2362 	.hw_fini = vcn_v3_0_hw_fini,
2363 	.suspend = vcn_v3_0_suspend,
2364 	.resume = vcn_v3_0_resume,
2365 	.wait_for_idle = vcn_v3_0_wait_for_idle,
2366 	.set_clockgating_state = vcn_v3_0_set_clockgating_state,
2367 	.set_powergating_state = vcn_set_powergating_state,
2368 	.dump_ip_state = amdgpu_vcn_dump_ip_state,
2369 	.print_ip_state = amdgpu_vcn_print_ip_state,
2370 };
2371 
2372 const struct amdgpu_ip_block_version vcn_v3_0_ip_block = {
2373 	.type = AMD_IP_BLOCK_TYPE_VCN,
2374 	.major = 3,
2375 	.minor = 0,
2376 	.rev = 0,
2377 	.funcs = &vcn_v3_0_ip_funcs,
2378 };
2379