xref: /linux/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c (revision 3fd6c59042dbba50391e30862beac979491145fe)
1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <drm/drm_drv.h>
26 
27 #include "amdgpu.h"
28 #include "amdgpu_vcn.h"
29 #include "soc15.h"
30 #include "soc15d.h"
31 #include "amdgpu_pm.h"
32 #include "amdgpu_psp.h"
33 #include "mmsch_v2_0.h"
34 #include "vcn_v2_0.h"
35 
36 #include "vcn/vcn_2_0_0_offset.h"
37 #include "vcn/vcn_2_0_0_sh_mask.h"
38 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
39 
40 #define VCN_VID_SOC_ADDRESS_2_0					0x1fa00
41 #define VCN1_VID_SOC_ADDRESS_3_0				0x48200
42 
43 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET			0x1fd
44 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET			0x503
45 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET			0x504
46 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET			0x505
47 #define mmUVD_NO_OP_INTERNAL_OFFSET				0x53f
48 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET			0x54a
49 #define mmUVD_SCRATCH9_INTERNAL_OFFSET				0xc01d
50 
51 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET			0x1e1
52 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET		0x5a6
53 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET		0x5a7
54 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET			0x1e2
55 
56 static const struct amdgpu_hwip_reg_entry vcn_reg_list_2_0[] = {
57 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_POWER_STATUS),
58 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_STATUS),
59 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID),
60 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID2),
61 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA0),
62 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA1),
63 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_CMD),
64 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI),
65 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO),
66 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI2),
67 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO2),
68 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI3),
69 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO3),
70 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI4),
71 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO4),
72 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR),
73 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR),
74 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR2),
75 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR2),
76 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR3),
77 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR3),
78 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR4),
79 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR4),
80 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE),
81 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE2),
82 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE3),
83 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE4),
84 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_CONFIG),
85 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_STATUS),
86 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_CTL),
87 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_DATA),
88 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_MASK),
89 	SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_PAUSE)
90 };
91 
92 static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev);
93 static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev);
94 static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev);
95 static int vcn_v2_0_set_powergating_state(void *handle,
96 				enum amd_powergating_state state);
97 static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
98 				int inst_idx, struct dpg_pause_state *new_state);
99 static int vcn_v2_0_start_sriov(struct amdgpu_device *adev);
100 /**
101  * vcn_v2_0_early_init - set function pointers and load microcode
102  *
103  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
104  *
105  * Set ring and irq function pointers
106  * Load microcode from filesystem
107  */
vcn_v2_0_early_init(struct amdgpu_ip_block * ip_block)108 static int vcn_v2_0_early_init(struct amdgpu_ip_block *ip_block)
109 {
110 	struct amdgpu_device *adev = ip_block->adev;
111 
112 	if (amdgpu_sriov_vf(adev))
113 		adev->vcn.num_enc_rings = 1;
114 	else
115 		adev->vcn.num_enc_rings = 2;
116 
117 	vcn_v2_0_set_dec_ring_funcs(adev);
118 	vcn_v2_0_set_enc_ring_funcs(adev);
119 	vcn_v2_0_set_irq_funcs(adev);
120 
121 	return amdgpu_vcn_early_init(adev);
122 }
123 
124 /**
125  * vcn_v2_0_sw_init - sw init for VCN block
126  *
127  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
128  *
129  * Load firmware and sw initialization
130  */
vcn_v2_0_sw_init(struct amdgpu_ip_block * ip_block)131 static int vcn_v2_0_sw_init(struct amdgpu_ip_block *ip_block)
132 {
133 	struct amdgpu_ring *ring;
134 	int i, r;
135 	uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_0);
136 	uint32_t *ptr;
137 	struct amdgpu_device *adev = ip_block->adev;
138 	volatile struct amdgpu_fw_shared *fw_shared;
139 
140 	/* VCN DEC TRAP */
141 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
142 			      VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT,
143 			      &adev->vcn.inst->irq);
144 	if (r)
145 		return r;
146 
147 	/* VCN ENC TRAP */
148 	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
149 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
150 				      i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
151 				      &adev->vcn.inst->irq);
152 		if (r)
153 			return r;
154 	}
155 
156 	r = amdgpu_vcn_sw_init(adev);
157 	if (r)
158 		return r;
159 
160 	amdgpu_vcn_setup_ucode(adev);
161 
162 	r = amdgpu_vcn_resume(adev);
163 	if (r)
164 		return r;
165 
166 	ring = &adev->vcn.inst->ring_dec;
167 
168 	ring->use_doorbell = true;
169 	ring->doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
170 	ring->vm_hub = AMDGPU_MMHUB0(0);
171 
172 	sprintf(ring->name, "vcn_dec");
173 	r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
174 			     AMDGPU_RING_PRIO_DEFAULT, NULL);
175 	if (r)
176 		return r;
177 
178 	adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
179 	adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
180 	adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
181 	adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
182 	adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
183 	adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
184 
185 	adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
186 	adev->vcn.inst->external.scratch9 = SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9);
187 	adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
188 	adev->vcn.inst->external.data0 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0);
189 	adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
190 	adev->vcn.inst->external.data1 = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1);
191 	adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
192 	adev->vcn.inst->external.cmd = SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD);
193 	adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
194 	adev->vcn.inst->external.nop = SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP);
195 
196 	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
197 		enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);
198 
199 		ring = &adev->vcn.inst->ring_enc[i];
200 		ring->use_doorbell = true;
201 		ring->vm_hub = AMDGPU_MMHUB0(0);
202 		if (!amdgpu_sriov_vf(adev))
203 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + i;
204 		else
205 			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + i;
206 		sprintf(ring->name, "vcn_enc%d", i);
207 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0,
208 				     hw_prio, NULL);
209 		if (r)
210 			return r;
211 	}
212 
213 	adev->vcn.pause_dpg_mode = vcn_v2_0_pause_dpg_mode;
214 
215 	r = amdgpu_virt_alloc_mm_table(adev);
216 	if (r)
217 		return r;
218 
219 	fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
220 	fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
221 
222 	if (amdgpu_vcnfw_log)
223 		amdgpu_vcn_fwlog_init(adev->vcn.inst);
224 
225 	/* Allocate memory for VCN IP Dump buffer */
226 	ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL);
227 	if (!ptr) {
228 		DRM_ERROR("Failed to allocate memory for VCN IP Dump\n");
229 		adev->vcn.ip_dump = NULL;
230 	} else {
231 		adev->vcn.ip_dump = ptr;
232 	}
233 
234 	return 0;
235 }
236 
237 /**
238  * vcn_v2_0_sw_fini - sw fini for VCN block
239  *
240  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
241  *
242  * VCN suspend and free up sw allocation
243  */
vcn_v2_0_sw_fini(struct amdgpu_ip_block * ip_block)244 static int vcn_v2_0_sw_fini(struct amdgpu_ip_block *ip_block)
245 {
246 	int r, idx;
247 	struct amdgpu_device *adev = ip_block->adev;
248 	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
249 
250 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
251 		fw_shared->present_flag_0 = 0;
252 		drm_dev_exit(idx);
253 	}
254 
255 	amdgpu_virt_free_mm_table(adev);
256 
257 	r = amdgpu_vcn_suspend(adev);
258 	if (r)
259 		return r;
260 
261 	r = amdgpu_vcn_sw_fini(adev);
262 
263 	kfree(adev->vcn.ip_dump);
264 
265 	return r;
266 }
267 
268 /**
269  * vcn_v2_0_hw_init - start and test VCN block
270  *
271  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
272  *
273  * Initialize the hardware, boot up the VCPU and do some testing
274  */
vcn_v2_0_hw_init(struct amdgpu_ip_block * ip_block)275 static int vcn_v2_0_hw_init(struct amdgpu_ip_block *ip_block)
276 {
277 	struct amdgpu_device *adev = ip_block->adev;
278 	struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
279 	int i, r;
280 
281 	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
282 					     ring->doorbell_index, 0);
283 
284 	if (amdgpu_sriov_vf(adev))
285 		vcn_v2_0_start_sriov(adev);
286 
287 	r = amdgpu_ring_test_helper(ring);
288 	if (r)
289 		return r;
290 
291 	//Disable vcn decode for sriov
292 	if (amdgpu_sriov_vf(adev))
293 		ring->sched.ready = false;
294 
295 	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
296 		ring = &adev->vcn.inst->ring_enc[i];
297 		r = amdgpu_ring_test_helper(ring);
298 		if (r)
299 			return r;
300 	}
301 
302 	return 0;
303 }
304 
305 /**
306  * vcn_v2_0_hw_fini - stop the hardware block
307  *
308  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
309  *
310  * Stop the VCN block, mark ring as not ready any more
311  */
vcn_v2_0_hw_fini(struct amdgpu_ip_block * ip_block)312 static int vcn_v2_0_hw_fini(struct amdgpu_ip_block *ip_block)
313 {
314 	struct amdgpu_device *adev = ip_block->adev;
315 
316 	cancel_delayed_work_sync(&adev->vcn.idle_work);
317 
318 	if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
319 	    (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
320 	      RREG32_SOC15(VCN, 0, mmUVD_STATUS)))
321 		vcn_v2_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
322 
323 	return 0;
324 }
325 
326 /**
327  * vcn_v2_0_suspend - suspend VCN block
328  *
329  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
330  *
331  * HW fini and suspend VCN block
332  */
vcn_v2_0_suspend(struct amdgpu_ip_block * ip_block)333 static int vcn_v2_0_suspend(struct amdgpu_ip_block *ip_block)
334 {
335 	int r;
336 
337 	r = vcn_v2_0_hw_fini(ip_block);
338 	if (r)
339 		return r;
340 
341 	r = amdgpu_vcn_suspend(ip_block->adev);
342 
343 	return r;
344 }
345 
346 /**
347  * vcn_v2_0_resume - resume VCN block
348  *
349  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
350  *
351  * Resume firmware and hw init VCN block
352  */
vcn_v2_0_resume(struct amdgpu_ip_block * ip_block)353 static int vcn_v2_0_resume(struct amdgpu_ip_block *ip_block)
354 {
355 	int r;
356 
357 	r = amdgpu_vcn_resume(ip_block->adev);
358 	if (r)
359 		return r;
360 
361 	r = vcn_v2_0_hw_init(ip_block);
362 
363 	return r;
364 }
365 
366 /**
367  * vcn_v2_0_mc_resume - memory controller programming
368  *
369  * @adev: amdgpu_device pointer
370  *
371  * Let the VCN memory controller know it's offsets
372  */
vcn_v2_0_mc_resume(struct amdgpu_device * adev)373 static void vcn_v2_0_mc_resume(struct amdgpu_device *adev)
374 {
375 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
376 	uint32_t offset;
377 
378 	if (amdgpu_sriov_vf(adev))
379 		return;
380 
381 	/* cache window 0: fw */
382 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
383 		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
384 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
385 		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
386 			(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
387 		WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
388 		offset = 0;
389 	} else {
390 		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
391 			lower_32_bits(adev->vcn.inst->gpu_addr));
392 		WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
393 			upper_32_bits(adev->vcn.inst->gpu_addr));
394 		offset = size;
395 		WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
396 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
397 	}
398 
399 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
400 
401 	/* cache window 1: stack */
402 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
403 		lower_32_bits(adev->vcn.inst->gpu_addr + offset));
404 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
405 		upper_32_bits(adev->vcn.inst->gpu_addr + offset));
406 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
407 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
408 
409 	/* cache window 2: context */
410 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
411 		lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
412 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
413 		upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
414 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
415 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
416 
417 	/* non-cache window */
418 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
419 		lower_32_bits(adev->vcn.inst->fw_shared.gpu_addr));
420 	WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
421 		upper_32_bits(adev->vcn.inst->fw_shared.gpu_addr));
422 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
423 	WREG32_SOC15(UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0,
424 		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
425 
426 	WREG32_SOC15(UVD, 0, mmUVD_GFX10_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
427 }
428 
vcn_v2_0_mc_resume_dpg_mode(struct amdgpu_device * adev,bool indirect)429 static void vcn_v2_0_mc_resume_dpg_mode(struct amdgpu_device *adev, bool indirect)
430 {
431 	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
432 	uint32_t offset;
433 
434 	/* cache window 0: fw */
435 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
436 		if (!indirect) {
437 			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
438 				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
439 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo), 0, indirect);
440 			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
441 				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
442 				(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi), 0, indirect);
443 			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
444 				UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
445 		} else {
446 			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
447 				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
448 			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
449 				UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
450 			WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
451 				UVD, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
452 		}
453 		offset = 0;
454 	} else {
455 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
456 			UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
457 			lower_32_bits(adev->vcn.inst->gpu_addr), 0, indirect);
458 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
459 			UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
460 			upper_32_bits(adev->vcn.inst->gpu_addr), 0, indirect);
461 		offset = size;
462 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
463 			UVD, 0, mmUVD_VCPU_CACHE_OFFSET0),
464 			AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
465 	}
466 
467 	if (!indirect)
468 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
469 			UVD, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
470 	else
471 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
472 			UVD, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
473 
474 	/* cache window 1: stack */
475 	if (!indirect) {
476 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
477 			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
478 			lower_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect);
479 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
480 			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
481 			upper_32_bits(adev->vcn.inst->gpu_addr + offset), 0, indirect);
482 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
483 			UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
484 	} else {
485 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
486 			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
487 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
488 			UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
489 		WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
490 			UVD, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
491 	}
492 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
493 		UVD, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
494 
495 	/* cache window 2: context */
496 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
497 		UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
498 		lower_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
499 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
500 		UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
501 		upper_32_bits(adev->vcn.inst->gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
502 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
503 		UVD, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
504 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
505 		UVD, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
506 
507 	/* non-cache window */
508 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
509 		UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
510 		lower_32_bits(adev->vcn.inst->fw_shared.gpu_addr), 0, indirect);
511 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
512 		UVD, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
513 		upper_32_bits(adev->vcn.inst->fw_shared.gpu_addr), 0, indirect);
514 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
515 		UVD, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
516 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
517 		UVD, 0, mmUVD_VCPU_NONCACHE_SIZE0),
518 		AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
519 
520 	/* VCN global tiling registers */
521 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
522 		UVD, 0, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
523 }
524 
525 /**
526  * vcn_v2_0_disable_clock_gating - disable VCN clock gating
527  *
528  * @adev: amdgpu_device pointer
529  *
530  * Disable clock gating for VCN block
531  */
vcn_v2_0_disable_clock_gating(struct amdgpu_device * adev)532 static void vcn_v2_0_disable_clock_gating(struct amdgpu_device *adev)
533 {
534 	uint32_t data;
535 
536 	if (amdgpu_sriov_vf(adev))
537 		return;
538 
539 	/* UVD disable CGC */
540 	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
541 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
542 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
543 	else
544 		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
545 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
546 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
547 	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
548 
549 	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
550 	data &= ~(UVD_CGC_GATE__SYS_MASK
551 		| UVD_CGC_GATE__UDEC_MASK
552 		| UVD_CGC_GATE__MPEG2_MASK
553 		| UVD_CGC_GATE__REGS_MASK
554 		| UVD_CGC_GATE__RBC_MASK
555 		| UVD_CGC_GATE__LMI_MC_MASK
556 		| UVD_CGC_GATE__LMI_UMC_MASK
557 		| UVD_CGC_GATE__IDCT_MASK
558 		| UVD_CGC_GATE__MPRD_MASK
559 		| UVD_CGC_GATE__MPC_MASK
560 		| UVD_CGC_GATE__LBSI_MASK
561 		| UVD_CGC_GATE__LRBBM_MASK
562 		| UVD_CGC_GATE__UDEC_RE_MASK
563 		| UVD_CGC_GATE__UDEC_CM_MASK
564 		| UVD_CGC_GATE__UDEC_IT_MASK
565 		| UVD_CGC_GATE__UDEC_DB_MASK
566 		| UVD_CGC_GATE__UDEC_MP_MASK
567 		| UVD_CGC_GATE__WCB_MASK
568 		| UVD_CGC_GATE__VCPU_MASK
569 		| UVD_CGC_GATE__SCPU_MASK);
570 	WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
571 
572 	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
573 	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
574 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
575 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
576 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
577 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
578 		| UVD_CGC_CTRL__SYS_MODE_MASK
579 		| UVD_CGC_CTRL__UDEC_MODE_MASK
580 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
581 		| UVD_CGC_CTRL__REGS_MODE_MASK
582 		| UVD_CGC_CTRL__RBC_MODE_MASK
583 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
584 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
585 		| UVD_CGC_CTRL__IDCT_MODE_MASK
586 		| UVD_CGC_CTRL__MPRD_MODE_MASK
587 		| UVD_CGC_CTRL__MPC_MODE_MASK
588 		| UVD_CGC_CTRL__LBSI_MODE_MASK
589 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
590 		| UVD_CGC_CTRL__WCB_MODE_MASK
591 		| UVD_CGC_CTRL__VCPU_MODE_MASK
592 		| UVD_CGC_CTRL__SCPU_MODE_MASK);
593 	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
594 
595 	/* turn on */
596 	data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
597 	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
598 		| UVD_SUVD_CGC_GATE__SIT_MASK
599 		| UVD_SUVD_CGC_GATE__SMP_MASK
600 		| UVD_SUVD_CGC_GATE__SCM_MASK
601 		| UVD_SUVD_CGC_GATE__SDB_MASK
602 		| UVD_SUVD_CGC_GATE__SRE_H264_MASK
603 		| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
604 		| UVD_SUVD_CGC_GATE__SIT_H264_MASK
605 		| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
606 		| UVD_SUVD_CGC_GATE__SCM_H264_MASK
607 		| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
608 		| UVD_SUVD_CGC_GATE__SDB_H264_MASK
609 		| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
610 		| UVD_SUVD_CGC_GATE__SCLR_MASK
611 		| UVD_SUVD_CGC_GATE__UVD_SC_MASK
612 		| UVD_SUVD_CGC_GATE__ENT_MASK
613 		| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
614 		| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
615 		| UVD_SUVD_CGC_GATE__SITE_MASK
616 		| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
617 		| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
618 		| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
619 		| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
620 		| UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
621 	WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
622 
623 	data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
624 	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
625 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
626 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
627 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
628 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
629 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
630 		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
631 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
632 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
633 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
634 	WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
635 }
636 
vcn_v2_0_clock_gating_dpg_mode(struct amdgpu_device * adev,uint8_t sram_sel,uint8_t indirect)637 static void vcn_v2_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
638 		uint8_t sram_sel, uint8_t indirect)
639 {
640 	uint32_t reg_data = 0;
641 
642 	/* enable sw clock gating control */
643 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
644 		reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
645 	else
646 		reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
647 	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
648 	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
649 	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
650 		 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
651 		 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
652 		 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
653 		 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
654 		 UVD_CGC_CTRL__SYS_MODE_MASK |
655 		 UVD_CGC_CTRL__UDEC_MODE_MASK |
656 		 UVD_CGC_CTRL__MPEG2_MODE_MASK |
657 		 UVD_CGC_CTRL__REGS_MODE_MASK |
658 		 UVD_CGC_CTRL__RBC_MODE_MASK |
659 		 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
660 		 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
661 		 UVD_CGC_CTRL__IDCT_MODE_MASK |
662 		 UVD_CGC_CTRL__MPRD_MODE_MASK |
663 		 UVD_CGC_CTRL__MPC_MODE_MASK |
664 		 UVD_CGC_CTRL__LBSI_MODE_MASK |
665 		 UVD_CGC_CTRL__LRBBM_MODE_MASK |
666 		 UVD_CGC_CTRL__WCB_MODE_MASK |
667 		 UVD_CGC_CTRL__VCPU_MODE_MASK |
668 		 UVD_CGC_CTRL__SCPU_MODE_MASK);
669 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
670 		UVD, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
671 
672 	/* turn off clock gating */
673 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
674 		UVD, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
675 
676 	/* turn on SUVD clock gating */
677 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
678 		UVD, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
679 
680 	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
681 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
682 		UVD, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
683 }
684 
685 /**
686  * vcn_v2_0_enable_clock_gating - enable VCN clock gating
687  *
688  * @adev: amdgpu_device pointer
689  *
690  * Enable clock gating for VCN block
691  */
vcn_v2_0_enable_clock_gating(struct amdgpu_device * adev)692 static void vcn_v2_0_enable_clock_gating(struct amdgpu_device *adev)
693 {
694 	uint32_t data = 0;
695 
696 	if (amdgpu_sriov_vf(adev))
697 		return;
698 
699 	/* enable UVD CGC */
700 	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
701 	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
702 		data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
703 	else
704 		data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
705 	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
706 	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
707 	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
708 
709 	data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
710 	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
711 		| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
712 		| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
713 		| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
714 		| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
715 		| UVD_CGC_CTRL__SYS_MODE_MASK
716 		| UVD_CGC_CTRL__UDEC_MODE_MASK
717 		| UVD_CGC_CTRL__MPEG2_MODE_MASK
718 		| UVD_CGC_CTRL__REGS_MODE_MASK
719 		| UVD_CGC_CTRL__RBC_MODE_MASK
720 		| UVD_CGC_CTRL__LMI_MC_MODE_MASK
721 		| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
722 		| UVD_CGC_CTRL__IDCT_MODE_MASK
723 		| UVD_CGC_CTRL__MPRD_MODE_MASK
724 		| UVD_CGC_CTRL__MPC_MODE_MASK
725 		| UVD_CGC_CTRL__LBSI_MODE_MASK
726 		| UVD_CGC_CTRL__LRBBM_MODE_MASK
727 		| UVD_CGC_CTRL__WCB_MODE_MASK
728 		| UVD_CGC_CTRL__VCPU_MODE_MASK
729 		| UVD_CGC_CTRL__SCPU_MODE_MASK);
730 	WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
731 
732 	data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
733 	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
734 		| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
735 		| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
736 		| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
737 		| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
738 		| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
739 		| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
740 		| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
741 		| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
742 		| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
743 	WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
744 }
745 
vcn_v2_0_disable_static_power_gating(struct amdgpu_device * adev)746 static void vcn_v2_0_disable_static_power_gating(struct amdgpu_device *adev)
747 {
748 	uint32_t data = 0;
749 
750 	if (amdgpu_sriov_vf(adev))
751 		return;
752 
753 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
754 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
755 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
756 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
757 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
758 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
759 			| 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
760 			| 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
761 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
762 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
763 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
764 
765 		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
766 		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS,
767 			UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON_2_0, 0xFFFFF);
768 	} else {
769 		data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
770 			| 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
771 			| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
772 			| 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
773 			| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
774 			| 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
775 			| 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
776 			| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
777 			| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
778 			| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
779 		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
780 		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0,  0xFFFFF);
781 	}
782 
783 	/* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS,
784 	 * UVDU_PWR_STATUS are 0 (power on) */
785 
786 	data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
787 	data &= ~0x103;
788 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
789 		data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
790 			UVD_POWER_STATUS__UVD_PG_EN_MASK;
791 
792 	WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
793 }
794 
vcn_v2_0_enable_static_power_gating(struct amdgpu_device * adev)795 static void vcn_v2_0_enable_static_power_gating(struct amdgpu_device *adev)
796 {
797 	uint32_t data = 0;
798 
799 	if (amdgpu_sriov_vf(adev))
800 		return;
801 
802 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
803 		/* Before power off, this indicator has to be turned on */
804 		data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
805 		data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
806 		data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
807 		WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
808 
809 
810 		data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
811 			| 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
812 			| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
813 			| 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
814 			| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
815 			| 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
816 			| 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
817 			| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
818 			| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
819 			| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT);
820 
821 		WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
822 
823 		data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
824 			| 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
825 			| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
826 			| 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
827 			| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
828 			| 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
829 			| 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
830 			| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
831 			| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
832 			| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT);
833 		SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFF);
834 	}
835 }
836 
vcn_v2_0_start_dpg_mode(struct amdgpu_device * adev,bool indirect)837 static int vcn_v2_0_start_dpg_mode(struct amdgpu_device *adev, bool indirect)
838 {
839 	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
840 	struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
841 	uint32_t rb_bufsz, tmp;
842 
843 	vcn_v2_0_enable_static_power_gating(adev);
844 
845 	/* enable dynamic power gating mode */
846 	tmp = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
847 	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
848 	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
849 	WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, tmp);
850 
851 	if (indirect)
852 		adev->vcn.inst->dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst->dpg_sram_cpu_addr;
853 
854 	/* enable clock gating */
855 	vcn_v2_0_clock_gating_dpg_mode(adev, 0, indirect);
856 
857 	/* enable VCPU clock */
858 	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
859 	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
860 	tmp |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK;
861 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
862 		UVD, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
863 
864 	/* disable master interupt */
865 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
866 		UVD, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
867 
868 	/* setup mmUVD_LMI_CTRL */
869 	tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
870 		UVD_LMI_CTRL__REQ_MODE_MASK |
871 		UVD_LMI_CTRL__CRC_RESET_MASK |
872 		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
873 		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
874 		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
875 		(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
876 		0x00100000L);
877 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
878 		UVD, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
879 
880 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
881 		UVD, 0, mmUVD_MPC_CNTL),
882 		0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
883 
884 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
885 		UVD, 0, mmUVD_MPC_SET_MUXA0),
886 		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
887 		 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
888 		 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
889 		 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
890 
891 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
892 		UVD, 0, mmUVD_MPC_SET_MUXB0),
893 		((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
894 		 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
895 		 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
896 		 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
897 
898 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
899 		UVD, 0, mmUVD_MPC_SET_MUX),
900 		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
901 		 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
902 		 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
903 
904 	vcn_v2_0_mc_resume_dpg_mode(adev, indirect);
905 
906 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
907 		UVD, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
908 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
909 		UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
910 
911 	/* release VCPU reset to boot */
912 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
913 		UVD, 0, mmUVD_SOFT_RESET), 0, 0, indirect);
914 
915 	/* enable LMI MC and UMC channels */
916 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
917 		UVD, 0, mmUVD_LMI_CTRL2),
918 		0x1F << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT, 0, indirect);
919 
920 	/* enable master interrupt */
921 	WREG32_SOC15_DPG_MODE(0, SOC15_DPG_MODE_OFFSET(
922 		UVD, 0, mmUVD_MASTINT_EN),
923 		UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
924 
925 	if (indirect)
926 		amdgpu_vcn_psp_update_sram(adev, 0, 0);
927 
928 	/* force RBC into idle state */
929 	rb_bufsz = order_base_2(ring->ring_size);
930 	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
931 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
932 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
933 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
934 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
935 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
936 
937 	/* Stall DPG before WPTR/RPTR reset */
938 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
939 		UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
940 		~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
941 	fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
942 
943 	/* set the write pointer delay */
944 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
945 
946 	/* set the wb address */
947 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
948 		(upper_32_bits(ring->gpu_addr) >> 2));
949 
950 	/* program the RB_BASE for ring buffer */
951 	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
952 		lower_32_bits(ring->gpu_addr));
953 	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
954 		upper_32_bits(ring->gpu_addr));
955 
956 	/* Initialize the ring buffer's read and write pointers */
957 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
958 
959 	WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
960 
961 	ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
962 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
963 		lower_32_bits(ring->wptr));
964 
965 	fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
966 	/* Unstall DPG */
967 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
968 		0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
969 	return 0;
970 }
971 
vcn_v2_0_start(struct amdgpu_device * adev)972 static int vcn_v2_0_start(struct amdgpu_device *adev)
973 {
974 	volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
975 	struct amdgpu_ring *ring = &adev->vcn.inst->ring_dec;
976 	uint32_t rb_bufsz, tmp;
977 	uint32_t lmi_swap_cntl;
978 	int i, j, r;
979 
980 	if (adev->pm.dpm_enabled)
981 		amdgpu_dpm_enable_uvd(adev, true);
982 
983 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
984 		return vcn_v2_0_start_dpg_mode(adev, adev->vcn.indirect_sram);
985 
986 	vcn_v2_0_disable_static_power_gating(adev);
987 
988 	/* set uvd status busy */
989 	tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
990 	WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
991 
992 	/*SW clock gating */
993 	vcn_v2_0_disable_clock_gating(adev);
994 
995 	/* enable VCPU clock */
996 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
997 		UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
998 
999 	/* disable master interrupt */
1000 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
1001 		~UVD_MASTINT_EN__VCPU_EN_MASK);
1002 
1003 	/* setup mmUVD_LMI_CTRL */
1004 	tmp = RREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL);
1005 	WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL, tmp |
1006 		UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK	|
1007 		UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1008 		UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1009 		UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1010 
1011 	/* setup mmUVD_MPC_CNTL */
1012 	tmp = RREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL);
1013 	tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1014 	tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1015 	WREG32_SOC15(VCN, 0, mmUVD_MPC_CNTL, tmp);
1016 
1017 	/* setup UVD_MPC_SET_MUXA0 */
1018 	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0,
1019 		((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1020 		(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1021 		(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1022 		(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1023 
1024 	/* setup UVD_MPC_SET_MUXB0 */
1025 	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0,
1026 		((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1027 		(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1028 		(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1029 		(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1030 
1031 	/* setup mmUVD_MPC_SET_MUX */
1032 	WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX,
1033 		((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1034 		(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1035 		(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1036 
1037 	vcn_v2_0_mc_resume(adev);
1038 
1039 	/* release VCPU reset to boot */
1040 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
1041 		~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1042 
1043 	/* enable LMI MC and UMC channels */
1044 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
1045 		~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1046 
1047 	tmp = RREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET);
1048 	tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1049 	tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1050 	WREG32_SOC15(VCN, 0, mmUVD_SOFT_RESET, tmp);
1051 
1052 	/* disable byte swapping */
1053 	lmi_swap_cntl = 0;
1054 #ifdef __BIG_ENDIAN
1055 	/* swap (8 in 32) RB and IB */
1056 	lmi_swap_cntl = 0xa;
1057 #endif
1058 	WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
1059 
1060 	for (i = 0; i < 10; ++i) {
1061 		uint32_t status;
1062 
1063 		for (j = 0; j < 100; ++j) {
1064 			status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
1065 			if (status & 2)
1066 				break;
1067 			mdelay(10);
1068 		}
1069 		r = 0;
1070 		if (status & 2)
1071 			break;
1072 
1073 		DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
1074 		WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1075 			UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1076 			~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1077 		mdelay(10);
1078 		WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
1079 			~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1080 		mdelay(10);
1081 		r = -1;
1082 	}
1083 
1084 	if (r) {
1085 		DRM_ERROR("VCN decode not responding, giving up!!!\n");
1086 		return r;
1087 	}
1088 
1089 	/* enable master interrupt */
1090 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
1091 		UVD_MASTINT_EN__VCPU_EN_MASK,
1092 		~UVD_MASTINT_EN__VCPU_EN_MASK);
1093 
1094 	/* clear the busy bit of VCN_STATUS */
1095 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
1096 		~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1097 
1098 	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_VMID, 0);
1099 
1100 	/* force RBC into idle state */
1101 	rb_bufsz = order_base_2(ring->ring_size);
1102 	tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1103 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1104 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1105 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1106 	tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1107 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1108 
1109 	fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1110 	/* program the RB_BASE for ring buffer */
1111 	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1112 		lower_32_bits(ring->gpu_addr));
1113 	WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1114 		upper_32_bits(ring->gpu_addr));
1115 
1116 	/* Initialize the ring buffer's read and write pointers */
1117 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1118 
1119 	ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1120 	WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1121 			lower_32_bits(ring->wptr));
1122 	fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1123 
1124 	fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1125 	ring = &adev->vcn.inst->ring_enc[0];
1126 	WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1127 	WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1128 	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1129 	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1130 	WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1131 	fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1132 
1133 	fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1134 	ring = &adev->vcn.inst->ring_enc[1];
1135 	WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1136 	WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1137 	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1138 	WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1139 	WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1140 	fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1141 
1142 	return 0;
1143 }
1144 
vcn_v2_0_stop_dpg_mode(struct amdgpu_device * adev)1145 static int vcn_v2_0_stop_dpg_mode(struct amdgpu_device *adev)
1146 {
1147 	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1148 	uint32_t tmp;
1149 
1150 	vcn_v2_0_pause_dpg_mode(adev, 0, &state);
1151 	/* Wait for power status to be 1 */
1152 	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
1153 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1154 
1155 	/* wait for read ptr to be equal to write ptr */
1156 	tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1157 	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1158 
1159 	tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1160 	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1161 
1162 	tmp = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1163 	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1164 
1165 	SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 1,
1166 		UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1167 
1168 	/* disable dynamic power gating mode */
1169 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
1170 			~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1171 
1172 	return 0;
1173 }
1174 
vcn_v2_0_stop(struct amdgpu_device * adev)1175 static int vcn_v2_0_stop(struct amdgpu_device *adev)
1176 {
1177 	uint32_t tmp;
1178 	int r;
1179 
1180 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1181 		r = vcn_v2_0_stop_dpg_mode(adev);
1182 		if (r)
1183 			return r;
1184 		goto power_off;
1185 	}
1186 
1187 	/* wait for uvd idle */
1188 	r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1189 	if (r)
1190 		return r;
1191 
1192 	tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1193 		UVD_LMI_STATUS__READ_CLEAN_MASK |
1194 		UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1195 		UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1196 	r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp);
1197 	if (r)
1198 		return r;
1199 
1200 	/* stall UMC channel */
1201 	tmp = RREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2);
1202 	tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1203 	WREG32_SOC15(VCN, 0, mmUVD_LMI_CTRL2, tmp);
1204 
1205 	tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1206 		UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1207 	r = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_LMI_STATUS, tmp, tmp);
1208 	if (r)
1209 		return r;
1210 
1211 	/* disable VCPU clock */
1212 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 0,
1213 		~(UVD_VCPU_CNTL__CLK_EN_MASK));
1214 
1215 	/* reset LMI UMC */
1216 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1217 		UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK,
1218 		~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
1219 
1220 	/* reset LMI */
1221 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1222 		UVD_SOFT_RESET__LMI_SOFT_RESET_MASK,
1223 		~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK);
1224 
1225 	/* reset VCPU */
1226 	WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1227 		UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1228 		~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1229 
1230 	/* clear status */
1231 	WREG32_SOC15(VCN, 0, mmUVD_STATUS, 0);
1232 
1233 	vcn_v2_0_enable_clock_gating(adev);
1234 	vcn_v2_0_enable_static_power_gating(adev);
1235 
1236 power_off:
1237 	if (adev->pm.dpm_enabled)
1238 		amdgpu_dpm_enable_uvd(adev, false);
1239 
1240 	return 0;
1241 }
1242 
vcn_v2_0_pause_dpg_mode(struct amdgpu_device * adev,int inst_idx,struct dpg_pause_state * new_state)1243 static int vcn_v2_0_pause_dpg_mode(struct amdgpu_device *adev,
1244 				int inst_idx, struct dpg_pause_state *new_state)
1245 {
1246 	struct amdgpu_ring *ring;
1247 	uint32_t reg_data = 0;
1248 	int ret_code;
1249 
1250 	/* pause/unpause if state is changed */
1251 	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1252 		DRM_DEBUG("dpg pause state changed %d -> %d",
1253 			adev->vcn.inst[inst_idx].pause_state.fw_based,	new_state->fw_based);
1254 		reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
1255 			(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1256 
1257 		if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1258 			ret_code = SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 0x1,
1259 				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1260 
1261 			if (!ret_code) {
1262 				volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
1263 				/* pause DPG */
1264 				reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1265 				WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1266 
1267 				/* wait for ACK */
1268 				SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
1269 					   UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1270 					   UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1271 
1272 				/* Stall DPG before WPTR/RPTR reset */
1273 				WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
1274 					   UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1275 					   ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1276 				/* Restore */
1277 				fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1278 				ring = &adev->vcn.inst->ring_enc[0];
1279 				ring->wptr = 0;
1280 				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1281 				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1282 				WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1283 				WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1284 				WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1285 				fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1286 
1287 				fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1288 				ring = &adev->vcn.inst->ring_enc[1];
1289 				ring->wptr = 0;
1290 				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1291 				WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1292 				WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1293 				WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1294 				WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1295 				fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1296 
1297 				fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1298 				WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1299 					   RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
1300 				fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1301 				/* Unstall DPG */
1302 				WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
1303 					   0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1304 
1305 				SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1306 					   UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1307 					   UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1308 			}
1309 		} else {
1310 			/* unpause dpg, no need to wait */
1311 			reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1312 			WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1313 		}
1314 		adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1315 	}
1316 
1317 	return 0;
1318 }
1319 
vcn_v2_0_is_idle(void * handle)1320 static bool vcn_v2_0_is_idle(void *handle)
1321 {
1322 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1323 
1324 	return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == UVD_STATUS__IDLE);
1325 }
1326 
vcn_v2_0_wait_for_idle(struct amdgpu_ip_block * ip_block)1327 static int vcn_v2_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
1328 {
1329 	struct amdgpu_device *adev = ip_block->adev;
1330 	int ret;
1331 
1332 	ret = SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE,
1333 		UVD_STATUS__IDLE);
1334 
1335 	return ret;
1336 }
1337 
vcn_v2_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)1338 static int vcn_v2_0_set_clockgating_state(void *handle,
1339 					  enum amd_clockgating_state state)
1340 {
1341 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1342 	bool enable = (state == AMD_CG_STATE_GATE);
1343 
1344 	if (amdgpu_sriov_vf(adev))
1345 		return 0;
1346 
1347 	if (enable) {
1348 		/* wait for STATUS to clear */
1349 		if (!vcn_v2_0_is_idle(handle))
1350 			return -EBUSY;
1351 		vcn_v2_0_enable_clock_gating(adev);
1352 	} else {
1353 		/* disable HW gating and enable Sw gating */
1354 		vcn_v2_0_disable_clock_gating(adev);
1355 	}
1356 	return 0;
1357 }
1358 
1359 /**
1360  * vcn_v2_0_dec_ring_get_rptr - get read pointer
1361  *
1362  * @ring: amdgpu_ring pointer
1363  *
1364  * Returns the current hardware read pointer
1365  */
vcn_v2_0_dec_ring_get_rptr(struct amdgpu_ring * ring)1366 static uint64_t vcn_v2_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1367 {
1368 	struct amdgpu_device *adev = ring->adev;
1369 
1370 	return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1371 }
1372 
1373 /**
1374  * vcn_v2_0_dec_ring_get_wptr - get write pointer
1375  *
1376  * @ring: amdgpu_ring pointer
1377  *
1378  * Returns the current hardware write pointer
1379  */
vcn_v2_0_dec_ring_get_wptr(struct amdgpu_ring * ring)1380 static uint64_t vcn_v2_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1381 {
1382 	struct amdgpu_device *adev = ring->adev;
1383 
1384 	if (ring->use_doorbell)
1385 		return *ring->wptr_cpu_addr;
1386 	else
1387 		return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
1388 }
1389 
1390 /**
1391  * vcn_v2_0_dec_ring_set_wptr - set write pointer
1392  *
1393  * @ring: amdgpu_ring pointer
1394  *
1395  * Commits the write pointer to the hardware
1396  */
vcn_v2_0_dec_ring_set_wptr(struct amdgpu_ring * ring)1397 static void vcn_v2_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1398 {
1399 	struct amdgpu_device *adev = ring->adev;
1400 
1401 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1402 		WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2,
1403 			lower_32_bits(ring->wptr) | 0x80000000);
1404 
1405 	if (ring->use_doorbell) {
1406 		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1407 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1408 	} else {
1409 		WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1410 	}
1411 }
1412 
1413 /**
1414  * vcn_v2_0_dec_ring_insert_start - insert a start command
1415  *
1416  * @ring: amdgpu_ring pointer
1417  *
1418  * Write a start command to the ring.
1419  */
vcn_v2_0_dec_ring_insert_start(struct amdgpu_ring * ring)1420 void vcn_v2_0_dec_ring_insert_start(struct amdgpu_ring *ring)
1421 {
1422 	struct amdgpu_device *adev = ring->adev;
1423 
1424 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1425 	amdgpu_ring_write(ring, 0);
1426 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1427 	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
1428 }
1429 
1430 /**
1431  * vcn_v2_0_dec_ring_insert_end - insert a end command
1432  *
1433  * @ring: amdgpu_ring pointer
1434  *
1435  * Write a end command to the ring.
1436  */
vcn_v2_0_dec_ring_insert_end(struct amdgpu_ring * ring)1437 void vcn_v2_0_dec_ring_insert_end(struct amdgpu_ring *ring)
1438 {
1439 	struct amdgpu_device *adev = ring->adev;
1440 
1441 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1442 	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_END << 1));
1443 }
1444 
1445 /**
1446  * vcn_v2_0_dec_ring_insert_nop - insert a nop command
1447  *
1448  * @ring: amdgpu_ring pointer
1449  * @count: the number of NOP packets to insert
1450  *
1451  * Write a nop command to the ring.
1452  */
vcn_v2_0_dec_ring_insert_nop(struct amdgpu_ring * ring,uint32_t count)1453 void vcn_v2_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1454 {
1455 	struct amdgpu_device *adev = ring->adev;
1456 	int i;
1457 
1458 	WARN_ON(ring->wptr % 2 || count % 2);
1459 
1460 	for (i = 0; i < count / 2; i++) {
1461 		amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.nop, 0));
1462 		amdgpu_ring_write(ring, 0);
1463 	}
1464 }
1465 
1466 /**
1467  * vcn_v2_0_dec_ring_emit_fence - emit an fence & trap command
1468  *
1469  * @ring: amdgpu_ring pointer
1470  * @addr: address
1471  * @seq: sequence number
1472  * @flags: fence related flags
1473  *
1474  * Write a fence and a trap command to the ring.
1475  */
vcn_v2_0_dec_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)1476 void vcn_v2_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1477 				unsigned flags)
1478 {
1479 	struct amdgpu_device *adev = ring->adev;
1480 
1481 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1482 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.context_id, 0));
1483 	amdgpu_ring_write(ring, seq);
1484 
1485 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1486 	amdgpu_ring_write(ring, addr & 0xffffffff);
1487 
1488 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1489 	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
1490 
1491 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1492 	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_FENCE << 1));
1493 
1494 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1495 	amdgpu_ring_write(ring, 0);
1496 
1497 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1498 	amdgpu_ring_write(ring, 0);
1499 
1500 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1501 
1502 	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_TRAP << 1));
1503 }
1504 
1505 /**
1506  * vcn_v2_0_dec_ring_emit_ib - execute indirect buffer
1507  *
1508  * @ring: amdgpu_ring pointer
1509  * @job: job to retrieve vmid from
1510  * @ib: indirect buffer to execute
1511  * @flags: unused
1512  *
1513  * Write ring commands to execute the indirect buffer
1514  */
vcn_v2_0_dec_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)1515 void vcn_v2_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
1516 			       struct amdgpu_job *job,
1517 			       struct amdgpu_ib *ib,
1518 			       uint32_t flags)
1519 {
1520 	struct amdgpu_device *adev = ring->adev;
1521 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1522 
1523 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.ib_vmid, 0));
1524 	amdgpu_ring_write(ring, vmid);
1525 
1526 	amdgpu_ring_write(ring,	PACKET0(adev->vcn.internal.ib_bar_low, 0));
1527 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1528 	amdgpu_ring_write(ring,	PACKET0(adev->vcn.internal.ib_bar_high, 0));
1529 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1530 	amdgpu_ring_write(ring,	PACKET0(adev->vcn.internal.ib_size, 0));
1531 	amdgpu_ring_write(ring, ib->length_dw);
1532 }
1533 
vcn_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)1534 void vcn_v2_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1535 				uint32_t val, uint32_t mask)
1536 {
1537 	struct amdgpu_device *adev = ring->adev;
1538 
1539 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1540 	amdgpu_ring_write(ring, reg << 2);
1541 
1542 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1543 	amdgpu_ring_write(ring, val);
1544 
1545 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.gp_scratch8, 0));
1546 	amdgpu_ring_write(ring, mask);
1547 
1548 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1549 
1550 	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_REG_READ_COND_WAIT << 1));
1551 }
1552 
vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)1553 void vcn_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
1554 				unsigned vmid, uint64_t pd_addr)
1555 {
1556 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1557 	uint32_t data0, data1, mask;
1558 
1559 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1560 
1561 	/* wait for register write */
1562 	data0 = hub->ctx0_ptb_addr_lo32 + vmid * hub->ctx_addr_distance;
1563 	data1 = lower_32_bits(pd_addr);
1564 	mask = 0xffffffff;
1565 	vcn_v2_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
1566 }
1567 
vcn_v2_0_dec_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)1568 void vcn_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring,
1569 				uint32_t reg, uint32_t val)
1570 {
1571 	struct amdgpu_device *adev = ring->adev;
1572 
1573 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data0, 0));
1574 	amdgpu_ring_write(ring, reg << 2);
1575 
1576 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.data1, 0));
1577 	amdgpu_ring_write(ring, val);
1578 
1579 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1580 
1581 	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_WRITE_REG << 1));
1582 }
1583 
1584 /**
1585  * vcn_v2_0_enc_ring_get_rptr - get enc read pointer
1586  *
1587  * @ring: amdgpu_ring pointer
1588  *
1589  * Returns the current hardware enc read pointer
1590  */
vcn_v2_0_enc_ring_get_rptr(struct amdgpu_ring * ring)1591 static uint64_t vcn_v2_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1592 {
1593 	struct amdgpu_device *adev = ring->adev;
1594 
1595 	if (ring == &adev->vcn.inst->ring_enc[0])
1596 		return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
1597 	else
1598 		return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
1599 }
1600 
1601  /**
1602  * vcn_v2_0_enc_ring_get_wptr - get enc write pointer
1603  *
1604  * @ring: amdgpu_ring pointer
1605  *
1606  * Returns the current hardware enc write pointer
1607  */
vcn_v2_0_enc_ring_get_wptr(struct amdgpu_ring * ring)1608 static uint64_t vcn_v2_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1609 {
1610 	struct amdgpu_device *adev = ring->adev;
1611 
1612 	if (ring == &adev->vcn.inst->ring_enc[0]) {
1613 		if (ring->use_doorbell)
1614 			return *ring->wptr_cpu_addr;
1615 		else
1616 			return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1617 	} else {
1618 		if (ring->use_doorbell)
1619 			return *ring->wptr_cpu_addr;
1620 		else
1621 			return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1622 	}
1623 }
1624 
1625  /**
1626  * vcn_v2_0_enc_ring_set_wptr - set enc write pointer
1627  *
1628  * @ring: amdgpu_ring pointer
1629  *
1630  * Commits the enc write pointer to the hardware
1631  */
vcn_v2_0_enc_ring_set_wptr(struct amdgpu_ring * ring)1632 static void vcn_v2_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1633 {
1634 	struct amdgpu_device *adev = ring->adev;
1635 
1636 	if (ring == &adev->vcn.inst->ring_enc[0]) {
1637 		if (ring->use_doorbell) {
1638 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1639 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1640 		} else {
1641 			WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1642 		}
1643 	} else {
1644 		if (ring->use_doorbell) {
1645 			*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1646 			WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1647 		} else {
1648 			WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1649 		}
1650 	}
1651 }
1652 
1653 /**
1654  * vcn_v2_0_enc_ring_emit_fence - emit an enc fence & trap command
1655  *
1656  * @ring: amdgpu_ring pointer
1657  * @addr: address
1658  * @seq: sequence number
1659  * @flags: fence related flags
1660  *
1661  * Write enc a fence and a trap command to the ring.
1662  */
vcn_v2_0_enc_ring_emit_fence(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)1663 void vcn_v2_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1664 				u64 seq, unsigned flags)
1665 {
1666 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1667 
1668 	amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
1669 	amdgpu_ring_write(ring, addr);
1670 	amdgpu_ring_write(ring, upper_32_bits(addr));
1671 	amdgpu_ring_write(ring, seq);
1672 	amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
1673 }
1674 
vcn_v2_0_enc_ring_insert_end(struct amdgpu_ring * ring)1675 void vcn_v2_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1676 {
1677 	amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1678 }
1679 
1680 /**
1681  * vcn_v2_0_enc_ring_emit_ib - enc execute indirect buffer
1682  *
1683  * @ring: amdgpu_ring pointer
1684  * @job: job to retrive vmid from
1685  * @ib: indirect buffer to execute
1686  * @flags: unused
1687  *
1688  * Write enc ring commands to execute the indirect buffer
1689  */
vcn_v2_0_enc_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)1690 void vcn_v2_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1691 			       struct amdgpu_job *job,
1692 			       struct amdgpu_ib *ib,
1693 			       uint32_t flags)
1694 {
1695 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1696 
1697 	amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1698 	amdgpu_ring_write(ring, vmid);
1699 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1700 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1701 	amdgpu_ring_write(ring, ib->length_dw);
1702 }
1703 
vcn_v2_0_enc_ring_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)1704 void vcn_v2_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1705 				uint32_t val, uint32_t mask)
1706 {
1707 	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1708 	amdgpu_ring_write(ring, reg << 2);
1709 	amdgpu_ring_write(ring, mask);
1710 	amdgpu_ring_write(ring, val);
1711 }
1712 
vcn_v2_0_enc_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned int vmid,uint64_t pd_addr)1713 void vcn_v2_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1714 				unsigned int vmid, uint64_t pd_addr)
1715 {
1716 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
1717 
1718 	pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1719 
1720 	/* wait for reg writes */
1721 	vcn_v2_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 +
1722 					vmid * hub->ctx_addr_distance,
1723 					lower_32_bits(pd_addr), 0xffffffff);
1724 }
1725 
vcn_v2_0_enc_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)1726 void vcn_v2_0_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val)
1727 {
1728 	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1729 	amdgpu_ring_write(ring,	reg << 2);
1730 	amdgpu_ring_write(ring, val);
1731 }
1732 
vcn_v2_0_set_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)1733 static int vcn_v2_0_set_interrupt_state(struct amdgpu_device *adev,
1734 					struct amdgpu_irq_src *source,
1735 					unsigned type,
1736 					enum amdgpu_interrupt_state state)
1737 {
1738 	return 0;
1739 }
1740 
vcn_v2_0_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1741 static int vcn_v2_0_process_interrupt(struct amdgpu_device *adev,
1742 				      struct amdgpu_irq_src *source,
1743 				      struct amdgpu_iv_entry *entry)
1744 {
1745 	DRM_DEBUG("IH: VCN TRAP\n");
1746 
1747 	switch (entry->src_id) {
1748 	case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
1749 		amdgpu_fence_process(&adev->vcn.inst->ring_dec);
1750 		break;
1751 	case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1752 		amdgpu_fence_process(&adev->vcn.inst->ring_enc[0]);
1753 		break;
1754 	case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
1755 		amdgpu_fence_process(&adev->vcn.inst->ring_enc[1]);
1756 		break;
1757 	default:
1758 		DRM_ERROR("Unhandled interrupt: %d %d\n",
1759 			  entry->src_id, entry->src_data[0]);
1760 		break;
1761 	}
1762 
1763 	return 0;
1764 }
1765 
vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring * ring)1766 int vcn_v2_0_dec_ring_test_ring(struct amdgpu_ring *ring)
1767 {
1768 	struct amdgpu_device *adev = ring->adev;
1769 	uint32_t tmp = 0;
1770 	unsigned i;
1771 	int r;
1772 
1773 	if (amdgpu_sriov_vf(adev))
1774 		return 0;
1775 
1776 	WREG32(adev->vcn.inst[ring->me].external.scratch9, 0xCAFEDEAD);
1777 	r = amdgpu_ring_alloc(ring, 4);
1778 	if (r)
1779 		return r;
1780 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.cmd, 0));
1781 	amdgpu_ring_write(ring, VCN_DEC_KMD_CMD | (VCN_DEC_CMD_PACKET_START << 1));
1782 	amdgpu_ring_write(ring, PACKET0(adev->vcn.internal.scratch9, 0));
1783 	amdgpu_ring_write(ring, 0xDEADBEEF);
1784 	amdgpu_ring_commit(ring);
1785 	for (i = 0; i < adev->usec_timeout; i++) {
1786 		tmp = RREG32(adev->vcn.inst[ring->me].external.scratch9);
1787 		if (tmp == 0xDEADBEEF)
1788 			break;
1789 		udelay(1);
1790 	}
1791 
1792 	if (i >= adev->usec_timeout)
1793 		r = -ETIMEDOUT;
1794 
1795 	return r;
1796 }
1797 
1798 
vcn_v2_0_set_powergating_state(void * handle,enum amd_powergating_state state)1799 static int vcn_v2_0_set_powergating_state(void *handle,
1800 					  enum amd_powergating_state state)
1801 {
1802 	/* This doesn't actually powergate the VCN block.
1803 	 * That's done in the dpm code via the SMC.  This
1804 	 * just re-inits the block as necessary.  The actual
1805 	 * gating still happens in the dpm code.  We should
1806 	 * revisit this when there is a cleaner line between
1807 	 * the smc and the hw blocks
1808 	 */
1809 	int ret;
1810 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1811 
1812 	if (amdgpu_sriov_vf(adev)) {
1813 		adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
1814 		return 0;
1815 	}
1816 
1817 	if (state == adev->vcn.cur_state)
1818 		return 0;
1819 
1820 	if (state == AMD_PG_STATE_GATE)
1821 		ret = vcn_v2_0_stop(adev);
1822 	else
1823 		ret = vcn_v2_0_start(adev);
1824 
1825 	if (!ret)
1826 		adev->vcn.cur_state = state;
1827 	return ret;
1828 }
1829 
vcn_v2_0_start_mmsch(struct amdgpu_device * adev,struct amdgpu_mm_table * table)1830 static int vcn_v2_0_start_mmsch(struct amdgpu_device *adev,
1831 				struct amdgpu_mm_table *table)
1832 {
1833 	uint32_t data = 0, loop;
1834 	uint64_t addr = table->gpu_addr;
1835 	struct mmsch_v2_0_init_header *header;
1836 	uint32_t size;
1837 	int i;
1838 
1839 	header = (struct mmsch_v2_0_init_header *)table->cpu_addr;
1840 	size = header->header_size + header->vcn_table_size;
1841 
1842 	/* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1843 	 * of memory descriptor location
1844 	 */
1845 	WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));
1846 	WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));
1847 
1848 	/* 2, update vmid of descriptor */
1849 	data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_VMID);
1850 	data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1851 	/* use domain0 for MM scheduler */
1852 	data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1853 	WREG32_SOC15(UVD, 0, mmMMSCH_VF_VMID, data);
1854 
1855 	/* 3, notify mmsch about the size of this descriptor */
1856 	WREG32_SOC15(UVD, 0, mmMMSCH_VF_CTX_SIZE, size);
1857 
1858 	/* 4, set resp to zero */
1859 	WREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1860 
1861 	adev->vcn.inst->ring_dec.wptr = 0;
1862 	adev->vcn.inst->ring_dec.wptr_old = 0;
1863 	vcn_v2_0_dec_ring_set_wptr(&adev->vcn.inst->ring_dec);
1864 
1865 	for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
1866 		adev->vcn.inst->ring_enc[i].wptr = 0;
1867 		adev->vcn.inst->ring_enc[i].wptr_old = 0;
1868 		vcn_v2_0_enc_ring_set_wptr(&adev->vcn.inst->ring_enc[i]);
1869 	}
1870 
1871 	/* 5, kick off the initialization and wait until
1872 	 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero
1873 	 */
1874 	WREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001);
1875 
1876 	data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP);
1877 	loop = 1000;
1878 	while ((data & 0x10000002) != 0x10000002) {
1879 		udelay(10);
1880 		data = RREG32_SOC15(UVD, 0, mmMMSCH_VF_MAILBOX_RESP);
1881 		loop--;
1882 		if (!loop)
1883 			break;
1884 	}
1885 
1886 	if (!loop) {
1887 		DRM_ERROR("failed to init MMSCH, " \
1888 			"mmMMSCH_VF_MAILBOX_RESP = 0x%08x\n", data);
1889 		return -EBUSY;
1890 	}
1891 
1892 	return 0;
1893 }
1894 
vcn_v2_0_start_sriov(struct amdgpu_device * adev)1895 static int vcn_v2_0_start_sriov(struct amdgpu_device *adev)
1896 {
1897 	int r;
1898 	uint32_t tmp;
1899 	struct amdgpu_ring *ring;
1900 	uint32_t offset, size;
1901 	uint32_t table_size = 0;
1902 	struct mmsch_v2_0_cmd_direct_write direct_wt = { {0} };
1903 	struct mmsch_v2_0_cmd_direct_read_modify_write direct_rd_mod_wt = { {0} };
1904 	struct mmsch_v2_0_cmd_end end = { {0} };
1905 	struct mmsch_v2_0_init_header *header;
1906 	uint32_t *init_table = adev->virt.mm_table.cpu_addr;
1907 	uint8_t i = 0;
1908 
1909 	header = (struct mmsch_v2_0_init_header *)init_table;
1910 	direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
1911 	direct_rd_mod_wt.cmd_header.command_type =
1912 		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1913 	end.cmd_header.command_type = MMSCH_COMMAND__END;
1914 
1915 	if (header->vcn_table_offset == 0 && header->vcn_table_size == 0) {
1916 		header->version = MMSCH_VERSION;
1917 		header->header_size = sizeof(struct mmsch_v2_0_init_header) >> 2;
1918 
1919 		header->vcn_table_offset = header->header_size;
1920 
1921 		init_table += header->vcn_table_offset;
1922 
1923 		size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw[0]->size + 4);
1924 
1925 		MMSCH_V2_0_INSERT_DIRECT_RD_MOD_WT(
1926 			SOC15_REG_OFFSET(UVD, i, mmUVD_STATUS),
1927 			0xFFFFFFFF, 0x00000004);
1928 
1929 		/* mc resume*/
1930 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1931 			MMSCH_V2_0_INSERT_DIRECT_WT(
1932 				SOC15_REG_OFFSET(UVD, i,
1933 					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1934 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo);
1935 			MMSCH_V2_0_INSERT_DIRECT_WT(
1936 				SOC15_REG_OFFSET(UVD, i,
1937 					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1938 				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi);
1939 			offset = 0;
1940 		} else {
1941 			MMSCH_V2_0_INSERT_DIRECT_WT(
1942 				SOC15_REG_OFFSET(UVD, i,
1943 					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1944 				lower_32_bits(adev->vcn.inst->gpu_addr));
1945 			MMSCH_V2_0_INSERT_DIRECT_WT(
1946 				SOC15_REG_OFFSET(UVD, i,
1947 					mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1948 				upper_32_bits(adev->vcn.inst->gpu_addr));
1949 			offset = size;
1950 		}
1951 
1952 		MMSCH_V2_0_INSERT_DIRECT_WT(
1953 			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET0),
1954 			0);
1955 		MMSCH_V2_0_INSERT_DIRECT_WT(
1956 			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE0),
1957 			size);
1958 
1959 		MMSCH_V2_0_INSERT_DIRECT_WT(
1960 			SOC15_REG_OFFSET(UVD, i,
1961 				mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1962 			lower_32_bits(adev->vcn.inst->gpu_addr + offset));
1963 		MMSCH_V2_0_INSERT_DIRECT_WT(
1964 			SOC15_REG_OFFSET(UVD, i,
1965 				mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1966 			upper_32_bits(adev->vcn.inst->gpu_addr + offset));
1967 		MMSCH_V2_0_INSERT_DIRECT_WT(
1968 			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET1),
1969 			0);
1970 		MMSCH_V2_0_INSERT_DIRECT_WT(
1971 			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE1),
1972 			AMDGPU_VCN_STACK_SIZE);
1973 
1974 		MMSCH_V2_0_INSERT_DIRECT_WT(
1975 			SOC15_REG_OFFSET(UVD, i,
1976 				mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1977 			lower_32_bits(adev->vcn.inst->gpu_addr + offset +
1978 				AMDGPU_VCN_STACK_SIZE));
1979 		MMSCH_V2_0_INSERT_DIRECT_WT(
1980 			SOC15_REG_OFFSET(UVD, i,
1981 				mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1982 			upper_32_bits(adev->vcn.inst->gpu_addr + offset +
1983 				AMDGPU_VCN_STACK_SIZE));
1984 		MMSCH_V2_0_INSERT_DIRECT_WT(
1985 			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_OFFSET2),
1986 			0);
1987 		MMSCH_V2_0_INSERT_DIRECT_WT(
1988 			SOC15_REG_OFFSET(UVD, i, mmUVD_VCPU_CACHE_SIZE2),
1989 			AMDGPU_VCN_CONTEXT_SIZE);
1990 
1991 		for (r = 0; r < adev->vcn.num_enc_rings; ++r) {
1992 			ring = &adev->vcn.inst->ring_enc[r];
1993 			ring->wptr = 0;
1994 			MMSCH_V2_0_INSERT_DIRECT_WT(
1995 				SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_LO),
1996 				lower_32_bits(ring->gpu_addr));
1997 			MMSCH_V2_0_INSERT_DIRECT_WT(
1998 				SOC15_REG_OFFSET(UVD, i, mmUVD_RB_BASE_HI),
1999 				upper_32_bits(ring->gpu_addr));
2000 			MMSCH_V2_0_INSERT_DIRECT_WT(
2001 				SOC15_REG_OFFSET(UVD, i, mmUVD_RB_SIZE),
2002 				ring->ring_size / 4);
2003 		}
2004 
2005 		ring = &adev->vcn.inst->ring_dec;
2006 		ring->wptr = 0;
2007 		MMSCH_V2_0_INSERT_DIRECT_WT(
2008 			SOC15_REG_OFFSET(UVD, i,
2009 				mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
2010 			lower_32_bits(ring->gpu_addr));
2011 		MMSCH_V2_0_INSERT_DIRECT_WT(
2012 			SOC15_REG_OFFSET(UVD, i,
2013 				mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
2014 			upper_32_bits(ring->gpu_addr));
2015 		/* force RBC into idle state */
2016 		tmp = order_base_2(ring->ring_size);
2017 		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
2018 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
2019 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
2020 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
2021 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
2022 		MMSCH_V2_0_INSERT_DIRECT_WT(
2023 			SOC15_REG_OFFSET(UVD, i, mmUVD_RBC_RB_CNTL), tmp);
2024 
2025 		/* add end packet */
2026 		tmp = sizeof(struct mmsch_v2_0_cmd_end);
2027 		memcpy((void *)init_table, &end, tmp);
2028 		table_size += (tmp / 4);
2029 		header->vcn_table_size = table_size;
2030 
2031 	}
2032 	return vcn_v2_0_start_mmsch(adev, &adev->virt.mm_table);
2033 }
2034 
vcn_v2_0_print_ip_state(struct amdgpu_ip_block * ip_block,struct drm_printer * p)2035 static void vcn_v2_0_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
2036 {
2037 	struct amdgpu_device *adev = ip_block->adev;
2038 	int i, j;
2039 	uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_0);
2040 	uint32_t inst_off, is_powered;
2041 
2042 	if (!adev->vcn.ip_dump)
2043 		return;
2044 
2045 	drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst);
2046 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2047 		if (adev->vcn.harvest_config & (1 << i)) {
2048 			drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i);
2049 			continue;
2050 		}
2051 
2052 		inst_off = i * reg_count;
2053 		is_powered = (adev->vcn.ip_dump[inst_off] &
2054 				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;
2055 
2056 		if (is_powered) {
2057 			drm_printf(p, "\nActive Instance:VCN%d\n", i);
2058 			for (j = 0; j < reg_count; j++)
2059 				drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_2_0[j].reg_name,
2060 					   adev->vcn.ip_dump[inst_off + j]);
2061 		} else {
2062 			drm_printf(p, "\nInactive Instance:VCN%d\n", i);
2063 		}
2064 	}
2065 }
2066 
vcn_v2_0_dump_ip_state(struct amdgpu_ip_block * ip_block)2067 static void vcn_v2_0_dump_ip_state(struct amdgpu_ip_block *ip_block)
2068 {
2069 	struct amdgpu_device *adev = ip_block->adev;
2070 	int i, j;
2071 	bool is_powered;
2072 	uint32_t inst_off;
2073 	uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_0);
2074 
2075 	if (!adev->vcn.ip_dump)
2076 		return;
2077 
2078 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2079 		if (adev->vcn.harvest_config & (1 << i))
2080 			continue;
2081 
2082 		inst_off = i * reg_count;
2083 		/* mmUVD_POWER_STATUS is always readable and is first element of the array */
2084 		adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, i, mmUVD_POWER_STATUS);
2085 		is_powered = (adev->vcn.ip_dump[inst_off] &
2086 				UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;
2087 
2088 		if (is_powered)
2089 			for (j = 1; j < reg_count; j++)
2090 				adev->vcn.ip_dump[inst_off + j] =
2091 					RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_2_0[j], i));
2092 	}
2093 }
2094 
2095 static const struct amd_ip_funcs vcn_v2_0_ip_funcs = {
2096 	.name = "vcn_v2_0",
2097 	.early_init = vcn_v2_0_early_init,
2098 	.sw_init = vcn_v2_0_sw_init,
2099 	.sw_fini = vcn_v2_0_sw_fini,
2100 	.hw_init = vcn_v2_0_hw_init,
2101 	.hw_fini = vcn_v2_0_hw_fini,
2102 	.suspend = vcn_v2_0_suspend,
2103 	.resume = vcn_v2_0_resume,
2104 	.is_idle = vcn_v2_0_is_idle,
2105 	.wait_for_idle = vcn_v2_0_wait_for_idle,
2106 	.set_clockgating_state = vcn_v2_0_set_clockgating_state,
2107 	.set_powergating_state = vcn_v2_0_set_powergating_state,
2108 	.dump_ip_state = vcn_v2_0_dump_ip_state,
2109 	.print_ip_state = vcn_v2_0_print_ip_state,
2110 };
2111 
2112 static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = {
2113 	.type = AMDGPU_RING_TYPE_VCN_DEC,
2114 	.align_mask = 0xf,
2115 	.secure_submission_supported = true,
2116 	.get_rptr = vcn_v2_0_dec_ring_get_rptr,
2117 	.get_wptr = vcn_v2_0_dec_ring_get_wptr,
2118 	.set_wptr = vcn_v2_0_dec_ring_set_wptr,
2119 	.emit_frame_size =
2120 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2121 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2122 		8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
2123 		14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
2124 		6,
2125 	.emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
2126 	.emit_ib = vcn_v2_0_dec_ring_emit_ib,
2127 	.emit_fence = vcn_v2_0_dec_ring_emit_fence,
2128 	.emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
2129 	.test_ring = vcn_v2_0_dec_ring_test_ring,
2130 	.test_ib = amdgpu_vcn_dec_ring_test_ib,
2131 	.insert_nop = vcn_v2_0_dec_ring_insert_nop,
2132 	.insert_start = vcn_v2_0_dec_ring_insert_start,
2133 	.insert_end = vcn_v2_0_dec_ring_insert_end,
2134 	.pad_ib = amdgpu_ring_generic_pad_ib,
2135 	.begin_use = amdgpu_vcn_ring_begin_use,
2136 	.end_use = amdgpu_vcn_ring_end_use,
2137 	.emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
2138 	.emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
2139 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2140 };
2141 
2142 static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = {
2143 	.type = AMDGPU_RING_TYPE_VCN_ENC,
2144 	.align_mask = 0x3f,
2145 	.nop = VCN_ENC_CMD_NO_OP,
2146 	.get_rptr = vcn_v2_0_enc_ring_get_rptr,
2147 	.get_wptr = vcn_v2_0_enc_ring_get_wptr,
2148 	.set_wptr = vcn_v2_0_enc_ring_set_wptr,
2149 	.emit_frame_size =
2150 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2151 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2152 		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2153 		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2154 		1, /* vcn_v2_0_enc_ring_insert_end */
2155 	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2156 	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
2157 	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
2158 	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2159 	.test_ring = amdgpu_vcn_enc_ring_test_ring,
2160 	.test_ib = amdgpu_vcn_enc_ring_test_ib,
2161 	.insert_nop = amdgpu_ring_insert_nop,
2162 	.insert_end = vcn_v2_0_enc_ring_insert_end,
2163 	.pad_ib = amdgpu_ring_generic_pad_ib,
2164 	.begin_use = amdgpu_vcn_ring_begin_use,
2165 	.end_use = amdgpu_vcn_ring_end_use,
2166 	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2167 	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2168 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2169 };
2170 
vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device * adev)2171 static void vcn_v2_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2172 {
2173 	adev->vcn.inst->ring_dec.funcs = &vcn_v2_0_dec_ring_vm_funcs;
2174 }
2175 
vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device * adev)2176 static void vcn_v2_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2177 {
2178 	int i;
2179 
2180 	for (i = 0; i < adev->vcn.num_enc_rings; ++i)
2181 		adev->vcn.inst->ring_enc[i].funcs = &vcn_v2_0_enc_ring_vm_funcs;
2182 }
2183 
2184 static const struct amdgpu_irq_src_funcs vcn_v2_0_irq_funcs = {
2185 	.set = vcn_v2_0_set_interrupt_state,
2186 	.process = vcn_v2_0_process_interrupt,
2187 };
2188 
vcn_v2_0_set_irq_funcs(struct amdgpu_device * adev)2189 static void vcn_v2_0_set_irq_funcs(struct amdgpu_device *adev)
2190 {
2191 	adev->vcn.inst->irq.num_types = adev->vcn.num_enc_rings + 1;
2192 	adev->vcn.inst->irq.funcs = &vcn_v2_0_irq_funcs;
2193 }
2194 
2195 const struct amdgpu_ip_block_version vcn_v2_0_ip_block =
2196 {
2197 		.type = AMD_IP_BLOCK_TYPE_VCN,
2198 		.major = 2,
2199 		.minor = 0,
2200 		.rev = 0,
2201 		.funcs = &vcn_v2_0_ip_funcs,
2202 };
2203