xref: /linux/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 /*
2  * Copyright 2023 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include "amdgpu.h"
25 #include "amdgpu_ip.h"
26 #include "amdgpu_jpeg.h"
27 #include "amdgpu_pm.h"
28 #include "soc15.h"
29 #include "soc15d.h"
30 #include "jpeg_v2_0.h"
31 #include "jpeg_v4_0_5.h"
32 #include "mmsch_v4_0.h"
33 
34 #include "vcn/vcn_4_0_5_offset.h"
35 #include "vcn/vcn_4_0_5_sh_mask.h"
36 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
37 
38 #define mmUVD_DPG_LMA_CTL						regUVD_DPG_LMA_CTL
39 #define mmUVD_DPG_LMA_CTL_BASE_IDX					regUVD_DPG_LMA_CTL_BASE_IDX
40 #define mmUVD_DPG_LMA_DATA						regUVD_DPG_LMA_DATA
41 #define mmUVD_DPG_LMA_DATA_BASE_IDX					regUVD_DPG_LMA_DATA_BASE_IDX
42 
43 #define regUVD_JPEG_PITCH_INTERNAL_OFFSET		0x401f
44 #define regJPEG_DEC_GFX10_ADDR_CONFIG_INTERNAL_OFFSET	0x4026
45 #define regJPEG_SYS_INT_EN_INTERNAL_OFFSET		0x4141
46 #define regJPEG_CGC_CTRL_INTERNAL_OFFSET		0x4161
47 #define regJPEG_CGC_GATE_INTERNAL_OFFSET		0x4160
48 #define regUVD_NO_OP_INTERNAL_OFFSET			0x0029
49 
50 static const struct amdgpu_hwip_reg_entry jpeg_reg_list_4_0_5[] = {
51 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_POWER_STATUS),
52 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_INT_STAT),
53 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC_RB_RPTR),
54 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC_RB_WPTR),
55 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC_RB_CNTL),
56 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC_RB_SIZE),
57 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JRBC_STATUS),
58 	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_ADDR_MODE),
59 	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_GFX10_ADDR_CONFIG),
60 	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_Y_GFX10_TILING_SURFACE),
61 	SOC15_REG_ENTRY_STR(JPEG, 0, regJPEG_DEC_UV_GFX10_TILING_SURFACE),
62 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_PITCH),
63 	SOC15_REG_ENTRY_STR(JPEG, 0, regUVD_JPEG_UV_PITCH),
64 };
65 
66 static void jpeg_v4_0_5_set_dec_ring_funcs(struct amdgpu_device *adev);
67 static void jpeg_v4_0_5_set_irq_funcs(struct amdgpu_device *adev);
68 static int jpeg_v4_0_5_set_powergating_state(struct amdgpu_ip_block *ip_block,
69 				enum amd_powergating_state state);
70 static void jpeg_v4_0_5_dec_ring_set_wptr(struct amdgpu_ring *ring);
71 
72 static int amdgpu_ih_clientid_jpeg[] = {
73 	SOC15_IH_CLIENTID_VCN,
74 	SOC15_IH_CLIENTID_VCN1
75 };
76 
77 
78 
79 /**
80  * jpeg_v4_0_5_early_init - set function pointers
81  *
82  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
83  *
84  * Set ring and irq function pointers
85  */
86 static int jpeg_v4_0_5_early_init(struct amdgpu_ip_block *ip_block)
87 {
88 	struct amdgpu_device *adev = ip_block->adev;
89 
90 	switch (amdgpu_ip_version(adev, UVD_HWIP, 0)) {
91 	case IP_VERSION(4, 0, 5):
92 		adev->jpeg.num_jpeg_inst = 1;
93 		break;
94 	case IP_VERSION(4, 0, 6):
95 		adev->jpeg.num_jpeg_inst = 2;
96 		break;
97 	default:
98 		DRM_DEV_ERROR(adev->dev,
99 			"Failed to init vcn ip block(UVD_HWIP:0x%x)\n",
100 			amdgpu_ip_version(adev, UVD_HWIP, 0));
101 		return -EINVAL;
102 	}
103 
104 	adev->jpeg.num_jpeg_rings = 1;
105 
106 	jpeg_v4_0_5_set_dec_ring_funcs(adev);
107 	jpeg_v4_0_5_set_irq_funcs(adev);
108 
109 	return 0;
110 }
111 
112 /**
113  * jpeg_v4_0_5_sw_init - sw init for JPEG block
114  *
115  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
116  *
117  * Load firmware and sw initialization
118  */
119 static int jpeg_v4_0_5_sw_init(struct amdgpu_ip_block *ip_block)
120 {
121 	struct amdgpu_device *adev = ip_block->adev;
122 	struct amdgpu_ring *ring;
123 	int r, i;
124 
125 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
126 		if (adev->jpeg.harvest_config & (1 << i))
127 			continue;
128 
129 		/* JPEG TRAP */
130 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_jpeg[i],
131 				VCN_4_0__SRCID__JPEG_DECODE, &adev->jpeg.inst[i].irq);
132 		if (r)
133 			return r;
134 
135 		/* JPEG DJPEG POISON EVENT */
136 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_jpeg[i],
137 			VCN_4_0__SRCID_DJPEG0_POISON, &adev->jpeg.inst[i].irq);
138 		if (r)
139 			return r;
140 
141 		/* JPEG EJPEG POISON EVENT */
142 		r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_jpeg[i],
143 			VCN_4_0__SRCID_EJPEG0_POISON, &adev->jpeg.inst[i].irq);
144 		if (r)
145 			return r;
146 	}
147 
148 	r = amdgpu_jpeg_sw_init(adev);
149 	if (r)
150 		return r;
151 
152 	r = amdgpu_jpeg_resume(adev);
153 	if (r)
154 		return r;
155 
156 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
157 		if (adev->jpeg.harvest_config & (1 << i))
158 			continue;
159 
160 		ring = adev->jpeg.inst[i].ring_dec;
161 		ring->use_doorbell = true;
162 		ring->vm_hub = AMDGPU_MMHUB0(0);
163 		ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + 8 * i;
164 		sprintf(ring->name, "jpeg_dec_%d", i);
165 		r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst[i].irq,
166 				     0, AMDGPU_RING_PRIO_DEFAULT, NULL);
167 		if (r)
168 			return r;
169 
170 		adev->jpeg.internal.jpeg_pitch[0] = regUVD_JPEG_PITCH_INTERNAL_OFFSET;
171 		adev->jpeg.inst[i].external.jpeg_pitch[0] = SOC15_REG_OFFSET(JPEG, i, regUVD_JPEG_PITCH);
172 	}
173 
174 	r = amdgpu_jpeg_reg_dump_init(adev, jpeg_reg_list_4_0_5, ARRAY_SIZE(jpeg_reg_list_4_0_5));
175 	if (r)
176 		return r;
177 
178 	adev->jpeg.supported_reset =
179 		amdgpu_get_soft_full_reset_mask(&adev->jpeg.inst[0].ring_dec[0]);
180 	if (!amdgpu_sriov_vf(adev))
181 		adev->jpeg.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
182 	r = amdgpu_jpeg_sysfs_reset_mask_init(adev);
183 	if (r)
184 		return r;
185 
186 	return 0;
187 }
188 
189 /**
190  * jpeg_v4_0_5_sw_fini - sw fini for JPEG block
191  *
192  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
193  *
194  * JPEG suspend and free up sw allocation
195  */
196 static int jpeg_v4_0_5_sw_fini(struct amdgpu_ip_block *ip_block)
197 {
198 	struct amdgpu_device *adev = ip_block->adev;
199 	int r;
200 
201 	r = amdgpu_jpeg_suspend(adev);
202 	if (r)
203 		return r;
204 
205 	amdgpu_jpeg_sysfs_reset_mask_fini(adev);
206 	r = amdgpu_jpeg_sw_fini(adev);
207 
208 	return r;
209 }
210 
211 /**
212  * jpeg_v4_0_5_hw_init - start and test JPEG block
213  *
214  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
215  *
216  */
217 static int jpeg_v4_0_5_hw_init(struct amdgpu_ip_block *ip_block)
218 {
219 	struct amdgpu_device *adev = ip_block->adev;
220 	struct amdgpu_ring *ring;
221 	int i, r = 0;
222 
223 	// TODO: Enable ring test with DPG support
224 	if (adev->pg_flags & AMD_PG_SUPPORT_JPEG_DPG) {
225 		return 0;
226 	}
227 
228 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
229 		if (adev->jpeg.harvest_config & (1 << i))
230 			continue;
231 
232 		ring = adev->jpeg.inst[i].ring_dec;
233 		r = amdgpu_ring_test_helper(ring);
234 		if (r)
235 			return r;
236 	}
237 
238 	return 0;
239 }
240 
241 /**
242  * jpeg_v4_0_5_hw_fini - stop the hardware block
243  *
244  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
245  *
246  * Stop the JPEG block, mark ring as not ready any more
247  */
248 static int jpeg_v4_0_5_hw_fini(struct amdgpu_ip_block *ip_block)
249 {
250 	struct amdgpu_device *adev = ip_block->adev;
251 	int i;
252 
253 	cancel_delayed_work_sync(&adev->jpeg.idle_work);
254 
255 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
256 		if (adev->jpeg.harvest_config & (1 << i))
257 			continue;
258 
259 		if (!amdgpu_sriov_vf(adev)) {
260 			if (adev->jpeg.cur_state != AMD_PG_STATE_GATE &&
261 			    RREG32_SOC15(JPEG, i, regUVD_JRBC_STATUS))
262 				jpeg_v4_0_5_set_powergating_state(ip_block, AMD_PG_STATE_GATE);
263 		}
264 	}
265 	return 0;
266 }
267 
268 /**
269  * jpeg_v4_0_5_suspend - suspend JPEG block
270  *
271  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
272  *
273  * HW fini and suspend JPEG block
274  */
275 static int jpeg_v4_0_5_suspend(struct amdgpu_ip_block *ip_block)
276 {
277 	int r;
278 
279 	r = jpeg_v4_0_5_hw_fini(ip_block);
280 	if (r)
281 		return r;
282 
283 	r = amdgpu_jpeg_suspend(ip_block->adev);
284 
285 	return r;
286 }
287 
288 /**
289  * jpeg_v4_0_5_resume - resume JPEG block
290  *
291  * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
292  *
293  * Resume firmware and hw init JPEG block
294  */
295 static int jpeg_v4_0_5_resume(struct amdgpu_ip_block *ip_block)
296 {
297 	int r;
298 
299 	r = amdgpu_jpeg_resume(ip_block->adev);
300 	if (r)
301 		return r;
302 
303 	r = jpeg_v4_0_5_hw_init(ip_block);
304 
305 	return r;
306 }
307 
308 static void jpeg_v4_0_5_disable_clock_gating(struct amdgpu_device *adev, int inst)
309 {
310 	uint32_t data = 0;
311 
312 	data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL);
313 	if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
314 		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
315 		data &= (~JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK);
316 	} else {
317 		data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
318 	}
319 
320 	data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
321 	data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
322 	WREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL, data);
323 
324 	data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE);
325 	data &= ~(JPEG_CGC_GATE__JPEG_DEC_MASK
326 		| JPEG_CGC_GATE__JPEG2_DEC_MASK
327 		| JPEG_CGC_GATE__JMCIF_MASK
328 		| JPEG_CGC_GATE__JRBBM_MASK);
329 	WREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE, data);
330 }
331 
332 static void jpeg_v4_0_5_enable_clock_gating(struct amdgpu_device *adev, int inst)
333 {
334 	uint32_t data = 0;
335 
336 	data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL);
337 	if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG) {
338 		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
339 		data |= JPEG_CGC_CTRL__JPEG_DEC_MODE_MASK;
340 	} else {
341 		data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
342 	}
343 
344 	data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
345 	data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
346 	WREG32_SOC15(JPEG, inst, regJPEG_CGC_CTRL, data);
347 
348 	data = RREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE);
349 	data |= (JPEG_CGC_GATE__JPEG_DEC_MASK
350 		|JPEG_CGC_GATE__JPEG2_DEC_MASK
351 		|JPEG_CGC_GATE__JMCIF_MASK
352 		|JPEG_CGC_GATE__JRBBM_MASK);
353 	WREG32_SOC15(JPEG, inst, regJPEG_CGC_GATE, data);
354 }
355 
356 static void jpeg_engine_4_0_5_dpg_clock_gating_mode(struct amdgpu_device *adev,
357 			int inst_idx, uint8_t indirect)
358 {
359 	uint32_t data = 0;
360 
361 	if (adev->cg_flags & AMD_CG_SUPPORT_JPEG_MGCG)
362 		data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
363 	else
364 		data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
365 
366 	data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
367 	data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
368 	WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_CGC_CTRL_INTERNAL_OFFSET, data, indirect);
369 
370 	data = 0;
371 	WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_CGC_GATE_INTERNAL_OFFSET,
372 				data, indirect);
373 }
374 
375 static int jpeg_v4_0_5_disable_static_power_gating(struct amdgpu_device *adev, int inst)
376 {
377 	if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
378 		WREG32(SOC15_REG_OFFSET(JPEG, inst, regUVD_IPX_DLDO_CONFIG),
379 			1 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
380 		SOC15_WAIT_ON_RREG(JPEG, inst, regUVD_IPX_DLDO_STATUS,
381 			0, UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
382 	}
383 
384 	/* disable anti hang mechanism */
385 	WREG32_P(SOC15_REG_OFFSET(JPEG, inst, regUVD_JPEG_POWER_STATUS), 0,
386 		~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
387 
388 	/* keep the JPEG in static PG mode */
389 	WREG32_P(SOC15_REG_OFFSET(JPEG, inst, regUVD_JPEG_POWER_STATUS), 0,
390 		~UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK);
391 
392 	return 0;
393 }
394 
395 static int jpeg_v4_0_5_enable_static_power_gating(struct amdgpu_device *adev, int inst)
396 {
397 	/* enable anti hang mechanism */
398 	WREG32_P(SOC15_REG_OFFSET(JPEG, inst, regUVD_JPEG_POWER_STATUS),
399 		UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK,
400 		~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK);
401 
402 	if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
403 		WREG32(SOC15_REG_OFFSET(JPEG, inst, regUVD_IPX_DLDO_CONFIG),
404 			2 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
405 		SOC15_WAIT_ON_RREG(JPEG, inst, regUVD_IPX_DLDO_STATUS,
406 			1 << UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS__SHIFT,
407 			UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
408 	}
409 
410 	return 0;
411 }
412 
413 /**
414  * jpeg_v4_0_5_start_dpg_mode - Jpeg start with dpg mode
415  *
416  * @adev: amdgpu_device pointer
417  * @inst_idx: instance number index
418  * @indirect: indirectly write sram
419  *
420  * Start JPEG block with dpg mode
421  */
422 static void jpeg_v4_0_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
423 {
424 	struct amdgpu_ring *ring = adev->jpeg.inst[inst_idx].ring_dec;
425 	uint32_t reg_data = 0;
426 
427 	/* enable anti hang mechanism */
428 	reg_data = RREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS);
429 	reg_data &= ~UVD_JPEG_POWER_STATUS__JPEG_POWER_STATUS_MASK;
430 	reg_data |=  0x1;
431 	WREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS, reg_data);
432 
433 	if (adev->pg_flags & AMD_PG_SUPPORT_JPEG) {
434 		WREG32(SOC15_REG_OFFSET(JPEG, inst_idx, regUVD_IPX_DLDO_CONFIG),
435 			2 << UVD_IPX_DLDO_CONFIG__ONO1_PWR_CONFIG__SHIFT);
436 		SOC15_WAIT_ON_RREG(JPEG, inst_idx, regUVD_IPX_DLDO_STATUS,
437 			1 << UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS__SHIFT,
438 			UVD_IPX_DLDO_STATUS__ONO1_PWR_STATUS_MASK);
439 	}
440 
441 	reg_data = RREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS);
442 	reg_data |= UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK;
443 	WREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS, reg_data);
444 
445 	if (indirect)
446 		adev->jpeg.inst[inst_idx].dpg_sram_curr_addr =
447 					(uint32_t *)adev->jpeg.inst[inst_idx].dpg_sram_cpu_addr;
448 
449 	jpeg_engine_4_0_5_dpg_clock_gating_mode(adev, inst_idx, indirect);
450 
451 	/* MJPEG global tiling registers */
452 	WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_DEC_GFX10_ADDR_CONFIG_INTERNAL_OFFSET,
453 	adev->gfx.config.gb_addr_config, indirect);
454 	/* enable System Interrupt for JRBC */
455 	WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regJPEG_SYS_INT_EN_INTERNAL_OFFSET,
456 	JPEG_SYS_INT_EN__DJRBC_MASK, indirect);
457 
458 	/* add nop to workaround PSP size check */
459 	WREG32_SOC15_JPEG_DPG_MODE(inst_idx, regUVD_NO_OP_INTERNAL_OFFSET, 0, indirect);
460 
461 	if (indirect)
462 		amdgpu_jpeg_psp_update_sram(adev, inst_idx, 0);
463 
464 	WREG32_SOC15(JPEG, inst_idx, regUVD_LMI_JRBC_RB_VMID, 0);
465 	WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
466 	WREG32_SOC15(JPEG, inst_idx, regUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
467 		lower_32_bits(ring->gpu_addr));
468 	WREG32_SOC15(JPEG, inst_idx, regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
469 		upper_32_bits(ring->gpu_addr));
470 	WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_RPTR, 0);
471 	WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_WPTR, 0);
472 	WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_CNTL, 0x00000002L);
473 	WREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_SIZE, ring->ring_size / 4);
474 	ring->wptr = RREG32_SOC15(JPEG, inst_idx, regUVD_JRBC_RB_WPTR);
475 }
476 
477 /**
478  * jpeg_v4_0_5_stop_dpg_mode - Jpeg stop with dpg mode
479  *
480  * @adev: amdgpu_device pointer
481  * @inst_idx: instance number index
482  *
483  * Stop JPEG block with dpg mode
484  */
485 static void jpeg_v4_0_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
486 {
487 	uint32_t reg_data = 0;
488 
489 	reg_data = RREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS);
490 	reg_data &= ~UVD_JPEG_POWER_STATUS__JPEG_PG_MODE_MASK;
491 	WREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS, reg_data);
492 
493 }
494 
495 /**
496  * jpeg_v4_0_5_start - start JPEG block
497  *
498  * @adev: amdgpu_device pointer
499  *
500  * Setup and start the JPEG block
501  */
502 static int jpeg_v4_0_5_start(struct amdgpu_device *adev)
503 {
504 	struct amdgpu_ring *ring;
505 	int r, i;
506 
507 	if (adev->pm.dpm_enabled)
508 		amdgpu_dpm_enable_jpeg(adev, true);
509 
510 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
511 		if (adev->jpeg.harvest_config & (1 << i))
512 			continue;
513 
514 		ring = adev->jpeg.inst[i].ring_dec;
515 		/* doorbell programming is done for every playback */
516 		adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
517 				(adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i, i);
518 
519 		WREG32_SOC15(VCN, i, regVCN_JPEG_DB_CTRL,
520 			ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT |
521 			VCN_JPEG_DB_CTRL__EN_MASK);
522 
523 		if (adev->pg_flags & AMD_PG_SUPPORT_JPEG_DPG) {
524 			jpeg_v4_0_5_start_dpg_mode(adev, i, adev->jpeg.indirect_sram);
525 			continue;
526 		}
527 
528 		/* disable power gating */
529 		r = jpeg_v4_0_5_disable_static_power_gating(adev, i);
530 		if (r)
531 			return r;
532 
533 		/* JPEG disable CGC */
534 		jpeg_v4_0_5_disable_clock_gating(adev, i);
535 
536 		/* MJPEG global tiling registers */
537 		WREG32_SOC15(JPEG, i, regJPEG_DEC_GFX10_ADDR_CONFIG,
538 			adev->gfx.config.gb_addr_config);
539 
540 		/* enable JMI channel */
541 		WREG32_P(SOC15_REG_OFFSET(JPEG, i, regUVD_JMI_CNTL), 0,
542 			~UVD_JMI_CNTL__SOFT_RESET_MASK);
543 
544 		/* enable System Interrupt for JRBC */
545 		WREG32_P(SOC15_REG_OFFSET(JPEG, i, regJPEG_SYS_INT_EN),
546 			JPEG_SYS_INT_EN__DJRBC_MASK,
547 			~JPEG_SYS_INT_EN__DJRBC_MASK);
548 
549 		WREG32_SOC15(JPEG, i, regUVD_LMI_JRBC_RB_VMID, 0);
550 		WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
551 		WREG32_SOC15(JPEG, i, regUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
552 			lower_32_bits(ring->gpu_addr));
553 		WREG32_SOC15(JPEG, i, regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
554 			upper_32_bits(ring->gpu_addr));
555 		WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_RPTR, 0);
556 		WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_WPTR, 0);
557 		WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_CNTL, 0x00000002L);
558 		WREG32_SOC15(JPEG, i, regUVD_JRBC_RB_SIZE, ring->ring_size / 4);
559 		ring->wptr = RREG32_SOC15(JPEG, i, regUVD_JRBC_RB_WPTR);
560 	}
561 
562 	return 0;
563 }
564 
565 /**
566  * jpeg_v4_0_5_stop - stop JPEG block
567  *
568  * @adev: amdgpu_device pointer
569  *
570  * stop the JPEG block
571  */
572 static int jpeg_v4_0_5_stop(struct amdgpu_device *adev)
573 {
574 	int r, i;
575 
576 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
577 		if (adev->jpeg.harvest_config & (1 << i))
578 			continue;
579 
580 		if (adev->pg_flags & AMD_PG_SUPPORT_JPEG_DPG) {
581 			jpeg_v4_0_5_stop_dpg_mode(adev, i);
582 			continue;
583 		}
584 
585 		/* reset JMI */
586 		WREG32_P(SOC15_REG_OFFSET(JPEG, i, regUVD_JMI_CNTL),
587 			UVD_JMI_CNTL__SOFT_RESET_MASK,
588 			~UVD_JMI_CNTL__SOFT_RESET_MASK);
589 
590 		jpeg_v4_0_5_enable_clock_gating(adev, i);
591 
592 		/* enable power gating */
593 		r = jpeg_v4_0_5_enable_static_power_gating(adev, i);
594 		if (r)
595 			return r;
596 	}
597 	if (adev->pm.dpm_enabled)
598 		amdgpu_dpm_enable_jpeg(adev, false);
599 
600 	return 0;
601 }
602 
603 /**
604  * jpeg_v4_0_5_dec_ring_get_rptr - get read pointer
605  *
606  * @ring: amdgpu_ring pointer
607  *
608  * Returns the current hardware read pointer
609  */
610 static uint64_t jpeg_v4_0_5_dec_ring_get_rptr(struct amdgpu_ring *ring)
611 {
612 	struct amdgpu_device *adev = ring->adev;
613 
614 	return RREG32_SOC15(JPEG, ring->me, regUVD_JRBC_RB_RPTR);
615 }
616 
617 /**
618  * jpeg_v4_0_5_dec_ring_get_wptr - get write pointer
619  *
620  * @ring: amdgpu_ring pointer
621  *
622  * Returns the current hardware write pointer
623  */
624 static uint64_t jpeg_v4_0_5_dec_ring_get_wptr(struct amdgpu_ring *ring)
625 {
626 	struct amdgpu_device *adev = ring->adev;
627 
628 	if (ring->use_doorbell)
629 		return *ring->wptr_cpu_addr;
630 	else
631 		return RREG32_SOC15(JPEG, ring->me, regUVD_JRBC_RB_WPTR);
632 }
633 
634 /**
635  * jpeg_v4_0_5_dec_ring_set_wptr - set write pointer
636  *
637  * @ring: amdgpu_ring pointer
638  *
639  * Commits the write pointer to the hardware
640  */
641 static void jpeg_v4_0_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
642 {
643 	struct amdgpu_device *adev = ring->adev;
644 
645 	if (ring->use_doorbell) {
646 		*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
647 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
648 	} else {
649 		WREG32_SOC15(JPEG, ring->me, regUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
650 	}
651 }
652 
653 static bool jpeg_v4_0_5_is_idle(struct amdgpu_ip_block *ip_block)
654 {
655 	struct amdgpu_device *adev = ip_block->adev;
656 	int i, ret = 1;
657 
658 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
659 		if (adev->jpeg.harvest_config & (1 << i))
660 			continue;
661 
662 		ret &= (((RREG32_SOC15(JPEG, i, regUVD_JRBC_STATUS) &
663 			UVD_JRBC_STATUS__RB_JOB_DONE_MASK) ==
664 			UVD_JRBC_STATUS__RB_JOB_DONE_MASK));
665 	}
666 	return ret;
667 }
668 
669 static int jpeg_v4_0_5_wait_for_idle(struct amdgpu_ip_block *ip_block)
670 {
671 	struct amdgpu_device *adev = ip_block->adev;
672 	int i;
673 
674 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
675 		if (adev->jpeg.harvest_config & (1 << i))
676 			continue;
677 
678 		return SOC15_WAIT_ON_RREG(JPEG, i, regUVD_JRBC_STATUS,
679 			UVD_JRBC_STATUS__RB_JOB_DONE_MASK,
680 			UVD_JRBC_STATUS__RB_JOB_DONE_MASK);
681 	}
682 
683 	return 0;
684 }
685 
686 static int jpeg_v4_0_5_set_clockgating_state(struct amdgpu_ip_block *ip_block,
687 					  enum amd_clockgating_state state)
688 {
689 	struct amdgpu_device *adev = ip_block->adev;
690 	bool enable = state == AMD_CG_STATE_GATE;
691 	int i;
692 
693 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
694 		if (adev->jpeg.harvest_config & (1 << i))
695 			continue;
696 
697 		if (enable) {
698 			if (!jpeg_v4_0_5_is_idle(ip_block))
699 				return -EBUSY;
700 
701 			jpeg_v4_0_5_enable_clock_gating(adev, i);
702 		} else {
703 			jpeg_v4_0_5_disable_clock_gating(adev, i);
704 		}
705 	}
706 
707 	return 0;
708 }
709 
710 static int jpeg_v4_0_5_set_powergating_state(struct amdgpu_ip_block *ip_block,
711 					  enum amd_powergating_state state)
712 {
713 	struct amdgpu_device *adev = ip_block->adev;
714 	int ret;
715 
716 	if (amdgpu_sriov_vf(adev)) {
717 		adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
718 		return 0;
719 	}
720 
721 	if (state == adev->jpeg.cur_state)
722 		return 0;
723 
724 	if (state == AMD_PG_STATE_GATE)
725 		ret = jpeg_v4_0_5_stop(adev);
726 	else
727 		ret = jpeg_v4_0_5_start(adev);
728 
729 	if (!ret)
730 		adev->jpeg.cur_state = state;
731 
732 	return ret;
733 }
734 
735 static int jpeg_v4_0_5_process_interrupt(struct amdgpu_device *adev,
736 				      struct amdgpu_irq_src *source,
737 				      struct amdgpu_iv_entry *entry)
738 {
739 	uint32_t ip_instance;
740 
741 	DRM_DEBUG("IH: JPEG TRAP\n");
742 
743 	switch (entry->client_id) {
744 	case SOC15_IH_CLIENTID_VCN:
745 		ip_instance = 0;
746 		break;
747 	case SOC15_IH_CLIENTID_VCN1:
748 		ip_instance = 1;
749 		break;
750 	default:
751 		DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
752 		return 0;
753 	}
754 
755 	switch (entry->src_id) {
756 	case VCN_4_0__SRCID__JPEG_DECODE:
757 		amdgpu_fence_process(adev->jpeg.inst[ip_instance].ring_dec);
758 		break;
759 	case VCN_4_0__SRCID_DJPEG0_POISON:
760 	case VCN_4_0__SRCID_EJPEG0_POISON:
761 		amdgpu_jpeg_process_poison_irq(adev, source, entry);
762 		break;
763 	default:
764 		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
765 			  entry->src_id, entry->src_data[0]);
766 		break;
767 	}
768 
769 	return 0;
770 }
771 
772 static int jpeg_v4_0_5_ring_reset(struct amdgpu_ring *ring,
773 				  unsigned int vmid,
774 				  struct amdgpu_fence *timedout_fence)
775 {
776 	struct amdgpu_device *adev = ring->adev;
777 	u32 pg_flags = adev->pg_flags;
778 	int r;
779 
780 	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
781 
782 	/*
783 	 * The DPG stop path only clears the JPEG_PG_MODE bit and never resets a
784 	 * hung JRBC, so the post-reset ring test times out and the driver falls
785 	 * back to a full MODE1 reset. Temporarily force the static power-gating
786 	 * path so the stop/start sequence actually power-cycles the JPEG block
787 	 * (JMI soft reset + static power off/on), matching the working jpeg_v4_0
788 	 * reset.
789 	 */
790 	adev->pg_flags &= ~AMD_PG_SUPPORT_JPEG_DPG;
791 	r = jpeg_v4_0_5_stop(adev);
792 	if (!r)
793 		r = jpeg_v4_0_5_start(adev);
794 	adev->pg_flags = pg_flags;
795 	if (r)
796 		return r;
797 
798 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
799 }
800 
801 static const struct amd_ip_funcs jpeg_v4_0_5_ip_funcs = {
802 	.name = "jpeg_v4_0_5",
803 	.early_init = jpeg_v4_0_5_early_init,
804 	.sw_init = jpeg_v4_0_5_sw_init,
805 	.sw_fini = jpeg_v4_0_5_sw_fini,
806 	.hw_init = jpeg_v4_0_5_hw_init,
807 	.hw_fini = jpeg_v4_0_5_hw_fini,
808 	.suspend = jpeg_v4_0_5_suspend,
809 	.resume = jpeg_v4_0_5_resume,
810 	.wait_for_idle = jpeg_v4_0_5_wait_for_idle,
811 	.set_clockgating_state = jpeg_v4_0_5_set_clockgating_state,
812 	.set_powergating_state = jpeg_v4_0_5_set_powergating_state,
813 	.dump_ip_state = amdgpu_jpeg_dump_ip_state,
814 	.print_ip_state = amdgpu_jpeg_print_ip_state,
815 };
816 
817 static const struct amdgpu_ring_funcs jpeg_v4_0_5_dec_ring_vm_funcs = {
818 	.type = AMDGPU_RING_TYPE_VCN_JPEG,
819 	.align_mask = 0xf,
820 	.no_user_fence = true,
821 	.get_rptr = jpeg_v4_0_5_dec_ring_get_rptr,
822 	.get_wptr = jpeg_v4_0_5_dec_ring_get_wptr,
823 	.set_wptr = jpeg_v4_0_5_dec_ring_set_wptr,
824 	.parse_cs = amdgpu_jpeg_dec_parse_cs,
825 	.emit_frame_size =
826 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
827 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
828 		8 + /* jpeg_v4_0_5_dec_ring_emit_vm_flush */
829 		18 + 18 + /* jpeg_v4_0_5_dec_ring_emit_fence x2 vm fence */
830 		8 + 16,
831 	.emit_ib_size = 22, /* jpeg_v4_0_5_dec_ring_emit_ib */
832 	.emit_ib = jpeg_v2_0_dec_ring_emit_ib,
833 	.emit_fence = jpeg_v2_0_dec_ring_emit_fence,
834 	.emit_vm_flush = jpeg_v2_0_dec_ring_emit_vm_flush,
835 	.test_ring = amdgpu_jpeg_dec_ring_test_ring,
836 	.test_ib = amdgpu_jpeg_dec_ring_test_ib,
837 	.insert_nop = jpeg_v2_0_dec_ring_nop,
838 	.insert_start = jpeg_v2_0_dec_ring_insert_start,
839 	.insert_end = jpeg_v2_0_dec_ring_insert_end,
840 	.pad_ib = amdgpu_ring_generic_pad_ib,
841 	.begin_use = amdgpu_jpeg_ring_begin_use,
842 	.end_use = amdgpu_jpeg_ring_end_use,
843 	.emit_wreg = jpeg_v2_0_dec_ring_emit_wreg,
844 	.emit_reg_wait = jpeg_v2_0_dec_ring_emit_reg_wait,
845 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
846 	.reset = jpeg_v4_0_5_ring_reset,
847 };
848 
849 static void jpeg_v4_0_5_set_dec_ring_funcs(struct amdgpu_device *adev)
850 {
851 	int i;
852 
853 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
854 		if (adev->jpeg.harvest_config & (1 << i))
855 			continue;
856 
857 		adev->jpeg.inst[i].ring_dec->funcs = &jpeg_v4_0_5_dec_ring_vm_funcs;
858 		adev->jpeg.inst[i].ring_dec->me = i;
859 	}
860 }
861 
862 static const struct amdgpu_irq_src_funcs jpeg_v4_0_5_irq_funcs = {
863 	.process = jpeg_v4_0_5_process_interrupt,
864 };
865 
866 static void jpeg_v4_0_5_set_irq_funcs(struct amdgpu_device *adev)
867 {
868 	int i;
869 
870 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
871 		if (adev->jpeg.harvest_config & (1 << i))
872 			continue;
873 
874 		adev->jpeg.inst[i].irq.num_types = 1;
875 		adev->jpeg.inst[i].irq.funcs = &jpeg_v4_0_5_irq_funcs;
876 	}
877 }
878 
879 const struct amdgpu_ip_block_version jpeg_v4_0_5_ip_block = {
880 	.type = AMD_IP_BLOCK_TYPE_JPEG,
881 	.major = 4,
882 	.minor = 0,
883 	.rev = 5,
884 	.funcs = &jpeg_v4_0_5_ip_funcs,
885 };
886 
887