1 /*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/firmware.h>
25 #include <drm/drm_drv.h>
26
27 #include "amdgpu.h"
28 #include "amdgpu_vcn.h"
29 #include "amdgpu_pm.h"
30 #include "soc15.h"
31 #include "soc15d.h"
32 #include "vcn_v2_0.h"
33 #include "mmsch_v1_0.h"
34 #include "vcn_v2_5.h"
35
36 #include "vcn/vcn_2_5_offset.h"
37 #include "vcn/vcn_2_5_sh_mask.h"
38 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
39
40 #define VCN_VID_SOC_ADDRESS_2_0 0x1fa00
41 #define VCN1_VID_SOC_ADDRESS_3_0 0x48200
42 #define VCN1_AON_SOC_ADDRESS_3_0 0x48000
43
44 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27
45 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f
46 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10
47 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11
48 #define mmUVD_NO_OP_INTERNAL_OFFSET 0x29
49 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66
50 #define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
51
52 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431
53 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4
54 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5
55 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c
56
57 #define VCN25_MAX_HW_INSTANCES_ARCTURUS 2
58
59 static const struct amdgpu_hwip_reg_entry vcn_reg_list_2_5[] = {
60 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_POWER_STATUS),
61 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_POWER_STATUS),
62 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_STATUS),
63 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID),
64 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_CONTEXT_ID2),
65 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA0),
66 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_DATA1),
67 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_GPCOM_VCPU_CMD),
68 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI),
69 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO),
70 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI2),
71 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO2),
72 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI3),
73 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO3),
74 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_HI4),
75 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_BASE_LO4),
76 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR),
77 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR),
78 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR2),
79 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR2),
80 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR3),
81 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR3),
82 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_RPTR4),
83 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_WPTR4),
84 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE),
85 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE2),
86 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE3),
87 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_RB_SIZE4),
88 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_CONFIG),
89 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_PGFSM_STATUS),
90 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_CTL),
91 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_DATA),
92 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_LMA_MASK),
93 SOC15_REG_ENTRY_STR(VCN, 0, mmUVD_DPG_PAUSE)
94 };
95
96 static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev);
97 static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev);
98 static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev);
99 static int vcn_v2_5_set_pg_state(struct amdgpu_vcn_inst *vinst,
100 enum amd_powergating_state state);
101 static int vcn_v2_5_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
102 struct dpg_pause_state *new_state);
103 static int vcn_v2_5_sriov_start(struct amdgpu_device *adev);
104 static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev);
105 static int vcn_v2_5_reset(struct amdgpu_vcn_inst *vinst);
106
107 static int amdgpu_ih_clientid_vcns[] = {
108 SOC15_IH_CLIENTID_VCN,
109 SOC15_IH_CLIENTID_VCN1
110 };
111
vcn_v2_5_idle_work_handler(struct work_struct * work)112 static void vcn_v2_5_idle_work_handler(struct work_struct *work)
113 {
114 struct amdgpu_vcn_inst *vcn_inst =
115 container_of(work, struct amdgpu_vcn_inst, idle_work.work);
116 struct amdgpu_device *adev = vcn_inst->adev;
117 unsigned int fences = 0, fence[AMDGPU_MAX_VCN_INSTANCES] = {0};
118 unsigned int i, j;
119 int r = 0;
120
121 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
122 struct amdgpu_vcn_inst *v = &adev->vcn.inst[i];
123
124 if (adev->vcn.harvest_config & (1 << i))
125 continue;
126
127 for (j = 0; j < v->num_enc_rings; ++j)
128 fence[i] += amdgpu_fence_count_emitted(&v->ring_enc[j]);
129
130 /* Only set DPG pause for VCN3 or below, VCN4 and above will be handled by FW */
131 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG &&
132 !v->using_unified_queue) {
133 struct dpg_pause_state new_state;
134
135 if (fence[i] ||
136 unlikely(atomic_read(&v->dpg_enc_submission_cnt)))
137 new_state.fw_based = VCN_DPG_STATE__PAUSE;
138 else
139 new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
140
141 v->pause_dpg_mode(v, &new_state);
142 }
143
144 fence[i] += amdgpu_fence_count_emitted(&v->ring_dec);
145 fences += fence[i];
146
147 }
148
149 if (!fences && !atomic_read(&adev->vcn.inst[0].total_submission_cnt)) {
150 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
151 AMD_PG_STATE_GATE);
152 mutex_lock(&adev->vcn.workload_profile_mutex);
153 if (adev->vcn.workload_profile_active) {
154 r = amdgpu_dpm_switch_power_profile(adev, PP_SMC_POWER_PROFILE_VIDEO,
155 false);
156 if (r)
157 dev_warn(adev->dev, "(%d) failed to disable video power profile mode\n", r);
158 adev->vcn.workload_profile_active = false;
159 }
160 mutex_unlock(&adev->vcn.workload_profile_mutex);
161 } else {
162 schedule_delayed_work(&adev->vcn.inst[0].idle_work, VCN_IDLE_TIMEOUT);
163 }
164 }
165
vcn_v2_5_ring_begin_use(struct amdgpu_ring * ring)166 static void vcn_v2_5_ring_begin_use(struct amdgpu_ring *ring)
167 {
168 struct amdgpu_device *adev = ring->adev;
169 struct amdgpu_vcn_inst *v = &adev->vcn.inst[ring->me];
170 int r = 0;
171
172 atomic_inc(&adev->vcn.inst[0].total_submission_cnt);
173
174 cancel_delayed_work_sync(&adev->vcn.inst[0].idle_work);
175
176 /* We can safely return early here because we've cancelled the
177 * the delayed work so there is no one else to set it to false
178 * and we don't care if someone else sets it to true.
179 */
180 if (adev->vcn.workload_profile_active)
181 goto pg_lock;
182
183 mutex_lock(&adev->vcn.workload_profile_mutex);
184 if (!adev->vcn.workload_profile_active) {
185 r = amdgpu_dpm_switch_power_profile(adev, PP_SMC_POWER_PROFILE_VIDEO,
186 true);
187 if (r)
188 dev_warn(adev->dev, "(%d) failed to switch to video power profile mode\n", r);
189 adev->vcn.workload_profile_active = true;
190 }
191 mutex_unlock(&adev->vcn.workload_profile_mutex);
192
193 pg_lock:
194 mutex_lock(&adev->vcn.inst[0].vcn_pg_lock);
195 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCN,
196 AMD_PG_STATE_UNGATE);
197
198 /* Only set DPG pause for VCN3 or below, VCN4 and above will be handled by FW */
199 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG &&
200 !v->using_unified_queue) {
201 struct dpg_pause_state new_state;
202
203 if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC) {
204 atomic_inc(&v->dpg_enc_submission_cnt);
205 new_state.fw_based = VCN_DPG_STATE__PAUSE;
206 } else {
207 unsigned int fences = 0;
208 unsigned int i;
209
210 for (i = 0; i < v->num_enc_rings; ++i)
211 fences += amdgpu_fence_count_emitted(&v->ring_enc[i]);
212
213 if (fences || atomic_read(&v->dpg_enc_submission_cnt))
214 new_state.fw_based = VCN_DPG_STATE__PAUSE;
215 else
216 new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
217 }
218 v->pause_dpg_mode(v, &new_state);
219 }
220 mutex_unlock(&adev->vcn.inst[0].vcn_pg_lock);
221 }
222
vcn_v2_5_ring_end_use(struct amdgpu_ring * ring)223 static void vcn_v2_5_ring_end_use(struct amdgpu_ring *ring)
224 {
225 struct amdgpu_device *adev = ring->adev;
226
227 /* Only set DPG pause for VCN3 or below, VCN4 and above will be handled by FW */
228 if (ring->adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG &&
229 ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC &&
230 !adev->vcn.inst[ring->me].using_unified_queue)
231 atomic_dec(&adev->vcn.inst[ring->me].dpg_enc_submission_cnt);
232
233 atomic_dec(&adev->vcn.inst[0].total_submission_cnt);
234
235 schedule_delayed_work(&adev->vcn.inst[0].idle_work,
236 VCN_IDLE_TIMEOUT);
237 }
238
239 /**
240 * vcn_v2_5_early_init - set function pointers and load microcode
241 *
242 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
243 *
244 * Set ring and irq function pointers
245 * Load microcode from filesystem
246 */
vcn_v2_5_early_init(struct amdgpu_ip_block * ip_block)247 static int vcn_v2_5_early_init(struct amdgpu_ip_block *ip_block)
248 {
249 struct amdgpu_device *adev = ip_block->adev;
250 int i, r;
251
252 if (amdgpu_sriov_vf(adev)) {
253 adev->vcn.num_vcn_inst = 2;
254 adev->vcn.harvest_config = 0;
255 for (i = 0; i < adev->vcn.num_vcn_inst; i++)
256 adev->vcn.inst[i].num_enc_rings = 1;
257 } else {
258 u32 harvest;
259 int i;
260
261 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
262 harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING);
263 if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK)
264 adev->vcn.harvest_config |= 1 << i;
265 adev->vcn.inst[i].num_enc_rings = 2;
266 }
267 if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
268 AMDGPU_VCN_HARVEST_VCN1))
269 /* both instances are harvested, disable the block */
270 return -ENOENT;
271 }
272
273 vcn_v2_5_set_dec_ring_funcs(adev);
274 vcn_v2_5_set_enc_ring_funcs(adev);
275 vcn_v2_5_set_irq_funcs(adev);
276 vcn_v2_5_set_ras_funcs(adev);
277
278 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
279 adev->vcn.inst[i].set_pg_state = vcn_v2_5_set_pg_state;
280
281 r = amdgpu_vcn_early_init(adev, i);
282 if (r)
283 return r;
284 }
285
286 return 0;
287 }
288
289 /**
290 * vcn_v2_5_sw_init - sw init for VCN block
291 *
292 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
293 *
294 * Load firmware and sw initialization
295 */
vcn_v2_5_sw_init(struct amdgpu_ip_block * ip_block)296 static int vcn_v2_5_sw_init(struct amdgpu_ip_block *ip_block)
297 {
298 struct amdgpu_ring *ring;
299 int i, j, r;
300 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_5);
301 uint32_t *ptr;
302 struct amdgpu_device *adev = ip_block->adev;
303
304 for (j = 0; j < adev->vcn.num_vcn_inst; j++) {
305 volatile struct amdgpu_fw_shared *fw_shared;
306
307 if (adev->vcn.harvest_config & (1 << j))
308 continue;
309 /* VCN DEC TRAP */
310 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
311 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[j].irq);
312 if (r)
313 return r;
314
315 /* VCN ENC TRAP */
316 for (i = 0; i < adev->vcn.inst[j].num_enc_rings; ++i) {
317 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
318 i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[j].irq);
319 if (r)
320 return r;
321 }
322
323 /* VCN POISON TRAP */
324 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
325 VCN_2_6__SRCID_UVD_POISON, &adev->vcn.inst[j].ras_poison_irq);
326 if (r)
327 return r;
328
329 r = amdgpu_vcn_sw_init(adev, j);
330 if (r)
331 return r;
332
333 /* Override the work func */
334 adev->vcn.inst[j].idle_work.work.func = vcn_v2_5_idle_work_handler;
335
336 amdgpu_vcn_setup_ucode(adev, j);
337
338 r = amdgpu_vcn_resume(adev, j);
339 if (r)
340 return r;
341
342 adev->vcn.inst[j].internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
343 adev->vcn.inst[j].internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
344 adev->vcn.inst[j].internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
345 adev->vcn.inst[j].internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
346 adev->vcn.inst[j].internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
347 adev->vcn.inst[j].internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
348
349 adev->vcn.inst[j].internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
350 adev->vcn.inst[j].external.scratch9 = SOC15_REG_OFFSET(VCN, j, mmUVD_SCRATCH9);
351 adev->vcn.inst[j].internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
352 adev->vcn.inst[j].external.data0 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA0);
353 adev->vcn.inst[j].internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
354 adev->vcn.inst[j].external.data1 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA1);
355 adev->vcn.inst[j].internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
356 adev->vcn.inst[j].external.cmd = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_CMD);
357 adev->vcn.inst[j].internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
358 adev->vcn.inst[j].external.nop = SOC15_REG_OFFSET(VCN, j, mmUVD_NO_OP);
359
360 ring = &adev->vcn.inst[j].ring_dec;
361 ring->use_doorbell = true;
362
363 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
364 (amdgpu_sriov_vf(adev) ? 2*j : 8*j);
365
366 if (amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(2, 5, 0))
367 ring->vm_hub = AMDGPU_MMHUB1(0);
368 else
369 ring->vm_hub = AMDGPU_MMHUB0(0);
370
371 sprintf(ring->name, "vcn_dec_%d", j);
372 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[j].irq,
373 0, AMDGPU_RING_PRIO_DEFAULT, NULL);
374 if (r)
375 return r;
376
377 for (i = 0; i < adev->vcn.inst[j].num_enc_rings; ++i) {
378 enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);
379
380 ring = &adev->vcn.inst[j].ring_enc[i];
381 ring->use_doorbell = true;
382
383 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
384 (amdgpu_sriov_vf(adev) ? (1 + i + 2*j) : (2 + i + 8*j));
385
386 if (amdgpu_ip_version(adev, UVD_HWIP, 0) ==
387 IP_VERSION(2, 5, 0))
388 ring->vm_hub = AMDGPU_MMHUB1(0);
389 else
390 ring->vm_hub = AMDGPU_MMHUB0(0);
391
392 sprintf(ring->name, "vcn_enc_%d.%d", j, i);
393 r = amdgpu_ring_init(adev, ring, 512,
394 &adev->vcn.inst[j].irq, 0,
395 hw_prio, NULL);
396 if (r)
397 return r;
398 }
399
400 fw_shared = adev->vcn.inst[j].fw_shared.cpu_addr;
401 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
402
403 if (amdgpu_vcnfw_log)
404 amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
405
406 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
407 adev->vcn.inst[j].pause_dpg_mode = vcn_v2_5_pause_dpg_mode;
408 adev->vcn.inst[j].reset = vcn_v2_5_reset;
409 }
410
411 adev->vcn.supported_reset =
412 amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]);
413 if (!amdgpu_sriov_vf(adev))
414 adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
415
416 if (amdgpu_sriov_vf(adev)) {
417 r = amdgpu_virt_alloc_mm_table(adev);
418 if (r)
419 return r;
420 }
421
422 r = amdgpu_vcn_ras_sw_init(adev);
423 if (r)
424 return r;
425
426 /* Allocate memory for VCN IP Dump buffer */
427 ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL);
428 if (!ptr) {
429 DRM_ERROR("Failed to allocate memory for VCN IP Dump\n");
430 adev->vcn.ip_dump = NULL;
431 } else {
432 adev->vcn.ip_dump = ptr;
433 }
434
435 r = amdgpu_vcn_sysfs_reset_mask_init(adev);
436 if (r)
437 return r;
438
439 return 0;
440 }
441
442 /**
443 * vcn_v2_5_sw_fini - sw fini for VCN block
444 *
445 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
446 *
447 * VCN suspend and free up sw allocation
448 */
vcn_v2_5_sw_fini(struct amdgpu_ip_block * ip_block)449 static int vcn_v2_5_sw_fini(struct amdgpu_ip_block *ip_block)
450 {
451 int i, r, idx;
452 struct amdgpu_device *adev = ip_block->adev;
453 volatile struct amdgpu_fw_shared *fw_shared;
454
455 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
456 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
457 if (adev->vcn.harvest_config & (1 << i))
458 continue;
459 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
460 fw_shared->present_flag_0 = 0;
461 }
462 drm_dev_exit(idx);
463 }
464
465
466 if (amdgpu_sriov_vf(adev))
467 amdgpu_virt_free_mm_table(adev);
468
469 amdgpu_vcn_sysfs_reset_mask_fini(adev);
470
471 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
472 r = amdgpu_vcn_suspend(adev, i);
473 if (r)
474 return r;
475 r = amdgpu_vcn_sw_fini(adev, i);
476 if (r)
477 return r;
478 }
479
480 kfree(adev->vcn.ip_dump);
481
482 return 0;
483 }
484
485 /**
486 * vcn_v2_5_hw_init - start and test VCN block
487 *
488 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
489 *
490 * Initialize the hardware, boot up the VCPU and do some testing
491 */
vcn_v2_5_hw_init(struct amdgpu_ip_block * ip_block)492 static int vcn_v2_5_hw_init(struct amdgpu_ip_block *ip_block)
493 {
494 struct amdgpu_device *adev = ip_block->adev;
495 struct amdgpu_ring *ring;
496 int i, j, r = 0;
497
498 if (amdgpu_sriov_vf(adev))
499 r = vcn_v2_5_sriov_start(adev);
500
501 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
502 if (adev->vcn.harvest_config & (1 << j))
503 continue;
504
505 if (amdgpu_sriov_vf(adev)) {
506 adev->vcn.inst[j].ring_enc[0].sched.ready = true;
507 adev->vcn.inst[j].ring_enc[1].sched.ready = false;
508 adev->vcn.inst[j].ring_enc[2].sched.ready = false;
509 adev->vcn.inst[j].ring_dec.sched.ready = true;
510 } else {
511
512 ring = &adev->vcn.inst[j].ring_dec;
513
514 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
515 ring->doorbell_index, j);
516
517 r = amdgpu_ring_test_helper(ring);
518 if (r)
519 return r;
520
521 for (i = 0; i < adev->vcn.inst[j].num_enc_rings; ++i) {
522 ring = &adev->vcn.inst[j].ring_enc[i];
523 r = amdgpu_ring_test_helper(ring);
524 if (r)
525 return r;
526 }
527 }
528 }
529
530 return r;
531 }
532
533 /**
534 * vcn_v2_5_hw_fini - stop the hardware block
535 *
536 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
537 *
538 * Stop the VCN block, mark ring as not ready any more
539 */
vcn_v2_5_hw_fini(struct amdgpu_ip_block * ip_block)540 static int vcn_v2_5_hw_fini(struct amdgpu_ip_block *ip_block)
541 {
542 struct amdgpu_device *adev = ip_block->adev;
543 int i;
544
545 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
546 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
547
548 if (adev->vcn.harvest_config & (1 << i))
549 continue;
550
551 cancel_delayed_work_sync(&vinst->idle_work);
552
553 if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
554 (vinst->cur_state != AMD_PG_STATE_GATE &&
555 RREG32_SOC15(VCN, i, mmUVD_STATUS)))
556 vinst->set_pg_state(vinst, AMD_PG_STATE_GATE);
557
558 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN))
559 amdgpu_irq_put(adev, &vinst->ras_poison_irq, 0);
560 }
561
562 return 0;
563 }
564
565 /**
566 * vcn_v2_5_suspend - suspend VCN block
567 *
568 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
569 *
570 * HW fini and suspend VCN block
571 */
vcn_v2_5_suspend(struct amdgpu_ip_block * ip_block)572 static int vcn_v2_5_suspend(struct amdgpu_ip_block *ip_block)
573 {
574 struct amdgpu_device *adev = ip_block->adev;
575 int r, i;
576
577 r = vcn_v2_5_hw_fini(ip_block);
578 if (r)
579 return r;
580
581 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
582 r = amdgpu_vcn_suspend(ip_block->adev, i);
583 if (r)
584 return r;
585 }
586
587 return 0;
588 }
589
590 /**
591 * vcn_v2_5_resume - resume VCN block
592 *
593 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
594 *
595 * Resume firmware and hw init VCN block
596 */
vcn_v2_5_resume(struct amdgpu_ip_block * ip_block)597 static int vcn_v2_5_resume(struct amdgpu_ip_block *ip_block)
598 {
599 struct amdgpu_device *adev = ip_block->adev;
600 int r, i;
601
602 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
603 r = amdgpu_vcn_resume(ip_block->adev, i);
604 if (r)
605 return r;
606 }
607
608 r = vcn_v2_5_hw_init(ip_block);
609
610 return r;
611 }
612
613 /**
614 * vcn_v2_5_mc_resume - memory controller programming
615 *
616 * @vinst: VCN instance
617 *
618 * Let the VCN memory controller know it's offsets
619 */
vcn_v2_5_mc_resume(struct amdgpu_vcn_inst * vinst)620 static void vcn_v2_5_mc_resume(struct amdgpu_vcn_inst *vinst)
621 {
622 struct amdgpu_device *adev = vinst->adev;
623 int i = vinst->inst;
624 uint32_t size;
625 uint32_t offset;
626
627 if (adev->vcn.harvest_config & (1 << i))
628 return;
629
630 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4);
631 /* cache window 0: fw */
632 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
633 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
634 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo));
635 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
636 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi));
637 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0, 0);
638 offset = 0;
639 } else {
640 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
641 lower_32_bits(adev->vcn.inst[i].gpu_addr));
642 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
643 upper_32_bits(adev->vcn.inst[i].gpu_addr));
644 offset = size;
645 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0,
646 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
647 }
648 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE0, size);
649
650 /* cache window 1: stack */
651 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
652 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
653 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
654 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
655 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET1, 0);
656 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
657
658 /* cache window 2: context */
659 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
660 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
661 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
662 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
663 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET2, 0);
664 WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
665
666 /* non-cache window */
667 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
668 lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
669 WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
670 upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
671 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
672 WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_SIZE0,
673 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
674 }
675
vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_vcn_inst * vinst,bool indirect)676 static void vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst,
677 bool indirect)
678 {
679 struct amdgpu_device *adev = vinst->adev;
680 int inst_idx = vinst->inst;
681 uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[inst_idx].fw->size + 4);
682 uint32_t offset;
683
684 /* cache window 0: fw */
685 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
686 if (!indirect) {
687 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
688 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
689 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
690 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
691 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
692 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
693 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
694 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
695 } else {
696 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
697 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
698 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
699 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
700 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
701 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
702 }
703 offset = 0;
704 } else {
705 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
706 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
707 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
708 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
709 VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
710 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
711 offset = size;
712 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
713 VCN, 0, mmUVD_VCPU_CACHE_OFFSET0),
714 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
715 }
716
717 if (!indirect)
718 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
719 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
720 else
721 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
722 VCN, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
723
724 /* cache window 1: stack */
725 if (!indirect) {
726 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
727 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
728 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
729 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
730 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
731 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
732 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
733 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
734 } else {
735 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
736 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
737 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
738 VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
739 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
740 VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
741 }
742 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
743 VCN, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
744
745 /* cache window 2: context */
746 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
747 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
748 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
749 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
750 VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
751 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
752 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
753 VCN, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
754 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
755 VCN, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
756
757 /* non-cache window */
758 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
759 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
760 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
761 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
762 VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
763 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
764 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
765 VCN, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
766 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
767 VCN, 0, mmUVD_VCPU_NONCACHE_SIZE0),
768 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
769
770 /* VCN global tiling registers */
771 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
772 VCN, 0, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
773 }
774
775 /**
776 * vcn_v2_5_disable_clock_gating - disable VCN clock gating
777 *
778 * @vinst: VCN instance
779 *
780 * Disable clock gating for VCN block
781 */
vcn_v2_5_disable_clock_gating(struct amdgpu_vcn_inst * vinst)782 static void vcn_v2_5_disable_clock_gating(struct amdgpu_vcn_inst *vinst)
783 {
784 struct amdgpu_device *adev = vinst->adev;
785 int i = vinst->inst;
786 uint32_t data;
787
788 if (adev->vcn.harvest_config & (1 << i))
789 return;
790 /* UVD disable CGC */
791 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
792 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
793 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
794 else
795 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
796 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
797 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
798 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
799
800 data = RREG32_SOC15(VCN, i, mmUVD_CGC_GATE);
801 data &= ~(UVD_CGC_GATE__SYS_MASK
802 | UVD_CGC_GATE__UDEC_MASK
803 | UVD_CGC_GATE__MPEG2_MASK
804 | UVD_CGC_GATE__REGS_MASK
805 | UVD_CGC_GATE__RBC_MASK
806 | UVD_CGC_GATE__LMI_MC_MASK
807 | UVD_CGC_GATE__LMI_UMC_MASK
808 | UVD_CGC_GATE__IDCT_MASK
809 | UVD_CGC_GATE__MPRD_MASK
810 | UVD_CGC_GATE__MPC_MASK
811 | UVD_CGC_GATE__LBSI_MASK
812 | UVD_CGC_GATE__LRBBM_MASK
813 | UVD_CGC_GATE__UDEC_RE_MASK
814 | UVD_CGC_GATE__UDEC_CM_MASK
815 | UVD_CGC_GATE__UDEC_IT_MASK
816 | UVD_CGC_GATE__UDEC_DB_MASK
817 | UVD_CGC_GATE__UDEC_MP_MASK
818 | UVD_CGC_GATE__WCB_MASK
819 | UVD_CGC_GATE__VCPU_MASK
820 | UVD_CGC_GATE__MMSCH_MASK);
821
822 WREG32_SOC15(VCN, i, mmUVD_CGC_GATE, data);
823
824 SOC15_WAIT_ON_RREG(VCN, i, mmUVD_CGC_GATE, 0, 0xFFFFFFFF);
825
826 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
827 data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
828 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
829 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
830 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
831 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
832 | UVD_CGC_CTRL__SYS_MODE_MASK
833 | UVD_CGC_CTRL__UDEC_MODE_MASK
834 | UVD_CGC_CTRL__MPEG2_MODE_MASK
835 | UVD_CGC_CTRL__REGS_MODE_MASK
836 | UVD_CGC_CTRL__RBC_MODE_MASK
837 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
838 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
839 | UVD_CGC_CTRL__IDCT_MODE_MASK
840 | UVD_CGC_CTRL__MPRD_MODE_MASK
841 | UVD_CGC_CTRL__MPC_MODE_MASK
842 | UVD_CGC_CTRL__LBSI_MODE_MASK
843 | UVD_CGC_CTRL__LRBBM_MODE_MASK
844 | UVD_CGC_CTRL__WCB_MODE_MASK
845 | UVD_CGC_CTRL__VCPU_MODE_MASK
846 | UVD_CGC_CTRL__MMSCH_MODE_MASK);
847 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
848
849 /* turn on */
850 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE);
851 data |= (UVD_SUVD_CGC_GATE__SRE_MASK
852 | UVD_SUVD_CGC_GATE__SIT_MASK
853 | UVD_SUVD_CGC_GATE__SMP_MASK
854 | UVD_SUVD_CGC_GATE__SCM_MASK
855 | UVD_SUVD_CGC_GATE__SDB_MASK
856 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
857 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
858 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
859 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
860 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
861 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
862 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
863 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
864 | UVD_SUVD_CGC_GATE__SCLR_MASK
865 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
866 | UVD_SUVD_CGC_GATE__ENT_MASK
867 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
868 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
869 | UVD_SUVD_CGC_GATE__SITE_MASK
870 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
871 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
872 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
873 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
874 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
875 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE, data);
876
877 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
878 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
879 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
880 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
881 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
882 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
883 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
884 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
885 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
886 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
887 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
888 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
889 }
890
vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_vcn_inst * vinst,uint8_t sram_sel,uint8_t indirect)891 static void vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst,
892 uint8_t sram_sel, uint8_t indirect)
893 {
894 struct amdgpu_device *adev = vinst->adev;
895 int inst_idx = vinst->inst;
896 uint32_t reg_data = 0;
897
898 /* enable sw clock gating control */
899 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
900 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
901 else
902 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
903 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
904 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
905 reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
906 UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
907 UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
908 UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
909 UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
910 UVD_CGC_CTRL__SYS_MODE_MASK |
911 UVD_CGC_CTRL__UDEC_MODE_MASK |
912 UVD_CGC_CTRL__MPEG2_MODE_MASK |
913 UVD_CGC_CTRL__REGS_MODE_MASK |
914 UVD_CGC_CTRL__RBC_MODE_MASK |
915 UVD_CGC_CTRL__LMI_MC_MODE_MASK |
916 UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
917 UVD_CGC_CTRL__IDCT_MODE_MASK |
918 UVD_CGC_CTRL__MPRD_MODE_MASK |
919 UVD_CGC_CTRL__MPC_MODE_MASK |
920 UVD_CGC_CTRL__LBSI_MODE_MASK |
921 UVD_CGC_CTRL__LRBBM_MODE_MASK |
922 UVD_CGC_CTRL__WCB_MODE_MASK |
923 UVD_CGC_CTRL__VCPU_MODE_MASK |
924 UVD_CGC_CTRL__MMSCH_MODE_MASK);
925 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
926 VCN, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
927
928 /* turn off clock gating */
929 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
930 VCN, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
931
932 /* turn on SUVD clock gating */
933 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
934 VCN, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
935
936 /* turn on sw mode in UVD_SUVD_CGC_CTRL */
937 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
938 VCN, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
939 }
940
941 /**
942 * vcn_v2_5_enable_clock_gating - enable VCN clock gating
943 *
944 * @vinst: VCN instance
945 *
946 * Enable clock gating for VCN block
947 */
vcn_v2_5_enable_clock_gating(struct amdgpu_vcn_inst * vinst)948 static void vcn_v2_5_enable_clock_gating(struct amdgpu_vcn_inst *vinst)
949 {
950 struct amdgpu_device *adev = vinst->adev;
951 int i = vinst->inst;
952 uint32_t data = 0;
953
954 if (adev->vcn.harvest_config & (1 << i))
955 return;
956 /* enable UVD CGC */
957 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
958 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
959 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
960 else
961 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
962 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
963 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
964 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
965
966 data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
967 data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
968 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
969 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
970 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
971 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
972 | UVD_CGC_CTRL__SYS_MODE_MASK
973 | UVD_CGC_CTRL__UDEC_MODE_MASK
974 | UVD_CGC_CTRL__MPEG2_MODE_MASK
975 | UVD_CGC_CTRL__REGS_MODE_MASK
976 | UVD_CGC_CTRL__RBC_MODE_MASK
977 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
978 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
979 | UVD_CGC_CTRL__IDCT_MODE_MASK
980 | UVD_CGC_CTRL__MPRD_MODE_MASK
981 | UVD_CGC_CTRL__MPC_MODE_MASK
982 | UVD_CGC_CTRL__LBSI_MODE_MASK
983 | UVD_CGC_CTRL__LRBBM_MODE_MASK
984 | UVD_CGC_CTRL__WCB_MODE_MASK
985 | UVD_CGC_CTRL__VCPU_MODE_MASK);
986 WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
987
988 data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
989 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
990 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
991 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
992 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
993 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
994 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
995 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
996 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
997 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
998 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
999 WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
1000 }
1001
vcn_v2_6_enable_ras(struct amdgpu_vcn_inst * vinst,bool indirect)1002 static void vcn_v2_6_enable_ras(struct amdgpu_vcn_inst *vinst,
1003 bool indirect)
1004 {
1005 struct amdgpu_device *adev = vinst->adev;
1006 int inst_idx = vinst->inst;
1007 uint32_t tmp;
1008
1009 if (amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(2, 6, 0))
1010 return;
1011
1012 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK |
1013 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK |
1014 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK |
1015 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK;
1016 WREG32_SOC15_DPG_MODE(inst_idx,
1017 SOC15_DPG_MODE_OFFSET(VCN, 0, mmVCN_RAS_CNTL),
1018 tmp, 0, indirect);
1019
1020 tmp = UVD_VCPU_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;
1021 WREG32_SOC15_DPG_MODE(inst_idx,
1022 SOC15_DPG_MODE_OFFSET(VCN, 0, mmUVD_VCPU_INT_EN),
1023 tmp, 0, indirect);
1024
1025 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;
1026 WREG32_SOC15_DPG_MODE(inst_idx,
1027 SOC15_DPG_MODE_OFFSET(VCN, 0, mmUVD_SYS_INT_EN),
1028 tmp, 0, indirect);
1029 }
1030
vcn_v2_5_start_dpg_mode(struct amdgpu_vcn_inst * vinst,bool indirect)1031 static int vcn_v2_5_start_dpg_mode(struct amdgpu_vcn_inst *vinst, bool indirect)
1032 {
1033 struct amdgpu_device *adev = vinst->adev;
1034 int inst_idx = vinst->inst;
1035 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1036 struct amdgpu_ring *ring;
1037 uint32_t rb_bufsz, tmp;
1038
1039 /* disable register anti-hang mechanism */
1040 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
1041 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1042 /* enable dynamic power gating mode */
1043 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
1044 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
1045 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
1046 WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
1047
1048 if (indirect)
1049 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
1050
1051 /* enable clock gating */
1052 vcn_v2_5_clock_gating_dpg_mode(vinst, 0, indirect);
1053
1054 /* enable VCPU clock */
1055 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1056 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1057 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
1058 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1059 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1060
1061 /* disable master interupt */
1062 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1063 VCN, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
1064
1065 /* setup mmUVD_LMI_CTRL */
1066 tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1067 UVD_LMI_CTRL__REQ_MODE_MASK |
1068 UVD_LMI_CTRL__CRC_RESET_MASK |
1069 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1070 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1071 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1072 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1073 0x00100000L);
1074 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1075 VCN, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
1076
1077 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1078 VCN, 0, mmUVD_MPC_CNTL),
1079 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
1080
1081 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1082 VCN, 0, mmUVD_MPC_SET_MUXA0),
1083 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1084 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1085 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1086 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
1087
1088 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1089 VCN, 0, mmUVD_MPC_SET_MUXB0),
1090 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1091 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1092 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1093 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1094
1095 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1096 VCN, 0, mmUVD_MPC_SET_MUX),
1097 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1098 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1099 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1100
1101 vcn_v2_5_mc_resume_dpg_mode(vinst, indirect);
1102
1103 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1104 VCN, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1105 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1106 VCN, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1107
1108 /* enable LMI MC and UMC channels */
1109 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1110 VCN, 0, mmUVD_LMI_CTRL2), 0, 0, indirect);
1111
1112 vcn_v2_6_enable_ras(vinst, indirect);
1113
1114 /* unblock VCPU register access */
1115 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1116 VCN, 0, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1117
1118 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1119 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1120 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1121 VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1122
1123 /* enable master interrupt */
1124 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1125 VCN, 0, mmUVD_MASTINT_EN),
1126 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1127
1128 if (indirect)
1129 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
1130
1131 ring = &adev->vcn.inst[inst_idx].ring_dec;
1132 /* force RBC into idle state */
1133 rb_bufsz = order_base_2(ring->ring_size);
1134 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1135 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1136 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1137 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1138 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1139 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1140
1141 /* Stall DPG before WPTR/RPTR reset */
1142 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1143 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1144 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1145 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1146
1147 /* set the write pointer delay */
1148 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1149
1150 /* set the wb address */
1151 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1152 (upper_32_bits(ring->gpu_addr) >> 2));
1153
1154 /* program the RB_BASE for ring buffer */
1155 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1156 lower_32_bits(ring->gpu_addr));
1157 WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1158 upper_32_bits(ring->gpu_addr));
1159
1160 /* Initialize the ring buffer's read and write pointers */
1161 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1162
1163 WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1164
1165 ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1166 WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1167 lower_32_bits(ring->wptr));
1168
1169 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1170 /* Unstall DPG */
1171 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1172 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1173
1174 /* Keeping one read-back to ensure all register writes are done,
1175 * otherwise it may introduce race conditions.
1176 */
1177 RREG32_SOC15(VCN, inst_idx, mmUVD_STATUS);
1178
1179 return 0;
1180 }
1181
vcn_v2_5_start(struct amdgpu_vcn_inst * vinst)1182 static int vcn_v2_5_start(struct amdgpu_vcn_inst *vinst)
1183 {
1184 struct amdgpu_device *adev = vinst->adev;
1185 int i = vinst->inst;
1186 volatile struct amdgpu_fw_shared *fw_shared =
1187 adev->vcn.inst[i].fw_shared.cpu_addr;
1188 struct amdgpu_ring *ring;
1189 uint32_t rb_bufsz, tmp;
1190 int j, k, r;
1191
1192 if (adev->vcn.harvest_config & (1 << i))
1193 return 0;
1194
1195 if (adev->pm.dpm_enabled)
1196 amdgpu_dpm_enable_vcn(adev, true, i);
1197
1198 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1199 return vcn_v2_5_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram);
1200
1201 /* disable register anti-hang mechanism */
1202 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS), 0,
1203 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1204
1205 /* set uvd status busy */
1206 tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1207 WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1208
1209 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1210 return 0;
1211
1212 /* SW clock gating */
1213 vcn_v2_5_disable_clock_gating(vinst);
1214
1215 /* enable VCPU clock */
1216 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1217 UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1218
1219 /* disable master interrupt */
1220 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1221 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1222
1223 /* setup mmUVD_LMI_CTRL */
1224 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1225 tmp &= ~0xff;
1226 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp | 0x8|
1227 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1228 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1229 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1230 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1231
1232 /* setup mmUVD_MPC_CNTL */
1233 tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1234 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1235 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1236 WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1237
1238 /* setup UVD_MPC_SET_MUXA0 */
1239 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1240 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1241 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1242 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1243 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1244
1245 /* setup UVD_MPC_SET_MUXB0 */
1246 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1247 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1248 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1249 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1250 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1251
1252 /* setup mmUVD_MPC_SET_MUX */
1253 WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1254 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1255 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1256 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1257
1258 vcn_v2_5_mc_resume(vinst);
1259
1260 /* VCN global tiling registers */
1261 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1262 adev->gfx.config.gb_addr_config);
1263 WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1264 adev->gfx.config.gb_addr_config);
1265
1266 /* enable LMI MC and UMC channels */
1267 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1268 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1269
1270 /* unblock VCPU register access */
1271 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1272 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1273
1274 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1275 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1276
1277 for (k = 0; k < 10; ++k) {
1278 uint32_t status;
1279
1280 for (j = 0; j < 100; ++j) {
1281 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1282 if (status & 2)
1283 break;
1284 if (amdgpu_emu_mode == 1)
1285 msleep(500);
1286 else
1287 mdelay(10);
1288 }
1289 r = 0;
1290 if (status & 2)
1291 break;
1292
1293 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
1294 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1295 UVD_VCPU_CNTL__BLK_RST_MASK,
1296 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1297 mdelay(10);
1298 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1299 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1300
1301 mdelay(10);
1302 r = -1;
1303 }
1304
1305 if (r) {
1306 DRM_ERROR("VCN decode not responding, giving up!!!\n");
1307 return r;
1308 }
1309
1310 /* enable master interrupt */
1311 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1312 UVD_MASTINT_EN__VCPU_EN_MASK,
1313 ~UVD_MASTINT_EN__VCPU_EN_MASK);
1314
1315 /* clear the busy bit of VCN_STATUS */
1316 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1317 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1318
1319 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1320
1321 ring = &adev->vcn.inst[i].ring_dec;
1322 /* force RBC into idle state */
1323 rb_bufsz = order_base_2(ring->ring_size);
1324 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1325 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1326 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1327 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1328 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1329 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1330
1331 fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1332 /* program the RB_BASE for ring buffer */
1333 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1334 lower_32_bits(ring->gpu_addr));
1335 WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1336 upper_32_bits(ring->gpu_addr));
1337
1338 /* Initialize the ring buffer's read and write pointers */
1339 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1340
1341 ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1342 WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1343 lower_32_bits(ring->wptr));
1344 fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1345
1346 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1347 ring = &adev->vcn.inst[i].ring_enc[0];
1348 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1349 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1350 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1351 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1352 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1353 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1354
1355 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1356 ring = &adev->vcn.inst[i].ring_enc[1];
1357 WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1358 WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1359 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1360 WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1361 WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1362 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1363
1364 /* Keeping one read-back to ensure all register writes are done,
1365 * otherwise it may introduce race conditions.
1366 */
1367 RREG32_SOC15(VCN, i, mmUVD_STATUS);
1368
1369 return 0;
1370 }
1371
vcn_v2_5_mmsch_start(struct amdgpu_device * adev,struct amdgpu_mm_table * table)1372 static int vcn_v2_5_mmsch_start(struct amdgpu_device *adev,
1373 struct amdgpu_mm_table *table)
1374 {
1375 uint32_t data = 0, loop = 0, size = 0;
1376 uint64_t addr = table->gpu_addr;
1377 struct mmsch_v1_1_init_header *header = NULL;
1378
1379 header = (struct mmsch_v1_1_init_header *)table->cpu_addr;
1380 size = header->total_size;
1381
1382 /*
1383 * 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of
1384 * memory descriptor location
1385 */
1386 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));
1387 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));
1388
1389 /* 2, update vmid of descriptor */
1390 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1391 data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1392 /* use domain0 for MM scheduler */
1393 data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1394 WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, data);
1395
1396 /* 3, notify mmsch about the size of this descriptor */
1397 WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1398
1399 /* 4, set resp to zero */
1400 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1401
1402 /*
1403 * 5, kick off the initialization and wait until
1404 * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero
1405 */
1406 WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001);
1407
1408 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1409 loop = 10;
1410 while ((data & 0x10000002) != 0x10000002) {
1411 udelay(100);
1412 data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1413 loop--;
1414 if (!loop)
1415 break;
1416 }
1417
1418 if (!loop) {
1419 dev_err(adev->dev,
1420 "failed to init MMSCH, mmMMSCH_VF_MAILBOX_RESP = %x\n",
1421 data);
1422 return -EBUSY;
1423 }
1424
1425 return 0;
1426 }
1427
vcn_v2_5_sriov_start(struct amdgpu_device * adev)1428 static int vcn_v2_5_sriov_start(struct amdgpu_device *adev)
1429 {
1430 struct amdgpu_ring *ring;
1431 uint32_t offset, size, tmp, i, rb_bufsz;
1432 uint32_t table_size = 0;
1433 struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };
1434 struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };
1435 struct mmsch_v1_0_cmd_end end = { { 0 } };
1436 uint32_t *init_table = adev->virt.mm_table.cpu_addr;
1437 struct mmsch_v1_1_init_header *header = (struct mmsch_v1_1_init_header *)init_table;
1438
1439 direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
1440 direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1441 end.cmd_header.command_type = MMSCH_COMMAND__END;
1442
1443 header->version = MMSCH_VERSION;
1444 header->total_size = sizeof(struct mmsch_v1_1_init_header) >> 2;
1445 init_table += header->total_size;
1446
1447 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1448 header->eng[i].table_offset = header->total_size;
1449 header->eng[i].init_status = 0;
1450 header->eng[i].table_size = 0;
1451
1452 table_size = 0;
1453
1454 MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(
1455 SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS),
1456 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1457
1458 size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4);
1459 /* mc resume*/
1460 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1461 MMSCH_V1_0_INSERT_DIRECT_WT(
1462 SOC15_REG_OFFSET(VCN, i,
1463 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1464 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1465 MMSCH_V1_0_INSERT_DIRECT_WT(
1466 SOC15_REG_OFFSET(VCN, i,
1467 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1468 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1469 offset = 0;
1470 MMSCH_V1_0_INSERT_DIRECT_WT(
1471 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0), 0);
1472 } else {
1473 MMSCH_V1_0_INSERT_DIRECT_WT(
1474 SOC15_REG_OFFSET(VCN, i,
1475 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1476 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1477 MMSCH_V1_0_INSERT_DIRECT_WT(
1478 SOC15_REG_OFFSET(VCN, i,
1479 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1480 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1481 offset = size;
1482 MMSCH_V1_0_INSERT_DIRECT_WT(
1483 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0),
1484 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1485 }
1486
1487 MMSCH_V1_0_INSERT_DIRECT_WT(
1488 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE0),
1489 size);
1490 MMSCH_V1_0_INSERT_DIRECT_WT(
1491 SOC15_REG_OFFSET(VCN, i,
1492 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1493 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1494 MMSCH_V1_0_INSERT_DIRECT_WT(
1495 SOC15_REG_OFFSET(VCN, i,
1496 mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1497 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1498 MMSCH_V1_0_INSERT_DIRECT_WT(
1499 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET1),
1500 0);
1501 MMSCH_V1_0_INSERT_DIRECT_WT(
1502 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE1),
1503 AMDGPU_VCN_STACK_SIZE);
1504 MMSCH_V1_0_INSERT_DIRECT_WT(
1505 SOC15_REG_OFFSET(VCN, i,
1506 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1507 lower_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1508 AMDGPU_VCN_STACK_SIZE));
1509 MMSCH_V1_0_INSERT_DIRECT_WT(
1510 SOC15_REG_OFFSET(VCN, i,
1511 mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1512 upper_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1513 AMDGPU_VCN_STACK_SIZE));
1514 MMSCH_V1_0_INSERT_DIRECT_WT(
1515 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET2),
1516 0);
1517 MMSCH_V1_0_INSERT_DIRECT_WT(
1518 SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE2),
1519 AMDGPU_VCN_CONTEXT_SIZE);
1520
1521 ring = &adev->vcn.inst[i].ring_enc[0];
1522 ring->wptr = 0;
1523
1524 MMSCH_V1_0_INSERT_DIRECT_WT(
1525 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_LO),
1526 lower_32_bits(ring->gpu_addr));
1527 MMSCH_V1_0_INSERT_DIRECT_WT(
1528 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_HI),
1529 upper_32_bits(ring->gpu_addr));
1530 MMSCH_V1_0_INSERT_DIRECT_WT(
1531 SOC15_REG_OFFSET(VCN, i, mmUVD_RB_SIZE),
1532 ring->ring_size / 4);
1533
1534 ring = &adev->vcn.inst[i].ring_dec;
1535 ring->wptr = 0;
1536 MMSCH_V1_0_INSERT_DIRECT_WT(
1537 SOC15_REG_OFFSET(VCN, i,
1538 mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1539 lower_32_bits(ring->gpu_addr));
1540 MMSCH_V1_0_INSERT_DIRECT_WT(
1541 SOC15_REG_OFFSET(VCN, i,
1542 mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1543 upper_32_bits(ring->gpu_addr));
1544
1545 /* force RBC into idle state */
1546 rb_bufsz = order_base_2(ring->ring_size);
1547 tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1548 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1549 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1550 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1551 tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1552 MMSCH_V1_0_INSERT_DIRECT_WT(
1553 SOC15_REG_OFFSET(VCN, i, mmUVD_RBC_RB_CNTL), tmp);
1554
1555 /* add end packet */
1556 memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
1557 table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1558 init_table += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1559
1560 /* refine header */
1561 header->eng[i].table_size = table_size;
1562 header->total_size += table_size;
1563 }
1564
1565 return vcn_v2_5_mmsch_start(adev, &adev->virt.mm_table);
1566 }
1567
vcn_v2_5_stop_dpg_mode(struct amdgpu_vcn_inst * vinst)1568 static int vcn_v2_5_stop_dpg_mode(struct amdgpu_vcn_inst *vinst)
1569 {
1570 struct amdgpu_device *adev = vinst->adev;
1571 int inst_idx = vinst->inst;
1572 uint32_t tmp;
1573
1574 /* Wait for power status to be 1 */
1575 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1576 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1577
1578 /* wait for read ptr to be equal to write ptr */
1579 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1580 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1581
1582 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1583 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1584
1585 tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1586 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1587
1588 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1589 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1590
1591 /* disable dynamic power gating mode */
1592 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1593 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1594
1595 /* Keeping one read-back to ensure all register writes are done,
1596 * otherwise it may introduce race conditions.
1597 */
1598 RREG32_SOC15(VCN, inst_idx, mmUVD_STATUS);
1599
1600 return 0;
1601 }
1602
vcn_v2_5_stop(struct amdgpu_vcn_inst * vinst)1603 static int vcn_v2_5_stop(struct amdgpu_vcn_inst *vinst)
1604 {
1605 struct amdgpu_device *adev = vinst->adev;
1606 int i = vinst->inst;
1607 uint32_t tmp;
1608 int r;
1609
1610 if (adev->vcn.harvest_config & (1 << i))
1611 return 0;
1612
1613 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1614 r = vcn_v2_5_stop_dpg_mode(vinst);
1615 goto done;
1616 }
1617
1618 /* wait for vcn idle */
1619 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1620 if (r)
1621 goto done;
1622
1623 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1624 UVD_LMI_STATUS__READ_CLEAN_MASK |
1625 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1626 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1627 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1628 if (r)
1629 goto done;
1630
1631 /* block LMI UMC channel */
1632 tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1633 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1634 WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1635
1636 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1637 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1638 r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1639 if (r)
1640 goto done;
1641
1642 /* block VCPU register access */
1643 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1644 UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1645 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1646
1647 /* reset VCPU */
1648 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1649 UVD_VCPU_CNTL__BLK_RST_MASK,
1650 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1651
1652 /* disable VCPU clock */
1653 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1654 ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1655
1656 /* clear status */
1657 WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1658
1659 vcn_v2_5_enable_clock_gating(vinst);
1660
1661 /* enable register anti-hang mechanism */
1662 WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS),
1663 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK,
1664 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1665
1666 /* Keeping one read-back to ensure all register writes are done,
1667 * otherwise it may introduce race conditions.
1668 */
1669 RREG32_SOC15(VCN, i, mmUVD_STATUS);
1670 done:
1671 if (adev->pm.dpm_enabled)
1672 amdgpu_dpm_enable_vcn(adev, false, i);
1673
1674 return r;
1675 }
1676
vcn_v2_5_pause_dpg_mode(struct amdgpu_vcn_inst * vinst,struct dpg_pause_state * new_state)1677 static int vcn_v2_5_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
1678 struct dpg_pause_state *new_state)
1679 {
1680 struct amdgpu_device *adev = vinst->adev;
1681 int inst_idx = vinst->inst;
1682 struct amdgpu_ring *ring;
1683 uint32_t reg_data = 0;
1684 int ret_code = 0;
1685
1686 /* pause/unpause if state is changed */
1687 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1688 DRM_DEBUG("dpg pause state changed %d -> %d",
1689 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1690 reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1691 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1692
1693 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1694 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1695 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1696
1697 if (!ret_code) {
1698 volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1699
1700 /* pause DPG */
1701 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1702 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1703
1704 /* wait for ACK */
1705 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1706 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1707 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1708
1709 /* Stall DPG before WPTR/RPTR reset */
1710 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1711 UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1712 ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1713
1714 /* Restore */
1715 fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1716 ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1717 ring->wptr = 0;
1718 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1719 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1720 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1721 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1722 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1723 fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1724
1725 fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1726 ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1727 ring->wptr = 0;
1728 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1729 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1730 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1731 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1732 WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1733 fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1734
1735 /* Unstall DPG */
1736 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1737 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1738
1739 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1740 UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1741 }
1742 } else {
1743 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1744 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1745 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1746 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1747 }
1748 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1749 }
1750
1751 return 0;
1752 }
1753
1754 /**
1755 * vcn_v2_5_dec_ring_get_rptr - get read pointer
1756 *
1757 * @ring: amdgpu_ring pointer
1758 *
1759 * Returns the current hardware read pointer
1760 */
vcn_v2_5_dec_ring_get_rptr(struct amdgpu_ring * ring)1761 static uint64_t vcn_v2_5_dec_ring_get_rptr(struct amdgpu_ring *ring)
1762 {
1763 struct amdgpu_device *adev = ring->adev;
1764
1765 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1766 }
1767
1768 /**
1769 * vcn_v2_5_dec_ring_get_wptr - get write pointer
1770 *
1771 * @ring: amdgpu_ring pointer
1772 *
1773 * Returns the current hardware write pointer
1774 */
vcn_v2_5_dec_ring_get_wptr(struct amdgpu_ring * ring)1775 static uint64_t vcn_v2_5_dec_ring_get_wptr(struct amdgpu_ring *ring)
1776 {
1777 struct amdgpu_device *adev = ring->adev;
1778
1779 if (ring->use_doorbell)
1780 return *ring->wptr_cpu_addr;
1781 else
1782 return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1783 }
1784
1785 /**
1786 * vcn_v2_5_dec_ring_set_wptr - set write pointer
1787 *
1788 * @ring: amdgpu_ring pointer
1789 *
1790 * Commits the write pointer to the hardware
1791 */
vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring * ring)1792 static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
1793 {
1794 struct amdgpu_device *adev = ring->adev;
1795
1796 if (ring->use_doorbell) {
1797 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1798 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1799 } else {
1800 WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1801 }
1802 }
1803
1804 static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {
1805 .type = AMDGPU_RING_TYPE_VCN_DEC,
1806 .align_mask = 0xf,
1807 .secure_submission_supported = true,
1808 .get_rptr = vcn_v2_5_dec_ring_get_rptr,
1809 .get_wptr = vcn_v2_5_dec_ring_get_wptr,
1810 .set_wptr = vcn_v2_5_dec_ring_set_wptr,
1811 .emit_frame_size =
1812 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1813 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1814 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1815 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1816 6,
1817 .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1818 .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1819 .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1820 .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1821 .test_ring = vcn_v2_0_dec_ring_test_ring,
1822 .test_ib = amdgpu_vcn_dec_ring_test_ib,
1823 .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1824 .insert_start = vcn_v2_0_dec_ring_insert_start,
1825 .insert_end = vcn_v2_0_dec_ring_insert_end,
1826 .pad_ib = amdgpu_ring_generic_pad_ib,
1827 .begin_use = vcn_v2_5_ring_begin_use,
1828 .end_use = vcn_v2_5_ring_end_use,
1829 .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1830 .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1831 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1832 .reset = amdgpu_vcn_ring_reset,
1833 };
1834
1835 /**
1836 * vcn_v2_5_enc_ring_get_rptr - get enc read pointer
1837 *
1838 * @ring: amdgpu_ring pointer
1839 *
1840 * Returns the current hardware enc read pointer
1841 */
vcn_v2_5_enc_ring_get_rptr(struct amdgpu_ring * ring)1842 static uint64_t vcn_v2_5_enc_ring_get_rptr(struct amdgpu_ring *ring)
1843 {
1844 struct amdgpu_device *adev = ring->adev;
1845
1846 if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1847 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1848 else
1849 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1850 }
1851
1852 /**
1853 * vcn_v2_5_enc_ring_get_wptr - get enc write pointer
1854 *
1855 * @ring: amdgpu_ring pointer
1856 *
1857 * Returns the current hardware enc write pointer
1858 */
vcn_v2_5_enc_ring_get_wptr(struct amdgpu_ring * ring)1859 static uint64_t vcn_v2_5_enc_ring_get_wptr(struct amdgpu_ring *ring)
1860 {
1861 struct amdgpu_device *adev = ring->adev;
1862
1863 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1864 if (ring->use_doorbell)
1865 return *ring->wptr_cpu_addr;
1866 else
1867 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1868 } else {
1869 if (ring->use_doorbell)
1870 return *ring->wptr_cpu_addr;
1871 else
1872 return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1873 }
1874 }
1875
1876 /**
1877 * vcn_v2_5_enc_ring_set_wptr - set enc write pointer
1878 *
1879 * @ring: amdgpu_ring pointer
1880 *
1881 * Commits the enc write pointer to the hardware
1882 */
vcn_v2_5_enc_ring_set_wptr(struct amdgpu_ring * ring)1883 static void vcn_v2_5_enc_ring_set_wptr(struct amdgpu_ring *ring)
1884 {
1885 struct amdgpu_device *adev = ring->adev;
1886
1887 if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1888 if (ring->use_doorbell) {
1889 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1890 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1891 } else {
1892 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1893 }
1894 } else {
1895 if (ring->use_doorbell) {
1896 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1897 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1898 } else {
1899 WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1900 }
1901 }
1902 }
1903
1904 static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
1905 .type = AMDGPU_RING_TYPE_VCN_ENC,
1906 .align_mask = 0x3f,
1907 .nop = VCN_ENC_CMD_NO_OP,
1908 .get_rptr = vcn_v2_5_enc_ring_get_rptr,
1909 .get_wptr = vcn_v2_5_enc_ring_get_wptr,
1910 .set_wptr = vcn_v2_5_enc_ring_set_wptr,
1911 .emit_frame_size =
1912 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1913 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1914 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1915 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1916 1, /* vcn_v2_0_enc_ring_insert_end */
1917 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1918 .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1919 .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1920 .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1921 .test_ring = amdgpu_vcn_enc_ring_test_ring,
1922 .test_ib = amdgpu_vcn_enc_ring_test_ib,
1923 .insert_nop = amdgpu_ring_insert_nop,
1924 .insert_end = vcn_v2_0_enc_ring_insert_end,
1925 .pad_ib = amdgpu_ring_generic_pad_ib,
1926 .begin_use = vcn_v2_5_ring_begin_use,
1927 .end_use = vcn_v2_5_ring_end_use,
1928 .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1929 .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1930 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1931 .reset = amdgpu_vcn_ring_reset,
1932 };
1933
vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device * adev)1934 static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev)
1935 {
1936 int i;
1937
1938 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1939 if (adev->vcn.harvest_config & (1 << i))
1940 continue;
1941 adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs;
1942 adev->vcn.inst[i].ring_dec.me = i;
1943 }
1944 }
1945
vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device * adev)1946 static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev)
1947 {
1948 int i, j;
1949
1950 for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
1951 if (adev->vcn.harvest_config & (1 << j))
1952 continue;
1953 for (i = 0; i < adev->vcn.inst[j].num_enc_rings; ++i) {
1954 adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs;
1955 adev->vcn.inst[j].ring_enc[i].me = j;
1956 }
1957 }
1958 }
1959
vcn_v2_5_reset(struct amdgpu_vcn_inst * vinst)1960 static int vcn_v2_5_reset(struct amdgpu_vcn_inst *vinst)
1961 {
1962 int r;
1963
1964 r = vcn_v2_5_stop(vinst);
1965 if (r)
1966 return r;
1967 return vcn_v2_5_start(vinst);
1968 }
1969
vcn_v2_5_is_idle(struct amdgpu_ip_block * ip_block)1970 static bool vcn_v2_5_is_idle(struct amdgpu_ip_block *ip_block)
1971 {
1972 struct amdgpu_device *adev = ip_block->adev;
1973 int i, ret = 1;
1974
1975 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1976 if (adev->vcn.harvest_config & (1 << i))
1977 continue;
1978
1979 ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
1980 }
1981
1982 return ret;
1983 }
1984
vcn_v2_5_wait_for_idle(struct amdgpu_ip_block * ip_block)1985 static int vcn_v2_5_wait_for_idle(struct amdgpu_ip_block *ip_block)
1986 {
1987 struct amdgpu_device *adev = ip_block->adev;
1988 int i, ret = 0;
1989
1990 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1991 if (adev->vcn.harvest_config & (1 << i))
1992 continue;
1993 ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
1994 UVD_STATUS__IDLE);
1995 if (ret)
1996 return ret;
1997 }
1998
1999 return ret;
2000 }
2001
vcn_v2_5_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)2002 static int vcn_v2_5_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2003 enum amd_clockgating_state state)
2004 {
2005 struct amdgpu_device *adev = ip_block->adev;
2006 bool enable = (state == AMD_CG_STATE_GATE);
2007 int i;
2008
2009 if (amdgpu_sriov_vf(adev))
2010 return 0;
2011
2012 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2013 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
2014
2015 if (enable) {
2016 if (!vcn_v2_5_is_idle(ip_block))
2017 return -EBUSY;
2018 vcn_v2_5_enable_clock_gating(vinst);
2019 } else {
2020 vcn_v2_5_disable_clock_gating(vinst);
2021 }
2022 }
2023
2024 return 0;
2025 }
2026
vcn_v2_5_set_pg_state(struct amdgpu_vcn_inst * vinst,enum amd_powergating_state state)2027 static int vcn_v2_5_set_pg_state(struct amdgpu_vcn_inst *vinst,
2028 enum amd_powergating_state state)
2029 {
2030 struct amdgpu_device *adev = vinst->adev;
2031 int ret;
2032
2033 if (amdgpu_sriov_vf(adev))
2034 return 0;
2035
2036 if (state == vinst->cur_state)
2037 return 0;
2038
2039 if (state == AMD_PG_STATE_GATE)
2040 ret = vcn_v2_5_stop(vinst);
2041 else
2042 ret = vcn_v2_5_start(vinst);
2043
2044 if (!ret)
2045 vinst->cur_state = state;
2046
2047 return ret;
2048 }
2049
vcn_v2_5_set_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)2050 static int vcn_v2_5_set_interrupt_state(struct amdgpu_device *adev,
2051 struct amdgpu_irq_src *source,
2052 unsigned type,
2053 enum amdgpu_interrupt_state state)
2054 {
2055 return 0;
2056 }
2057
vcn_v2_6_set_ras_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)2058 static int vcn_v2_6_set_ras_interrupt_state(struct amdgpu_device *adev,
2059 struct amdgpu_irq_src *source,
2060 unsigned int type,
2061 enum amdgpu_interrupt_state state)
2062 {
2063 return 0;
2064 }
2065
vcn_v2_5_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)2066 static int vcn_v2_5_process_interrupt(struct amdgpu_device *adev,
2067 struct amdgpu_irq_src *source,
2068 struct amdgpu_iv_entry *entry)
2069 {
2070 uint32_t ip_instance;
2071
2072 switch (entry->client_id) {
2073 case SOC15_IH_CLIENTID_VCN:
2074 ip_instance = 0;
2075 break;
2076 case SOC15_IH_CLIENTID_VCN1:
2077 ip_instance = 1;
2078 break;
2079 default:
2080 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2081 return 0;
2082 }
2083
2084 DRM_DEBUG("IH: VCN TRAP\n");
2085
2086 switch (entry->src_id) {
2087 case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2088 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2089 break;
2090 case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2091 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2092 break;
2093 case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2094 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2095 break;
2096 default:
2097 DRM_ERROR("Unhandled interrupt: %d %d\n",
2098 entry->src_id, entry->src_data[0]);
2099 break;
2100 }
2101
2102 return 0;
2103 }
2104
2105 static const struct amdgpu_irq_src_funcs vcn_v2_5_irq_funcs = {
2106 .set = vcn_v2_5_set_interrupt_state,
2107 .process = vcn_v2_5_process_interrupt,
2108 };
2109
2110 static const struct amdgpu_irq_src_funcs vcn_v2_6_ras_irq_funcs = {
2111 .set = vcn_v2_6_set_ras_interrupt_state,
2112 .process = amdgpu_vcn_process_poison_irq,
2113 };
2114
vcn_v2_5_set_irq_funcs(struct amdgpu_device * adev)2115 static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev)
2116 {
2117 int i;
2118
2119 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2120 if (adev->vcn.harvest_config & (1 << i))
2121 continue;
2122 adev->vcn.inst[i].irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
2123 adev->vcn.inst[i].irq.funcs = &vcn_v2_5_irq_funcs;
2124
2125 adev->vcn.inst[i].ras_poison_irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
2126 adev->vcn.inst[i].ras_poison_irq.funcs = &vcn_v2_6_ras_irq_funcs;
2127 }
2128 }
2129
vcn_v2_5_print_ip_state(struct amdgpu_ip_block * ip_block,struct drm_printer * p)2130 static void vcn_v2_5_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
2131 {
2132 struct amdgpu_device *adev = ip_block->adev;
2133 int i, j;
2134 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_5);
2135 uint32_t inst_off, is_powered;
2136
2137 if (!adev->vcn.ip_dump)
2138 return;
2139
2140 drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst);
2141 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2142 if (adev->vcn.harvest_config & (1 << i)) {
2143 drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i);
2144 continue;
2145 }
2146
2147 inst_off = i * reg_count;
2148 is_powered = (adev->vcn.ip_dump[inst_off] &
2149 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;
2150
2151 if (is_powered) {
2152 drm_printf(p, "\nActive Instance:VCN%d\n", i);
2153 for (j = 0; j < reg_count; j++)
2154 drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_2_5[j].reg_name,
2155 adev->vcn.ip_dump[inst_off + j]);
2156 } else {
2157 drm_printf(p, "\nInactive Instance:VCN%d\n", i);
2158 }
2159 }
2160 }
2161
vcn_v2_5_dump_ip_state(struct amdgpu_ip_block * ip_block)2162 static void vcn_v2_5_dump_ip_state(struct amdgpu_ip_block *ip_block)
2163 {
2164 struct amdgpu_device *adev = ip_block->adev;
2165 int i, j;
2166 bool is_powered;
2167 uint32_t inst_off;
2168 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_2_5);
2169
2170 if (!adev->vcn.ip_dump)
2171 return;
2172
2173 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2174 if (adev->vcn.harvest_config & (1 << i))
2175 continue;
2176
2177 inst_off = i * reg_count;
2178 /* mmUVD_POWER_STATUS is always readable and is first element of the array */
2179 adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, i, mmUVD_POWER_STATUS);
2180 is_powered = (adev->vcn.ip_dump[inst_off] &
2181 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1;
2182
2183 if (is_powered)
2184 for (j = 1; j < reg_count; j++)
2185 adev->vcn.ip_dump[inst_off + j] =
2186 RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_2_5[j], i));
2187 }
2188 }
2189
2190 static const struct amd_ip_funcs vcn_v2_5_ip_funcs = {
2191 .name = "vcn_v2_5",
2192 .early_init = vcn_v2_5_early_init,
2193 .sw_init = vcn_v2_5_sw_init,
2194 .sw_fini = vcn_v2_5_sw_fini,
2195 .hw_init = vcn_v2_5_hw_init,
2196 .hw_fini = vcn_v2_5_hw_fini,
2197 .suspend = vcn_v2_5_suspend,
2198 .resume = vcn_v2_5_resume,
2199 .is_idle = vcn_v2_5_is_idle,
2200 .wait_for_idle = vcn_v2_5_wait_for_idle,
2201 .set_clockgating_state = vcn_v2_5_set_clockgating_state,
2202 .set_powergating_state = vcn_set_powergating_state,
2203 .dump_ip_state = vcn_v2_5_dump_ip_state,
2204 .print_ip_state = vcn_v2_5_print_ip_state,
2205 };
2206
2207 static const struct amd_ip_funcs vcn_v2_6_ip_funcs = {
2208 .name = "vcn_v2_6",
2209 .early_init = vcn_v2_5_early_init,
2210 .sw_init = vcn_v2_5_sw_init,
2211 .sw_fini = vcn_v2_5_sw_fini,
2212 .hw_init = vcn_v2_5_hw_init,
2213 .hw_fini = vcn_v2_5_hw_fini,
2214 .suspend = vcn_v2_5_suspend,
2215 .resume = vcn_v2_5_resume,
2216 .is_idle = vcn_v2_5_is_idle,
2217 .wait_for_idle = vcn_v2_5_wait_for_idle,
2218 .set_clockgating_state = vcn_v2_5_set_clockgating_state,
2219 .set_powergating_state = vcn_set_powergating_state,
2220 .dump_ip_state = vcn_v2_5_dump_ip_state,
2221 .print_ip_state = vcn_v2_5_print_ip_state,
2222 };
2223
2224 const struct amdgpu_ip_block_version vcn_v2_5_ip_block =
2225 {
2226 .type = AMD_IP_BLOCK_TYPE_VCN,
2227 .major = 2,
2228 .minor = 5,
2229 .rev = 0,
2230 .funcs = &vcn_v2_5_ip_funcs,
2231 };
2232
2233 const struct amdgpu_ip_block_version vcn_v2_6_ip_block =
2234 {
2235 .type = AMD_IP_BLOCK_TYPE_VCN,
2236 .major = 2,
2237 .minor = 6,
2238 .rev = 0,
2239 .funcs = &vcn_v2_6_ip_funcs,
2240 };
2241
vcn_v2_6_query_poison_by_instance(struct amdgpu_device * adev,uint32_t instance,uint32_t sub_block)2242 static uint32_t vcn_v2_6_query_poison_by_instance(struct amdgpu_device *adev,
2243 uint32_t instance, uint32_t sub_block)
2244 {
2245 uint32_t poison_stat = 0, reg_value = 0;
2246
2247 switch (sub_block) {
2248 case AMDGPU_VCN_V2_6_VCPU_VCODEC:
2249 reg_value = RREG32_SOC15(VCN, instance, mmUVD_RAS_VCPU_VCODEC_STATUS);
2250 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
2251 break;
2252 default:
2253 break;
2254 }
2255
2256 if (poison_stat)
2257 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
2258 instance, sub_block);
2259
2260 return poison_stat;
2261 }
2262
vcn_v2_6_query_poison_status(struct amdgpu_device * adev)2263 static bool vcn_v2_6_query_poison_status(struct amdgpu_device *adev)
2264 {
2265 uint32_t inst, sub;
2266 uint32_t poison_stat = 0;
2267
2268 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
2269 for (sub = 0; sub < AMDGPU_VCN_V2_6_MAX_SUB_BLOCK; sub++)
2270 poison_stat +=
2271 vcn_v2_6_query_poison_by_instance(adev, inst, sub);
2272
2273 return !!poison_stat;
2274 }
2275
2276 const struct amdgpu_ras_block_hw_ops vcn_v2_6_ras_hw_ops = {
2277 .query_poison_status = vcn_v2_6_query_poison_status,
2278 };
2279
2280 static struct amdgpu_vcn_ras vcn_v2_6_ras = {
2281 .ras_block = {
2282 .hw_ops = &vcn_v2_6_ras_hw_ops,
2283 .ras_late_init = amdgpu_vcn_ras_late_init,
2284 },
2285 };
2286
vcn_v2_5_set_ras_funcs(struct amdgpu_device * adev)2287 static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev)
2288 {
2289 switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
2290 case IP_VERSION(2, 6, 0):
2291 adev->vcn.ras = &vcn_v2_6_ras;
2292 break;
2293 default:
2294 break;
2295 }
2296 }
2297