1 /*
2 * Copyright 2016-2024 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26
27 #include <linux/firmware.h>
28 #include <linux/module.h>
29 #include <linux/dmi.h>
30 #include <linux/pci.h>
31 #include <linux/debugfs.h>
32 #include <drm/drm_drv.h>
33
34 #include "amdgpu.h"
35 #include "amdgpu_pm.h"
36 #include "amdgpu_vcn.h"
37 #include "soc15d.h"
38
39 /* Firmware Names */
40 #define FIRMWARE_RAVEN "amdgpu/raven_vcn.bin"
41 #define FIRMWARE_PICASSO "amdgpu/picasso_vcn.bin"
42 #define FIRMWARE_RAVEN2 "amdgpu/raven2_vcn.bin"
43 #define FIRMWARE_ARCTURUS "amdgpu/arcturus_vcn.bin"
44 #define FIRMWARE_RENOIR "amdgpu/renoir_vcn.bin"
45 #define FIRMWARE_GREEN_SARDINE "amdgpu/green_sardine_vcn.bin"
46 #define FIRMWARE_NAVI10 "amdgpu/navi10_vcn.bin"
47 #define FIRMWARE_NAVI14 "amdgpu/navi14_vcn.bin"
48 #define FIRMWARE_NAVI12 "amdgpu/navi12_vcn.bin"
49 #define FIRMWARE_SIENNA_CICHLID "amdgpu/sienna_cichlid_vcn.bin"
50 #define FIRMWARE_NAVY_FLOUNDER "amdgpu/navy_flounder_vcn.bin"
51 #define FIRMWARE_VANGOGH "amdgpu/vangogh_vcn.bin"
52 #define FIRMWARE_DIMGREY_CAVEFISH "amdgpu/dimgrey_cavefish_vcn.bin"
53 #define FIRMWARE_ALDEBARAN "amdgpu/aldebaran_vcn.bin"
54 #define FIRMWARE_BEIGE_GOBY "amdgpu/beige_goby_vcn.bin"
55 #define FIRMWARE_YELLOW_CARP "amdgpu/yellow_carp_vcn.bin"
56 #define FIRMWARE_VCN_3_1_2 "amdgpu/vcn_3_1_2.bin"
57 #define FIRMWARE_VCN4_0_0 "amdgpu/vcn_4_0_0.bin"
58 #define FIRMWARE_VCN4_0_2 "amdgpu/vcn_4_0_2.bin"
59 #define FIRMWARE_VCN4_0_3 "amdgpu/vcn_4_0_3.bin"
60 #define FIRMWARE_VCN4_0_4 "amdgpu/vcn_4_0_4.bin"
61 #define FIRMWARE_VCN4_0_5 "amdgpu/vcn_4_0_5.bin"
62 #define FIRMWARE_VCN4_0_6 "amdgpu/vcn_4_0_6.bin"
63 #define FIRMWARE_VCN4_0_6_1 "amdgpu/vcn_4_0_6_1.bin"
64 #define FIRMWARE_VCN5_0_0 "amdgpu/vcn_5_0_0.bin"
65 #define FIRMWARE_VCN5_0_1 "amdgpu/vcn_5_0_1.bin"
66
67 MODULE_FIRMWARE(FIRMWARE_RAVEN);
68 MODULE_FIRMWARE(FIRMWARE_PICASSO);
69 MODULE_FIRMWARE(FIRMWARE_RAVEN2);
70 MODULE_FIRMWARE(FIRMWARE_ARCTURUS);
71 MODULE_FIRMWARE(FIRMWARE_RENOIR);
72 MODULE_FIRMWARE(FIRMWARE_GREEN_SARDINE);
73 MODULE_FIRMWARE(FIRMWARE_ALDEBARAN);
74 MODULE_FIRMWARE(FIRMWARE_NAVI10);
75 MODULE_FIRMWARE(FIRMWARE_NAVI14);
76 MODULE_FIRMWARE(FIRMWARE_NAVI12);
77 MODULE_FIRMWARE(FIRMWARE_SIENNA_CICHLID);
78 MODULE_FIRMWARE(FIRMWARE_NAVY_FLOUNDER);
79 MODULE_FIRMWARE(FIRMWARE_VANGOGH);
80 MODULE_FIRMWARE(FIRMWARE_DIMGREY_CAVEFISH);
81 MODULE_FIRMWARE(FIRMWARE_BEIGE_GOBY);
82 MODULE_FIRMWARE(FIRMWARE_YELLOW_CARP);
83 MODULE_FIRMWARE(FIRMWARE_VCN_3_1_2);
84 MODULE_FIRMWARE(FIRMWARE_VCN4_0_0);
85 MODULE_FIRMWARE(FIRMWARE_VCN4_0_2);
86 MODULE_FIRMWARE(FIRMWARE_VCN4_0_3);
87 MODULE_FIRMWARE(FIRMWARE_VCN4_0_4);
88 MODULE_FIRMWARE(FIRMWARE_VCN4_0_5);
89 MODULE_FIRMWARE(FIRMWARE_VCN4_0_6);
90 MODULE_FIRMWARE(FIRMWARE_VCN4_0_6_1);
91 MODULE_FIRMWARE(FIRMWARE_VCN5_0_0);
92 MODULE_FIRMWARE(FIRMWARE_VCN5_0_1);
93
94 static void amdgpu_vcn_idle_work_handler(struct work_struct *work);
95
amdgpu_vcn_early_init(struct amdgpu_device * adev,int i)96 int amdgpu_vcn_early_init(struct amdgpu_device *adev, int i)
97 {
98 char ucode_prefix[25];
99 int r;
100
101 adev->vcn.inst[i].adev = adev;
102 adev->vcn.inst[i].inst = i;
103 amdgpu_ucode_ip_version_decode(adev, UVD_HWIP, ucode_prefix, sizeof(ucode_prefix));
104
105 if (i != 0 && adev->vcn.per_inst_fw) {
106 r = amdgpu_ucode_request(adev, &adev->vcn.inst[i].fw,
107 AMDGPU_UCODE_REQUIRED,
108 "amdgpu/%s_%d.bin", ucode_prefix, i);
109 if (r)
110 amdgpu_ucode_release(&adev->vcn.inst[i].fw);
111 } else {
112 if (!adev->vcn.inst[0].fw) {
113 r = amdgpu_ucode_request(adev, &adev->vcn.inst[0].fw,
114 AMDGPU_UCODE_REQUIRED,
115 "amdgpu/%s.bin", ucode_prefix);
116 if (r)
117 amdgpu_ucode_release(&adev->vcn.inst[0].fw);
118 } else {
119 r = 0;
120 }
121 adev->vcn.inst[i].fw = adev->vcn.inst[0].fw;
122 }
123
124 return r;
125 }
126
amdgpu_vcn_sw_init(struct amdgpu_device * adev,int i)127 int amdgpu_vcn_sw_init(struct amdgpu_device *adev, int i)
128 {
129 unsigned long bo_size;
130 const struct common_firmware_header *hdr;
131 unsigned char fw_check;
132 unsigned int fw_shared_size, log_offset;
133 int r;
134
135 mutex_init(&adev->vcn.inst[i].vcn1_jpeg1_workaround);
136 mutex_init(&adev->vcn.inst[i].vcn_pg_lock);
137 mutex_init(&adev->vcn.inst[i].engine_reset_mutex);
138 atomic_set(&adev->vcn.inst[i].total_submission_cnt, 0);
139 INIT_DELAYED_WORK(&adev->vcn.inst[i].idle_work, amdgpu_vcn_idle_work_handler);
140 atomic_set(&adev->vcn.inst[i].dpg_enc_submission_cnt, 0);
141 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
142 (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
143 adev->vcn.inst[i].indirect_sram = true;
144
145 /*
146 * Some Steam Deck's BIOS versions are incompatible with the
147 * indirect SRAM mode, leading to amdgpu being unable to get
148 * properly probed (and even potentially crashing the kernel).
149 * Hence, check for these versions here - notice this is
150 * restricted to Vangogh (Deck's APU).
151 */
152 if (amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(3, 0, 2)) {
153 const char *bios_ver = dmi_get_system_info(DMI_BIOS_VERSION);
154
155 if (bios_ver && (!strncmp("F7A0113", bios_ver, 7) ||
156 !strncmp("F7A0114", bios_ver, 7))) {
157 adev->vcn.inst[i].indirect_sram = false;
158 dev_info(adev->dev,
159 "Steam Deck quirk: indirect SRAM disabled on BIOS %s\n", bios_ver);
160 }
161 }
162
163 /* from vcn4 and above, only unified queue is used */
164 adev->vcn.inst[i].using_unified_queue =
165 amdgpu_ip_version(adev, UVD_HWIP, 0) >= IP_VERSION(4, 0, 0);
166
167 hdr = (const struct common_firmware_header *)adev->vcn.inst[i].fw->data;
168 adev->vcn.inst[i].fw_version = le32_to_cpu(hdr->ucode_version);
169 adev->vcn.fw_version = le32_to_cpu(hdr->ucode_version);
170
171 /* Bit 20-23, it is encode major and non-zero for new naming convention.
172 * This field is part of version minor and DRM_DISABLED_FLAG in old naming
173 * convention. Since the l:wq!atest version minor is 0x5B and DRM_DISABLED_FLAG
174 * is zero in old naming convention, this field is always zero so far.
175 * These four bits are used to tell which naming convention is present.
176 */
177 fw_check = (le32_to_cpu(hdr->ucode_version) >> 20) & 0xf;
178 if (fw_check) {
179 unsigned int dec_ver, enc_major, enc_minor, vep, fw_rev;
180
181 fw_rev = le32_to_cpu(hdr->ucode_version) & 0xfff;
182 enc_minor = (le32_to_cpu(hdr->ucode_version) >> 12) & 0xff;
183 enc_major = fw_check;
184 dec_ver = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xf;
185 vep = (le32_to_cpu(hdr->ucode_version) >> 28) & 0xf;
186 dev_info(adev->dev,
187 "Found VCN firmware Version ENC: %u.%u DEC: %u VEP: %u Revision: %u\n",
188 enc_major, enc_minor, dec_ver, vep, fw_rev);
189 } else {
190 unsigned int version_major, version_minor, family_id;
191
192 family_id = le32_to_cpu(hdr->ucode_version) & 0xff;
193 version_major = (le32_to_cpu(hdr->ucode_version) >> 24) & 0xff;
194 version_minor = (le32_to_cpu(hdr->ucode_version) >> 8) & 0xff;
195 dev_info(adev->dev, "Found VCN firmware Version: %u.%u Family ID: %u\n",
196 version_major, version_minor, family_id);
197 }
198
199 bo_size = AMDGPU_VCN_STACK_SIZE + AMDGPU_VCN_CONTEXT_SIZE;
200 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
201 bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
202
203 if (amdgpu_ip_version(adev, UVD_HWIP, 0) >= IP_VERSION(5, 0, 0)) {
204 fw_shared_size = AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn5_fw_shared));
205 log_offset = offsetof(struct amdgpu_vcn5_fw_shared, fw_log);
206 } else if (amdgpu_ip_version(adev, UVD_HWIP, 0) >= IP_VERSION(4, 0, 0)) {
207 fw_shared_size = AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared));
208 log_offset = offsetof(struct amdgpu_vcn4_fw_shared, fw_log);
209 } else {
210 fw_shared_size = AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared));
211 log_offset = offsetof(struct amdgpu_fw_shared, fw_log);
212 }
213
214 bo_size += fw_shared_size;
215
216 if (amdgpu_vcnfw_log)
217 bo_size += AMDGPU_VCNFW_LOG_SIZE;
218
219 r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE,
220 AMDGPU_GEM_DOMAIN_VRAM |
221 AMDGPU_GEM_DOMAIN_GTT,
222 &adev->vcn.inst[i].vcpu_bo,
223 &adev->vcn.inst[i].gpu_addr,
224 &adev->vcn.inst[i].cpu_addr);
225 if (r) {
226 dev_err(adev->dev, "(%d) failed to allocate vcn bo\n", r);
227 return r;
228 }
229
230 adev->vcn.inst[i].fw_shared.cpu_addr = adev->vcn.inst[i].cpu_addr +
231 bo_size - fw_shared_size;
232 adev->vcn.inst[i].fw_shared.gpu_addr = adev->vcn.inst[i].gpu_addr +
233 bo_size - fw_shared_size;
234
235 adev->vcn.inst[i].fw_shared.mem_size = fw_shared_size;
236
237 if (amdgpu_vcnfw_log) {
238 adev->vcn.inst[i].fw_shared.cpu_addr -= AMDGPU_VCNFW_LOG_SIZE;
239 adev->vcn.inst[i].fw_shared.gpu_addr -= AMDGPU_VCNFW_LOG_SIZE;
240 adev->vcn.inst[i].fw_shared.log_offset = log_offset;
241 }
242
243 if (adev->vcn.inst[i].indirect_sram) {
244 r = amdgpu_bo_create_kernel(adev, 64 * 2 * 4, PAGE_SIZE,
245 AMDGPU_GEM_DOMAIN_VRAM |
246 AMDGPU_GEM_DOMAIN_GTT,
247 &adev->vcn.inst[i].dpg_sram_bo,
248 &adev->vcn.inst[i].dpg_sram_gpu_addr,
249 &adev->vcn.inst[i].dpg_sram_cpu_addr);
250 if (r) {
251 dev_err(adev->dev, "VCN %d (%d) failed to allocate DPG bo\n", i, r);
252 return r;
253 }
254 }
255
256 return 0;
257 }
258
amdgpu_vcn_sw_fini(struct amdgpu_device * adev,int i)259 int amdgpu_vcn_sw_fini(struct amdgpu_device *adev, int i)
260 {
261 int j;
262
263 if (adev->vcn.harvest_config & (1 << i))
264 return 0;
265
266 amdgpu_bo_free_kernel(
267 &adev->vcn.inst[i].dpg_sram_bo,
268 &adev->vcn.inst[i].dpg_sram_gpu_addr,
269 (void **)&adev->vcn.inst[i].dpg_sram_cpu_addr);
270
271 kvfree(adev->vcn.inst[i].saved_bo);
272
273 amdgpu_bo_free_kernel(&adev->vcn.inst[i].vcpu_bo,
274 &adev->vcn.inst[i].gpu_addr,
275 (void **)&adev->vcn.inst[i].cpu_addr);
276
277 amdgpu_ring_fini(&adev->vcn.inst[i].ring_dec);
278
279 for (j = 0; j < adev->vcn.inst[i].num_enc_rings; ++j)
280 amdgpu_ring_fini(&adev->vcn.inst[i].ring_enc[j]);
281
282 if (adev->vcn.per_inst_fw) {
283 amdgpu_ucode_release(&adev->vcn.inst[i].fw);
284 } else {
285 amdgpu_ucode_release(&adev->vcn.inst[0].fw);
286 adev->vcn.inst[i].fw = NULL;
287 }
288 mutex_destroy(&adev->vcn.inst[i].vcn_pg_lock);
289 mutex_destroy(&adev->vcn.inst[i].vcn1_jpeg1_workaround);
290
291 return 0;
292 }
293
amdgpu_vcn_is_disabled_vcn(struct amdgpu_device * adev,enum vcn_ring_type type,uint32_t vcn_instance)294 bool amdgpu_vcn_is_disabled_vcn(struct amdgpu_device *adev, enum vcn_ring_type type, uint32_t vcn_instance)
295 {
296 bool ret = false;
297 int vcn_config = adev->vcn.inst[vcn_instance].vcn_config;
298
299 if ((type == VCN_ENCODE_RING) && (vcn_config & VCN_BLOCK_ENCODE_DISABLE_MASK))
300 ret = true;
301 else if ((type == VCN_DECODE_RING) && (vcn_config & VCN_BLOCK_DECODE_DISABLE_MASK))
302 ret = true;
303 else if ((type == VCN_UNIFIED_RING) && (vcn_config & VCN_BLOCK_QUEUE_DISABLE_MASK))
304 ret = true;
305
306 return ret;
307 }
308
amdgpu_vcn_save_vcpu_bo_inst(struct amdgpu_device * adev,int i)309 static int amdgpu_vcn_save_vcpu_bo_inst(struct amdgpu_device *adev, int i)
310 {
311 unsigned int size;
312 void *ptr;
313 int idx;
314
315 if (adev->vcn.harvest_config & (1 << i))
316 return 0;
317 if (adev->vcn.inst[i].vcpu_bo == NULL)
318 return 0;
319
320 size = amdgpu_bo_size(adev->vcn.inst[i].vcpu_bo);
321 ptr = adev->vcn.inst[i].cpu_addr;
322
323 adev->vcn.inst[i].saved_bo = kvmalloc(size, GFP_KERNEL);
324 if (!adev->vcn.inst[i].saved_bo)
325 return -ENOMEM;
326
327 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
328 memcpy_fromio(adev->vcn.inst[i].saved_bo, ptr, size);
329 drm_dev_exit(idx);
330 }
331
332 return 0;
333 }
334
amdgpu_vcn_save_vcpu_bo(struct amdgpu_device * adev)335 int amdgpu_vcn_save_vcpu_bo(struct amdgpu_device *adev)
336 {
337 int ret, i;
338
339 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
340 ret = amdgpu_vcn_save_vcpu_bo_inst(adev, i);
341 if (ret)
342 return ret;
343 }
344
345 return 0;
346 }
347
amdgpu_vcn_suspend(struct amdgpu_device * adev,int i)348 int amdgpu_vcn_suspend(struct amdgpu_device *adev, int i)
349 {
350 bool in_ras_intr = amdgpu_ras_intr_triggered();
351
352 if (adev->vcn.harvest_config & (1 << i))
353 return 0;
354
355 cancel_delayed_work_sync(&adev->vcn.inst[i].idle_work);
356
357 /* err_event_athub and dpc recovery will corrupt VCPU buffer, so we need to
358 * restore fw data and clear buffer in amdgpu_vcn_resume() */
359 if (in_ras_intr || adev->pcie_reset_ctx.in_link_reset)
360 return 0;
361
362 return amdgpu_vcn_save_vcpu_bo_inst(adev, i);
363 }
364
amdgpu_vcn_resume(struct amdgpu_device * adev,int i)365 int amdgpu_vcn_resume(struct amdgpu_device *adev, int i)
366 {
367 unsigned int size;
368 void *ptr;
369 int idx;
370
371 if (adev->vcn.harvest_config & (1 << i))
372 return 0;
373 if (adev->vcn.inst[i].vcpu_bo == NULL)
374 return -EINVAL;
375
376 size = amdgpu_bo_size(adev->vcn.inst[i].vcpu_bo);
377 ptr = adev->vcn.inst[i].cpu_addr;
378
379 if (adev->vcn.inst[i].saved_bo != NULL) {
380 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
381 memcpy_toio(ptr, adev->vcn.inst[i].saved_bo, size);
382 drm_dev_exit(idx);
383 }
384 kvfree(adev->vcn.inst[i].saved_bo);
385 adev->vcn.inst[i].saved_bo = NULL;
386 } else {
387 const struct common_firmware_header *hdr;
388 unsigned int offset;
389
390 hdr = (const struct common_firmware_header *)adev->vcn.inst[i].fw->data;
391 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
392 offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
393 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
394 memcpy_toio(adev->vcn.inst[i].cpu_addr,
395 adev->vcn.inst[i].fw->data + offset,
396 le32_to_cpu(hdr->ucode_size_bytes));
397 drm_dev_exit(idx);
398 }
399 size -= le32_to_cpu(hdr->ucode_size_bytes);
400 ptr += le32_to_cpu(hdr->ucode_size_bytes);
401 }
402 memset_io(ptr, 0, size);
403 }
404
405 return 0;
406 }
407
amdgpu_vcn_idle_work_handler(struct work_struct * work)408 static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
409 {
410 struct amdgpu_vcn_inst *vcn_inst =
411 container_of(work, struct amdgpu_vcn_inst, idle_work.work);
412 struct amdgpu_device *adev = vcn_inst->adev;
413 unsigned int fences = 0, fence[AMDGPU_MAX_VCN_INSTANCES] = {0};
414 unsigned int i = vcn_inst->inst, j;
415 int r = 0;
416
417 if (adev->vcn.harvest_config & (1 << i))
418 return;
419
420 for (j = 0; j < adev->vcn.inst[i].num_enc_rings; ++j)
421 fence[i] += amdgpu_fence_count_emitted(&vcn_inst->ring_enc[j]);
422
423 /* Only set DPG pause for VCN3 or below, VCN4 and above will be handled by FW */
424 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG &&
425 !adev->vcn.inst[i].using_unified_queue) {
426 struct dpg_pause_state new_state;
427
428 if (fence[i] ||
429 unlikely(atomic_read(&vcn_inst->dpg_enc_submission_cnt)))
430 new_state.fw_based = VCN_DPG_STATE__PAUSE;
431 else
432 new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
433
434 adev->vcn.inst[i].pause_dpg_mode(vcn_inst, &new_state);
435 }
436
437 fence[i] += amdgpu_fence_count_emitted(&vcn_inst->ring_dec);
438 fences += fence[i];
439
440 if (!fences && !atomic_read(&vcn_inst->total_submission_cnt)) {
441 vcn_inst->set_pg_state(vcn_inst, AMD_PG_STATE_GATE);
442 mutex_lock(&adev->vcn.workload_profile_mutex);
443 if (adev->vcn.workload_profile_active) {
444 r = amdgpu_dpm_switch_power_profile(adev, PP_SMC_POWER_PROFILE_VIDEO,
445 false);
446 if (r)
447 dev_warn(adev->dev, "(%d) failed to disable video power profile mode\n", r);
448 adev->vcn.workload_profile_active = false;
449 }
450 mutex_unlock(&adev->vcn.workload_profile_mutex);
451 } else {
452 schedule_delayed_work(&vcn_inst->idle_work, VCN_IDLE_TIMEOUT);
453 }
454 }
455
amdgpu_vcn_ring_begin_use(struct amdgpu_ring * ring)456 void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
457 {
458 struct amdgpu_device *adev = ring->adev;
459 struct amdgpu_vcn_inst *vcn_inst = &adev->vcn.inst[ring->me];
460 int r = 0;
461
462 atomic_inc(&vcn_inst->total_submission_cnt);
463
464 cancel_delayed_work_sync(&vcn_inst->idle_work);
465
466 /* We can safely return early here because we've cancelled the
467 * the delayed work so there is no one else to set it to false
468 * and we don't care if someone else sets it to true.
469 */
470 if (adev->vcn.workload_profile_active)
471 goto pg_lock;
472
473 mutex_lock(&adev->vcn.workload_profile_mutex);
474 if (!adev->vcn.workload_profile_active) {
475 r = amdgpu_dpm_switch_power_profile(adev, PP_SMC_POWER_PROFILE_VIDEO,
476 true);
477 if (r)
478 dev_warn(adev->dev, "(%d) failed to switch to video power profile mode\n", r);
479 adev->vcn.workload_profile_active = true;
480 }
481 mutex_unlock(&adev->vcn.workload_profile_mutex);
482
483 pg_lock:
484 mutex_lock(&vcn_inst->vcn_pg_lock);
485 vcn_inst->set_pg_state(vcn_inst, AMD_PG_STATE_UNGATE);
486
487 /* Only set DPG pause for VCN3 or below, VCN4 and above will be handled by FW */
488 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG &&
489 !vcn_inst->using_unified_queue) {
490 struct dpg_pause_state new_state;
491
492 if (ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC) {
493 atomic_inc(&vcn_inst->dpg_enc_submission_cnt);
494 new_state.fw_based = VCN_DPG_STATE__PAUSE;
495 } else {
496 unsigned int fences = 0;
497 unsigned int i;
498
499 for (i = 0; i < vcn_inst->num_enc_rings; ++i)
500 fences += amdgpu_fence_count_emitted(&vcn_inst->ring_enc[i]);
501
502 if (fences || atomic_read(&vcn_inst->dpg_enc_submission_cnt))
503 new_state.fw_based = VCN_DPG_STATE__PAUSE;
504 else
505 new_state.fw_based = VCN_DPG_STATE__UNPAUSE;
506 }
507
508 vcn_inst->pause_dpg_mode(vcn_inst, &new_state);
509 }
510 mutex_unlock(&vcn_inst->vcn_pg_lock);
511 }
512
amdgpu_vcn_ring_end_use(struct amdgpu_ring * ring)513 void amdgpu_vcn_ring_end_use(struct amdgpu_ring *ring)
514 {
515 struct amdgpu_device *adev = ring->adev;
516
517 /* Only set DPG pause for VCN3 or below, VCN4 and above will be handled by FW */
518 if (ring->adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG &&
519 ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC &&
520 !adev->vcn.inst[ring->me].using_unified_queue)
521 atomic_dec(&ring->adev->vcn.inst[ring->me].dpg_enc_submission_cnt);
522
523 atomic_dec(&ring->adev->vcn.inst[ring->me].total_submission_cnt);
524
525 schedule_delayed_work(&ring->adev->vcn.inst[ring->me].idle_work,
526 VCN_IDLE_TIMEOUT);
527 }
528
amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring * ring)529 int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
530 {
531 struct amdgpu_device *adev = ring->adev;
532 uint32_t tmp = 0;
533 unsigned int i;
534 int r;
535
536 /* VCN in SRIOV does not support direct register read/write */
537 if (amdgpu_sriov_vf(adev))
538 return 0;
539
540 WREG32(adev->vcn.inst[ring->me].external.scratch9, 0xCAFEDEAD);
541 r = amdgpu_ring_alloc(ring, 3);
542 if (r)
543 return r;
544 amdgpu_ring_write(ring, PACKET0(adev->vcn.inst[ring->me].internal.scratch9, 0));
545 amdgpu_ring_write(ring, 0xDEADBEEF);
546 amdgpu_ring_commit(ring);
547 for (i = 0; i < adev->usec_timeout; i++) {
548 tmp = RREG32(adev->vcn.inst[ring->me].external.scratch9);
549 if (tmp == 0xDEADBEEF)
550 break;
551 udelay(1);
552 }
553
554 if (i >= adev->usec_timeout)
555 r = -ETIMEDOUT;
556
557 return r;
558 }
559
amdgpu_vcn_dec_sw_ring_test_ring(struct amdgpu_ring * ring)560 int amdgpu_vcn_dec_sw_ring_test_ring(struct amdgpu_ring *ring)
561 {
562 struct amdgpu_device *adev = ring->adev;
563 uint32_t rptr;
564 unsigned int i;
565 int r;
566
567 if (amdgpu_sriov_vf(adev))
568 return 0;
569
570 r = amdgpu_ring_alloc(ring, 16);
571 if (r)
572 return r;
573
574 rptr = amdgpu_ring_get_rptr(ring);
575
576 amdgpu_ring_write(ring, VCN_DEC_SW_CMD_END);
577 amdgpu_ring_commit(ring);
578
579 for (i = 0; i < adev->usec_timeout; i++) {
580 if (amdgpu_ring_get_rptr(ring) != rptr)
581 break;
582 udelay(1);
583 }
584
585 if (i >= adev->usec_timeout)
586 r = -ETIMEDOUT;
587
588 return r;
589 }
590
amdgpu_vcn_dec_send_msg(struct amdgpu_ring * ring,struct amdgpu_ib * ib_msg,struct dma_fence ** fence)591 static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring,
592 struct amdgpu_ib *ib_msg,
593 struct dma_fence **fence)
594 {
595 u64 addr = AMDGPU_GPU_PAGE_ALIGN(ib_msg->gpu_addr);
596 struct amdgpu_device *adev = ring->adev;
597 struct dma_fence *f = NULL;
598 struct amdgpu_job *job;
599 struct amdgpu_ib *ib;
600 int i, r;
601
602 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
603 64, AMDGPU_IB_POOL_DIRECT,
604 &job);
605 if (r)
606 goto err;
607
608 ib = &job->ibs[0];
609 ib->ptr[0] = PACKET0(adev->vcn.inst[ring->me].internal.data0, 0);
610 ib->ptr[1] = addr;
611 ib->ptr[2] = PACKET0(adev->vcn.inst[ring->me].internal.data1, 0);
612 ib->ptr[3] = addr >> 32;
613 ib->ptr[4] = PACKET0(adev->vcn.inst[ring->me].internal.cmd, 0);
614 ib->ptr[5] = 0;
615 for (i = 6; i < 16; i += 2) {
616 ib->ptr[i] = PACKET0(adev->vcn.inst[ring->me].internal.nop, 0);
617 ib->ptr[i+1] = 0;
618 }
619 ib->length_dw = 16;
620
621 r = amdgpu_job_submit_direct(job, ring, &f);
622 if (r)
623 goto err_free;
624
625 amdgpu_ib_free(ib_msg, f);
626
627 if (fence)
628 *fence = dma_fence_get(f);
629 dma_fence_put(f);
630
631 return 0;
632
633 err_free:
634 amdgpu_job_free(job);
635 err:
636 amdgpu_ib_free(ib_msg, f);
637 return r;
638 }
639
amdgpu_vcn_dec_get_create_msg(struct amdgpu_ring * ring,uint32_t handle,struct amdgpu_ib * ib)640 static int amdgpu_vcn_dec_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
641 struct amdgpu_ib *ib)
642 {
643 struct amdgpu_device *adev = ring->adev;
644 uint32_t *msg;
645 int r, i;
646
647 memset(ib, 0, sizeof(*ib));
648 r = amdgpu_ib_get(adev, NULL, AMDGPU_GPU_PAGE_SIZE * 2,
649 AMDGPU_IB_POOL_DIRECT,
650 ib);
651 if (r)
652 return r;
653
654 msg = (uint32_t *)AMDGPU_GPU_PAGE_ALIGN((unsigned long)ib->ptr);
655 msg[0] = cpu_to_le32(0x00000028);
656 msg[1] = cpu_to_le32(0x00000038);
657 msg[2] = cpu_to_le32(0x00000001);
658 msg[3] = cpu_to_le32(0x00000000);
659 msg[4] = cpu_to_le32(handle);
660 msg[5] = cpu_to_le32(0x00000000);
661 msg[6] = cpu_to_le32(0x00000001);
662 msg[7] = cpu_to_le32(0x00000028);
663 msg[8] = cpu_to_le32(0x00000010);
664 msg[9] = cpu_to_le32(0x00000000);
665 msg[10] = cpu_to_le32(0x00000007);
666 msg[11] = cpu_to_le32(0x00000000);
667 msg[12] = cpu_to_le32(0x00000780);
668 msg[13] = cpu_to_le32(0x00000440);
669 for (i = 14; i < 1024; ++i)
670 msg[i] = cpu_to_le32(0x0);
671
672 return 0;
673 }
674
amdgpu_vcn_dec_get_destroy_msg(struct amdgpu_ring * ring,uint32_t handle,struct amdgpu_ib * ib)675 static int amdgpu_vcn_dec_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
676 struct amdgpu_ib *ib)
677 {
678 struct amdgpu_device *adev = ring->adev;
679 uint32_t *msg;
680 int r, i;
681
682 memset(ib, 0, sizeof(*ib));
683 r = amdgpu_ib_get(adev, NULL, AMDGPU_GPU_PAGE_SIZE * 2,
684 AMDGPU_IB_POOL_DIRECT,
685 ib);
686 if (r)
687 return r;
688
689 msg = (uint32_t *)AMDGPU_GPU_PAGE_ALIGN((unsigned long)ib->ptr);
690 msg[0] = cpu_to_le32(0x00000028);
691 msg[1] = cpu_to_le32(0x00000018);
692 msg[2] = cpu_to_le32(0x00000000);
693 msg[3] = cpu_to_le32(0x00000002);
694 msg[4] = cpu_to_le32(handle);
695 msg[5] = cpu_to_le32(0x00000000);
696 for (i = 6; i < 1024; ++i)
697 msg[i] = cpu_to_le32(0x0);
698
699 return 0;
700 }
701
amdgpu_vcn_dec_ring_test_ib(struct amdgpu_ring * ring,long timeout)702 int amdgpu_vcn_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout)
703 {
704 struct dma_fence *fence = NULL;
705 struct amdgpu_ib ib;
706 long r;
707
708 r = amdgpu_vcn_dec_get_create_msg(ring, 1, &ib);
709 if (r)
710 goto error;
711
712 r = amdgpu_vcn_dec_send_msg(ring, &ib, NULL);
713 if (r)
714 goto error;
715 r = amdgpu_vcn_dec_get_destroy_msg(ring, 1, &ib);
716 if (r)
717 goto error;
718
719 r = amdgpu_vcn_dec_send_msg(ring, &ib, &fence);
720 if (r)
721 goto error;
722
723 r = dma_fence_wait_timeout(fence, false, timeout);
724 if (r == 0)
725 r = -ETIMEDOUT;
726 else if (r > 0)
727 r = 0;
728
729 dma_fence_put(fence);
730 error:
731 return r;
732 }
733
amdgpu_vcn_unified_ring_ib_header(struct amdgpu_ib * ib,uint32_t ib_pack_in_dw,bool enc)734 static uint32_t *amdgpu_vcn_unified_ring_ib_header(struct amdgpu_ib *ib,
735 uint32_t ib_pack_in_dw, bool enc)
736 {
737 uint32_t *ib_checksum;
738
739 ib->ptr[ib->length_dw++] = 0x00000010; /* single queue checksum */
740 ib->ptr[ib->length_dw++] = 0x30000002;
741 ib_checksum = &ib->ptr[ib->length_dw++];
742 ib->ptr[ib->length_dw++] = ib_pack_in_dw;
743
744 ib->ptr[ib->length_dw++] = 0x00000010; /* engine info */
745 ib->ptr[ib->length_dw++] = 0x30000001;
746 ib->ptr[ib->length_dw++] = enc ? 0x2 : 0x3;
747 ib->ptr[ib->length_dw++] = ib_pack_in_dw * sizeof(uint32_t);
748
749 return ib_checksum;
750 }
751
amdgpu_vcn_unified_ring_ib_checksum(uint32_t ** ib_checksum,uint32_t ib_pack_in_dw)752 static void amdgpu_vcn_unified_ring_ib_checksum(uint32_t **ib_checksum,
753 uint32_t ib_pack_in_dw)
754 {
755 uint32_t i;
756 uint32_t checksum = 0;
757
758 for (i = 0; i < ib_pack_in_dw; i++)
759 checksum += *(*ib_checksum + 2 + i);
760
761 **ib_checksum = checksum;
762 }
763
amdgpu_vcn_dec_sw_send_msg(struct amdgpu_ring * ring,struct amdgpu_ib * ib_msg,struct dma_fence ** fence)764 static int amdgpu_vcn_dec_sw_send_msg(struct amdgpu_ring *ring,
765 struct amdgpu_ib *ib_msg,
766 struct dma_fence **fence)
767 {
768 struct amdgpu_vcn_decode_buffer *decode_buffer = NULL;
769 unsigned int ib_size_dw = 64;
770 struct amdgpu_device *adev = ring->adev;
771 struct dma_fence *f = NULL;
772 struct amdgpu_job *job;
773 struct amdgpu_ib *ib;
774 uint64_t addr = AMDGPU_GPU_PAGE_ALIGN(ib_msg->gpu_addr);
775 uint32_t *ib_checksum;
776 uint32_t ib_pack_in_dw;
777 int i, r;
778
779 if (adev->vcn.inst[ring->me].using_unified_queue)
780 ib_size_dw += 8;
781
782 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
783 ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
784 &job);
785 if (r)
786 goto err;
787
788 ib = &job->ibs[0];
789 ib->length_dw = 0;
790
791 /* single queue headers */
792 if (adev->vcn.inst[ring->me].using_unified_queue) {
793 ib_pack_in_dw = sizeof(struct amdgpu_vcn_decode_buffer) / sizeof(uint32_t)
794 + 4 + 2; /* engine info + decoding ib in dw */
795 ib_checksum = amdgpu_vcn_unified_ring_ib_header(ib, ib_pack_in_dw, false);
796 }
797
798 ib->ptr[ib->length_dw++] = sizeof(struct amdgpu_vcn_decode_buffer) + 8;
799 ib->ptr[ib->length_dw++] = cpu_to_le32(AMDGPU_VCN_IB_FLAG_DECODE_BUFFER);
800 decode_buffer = (struct amdgpu_vcn_decode_buffer *)&(ib->ptr[ib->length_dw]);
801 ib->length_dw += sizeof(struct amdgpu_vcn_decode_buffer) / 4;
802 memset(decode_buffer, 0, sizeof(struct amdgpu_vcn_decode_buffer));
803
804 decode_buffer->valid_buf_flag |= cpu_to_le32(AMDGPU_VCN_CMD_FLAG_MSG_BUFFER);
805 decode_buffer->msg_buffer_address_hi = cpu_to_le32(addr >> 32);
806 decode_buffer->msg_buffer_address_lo = cpu_to_le32(addr);
807
808 for (i = ib->length_dw; i < ib_size_dw; ++i)
809 ib->ptr[i] = 0x0;
810
811 if (adev->vcn.inst[ring->me].using_unified_queue)
812 amdgpu_vcn_unified_ring_ib_checksum(&ib_checksum, ib_pack_in_dw);
813
814 r = amdgpu_job_submit_direct(job, ring, &f);
815 if (r)
816 goto err_free;
817
818 amdgpu_ib_free(ib_msg, f);
819
820 if (fence)
821 *fence = dma_fence_get(f);
822 dma_fence_put(f);
823
824 return 0;
825
826 err_free:
827 amdgpu_job_free(job);
828 err:
829 amdgpu_ib_free(ib_msg, f);
830 return r;
831 }
832
amdgpu_vcn_dec_sw_ring_test_ib(struct amdgpu_ring * ring,long timeout)833 int amdgpu_vcn_dec_sw_ring_test_ib(struct amdgpu_ring *ring, long timeout)
834 {
835 struct dma_fence *fence = NULL;
836 struct amdgpu_ib ib;
837 long r;
838
839 r = amdgpu_vcn_dec_get_create_msg(ring, 1, &ib);
840 if (r)
841 goto error;
842
843 r = amdgpu_vcn_dec_sw_send_msg(ring, &ib, NULL);
844 if (r)
845 goto error;
846 r = amdgpu_vcn_dec_get_destroy_msg(ring, 1, &ib);
847 if (r)
848 goto error;
849
850 r = amdgpu_vcn_dec_sw_send_msg(ring, &ib, &fence);
851 if (r)
852 goto error;
853
854 r = dma_fence_wait_timeout(fence, false, timeout);
855 if (r == 0)
856 r = -ETIMEDOUT;
857 else if (r > 0)
858 r = 0;
859
860 dma_fence_put(fence);
861 error:
862 return r;
863 }
864
amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring * ring)865 int amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring *ring)
866 {
867 struct amdgpu_device *adev = ring->adev;
868 uint32_t rptr;
869 unsigned int i;
870 int r;
871
872 if (amdgpu_sriov_vf(adev))
873 return 0;
874
875 r = amdgpu_ring_alloc(ring, 16);
876 if (r)
877 return r;
878
879 rptr = amdgpu_ring_get_rptr(ring);
880
881 amdgpu_ring_write(ring, VCN_ENC_CMD_END);
882 amdgpu_ring_commit(ring);
883
884 for (i = 0; i < adev->usec_timeout; i++) {
885 if (amdgpu_ring_get_rptr(ring) != rptr)
886 break;
887 udelay(1);
888 }
889
890 if (i >= adev->usec_timeout)
891 r = -ETIMEDOUT;
892
893 return r;
894 }
895
amdgpu_vcn_enc_get_create_msg(struct amdgpu_ring * ring,uint32_t handle,struct amdgpu_ib * ib_msg,struct dma_fence ** fence)896 static int amdgpu_vcn_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
897 struct amdgpu_ib *ib_msg,
898 struct dma_fence **fence)
899 {
900 unsigned int ib_size_dw = 16;
901 struct amdgpu_device *adev = ring->adev;
902 struct amdgpu_job *job;
903 struct amdgpu_ib *ib;
904 struct dma_fence *f = NULL;
905 uint32_t *ib_checksum = NULL;
906 uint64_t addr;
907 int i, r;
908
909 if (adev->vcn.inst[ring->me].using_unified_queue)
910 ib_size_dw += 8;
911
912 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
913 ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
914 &job);
915 if (r)
916 return r;
917
918 ib = &job->ibs[0];
919 addr = AMDGPU_GPU_PAGE_ALIGN(ib_msg->gpu_addr);
920
921 ib->length_dw = 0;
922
923 if (adev->vcn.inst[ring->me].using_unified_queue)
924 ib_checksum = amdgpu_vcn_unified_ring_ib_header(ib, 0x11, true);
925
926 ib->ptr[ib->length_dw++] = 0x00000018;
927 ib->ptr[ib->length_dw++] = 0x00000001; /* session info */
928 ib->ptr[ib->length_dw++] = handle;
929 ib->ptr[ib->length_dw++] = upper_32_bits(addr);
930 ib->ptr[ib->length_dw++] = addr;
931 ib->ptr[ib->length_dw++] = 0x00000000;
932
933 ib->ptr[ib->length_dw++] = 0x00000014;
934 ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
935 ib->ptr[ib->length_dw++] = 0x0000001c;
936 ib->ptr[ib->length_dw++] = 0x00000000;
937 ib->ptr[ib->length_dw++] = 0x00000000;
938
939 ib->ptr[ib->length_dw++] = 0x00000008;
940 ib->ptr[ib->length_dw++] = 0x08000001; /* op initialize */
941
942 for (i = ib->length_dw; i < ib_size_dw; ++i)
943 ib->ptr[i] = 0x0;
944
945 if (adev->vcn.inst[ring->me].using_unified_queue)
946 amdgpu_vcn_unified_ring_ib_checksum(&ib_checksum, 0x11);
947
948 r = amdgpu_job_submit_direct(job, ring, &f);
949 if (r)
950 goto err;
951
952 if (fence)
953 *fence = dma_fence_get(f);
954 dma_fence_put(f);
955
956 return 0;
957
958 err:
959 amdgpu_job_free(job);
960 return r;
961 }
962
amdgpu_vcn_enc_get_destroy_msg(struct amdgpu_ring * ring,uint32_t handle,struct amdgpu_ib * ib_msg,struct dma_fence ** fence)963 static int amdgpu_vcn_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
964 struct amdgpu_ib *ib_msg,
965 struct dma_fence **fence)
966 {
967 unsigned int ib_size_dw = 16;
968 struct amdgpu_device *adev = ring->adev;
969 struct amdgpu_job *job;
970 struct amdgpu_ib *ib;
971 struct dma_fence *f = NULL;
972 uint32_t *ib_checksum = NULL;
973 uint64_t addr;
974 int i, r;
975
976 if (adev->vcn.inst[ring->me].using_unified_queue)
977 ib_size_dw += 8;
978
979 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
980 ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
981 &job);
982 if (r)
983 return r;
984
985 ib = &job->ibs[0];
986 addr = AMDGPU_GPU_PAGE_ALIGN(ib_msg->gpu_addr);
987
988 ib->length_dw = 0;
989
990 if (adev->vcn.inst[ring->me].using_unified_queue)
991 ib_checksum = amdgpu_vcn_unified_ring_ib_header(ib, 0x11, true);
992
993 ib->ptr[ib->length_dw++] = 0x00000018;
994 ib->ptr[ib->length_dw++] = 0x00000001;
995 ib->ptr[ib->length_dw++] = handle;
996 ib->ptr[ib->length_dw++] = upper_32_bits(addr);
997 ib->ptr[ib->length_dw++] = addr;
998 ib->ptr[ib->length_dw++] = 0x00000000;
999
1000 ib->ptr[ib->length_dw++] = 0x00000014;
1001 ib->ptr[ib->length_dw++] = 0x00000002;
1002 ib->ptr[ib->length_dw++] = 0x0000001c;
1003 ib->ptr[ib->length_dw++] = 0x00000000;
1004 ib->ptr[ib->length_dw++] = 0x00000000;
1005
1006 ib->ptr[ib->length_dw++] = 0x00000008;
1007 ib->ptr[ib->length_dw++] = 0x08000002; /* op close session */
1008
1009 for (i = ib->length_dw; i < ib_size_dw; ++i)
1010 ib->ptr[i] = 0x0;
1011
1012 if (adev->vcn.inst[ring->me].using_unified_queue)
1013 amdgpu_vcn_unified_ring_ib_checksum(&ib_checksum, 0x11);
1014
1015 r = amdgpu_job_submit_direct(job, ring, &f);
1016 if (r)
1017 goto err;
1018
1019 if (fence)
1020 *fence = dma_fence_get(f);
1021 dma_fence_put(f);
1022
1023 return 0;
1024
1025 err:
1026 amdgpu_job_free(job);
1027 return r;
1028 }
1029
amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring * ring,long timeout)1030 int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1031 {
1032 struct amdgpu_device *adev = ring->adev;
1033 struct dma_fence *fence = NULL;
1034 struct amdgpu_ib ib;
1035 long r;
1036
1037 memset(&ib, 0, sizeof(ib));
1038 r = amdgpu_ib_get(adev, NULL, (128 << 10) + AMDGPU_GPU_PAGE_SIZE,
1039 AMDGPU_IB_POOL_DIRECT,
1040 &ib);
1041 if (r)
1042 return r;
1043
1044 r = amdgpu_vcn_enc_get_create_msg(ring, 1, &ib, NULL);
1045 if (r)
1046 goto error;
1047
1048 r = amdgpu_vcn_enc_get_destroy_msg(ring, 1, &ib, &fence);
1049 if (r)
1050 goto error;
1051
1052 r = dma_fence_wait_timeout(fence, false, timeout);
1053 if (r == 0)
1054 r = -ETIMEDOUT;
1055 else if (r > 0)
1056 r = 0;
1057
1058 error:
1059 amdgpu_ib_free(&ib, fence);
1060 dma_fence_put(fence);
1061
1062 return r;
1063 }
1064
amdgpu_vcn_unified_ring_test_ib(struct amdgpu_ring * ring,long timeout)1065 int amdgpu_vcn_unified_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1066 {
1067 struct amdgpu_device *adev = ring->adev;
1068 long r;
1069
1070 if ((amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(4, 0, 3)) &&
1071 (amdgpu_ip_version(adev, UVD_HWIP, 0) != IP_VERSION(5, 0, 1))) {
1072 r = amdgpu_vcn_enc_ring_test_ib(ring, timeout);
1073 if (r)
1074 goto error;
1075 }
1076
1077 r = amdgpu_vcn_dec_sw_ring_test_ib(ring, timeout);
1078
1079 error:
1080 return r;
1081 }
1082
amdgpu_vcn_get_enc_ring_prio(int ring)1083 enum amdgpu_ring_priority_level amdgpu_vcn_get_enc_ring_prio(int ring)
1084 {
1085 switch (ring) {
1086 case 0:
1087 return AMDGPU_RING_PRIO_0;
1088 case 1:
1089 return AMDGPU_RING_PRIO_1;
1090 case 2:
1091 return AMDGPU_RING_PRIO_2;
1092 default:
1093 return AMDGPU_RING_PRIO_0;
1094 }
1095 }
1096
amdgpu_vcn_setup_ucode(struct amdgpu_device * adev,int i)1097 void amdgpu_vcn_setup_ucode(struct amdgpu_device *adev, int i)
1098 {
1099 unsigned int idx;
1100
1101 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1102 const struct common_firmware_header *hdr;
1103
1104 if (adev->vcn.harvest_config & (1 << i))
1105 return;
1106
1107 if ((amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(4, 0, 3) ||
1108 amdgpu_ip_version(adev, UVD_HWIP, 0) == IP_VERSION(5, 0, 1))
1109 && (i > 0))
1110 return;
1111
1112 hdr = (const struct common_firmware_header *)adev->vcn.inst[i].fw->data;
1113 /* currently only support 2 FW instances */
1114 if (i >= 2) {
1115 dev_info(adev->dev, "More then 2 VCN FW instances!\n");
1116 return;
1117 }
1118 idx = AMDGPU_UCODE_ID_VCN + i;
1119 adev->firmware.ucode[idx].ucode_id = idx;
1120 adev->firmware.ucode[idx].fw = adev->vcn.inst[i].fw;
1121 adev->firmware.fw_size +=
1122 ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
1123 }
1124 }
1125
1126 /*
1127 * debugfs for mapping vcn firmware log buffer.
1128 */
1129 #if defined(CONFIG_DEBUG_FS)
amdgpu_debugfs_vcn_fwlog_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1130 static ssize_t amdgpu_debugfs_vcn_fwlog_read(struct file *f, char __user *buf,
1131 size_t size, loff_t *pos)
1132 {
1133 struct amdgpu_vcn_inst *vcn;
1134 void *log_buf;
1135 volatile struct amdgpu_vcn_fwlog *plog;
1136 unsigned int read_pos, write_pos, available, i, read_bytes = 0;
1137 unsigned int read_num[2] = {0};
1138
1139 vcn = file_inode(f)->i_private;
1140 if (!vcn)
1141 return -ENODEV;
1142
1143 if (!vcn->fw_shared.cpu_addr || !amdgpu_vcnfw_log)
1144 return -EFAULT;
1145
1146 log_buf = vcn->fw_shared.cpu_addr + vcn->fw_shared.mem_size;
1147
1148 plog = (volatile struct amdgpu_vcn_fwlog *)log_buf;
1149 read_pos = plog->rptr;
1150 write_pos = plog->wptr;
1151
1152 if (read_pos > AMDGPU_VCNFW_LOG_SIZE || write_pos > AMDGPU_VCNFW_LOG_SIZE)
1153 return -EFAULT;
1154
1155 if (!size || (read_pos == write_pos))
1156 return 0;
1157
1158 if (write_pos > read_pos) {
1159 available = write_pos - read_pos;
1160 read_num[0] = min_t(size_t, size, available);
1161 } else {
1162 read_num[0] = AMDGPU_VCNFW_LOG_SIZE - read_pos;
1163 available = read_num[0] + write_pos - plog->header_size;
1164 if (size > available)
1165 read_num[1] = write_pos - plog->header_size;
1166 else if (size > read_num[0])
1167 read_num[1] = size - read_num[0];
1168 else
1169 read_num[0] = size;
1170 }
1171
1172 for (i = 0; i < 2; i++) {
1173 if (read_num[i]) {
1174 if (read_pos == AMDGPU_VCNFW_LOG_SIZE)
1175 read_pos = plog->header_size;
1176 if (read_num[i] == copy_to_user((buf + read_bytes),
1177 (log_buf + read_pos), read_num[i]))
1178 return -EFAULT;
1179
1180 read_bytes += read_num[i];
1181 read_pos += read_num[i];
1182 }
1183 }
1184
1185 plog->rptr = read_pos;
1186 *pos += read_bytes;
1187 return read_bytes;
1188 }
1189
1190 static const struct file_operations amdgpu_debugfs_vcnfwlog_fops = {
1191 .owner = THIS_MODULE,
1192 .read = amdgpu_debugfs_vcn_fwlog_read,
1193 .llseek = default_llseek
1194 };
1195 #endif
1196
amdgpu_debugfs_vcn_fwlog_init(struct amdgpu_device * adev,uint8_t i,struct amdgpu_vcn_inst * vcn)1197 void amdgpu_debugfs_vcn_fwlog_init(struct amdgpu_device *adev, uint8_t i,
1198 struct amdgpu_vcn_inst *vcn)
1199 {
1200 #if defined(CONFIG_DEBUG_FS)
1201 struct drm_minor *minor = adev_to_drm(adev)->primary;
1202 struct dentry *root = minor->debugfs_root;
1203 char name[32];
1204
1205 sprintf(name, "amdgpu_vcn_%d_fwlog", i);
1206 debugfs_create_file_size(name, S_IFREG | 0444, root, vcn,
1207 &amdgpu_debugfs_vcnfwlog_fops,
1208 AMDGPU_VCNFW_LOG_SIZE);
1209 #endif
1210 }
1211
amdgpu_vcn_fwlog_init(struct amdgpu_vcn_inst * vcn)1212 void amdgpu_vcn_fwlog_init(struct amdgpu_vcn_inst *vcn)
1213 {
1214 #if defined(CONFIG_DEBUG_FS)
1215 volatile uint32_t *flag = vcn->fw_shared.cpu_addr;
1216 void *fw_log_cpu_addr = vcn->fw_shared.cpu_addr + vcn->fw_shared.mem_size;
1217 uint64_t fw_log_gpu_addr = vcn->fw_shared.gpu_addr + vcn->fw_shared.mem_size;
1218 volatile struct amdgpu_vcn_fwlog *log_buf = fw_log_cpu_addr;
1219 volatile struct amdgpu_fw_shared_fw_logging *fw_log = vcn->fw_shared.cpu_addr
1220 + vcn->fw_shared.log_offset;
1221 *flag |= cpu_to_le32(AMDGPU_VCN_FW_LOGGING_FLAG);
1222 fw_log->is_enabled = 1;
1223 fw_log->addr_lo = cpu_to_le32(fw_log_gpu_addr & 0xFFFFFFFF);
1224 fw_log->addr_hi = cpu_to_le32(fw_log_gpu_addr >> 32);
1225 fw_log->size = cpu_to_le32(AMDGPU_VCNFW_LOG_SIZE);
1226
1227 log_buf->header_size = sizeof(struct amdgpu_vcn_fwlog);
1228 log_buf->buffer_size = AMDGPU_VCNFW_LOG_SIZE;
1229 log_buf->rptr = log_buf->header_size;
1230 log_buf->wptr = log_buf->header_size;
1231 log_buf->wrapped = 0;
1232 #endif
1233 }
1234
amdgpu_vcn_process_poison_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1235 int amdgpu_vcn_process_poison_irq(struct amdgpu_device *adev,
1236 struct amdgpu_irq_src *source,
1237 struct amdgpu_iv_entry *entry)
1238 {
1239 struct ras_common_if *ras_if = adev->vcn.ras_if;
1240 struct ras_dispatch_if ih_data = {
1241 .entry = entry,
1242 };
1243
1244 if (!ras_if)
1245 return 0;
1246
1247 if (!amdgpu_sriov_vf(adev)) {
1248 ih_data.head = *ras_if;
1249 amdgpu_ras_interrupt_dispatch(adev, &ih_data);
1250 } else {
1251 if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
1252 adev->virt.ops->ras_poison_handler(adev, ras_if->block);
1253 else
1254 dev_warn(adev->dev,
1255 "No ras_poison_handler interface in SRIOV for VCN!\n");
1256 }
1257
1258 return 0;
1259 }
1260
amdgpu_vcn_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)1261 int amdgpu_vcn_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1262 {
1263 int r, i;
1264
1265 r = amdgpu_ras_block_late_init(adev, ras_block);
1266 if (r)
1267 return r;
1268
1269 if (amdgpu_ras_is_supported(adev, ras_block->block)) {
1270 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1271 if (adev->vcn.harvest_config & (1 << i) ||
1272 !adev->vcn.inst[i].ras_poison_irq.funcs)
1273 continue;
1274
1275 r = amdgpu_irq_get(adev, &adev->vcn.inst[i].ras_poison_irq, 0);
1276 if (r)
1277 goto late_fini;
1278 }
1279 }
1280 return 0;
1281
1282 late_fini:
1283 amdgpu_ras_block_late_fini(adev, ras_block);
1284 return r;
1285 }
1286
amdgpu_vcn_ras_sw_init(struct amdgpu_device * adev)1287 int amdgpu_vcn_ras_sw_init(struct amdgpu_device *adev)
1288 {
1289 int err;
1290 struct amdgpu_vcn_ras *ras;
1291
1292 if (!adev->vcn.ras)
1293 return 0;
1294
1295 ras = adev->vcn.ras;
1296 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
1297 if (err) {
1298 dev_err(adev->dev, "Failed to register vcn ras block!\n");
1299 return err;
1300 }
1301
1302 strcpy(ras->ras_block.ras_comm.name, "vcn");
1303 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__VCN;
1304 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
1305 adev->vcn.ras_if = &ras->ras_block.ras_comm;
1306
1307 if (!ras->ras_block.ras_late_init)
1308 ras->ras_block.ras_late_init = amdgpu_vcn_ras_late_init;
1309
1310 return 0;
1311 }
1312
amdgpu_vcn_psp_update_sram(struct amdgpu_device * adev,int inst_idx,enum AMDGPU_UCODE_ID ucode_id)1313 int amdgpu_vcn_psp_update_sram(struct amdgpu_device *adev, int inst_idx,
1314 enum AMDGPU_UCODE_ID ucode_id)
1315 {
1316 struct amdgpu_firmware_info ucode = {
1317 .ucode_id = (ucode_id ? ucode_id :
1318 (inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
1319 AMDGPU_UCODE_ID_VCN0_RAM)),
1320 .mc_addr = adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
1321 .ucode_size = ((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
1322 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr),
1323 };
1324
1325 return psp_execute_ip_fw_load(&adev->psp, &ucode);
1326 }
1327
amdgpu_get_vcn_reset_mask(struct device * dev,struct device_attribute * attr,char * buf)1328 static ssize_t amdgpu_get_vcn_reset_mask(struct device *dev,
1329 struct device_attribute *attr,
1330 char *buf)
1331 {
1332 struct drm_device *ddev = dev_get_drvdata(dev);
1333 struct amdgpu_device *adev = drm_to_adev(ddev);
1334
1335 if (!adev)
1336 return -ENODEV;
1337
1338 return amdgpu_show_reset_mask(buf, adev->vcn.supported_reset);
1339 }
1340
1341 static DEVICE_ATTR(vcn_reset_mask, 0444,
1342 amdgpu_get_vcn_reset_mask, NULL);
1343
amdgpu_vcn_sysfs_reset_mask_init(struct amdgpu_device * adev)1344 int amdgpu_vcn_sysfs_reset_mask_init(struct amdgpu_device *adev)
1345 {
1346 int r = 0;
1347
1348 if (adev->vcn.num_vcn_inst) {
1349 r = device_create_file(adev->dev, &dev_attr_vcn_reset_mask);
1350 if (r)
1351 return r;
1352 }
1353
1354 return r;
1355 }
1356
amdgpu_vcn_sysfs_reset_mask_fini(struct amdgpu_device * adev)1357 void amdgpu_vcn_sysfs_reset_mask_fini(struct amdgpu_device *adev)
1358 {
1359 if (adev->dev->kobj.sd) {
1360 if (adev->vcn.num_vcn_inst)
1361 device_remove_file(adev->dev, &dev_attr_vcn_reset_mask);
1362 }
1363 }
1364
1365 /*
1366 * debugfs to enable/disable vcn job submission to specific core or
1367 * instance. It is created only if the queue type is unified.
1368 */
1369 #if defined(CONFIG_DEBUG_FS)
amdgpu_debugfs_vcn_sched_mask_set(void * data,u64 val)1370 static int amdgpu_debugfs_vcn_sched_mask_set(void *data, u64 val)
1371 {
1372 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1373 u32 i;
1374 u64 mask;
1375 struct amdgpu_ring *ring;
1376
1377 if (!adev)
1378 return -ENODEV;
1379
1380 mask = (1ULL << adev->vcn.num_vcn_inst) - 1;
1381 if ((val & mask) == 0)
1382 return -EINVAL;
1383 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1384 ring = &adev->vcn.inst[i].ring_enc[0];
1385 if (val & (1ULL << i))
1386 ring->sched.ready = true;
1387 else
1388 ring->sched.ready = false;
1389 }
1390 /* publish sched.ready flag update effective immediately across smp */
1391 smp_rmb();
1392 return 0;
1393 }
1394
amdgpu_debugfs_vcn_sched_mask_get(void * data,u64 * val)1395 static int amdgpu_debugfs_vcn_sched_mask_get(void *data, u64 *val)
1396 {
1397 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1398 u32 i;
1399 u64 mask = 0;
1400 struct amdgpu_ring *ring;
1401
1402 if (!adev)
1403 return -ENODEV;
1404 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1405 ring = &adev->vcn.inst[i].ring_enc[0];
1406 if (ring->sched.ready)
1407 mask |= 1ULL << i;
1408 }
1409 *val = mask;
1410 return 0;
1411 }
1412
1413 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_vcn_sched_mask_fops,
1414 amdgpu_debugfs_vcn_sched_mask_get,
1415 amdgpu_debugfs_vcn_sched_mask_set, "%llx\n");
1416 #endif
1417
amdgpu_debugfs_vcn_sched_mask_init(struct amdgpu_device * adev)1418 void amdgpu_debugfs_vcn_sched_mask_init(struct amdgpu_device *adev)
1419 {
1420 #if defined(CONFIG_DEBUG_FS)
1421 struct drm_minor *minor = adev_to_drm(adev)->primary;
1422 struct dentry *root = minor->debugfs_root;
1423 char name[32];
1424
1425 if (adev->vcn.num_vcn_inst <= 1 || !adev->vcn.inst[0].using_unified_queue)
1426 return;
1427 sprintf(name, "amdgpu_vcn_sched_mask");
1428 debugfs_create_file(name, 0600, root, adev,
1429 &amdgpu_debugfs_vcn_sched_mask_fops);
1430 #endif
1431 }
1432
1433 /**
1434 * vcn_set_powergating_state - set VCN block powergating state
1435 *
1436 * @ip_block: amdgpu_ip_block pointer
1437 * @state: power gating state
1438 *
1439 * Set VCN block powergating state
1440 */
vcn_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)1441 int vcn_set_powergating_state(struct amdgpu_ip_block *ip_block,
1442 enum amd_powergating_state state)
1443 {
1444 struct amdgpu_device *adev = ip_block->adev;
1445 int ret = 0, i;
1446
1447 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1448 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
1449
1450 ret |= vinst->set_pg_state(vinst, state);
1451 }
1452
1453 return ret;
1454 }
1455
1456 /**
1457 * amdgpu_vcn_reset_engine - Reset a specific VCN engine
1458 * @adev: Pointer to the AMDGPU device
1459 * @instance_id: VCN engine instance to reset
1460 *
1461 * Returns: 0 on success, or a negative error code on failure.
1462 */
amdgpu_vcn_reset_engine(struct amdgpu_device * adev,uint32_t instance_id)1463 static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev,
1464 uint32_t instance_id)
1465 {
1466 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[instance_id];
1467 int r, i;
1468
1469 mutex_lock(&vinst->engine_reset_mutex);
1470 /* Stop the scheduler's work queue for the dec and enc rings if they are running.
1471 * This ensures that no new tasks are submitted to the queues while
1472 * the reset is in progress.
1473 */
1474 drm_sched_wqueue_stop(&vinst->ring_dec.sched);
1475 for (i = 0; i < vinst->num_enc_rings; i++)
1476 drm_sched_wqueue_stop(&vinst->ring_enc[i].sched);
1477
1478 /* Perform the VCN reset for the specified instance */
1479 r = vinst->reset(vinst);
1480 if (r)
1481 goto unlock;
1482 r = amdgpu_ring_test_ring(&vinst->ring_dec);
1483 if (r)
1484 goto unlock;
1485 for (i = 0; i < vinst->num_enc_rings; i++) {
1486 r = amdgpu_ring_test_ring(&vinst->ring_enc[i]);
1487 if (r)
1488 goto unlock;
1489 }
1490 amdgpu_fence_driver_force_completion(&vinst->ring_dec);
1491 for (i = 0; i < vinst->num_enc_rings; i++)
1492 amdgpu_fence_driver_force_completion(&vinst->ring_enc[i]);
1493
1494 /* Restart the scheduler's work queue for the dec and enc rings
1495 * if they were stopped by this function. This allows new tasks
1496 * to be submitted to the queues after the reset is complete.
1497 */
1498 drm_sched_wqueue_start(&vinst->ring_dec.sched);
1499 for (i = 0; i < vinst->num_enc_rings; i++)
1500 drm_sched_wqueue_start(&vinst->ring_enc[i].sched);
1501
1502 unlock:
1503 mutex_unlock(&vinst->engine_reset_mutex);
1504
1505 return r;
1506 }
1507
1508 /**
1509 * amdgpu_vcn_ring_reset - Reset a VCN ring
1510 * @ring: ring to reset
1511 * @vmid: vmid of guilty job
1512 * @timedout_fence: fence of timed out job
1513 *
1514 * This helper is for VCN blocks without unified queues because
1515 * resetting the engine resets all queues in that case. With
1516 * unified queues we have one queue per engine.
1517 * Returns: 0 on success, or a negative error code on failure.
1518 */
amdgpu_vcn_ring_reset(struct amdgpu_ring * ring,unsigned int vmid,struct amdgpu_fence * timedout_fence)1519 int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring,
1520 unsigned int vmid,
1521 struct amdgpu_fence *timedout_fence)
1522 {
1523 struct amdgpu_device *adev = ring->adev;
1524
1525 if (adev->vcn.inst[ring->me].using_unified_queue)
1526 return -EINVAL;
1527
1528 return amdgpu_vcn_reset_engine(adev, ring->me);
1529 }
1530