xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 /*
2  * Copyright 2013 Advanced Micro Devices, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  * Authors: Christian König <christian.koenig@amd.com>
26  */
27 
28 #include <linux/firmware.h>
29 #include <linux/module.h>
30 
31 #include <drm/drm.h>
32 #include <drm/drm_drv.h>
33 
34 #include "amdgpu.h"
35 #include "amdgpu_pm.h"
36 #include "amdgpu_vce.h"
37 #include "amdgpu_cs.h"
38 #include "cikd.h"
39 
40 /* 1 second timeout */
41 #define VCE_IDLE_TIMEOUT	msecs_to_jiffies(1000)
42 
43 /* Firmware Names */
44 #ifdef CONFIG_DRM_AMDGPU_SI
45 #define FIRMWARE_VCE_V1_0	"amdgpu/vce_1_0_0.bin"
46 #endif
47 #ifdef CONFIG_DRM_AMDGPU_CIK
48 #define FIRMWARE_BONAIRE	"amdgpu/bonaire_vce.bin"
49 #define FIRMWARE_KABINI	"amdgpu/kabini_vce.bin"
50 #define FIRMWARE_KAVERI	"amdgpu/kaveri_vce.bin"
51 #define FIRMWARE_HAWAII	"amdgpu/hawaii_vce.bin"
52 #define FIRMWARE_MULLINS	"amdgpu/mullins_vce.bin"
53 #endif
54 #define FIRMWARE_TONGA		"amdgpu/tonga_vce.bin"
55 #define FIRMWARE_CARRIZO	"amdgpu/carrizo_vce.bin"
56 #define FIRMWARE_FIJI		"amdgpu/fiji_vce.bin"
57 #define FIRMWARE_STONEY		"amdgpu/stoney_vce.bin"
58 #define FIRMWARE_POLARIS10	"amdgpu/polaris10_vce.bin"
59 #define FIRMWARE_POLARIS11	"amdgpu/polaris11_vce.bin"
60 #define FIRMWARE_POLARIS12	"amdgpu/polaris12_vce.bin"
61 #define FIRMWARE_VEGAM		"amdgpu/vegam_vce.bin"
62 
63 #define FIRMWARE_VEGA10		"amdgpu/vega10_vce.bin"
64 #define FIRMWARE_VEGA12		"amdgpu/vega12_vce.bin"
65 #define FIRMWARE_VEGA20		"amdgpu/vega20_vce.bin"
66 
67 #ifdef CONFIG_DRM_AMDGPU_SI
68 MODULE_FIRMWARE(FIRMWARE_VCE_V1_0);
69 #endif
70 #ifdef CONFIG_DRM_AMDGPU_CIK
71 MODULE_FIRMWARE(FIRMWARE_BONAIRE);
72 MODULE_FIRMWARE(FIRMWARE_KABINI);
73 MODULE_FIRMWARE(FIRMWARE_KAVERI);
74 MODULE_FIRMWARE(FIRMWARE_HAWAII);
75 MODULE_FIRMWARE(FIRMWARE_MULLINS);
76 #endif
77 MODULE_FIRMWARE(FIRMWARE_TONGA);
78 MODULE_FIRMWARE(FIRMWARE_CARRIZO);
79 MODULE_FIRMWARE(FIRMWARE_FIJI);
80 MODULE_FIRMWARE(FIRMWARE_STONEY);
81 MODULE_FIRMWARE(FIRMWARE_POLARIS10);
82 MODULE_FIRMWARE(FIRMWARE_POLARIS11);
83 MODULE_FIRMWARE(FIRMWARE_POLARIS12);
84 MODULE_FIRMWARE(FIRMWARE_VEGAM);
85 
86 MODULE_FIRMWARE(FIRMWARE_VEGA10);
87 MODULE_FIRMWARE(FIRMWARE_VEGA12);
88 MODULE_FIRMWARE(FIRMWARE_VEGA20);
89 
90 static void amdgpu_vce_idle_work_handler(struct work_struct *work);
91 static int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
92 				     struct dma_fence **fence);
93 static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
94 				      bool direct, struct dma_fence **fence);
95 
96 /**
97  * amdgpu_vce_firmware_name() - determine the firmware file name for VCE
98  *
99  * @adev: amdgpu_device pointer
100  *
101  * Each chip that has VCE IP may need a different firmware.
102  * This function returns the name of the VCE firmware file
103  * appropriate for the current chip.
104  */
105 static const char *amdgpu_vce_firmware_name(struct amdgpu_device *adev)
106 {
107 	switch (adev->asic_type) {
108 #ifdef CONFIG_DRM_AMDGPU_SI
109 	case CHIP_PITCAIRN:
110 	case CHIP_TAHITI:
111 	case CHIP_VERDE:
112 		return FIRMWARE_VCE_V1_0;
113 #endif
114 #ifdef CONFIG_DRM_AMDGPU_CIK
115 	case CHIP_BONAIRE:
116 		return FIRMWARE_BONAIRE;
117 	case CHIP_KAVERI:
118 		return FIRMWARE_KAVERI;
119 	case CHIP_KABINI:
120 		return FIRMWARE_KABINI;
121 	case CHIP_HAWAII:
122 		return FIRMWARE_HAWAII;
123 	case CHIP_MULLINS:
124 		return FIRMWARE_MULLINS;
125 #endif
126 	case CHIP_TONGA:
127 		return  FIRMWARE_TONGA;
128 	case CHIP_CARRIZO:
129 		return  FIRMWARE_CARRIZO;
130 	case CHIP_FIJI:
131 		return  FIRMWARE_FIJI;
132 	case CHIP_STONEY:
133 		return  FIRMWARE_STONEY;
134 	case CHIP_POLARIS10:
135 		return  FIRMWARE_POLARIS10;
136 	case CHIP_POLARIS11:
137 		return  FIRMWARE_POLARIS11;
138 	case CHIP_POLARIS12:
139 		return  FIRMWARE_POLARIS12;
140 	case CHIP_VEGAM:
141 		return  FIRMWARE_VEGAM;
142 	case CHIP_VEGA10:
143 		return  FIRMWARE_VEGA10;
144 	case CHIP_VEGA12:
145 		return  FIRMWARE_VEGA12;
146 	case CHIP_VEGA20:
147 		return  FIRMWARE_VEGA20;
148 
149 	default:
150 		return NULL;
151 	}
152 }
153 
154 /**
155  * amdgpu_vce_early_init() - try to load VCE firmware
156  *
157  * @adev: amdgpu_device pointer
158  *
159  * Tries to load the VCE firmware.
160  *
161  * When not found, returns ENOENT so that the driver can
162  * still load and initialize the rest of the IP blocks.
163  * The GPU can function just fine without VCE, they will just
164  * not support video encoding.
165  */
166 int amdgpu_vce_early_init(struct amdgpu_device *adev)
167 {
168 	const char *fw_name = amdgpu_vce_firmware_name(adev);
169 	const struct common_firmware_header *hdr;
170 	unsigned int ucode_version, version_major, version_minor, binary_id;
171 	int r;
172 
173 	if (!fw_name)
174 		return -ENOENT;
175 
176 	r = amdgpu_ucode_request(adev, &adev->vce.fw, AMDGPU_UCODE_REQUIRED, "%s", fw_name);
177 	if (r) {
178 		dev_err(adev->dev,
179 			"amdgpu_vce: Firmware \"%s\" not found or failed to validate (%d)\n",
180 			fw_name, r);
181 
182 		amdgpu_ucode_release(&adev->vce.fw);
183 		return -ENOENT;
184 	}
185 
186 	hdr = (const struct common_firmware_header *)adev->vce.fw->data;
187 
188 	ucode_version = le32_to_cpu(hdr->ucode_version);
189 	version_major = (ucode_version >> 20) & 0xfff;
190 	version_minor = (ucode_version >> 8) & 0xfff;
191 	binary_id = ucode_version & 0xff;
192 	dev_info(adev->dev, "Found VCE firmware Version: %d.%d Binary ID: %d\n",
193 		version_major, version_minor, binary_id);
194 	adev->vce.fw_version = ((version_major << 24) | (version_minor << 16) |
195 				(binary_id << 8));
196 
197 	return 0;
198 }
199 
200 /**
201  * amdgpu_vce_sw_init() - allocate memory for VCE BO
202  *
203  * @adev: amdgpu_device pointer
204  * @size: size for the new BO
205  *
206  * First step to get VCE online: allocate memory for VCE BO.
207  * The VCE firmware binary is copied into the VCE BO later,
208  * in amdgpu_vce_resume. The VCE executes its code from the
209  * VCE BO and also uses the space in this BO for its stack and data.
210  *
211  * Ideally this BO should be placed in VRAM for optimal performance,
212  * although technically it also runs from system RAM (albeit slowly).
213  */
214 int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
215 {
216 	int i, r;
217 
218 	if (!adev->vce.fw)
219 		return -ENOENT;
220 
221 	r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
222 				    AMDGPU_GEM_DOMAIN_VRAM |
223 				    AMDGPU_GEM_DOMAIN_GTT,
224 				    &adev->vce.vcpu_bo,
225 				    &adev->vce.gpu_addr, &adev->vce.cpu_addr);
226 	if (r) {
227 		dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);
228 		return r;
229 	}
230 
231 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
232 		atomic_set(&adev->vce.handles[i], 0);
233 		adev->vce.filp[i] = NULL;
234 	}
235 
236 	INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler);
237 	mutex_init(&adev->vce.idle_mutex);
238 
239 	return 0;
240 }
241 
242 /**
243  * amdgpu_vce_sw_fini - free memory
244  *
245  * @adev: amdgpu_device pointer
246  *
247  * Last step on VCE teardown, free firmware memory
248  */
249 int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
250 {
251 	unsigned int i;
252 
253 	if (adev->vce.vcpu_bo == NULL)
254 		return 0;
255 
256 	if (adev->vce.entity.rq)
257 		drm_sched_entity_destroy(&adev->vce.entity);
258 
259 	for (i = 0; i < adev->vce.num_rings; i++)
260 		amdgpu_ring_fini(&adev->vce.ring[i]);
261 
262 	amdgpu_ucode_release(&adev->vce.fw);
263 	mutex_destroy(&adev->vce.idle_mutex);
264 
265 	amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, &adev->vce.gpu_addr,
266 		(void **)&adev->vce.cpu_addr);
267 
268 	return 0;
269 }
270 
271 /**
272  * amdgpu_vce_entity_init - init entity
273  *
274  * @adev: amdgpu_device pointer
275  * @ring: amdgpu_ring pointer to check
276  *
277  * Initialize the entity used for handle management in the kernel driver.
278  */
279 int amdgpu_vce_entity_init(struct amdgpu_device *adev, struct amdgpu_ring *ring)
280 {
281 	if (ring == &adev->vce.ring[0]) {
282 		struct drm_gpu_scheduler *sched = &ring->sched;
283 		int r;
284 
285 		r = drm_sched_entity_init(&adev->vce.entity, DRM_SCHED_PRIORITY_NORMAL,
286 					  &sched, 1, NULL);
287 		if (r != 0) {
288 			DRM_ERROR("Failed setting up VCE run queue.\n");
289 			return r;
290 		}
291 	}
292 
293 	return 0;
294 }
295 
296 /**
297  * amdgpu_vce_suspend - unpin VCE fw memory
298  *
299  * @adev: amdgpu_device pointer
300  *
301  */
302 int amdgpu_vce_suspend(struct amdgpu_device *adev)
303 {
304 	int i;
305 
306 	cancel_delayed_work_sync(&adev->vce.idle_work);
307 
308 	if (adev->vce.vcpu_bo == NULL)
309 		return 0;
310 
311 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
312 		if (atomic_read(&adev->vce.handles[i]))
313 			break;
314 
315 	if (i == AMDGPU_MAX_VCE_HANDLES)
316 		return 0;
317 
318 	/* TODO: suspending running encoding sessions isn't supported */
319 	return -EINVAL;
320 }
321 
322 /**
323  * amdgpu_vce_resume - pin VCE fw memory
324  *
325  * @adev: amdgpu_device pointer
326  *
327  */
328 int amdgpu_vce_resume(struct amdgpu_device *adev)
329 {
330 	const struct common_firmware_header *hdr;
331 	unsigned int offset;
332 	int idx;
333 
334 	if (adev->vce.vcpu_bo == NULL)
335 		return -EINVAL;
336 
337 	hdr = (const struct common_firmware_header *)adev->vce.fw->data;
338 	offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
339 
340 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
341 		memset_io(adev->vce.cpu_addr, 0, amdgpu_bo_size(adev->vce.vcpu_bo));
342 		memcpy_toio(adev->vce.cpu_addr, adev->vce.fw->data + offset,
343 			    adev->vce.fw->size - offset);
344 		drm_dev_exit(idx);
345 	}
346 
347 	return 0;
348 }
349 
350 /**
351  * amdgpu_vce_idle_work_handler - power off VCE
352  *
353  * @work: pointer to work structure
354  *
355  * power of VCE when it's not used any more
356  */
357 static void amdgpu_vce_idle_work_handler(struct work_struct *work)
358 {
359 	struct amdgpu_device *adev =
360 		container_of(work, struct amdgpu_device, vce.idle_work.work);
361 	unsigned int i, count = 0;
362 
363 	for (i = 0; i < adev->vce.num_rings; i++)
364 		count += amdgpu_fence_count_emitted(&adev->vce.ring[i]);
365 
366 	if (count == 0) {
367 		if (adev->pm.dpm_enabled) {
368 			amdgpu_dpm_enable_vce(adev, false);
369 		} else {
370 			amdgpu_asic_set_vce_clocks(adev, 0, 0);
371 			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
372 							       AMD_PG_STATE_GATE);
373 			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
374 							       AMD_CG_STATE_GATE);
375 		}
376 	} else {
377 		schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT);
378 	}
379 }
380 
381 /**
382  * amdgpu_vce_ring_begin_use - power up VCE
383  *
384  * @ring: amdgpu ring
385  *
386  * Make sure VCE is powerd up when we want to use it
387  */
388 void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring)
389 {
390 	struct amdgpu_device *adev = ring->adev;
391 	bool set_clocks;
392 
393 	if (amdgpu_sriov_vf(adev))
394 		return;
395 
396 	mutex_lock(&adev->vce.idle_mutex);
397 	set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work);
398 	if (set_clocks) {
399 		if (adev->pm.dpm_enabled) {
400 			amdgpu_dpm_enable_vce(adev, true);
401 		} else {
402 			amdgpu_asic_set_vce_clocks(adev, 53300, 40000);
403 			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
404 							       AMD_CG_STATE_UNGATE);
405 			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
406 							       AMD_PG_STATE_UNGATE);
407 
408 		}
409 	}
410 	mutex_unlock(&adev->vce.idle_mutex);
411 }
412 
413 /**
414  * amdgpu_vce_ring_end_use - power VCE down
415  *
416  * @ring: amdgpu ring
417  *
418  * Schedule work to power VCE down again
419  */
420 void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring)
421 {
422 	if (!amdgpu_sriov_vf(ring->adev))
423 		schedule_delayed_work(&ring->adev->vce.idle_work, VCE_IDLE_TIMEOUT);
424 }
425 
426 /**
427  * amdgpu_vce_free_handles - free still open VCE handles
428  *
429  * @adev: amdgpu_device pointer
430  * @filp: drm file pointer
431  *
432  * Close all VCE handles still open by this file pointer
433  */
434 void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
435 {
436 	struct amdgpu_ring *ring = &adev->vce.ring[0];
437 	int i, r;
438 
439 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
440 		uint32_t handle = atomic_read(&adev->vce.handles[i]);
441 
442 		if (!handle || adev->vce.filp[i] != filp)
443 			continue;
444 
445 		r = amdgpu_vce_get_destroy_msg(ring, handle, false, NULL);
446 		if (r)
447 			DRM_ERROR("Error destroying VCE handle (%d)!\n", r);
448 
449 		adev->vce.filp[i] = NULL;
450 		atomic_set(&adev->vce.handles[i], 0);
451 	}
452 }
453 
454 /**
455  * amdgpu_vce_get_create_msg - generate a VCE create msg
456  *
457  * @ring: ring we should submit the msg to
458  * @handle: VCE session handle to use
459  * @fence: optional fence to return
460  *
461  * Open up a stream for HW test
462  */
463 static int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
464 				     struct dma_fence **fence)
465 {
466 	const unsigned int ib_size_dw = 1024;
467 	struct amdgpu_job *job;
468 	struct amdgpu_ib *ib;
469 	struct amdgpu_ib ib_msg;
470 	struct dma_fence *f = NULL;
471 	uint64_t addr;
472 	int i, r;
473 
474 	r = amdgpu_job_alloc_with_ib(ring->adev, &ring->adev->vce.entity,
475 				     AMDGPU_FENCE_OWNER_UNDEFINED,
476 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
477 				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
478 				     &job);
479 	if (r)
480 		return r;
481 
482 	memset(&ib_msg, 0, sizeof(ib_msg));
483 	/* only one gpu page is needed, alloc +1 page to make addr aligned. */
484 	r = amdgpu_ib_get(ring->adev, NULL, AMDGPU_GPU_PAGE_SIZE * 2,
485 			  AMDGPU_IB_POOL_DIRECT,
486 			  &ib_msg);
487 	if (r)
488 		goto err;
489 
490 	ib = &job->ibs[0];
491 	/* let addr point to page boundary */
492 	addr = AMDGPU_GPU_PAGE_ALIGN(ib_msg.gpu_addr);
493 
494 	/* stitch together an VCE create msg */
495 	ib->length_dw = 0;
496 	ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
497 	ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
498 	ib->ptr[ib->length_dw++] = handle;
499 
500 	if ((ring->adev->vce.fw_version >> 24) >= 52)
501 		ib->ptr[ib->length_dw++] = 0x00000040; /* len */
502 	else
503 		ib->ptr[ib->length_dw++] = 0x00000030; /* len */
504 	ib->ptr[ib->length_dw++] = 0x01000001; /* create cmd */
505 	ib->ptr[ib->length_dw++] = 0x00000000;
506 	ib->ptr[ib->length_dw++] = 0x00000042;
507 	ib->ptr[ib->length_dw++] = 0x0000000a;
508 	ib->ptr[ib->length_dw++] = 0x00000001;
509 	ib->ptr[ib->length_dw++] = 0x00000080;
510 	ib->ptr[ib->length_dw++] = 0x00000060;
511 	ib->ptr[ib->length_dw++] = 0x00000100;
512 	ib->ptr[ib->length_dw++] = 0x00000100;
513 	ib->ptr[ib->length_dw++] = 0x0000000c;
514 	ib->ptr[ib->length_dw++] = 0x00000000;
515 	if ((ring->adev->vce.fw_version >> 24) >= 52) {
516 		ib->ptr[ib->length_dw++] = 0x00000000;
517 		ib->ptr[ib->length_dw++] = 0x00000000;
518 		ib->ptr[ib->length_dw++] = 0x00000000;
519 		ib->ptr[ib->length_dw++] = 0x00000000;
520 	}
521 
522 	ib->ptr[ib->length_dw++] = 0x00000014; /* len */
523 	ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */
524 	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
525 	ib->ptr[ib->length_dw++] = addr;
526 	ib->ptr[ib->length_dw++] = 0x00000001;
527 
528 	for (i = ib->length_dw; i < ib_size_dw; ++i)
529 		ib->ptr[i] = 0x0;
530 
531 	r = amdgpu_job_submit_direct(job, ring, &f);
532 	amdgpu_ib_free(&ib_msg, f);
533 	if (r)
534 		goto err;
535 
536 	if (fence)
537 		*fence = dma_fence_get(f);
538 	dma_fence_put(f);
539 	return 0;
540 
541 err:
542 	amdgpu_job_free(job);
543 	return r;
544 }
545 
546 /**
547  * amdgpu_vce_get_destroy_msg - generate a VCE destroy msg
548  *
549  * @ring: ring we should submit the msg to
550  * @handle: VCE session handle to use
551  * @direct: direct or delayed pool
552  * @fence: optional fence to return
553  *
554  * Close up a stream for HW test or if userspace failed to do so
555  */
556 static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
557 				      bool direct, struct dma_fence **fence)
558 {
559 	const unsigned int ib_size_dw = 1024;
560 	struct amdgpu_job *job;
561 	struct amdgpu_ib *ib;
562 	struct dma_fence *f = NULL;
563 	int i, r;
564 
565 	r = amdgpu_job_alloc_with_ib(ring->adev, &ring->adev->vce.entity,
566 				     AMDGPU_FENCE_OWNER_UNDEFINED,
567 				     ib_size_dw * 4,
568 				     direct ? AMDGPU_IB_POOL_DIRECT :
569 				     AMDGPU_IB_POOL_DELAYED,
570 				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
571 				     &job);
572 	if (r)
573 		return r;
574 
575 	ib = &job->ibs[0];
576 
577 	/* stitch together an VCE destroy msg */
578 	ib->length_dw = 0;
579 	ib->ptr[ib->length_dw++] = 0x0000000c; /* len */
580 	ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
581 	ib->ptr[ib->length_dw++] = handle;
582 
583 	ib->ptr[ib->length_dw++] = 0x00000020; /* len */
584 	ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
585 	ib->ptr[ib->length_dw++] = 0xffffffff; /* next task info, set to 0xffffffff if no */
586 	ib->ptr[ib->length_dw++] = 0x00000001; /* destroy session */
587 	ib->ptr[ib->length_dw++] = 0x00000000;
588 	ib->ptr[ib->length_dw++] = 0x00000000;
589 	ib->ptr[ib->length_dw++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */
590 	ib->ptr[ib->length_dw++] = 0x00000000;
591 
592 	ib->ptr[ib->length_dw++] = 0x00000008; /* len */
593 	ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */
594 
595 	for (i = ib->length_dw; i < ib_size_dw; ++i)
596 		ib->ptr[i] = 0x0;
597 
598 	if (direct)
599 		r = amdgpu_job_submit_direct(job, ring, &f);
600 	else
601 		f = amdgpu_job_submit(job);
602 	if (r)
603 		goto err;
604 
605 	if (fence)
606 		*fence = dma_fence_get(f);
607 	dma_fence_put(f);
608 	return 0;
609 
610 err:
611 	amdgpu_job_free(job);
612 	return r;
613 }
614 
615 /**
616  * amdgpu_vce_validate_bo - make sure not to cross 4GB boundary
617  *
618  * @p: cs parser
619  * @ib: indirect buffer to use
620  * @lo: address of lower dword
621  * @hi: address of higher dword
622  * @size: minimum size
623  * @index: bs/fb index
624  *
625  * Make sure that no BO cross a 4GB boundary.
626  */
627 static int amdgpu_vce_validate_bo(struct amdgpu_cs_parser *p,
628 				  struct amdgpu_ib *ib, int lo, int hi,
629 				  unsigned int size, int32_t index)
630 {
631 	int64_t offset = ((uint64_t)size) * ((int64_t)index);
632 	struct ttm_operation_ctx ctx = { false, false };
633 	struct amdgpu_bo_va_mapping *mapping;
634 	unsigned int i, fpfn, lpfn;
635 	struct amdgpu_bo *bo;
636 	uint64_t addr;
637 	int r;
638 
639 	addr = ((uint64_t)amdgpu_ib_get_value(ib, lo)) |
640 	       ((uint64_t)amdgpu_ib_get_value(ib, hi)) << 32;
641 	if (index >= 0) {
642 		addr += offset;
643 		fpfn = PAGE_ALIGN(offset) >> PAGE_SHIFT;
644 		lpfn = 0x100000000ULL >> PAGE_SHIFT;
645 	} else {
646 		fpfn = 0;
647 		lpfn = (0x100000000ULL - PAGE_ALIGN(offset)) >> PAGE_SHIFT;
648 	}
649 
650 	r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
651 	if (r) {
652 		DRM_ERROR("Can't find BO for addr 0x%010llx %d %d %d %d\n",
653 			  addr, lo, hi, size, index);
654 		return r;
655 	}
656 
657 	for (i = 0; i < bo->placement.num_placement; ++i) {
658 		bo->placements[i].fpfn = max(bo->placements[i].fpfn, fpfn);
659 		bo->placements[i].lpfn = bo->placements[i].lpfn ?
660 			min(bo->placements[i].lpfn, lpfn) : lpfn;
661 	}
662 	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
663 }
664 
665 
666 /**
667  * amdgpu_vce_cs_reloc - command submission relocation
668  *
669  * @p: parser context
670  * @ib: indirect buffer to use
671  * @lo: address of lower dword
672  * @hi: address of higher dword
673  * @size: minimum size
674  * @index: bs/fb index
675  *
676  * Patch relocation inside command stream with real buffer address
677  */
678 static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, struct amdgpu_ib *ib,
679 			       int lo, int hi, unsigned int size, uint32_t index)
680 {
681 	struct amdgpu_bo_va_mapping *mapping;
682 	struct amdgpu_bo *bo;
683 	uint64_t addr;
684 	int r;
685 
686 	if (lo >= ib->length_dw || hi >= ib->length_dw)
687 		return -EINVAL;
688 
689 	if (index == 0xffffffff)
690 		index = 0;
691 
692 	addr = ((uint64_t)amdgpu_ib_get_value(ib, lo)) |
693 	       ((uint64_t)amdgpu_ib_get_value(ib, hi)) << 32;
694 	addr += ((uint64_t)size) * ((uint64_t)index);
695 
696 	r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
697 	if (r) {
698 		DRM_ERROR("Can't find BO for addr 0x%010llx %d %d %d %d\n",
699 			  addr, lo, hi, size, index);
700 		return r;
701 	}
702 
703 	if ((addr + (uint64_t)size) >
704 	    (mapping->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
705 		DRM_ERROR("BO too small for addr 0x%010llx %d %d\n",
706 			  addr, lo, hi);
707 		return -EINVAL;
708 	}
709 
710 	addr -= mapping->start * AMDGPU_GPU_PAGE_SIZE;
711 	addr += amdgpu_bo_gpu_offset(bo);
712 	addr -= ((uint64_t)size) * ((uint64_t)index);
713 
714 	amdgpu_ib_set_value(ib, lo, lower_32_bits(addr));
715 	amdgpu_ib_set_value(ib, hi, upper_32_bits(addr));
716 
717 	return 0;
718 }
719 
720 /**
721  * amdgpu_vce_validate_handle - validate stream handle
722  *
723  * @p: parser context
724  * @handle: handle to validate
725  * @allocated: allocated a new handle?
726  *
727  * Validates the handle and return the found session index or -EINVAL
728  * we don't have another free session index.
729  */
730 static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p,
731 				      uint32_t handle, uint32_t *allocated)
732 {
733 	unsigned int i;
734 
735 	/* validate the handle */
736 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
737 		if (atomic_read(&p->adev->vce.handles[i]) == handle) {
738 			if (p->adev->vce.filp[i] != p->filp) {
739 				DRM_ERROR("VCE handle collision detected!\n");
740 				return -EINVAL;
741 			}
742 			return i;
743 		}
744 	}
745 
746 	/* handle not found try to alloc a new one */
747 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
748 		if (!atomic_cmpxchg(&p->adev->vce.handles[i], 0, handle)) {
749 			p->adev->vce.filp[i] = p->filp;
750 			p->adev->vce.img_size[i] = 0;
751 			*allocated |= 1 << i;
752 			return i;
753 		}
754 	}
755 
756 	DRM_ERROR("No more free VCE handles!\n");
757 	return -EINVAL;
758 }
759 
760 /**
761  * amdgpu_vce_ring_parse_cs - parse and validate the command stream
762  *
763  * @p: parser context
764  * @job: the job to parse
765  * @ib: the IB to patch
766  */
767 int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p,
768 			     struct amdgpu_job *job,
769 			     struct amdgpu_ib *ib)
770 {
771 	unsigned int fb_idx = 0, bs_idx = 0;
772 	int session_idx = -1;
773 	uint32_t destroyed = 0;
774 	uint32_t created = 0;
775 	uint32_t allocated = 0;
776 	uint32_t tmp, handle = 0;
777 	uint32_t dummy = 0xffffffff;
778 	uint32_t *size = &dummy;
779 	unsigned int idx;
780 	int i, r = 0;
781 
782 	job->vm = NULL;
783 
784 	for (idx = 0; idx < ib->length_dw;) {
785 		uint32_t len = amdgpu_ib_get_value(ib, idx);
786 		uint32_t cmd = amdgpu_ib_get_value(ib, idx + 1);
787 
788 		if ((len < 8) || (len & 3)) {
789 			DRM_ERROR("invalid VCE command length (%d)!\n", len);
790 			r = -EINVAL;
791 			goto out;
792 		}
793 
794 		switch (cmd) {
795 		case 0x00000002: /* task info */
796 			fb_idx = amdgpu_ib_get_value(ib, idx + 6);
797 			bs_idx = amdgpu_ib_get_value(ib, idx + 7);
798 			break;
799 
800 		case 0x03000001: /* encode */
801 			r = amdgpu_vce_validate_bo(p, ib, idx + 10, idx + 9,
802 						   0, 0);
803 			if (r)
804 				goto out;
805 
806 			r = amdgpu_vce_validate_bo(p, ib, idx + 12, idx + 11,
807 						   0, 0);
808 			if (r)
809 				goto out;
810 			break;
811 
812 		case 0x05000001: /* context buffer */
813 			r = amdgpu_vce_validate_bo(p, ib, idx + 3, idx + 2,
814 						   0, 0);
815 			if (r)
816 				goto out;
817 			break;
818 
819 		case 0x05000004: /* video bitstream buffer */
820 			tmp = amdgpu_ib_get_value(ib, idx + 4);
821 			r = amdgpu_vce_validate_bo(p, ib, idx + 3, idx + 2,
822 						   tmp, bs_idx);
823 			if (r)
824 				goto out;
825 			break;
826 
827 		case 0x05000005: /* feedback buffer */
828 			r = amdgpu_vce_validate_bo(p, ib, idx + 3, idx + 2,
829 						   4096, fb_idx);
830 			if (r)
831 				goto out;
832 			break;
833 
834 		case 0x0500000d: /* MV buffer */
835 			r = amdgpu_vce_validate_bo(p, ib, idx + 3, idx + 2,
836 						   0, 0);
837 			if (r)
838 				goto out;
839 
840 			r = amdgpu_vce_validate_bo(p, ib, idx + 8, idx + 7,
841 						   0, 0);
842 			if (r)
843 				goto out;
844 			break;
845 		}
846 
847 		idx += len / 4;
848 	}
849 
850 	for (idx = 0; idx < ib->length_dw;) {
851 		uint32_t len = amdgpu_ib_get_value(ib, idx);
852 		uint32_t cmd = amdgpu_ib_get_value(ib, idx + 1);
853 
854 		switch (cmd) {
855 		case 0x00000001: /* session */
856 			handle = amdgpu_ib_get_value(ib, idx + 2);
857 			session_idx = amdgpu_vce_validate_handle(p, handle,
858 								 &allocated);
859 			if (session_idx < 0) {
860 				r = session_idx;
861 				goto out;
862 			}
863 			size = &p->adev->vce.img_size[session_idx];
864 			break;
865 
866 		case 0x00000002: /* task info */
867 			fb_idx = amdgpu_ib_get_value(ib, idx + 6);
868 			bs_idx = amdgpu_ib_get_value(ib, idx + 7);
869 			break;
870 
871 		case 0x01000001: /* create */
872 			created |= 1 << session_idx;
873 			if (destroyed & (1 << session_idx)) {
874 				destroyed &= ~(1 << session_idx);
875 				allocated |= 1 << session_idx;
876 
877 			} else if (!(allocated & (1 << session_idx))) {
878 				DRM_ERROR("Handle already in use!\n");
879 				r = -EINVAL;
880 				goto out;
881 			}
882 
883 			uint32_t width, height;
884 			width = amdgpu_ib_get_value(ib, idx + 8);
885 			height = amdgpu_ib_get_value(ib, idx + 10);
886 
887 			if (width == 0 || height == 0 ||
888 			    width > 4096 || height > 2304) {
889 				DRM_ERROR("invalid VCE image size: %ux%u\n",
890 					  width, height);
891 				r = -EINVAL;
892 				goto out;
893 			}
894 
895 			*size = width * height * 8 * 3 / 2;
896 
897 			break;
898 
899 		case 0x04000001: /* config extension */
900 		case 0x04000002: /* pic control */
901 		case 0x04000005: /* rate control */
902 		case 0x04000007: /* motion estimation */
903 		case 0x04000008: /* rdo */
904 		case 0x04000009: /* vui */
905 		case 0x05000002: /* auxiliary buffer */
906 		case 0x05000009: /* clock table */
907 			break;
908 
909 		case 0x0500000c: /* hw config */
910 			switch (p->adev->asic_type) {
911 #ifdef CONFIG_DRM_AMDGPU_CIK
912 			case CHIP_KAVERI:
913 			case CHIP_MULLINS:
914 #endif
915 			case CHIP_CARRIZO:
916 				break;
917 			default:
918 				r = -EINVAL;
919 				goto out;
920 			}
921 			break;
922 
923 		case 0x03000001: /* encode */
924 			r = amdgpu_vce_cs_reloc(p, ib, idx + 10, idx + 9,
925 						*size, 0);
926 			if (r)
927 				goto out;
928 
929 			r = amdgpu_vce_cs_reloc(p, ib, idx + 12, idx + 11,
930 						*size / 3, 0);
931 			if (r)
932 				goto out;
933 			break;
934 
935 		case 0x02000001: /* destroy */
936 			destroyed |= 1 << session_idx;
937 			break;
938 
939 		case 0x05000001: /* context buffer */
940 			r = amdgpu_vce_cs_reloc(p, ib, idx + 3, idx + 2,
941 						*size * 2, 0);
942 			if (r)
943 				goto out;
944 			break;
945 
946 		case 0x05000004: /* video bitstream buffer */
947 			tmp = amdgpu_ib_get_value(ib, idx + 4);
948 			r = amdgpu_vce_cs_reloc(p, ib, idx + 3, idx + 2,
949 						tmp, bs_idx);
950 			if (r)
951 				goto out;
952 			break;
953 
954 		case 0x05000005: /* feedback buffer */
955 			r = amdgpu_vce_cs_reloc(p, ib, idx + 3, idx + 2,
956 						4096, fb_idx);
957 			if (r)
958 				goto out;
959 			break;
960 
961 		case 0x0500000d: /* MV buffer */
962 			r = amdgpu_vce_cs_reloc(p, ib, idx + 3,
963 						idx + 2, *size, 0);
964 			if (r)
965 				goto out;
966 
967 			r = amdgpu_vce_cs_reloc(p, ib, idx + 8,
968 						idx + 7, *size / 12, 0);
969 			if (r)
970 				goto out;
971 			break;
972 
973 		default:
974 			DRM_ERROR("invalid VCE command (0x%x)!\n", cmd);
975 			r = -EINVAL;
976 			goto out;
977 		}
978 
979 		if (session_idx == -1) {
980 			DRM_ERROR("no session command at start of IB\n");
981 			r = -EINVAL;
982 			goto out;
983 		}
984 
985 		idx += len / 4;
986 	}
987 
988 	if (allocated & ~created) {
989 		DRM_ERROR("New session without create command!\n");
990 		r = -ENOENT;
991 	}
992 
993 out:
994 	if (!r) {
995 		/* No error, free all destroyed handle slots */
996 		tmp = destroyed;
997 	} else {
998 		/* Error during parsing, free all allocated handle slots */
999 		tmp = allocated;
1000 	}
1001 
1002 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
1003 		if (tmp & (1 << i))
1004 			atomic_set(&p->adev->vce.handles[i], 0);
1005 
1006 	return r;
1007 }
1008 
1009 /**
1010  * amdgpu_vce_ring_parse_cs_vm - parse the command stream in VM mode
1011  *
1012  * @p: parser context
1013  * @job: the job to parse
1014  * @ib: the IB to patch
1015  */
1016 int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p,
1017 				struct amdgpu_job *job,
1018 				struct amdgpu_ib *ib)
1019 {
1020 	int session_idx = -1;
1021 	uint32_t destroyed = 0;
1022 	uint32_t created = 0;
1023 	uint32_t allocated = 0;
1024 	uint32_t tmp, handle = 0;
1025 	int i, r = 0, idx = 0;
1026 
1027 	while (idx < ib->length_dw) {
1028 		uint32_t len = amdgpu_ib_get_value(ib, idx);
1029 		uint32_t cmd = amdgpu_ib_get_value(ib, idx + 1);
1030 
1031 		if ((len < 8) || (len & 3)) {
1032 			DRM_ERROR("invalid VCE command length (%d)!\n", len);
1033 			r = -EINVAL;
1034 			goto out;
1035 		}
1036 
1037 		switch (cmd) {
1038 		case 0x00000001: /* session */
1039 			handle = amdgpu_ib_get_value(ib, idx + 2);
1040 			session_idx = amdgpu_vce_validate_handle(p, handle,
1041 								 &allocated);
1042 			if (session_idx < 0) {
1043 				r = session_idx;
1044 				goto out;
1045 			}
1046 			break;
1047 
1048 		case 0x01000001: /* create */
1049 			created |= 1 << session_idx;
1050 			if (destroyed & (1 << session_idx)) {
1051 				destroyed &= ~(1 << session_idx);
1052 				allocated |= 1 << session_idx;
1053 
1054 			} else if (!(allocated & (1 << session_idx))) {
1055 				DRM_ERROR("Handle already in use!\n");
1056 				r = -EINVAL;
1057 				goto out;
1058 			}
1059 
1060 			break;
1061 
1062 		case 0x02000001: /* destroy */
1063 			destroyed |= 1 << session_idx;
1064 			break;
1065 
1066 		default:
1067 			break;
1068 		}
1069 
1070 		if (session_idx == -1) {
1071 			DRM_ERROR("no session command at start of IB\n");
1072 			r = -EINVAL;
1073 			goto out;
1074 		}
1075 
1076 		idx += len / 4;
1077 	}
1078 
1079 	if (allocated & ~created) {
1080 		DRM_ERROR("New session without create command!\n");
1081 		r = -ENOENT;
1082 	}
1083 
1084 out:
1085 	if (!r) {
1086 		/* No error, free all destroyed handle slots */
1087 		tmp = destroyed;
1088 	} else {
1089 		/* Error during parsing, free all allocated handle slots */
1090 		tmp = allocated;
1091 	}
1092 
1093 	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
1094 		if (tmp & (1 << i))
1095 			atomic_set(&p->adev->vce.handles[i], 0);
1096 
1097 	return r;
1098 }
1099 
1100 /**
1101  * amdgpu_vce_ring_emit_ib - execute indirect buffer
1102  *
1103  * @ring: engine to use
1104  * @job: job to retrieve vmid from
1105  * @ib: the IB to execute
1106  * @flags: unused
1107  *
1108  */
1109 void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring,
1110 				struct amdgpu_job *job,
1111 				struct amdgpu_ib *ib,
1112 				uint32_t flags)
1113 {
1114 	amdgpu_ring_write(ring, VCE_CMD_IB);
1115 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1116 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1117 	amdgpu_ring_write(ring, ib->length_dw);
1118 }
1119 
1120 /**
1121  * amdgpu_vce_ring_emit_fence - add a fence command to the ring
1122  *
1123  * @ring: engine to use
1124  * @addr: address
1125  * @seq: sequence number
1126  * @flags: fence related flags
1127  *
1128  */
1129 void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1130 				unsigned int flags)
1131 {
1132 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1133 
1134 	amdgpu_ring_write(ring, VCE_CMD_FENCE);
1135 	amdgpu_ring_write(ring, addr);
1136 	amdgpu_ring_write(ring, upper_32_bits(addr));
1137 	amdgpu_ring_write(ring, seq);
1138 	amdgpu_ring_write(ring, VCE_CMD_TRAP);
1139 	amdgpu_ring_write(ring, VCE_CMD_END);
1140 }
1141 
1142 /**
1143  * amdgpu_vce_ring_test_ring - test if VCE ring is working
1144  *
1145  * @ring: the engine to test on
1146  *
1147  */
1148 int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
1149 {
1150 	struct amdgpu_device *adev = ring->adev;
1151 	uint32_t rptr;
1152 	unsigned int i;
1153 	int r, timeout = adev->usec_timeout;
1154 
1155 	/* skip ring test for sriov*/
1156 	if (amdgpu_sriov_vf(adev))
1157 		return 0;
1158 
1159 	r = amdgpu_ring_alloc(ring, 16);
1160 	if (r)
1161 		return r;
1162 
1163 	rptr = amdgpu_ring_get_rptr(ring);
1164 
1165 	amdgpu_ring_write(ring, VCE_CMD_END);
1166 	amdgpu_ring_commit(ring);
1167 
1168 	for (i = 0; i < timeout; i++) {
1169 		if (amdgpu_ring_get_rptr(ring) != rptr)
1170 			break;
1171 		udelay(1);
1172 	}
1173 
1174 	if (i >= timeout)
1175 		r = -ETIMEDOUT;
1176 
1177 	return r;
1178 }
1179 
1180 /**
1181  * amdgpu_vce_ring_test_ib - test if VCE IBs are working
1182  *
1183  * @ring: the engine to test on
1184  * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1185  *
1186  */
1187 int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1188 {
1189 	struct dma_fence *fence = NULL;
1190 	long r;
1191 
1192 	/* skip vce ring1/2 ib test for now, since it's not reliable */
1193 	if (ring != &ring->adev->vce.ring[0])
1194 		return 0;
1195 
1196 	r = amdgpu_vce_get_create_msg(ring, 1, NULL);
1197 	if (r)
1198 		goto error;
1199 
1200 	r = amdgpu_vce_get_destroy_msg(ring, 1, true, &fence);
1201 	if (r)
1202 		goto error;
1203 
1204 	r = dma_fence_wait_timeout(fence, false, timeout);
1205 	if (r == 0)
1206 		r = -ETIMEDOUT;
1207 	else if (r > 0)
1208 		r = 0;
1209 
1210 error:
1211 	dma_fence_put(fence);
1212 	return r;
1213 }
1214 
1215 enum amdgpu_ring_priority_level amdgpu_vce_get_ring_prio(int ring)
1216 {
1217 	switch (ring) {
1218 	case 0:
1219 		return AMDGPU_RING_PRIO_0;
1220 	case 1:
1221 		return AMDGPU_RING_PRIO_1;
1222 	case 2:
1223 		return AMDGPU_RING_PRIO_2;
1224 	default:
1225 		return AMDGPU_RING_PRIO_0;
1226 	}
1227 }
1228