xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright 2023 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/debugfs.h>
28 #include <drm/drm_exec.h>
29 #include <drm/drm_drv.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_umsch_mm.h"
33 #include "umsch_mm_v4_0.h"
34 
35 struct umsch_mm_test_ctx_data {
36 	uint8_t process_csa[PAGE_SIZE];
37 	uint8_t vpe_ctx_csa[PAGE_SIZE];
38 	uint8_t vcn_ctx_csa[PAGE_SIZE];
39 };
40 
41 struct umsch_mm_test_mqd_data {
42 	uint8_t vpe_mqd[PAGE_SIZE];
43 	uint8_t vcn_mqd[PAGE_SIZE];
44 };
45 
46 struct umsch_mm_test_ring_data {
47 	uint8_t vpe_ring[PAGE_SIZE];
48 	uint8_t vpe_ib[PAGE_SIZE];
49 	uint8_t vcn_ring[PAGE_SIZE];
50 	uint8_t vcn_ib[PAGE_SIZE];
51 };
52 
53 struct umsch_mm_test_queue_info {
54 	uint64_t mqd_addr;
55 	uint64_t csa_addr;
56 	uint32_t doorbell_offset_0;
57 	uint32_t doorbell_offset_1;
58 	enum UMSCH_SWIP_ENGINE_TYPE engine;
59 };
60 
61 struct umsch_mm_test {
62 	struct amdgpu_bo	*ctx_data_obj;
63 	uint64_t		ctx_data_gpu_addr;
64 	uint32_t		*ctx_data_cpu_addr;
65 
66 	struct amdgpu_bo	*mqd_data_obj;
67 	uint64_t		mqd_data_gpu_addr;
68 	uint32_t		*mqd_data_cpu_addr;
69 
70 	struct amdgpu_bo	*ring_data_obj;
71 	uint64_t		ring_data_gpu_addr;
72 	uint32_t		*ring_data_cpu_addr;
73 
74 
75 	struct amdgpu_vm	*vm;
76 	struct amdgpu_bo_va	*bo_va;
77 	uint32_t		pasid;
78 	uint32_t		vm_cntx_cntl;
79 	uint32_t		num_queues;
80 };
81 
map_ring_data(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo * bo,struct amdgpu_bo_va ** bo_va,uint64_t addr,uint32_t size)82 static int map_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
83 			  struct amdgpu_bo *bo, struct amdgpu_bo_va **bo_va,
84 			  uint64_t addr, uint32_t size)
85 {
86 	struct amdgpu_sync sync;
87 	struct drm_exec exec;
88 	int r;
89 
90 	amdgpu_sync_create(&sync);
91 
92 	drm_exec_init(&exec, 0, 0);
93 	drm_exec_until_all_locked(&exec) {
94 		r = drm_exec_lock_obj(&exec, &bo->tbo.base);
95 		drm_exec_retry_on_contention(&exec);
96 		if (unlikely(r))
97 			goto error_fini_exec;
98 
99 		r = amdgpu_vm_lock_pd(vm, &exec, 0);
100 		drm_exec_retry_on_contention(&exec);
101 		if (unlikely(r))
102 			goto error_fini_exec;
103 	}
104 
105 	*bo_va = amdgpu_vm_bo_add(adev, vm, bo);
106 	if (!*bo_va) {
107 		r = -ENOMEM;
108 		goto error_fini_exec;
109 	}
110 
111 	r = amdgpu_vm_bo_map(adev, *bo_va, addr, 0, size,
112 			     AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
113 			     AMDGPU_PTE_EXECUTABLE);
114 
115 	if (r)
116 		goto error_del_bo_va;
117 
118 
119 	r = amdgpu_vm_bo_update(adev, *bo_va, false);
120 	if (r)
121 		goto error_del_bo_va;
122 
123 	amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update);
124 
125 	r = amdgpu_vm_update_pdes(adev, vm, false);
126 	if (r)
127 		goto error_del_bo_va;
128 
129 	amdgpu_sync_fence(&sync, vm->last_update);
130 
131 	amdgpu_sync_wait(&sync, false);
132 	drm_exec_fini(&exec);
133 
134 	amdgpu_sync_free(&sync);
135 
136 	return 0;
137 
138 error_del_bo_va:
139 	amdgpu_vm_bo_del(adev, *bo_va);
140 	amdgpu_sync_free(&sync);
141 
142 error_fini_exec:
143 	drm_exec_fini(&exec);
144 	amdgpu_sync_free(&sync);
145 	return r;
146 }
147 
unmap_ring_data(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo * bo,struct amdgpu_bo_va * bo_va,uint64_t addr)148 static int unmap_ring_data(struct amdgpu_device *adev, struct amdgpu_vm *vm,
149 			    struct amdgpu_bo *bo, struct amdgpu_bo_va *bo_va,
150 			    uint64_t addr)
151 {
152 	struct drm_exec exec;
153 	long r;
154 
155 	drm_exec_init(&exec, 0, 0);
156 	drm_exec_until_all_locked(&exec) {
157 		r = drm_exec_lock_obj(&exec, &bo->tbo.base);
158 		drm_exec_retry_on_contention(&exec);
159 		if (unlikely(r))
160 			goto out_unlock;
161 
162 		r = amdgpu_vm_lock_pd(vm, &exec, 0);
163 		drm_exec_retry_on_contention(&exec);
164 		if (unlikely(r))
165 			goto out_unlock;
166 	}
167 
168 
169 	r = amdgpu_vm_bo_unmap(adev, bo_va, addr);
170 	if (r)
171 		goto out_unlock;
172 
173 	amdgpu_vm_bo_del(adev, bo_va);
174 
175 out_unlock:
176 	drm_exec_fini(&exec);
177 
178 	return r;
179 }
180 
setup_vpe_queue(struct amdgpu_device * adev,struct umsch_mm_test * test,struct umsch_mm_test_queue_info * qinfo)181 static void setup_vpe_queue(struct amdgpu_device *adev,
182 			    struct umsch_mm_test *test,
183 			    struct umsch_mm_test_queue_info *qinfo)
184 {
185 	struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
186 	uint64_t ring_gpu_addr = test->ring_data_gpu_addr;
187 
188 	mqd->rb_base_lo = (ring_gpu_addr >> 8);
189 	mqd->rb_base_hi = (ring_gpu_addr >> 40);
190 	mqd->rb_size = PAGE_SIZE / 4;
191 	mqd->wptr_val = 0;
192 	mqd->rptr_val = 0;
193 	mqd->unmapped = 1;
194 
195 	if (adev->vpe.collaborate_mode)
196 		memcpy(++mqd, test->mqd_data_cpu_addr, sizeof(struct MQD_INFO));
197 
198 	qinfo->mqd_addr = test->mqd_data_gpu_addr;
199 	qinfo->csa_addr = test->ctx_data_gpu_addr +
200 		offsetof(struct umsch_mm_test_ctx_data, vpe_ctx_csa);
201 	qinfo->doorbell_offset_0 = 0;
202 	qinfo->doorbell_offset_1 = 0;
203 }
204 
setup_vcn_queue(struct amdgpu_device * adev,struct umsch_mm_test * test,struct umsch_mm_test_queue_info * qinfo)205 static void setup_vcn_queue(struct amdgpu_device *adev,
206 			    struct umsch_mm_test *test,
207 			    struct umsch_mm_test_queue_info *qinfo)
208 {
209 }
210 
add_test_queue(struct amdgpu_device * adev,struct umsch_mm_test * test,struct umsch_mm_test_queue_info * qinfo)211 static int add_test_queue(struct amdgpu_device *adev,
212 			  struct umsch_mm_test *test,
213 			  struct umsch_mm_test_queue_info *qinfo)
214 {
215 	struct umsch_mm_add_queue_input queue_input = {};
216 	int r;
217 
218 	queue_input.process_id = test->pasid;
219 	queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(test->vm->root.bo);
220 
221 	queue_input.process_va_start = 0;
222 	queue_input.process_va_end = (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT;
223 
224 	queue_input.process_quantum = 100000; /* 10ms */
225 	queue_input.process_csa_addr = test->ctx_data_gpu_addr +
226 				       offsetof(struct umsch_mm_test_ctx_data, process_csa);
227 
228 	queue_input.context_quantum = 10000; /* 1ms */
229 	queue_input.context_csa_addr = qinfo->csa_addr;
230 
231 	queue_input.inprocess_context_priority = CONTEXT_PRIORITY_LEVEL_NORMAL;
232 	queue_input.context_global_priority_level = CONTEXT_PRIORITY_LEVEL_NORMAL;
233 	queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
234 	queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
235 
236 	queue_input.engine_type = qinfo->engine;
237 	queue_input.mqd_addr = qinfo->mqd_addr;
238 	queue_input.vm_context_cntl = test->vm_cntx_cntl;
239 
240 	amdgpu_umsch_mm_lock(&adev->umsch_mm);
241 	r = adev->umsch_mm.funcs->add_queue(&adev->umsch_mm, &queue_input);
242 	amdgpu_umsch_mm_unlock(&adev->umsch_mm);
243 	if (r)
244 		return r;
245 
246 	return 0;
247 }
248 
remove_test_queue(struct amdgpu_device * adev,struct umsch_mm_test * test,struct umsch_mm_test_queue_info * qinfo)249 static int remove_test_queue(struct amdgpu_device *adev,
250 			     struct umsch_mm_test *test,
251 			     struct umsch_mm_test_queue_info *qinfo)
252 {
253 	struct umsch_mm_remove_queue_input queue_input = {};
254 	int r;
255 
256 	queue_input.doorbell_offset_0 = qinfo->doorbell_offset_0;
257 	queue_input.doorbell_offset_1 = qinfo->doorbell_offset_1;
258 	queue_input.context_csa_addr = qinfo->csa_addr;
259 
260 	amdgpu_umsch_mm_lock(&adev->umsch_mm);
261 	r = adev->umsch_mm.funcs->remove_queue(&adev->umsch_mm, &queue_input);
262 	amdgpu_umsch_mm_unlock(&adev->umsch_mm);
263 	if (r)
264 		return r;
265 
266 	return 0;
267 }
268 
submit_vpe_queue(struct amdgpu_device * adev,struct umsch_mm_test * test)269 static int submit_vpe_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
270 {
271 	struct MQD_INFO *mqd = (struct MQD_INFO *)test->mqd_data_cpu_addr;
272 	uint32_t *ring = test->ring_data_cpu_addr +
273 		offsetof(struct umsch_mm_test_ring_data, vpe_ring) / 4;
274 	uint32_t *ib = test->ring_data_cpu_addr +
275 		offsetof(struct umsch_mm_test_ring_data, vpe_ib) / 4;
276 	uint64_t ib_gpu_addr = test->ring_data_gpu_addr +
277 		offsetof(struct umsch_mm_test_ring_data, vpe_ib);
278 	uint32_t *fence = ib + 2048 / 4;
279 	uint64_t fence_gpu_addr = ib_gpu_addr + 2048;
280 	const uint32_t test_pattern = 0xdeadbeef;
281 	int i;
282 
283 	ib[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0);
284 	ib[1] = lower_32_bits(fence_gpu_addr);
285 	ib[2] = upper_32_bits(fence_gpu_addr);
286 	ib[3] = test_pattern;
287 
288 	ring[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_INDIRECT, 0);
289 	ring[1] = (ib_gpu_addr & 0xffffffe0);
290 	ring[2] = upper_32_bits(ib_gpu_addr);
291 	ring[3] = 4;
292 	ring[4] = 0;
293 	ring[5] = 0;
294 
295 	mqd->wptr_val = (6 << 2);
296 	if (adev->vpe.collaborate_mode)
297 		(++mqd)->wptr_val = (6 << 2);
298 
299 	WDOORBELL32(adev->umsch_mm.agdb_index[CONTEXT_PRIORITY_LEVEL_NORMAL], mqd->wptr_val);
300 
301 	for (i = 0; i < adev->usec_timeout; i++) {
302 		if (*fence == test_pattern)
303 			return 0;
304 		udelay(1);
305 	}
306 
307 	dev_err(adev->dev, "vpe queue submission timeout\n");
308 
309 	return -ETIMEDOUT;
310 }
311 
submit_vcn_queue(struct amdgpu_device * adev,struct umsch_mm_test * test)312 static int submit_vcn_queue(struct amdgpu_device *adev, struct umsch_mm_test *test)
313 {
314 	return 0;
315 }
316 
setup_umsch_mm_test(struct amdgpu_device * adev,struct umsch_mm_test * test)317 static int setup_umsch_mm_test(struct amdgpu_device *adev,
318 			  struct umsch_mm_test *test)
319 {
320 	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
321 	int r;
322 
323 	test->vm_cntx_cntl = hub->vm_cntx_cntl;
324 
325 	test->vm = kzalloc(sizeof(*test->vm), GFP_KERNEL);
326 	if (!test->vm) {
327 		r = -ENOMEM;
328 		return r;
329 	}
330 
331 	r = amdgpu_vm_init(adev, test->vm, -1);
332 	if (r)
333 		goto error_free_vm;
334 
335 	r = amdgpu_pasid_alloc(16);
336 	if (r < 0)
337 		goto error_fini_vm;
338 	test->pasid = r;
339 
340 	r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ctx_data),
341 				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
342 				    &test->ctx_data_obj,
343 				    &test->ctx_data_gpu_addr,
344 				    (void **)&test->ctx_data_cpu_addr);
345 	if (r)
346 		goto error_free_pasid;
347 
348 	memset(test->ctx_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ctx_data));
349 
350 	r = amdgpu_bo_create_kernel(adev, PAGE_SIZE,
351 				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
352 				    &test->mqd_data_obj,
353 				    &test->mqd_data_gpu_addr,
354 				    (void **)&test->mqd_data_cpu_addr);
355 	if (r)
356 		goto error_free_ctx_data_obj;
357 
358 	memset(test->mqd_data_cpu_addr, 0, PAGE_SIZE);
359 
360 	r = amdgpu_bo_create_kernel(adev, sizeof(struct umsch_mm_test_ring_data),
361 				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
362 				    &test->ring_data_obj,
363 				    NULL,
364 				    (void **)&test->ring_data_cpu_addr);
365 	if (r)
366 		goto error_free_mqd_data_obj;
367 
368 	memset(test->ring_data_cpu_addr, 0, sizeof(struct umsch_mm_test_ring_data));
369 
370 	test->ring_data_gpu_addr = AMDGPU_VA_RESERVED_BOTTOM;
371 	r = map_ring_data(adev, test->vm, test->ring_data_obj, &test->bo_va,
372 			  test->ring_data_gpu_addr, sizeof(struct umsch_mm_test_ring_data));
373 	if (r)
374 		goto error_free_ring_data_obj;
375 
376 	return 0;
377 
378 error_free_ring_data_obj:
379 	amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
380 			      (void **)&test->ring_data_cpu_addr);
381 error_free_mqd_data_obj:
382 	amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
383 			      (void **)&test->mqd_data_cpu_addr);
384 error_free_ctx_data_obj:
385 	amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
386 			      (void **)&test->ctx_data_cpu_addr);
387 error_free_pasid:
388 	amdgpu_pasid_free(test->pasid);
389 error_fini_vm:
390 	amdgpu_vm_fini(adev, test->vm);
391 error_free_vm:
392 	kfree(test->vm);
393 
394 	return r;
395 }
396 
cleanup_umsch_mm_test(struct amdgpu_device * adev,struct umsch_mm_test * test)397 static void cleanup_umsch_mm_test(struct amdgpu_device *adev,
398 				  struct umsch_mm_test *test)
399 {
400 	unmap_ring_data(adev, test->vm, test->ring_data_obj,
401 			test->bo_va, test->ring_data_gpu_addr);
402 	amdgpu_bo_free_kernel(&test->mqd_data_obj, &test->mqd_data_gpu_addr,
403 			      (void **)&test->mqd_data_cpu_addr);
404 	amdgpu_bo_free_kernel(&test->ring_data_obj, NULL,
405 			      (void **)&test->ring_data_cpu_addr);
406 	amdgpu_bo_free_kernel(&test->ctx_data_obj, &test->ctx_data_gpu_addr,
407 			       (void **)&test->ctx_data_cpu_addr);
408 	amdgpu_pasid_free(test->pasid);
409 	amdgpu_vm_fini(adev, test->vm);
410 	kfree(test->vm);
411 }
412 
setup_test_queues(struct amdgpu_device * adev,struct umsch_mm_test * test,struct umsch_mm_test_queue_info * qinfo)413 static int setup_test_queues(struct amdgpu_device *adev,
414 			     struct umsch_mm_test *test,
415 			     struct umsch_mm_test_queue_info *qinfo)
416 {
417 	int i, r;
418 
419 	for (i = 0; i < test->num_queues; i++) {
420 		if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
421 			setup_vpe_queue(adev, test, &qinfo[i]);
422 		else
423 			setup_vcn_queue(adev, test, &qinfo[i]);
424 
425 		r = add_test_queue(adev, test, &qinfo[i]);
426 		if (r)
427 			return r;
428 	}
429 
430 	return 0;
431 }
432 
submit_test_queues(struct amdgpu_device * adev,struct umsch_mm_test * test,struct umsch_mm_test_queue_info * qinfo)433 static int submit_test_queues(struct amdgpu_device *adev,
434 			      struct umsch_mm_test *test,
435 			      struct umsch_mm_test_queue_info *qinfo)
436 {
437 	int i, r;
438 
439 	for (i = 0; i < test->num_queues; i++) {
440 		if (qinfo[i].engine == UMSCH_SWIP_ENGINE_TYPE_VPE)
441 			r = submit_vpe_queue(adev, test);
442 		else
443 			r = submit_vcn_queue(adev, test);
444 		if (r)
445 			return r;
446 	}
447 
448 	return 0;
449 }
450 
cleanup_test_queues(struct amdgpu_device * adev,struct umsch_mm_test * test,struct umsch_mm_test_queue_info * qinfo)451 static void cleanup_test_queues(struct amdgpu_device *adev,
452 			      struct umsch_mm_test *test,
453 			      struct umsch_mm_test_queue_info *qinfo)
454 {
455 	int i;
456 
457 	for (i = 0; i < test->num_queues; i++)
458 		remove_test_queue(adev, test, &qinfo[i]);
459 }
460 
umsch_mm_test(struct amdgpu_device * adev)461 static int umsch_mm_test(struct amdgpu_device *adev)
462 {
463 	struct umsch_mm_test_queue_info qinfo[] = {
464 		{ .engine = UMSCH_SWIP_ENGINE_TYPE_VPE },
465 	};
466 	struct umsch_mm_test test = { .num_queues = ARRAY_SIZE(qinfo) };
467 	int r;
468 
469 	r = setup_umsch_mm_test(adev, &test);
470 	if (r)
471 		return r;
472 
473 	r = setup_test_queues(adev, &test, qinfo);
474 	if (r)
475 		goto cleanup;
476 
477 	r = submit_test_queues(adev, &test, qinfo);
478 	if (r)
479 		goto cleanup;
480 
481 	cleanup_test_queues(adev, &test, qinfo);
482 	cleanup_umsch_mm_test(adev, &test);
483 
484 	return 0;
485 
486 cleanup:
487 	cleanup_test_queues(adev, &test, qinfo);
488 	cleanup_umsch_mm_test(adev, &test);
489 	return r;
490 }
491 
amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm * umsch,void * pkt,int ndws)492 int amdgpu_umsch_mm_submit_pkt(struct amdgpu_umsch_mm *umsch, void *pkt, int ndws)
493 {
494 	struct amdgpu_ring *ring = &umsch->ring;
495 
496 	if (amdgpu_ring_alloc(ring, ndws))
497 		return -ENOMEM;
498 
499 	amdgpu_ring_write_multiple(ring, pkt, ndws);
500 	amdgpu_ring_commit(ring);
501 
502 	return 0;
503 }
504 
amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm * umsch)505 int amdgpu_umsch_mm_query_fence(struct amdgpu_umsch_mm *umsch)
506 {
507 	struct amdgpu_ring *ring = &umsch->ring;
508 	struct amdgpu_device *adev = ring->adev;
509 	int r;
510 
511 	r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, adev->usec_timeout);
512 	if (r < 1) {
513 		dev_err(adev->dev, "ring umsch timeout, emitted fence %u\n",
514 			ring->fence_drv.sync_seq);
515 		return -ETIMEDOUT;
516 	}
517 
518 	return 0;
519 }
520 
umsch_mm_ring_set_wptr(struct amdgpu_ring * ring)521 static void umsch_mm_ring_set_wptr(struct amdgpu_ring *ring)
522 {
523 	struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
524 	struct amdgpu_device *adev = ring->adev;
525 
526 	if (ring->use_doorbell)
527 		WDOORBELL32(ring->doorbell_index, ring->wptr << 2);
528 	else
529 		WREG32(umsch->rb_wptr, ring->wptr << 2);
530 }
531 
umsch_mm_ring_get_rptr(struct amdgpu_ring * ring)532 static u64 umsch_mm_ring_get_rptr(struct amdgpu_ring *ring)
533 {
534 	struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
535 	struct amdgpu_device *adev = ring->adev;
536 
537 	return RREG32(umsch->rb_rptr);
538 }
539 
umsch_mm_ring_get_wptr(struct amdgpu_ring * ring)540 static u64 umsch_mm_ring_get_wptr(struct amdgpu_ring *ring)
541 {
542 	struct amdgpu_umsch_mm *umsch = (struct amdgpu_umsch_mm *)ring;
543 	struct amdgpu_device *adev = ring->adev;
544 
545 	return RREG32(umsch->rb_wptr);
546 }
547 
548 static const struct amdgpu_ring_funcs umsch_v4_0_ring_funcs = {
549 	.type = AMDGPU_RING_TYPE_UMSCH_MM,
550 	.align_mask = 0,
551 	.nop = 0,
552 	.support_64bit_ptrs = false,
553 	.get_rptr = umsch_mm_ring_get_rptr,
554 	.get_wptr = umsch_mm_ring_get_wptr,
555 	.set_wptr = umsch_mm_ring_set_wptr,
556 	.insert_nop = amdgpu_ring_insert_nop,
557 };
558 
amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm * umsch)559 int amdgpu_umsch_mm_ring_init(struct amdgpu_umsch_mm *umsch)
560 {
561 	struct amdgpu_device *adev = container_of(umsch, struct amdgpu_device, umsch_mm);
562 	struct amdgpu_ring *ring = &umsch->ring;
563 
564 	ring->vm_hub = AMDGPU_MMHUB0(0);
565 	ring->use_doorbell = true;
566 	ring->no_scheduler = true;
567 	ring->doorbell_index = (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1) + 6;
568 
569 	snprintf(ring->name, sizeof(ring->name), "umsch");
570 
571 	return amdgpu_ring_init(adev, ring, 1024, NULL, 0, AMDGPU_RING_PRIO_DEFAULT, NULL);
572 }
573 
amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm * umsch)574 int amdgpu_umsch_mm_init_microcode(struct amdgpu_umsch_mm *umsch)
575 {
576 	const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
577 	struct amdgpu_device *adev = umsch->ring.adev;
578 	const char *fw_name = NULL;
579 	int r;
580 
581 	switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
582 	case IP_VERSION(4, 0, 5):
583 	case IP_VERSION(4, 0, 6):
584 		fw_name = "amdgpu/umsch_mm_4_0_0.bin";
585 		break;
586 	default:
587 		break;
588 	}
589 
590 	r = amdgpu_ucode_request(adev, &adev->umsch_mm.fw, AMDGPU_UCODE_REQUIRED,
591 				 "%s", fw_name);
592 	if (r) {
593 		release_firmware(adev->umsch_mm.fw);
594 		adev->umsch_mm.fw = NULL;
595 		return r;
596 	}
597 
598 	umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)adev->umsch_mm.fw->data;
599 
600 	adev->umsch_mm.ucode_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
601 	adev->umsch_mm.data_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
602 
603 	adev->umsch_mm.irq_start_addr =
604 		le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_lo) |
605 		((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_irq_start_addr_hi)) << 32);
606 	adev->umsch_mm.uc_start_addr =
607 		le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_lo) |
608 		((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_uc_start_addr_hi)) << 32);
609 	adev->umsch_mm.data_start_addr =
610 		le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_lo) |
611 		((uint64_t)(le32_to_cpu(umsch_mm_hdr->umsch_mm_data_start_addr_hi)) << 32);
612 
613 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
614 		struct amdgpu_firmware_info *info;
615 
616 		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_UCODE];
617 		info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_UCODE;
618 		info->fw = adev->umsch_mm.fw;
619 		adev->firmware.fw_size +=
620 			ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes), PAGE_SIZE);
621 
622 		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_UMSCH_MM_DATA];
623 		info->ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_DATA;
624 		info->fw = adev->umsch_mm.fw;
625 		adev->firmware.fw_size +=
626 			ALIGN(le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes), PAGE_SIZE);
627 	}
628 
629 	return 0;
630 }
631 
amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm * umsch)632 int amdgpu_umsch_mm_allocate_ucode_buffer(struct amdgpu_umsch_mm *umsch)
633 {
634 	const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
635 	struct amdgpu_device *adev = umsch->ring.adev;
636 	const __le32 *fw_data;
637 	uint32_t fw_size;
638 	int r;
639 
640 	umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
641 		       adev->umsch_mm.fw->data;
642 
643 	fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
644 		  le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_offset_bytes));
645 	fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_size_bytes);
646 
647 	r = amdgpu_bo_create_reserved(adev, fw_size,
648 				      4 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
649 				      &adev->umsch_mm.ucode_fw_obj,
650 				      &adev->umsch_mm.ucode_fw_gpu_addr,
651 				      (void **)&adev->umsch_mm.ucode_fw_ptr);
652 	if (r) {
653 		dev_err(adev->dev, "(%d) failed to create umsch_mm fw ucode bo\n", r);
654 		return r;
655 	}
656 
657 	memcpy(adev->umsch_mm.ucode_fw_ptr, fw_data, fw_size);
658 
659 	amdgpu_bo_kunmap(adev->umsch_mm.ucode_fw_obj);
660 	amdgpu_bo_unreserve(adev->umsch_mm.ucode_fw_obj);
661 	return 0;
662 }
663 
amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm * umsch)664 int amdgpu_umsch_mm_allocate_ucode_data_buffer(struct amdgpu_umsch_mm *umsch)
665 {
666 	const struct umsch_mm_firmware_header_v1_0 *umsch_mm_hdr;
667 	struct amdgpu_device *adev = umsch->ring.adev;
668 	const __le32 *fw_data;
669 	uint32_t fw_size;
670 	int r;
671 
672 	umsch_mm_hdr = (const struct umsch_mm_firmware_header_v1_0 *)
673 		       adev->umsch_mm.fw->data;
674 
675 	fw_data = (const __le32 *)(adev->umsch_mm.fw->data +
676 		  le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_offset_bytes));
677 	fw_size = le32_to_cpu(umsch_mm_hdr->umsch_mm_ucode_data_size_bytes);
678 
679 	r = amdgpu_bo_create_reserved(adev, fw_size,
680 				      64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
681 				      &adev->umsch_mm.data_fw_obj,
682 				      &adev->umsch_mm.data_fw_gpu_addr,
683 				      (void **)&adev->umsch_mm.data_fw_ptr);
684 	if (r) {
685 		dev_err(adev->dev, "(%d) failed to create umsch_mm fw data bo\n", r);
686 		return r;
687 	}
688 
689 	memcpy(adev->umsch_mm.data_fw_ptr, fw_data, fw_size);
690 
691 	amdgpu_bo_kunmap(adev->umsch_mm.data_fw_obj);
692 	amdgpu_bo_unreserve(adev->umsch_mm.data_fw_obj);
693 	return 0;
694 }
695 
amdgpu_umsch_mm_psp_execute_cmd_buf(struct amdgpu_umsch_mm * umsch)696 int amdgpu_umsch_mm_psp_execute_cmd_buf(struct amdgpu_umsch_mm *umsch)
697 {
698 	struct amdgpu_device *adev = umsch->ring.adev;
699 	struct amdgpu_firmware_info ucode = {
700 		.ucode_id = AMDGPU_UCODE_ID_UMSCH_MM_CMD_BUFFER,
701 		.mc_addr = adev->umsch_mm.cmd_buf_gpu_addr,
702 		.ucode_size = ((uintptr_t)adev->umsch_mm.cmd_buf_curr_ptr -
703 			      (uintptr_t)adev->umsch_mm.cmd_buf_ptr),
704 	};
705 
706 	return psp_execute_ip_fw_load(&adev->psp, &ucode);
707 }
708 
umsch_mm_agdb_index_init(struct amdgpu_device * adev)709 static void umsch_mm_agdb_index_init(struct amdgpu_device *adev)
710 {
711 	uint32_t umsch_mm_agdb_start;
712 	int i;
713 
714 	umsch_mm_agdb_start = adev->doorbell_index.max_assignment + 1;
715 	umsch_mm_agdb_start = roundup(umsch_mm_agdb_start, 1024);
716 	umsch_mm_agdb_start += (AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1);
717 
718 	for (i = 0; i < CONTEXT_PRIORITY_NUM_LEVELS; i++)
719 		adev->umsch_mm.agdb_index[i] = umsch_mm_agdb_start + i;
720 }
721 
umsch_mm_init(struct amdgpu_device * adev)722 static int umsch_mm_init(struct amdgpu_device *adev)
723 {
724 	int r;
725 
726 	adev->umsch_mm.vmid_mask_mm_vpe = 0xf00;
727 	adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE);
728 	adev->umsch_mm.vpe_hqd_mask = 0xfe;
729 
730 	r = amdgpu_device_wb_get(adev, &adev->umsch_mm.wb_index);
731 	if (r) {
732 		dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r);
733 		return r;
734 	}
735 
736 	adev->umsch_mm.sch_ctx_gpu_addr = adev->wb.gpu_addr +
737 					  (adev->umsch_mm.wb_index * 4);
738 
739 	r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
740 				    AMDGPU_GEM_DOMAIN_GTT,
741 				    &adev->umsch_mm.cmd_buf_obj,
742 				    &adev->umsch_mm.cmd_buf_gpu_addr,
743 				    (void **)&adev->umsch_mm.cmd_buf_ptr);
744 	if (r) {
745 		dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r);
746 		amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
747 		return r;
748 	}
749 
750 	r = amdgpu_bo_create_kernel(adev, AMDGPU_UMSCHFW_LOG_SIZE, PAGE_SIZE,
751 				    AMDGPU_GEM_DOMAIN_VRAM |
752 				    AMDGPU_GEM_DOMAIN_GTT,
753 				    &adev->umsch_mm.dbglog_bo,
754 				    &adev->umsch_mm.log_gpu_addr,
755 				    &adev->umsch_mm.log_cpu_addr);
756 	if (r) {
757 		dev_err(adev->dev, "(%d) failed to allocate umsch debug bo\n", r);
758 		return r;
759 	}
760 
761 	mutex_init(&adev->umsch_mm.mutex_hidden);
762 
763 	umsch_mm_agdb_index_init(adev);
764 
765 	return 0;
766 }
767 
768 
umsch_mm_early_init(struct amdgpu_ip_block * ip_block)769 static int umsch_mm_early_init(struct amdgpu_ip_block *ip_block)
770 {
771 	struct amdgpu_device *adev = ip_block->adev;
772 
773 	switch (amdgpu_ip_version(adev, VCN_HWIP, 0)) {
774 	case IP_VERSION(4, 0, 5):
775 	case IP_VERSION(4, 0, 6):
776 		umsch_mm_v4_0_set_funcs(&adev->umsch_mm);
777 		break;
778 	default:
779 		return -EINVAL;
780 	}
781 
782 	adev->umsch_mm.ring.funcs = &umsch_v4_0_ring_funcs;
783 	umsch_mm_set_regs(&adev->umsch_mm);
784 
785 	return 0;
786 }
787 
umsch_mm_late_init(struct amdgpu_ip_block * ip_block)788 static int umsch_mm_late_init(struct amdgpu_ip_block *ip_block)
789 {
790 	struct amdgpu_device *adev = ip_block->adev;
791 
792 	if (amdgpu_in_reset(adev) || adev->in_s0ix || adev->in_suspend)
793 		return 0;
794 
795 	return umsch_mm_test(adev);
796 }
797 
umsch_mm_sw_init(struct amdgpu_ip_block * ip_block)798 static int umsch_mm_sw_init(struct amdgpu_ip_block *ip_block)
799 {
800 	struct amdgpu_device *adev = ip_block->adev;
801 	int r;
802 
803 	r = umsch_mm_init(adev);
804 	if (r)
805 		return r;
806 
807 	amdgpu_umsch_fwlog_init(&adev->umsch_mm);
808 	r = umsch_mm_ring_init(&adev->umsch_mm);
809 	if (r)
810 		return r;
811 
812 	r = umsch_mm_init_microcode(&adev->umsch_mm);
813 	if (r)
814 		return r;
815 
816 	return 0;
817 }
818 
umsch_mm_sw_fini(struct amdgpu_ip_block * ip_block)819 static int umsch_mm_sw_fini(struct amdgpu_ip_block *ip_block)
820 {
821 	struct amdgpu_device *adev = ip_block->adev;
822 
823 	release_firmware(adev->umsch_mm.fw);
824 	adev->umsch_mm.fw = NULL;
825 
826 	amdgpu_ring_fini(&adev->umsch_mm.ring);
827 
828 	mutex_destroy(&adev->umsch_mm.mutex_hidden);
829 
830 	amdgpu_bo_free_kernel(&adev->umsch_mm.cmd_buf_obj,
831 			      &adev->umsch_mm.cmd_buf_gpu_addr,
832 			      (void **)&adev->umsch_mm.cmd_buf_ptr);
833 
834 	amdgpu_bo_free_kernel(&adev->umsch_mm.dbglog_bo,
835 				    &adev->umsch_mm.log_gpu_addr,
836 				    (void **)&adev->umsch_mm.log_cpu_addr);
837 
838 	amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
839 
840 	return 0;
841 }
842 
umsch_mm_hw_init(struct amdgpu_ip_block * ip_block)843 static int umsch_mm_hw_init(struct amdgpu_ip_block *ip_block)
844 {
845 	struct amdgpu_device *adev = ip_block->adev;
846 	int r;
847 
848 	r = umsch_mm_load_microcode(&adev->umsch_mm);
849 	if (r)
850 		return r;
851 
852 	umsch_mm_ring_start(&adev->umsch_mm);
853 
854 	r = umsch_mm_set_hw_resources(&adev->umsch_mm);
855 	if (r)
856 		return r;
857 
858 	return 0;
859 }
860 
umsch_mm_hw_fini(struct amdgpu_ip_block * ip_block)861 static int umsch_mm_hw_fini(struct amdgpu_ip_block *ip_block)
862 {
863 	struct amdgpu_device *adev = ip_block->adev;
864 
865 	umsch_mm_ring_stop(&adev->umsch_mm);
866 
867 	amdgpu_bo_free_kernel(&adev->umsch_mm.data_fw_obj,
868 			      &adev->umsch_mm.data_fw_gpu_addr,
869 			      (void **)&adev->umsch_mm.data_fw_ptr);
870 
871 	amdgpu_bo_free_kernel(&adev->umsch_mm.ucode_fw_obj,
872 			      &adev->umsch_mm.ucode_fw_gpu_addr,
873 			      (void **)&adev->umsch_mm.ucode_fw_ptr);
874 	return 0;
875 }
876 
umsch_mm_suspend(struct amdgpu_ip_block * ip_block)877 static int umsch_mm_suspend(struct amdgpu_ip_block *ip_block)
878 {
879 	return umsch_mm_hw_fini(ip_block);
880 }
881 
umsch_mm_resume(struct amdgpu_ip_block * ip_block)882 static int umsch_mm_resume(struct amdgpu_ip_block *ip_block)
883 {
884 	return umsch_mm_hw_init(ip_block);
885 }
886 
amdgpu_umsch_fwlog_init(struct amdgpu_umsch_mm * umsch_mm)887 void amdgpu_umsch_fwlog_init(struct amdgpu_umsch_mm *umsch_mm)
888 {
889 #if defined(CONFIG_DEBUG_FS)
890 	void *fw_log_cpu_addr = umsch_mm->log_cpu_addr;
891 	volatile struct amdgpu_umsch_fwlog *log_buf = fw_log_cpu_addr;
892 
893 	log_buf->header_size = sizeof(struct amdgpu_umsch_fwlog);
894 	log_buf->buffer_size = AMDGPU_UMSCHFW_LOG_SIZE;
895 	log_buf->rptr = log_buf->header_size;
896 	log_buf->wptr = log_buf->header_size;
897 	log_buf->wrapped = 0;
898 #endif
899 }
900 
901 /*
902  * debugfs for mapping umsch firmware log buffer.
903  */
904 #if defined(CONFIG_DEBUG_FS)
amdgpu_debugfs_umsch_fwlog_read(struct file * f,char __user * buf,size_t size,loff_t * pos)905 static ssize_t amdgpu_debugfs_umsch_fwlog_read(struct file *f, char __user *buf,
906 					     size_t size, loff_t *pos)
907 {
908 	struct amdgpu_umsch_mm *umsch_mm;
909 	void *log_buf;
910 	volatile struct amdgpu_umsch_fwlog *plog;
911 	unsigned int read_pos, write_pos, available, i, read_bytes = 0;
912 	unsigned int read_num[2] = {0};
913 
914 	umsch_mm = file_inode(f)->i_private;
915 	if (!umsch_mm)
916 		return -ENODEV;
917 
918 	if (!umsch_mm->log_cpu_addr)
919 		return -EFAULT;
920 
921 	log_buf = umsch_mm->log_cpu_addr;
922 
923 	plog = (volatile struct amdgpu_umsch_fwlog *)log_buf;
924 	read_pos = plog->rptr;
925 	write_pos = plog->wptr;
926 
927 	if (read_pos > AMDGPU_UMSCHFW_LOG_SIZE || write_pos > AMDGPU_UMSCHFW_LOG_SIZE)
928 		return -EFAULT;
929 
930 	if (!size || (read_pos == write_pos))
931 		return 0;
932 
933 	if (write_pos > read_pos) {
934 		available = write_pos - read_pos;
935 		read_num[0] = min_t(size_t, size, available);
936 	} else {
937 		read_num[0] = AMDGPU_UMSCHFW_LOG_SIZE - read_pos;
938 		available = read_num[0] + write_pos - plog->header_size;
939 		if (size > available)
940 			read_num[1] = write_pos - plog->header_size;
941 		else if (size > read_num[0])
942 			read_num[1] = size - read_num[0];
943 		else
944 			read_num[0] = size;
945 	}
946 
947 	for (i = 0; i < 2; i++) {
948 		if (read_num[i]) {
949 			if (read_pos == AMDGPU_UMSCHFW_LOG_SIZE)
950 				read_pos = plog->header_size;
951 			if (read_num[i] == copy_to_user((buf + read_bytes),
952 							(log_buf + read_pos), read_num[i]))
953 				return -EFAULT;
954 
955 			read_bytes += read_num[i];
956 			read_pos += read_num[i];
957 		}
958 	}
959 
960 	plog->rptr = read_pos;
961 	*pos += read_bytes;
962 	return read_bytes;
963 }
964 
965 static const struct file_operations amdgpu_debugfs_umschfwlog_fops = {
966 	.owner = THIS_MODULE,
967 	.read = amdgpu_debugfs_umsch_fwlog_read,
968 	.llseek = default_llseek
969 };
970 #endif
971 
amdgpu_debugfs_umsch_fwlog_init(struct amdgpu_device * adev,struct amdgpu_umsch_mm * umsch_mm)972 void amdgpu_debugfs_umsch_fwlog_init(struct amdgpu_device *adev,
973 			struct amdgpu_umsch_mm *umsch_mm)
974 {
975 #if defined(CONFIG_DEBUG_FS)
976 	struct drm_minor *minor = adev_to_drm(adev)->primary;
977 	struct dentry *root = minor->debugfs_root;
978 	char name[32];
979 
980 	sprintf(name, "amdgpu_umsch_fwlog");
981 	debugfs_create_file_size(name, S_IFREG | 0444, root, umsch_mm,
982 				 &amdgpu_debugfs_umschfwlog_fops,
983 				 AMDGPU_UMSCHFW_LOG_SIZE);
984 #endif
985 }
986 
987 static const struct amd_ip_funcs umsch_mm_v4_0_ip_funcs = {
988 	.name = "umsch_mm_v4_0",
989 	.early_init = umsch_mm_early_init,
990 	.late_init = umsch_mm_late_init,
991 	.sw_init = umsch_mm_sw_init,
992 	.sw_fini = umsch_mm_sw_fini,
993 	.hw_init = umsch_mm_hw_init,
994 	.hw_fini = umsch_mm_hw_fini,
995 	.suspend = umsch_mm_suspend,
996 	.resume = umsch_mm_resume,
997 };
998 
999 const struct amdgpu_ip_block_version umsch_mm_v4_0_ip_block = {
1000 	.type = AMD_IP_BLOCK_TYPE_UMSCH_MM,
1001 	.major = 4,
1002 	.minor = 0,
1003 	.rev = 0,
1004 	.funcs = &umsch_mm_v4_0_ip_funcs,
1005 };
1006