xref: /linux/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright 2024 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 #include <drm/drm_drv.h>
25 #include "amdgpu.h"
26 #include "amdgpu_gfx.h"
27 #include "mes_userqueue.h"
28 #include "amdgpu_userq_fence.h"
29 #include "amdgpu_trace.h"
30 
31 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
32 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
33 
34 static int
35 mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
36 			      struct amdgpu_userq_mgr *uq_mgr,
37 			      struct amdgpu_usermode_queue *queue,
38 			      uint64_t wptr)
39 {
40 	struct amdgpu_bo_va_mapping *wptr_mapping;
41 	struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;
42 	struct amdgpu_bo *obj;
43 	struct amdgpu_vm *vm = queue->vm;
44 	struct drm_exec exec;
45 	int ret;
46 
47 	wptr &= AMDGPU_GMC_HOLE_MASK;
48 
49 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
50 	drm_exec_until_all_locked(&exec) {
51 		ret = amdgpu_vm_lock_pd(vm, &exec, 1);
52 		drm_exec_retry_on_contention(&exec);
53 		if (unlikely(ret))
54 			goto fail_lock;
55 
56 		wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr);
57 		if (!wptr_mapping || !wptr_mapping->bo_va->base.bo) {
58 			ret = -EINVAL;
59 			goto fail_lock;
60 		}
61 
62 		obj = wptr_mapping->bo_va->base.bo;
63 		ret = drm_exec_prepare_obj(&exec, &obj->tbo.base,
64 					   TTM_NUM_MOVE_FENCES + 1);
65 		drm_exec_retry_on_contention(&exec);
66 		if (unlikely(ret))
67 			goto fail_lock;
68 	}
69 
70 	wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo);
71 	if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) {
72 		ret = -EINVAL;
73 		goto fail_map;
74 	}
75 	/* Keep WPTR BO under eviction-fence control instead of pinning. */
76 	ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj);
77 	if (ret) {
78 		DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret);
79 		goto fail_map;
80 	}
81 
82 	ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo);
83 	if (ret) {
84 		DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret);
85 		goto fail_map;
86 	}
87 
88 	queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
89 
90 	drm_exec_fini(&exec);
91 	return 0;
92 fail_map:
93 	amdgpu_bo_unref(&wptr_obj->obj);
94 fail_lock:
95 	drm_exec_fini(&exec);
96 	return ret;
97 
98 }
99 
100 static int convert_to_mes_priority(int priority)
101 {
102 	switch (priority) {
103 	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW:
104 	default:
105 		return AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
106 	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW:
107 		return AMDGPU_MES_PRIORITY_LEVEL_LOW;
108 	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH:
109 		return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM;
110 	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH:
111 		return AMDGPU_MES_PRIORITY_LEVEL_HIGH;
112 	}
113 }
114 
115 static int mes_userq_map(struct amdgpu_usermode_queue *queue)
116 {
117 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
118 	struct amdgpu_device *adev = uq_mgr->adev;
119 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
120 	struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
121 	struct mes_add_queue_input queue_input;
122 	struct amdgpu_mes *mes = &adev->mes;
123 	int r;
124 
125 	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
126 
127 	queue_input.process_va_start = 0;
128 	queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
129 
130 	/* set process quantum to 10 ms and gang quantum to 1 ms as default */
131 	queue_input.process_quantum = 100000;
132 	queue_input.gang_quantum = 10000;
133 	queue_input.paging = false;
134 
135 	queue_input.process_context_addr = uq_mgr->proc_ctx_obj.gpu_addr;
136 	queue_input.gang_context_addr = ctx->gpu_addr;
137 	queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
138 	queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority);
139 
140 	queue_input.process_id = queue->vm->pasid;
141 	queue_input.queue_type = queue->queue_type;
142 	queue_input.mqd_addr = queue->mqd.gpu_addr;
143 	queue_input.wptr_addr = userq_props->wptr_gpu_addr;
144 	queue_input.queue_size = userq_props->queue_size >> 2;
145 	queue_input.doorbell_offset = userq_props->doorbell_index;
146 	queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
147 	queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
148 
149 	if (mes->use_rs64mem) {
150 		if (!uq_mgr->proc_ctx_allocated) {
151 			r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index);
152 			if (r) {
153 				DRM_ERROR("Failed to allocate userq process index err:%d\n", r);
154 				return r;
155 			}
156 			uq_mgr->proc_ctx_allocated = true;
157 		}
158 
159 		r = amdgpu_mes_alloc_gang_ctx_index(mes, &queue->gang_ctx_array_index);
160 		if (r) {
161 			DRM_ERROR("Failed to allocate userq gang index err:%d\n", r);
162 			return r;
163 		}
164 		queue_input.process_context_array_index = uq_mgr->proc_ctx_array_index;
165 		queue_input.gang_context_array_index = queue->gang_ctx_array_index;
166 	}
167 	amdgpu_mes_lock(&adev->mes);
168 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
169 	amdgpu_mes_unlock(&adev->mes);
170 	if (r) {
171 		DRM_ERROR("Failed to map queue in HW, err (%d)\n", r);
172 		return r;
173 	}
174 
175 	amdgpu_mes_userq_queue_mapped(adev);
176 
177 	DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
178 	return 0;
179 }
180 
181 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
182 {
183 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
184 	struct amdgpu_device *adev = uq_mgr->adev;
185 	struct mes_remove_queue_input queue_input;
186 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
187 	struct amdgpu_mes *mes = &adev->mes;
188 	int r;
189 
190 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
191 	queue_input.gang_context_array_index = queue->gang_ctx_array_index;
192 	queue_input.doorbell_offset = queue->doorbell_index;
193 	queue_input.gang_context_addr = ctx->gpu_addr;
194 	queue_input.queue_type = queue->queue_type;
195 
196 	amdgpu_mes_lock(&adev->mes);
197 	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
198 	amdgpu_mes_unlock(&adev->mes);
199 	if (mes->use_rs64mem)
200 		amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
201 	if (r) {
202 		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
203 		return r;
204 	}
205 
206 	amdgpu_mes_userq_queue_unmapped(adev);
207 	return 0;
208 }
209 
210 int mes_userq_reset_hw(struct amdgpu_usermode_queue *queue,
211 		       struct mes_reset_queue_input *input)
212 {
213 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
214 	struct amdgpu_device *adev = uq_mgr->adev;
215 	int r;
216 
217 	amdgpu_mes_lock(&adev->mes);
218 	r = adev->mes.funcs->reset_hw_queue(&adev->mes, input);
219 	amdgpu_mes_unlock(&adev->mes);
220 	if (r)
221 		return r;
222 
223 	r = mes_userq_unmap(queue);
224 	if (r)
225 		return r;
226 
227 	/* Prevent the restore worker from resubmitting the guilty job. */
228 	trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
229 	queue->state = AMDGPU_USERQ_STATE_HUNG;
230 	return 0;
231 }
232 
233 int mes_userq_reset(struct amdgpu_usermode_queue *queue)
234 {
235 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
236 	struct amdgpu_device *adev = uq_mgr->adev;
237 	struct mes_reset_queue_input queue_input;
238 
239 	/* already reset by an earlier job's hang-detect; just signal and bail */
240 	if (queue->state == AMDGPU_USERQ_STATE_HUNG)
241 		return 0;
242 
243 	/* A GFX pipe reset also affects the kernel rings on that pipe. */
244 	if (queue->queue_type == AMDGPU_HW_IP_GFX &&
245 	    adev->gfx.funcs->detect_hung_queue &&
246 	    amdgpu_gfx_me_pipe_reset_supported(adev))
247 		return amdgpu_gfx_reset_mes_gfx(adev, NULL, NULL, queue, 0, true);
248 
249 	memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input));
250 	queue_input.doorbell_offset = queue->doorbell_index;
251 	queue_input.queue_type = queue->queue_type;
252 
253 	/*
254 	 * The MES packet reset fails once the hung queue wedges MES. For SDMA the
255 	 * queue is still on its HW slot, so reset it over MMIO instead: recover
256 	 * the (instance, queue_id) slot from the doorbell.
257 	 */
258 	if (queue->queue_type == AMDGPU_HW_IP_DMA &&
259 	    adev->sdma.instance[0].funcs &&
260 	    adev->sdma.instance[0].funcs->detect_hung_queue) {
261 		u32 instance, hw_queue_id;
262 
263 		if (adev->sdma.instance[0].funcs->detect_hung_queue(adev,
264 				queue->doorbell_index, &instance, &hw_queue_id)) {
265 			queue_input.use_mmio = true;
266 			queue_input.me_id = instance;
267 			queue_input.queue_id = hw_queue_id;
268 		} else {
269 			dev_warn(adev->dev,
270 				 "SDMA userq (doorbell %llu) not on any HW slot; falling back to MES reset\n",
271 				 queue->doorbell_index);
272 		}
273 	}
274 
275 	return mes_userq_reset_hw(queue, &queue_input);
276 }
277 
278 int mes_userq_reset_queue(struct amdgpu_device *adev,
279 			  struct amdgpu_usermode_queue *guilty_uq,
280 			  int queue_type,
281 			  unsigned int pipe,
282 			  unsigned int queue,
283 			  unsigned int db)
284 {
285 	struct amdgpu_usermode_queue *uq;
286 	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
287 	unsigned long uq_id;
288 	int r;
289 
290 	xa_for_each(&adev->userq_doorbell_xa, uq_id, uq) {
291 		if (uq->queue_type == queue_type) {
292 			if (uq == guilty_uq)
293 				continue;
294 			if (uq->doorbell_index == db) {
295 				uq->state = AMDGPU_USERQ_STATE_HUNG;
296 				if (use_mmio)
297 					r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 0);
298 				else
299 					r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 0);
300 				if (r)
301 					return r;
302 				r = mes_userq_unmap(uq);
303 				if (r)
304 					return r;
305 				amdgpu_userq_fence_driver_force_completion(uq);
306 				break;
307 			}
308 		}
309 	}
310 	return 0;
311 }
312 
313 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
314 				      struct amdgpu_usermode_queue *queue,
315 				      struct drm_amdgpu_userq_in *mqd_user)
316 {
317 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
318 	int r, size;
319 
320 	/* The FW expects at least one page space allocated for gang ctx. */
321 	size = AMDGPU_USERQ_GANG_CTX_SZ;
322 	r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0,
323 				    AMDGPU_GEM_DOMAIN_GTT,
324 				    &ctx->obj, &ctx->gpu_addr,
325 				    &ctx->cpu_ptr);
326 	if (r) {
327 		DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r);
328 		return r;
329 	}
330 
331 	memset(ctx->cpu_ptr, 0, size);
332 	return 0;
333 }
334 
335 static int mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr *uq_mgr)
336 {
337 	int r = 0;
338 
339 	mutex_lock(&uq_mgr->proc_ctx_lock);
340 	/* This check is a necessary because amdgpu_bo_create_kernel()
341 	 * calls helpers like amdgpu_bo_pin() and memset() unconditionally
342 	 */
343 	if (!uq_mgr->proc_ctx_obj.obj) {
344 		r = amdgpu_bo_create_kernel(uq_mgr->adev, AMDGPU_USERQ_PROC_CTX_SZ,
345 					    0, AMDGPU_GEM_DOMAIN_GTT,
346 					    &uq_mgr->proc_ctx_obj.obj,
347 					    &uq_mgr->proc_ctx_obj.gpu_addr,
348 					    &uq_mgr->proc_ctx_obj.cpu_ptr);
349 
350 		if (!r)
351 			memset(uq_mgr->proc_ctx_obj.cpu_ptr, 0, AMDGPU_USERQ_PROC_CTX_SZ);
352 	}
353 
354 	mutex_unlock(&uq_mgr->proc_ctx_lock);
355 
356 	return r;
357 }
358 
359 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
360 				struct drm_amdgpu_userq_in *args_in)
361 {
362 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
363 	struct amdgpu_device *adev = uq_mgr->adev;
364 	struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type];
365 	struct drm_amdgpu_userq_in *mqd_user = args_in;
366 	struct amdgpu_mqd_prop *userq_props;
367 	int r;
368 
369 	/* Structure to initialize MQD for userqueue using generic MQD init function */
370 	userq_props = kzalloc_obj(struct amdgpu_mqd_prop);
371 	if (!userq_props) {
372 		DRM_ERROR("Failed to allocate memory for userq_props\n");
373 		return -ENOMEM;
374 	}
375 
376 	r = amdgpu_bo_create_kernel(adev,
377 				    AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size),
378 				    0, AMDGPU_GEM_DOMAIN_GTT,
379 				    &queue->mqd.obj, &queue->mqd.gpu_addr,
380 				    &queue->mqd.cpu_ptr);
381 	if (r) {
382 		DRM_ERROR("Failed to create MQD object for userqueue\n");
383 		goto free_props;
384 	}
385 
386 	memset(queue->mqd.cpu_ptr, 0,
387 	       AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size));
388 
389 	/* Initialize the MQD BO with user given values */
390 	userq_props->wptr_gpu_addr = mqd_user->wptr_va;
391 	userq_props->rptr_gpu_addr = mqd_user->rptr_va;
392 	userq_props->queue_size = mqd_user->queue_size;
393 	userq_props->hqd_base_gpu_addr = mqd_user->queue_va;
394 	userq_props->mqd_gpu_addr = queue->mqd.gpu_addr;
395 	userq_props->use_doorbell = true;
396 	userq_props->doorbell_index = queue->doorbell_index;
397 	userq_props->fence_address = queue->fence_drv->gpu_addr;
398 
399 	if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
400 		struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
401 
402 		if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
403 			DRM_ERROR("Invalid compute IP MQD size\n");
404 			r = -EINVAL;
405 			goto free_mqd;
406 		}
407 
408 		compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
409 		if (IS_ERR(compute_mqd)) {
410 			r = PTR_ERR(compute_mqd);
411 			goto free_mqd;
412 		}
413 
414 		r = amdgpu_bo_reserve(queue->vm->root.bo, false);
415 		if (r) {
416 			kfree(compute_mqd);
417 			goto free_mqd;
418 		}
419 		r = amdgpu_userq_input_va_validate(adev, queue,
420 						   compute_mqd->eop_va, 2048,
421 						   &queue->userq_vas.va.eop);
422 		amdgpu_bo_unreserve(queue->vm->root.bo);
423 		if (r) {
424 			kfree(compute_mqd);
425 			goto free_mqd;
426 		}
427 
428 		userq_props->eop_gpu_addr = compute_mqd->eop_va;
429 		userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
430 		userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
431 		userq_props->hqd_active = false;
432 		userq_props->tmz_queue =
433 			mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
434 		kfree(compute_mqd);
435 	} else if (queue->queue_type == AMDGPU_HW_IP_GFX) {
436 		struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11;
437 		struct amdgpu_gfx_shadow_info shadow_info;
438 
439 		if (adev->gfx.funcs->get_gfx_shadow_info) {
440 			adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true);
441 		} else {
442 			r = -EINVAL;
443 			goto free_mqd;
444 		}
445 
446 		if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) {
447 			DRM_ERROR("Invalid GFX MQD\n");
448 			r = -EINVAL;
449 			goto free_mqd;
450 		}
451 
452 		mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
453 		if (IS_ERR(mqd_gfx_v11)) {
454 			r = PTR_ERR(mqd_gfx_v11);
455 			goto free_mqd;
456 		}
457 
458 		userq_props->shadow_addr = mqd_gfx_v11->shadow_va;
459 		userq_props->csa_addr = mqd_gfx_v11->csa_va;
460 		userq_props->tmz_queue =
461 			mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
462 
463 		r = amdgpu_bo_reserve(queue->vm->root.bo, false);
464 		if (r) {
465 			kfree(mqd_gfx_v11);
466 			goto free_mqd;
467 		}
468 		r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
469 						   shadow_info.shadow_size,
470 						   &queue->userq_vas.va.shadow);
471 		if (r) {
472 			amdgpu_bo_unreserve(queue->vm->root.bo);
473 			kfree(mqd_gfx_v11);
474 			goto free_mqd;
475 		}
476 
477 		r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
478 						   shadow_info.csa_size,
479 						   &queue->userq_vas.va.csa);
480 		amdgpu_bo_unreserve(queue->vm->root.bo);
481 		if (r) {
482 			kfree(mqd_gfx_v11);
483 			goto free_mqd;
484 		}
485 
486 		kfree(mqd_gfx_v11);
487 	} else if (queue->queue_type == AMDGPU_HW_IP_DMA) {
488 		struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11;
489 
490 		if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) {
491 			DRM_ERROR("Invalid SDMA MQD\n");
492 			r = -EINVAL;
493 			goto free_mqd;
494 		}
495 
496 		mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
497 		if (IS_ERR(mqd_sdma_v11)) {
498 			r = PTR_ERR(mqd_sdma_v11);
499 			goto free_mqd;
500 		}
501 
502 		r = amdgpu_bo_reserve(queue->vm->root.bo, false);
503 		if (r) {
504 			kfree(mqd_sdma_v11);
505 			goto free_mqd;
506 		}
507 		r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
508 						   32,
509 						   &queue->userq_vas.va.csa);
510 		amdgpu_bo_unreserve(queue->vm->root.bo);
511 		if (r) {
512 			kfree(mqd_sdma_v11);
513 			goto free_mqd;
514 		}
515 
516 		userq_props->csa_addr = mqd_sdma_v11->csa_va;
517 		kfree(mqd_sdma_v11);
518 	}
519 
520 	queue->userq_prop = userq_props;
521 
522 	r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);
523 	if (r) {
524 		DRM_ERROR("Failed to initialize MQD for userqueue\n");
525 		goto free_mqd;
526 	}
527 
528 	/* Create per-process MES process context BO */
529 	r = mes_userq_create_proc_ctx_space(uq_mgr);
530 	if (r) {
531 		DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r);
532 		goto free_mqd;
533 	}
534 
535 	/* Create BO of a gang for FW operations */
536 	r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user);
537 	if (r) {
538 		DRM_ERROR("Failed to allocate BO for userqueue (%d)", r);
539 		goto free_mqd;
540 	}
541 
542 	/* FW expects WPTR BOs to be mapped into GART */
543 	r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);
544 	if (r) {
545 		DRM_ERROR("Failed to create WPTR mapping\n");
546 		goto free_ctx;
547 	}
548 
549 	return 0;
550 
551 free_ctx:
552 	amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
553 			      &queue->fw_obj.cpu_ptr);
554 
555 free_mqd:
556 	amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
557 			      &queue->mqd.cpu_ptr);
558 
559 free_props:
560 	kfree(userq_props);
561 
562 	return r;
563 }
564 
565 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)
566 {
567 
568 	amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
569 			      &queue->fw_obj.cpu_ptr);
570 	kfree(queue->userq_prop);
571 	amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
572 			      &queue->mqd.cpu_ptr);
573 
574 	amdgpu_bo_unref(&queue->wptr_obj.obj);
575 }
576 
577 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
578 {
579 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
580 	struct amdgpu_device *adev = uq_mgr->adev;
581 	struct mes_suspend_gang_input queue_input;
582 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
583 	signed long timeout = 2100000; /* 2100 ms */
584 	u64 fence_gpu_addr;
585 	u32 fence_offset;
586 	u64 *fence_ptr;
587 	int i, r;
588 
589 	if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
590 		return 0;
591 	r = amdgpu_wb_get(adev, &fence_offset);
592 	if (r)
593 		return r;
594 
595 	fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4);
596 	fence_ptr = (u64 *)&adev->wb.wb[fence_offset];
597 	*fence_ptr = 0;
598 
599 	memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input));
600 	queue_input.gang_context_addr = ctx->gpu_addr;
601 	queue_input.suspend_fence_addr = fence_gpu_addr;
602 	queue_input.suspend_fence_value = 1;
603 	amdgpu_mes_lock(&adev->mes);
604 	r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input);
605 	amdgpu_mes_unlock(&adev->mes);
606 	if (r) {
607 		DRM_ERROR("Failed to suspend gang: %d\n", r);
608 		goto out;
609 	}
610 
611 	for (i = 0; i < timeout; i++) {
612 		if (*fence_ptr == 1)
613 			goto out;
614 		udelay(1);
615 	}
616 	r = -ETIMEDOUT;
617 
618 out:
619 	amdgpu_wb_free(adev, fence_offset);
620 	return r;
621 }
622 
623 static int mes_userq_restore(struct amdgpu_usermode_queue *queue)
624 {
625 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
626 	struct amdgpu_device *adev = uq_mgr->adev;
627 	struct mes_resume_gang_input queue_input;
628 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
629 	int r;
630 
631 	if (queue->state == AMDGPU_USERQ_STATE_HUNG)
632 		return -EINVAL;
633 	if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED)
634 		return 0;
635 
636 	memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input));
637 	queue_input.gang_context_addr = ctx->gpu_addr;
638 
639 	amdgpu_mes_lock(&adev->mes);
640 	r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input);
641 	amdgpu_mes_unlock(&adev->mes);
642 	if (r)
643 		dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r);
644 	return r;
645 }
646 
647 const struct amdgpu_userq_funcs userq_mes_funcs = {
648 	.mqd_create = mes_userq_mqd_create,
649 	.mqd_destroy = mes_userq_mqd_destroy,
650 	.unmap = mes_userq_unmap,
651 	.map = mes_userq_map,
652 	.preempt = mes_userq_preempt,
653 	.restore = mes_userq_restore,
654 	.reset = mes_userq_reset,
655 };
656