1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright 2024 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24 #include <drm/drm_drv.h>
25 #include "amdgpu.h"
26 #include "amdgpu_gfx.h"
27 #include "mes_userqueue.h"
28 #include "amdgpu_userq_fence.h"
29
30 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
31 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
32
33 static int
mes_userq_create_wptr_mapping(struct amdgpu_device * adev,struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_usermode_queue * queue,uint64_t wptr)34 mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
35 struct amdgpu_userq_mgr *uq_mgr,
36 struct amdgpu_usermode_queue *queue,
37 uint64_t wptr)
38 {
39 struct amdgpu_bo_va_mapping *wptr_mapping;
40 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;
41 struct amdgpu_bo *obj;
42 struct amdgpu_vm *vm = queue->vm;
43 struct drm_exec exec;
44 int ret;
45
46 wptr &= AMDGPU_GMC_HOLE_MASK;
47
48 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
49 drm_exec_until_all_locked(&exec) {
50 ret = amdgpu_vm_lock_pd(vm, &exec, 1);
51 drm_exec_retry_on_contention(&exec);
52 if (unlikely(ret))
53 goto fail_lock;
54
55 wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT);
56 if (!wptr_mapping) {
57 ret = -EINVAL;
58 goto fail_lock;
59 }
60
61 obj = wptr_mapping->bo_va->base.bo;
62 ret = drm_exec_lock_obj(&exec, &obj->tbo.base);
63 drm_exec_retry_on_contention(&exec);
64 if (unlikely(ret))
65 goto fail_lock;
66 }
67
68 wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo);
69 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) {
70 ret = -EINVAL;
71 goto fail_map;
72 }
73
74 /* TODO use eviction fence instead of pinning. */
75 ret = amdgpu_bo_pin(wptr_obj->obj, AMDGPU_GEM_DOMAIN_GTT);
76 if (ret) {
77 DRM_ERROR("Failed to pin wptr bo. ret %d\n", ret);
78 goto fail_map;
79 }
80
81 ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo);
82 if (ret) {
83 DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret);
84 goto fail_alloc_gart;
85 }
86
87 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
88
89 drm_exec_fini(&exec);
90 return 0;
91
92 fail_alloc_gart:
93 amdgpu_bo_unpin(wptr_obj->obj);
94 fail_map:
95 amdgpu_bo_unref(&wptr_obj->obj);
96 fail_lock:
97 drm_exec_fini(&exec);
98 return ret;
99
100 }
101
convert_to_mes_priority(int priority)102 static int convert_to_mes_priority(int priority)
103 {
104 switch (priority) {
105 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW:
106 default:
107 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
108 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW:
109 return AMDGPU_MES_PRIORITY_LEVEL_LOW;
110 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH:
111 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM;
112 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH:
113 return AMDGPU_MES_PRIORITY_LEVEL_HIGH;
114 }
115 }
116
mes_userq_map(struct amdgpu_usermode_queue * queue)117 static int mes_userq_map(struct amdgpu_usermode_queue *queue)
118 {
119 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
120 struct amdgpu_device *adev = uq_mgr->adev;
121 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
122 struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
123 struct mes_add_queue_input queue_input;
124 int r;
125
126 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
127
128 queue_input.process_va_start = 0;
129 queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
130
131 /* set process quantum to 10 ms and gang quantum to 1 ms as default */
132 queue_input.process_quantum = 100000;
133 queue_input.gang_quantum = 10000;
134 queue_input.paging = false;
135
136 queue_input.process_context_addr = ctx->gpu_addr;
137 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
138 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
139 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority);
140
141 queue_input.process_id = queue->vm->pasid;
142 queue_input.queue_type = queue->queue_type;
143 queue_input.mqd_addr = queue->mqd.gpu_addr;
144 queue_input.wptr_addr = userq_props->wptr_gpu_addr;
145 queue_input.queue_size = userq_props->queue_size >> 2;
146 queue_input.doorbell_offset = userq_props->doorbell_index;
147 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
148 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
149
150 amdgpu_mes_lock(&adev->mes);
151 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
152 amdgpu_mes_unlock(&adev->mes);
153 if (r) {
154 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r);
155 return r;
156 }
157
158 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
159 return 0;
160 }
161
mes_userq_unmap(struct amdgpu_usermode_queue * queue)162 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
163 {
164 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
165 struct amdgpu_device *adev = uq_mgr->adev;
166 struct mes_remove_queue_input queue_input;
167 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
168 int r;
169
170 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
171 queue_input.doorbell_offset = queue->doorbell_index;
172 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
173
174 amdgpu_mes_lock(&adev->mes);
175 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
176 amdgpu_mes_unlock(&adev->mes);
177 if (r)
178 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
179 return r;
180 }
181
mes_userq_create_ctx_space(struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_usermode_queue * queue,struct drm_amdgpu_userq_in * mqd_user)182 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
183 struct amdgpu_usermode_queue *queue,
184 struct drm_amdgpu_userq_in *mqd_user)
185 {
186 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
187 int r, size;
188
189 /*
190 * The FW expects at least one page space allocated for
191 * process ctx and gang ctx each. Create an object
192 * for the same.
193 */
194 size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ;
195 r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0,
196 AMDGPU_GEM_DOMAIN_GTT,
197 &ctx->obj, &ctx->gpu_addr,
198 &ctx->cpu_ptr);
199 if (r) {
200 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r);
201 return r;
202 }
203
204 memset(ctx->cpu_ptr, 0, size);
205 return 0;
206 }
207
mes_userq_detect_and_reset(struct amdgpu_device * adev,int queue_type)208 static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
209 int queue_type)
210 {
211 int db_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
212 struct mes_detect_and_reset_queue_input input;
213 struct amdgpu_usermode_queue *queue;
214 unsigned int hung_db_num = 0;
215 unsigned long queue_id;
216 u32 db_array[8];
217 bool found_hung_queue = false;
218 int r, i;
219
220 if (db_array_size > 8) {
221 dev_err(adev->dev, "DB array size (%d vs 8) too small\n",
222 db_array_size);
223 return -EINVAL;
224 }
225
226 memset(&input, 0x0, sizeof(struct mes_detect_and_reset_queue_input));
227
228 input.queue_type = queue_type;
229
230 amdgpu_mes_lock(&adev->mes);
231 r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false,
232 &hung_db_num, db_array, 0);
233 amdgpu_mes_unlock(&adev->mes);
234 if (r) {
235 dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r);
236 } else if (hung_db_num) {
237 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
238 if (queue->queue_type == queue_type) {
239 for (i = 0; i < hung_db_num; i++) {
240 if (queue->doorbell_index == db_array[i]) {
241 queue->state = AMDGPU_USERQ_STATE_HUNG;
242 found_hung_queue = true;
243 atomic_inc(&adev->gpu_reset_counter);
244 amdgpu_userq_fence_driver_force_completion(queue);
245 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL);
246 }
247 }
248 }
249 }
250 }
251
252 if (found_hung_queue) {
253 /* Resume scheduling after hang recovery */
254 r = amdgpu_mes_resume(adev);
255 }
256
257 return r;
258 }
259
mes_userq_mqd_create(struct amdgpu_usermode_queue * queue,struct drm_amdgpu_userq_in * args_in)260 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
261 struct drm_amdgpu_userq_in *args_in)
262 {
263 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
264 struct amdgpu_device *adev = uq_mgr->adev;
265 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type];
266 struct drm_amdgpu_userq_in *mqd_user = args_in;
267 struct amdgpu_mqd_prop *userq_props;
268 int r;
269
270 /* Structure to initialize MQD for userqueue using generic MQD init function */
271 userq_props = kzalloc_obj(struct amdgpu_mqd_prop);
272 if (!userq_props) {
273 DRM_ERROR("Failed to allocate memory for userq_props\n");
274 return -ENOMEM;
275 }
276
277 r = amdgpu_bo_create_kernel(adev,
278 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size),
279 0, AMDGPU_GEM_DOMAIN_GTT,
280 &queue->mqd.obj, &queue->mqd.gpu_addr,
281 &queue->mqd.cpu_ptr);
282 if (r) {
283 DRM_ERROR("Failed to create MQD object for userqueue\n");
284 goto free_props;
285 }
286
287 memset(queue->mqd.cpu_ptr, 0,
288 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size));
289
290 /* Initialize the MQD BO with user given values */
291 userq_props->wptr_gpu_addr = mqd_user->wptr_va;
292 userq_props->rptr_gpu_addr = mqd_user->rptr_va;
293 userq_props->queue_size = mqd_user->queue_size;
294 userq_props->hqd_base_gpu_addr = mqd_user->queue_va;
295 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr;
296 userq_props->use_doorbell = true;
297 userq_props->doorbell_index = queue->doorbell_index;
298 userq_props->fence_address = queue->fence_drv->gpu_addr;
299
300 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
301 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
302
303 if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
304 DRM_ERROR("Invalid compute IP MQD size\n");
305 r = -EINVAL;
306 goto free_mqd;
307 }
308
309 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
310 if (IS_ERR(compute_mqd)) {
311 DRM_ERROR("Failed to read user MQD\n");
312 r = -ENOMEM;
313 goto free_mqd;
314 }
315
316 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
317 if (r) {
318 kfree(compute_mqd);
319 goto free_mqd;
320 }
321 r = amdgpu_userq_input_va_validate(adev, queue,
322 compute_mqd->eop_va, 2048,
323 &queue->userq_vas.va.eop);
324 amdgpu_bo_unreserve(queue->vm->root.bo);
325 if (r) {
326 kfree(compute_mqd);
327 goto free_mqd;
328 }
329
330 userq_props->eop_gpu_addr = compute_mqd->eop_va;
331 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
332 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
333 userq_props->hqd_active = false;
334 userq_props->tmz_queue =
335 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
336 kfree(compute_mqd);
337 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) {
338 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11;
339 struct amdgpu_gfx_shadow_info shadow_info;
340
341 if (adev->gfx.funcs->get_gfx_shadow_info) {
342 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true);
343 } else {
344 r = -EINVAL;
345 goto free_mqd;
346 }
347
348 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) {
349 DRM_ERROR("Invalid GFX MQD\n");
350 r = -EINVAL;
351 goto free_mqd;
352 }
353
354 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
355 if (IS_ERR(mqd_gfx_v11)) {
356 DRM_ERROR("Failed to read user MQD\n");
357 r = -ENOMEM;
358 goto free_mqd;
359 }
360
361 userq_props->shadow_addr = mqd_gfx_v11->shadow_va;
362 userq_props->csa_addr = mqd_gfx_v11->csa_va;
363 userq_props->tmz_queue =
364 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
365
366 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
367 if (r) {
368 kfree(mqd_gfx_v11);
369 goto free_mqd;
370 }
371 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
372 shadow_info.shadow_size,
373 &queue->userq_vas.va.shadow);
374 if (r) {
375 amdgpu_bo_unreserve(queue->vm->root.bo);
376 kfree(mqd_gfx_v11);
377 goto free_mqd;
378 }
379
380 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
381 shadow_info.csa_size,
382 &queue->userq_vas.va.csa);
383 amdgpu_bo_unreserve(queue->vm->root.bo);
384 if (r) {
385 kfree(mqd_gfx_v11);
386 goto free_mqd;
387 }
388
389 kfree(mqd_gfx_v11);
390 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) {
391 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11;
392
393 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) {
394 DRM_ERROR("Invalid SDMA MQD\n");
395 r = -EINVAL;
396 goto free_mqd;
397 }
398
399 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
400 if (IS_ERR(mqd_sdma_v11)) {
401 DRM_ERROR("Failed to read sdma user MQD\n");
402 r = -ENOMEM;
403 goto free_mqd;
404 }
405
406 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
407 if (r) {
408 kfree(mqd_sdma_v11);
409 goto free_mqd;
410 }
411 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
412 32,
413 &queue->userq_vas.va.csa);
414 amdgpu_bo_unreserve(queue->vm->root.bo);
415 if (r) {
416 kfree(mqd_sdma_v11);
417 goto free_mqd;
418 }
419
420 userq_props->csa_addr = mqd_sdma_v11->csa_va;
421 kfree(mqd_sdma_v11);
422 }
423
424 queue->userq_prop = userq_props;
425
426 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);
427 if (r) {
428 DRM_ERROR("Failed to initialize MQD for userqueue\n");
429 goto free_mqd;
430 }
431
432 /* Create BO for FW operations */
433 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user);
434 if (r) {
435 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r);
436 goto free_mqd;
437 }
438
439 /* FW expects WPTR BOs to be mapped into GART */
440 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);
441 if (r) {
442 DRM_ERROR("Failed to create WPTR mapping\n");
443 goto free_ctx;
444 }
445
446 return 0;
447
448 free_ctx:
449 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
450 &queue->fw_obj.cpu_ptr);
451
452 free_mqd:
453 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
454 &queue->mqd.cpu_ptr);
455
456 free_props:
457 kfree(userq_props);
458
459 return r;
460 }
461
mes_userq_mqd_destroy(struct amdgpu_usermode_queue * queue)462 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)
463 {
464
465 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
466 &queue->fw_obj.cpu_ptr);
467 kfree(queue->userq_prop);
468 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
469 &queue->mqd.cpu_ptr);
470
471 amdgpu_bo_reserve(queue->wptr_obj.obj, true);
472 amdgpu_bo_unpin(queue->wptr_obj.obj);
473 amdgpu_bo_unreserve(queue->wptr_obj.obj);
474 amdgpu_bo_unref(&queue->wptr_obj.obj);
475 }
476
mes_userq_preempt(struct amdgpu_usermode_queue * queue)477 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
478 {
479 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
480 struct amdgpu_device *adev = uq_mgr->adev;
481 struct mes_suspend_gang_input queue_input;
482 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
483 signed long timeout = 2100000; /* 2100 ms */
484 u64 fence_gpu_addr;
485 u32 fence_offset;
486 u64 *fence_ptr;
487 int i, r;
488
489 if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
490 return 0;
491 r = amdgpu_device_wb_get(adev, &fence_offset);
492 if (r)
493 return r;
494
495 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4);
496 fence_ptr = (u64 *)&adev->wb.wb[fence_offset];
497 *fence_ptr = 0;
498
499 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input));
500 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
501 queue_input.suspend_fence_addr = fence_gpu_addr;
502 queue_input.suspend_fence_value = 1;
503 amdgpu_mes_lock(&adev->mes);
504 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input);
505 amdgpu_mes_unlock(&adev->mes);
506 if (r) {
507 DRM_ERROR("Failed to suspend gang: %d\n", r);
508 goto out;
509 }
510
511 for (i = 0; i < timeout; i++) {
512 if (*fence_ptr == 1)
513 goto out;
514 udelay(1);
515 }
516 r = -ETIMEDOUT;
517
518 out:
519 amdgpu_device_wb_free(adev, fence_offset);
520 return r;
521 }
522
mes_userq_restore(struct amdgpu_usermode_queue * queue)523 static int mes_userq_restore(struct amdgpu_usermode_queue *queue)
524 {
525 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
526 struct amdgpu_device *adev = uq_mgr->adev;
527 struct mes_resume_gang_input queue_input;
528 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
529 int r;
530
531 if (queue->state == AMDGPU_USERQ_STATE_HUNG)
532 return -EINVAL;
533 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED)
534 return 0;
535
536 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input));
537 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
538
539 amdgpu_mes_lock(&adev->mes);
540 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input);
541 amdgpu_mes_unlock(&adev->mes);
542 if (r)
543 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r);
544 return r;
545 }
546
547 const struct amdgpu_userq_funcs userq_mes_funcs = {
548 .mqd_create = mes_userq_mqd_create,
549 .mqd_destroy = mes_userq_mqd_destroy,
550 .unmap = mes_userq_unmap,
551 .map = mes_userq_map,
552 .detect_and_reset = mes_userq_detect_and_reset,
553 .preempt = mes_userq_preempt,
554 .restore = mes_userq_restore,
555 };
556