1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright 2024 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24 #include <drm/drm_drv.h>
25 #include "amdgpu.h"
26 #include "amdgpu_gfx.h"
27 #include "mes_userqueue.h"
28 #include "amdgpu_userq_fence.h"
29
30 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
31 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
32
33 static int
mes_userq_create_wptr_mapping(struct amdgpu_device * adev,struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_usermode_queue * queue,uint64_t wptr)34 mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
35 struct amdgpu_userq_mgr *uq_mgr,
36 struct amdgpu_usermode_queue *queue,
37 uint64_t wptr)
38 {
39 struct amdgpu_bo_va_mapping *wptr_mapping;
40 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;
41 struct amdgpu_bo *obj;
42 struct amdgpu_vm *vm = queue->vm;
43 struct drm_exec exec;
44 int ret;
45
46 wptr &= AMDGPU_GMC_HOLE_MASK;
47
48 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
49 drm_exec_until_all_locked(&exec) {
50 ret = amdgpu_vm_lock_pd(vm, &exec, 1);
51 drm_exec_retry_on_contention(&exec);
52 if (unlikely(ret))
53 goto fail_lock;
54
55 wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT);
56 if (!wptr_mapping) {
57 ret = -EINVAL;
58 goto fail_lock;
59 }
60
61 obj = wptr_mapping->bo_va->base.bo;
62 ret = drm_exec_lock_obj(&exec, &obj->tbo.base);
63 drm_exec_retry_on_contention(&exec);
64 if (unlikely(ret))
65 goto fail_lock;
66 }
67
68 wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo);
69 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) {
70 ret = -EINVAL;
71 goto fail_map;
72 }
73 /* Keep WPTR BO under eviction-fence control instead of pinning. */
74 ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj);
75 if (ret) {
76 DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret);
77 goto fail_map;
78 }
79
80 ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo);
81 if (ret) {
82 DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret);
83 goto fail_map;
84 }
85
86 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
87
88 drm_exec_fini(&exec);
89 return 0;
90 fail_map:
91 amdgpu_bo_unref(&wptr_obj->obj);
92 fail_lock:
93 drm_exec_fini(&exec);
94 return ret;
95
96 }
97
convert_to_mes_priority(int priority)98 static int convert_to_mes_priority(int priority)
99 {
100 switch (priority) {
101 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW:
102 default:
103 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
104 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW:
105 return AMDGPU_MES_PRIORITY_LEVEL_LOW;
106 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH:
107 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM;
108 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH:
109 return AMDGPU_MES_PRIORITY_LEVEL_HIGH;
110 }
111 }
112
mes_userq_map(struct amdgpu_usermode_queue * queue)113 static int mes_userq_map(struct amdgpu_usermode_queue *queue)
114 {
115 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
116 struct amdgpu_device *adev = uq_mgr->adev;
117 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
118 struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
119 struct mes_add_queue_input queue_input;
120 int r;
121
122 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
123
124 queue_input.process_va_start = 0;
125 queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
126
127 /* set process quantum to 10 ms and gang quantum to 1 ms as default */
128 queue_input.process_quantum = 100000;
129 queue_input.gang_quantum = 10000;
130 queue_input.paging = false;
131
132 queue_input.process_context_addr = ctx->gpu_addr;
133 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
134 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
135 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority);
136
137 queue_input.process_id = queue->vm->pasid;
138 queue_input.queue_type = queue->queue_type;
139 queue_input.mqd_addr = queue->mqd.gpu_addr;
140 queue_input.wptr_addr = userq_props->wptr_gpu_addr;
141 queue_input.queue_size = userq_props->queue_size >> 2;
142 queue_input.doorbell_offset = userq_props->doorbell_index;
143 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
144 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
145
146 amdgpu_mes_lock(&adev->mes);
147 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
148 amdgpu_mes_unlock(&adev->mes);
149 if (r) {
150 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r);
151 return r;
152 }
153
154 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
155 return 0;
156 }
157
mes_userq_unmap(struct amdgpu_usermode_queue * queue)158 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
159 {
160 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
161 struct amdgpu_device *adev = uq_mgr->adev;
162 struct mes_remove_queue_input queue_input;
163 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
164 int r;
165
166 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
167 queue_input.doorbell_offset = queue->doorbell_index;
168 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
169
170 amdgpu_mes_lock(&adev->mes);
171 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
172 amdgpu_mes_unlock(&adev->mes);
173 if (r)
174 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
175 return r;
176 }
177
mes_userq_create_ctx_space(struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_usermode_queue * queue,struct drm_amdgpu_userq_in * mqd_user)178 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
179 struct amdgpu_usermode_queue *queue,
180 struct drm_amdgpu_userq_in *mqd_user)
181 {
182 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
183 int r, size;
184
185 /*
186 * The FW expects at least one page space allocated for
187 * process ctx and gang ctx each. Create an object
188 * for the same.
189 */
190 size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ;
191 r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0,
192 AMDGPU_GEM_DOMAIN_GTT,
193 &ctx->obj, &ctx->gpu_addr,
194 &ctx->cpu_ptr);
195 if (r) {
196 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r);
197 return r;
198 }
199
200 memset(ctx->cpu_ptr, 0, size);
201 return 0;
202 }
203
mes_userq_detect_and_reset(struct amdgpu_device * adev,int queue_type)204 static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
205 int queue_type)
206 {
207 int db_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
208 struct mes_detect_and_reset_queue_input input;
209 struct amdgpu_usermode_queue *queue;
210 unsigned int hung_db_num = 0;
211 unsigned long queue_id;
212 u32 db_array[8];
213 bool found_hung_queue = false;
214 int r, i;
215
216 if (db_array_size > 8) {
217 dev_err(adev->dev, "DB array size (%d vs 8) too small\n",
218 db_array_size);
219 return -EINVAL;
220 }
221
222 memset(&input, 0x0, sizeof(struct mes_detect_and_reset_queue_input));
223
224 input.queue_type = queue_type;
225
226 amdgpu_mes_lock(&adev->mes);
227 r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false,
228 &hung_db_num, db_array, 0);
229 amdgpu_mes_unlock(&adev->mes);
230 if (r) {
231 dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r);
232 } else if (hung_db_num) {
233 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
234 if (queue->queue_type == queue_type) {
235 for (i = 0; i < hung_db_num; i++) {
236 if (queue->doorbell_index == db_array[i]) {
237 queue->state = AMDGPU_USERQ_STATE_HUNG;
238 found_hung_queue = true;
239 atomic_inc(&adev->gpu_reset_counter);
240 amdgpu_userq_fence_driver_force_completion(queue);
241 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL);
242 }
243 }
244 }
245 }
246 }
247
248 if (found_hung_queue) {
249 /* Resume scheduling after hang recovery */
250 r = amdgpu_mes_resume(adev, input.xcc_id);
251 }
252
253 return r;
254 }
255
mes_userq_mqd_create(struct amdgpu_usermode_queue * queue,struct drm_amdgpu_userq_in * args_in)256 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
257 struct drm_amdgpu_userq_in *args_in)
258 {
259 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
260 struct amdgpu_device *adev = uq_mgr->adev;
261 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type];
262 struct drm_amdgpu_userq_in *mqd_user = args_in;
263 struct amdgpu_mqd_prop *userq_props;
264 int r;
265
266 /* Structure to initialize MQD for userqueue using generic MQD init function */
267 userq_props = kzalloc_obj(struct amdgpu_mqd_prop);
268 if (!userq_props) {
269 DRM_ERROR("Failed to allocate memory for userq_props\n");
270 return -ENOMEM;
271 }
272
273 r = amdgpu_bo_create_kernel(adev,
274 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size),
275 0, AMDGPU_GEM_DOMAIN_GTT,
276 &queue->mqd.obj, &queue->mqd.gpu_addr,
277 &queue->mqd.cpu_ptr);
278 if (r) {
279 DRM_ERROR("Failed to create MQD object for userqueue\n");
280 goto free_props;
281 }
282
283 memset(queue->mqd.cpu_ptr, 0,
284 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size));
285
286 /* Initialize the MQD BO with user given values */
287 userq_props->wptr_gpu_addr = mqd_user->wptr_va;
288 userq_props->rptr_gpu_addr = mqd_user->rptr_va;
289 userq_props->queue_size = mqd_user->queue_size;
290 userq_props->hqd_base_gpu_addr = mqd_user->queue_va;
291 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr;
292 userq_props->use_doorbell = true;
293 userq_props->doorbell_index = queue->doorbell_index;
294 userq_props->fence_address = queue->fence_drv->gpu_addr;
295
296 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
297 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
298
299 if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
300 DRM_ERROR("Invalid compute IP MQD size\n");
301 r = -EINVAL;
302 goto free_mqd;
303 }
304
305 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
306 if (IS_ERR(compute_mqd)) {
307 DRM_ERROR("Failed to read user MQD\n");
308 r = -ENOMEM;
309 goto free_mqd;
310 }
311
312 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
313 if (r) {
314 kfree(compute_mqd);
315 goto free_mqd;
316 }
317 r = amdgpu_userq_input_va_validate(adev, queue,
318 compute_mqd->eop_va, 2048,
319 &queue->userq_vas.va.eop);
320 amdgpu_bo_unreserve(queue->vm->root.bo);
321 if (r) {
322 kfree(compute_mqd);
323 goto free_mqd;
324 }
325
326 userq_props->eop_gpu_addr = compute_mqd->eop_va;
327 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
328 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
329 userq_props->hqd_active = false;
330 userq_props->tmz_queue =
331 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
332 kfree(compute_mqd);
333 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) {
334 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11;
335 struct amdgpu_gfx_shadow_info shadow_info;
336
337 if (adev->gfx.funcs->get_gfx_shadow_info) {
338 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true);
339 } else {
340 r = -EINVAL;
341 goto free_mqd;
342 }
343
344 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) {
345 DRM_ERROR("Invalid GFX MQD\n");
346 r = -EINVAL;
347 goto free_mqd;
348 }
349
350 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
351 if (IS_ERR(mqd_gfx_v11)) {
352 DRM_ERROR("Failed to read user MQD\n");
353 r = -ENOMEM;
354 goto free_mqd;
355 }
356
357 userq_props->shadow_addr = mqd_gfx_v11->shadow_va;
358 userq_props->csa_addr = mqd_gfx_v11->csa_va;
359 userq_props->tmz_queue =
360 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
361
362 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
363 if (r) {
364 kfree(mqd_gfx_v11);
365 goto free_mqd;
366 }
367 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
368 shadow_info.shadow_size,
369 &queue->userq_vas.va.shadow);
370 if (r) {
371 amdgpu_bo_unreserve(queue->vm->root.bo);
372 kfree(mqd_gfx_v11);
373 goto free_mqd;
374 }
375
376 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
377 shadow_info.csa_size,
378 &queue->userq_vas.va.csa);
379 amdgpu_bo_unreserve(queue->vm->root.bo);
380 if (r) {
381 kfree(mqd_gfx_v11);
382 goto free_mqd;
383 }
384
385 kfree(mqd_gfx_v11);
386 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) {
387 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11;
388
389 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) {
390 DRM_ERROR("Invalid SDMA MQD\n");
391 r = -EINVAL;
392 goto free_mqd;
393 }
394
395 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
396 if (IS_ERR(mqd_sdma_v11)) {
397 DRM_ERROR("Failed to read sdma user MQD\n");
398 r = -ENOMEM;
399 goto free_mqd;
400 }
401
402 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
403 if (r) {
404 kfree(mqd_sdma_v11);
405 goto free_mqd;
406 }
407 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
408 32,
409 &queue->userq_vas.va.csa);
410 amdgpu_bo_unreserve(queue->vm->root.bo);
411 if (r) {
412 kfree(mqd_sdma_v11);
413 goto free_mqd;
414 }
415
416 userq_props->csa_addr = mqd_sdma_v11->csa_va;
417 kfree(mqd_sdma_v11);
418 }
419
420 queue->userq_prop = userq_props;
421
422 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);
423 if (r) {
424 DRM_ERROR("Failed to initialize MQD for userqueue\n");
425 goto free_mqd;
426 }
427
428 /* Create BO for FW operations */
429 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user);
430 if (r) {
431 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r);
432 goto free_mqd;
433 }
434
435 /* FW expects WPTR BOs to be mapped into GART */
436 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);
437 if (r) {
438 DRM_ERROR("Failed to create WPTR mapping\n");
439 goto free_ctx;
440 }
441
442 return 0;
443
444 free_ctx:
445 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
446 &queue->fw_obj.cpu_ptr);
447
448 free_mqd:
449 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
450 &queue->mqd.cpu_ptr);
451
452 free_props:
453 kfree(userq_props);
454
455 return r;
456 }
457
mes_userq_mqd_destroy(struct amdgpu_usermode_queue * queue)458 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)
459 {
460
461 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
462 &queue->fw_obj.cpu_ptr);
463 kfree(queue->userq_prop);
464 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
465 &queue->mqd.cpu_ptr);
466
467 amdgpu_bo_unref(&queue->wptr_obj.obj);
468 }
469
mes_userq_preempt(struct amdgpu_usermode_queue * queue)470 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
471 {
472 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
473 struct amdgpu_device *adev = uq_mgr->adev;
474 struct mes_suspend_gang_input queue_input;
475 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
476 signed long timeout = 2100000; /* 2100 ms */
477 u64 fence_gpu_addr;
478 u32 fence_offset;
479 u64 *fence_ptr;
480 int i, r;
481
482 if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
483 return 0;
484 r = amdgpu_device_wb_get(adev, &fence_offset);
485 if (r)
486 return r;
487
488 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4);
489 fence_ptr = (u64 *)&adev->wb.wb[fence_offset];
490 *fence_ptr = 0;
491
492 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input));
493 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
494 queue_input.suspend_fence_addr = fence_gpu_addr;
495 queue_input.suspend_fence_value = 1;
496 amdgpu_mes_lock(&adev->mes);
497 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input);
498 amdgpu_mes_unlock(&adev->mes);
499 if (r) {
500 DRM_ERROR("Failed to suspend gang: %d\n", r);
501 goto out;
502 }
503
504 for (i = 0; i < timeout; i++) {
505 if (*fence_ptr == 1)
506 goto out;
507 udelay(1);
508 }
509 r = -ETIMEDOUT;
510
511 out:
512 amdgpu_device_wb_free(adev, fence_offset);
513 return r;
514 }
515
mes_userq_restore(struct amdgpu_usermode_queue * queue)516 static int mes_userq_restore(struct amdgpu_usermode_queue *queue)
517 {
518 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
519 struct amdgpu_device *adev = uq_mgr->adev;
520 struct mes_resume_gang_input queue_input;
521 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
522 int r;
523
524 if (queue->state == AMDGPU_USERQ_STATE_HUNG)
525 return -EINVAL;
526 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED)
527 return 0;
528
529 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input));
530 queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
531
532 amdgpu_mes_lock(&adev->mes);
533 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input);
534 amdgpu_mes_unlock(&adev->mes);
535 if (r)
536 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r);
537 return r;
538 }
539
540 const struct amdgpu_userq_funcs userq_mes_funcs = {
541 .mqd_create = mes_userq_mqd_create,
542 .mqd_destroy = mes_userq_mqd_destroy,
543 .unmap = mes_userq_unmap,
544 .map = mes_userq_map,
545 .detect_and_reset = mes_userq_detect_and_reset,
546 .preempt = mes_userq_preempt,
547 .restore = mes_userq_restore,
548 };
549