1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright 2024 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24 #include <drm/drm_drv.h>
25 #include "amdgpu.h"
26 #include "amdgpu_gfx.h"
27 #include "mes_userqueue.h"
28 #include "amdgpu_userq_fence.h"
29 #include "amdgpu_trace.h"
30
31 #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
32 #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
33
34 static int
mes_userq_create_wptr_mapping(struct amdgpu_device * adev,struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_usermode_queue * queue,uint64_t wptr)35 mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
36 struct amdgpu_userq_mgr *uq_mgr,
37 struct amdgpu_usermode_queue *queue,
38 uint64_t wptr)
39 {
40 struct amdgpu_bo_va_mapping *wptr_mapping;
41 struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;
42 struct amdgpu_bo *obj;
43 struct amdgpu_vm *vm = queue->vm;
44 struct drm_exec exec;
45 int ret;
46
47 wptr &= AMDGPU_GMC_HOLE_MASK;
48
49 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
50 drm_exec_until_all_locked(&exec) {
51 ret = amdgpu_vm_lock_pd(vm, &exec, 1);
52 drm_exec_retry_on_contention(&exec);
53 if (unlikely(ret))
54 goto fail_lock;
55
56 wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT);
57 if (!wptr_mapping) {
58 ret = -EINVAL;
59 goto fail_lock;
60 }
61
62 obj = wptr_mapping->bo_va->base.bo;
63 ret = drm_exec_lock_obj(&exec, &obj->tbo.base);
64 drm_exec_retry_on_contention(&exec);
65 if (unlikely(ret))
66 goto fail_lock;
67 }
68
69 wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo);
70 if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) {
71 ret = -EINVAL;
72 goto fail_map;
73 }
74 /* Keep WPTR BO under eviction-fence control instead of pinning. */
75 ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj);
76 if (ret) {
77 DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret);
78 goto fail_map;
79 }
80
81 ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo);
82 if (ret) {
83 DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret);
84 goto fail_map;
85 }
86
87 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
88
89 drm_exec_fini(&exec);
90 return 0;
91 fail_map:
92 amdgpu_bo_unref(&wptr_obj->obj);
93 fail_lock:
94 drm_exec_fini(&exec);
95 return ret;
96
97 }
98
convert_to_mes_priority(int priority)99 static int convert_to_mes_priority(int priority)
100 {
101 switch (priority) {
102 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW:
103 default:
104 return AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
105 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW:
106 return AMDGPU_MES_PRIORITY_LEVEL_LOW;
107 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH:
108 return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM;
109 case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH:
110 return AMDGPU_MES_PRIORITY_LEVEL_HIGH;
111 }
112 }
113
mes_userq_map(struct amdgpu_usermode_queue * queue)114 static int mes_userq_map(struct amdgpu_usermode_queue *queue)
115 {
116 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
117 struct amdgpu_device *adev = uq_mgr->adev;
118 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
119 struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
120 struct mes_add_queue_input queue_input;
121 struct amdgpu_mes *mes = &adev->mes;
122 int r;
123
124 memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
125
126 queue_input.process_va_start = 0;
127 queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
128
129 /* set process quantum to 10 ms and gang quantum to 1 ms as default */
130 queue_input.process_quantum = 100000;
131 queue_input.gang_quantum = 10000;
132 queue_input.paging = false;
133
134 queue_input.process_context_addr = uq_mgr->proc_ctx_obj.gpu_addr;
135 queue_input.gang_context_addr = ctx->gpu_addr;
136 queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
137 queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority);
138
139 queue_input.process_id = queue->vm->pasid;
140 queue_input.queue_type = queue->queue_type;
141 queue_input.mqd_addr = queue->mqd.gpu_addr;
142 queue_input.wptr_addr = userq_props->wptr_gpu_addr;
143 queue_input.queue_size = userq_props->queue_size >> 2;
144 queue_input.doorbell_offset = userq_props->doorbell_index;
145 queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
146 queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
147
148 if (mes->use_rs64mem) {
149 if (!uq_mgr->proc_ctx_allocated) {
150 r = amdgpu_mes_alloc_proc_ctx_index(mes, &uq_mgr->proc_ctx_array_index);
151 if (r) {
152 DRM_ERROR("Failed to allocate userq process index err:%d\n", r);
153 return r;
154 }
155 uq_mgr->proc_ctx_allocated = true;
156 }
157
158 r = amdgpu_mes_alloc_gang_ctx_index(mes, &queue->gang_ctx_array_index);
159 if (r) {
160 DRM_ERROR("Failed to allocate userq gang index err:%d\n", r);
161 return r;
162 }
163 queue_input.process_context_array_index = uq_mgr->proc_ctx_array_index;
164 queue_input.gang_context_array_index = queue->gang_ctx_array_index;
165 }
166 amdgpu_mes_lock(&adev->mes);
167 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
168 amdgpu_mes_unlock(&adev->mes);
169 if (r) {
170 DRM_ERROR("Failed to map queue in HW, err (%d)\n", r);
171 return r;
172 }
173
174 DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
175 return 0;
176 }
177
mes_userq_unmap(struct amdgpu_usermode_queue * queue)178 static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
179 {
180 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
181 struct amdgpu_device *adev = uq_mgr->adev;
182 struct mes_remove_queue_input queue_input;
183 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
184 struct amdgpu_mes *mes = &adev->mes;
185 int r;
186
187 memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
188 queue_input.gang_context_array_index = queue->gang_ctx_array_index;
189 queue_input.doorbell_offset = queue->doorbell_index;
190 queue_input.gang_context_addr = ctx->gpu_addr;
191 queue_input.queue_type = queue->queue_type;
192
193 amdgpu_mes_lock(&adev->mes);
194 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
195 amdgpu_mes_unlock(&adev->mes);
196 if (mes->use_rs64mem)
197 amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
198 if (r)
199 DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
200 return r;
201 }
202
mes_userq_reset(struct amdgpu_usermode_queue * queue)203 int mes_userq_reset(struct amdgpu_usermode_queue *queue)
204 {
205 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
206 struct amdgpu_device *adev = uq_mgr->adev;
207 struct mes_reset_queue_input queue_input;
208 int r;
209
210 /* XXX: add a FW version check for SDMA per queue reset */
211 memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input));
212 queue_input.doorbell_offset = queue->doorbell_index;
213 queue_input.queue_type = queue->queue_type;
214
215 amdgpu_mes_lock(&adev->mes);
216 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);
217 amdgpu_mes_unlock(&adev->mes);
218 if (r)
219 return r;
220
221 /* mes_userq_unmap() does not update queue->state; mark it UNMAPPED so the
222 * destroy path does not issue a second REMOVE_QUEUE for the removed queue.
223 */
224 r = mes_userq_unmap(queue);
225 if (!r) {
226 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED);
227 queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
228 }
229 return r;
230 }
231
mes_userq_reset_queue(struct amdgpu_device * adev,struct amdgpu_usermode_queue * guilty_uq,int queue_type,unsigned int pipe,unsigned int queue,unsigned int db)232 int mes_userq_reset_queue(struct amdgpu_device *adev,
233 struct amdgpu_usermode_queue *guilty_uq,
234 int queue_type,
235 unsigned int pipe,
236 unsigned int queue,
237 unsigned int db)
238 {
239 struct amdgpu_usermode_queue *uq;
240 bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
241 unsigned long uq_id;
242 int r;
243
244 xa_for_each(&adev->userq_doorbell_xa, uq_id, uq) {
245 if (uq->queue_type == queue_type) {
246 if (uq == guilty_uq)
247 continue;
248 if (uq->doorbell_index == db) {
249 uq->state = AMDGPU_USERQ_STATE_HUNG;
250 if (use_mmio)
251 r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 0);
252 else
253 r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 0);
254 if (r)
255 return r;
256 r = mes_userq_unmap(uq);
257 if (r)
258 return r;
259 amdgpu_userq_fence_driver_force_completion(uq);
260 break;
261 }
262 }
263 }
264 return 0;
265 }
266
mes_userq_create_ctx_space(struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_usermode_queue * queue,struct drm_amdgpu_userq_in * mqd_user)267 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
268 struct amdgpu_usermode_queue *queue,
269 struct drm_amdgpu_userq_in *mqd_user)
270 {
271 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
272 int r, size;
273
274 /* The FW expects at least one page space allocated for gang ctx. */
275 size = AMDGPU_USERQ_GANG_CTX_SZ;
276 r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0,
277 AMDGPU_GEM_DOMAIN_GTT,
278 &ctx->obj, &ctx->gpu_addr,
279 &ctx->cpu_ptr);
280 if (r) {
281 DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r);
282 return r;
283 }
284
285 memset(ctx->cpu_ptr, 0, size);
286 return 0;
287 }
288
mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr * uq_mgr)289 static int mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr *uq_mgr)
290 {
291 int r = 0;
292
293 mutex_lock(&uq_mgr->proc_ctx_lock);
294 /* This check is a necessary because amdgpu_bo_create_kernel()
295 * calls helpers like amdgpu_bo_pin() and memset() unconditionally
296 */
297 if (!uq_mgr->proc_ctx_obj.obj) {
298 r = amdgpu_bo_create_kernel(uq_mgr->adev, AMDGPU_USERQ_PROC_CTX_SZ,
299 0, AMDGPU_GEM_DOMAIN_GTT,
300 &uq_mgr->proc_ctx_obj.obj,
301 &uq_mgr->proc_ctx_obj.gpu_addr,
302 &uq_mgr->proc_ctx_obj.cpu_ptr);
303
304 if (!r)
305 memset(uq_mgr->proc_ctx_obj.cpu_ptr, 0, AMDGPU_USERQ_PROC_CTX_SZ);
306 }
307
308 mutex_unlock(&uq_mgr->proc_ctx_lock);
309
310 return r;
311 }
312
mes_userq_mqd_create(struct amdgpu_usermode_queue * queue,struct drm_amdgpu_userq_in * args_in)313 static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
314 struct drm_amdgpu_userq_in *args_in)
315 {
316 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
317 struct amdgpu_device *adev = uq_mgr->adev;
318 struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type];
319 struct drm_amdgpu_userq_in *mqd_user = args_in;
320 struct amdgpu_mqd_prop *userq_props;
321 int r;
322
323 /* Structure to initialize MQD for userqueue using generic MQD init function */
324 userq_props = kzalloc_obj(struct amdgpu_mqd_prop);
325 if (!userq_props) {
326 DRM_ERROR("Failed to allocate memory for userq_props\n");
327 return -ENOMEM;
328 }
329
330 r = amdgpu_bo_create_kernel(adev,
331 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size),
332 0, AMDGPU_GEM_DOMAIN_GTT,
333 &queue->mqd.obj, &queue->mqd.gpu_addr,
334 &queue->mqd.cpu_ptr);
335 if (r) {
336 DRM_ERROR("Failed to create MQD object for userqueue\n");
337 goto free_props;
338 }
339
340 memset(queue->mqd.cpu_ptr, 0,
341 AMDGPU_MQD_SIZE_ALIGN(mqd_hw_default->mqd_size));
342
343 /* Initialize the MQD BO with user given values */
344 userq_props->wptr_gpu_addr = mqd_user->wptr_va;
345 userq_props->rptr_gpu_addr = mqd_user->rptr_va;
346 userq_props->queue_size = mqd_user->queue_size;
347 userq_props->hqd_base_gpu_addr = mqd_user->queue_va;
348 userq_props->mqd_gpu_addr = queue->mqd.gpu_addr;
349 userq_props->use_doorbell = true;
350 userq_props->doorbell_index = queue->doorbell_index;
351 userq_props->fence_address = queue->fence_drv->gpu_addr;
352
353 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
354 struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
355
356 if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
357 DRM_ERROR("Invalid compute IP MQD size\n");
358 r = -EINVAL;
359 goto free_mqd;
360 }
361
362 compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
363 if (IS_ERR(compute_mqd)) {
364 DRM_ERROR("Failed to read user MQD\n");
365 r = -ENOMEM;
366 goto free_mqd;
367 }
368
369 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
370 if (r) {
371 kfree(compute_mqd);
372 goto free_mqd;
373 }
374 r = amdgpu_userq_input_va_validate(adev, queue,
375 compute_mqd->eop_va, 2048,
376 &queue->userq_vas.va.eop);
377 amdgpu_bo_unreserve(queue->vm->root.bo);
378 if (r) {
379 kfree(compute_mqd);
380 goto free_mqd;
381 }
382
383 userq_props->eop_gpu_addr = compute_mqd->eop_va;
384 userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
385 userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
386 userq_props->hqd_active = false;
387 userq_props->tmz_queue =
388 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
389 kfree(compute_mqd);
390 } else if (queue->queue_type == AMDGPU_HW_IP_GFX) {
391 struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11;
392 struct amdgpu_gfx_shadow_info shadow_info;
393
394 if (adev->gfx.funcs->get_gfx_shadow_info) {
395 adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true);
396 } else {
397 r = -EINVAL;
398 goto free_mqd;
399 }
400
401 if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) {
402 DRM_ERROR("Invalid GFX MQD\n");
403 r = -EINVAL;
404 goto free_mqd;
405 }
406
407 mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
408 if (IS_ERR(mqd_gfx_v11)) {
409 DRM_ERROR("Failed to read user MQD\n");
410 r = -ENOMEM;
411 goto free_mqd;
412 }
413
414 userq_props->shadow_addr = mqd_gfx_v11->shadow_va;
415 userq_props->csa_addr = mqd_gfx_v11->csa_va;
416 userq_props->tmz_queue =
417 mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
418
419 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
420 if (r) {
421 kfree(mqd_gfx_v11);
422 goto free_mqd;
423 }
424 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->shadow_va,
425 shadow_info.shadow_size,
426 &queue->userq_vas.va.shadow);
427 if (r) {
428 amdgpu_bo_unreserve(queue->vm->root.bo);
429 kfree(mqd_gfx_v11);
430 goto free_mqd;
431 }
432
433 r = amdgpu_userq_input_va_validate(adev, queue, mqd_gfx_v11->csa_va,
434 shadow_info.csa_size,
435 &queue->userq_vas.va.csa);
436 amdgpu_bo_unreserve(queue->vm->root.bo);
437 if (r) {
438 kfree(mqd_gfx_v11);
439 goto free_mqd;
440 }
441
442 kfree(mqd_gfx_v11);
443 } else if (queue->queue_type == AMDGPU_HW_IP_DMA) {
444 struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11;
445
446 if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) {
447 DRM_ERROR("Invalid SDMA MQD\n");
448 r = -EINVAL;
449 goto free_mqd;
450 }
451
452 mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
453 if (IS_ERR(mqd_sdma_v11)) {
454 DRM_ERROR("Failed to read sdma user MQD\n");
455 r = -ENOMEM;
456 goto free_mqd;
457 }
458
459 r = amdgpu_bo_reserve(queue->vm->root.bo, false);
460 if (r) {
461 kfree(mqd_sdma_v11);
462 goto free_mqd;
463 }
464 r = amdgpu_userq_input_va_validate(adev, queue, mqd_sdma_v11->csa_va,
465 32,
466 &queue->userq_vas.va.csa);
467 amdgpu_bo_unreserve(queue->vm->root.bo);
468 if (r) {
469 kfree(mqd_sdma_v11);
470 goto free_mqd;
471 }
472
473 userq_props->csa_addr = mqd_sdma_v11->csa_va;
474 kfree(mqd_sdma_v11);
475 }
476
477 queue->userq_prop = userq_props;
478
479 r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);
480 if (r) {
481 DRM_ERROR("Failed to initialize MQD for userqueue\n");
482 goto free_mqd;
483 }
484
485 /* Create per-process MES process context BO */
486 r = mes_userq_create_proc_ctx_space(uq_mgr);
487 if (r) {
488 DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r);
489 goto free_mqd;
490 }
491
492 /* Create BO of a gang for FW operations */
493 r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user);
494 if (r) {
495 DRM_ERROR("Failed to allocate BO for userqueue (%d)", r);
496 goto free_mqd;
497 }
498
499 /* FW expects WPTR BOs to be mapped into GART */
500 r = mes_userq_create_wptr_mapping(adev, uq_mgr, queue, userq_props->wptr_gpu_addr);
501 if (r) {
502 DRM_ERROR("Failed to create WPTR mapping\n");
503 goto free_ctx;
504 }
505
506 return 0;
507
508 free_ctx:
509 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
510 &queue->fw_obj.cpu_ptr);
511
512 free_mqd:
513 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
514 &queue->mqd.cpu_ptr);
515
516 free_props:
517 kfree(userq_props);
518
519 return r;
520 }
521
mes_userq_mqd_destroy(struct amdgpu_usermode_queue * queue)522 static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)
523 {
524
525 amdgpu_bo_free_kernel(&queue->fw_obj.obj, &queue->fw_obj.gpu_addr,
526 &queue->fw_obj.cpu_ptr);
527 kfree(queue->userq_prop);
528 amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
529 &queue->mqd.cpu_ptr);
530
531 amdgpu_bo_unref(&queue->wptr_obj.obj);
532 }
533
mes_userq_preempt(struct amdgpu_usermode_queue * queue)534 static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
535 {
536 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
537 struct amdgpu_device *adev = uq_mgr->adev;
538 struct mes_suspend_gang_input queue_input;
539 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
540 signed long timeout = 2100000; /* 2100 ms */
541 u64 fence_gpu_addr;
542 u32 fence_offset;
543 u64 *fence_ptr;
544 int i, r;
545
546 if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
547 return 0;
548 r = amdgpu_wb_get(adev, &fence_offset);
549 if (r)
550 return r;
551
552 fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4);
553 fence_ptr = (u64 *)&adev->wb.wb[fence_offset];
554 *fence_ptr = 0;
555
556 memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input));
557 queue_input.gang_context_addr = ctx->gpu_addr;
558 queue_input.suspend_fence_addr = fence_gpu_addr;
559 queue_input.suspend_fence_value = 1;
560 amdgpu_mes_lock(&adev->mes);
561 r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input);
562 amdgpu_mes_unlock(&adev->mes);
563 if (r) {
564 DRM_ERROR("Failed to suspend gang: %d\n", r);
565 goto out;
566 }
567
568 for (i = 0; i < timeout; i++) {
569 if (*fence_ptr == 1)
570 goto out;
571 udelay(1);
572 }
573 r = -ETIMEDOUT;
574
575 out:
576 amdgpu_wb_free(adev, fence_offset);
577 return r;
578 }
579
mes_userq_restore(struct amdgpu_usermode_queue * queue)580 static int mes_userq_restore(struct amdgpu_usermode_queue *queue)
581 {
582 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
583 struct amdgpu_device *adev = uq_mgr->adev;
584 struct mes_resume_gang_input queue_input;
585 struct amdgpu_userq_obj *ctx = &queue->fw_obj;
586 int r;
587
588 if (queue->state == AMDGPU_USERQ_STATE_HUNG)
589 return -EINVAL;
590 if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED)
591 return 0;
592
593 memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input));
594 queue_input.gang_context_addr = ctx->gpu_addr;
595
596 amdgpu_mes_lock(&adev->mes);
597 r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input);
598 amdgpu_mes_unlock(&adev->mes);
599 if (r)
600 dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r);
601 return r;
602 }
603
604 const struct amdgpu_userq_funcs userq_mes_funcs = {
605 .mqd_create = mes_userq_mqd_create,
606 .mqd_destroy = mes_userq_mqd_destroy,
607 .unmap = mes_userq_unmap,
608 .map = mes_userq_map,
609 .preempt = mes_userq_preempt,
610 .restore = mes_userq_restore,
611 .reset = mes_userq_reset,
612 };
613