1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright 2023 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25 #include <drm/drm_auth.h>
26 #include <drm/drm_exec.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/overflow.h>
29 #include <drm/drm_drv.h>
30
31 #include "amdgpu.h"
32 #include "amdgpu_reset.h"
33 #include "amdgpu_vm.h"
34 #include "amdgpu_userq.h"
35 #include "amdgpu_hmm.h"
36 #include "amdgpu_userq_fence.h"
37 #include "amdgpu_trace.h"
38
amdgpu_userq_get_supported_ip_mask(struct amdgpu_device * adev)39 u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
40 {
41 int i;
42 u32 userq_ip_mask = 0;
43
44 for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {
45 if (adev->userq_funcs[i])
46 userq_ip_mask |= (1 << i);
47 }
48
49 return userq_ip_mask;
50 }
51
amdgpu_userq_is_reset_type_supported(struct amdgpu_device * adev,enum amdgpu_ring_type ring_type,int reset_type)52 static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
53 enum amdgpu_ring_type ring_type, int reset_type)
54 {
55
56 if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX)
57 return false;
58
59 switch (ring_type) {
60 case AMDGPU_RING_TYPE_GFX:
61 if (adev->gfx.gfx_supported_reset & reset_type)
62 return true;
63 break;
64 case AMDGPU_RING_TYPE_COMPUTE:
65 if (adev->gfx.compute_supported_reset & reset_type)
66 return true;
67 break;
68 case AMDGPU_RING_TYPE_SDMA:
69 if (adev->sdma.supported_reset & reset_type)
70 return true;
71 break;
72 case AMDGPU_RING_TYPE_VCN_DEC:
73 case AMDGPU_RING_TYPE_VCN_ENC:
74 if (adev->vcn.supported_reset & reset_type)
75 return true;
76 break;
77 case AMDGPU_RING_TYPE_VCN_JPEG:
78 if (adev->jpeg.supported_reset & reset_type)
79 return true;
80 break;
81 default:
82 break;
83 }
84 return false;
85 }
86
amdgpu_userq_mgr_reset_work(struct work_struct * work)87 static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
88 {
89 struct amdgpu_userq_mgr *uq_mgr =
90 container_of(work, struct amdgpu_userq_mgr,
91 reset_work);
92 struct amdgpu_device *adev = uq_mgr->adev;
93 struct amdgpu_reset_context reset_context;
94
95 if (unlikely(adev->debug_disable_gpu_ring_reset)) {
96 dev_err(adev->dev, "userq reset disabled by debug mask\n");
97 return;
98 }
99
100 /*
101 * If GPU recovery feature is disabled system-wide,
102 * skip all reset detection logic
103 */
104 if (!amdgpu_gpu_recovery)
105 return;
106
107 memset(&reset_context, 0, sizeof(reset_context));
108
109 reset_context.method = AMD_RESET_METHOD_NONE;
110 reset_context.reset_req_dev = adev;
111 reset_context.src = AMDGPU_RESET_SRC_USERQ;
112 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
113 /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
114
115 amdgpu_device_gpu_recover(adev, NULL, &reset_context);
116 }
117
amdgpu_userq_hang_detect_work(struct work_struct * work)118 static void amdgpu_userq_hang_detect_work(struct work_struct *work)
119 {
120 struct amdgpu_usermode_queue *queue =
121 container_of(work, struct amdgpu_usermode_queue,
122 hang_detect_work.work);
123 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
124 struct amdgpu_device *adev = uq_mgr->adev;
125 const struct amdgpu_userq_funcs *userq_funcs =
126 adev->userq_funcs[queue->queue_type];
127 struct drm_wedge_task_info *info = NULL;
128 struct amdgpu_task_info *ti = NULL;
129 bool gpu_reset = false;
130
131 if (unlikely(adev->debug_disable_gpu_ring_reset)) {
132 dev_err(adev->dev, "userq reset disabled by debug mask\n");
133 return;
134 }
135
136 /*
137 * If GPU recovery feature is disabled system-wide,
138 * skip all reset detection logic
139 */
140 if (!amdgpu_gpu_recovery)
141 return;
142
143 if (queue->vm && queue->vm->pasid) {
144 ti = amdgpu_vm_get_task_info_pasid(adev, queue->vm->pasid);
145 if (ti) {
146 amdgpu_vm_print_task_info(adev, ti);
147 info = &ti->task;
148 }
149 }
150
151 if (amdgpu_userq_is_reset_type_supported(adev, queue->queue_type,
152 AMDGPU_RESET_TYPE_PER_QUEUE)) {
153 int r;
154
155 if (queue->queue_type == AMDGPU_HW_IP_COMPUTE)
156 r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL,
157 queue, NULL, NULL);
158 else
159 r = userq_funcs->reset(queue);
160 if (r) {
161 gpu_reset = true;
162 } else {
163 atomic_inc(&adev->gpu_reset_counter);
164 amdgpu_userq_fence_driver_force_completion(queue);
165 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);
166 }
167 } else {
168 gpu_reset = true;
169 }
170 amdgpu_vm_put_task_info(ti);
171
172 /*
173 * Don't schedule the work here! Scheduling or queue work from one reset
174 * handler to another is illegal if you don't take extra precautions!
175 */
176 if (gpu_reset)
177 amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
178 }
179
180 /*
181 * Start hang detection for a user queue fence. A delayed work will be scheduled
182 * to reset the queues when the fence doesn't signal in time.
183 */
amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue * queue)184 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
185 {
186 struct amdgpu_device *adev;
187 unsigned long timeout_jiffies;
188
189 adev = queue->userq_mgr->adev;
190 /* Determine timeout based on queue type */
191 switch (queue->queue_type) {
192 case AMDGPU_RING_TYPE_GFX:
193 timeout_jiffies = adev->gfx_timeout;
194 break;
195 case AMDGPU_RING_TYPE_COMPUTE:
196 timeout_jiffies = adev->compute_timeout;
197 break;
198 case AMDGPU_RING_TYPE_SDMA:
199 timeout_jiffies = adev->sdma_timeout;
200 break;
201 default:
202 timeout_jiffies = adev->gfx_timeout;
203 break;
204 }
205
206 queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
207 timeout_jiffies);
208 }
209
amdgpu_userq_process_fence_irq(struct amdgpu_device * adev,u32 doorbell)210 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
211 {
212 struct xarray *xa = &adev->userq_doorbell_xa;
213 struct amdgpu_usermode_queue *queue;
214 unsigned long flags;
215 int r;
216
217 xa_lock_irqsave(xa, flags);
218 queue = xa_load(xa, doorbell);
219 if (queue) {
220 r = amdgpu_userq_fence_driver_process(queue->fence_drv);
221 /*
222 * We are in interrupt context here, this *can't* wait for
223 * reset work to finish.
224 */
225 if (r >= 0)
226 cancel_delayed_work(&queue->hang_detect_work);
227
228 /* Restart the timer when there are still fences pending */
229 if (r == 1)
230 amdgpu_userq_start_hang_detect_work(queue);
231 }
232 xa_unlock_irqrestore(xa, flags);
233 }
234
amdgpu_userq_input_va_validate(struct amdgpu_device * adev,struct amdgpu_usermode_queue * queue,u64 addr,u64 expected_size,u64 * va_out)235 int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
236 struct amdgpu_usermode_queue *queue,
237 u64 addr, u64 expected_size,
238 u64 *va_out)
239 {
240 struct amdgpu_bo_va_mapping *va_map;
241 struct amdgpu_vm *vm = queue->vm;
242 u64 start_addr;
243 u64 end_addr;
244 u64 start_page;
245
246 /* Caller must hold vm->root.bo reservation */
247 dma_resv_assert_held(queue->vm->root.bo->tbo.base.resv);
248
249 if (!expected_size)
250 return -EINVAL;
251
252 start_addr = addr & AMDGPU_GMC_HOLE_MASK;
253 if (check_add_overflow(start_addr, expected_size - 1, &end_addr))
254 return -EINVAL;
255
256 start_page = start_addr >> AMDGPU_GPU_PAGE_SHIFT;
257
258 va_map = amdgpu_vm_bo_lookup_mapping(vm, start_page);
259 if (!va_map)
260 return -EINVAL;
261
262 /* Lookup guarantees start_page is mapped; ensure full span is covered. */
263 if ((end_addr >> AMDGPU_GPU_PAGE_SHIFT) <= va_map->last) {
264 va_map->bo_va->userq_va_mapped = true;
265 *va_out = start_page;
266 return 0;
267 }
268
269 return -EINVAL;
270 }
271
amdgpu_userq_buffer_va_mapped(struct amdgpu_vm * vm,u64 addr)272 static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
273 {
274 struct amdgpu_bo_va_mapping *mapping;
275 bool r;
276
277 dma_resv_assert_held(vm->root.bo->tbo.base.resv);
278
279 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
280 if (!IS_ERR_OR_NULL(mapping) && mapping->bo_va->userq_va_mapped)
281 r = true;
282 else
283 r = false;
284
285 return r;
286 }
287
amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue * queue)288 static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue)
289 {
290 int i;
291 bool mapped;
292
293 for (i = 0; i < ARRAY_SIZE(queue->userq_vas.va_array); i++) {
294 if (!queue->userq_vas.va_array[i])
295 continue;
296
297 mapped = amdgpu_userq_buffer_va_mapped(queue->vm,
298 queue->userq_vas.va_array[i]);
299 dev_dbg(queue->userq_mgr->adev->dev,
300 "validate the userq mapping:%p va:%llx r:%d\n",
301 queue, queue->userq_vas.va_array[i], mapped);
302
303 if (!mapped)
304 return false;
305 }
306
307 return true;
308 }
309
310
311
amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue * queue)312 static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
313 {
314 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
315 struct amdgpu_device *adev = uq_mgr->adev;
316 const struct amdgpu_userq_funcs *userq_funcs =
317 adev->userq_funcs[queue->queue_type];
318 int r;
319
320 if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
321 trace_amdgpu_userq_state_start(queue);
322
323 r = userq_funcs->preempt(queue);
324 if (r) {
325 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
326 queue->state = AMDGPU_USERQ_STATE_HUNG;
327 return r;
328 } else {
329 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_PREEMPTED);
330 queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
331 }
332 }
333 return 0;
334 }
335
amdgpu_userq_restore_helper(struct amdgpu_usermode_queue * queue)336 static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
337 {
338 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
339 struct amdgpu_device *adev = uq_mgr->adev;
340 const struct amdgpu_userq_funcs *userq_funcs =
341 adev->userq_funcs[queue->queue_type];
342 int r = 0;
343
344 if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {
345 trace_amdgpu_userq_state_start(queue);
346
347 r = userq_funcs->restore(queue);
348 if (r) {
349 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
350 queue->state = AMDGPU_USERQ_STATE_HUNG;
351 } else {
352 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
353 queue->state = AMDGPU_USERQ_STATE_MAPPED;
354 }
355 }
356
357 return r;
358 }
359
amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue * queue)360 static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
361 {
362 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
363 struct amdgpu_device *adev = uq_mgr->adev;
364 const struct amdgpu_userq_funcs *userq_funcs =
365 adev->userq_funcs[queue->queue_type];
366 int r;
367
368 if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
369 (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
370 trace_amdgpu_userq_state_start(queue);
371
372 r = userq_funcs->unmap(queue);
373 if (r) {
374 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
375 queue->state = AMDGPU_USERQ_STATE_HUNG;
376 return r;
377 } else {
378 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED);
379 queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
380 }
381 }
382
383 return 0;
384 }
385
amdgpu_userq_map_helper(struct amdgpu_usermode_queue * queue)386 static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
387 {
388 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
389 struct amdgpu_device *adev = uq_mgr->adev;
390 const struct amdgpu_userq_funcs *userq_funcs =
391 adev->userq_funcs[queue->queue_type];
392 int r;
393
394 if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
395 trace_amdgpu_userq_state_start(queue);
396
397 r = userq_funcs->map(queue);
398 if (r) {
399 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
400 queue->state = AMDGPU_USERQ_STATE_HUNG;
401 return r;
402 } else {
403 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
404 queue->state = AMDGPU_USERQ_STATE_MAPPED;
405 }
406 }
407
408 return 0;
409 }
410
amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue * queue)411 static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
412 {
413 struct dma_fence *f = queue->last_fence;
414
415 if (!f)
416 return;
417
418 dma_fence_wait(f, false);
419 }
420
amdgpu_userq_detach_doorbell(struct amdgpu_usermode_queue * queue)421 static void amdgpu_userq_detach_doorbell(struct amdgpu_usermode_queue *queue)
422 {
423 struct amdgpu_device *adev = queue->userq_mgr->adev;
424
425 down_read(&adev->reset_domain->sem);
426 xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index);
427 up_read(&adev->reset_domain->sem);
428 }
429
430 /**
431 * amdgpu_userq_ensure_ev_fence - ensure a valid, unsignaled eviction fence exists
432 * @uq_mgr: the usermode queue manager for this process
433 * @evf_mgr: the eviction fence manager to check and rearm
434 *
435 * Ensures that a valid and not yet signaled eviction fence is attached to the
436 * usermode queue before any queue operations proceed. If it is signalled, then
437 * rearm a new eviction fence.
438 */
439 void
amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_eviction_fence_mgr * evf_mgr)440 amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
441 struct amdgpu_eviction_fence_mgr *evf_mgr)
442 {
443 struct dma_fence *ev_fence;
444
445 retry:
446 /* Flush any pending resume work to create ev_fence */
447 flush_delayed_work(&uq_mgr->resume_work);
448
449 mutex_lock(&uq_mgr->userq_mutex);
450 ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
451 if (dma_fence_is_signaled(ev_fence)) {
452 dma_fence_put(ev_fence);
453 mutex_unlock(&uq_mgr->userq_mutex);
454 /*
455 * Looks like there was no pending resume work,
456 * add one now to create a valid eviction fence
457 */
458 schedule_delayed_work(&uq_mgr->resume_work, 0);
459 goto retry;
460 }
461 dma_fence_put(ev_fence);
462 }
463
464
465
466 static int
amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_db_info * db_info,struct drm_file * filp,u64 * index)467 amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,
468 struct amdgpu_db_info *db_info,
469 struct drm_file *filp,
470 u64 *index)
471 {
472 u64 doorbell_index;
473 struct drm_gem_object *gobj;
474 struct amdgpu_userq_obj *db_obj = db_info->db_obj;
475 struct amdgpu_bo *abo;
476 int r, db_size;
477
478 gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle);
479 if (gobj == NULL) {
480 drm_file_err(uq_mgr->file, "Can't find GEM object for doorbell\n");
481 return -EINVAL;
482 }
483
484 /*
485 * Pinning a regular BO into the doorbell domain would discard its
486 * contents, possibly those of a buffer shared by another client.
487 */
488 abo = gem_to_amdgpu_bo(gobj);
489 if (!(abo->preferred_domains & AMDGPU_GEM_DOMAIN_DOORBELL)) {
490 drm_gem_object_put(gobj);
491 return -EINVAL;
492 }
493
494 db_obj->obj = amdgpu_bo_ref(abo);
495 drm_gem_object_put(gobj);
496
497 r = amdgpu_bo_reserve(db_obj->obj, true);
498 if (r) {
499 drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");
500 goto unref_bo;
501 }
502
503 /* Pin the BO before generating the index, unpin in queue destroy */
504 r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL);
505 if (r) {
506 drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");
507 goto unresv_bo;
508 }
509
510 switch (db_info->queue_type) {
511 case AMDGPU_HW_IP_GFX:
512 case AMDGPU_HW_IP_COMPUTE:
513 case AMDGPU_HW_IP_DMA:
514 db_size = sizeof(u64);
515 break;
516 default:
517 drm_file_err(uq_mgr->file, "[Usermode queues] IP %d not support\n",
518 db_info->queue_type);
519 r = -EINVAL;
520 goto unpin_bo;
521 }
522
523 /* Validate doorbell_offset is within the doorbell BO */
524 if ((u64)db_info->doorbell_offset * db_size + db_size >
525 amdgpu_bo_size(db_obj->obj)) {
526 r = -EINVAL;
527 goto unpin_bo;
528 }
529
530 doorbell_index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj,
531 db_info->doorbell_offset, db_size);
532 drm_dbg_driver(adev_to_drm(uq_mgr->adev),
533 "[Usermode queues] doorbell index=%lld\n", doorbell_index);
534 amdgpu_bo_unreserve(db_obj->obj);
535 *index = doorbell_index;
536 return 0;
537
538 unpin_bo:
539 amdgpu_bo_unpin(db_obj->obj);
540 unresv_bo:
541 amdgpu_bo_unreserve(db_obj->obj);
542 unref_bo:
543 amdgpu_bo_unref(&db_obj->obj);
544 return r;
545 }
546
547 static int
amdgpu_userq_destroy(struct amdgpu_userq_mgr * uq_mgr,struct amdgpu_usermode_queue * queue)548 amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue)
549 {
550 struct amdgpu_device *adev = uq_mgr->adev;
551 const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];
552 int r = 0;
553
554 trace_amdgpu_userq_destroy_start(queue);
555
556 cancel_delayed_work_sync(&uq_mgr->resume_work);
557
558 mutex_lock(&uq_mgr->userq_mutex);
559 amdgpu_userq_wait_for_last_fence(queue);
560
561 amdgpu_userq_detach_doorbell(queue);
562 cancel_delayed_work_sync(&queue->hang_detect_work);
563
564 #if defined(CONFIG_DEBUG_FS)
565 debugfs_remove_recursive(queue->debugfs_queue);
566 #endif
567 r = amdgpu_userq_unmap_helper(queue);
568 atomic_dec(&uq_mgr->userq_count[queue->queue_type]);
569 amdgpu_userq_fence_driver_free(queue);
570 queue->fence_drv = NULL;
571 mutex_unlock(&uq_mgr->userq_mutex);
572
573 /*
574 * A failed unmap means MES could not remove the hung queue and is now
575 * unresponsive. Recover the GPU here so the wedged MES does not fail
576 * the next, unrelated queue submission and trigger a reset attributed
577 * to an innocent workload.
578 */
579 if (r)
580 queue_work(adev->reset_domain->wq, &uq_mgr->reset_work);
581
582 uq_funcs->mqd_destroy(queue);
583 queue->userq_mgr = NULL;
584
585 amdgpu_bo_reserve(queue->db_obj.obj, true);
586 amdgpu_bo_unpin(queue->db_obj.obj);
587 amdgpu_bo_unreserve(queue->db_obj.obj);
588 amdgpu_bo_unref(&queue->db_obj.obj);
589
590 trace_amdgpu_userq_destroy_end(queue, r);
591 kfree(queue);
592
593 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
594
595 return r;
596 }
597
amdgpu_userq_kref_destroy(struct kref * kref)598 static void amdgpu_userq_kref_destroy(struct kref *kref)
599 {
600 int r;
601 struct amdgpu_usermode_queue *queue =
602 container_of(kref, struct amdgpu_usermode_queue, refcount);
603 struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
604
605 r = amdgpu_userq_destroy(uq_mgr, queue);
606 if (r)
607 drm_file_err(uq_mgr->file, "Failed to destroy usermode queue %d\n", r);
608 }
609
amdgpu_userq_get(struct amdgpu_userq_mgr * uq_mgr,u32 qid)610 struct amdgpu_usermode_queue *amdgpu_userq_get(struct amdgpu_userq_mgr *uq_mgr, u32 qid)
611 {
612 struct amdgpu_usermode_queue *queue;
613
614 xa_lock(&uq_mgr->userq_xa);
615 queue = xa_load(&uq_mgr->userq_xa, qid);
616 if (queue)
617 kref_get(&queue->refcount);
618 xa_unlock(&uq_mgr->userq_xa);
619
620 return queue;
621 }
622
amdgpu_userq_put(struct amdgpu_usermode_queue * queue)623 void amdgpu_userq_put(struct amdgpu_usermode_queue *queue)
624 {
625 if (queue)
626 kref_put(&queue->refcount, amdgpu_userq_kref_destroy);
627 }
628
amdgpu_userq_priority_permit(struct drm_file * filp,int priority)629 static int amdgpu_userq_priority_permit(struct drm_file *filp,
630 int priority)
631 {
632 if (priority < AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH)
633 return 0;
634
635 if (capable(CAP_SYS_NICE))
636 return 0;
637
638 if (drm_is_current_master(filp))
639 return 0;
640
641 return -EACCES;
642 }
643
644 static int
amdgpu_userq_create(struct drm_file * filp,union drm_amdgpu_userq * args)645 amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
646 {
647 struct amdgpu_fpriv *fpriv = filp->driver_priv;
648 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
649 struct amdgpu_device *adev = uq_mgr->adev;
650 const struct amdgpu_userq_funcs *uq_funcs;
651 struct amdgpu_usermode_queue *queue;
652 struct amdgpu_db_info db_info;
653 uint64_t index;
654 int priority;
655 u32 qid;
656 int r;
657
658 priority =
659 (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK)
660 >> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
661 r = amdgpu_userq_priority_permit(filp, priority);
662 if (r)
663 return r;
664
665 r = pm_runtime_resume_and_get(adev_to_drm(adev)->dev);
666 if (r < 0) {
667 drm_file_err(uq_mgr->file, "pm_runtime_resume_and_get() failed for userqueue create\n");
668 return r;
669 }
670
671 uq_funcs = adev->userq_funcs[args->in.ip_type];
672 if (!uq_funcs) {
673 r = -EINVAL;
674 goto err_pm_runtime;
675 }
676
677 queue = kzalloc_obj(struct amdgpu_usermode_queue);
678 if (!queue) {
679 r = -ENOMEM;
680 goto err_pm_runtime;
681 }
682
683 kref_init(&queue->refcount);
684 queue->doorbell_handle = args->in.doorbell_handle;
685 queue->queue_type = args->in.ip_type;
686 queue->vm = &fpriv->vm;
687 queue->priority = priority;
688 queue->xcp_id = (fpriv->xcp_id != AMDGPU_XCP_NO_PARTITION) ?
689 fpriv->xcp_id : 0;
690 queue->userq_mgr = uq_mgr;
691 INIT_DELAYED_WORK(&queue->hang_detect_work,
692 amdgpu_userq_hang_detect_work);
693
694 r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
695 if (r)
696 goto free_queue;
697
698 xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
699 mutex_init(&queue->fence_drv_lock);
700 /* Make sure the queue can actually run with those virtual addresses. */
701 r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
702 if (r)
703 goto free_fence_drv;
704
705 if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va,
706 args->in.queue_size,
707 &queue->userq_vas.va.queue_rb) ||
708 amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
709 sizeof(u64),
710 &queue->userq_vas.va.rptr) ||
711 amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
712 sizeof(u64),
713 &queue->userq_vas.va.wptr)) {
714 r = -EINVAL;
715 amdgpu_bo_unreserve(fpriv->vm.root.bo);
716 goto free_fence_drv;
717 }
718 amdgpu_bo_unreserve(fpriv->vm.root.bo);
719
720 /* Convert relative doorbell offset into absolute doorbell index */
721 db_info.queue_type = queue->queue_type;
722 db_info.doorbell_handle = queue->doorbell_handle;
723 db_info.db_obj = &queue->db_obj;
724 db_info.doorbell_offset = args->in.doorbell_offset;
725 r = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp, &index);
726 if (r) {
727 drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
728 goto free_fence_drv;
729 }
730
731 queue->doorbell_index = index;
732 queue->doorbell_offset = (u32)args->in.doorbell_offset;
733 trace_amdgpu_userq_create_start(queue);
734 r = uq_funcs->mqd_create(queue, &args->in);
735 if (r) {
736 drm_file_err(uq_mgr->file, "Failed to create Queue\n");
737 goto clean_doorbell_bo;
738 }
739
740 /* Update VM owner at userq submit-time for page-fault attribution. */
741 amdgpu_vm_set_task_info(&fpriv->vm);
742
743 r = xa_insert_irq(&adev->userq_doorbell_xa, index, queue,
744 GFP_KERNEL);
745 if (r)
746 goto clean_mqd;
747
748 amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
749
750 /* don't map the queue if scheduling is halted */
751 if (!adev->userq_halt_for_enforce_isolation ||
752 ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
753 (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
754 /* Serialize the map against an in-progress GPU reset (MES is
755 * unresponsive during recovery), matching amdgpu_userq_detach_doorbell().
756 */
757 down_read(&adev->reset_domain->sem);
758 r = amdgpu_userq_map_helper(queue);
759 up_read(&adev->reset_domain->sem);
760 if (r) {
761 drm_file_err(uq_mgr->file, "Failed to map Queue\n");
762 trace_amdgpu_userq_create_end(queue, r);
763 mutex_unlock(&uq_mgr->userq_mutex);
764 goto erase_doorbell;
765 }
766 }
767
768 atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
769 mutex_unlock(&uq_mgr->userq_mutex);
770
771 r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
772 XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
773 GFP_KERNEL);
774 if (r) {
775 /*
776 * This drops the last reference which should take care of
777 * all cleanup.
778 */
779 trace_amdgpu_userq_create_end(queue, r);
780 amdgpu_userq_put(queue);
781 return r;
782 }
783
784 amdgpu_debugfs_userq_init(filp, queue, qid);
785 trace_amdgpu_userq_create_end(queue, 0);
786 args->out.queue_id = qid;
787 return 0;
788
789 erase_doorbell:
790 xa_erase_irq(&adev->userq_doorbell_xa, index);
791 clean_mqd:
792 uq_funcs->mqd_destroy(queue);
793 clean_doorbell_bo:
794 amdgpu_bo_reserve(queue->db_obj.obj, true);
795 amdgpu_bo_unpin(queue->db_obj.obj);
796 amdgpu_bo_unreserve(queue->db_obj.obj);
797 amdgpu_bo_unref(&queue->db_obj.obj);
798 free_fence_drv:
799 amdgpu_userq_fence_driver_free(queue);
800 free_queue:
801 trace_amdgpu_userq_create_end(queue, r);
802 kfree(queue);
803 err_pm_runtime:
804 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
805 return r;
806 }
807
amdgpu_userq_input_args_validate(struct drm_device * dev,union drm_amdgpu_userq * args,struct drm_file * filp)808 static int amdgpu_userq_input_args_validate(struct drm_device *dev,
809 union drm_amdgpu_userq *args,
810 struct drm_file *filp)
811 {
812 struct amdgpu_device *adev = drm_to_adev(dev);
813
814 switch (args->in.op) {
815 case AMDGPU_USERQ_OP_CREATE:
816 if (args->in.flags & ~(AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK |
817 AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE))
818 return -EINVAL;
819 /* Usermode queues are only supported for GFX IP as of now */
820 if (args->in.ip_type != AMDGPU_HW_IP_GFX &&
821 args->in.ip_type != AMDGPU_HW_IP_DMA &&
822 args->in.ip_type != AMDGPU_HW_IP_COMPUTE) {
823 drm_file_err(filp, "Usermode queue doesn't support IP type %u\n",
824 args->in.ip_type);
825 return -EINVAL;
826 }
827
828 if ((args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE) &&
829 (args->in.ip_type != AMDGPU_HW_IP_GFX) &&
830 (args->in.ip_type != AMDGPU_HW_IP_COMPUTE) &&
831 !amdgpu_is_tmz(adev)) {
832 drm_file_err(filp, "Secure only supported on GFX/Compute queues\n");
833 return -EINVAL;
834 }
835
836 if (args->in.queue_va == AMDGPU_BO_INVALID_OFFSET ||
837 args->in.queue_va == 0 ||
838 args->in.queue_size == 0) {
839 drm_file_err(filp, "invalidate userq queue va or size\n");
840 return -EINVAL;
841 }
842
843 if (!is_power_of_2(args->in.queue_size)) {
844 drm_file_err(filp, "Queue size must be a power of 2\n");
845 return -EINVAL;
846 }
847
848 if (args->in.queue_size < AMDGPU_GPU_PAGE_SIZE) {
849 drm_file_err(filp, "Queue size smaller than AMDGPU_GPU_PAGE_SIZE\n");
850 return -EINVAL;
851 }
852
853 if (!args->in.wptr_va || !args->in.rptr_va) {
854 drm_file_err(filp, "invalidate userq queue rptr or wptr\n");
855 return -EINVAL;
856 }
857
858 if (!IS_ALIGNED(args->in.wptr_va, sizeof(u64)) ||
859 !IS_ALIGNED(args->in.rptr_va, sizeof(u64))) {
860 drm_file_err(filp, "user queue rptr or wptr is not 8-byte aligned\n");
861 return -EINVAL;
862 }
863 break;
864 case AMDGPU_USERQ_OP_FREE:
865 if (args->in.ip_type ||
866 args->in.doorbell_handle ||
867 args->in.doorbell_offset ||
868 args->in.flags ||
869 args->in.queue_va ||
870 args->in.queue_size ||
871 args->in.rptr_va ||
872 args->in.wptr_va ||
873 args->in.mqd ||
874 args->in.mqd_size)
875 return -EINVAL;
876 break;
877 default:
878 return -EINVAL;
879 }
880
881 return 0;
882 }
883
amdgpu_userq_enabled(struct drm_device * dev)884 bool amdgpu_userq_enabled(struct drm_device *dev)
885 {
886 struct amdgpu_device *adev = drm_to_adev(dev);
887 int i;
888
889 for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {
890 if (adev->userq_funcs[i])
891 return true;
892 }
893
894 return false;
895 }
896
amdgpu_userq_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)897 int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
898 struct drm_file *filp)
899 {
900 union drm_amdgpu_userq *args = data;
901 struct amdgpu_fpriv *fpriv = filp->driver_priv;
902 struct amdgpu_usermode_queue *queue;
903 int r = 0;
904
905 if (!amdgpu_userq_enabled(dev))
906 return -ENOTSUPP;
907
908 if (amdgpu_userq_input_args_validate(dev, args, filp) < 0)
909 return -EINVAL;
910
911 switch (args->in.op) {
912 case AMDGPU_USERQ_OP_CREATE:
913 r = amdgpu_userq_create(filp, args);
914 if (r)
915 drm_file_err(filp, "Failed to create usermode queue\n");
916 break;
917
918 case AMDGPU_USERQ_OP_FREE: {
919 xa_lock(&fpriv->userq_mgr.userq_xa);
920 queue = __xa_erase(&fpriv->userq_mgr.userq_xa, args->in.queue_id);
921 xa_unlock(&fpriv->userq_mgr.userq_xa);
922 if (!queue)
923 return -ENOENT;
924
925 amdgpu_userq_put(queue);
926 break;
927 }
928
929 default:
930 drm_dbg_driver(dev, "Invalid user queue op specified: %d\n", args->in.op);
931 return -EINVAL;
932 }
933
934 return r;
935 }
936
937 static int
amdgpu_userq_restore_all(struct amdgpu_userq_mgr * uq_mgr)938 amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
939 {
940 struct amdgpu_usermode_queue *queue;
941 unsigned long queue_id;
942 int ret = 0, r;
943
944 mutex_lock(&uq_mgr->userq_mutex);
945 /* Resume all the queues for this process */
946 xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
947
948 if (!amdgpu_userq_buffer_vas_mapped(queue)) {
949 drm_file_err(uq_mgr->file,
950 "trying restore queue without va mapping\n");
951 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_INVALID_VA);
952 queue->state = AMDGPU_USERQ_STATE_INVALID_VA;
953 continue;
954 }
955
956 r = amdgpu_userq_map_helper(queue);
957 if (r)
958 ret = r;
959 }
960 mutex_unlock(&uq_mgr->userq_mutex);
961
962 if (ret)
963 drm_file_err(uq_mgr->file,
964 "Failed to map all the queues, restore failed ret=%d\n", ret);
965 return ret;
966 }
967
amdgpu_userq_validate_vm(void * param,struct amdgpu_bo * bo)968 static int amdgpu_userq_validate_vm(void *param, struct amdgpu_bo *bo)
969 {
970 struct ttm_operation_ctx ctx = { false, false };
971
972 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
973 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
974 }
975
976 /* Handle all BOs on the invalidated list, validate them and update the PTs */
977 static int
amdgpu_userq_bo_validate(struct amdgpu_device * adev,struct drm_exec * exec,struct amdgpu_vm * vm)978 amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
979 struct amdgpu_vm *vm)
980 {
981 struct ttm_operation_ctx ctx = { false, false };
982 struct amdgpu_bo_va *bo_va;
983 struct amdgpu_bo *bo, *tmp;
984 int ret;
985
986 spin_lock(&vm->individual_lock);
987 while (!list_empty(&vm->individual.evicted)) {
988 bo_va = list_first_entry(&vm->individual.evicted,
989 struct amdgpu_bo_va,
990 base.vm_status);
991 bo = bo_va->base.bo;
992 tmp = amdgpu_bo_ref(bo);
993 spin_unlock(&vm->individual_lock);
994
995 ret = drm_exec_prepare_obj(exec, &bo->tbo.base,
996 TTM_NUM_MOVE_FENCES + 1);
997 amdgpu_bo_unref(&tmp);
998 if (unlikely(ret))
999 return ret;
1000
1001 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1002 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1003 if (ret)
1004 return ret;
1005
1006 /* This moves the bo_va to the idle list */
1007 ret = amdgpu_vm_bo_update(adev, bo_va, false);
1008 if (ret)
1009 return ret;
1010
1011 spin_lock(&vm->individual_lock);
1012 }
1013 spin_unlock(&vm->individual_lock);
1014
1015 return 0;
1016 }
1017
1018 /* Make sure the whole VM is ready to be used */
1019 static int
amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr * uq_mgr)1020 amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr)
1021 {
1022 struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
1023 bool invalidated = false, new_addition = false;
1024 struct ttm_operation_ctx ctx = { true, false };
1025 struct amdgpu_device *adev = uq_mgr->adev;
1026 struct amdgpu_hmm_range *range;
1027 struct amdgpu_vm *vm = &fpriv->vm;
1028 unsigned long key, tmp_key;
1029 struct amdgpu_bo_va *bo_va;
1030 struct amdgpu_usermode_queue *queue;
1031 struct amdgpu_bo *bo;
1032 struct drm_exec exec;
1033 struct xarray xa;
1034 int ret;
1035
1036 xa_init(&xa);
1037
1038 retry_lock:
1039 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
1040 drm_exec_until_all_locked(&exec) {
1041 ret = amdgpu_vm_lock_pd(vm, &exec, 1);
1042 drm_exec_retry_on_contention(&exec);
1043 if (unlikely(ret))
1044 goto unlock_all;
1045
1046 ret = amdgpu_vm_lock_individual(vm, &exec, TTM_NUM_MOVE_FENCES + 1);
1047 drm_exec_retry_on_contention(&exec);
1048 if (unlikely(ret))
1049 goto unlock_all;
1050
1051 /* This validates PDs, PTs and per VM BOs */
1052 ret = amdgpu_vm_validate(adev, vm, NULL,
1053 amdgpu_userq_validate_vm,
1054 NULL);
1055 if (unlikely(ret))
1056 goto unlock_all;
1057
1058 /* This locks and validates the remaining evicted BOs */
1059 ret = amdgpu_userq_bo_validate(adev, &exec, vm);
1060 drm_exec_retry_on_contention(&exec);
1061 if (unlikely(ret))
1062 goto unlock_all;
1063
1064 /*
1065 * WPTR BOs are VM-mapped, but each BO has its own reservation
1066 * object. Lock them into this drm_exec ww context so the later
1067 * amdgpu_bo_gpu_offset() reads are done with the BO resv locked.
1068 */
1069 xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) {
1070 struct ttm_operation_ctx wptr_ctx = { false, false };
1071
1072 bo = queue->wptr_obj.obj;
1073 if (!bo)
1074 continue;
1075
1076 ret = drm_exec_prepare_obj(&exec, &bo->tbo.base,
1077 TTM_NUM_MOVE_FENCES + 1);
1078 drm_exec_retry_on_contention(&exec);
1079 if (unlikely(ret))
1080 goto unlock_all;
1081
1082 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1083 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &wptr_ctx);
1084 if (unlikely(ret))
1085 goto unlock_all;
1086 }
1087 }
1088
1089 if (invalidated) {
1090 xa_for_each(&xa, tmp_key, range) {
1091 bo = range->bo;
1092 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
1093 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1094 if (ret)
1095 goto unlock_all;
1096
1097 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
1098
1099 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
1100 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1101 if (ret)
1102 goto unlock_all;
1103 }
1104 invalidated = false;
1105 }
1106
1107 ret = amdgpu_vm_handle_moved(adev, vm, NULL);
1108 if (ret)
1109 goto unlock_all;
1110
1111 /*
1112 * PRT/sparse mappings are kept off the vm_bo state lists, so
1113 * amdgpu_vm_handle_moved() does not touch them. Refresh their PTEs
1114 * explicitly here (as the CS path does) so sparse mappings survive a
1115 * VRAM-lost reset.
1116 */
1117 ret = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
1118 if (ret)
1119 goto unlock_all;
1120
1121 key = 0;
1122 /* Validate User Ptr BOs */
1123 list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
1124 bo = bo_va->base.bo;
1125 if (!bo)
1126 continue;
1127
1128 if (!amdgpu_ttm_tt_is_userptr(bo->tbo.ttm))
1129 continue;
1130
1131 range = xa_load(&xa, key);
1132 if (range && range->bo != bo) {
1133 xa_erase(&xa, key);
1134 amdgpu_hmm_range_free(range);
1135 range = NULL;
1136 }
1137
1138 if (!range) {
1139 range = amdgpu_hmm_range_alloc(bo);
1140 if (!range) {
1141 ret = -ENOMEM;
1142 goto unlock_all;
1143 }
1144
1145 xa_store(&xa, key, range, GFP_KERNEL);
1146 new_addition = true;
1147 }
1148 key++;
1149 }
1150
1151 if (new_addition) {
1152 drm_exec_fini(&exec);
1153 xa_for_each(&xa, tmp_key, range) {
1154 if (!range)
1155 continue;
1156 bo = range->bo;
1157 ret = amdgpu_ttm_tt_get_user_pages(bo, range);
1158 if (ret)
1159 goto free_ranges;
1160 }
1161
1162 invalidated = true;
1163 new_addition = false;
1164 goto retry_lock;
1165 }
1166
1167 ret = amdgpu_vm_update_pdes(adev, vm, false);
1168 if (ret)
1169 goto unlock_all;
1170
1171 /*
1172 * We need to wait for all VM updates to finish before restarting the
1173 * queues. Using the idle list like that is now ok since everything is
1174 * locked in place.
1175 */
1176 list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
1177 dma_fence_wait(bo_va->last_pt_update, false);
1178 /*
1179 * The PRT bo_va is kept off the state lists, so its PTE update fence
1180 * lands in prt_va->last_pt_update rather than vm->last_update; wait on
1181 * it explicitly (as the CS path syncs it) before restarting queues.
1182 */
1183 dma_fence_wait(fpriv->prt_va->last_pt_update, false);
1184 dma_fence_wait(vm->last_update, false);
1185
1186 xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) {
1187 bo = queue->wptr_obj.obj;
1188 if (!bo) {
1189 ret = -EINVAL;
1190 goto unlock_all;
1191 }
1192
1193 ret = amdgpu_ttm_alloc_gart(&bo->tbo);
1194 if (unlikely(ret)) {
1195 drm_file_err(uq_mgr->file,
1196 "failed to bind wptr bo to gart on resume, qid=%lu ret=%d\n",
1197 tmp_key, ret);
1198 goto unlock_all;
1199 }
1200
1201 queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(bo);
1202 }
1203
1204 ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec);
1205 if (ret) {
1206 drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n");
1207 goto unlock_all;
1208 }
1209
1210 ret = amdgpu_userq_restore_all(uq_mgr);
1211
1212 unlock_all:
1213 drm_exec_fini(&exec);
1214 free_ranges:
1215 xa_for_each(&xa, tmp_key, range) {
1216 if (!range)
1217 continue;
1218 bo = range->bo;
1219 amdgpu_hmm_range_free(range);
1220 }
1221 xa_destroy(&xa);
1222 return ret;
1223 }
1224
amdgpu_userq_restore_worker(struct work_struct * work)1225 static void amdgpu_userq_restore_worker(struct work_struct *work)
1226 {
1227 struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work);
1228 struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
1229 struct dma_fence *ev_fence;
1230 int ret;
1231
1232 ev_fence = amdgpu_evf_mgr_get_fence(&fpriv->evf_mgr);
1233 if (!dma_fence_is_signaled(ev_fence))
1234 goto put_fence;
1235
1236 ret = amdgpu_userq_vm_validate_and_restore_queue(uq_mgr);
1237 if (ret) {
1238 drm_file_err(uq_mgr->file, "Failed to validate BOs to restore ret=%d\n", ret);
1239 goto put_fence;
1240 }
1241
1242 put_fence:
1243 dma_fence_put(ev_fence);
1244 }
1245
amdgpu_userq_process_reset_irq(struct amdgpu_device * adev,u32 pasid,u32 doorbell_offset)1246 void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev,
1247 u32 pasid, u32 doorbell_offset)
1248 {
1249 struct xarray *xa = &adev->userq_doorbell_xa;
1250 struct amdgpu_usermode_queue *queue;
1251 unsigned long flags, idx;
1252
1253 xa_lock_irqsave(xa, flags);
1254 xa_for_each(xa, idx, queue) {
1255 if (queue->vm && queue->vm->pasid == pasid &&
1256 queue->doorbell_offset == doorbell_offset) {
1257 amdgpu_userq_start_hang_detect_work(queue);
1258 break;
1259 }
1260 }
1261 xa_unlock_irqrestore(xa, flags);
1262 }
1263
1264 static int
amdgpu_userq_evict_all(struct amdgpu_userq_mgr * uq_mgr)1265 amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
1266 {
1267 struct amdgpu_usermode_queue *queue;
1268 unsigned long queue_id;
1269 int ret = 0, r;
1270
1271 /* Try to unmap all the queues in this process ctx */
1272 xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
1273 r = amdgpu_userq_unmap_helper(queue);
1274 if (r)
1275 ret = r;
1276 }
1277
1278 if (ret) {
1279 drm_file_err(uq_mgr->file,
1280 "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
1281 amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
1282 &uq_mgr->reset_work);
1283 flush_work(&uq_mgr->reset_work);
1284 }
1285 return ret;
1286 }
1287
1288 void
amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr * uq_mgr)1289 amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
1290 {
1291 struct amdgpu_usermode_queue *queue;
1292 unsigned long queue_id;
1293
1294 xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
1295 struct dma_fence *f = queue->last_fence;
1296
1297 if (!f)
1298 continue;
1299
1300 dma_fence_wait(f, false);
1301 }
1302 }
1303
1304 void
amdgpu_userq_evict(struct amdgpu_userq_mgr * uq_mgr)1305 amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr)
1306 {
1307 amdgpu_userq_evict_all(uq_mgr);
1308 }
1309
amdgpu_userq_mgr_init(struct amdgpu_userq_mgr * userq_mgr,struct drm_file * file_priv,struct amdgpu_device * adev)1310 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
1311 struct amdgpu_device *adev)
1312 {
1313 mutex_init(&userq_mgr->userq_mutex);
1314 xa_init_flags(&userq_mgr->userq_xa, XA_FLAGS_ALLOC);
1315 userq_mgr->adev = adev;
1316 userq_mgr->file = file_priv;
1317 userq_mgr->proc_ctx_allocated = false;
1318 mutex_init(&userq_mgr->proc_ctx_lock);
1319
1320 INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
1321 INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
1322 return 0;
1323 }
1324
amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device * adev)1325 void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev)
1326 {
1327 struct xarray *xa = &adev->userq_doorbell_xa;
1328 struct amdgpu_usermode_queue *queue;
1329 unsigned long flags, queue_id;
1330
1331 xa_lock_irqsave(xa, flags);
1332 xa_for_each(xa, queue_id, queue) {
1333 cancel_delayed_work(&queue->hang_detect_work);
1334 cancel_work(&queue->userq_mgr->reset_work);
1335 }
1336 xa_unlock_irqrestore(xa, flags);
1337 }
1338
amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr * userq_mgr)1339 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
1340 {
1341 cancel_delayed_work_sync(&userq_mgr->resume_work);
1342 }
1343
amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr * userq_mgr)1344 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
1345 {
1346 struct amdgpu_mes *mes = &userq_mgr->adev->mes;
1347 struct amdgpu_usermode_queue *queue;
1348 unsigned long queue_id = 0;
1349
1350 for (;;) {
1351 xa_lock(&userq_mgr->userq_xa);
1352 queue = xa_find(&userq_mgr->userq_xa, &queue_id, ULONG_MAX,
1353 XA_PRESENT);
1354 if (queue)
1355 __xa_erase(&userq_mgr->userq_xa, queue_id);
1356 xa_unlock(&userq_mgr->userq_xa);
1357
1358 if (!queue)
1359 break;
1360
1361 amdgpu_userq_put(queue);
1362 }
1363
1364 xa_destroy(&userq_mgr->userq_xa);
1365
1366 /*
1367 * Drain any in-flight reset_work. By this point all queues are freed
1368 * and userq_count is 0, so if reset_work starts now it exits early.
1369 * We still need to wait in case it was already executing gpu_recover.
1370 */
1371 cancel_work_sync(&userq_mgr->reset_work);
1372
1373 if (userq_mgr->proc_ctx_allocated) {
1374 amdgpu_mes_free_proc_ctx_index(mes, userq_mgr->proc_ctx_array_index);
1375 userq_mgr->proc_ctx_allocated = false;
1376 }
1377 amdgpu_bo_free_kernel(&userq_mgr->proc_ctx_obj.obj,
1378 &userq_mgr->proc_ctx_obj.gpu_addr,
1379 &userq_mgr->proc_ctx_obj.cpu_ptr);
1380
1381 mutex_destroy(&userq_mgr->proc_ctx_lock);
1382 mutex_destroy(&userq_mgr->userq_mutex);
1383 }
1384
amdgpu_userq_suspend(struct amdgpu_device * adev)1385 int amdgpu_userq_suspend(struct amdgpu_device *adev)
1386 {
1387 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1388 struct amdgpu_usermode_queue *queue;
1389 struct amdgpu_userq_mgr *uqm;
1390 unsigned long queue_id;
1391 int r;
1392
1393 if (!ip_mask)
1394 return 0;
1395
1396 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1397 uqm = queue->userq_mgr;
1398 cancel_delayed_work_sync(&uqm->resume_work);
1399 guard(mutex)(&uqm->userq_mutex);
1400 if (adev->in_s0ix)
1401 r = amdgpu_userq_preempt_helper(queue);
1402 else
1403 r = amdgpu_userq_unmap_helper(queue);
1404 if (r)
1405 return r;
1406 }
1407 return 0;
1408 }
1409
amdgpu_userq_resume(struct amdgpu_device * adev)1410 int amdgpu_userq_resume(struct amdgpu_device *adev)
1411 {
1412 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1413 struct amdgpu_usermode_queue *queue;
1414 struct amdgpu_userq_mgr *uqm;
1415 unsigned long queue_id;
1416 int r;
1417
1418 if (!ip_mask)
1419 return 0;
1420
1421 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1422 uqm = queue->userq_mgr;
1423 guard(mutex)(&uqm->userq_mutex);
1424 if (adev->in_s0ix)
1425 r = amdgpu_userq_restore_helper(queue);
1426 else
1427 r = amdgpu_userq_map_helper(queue);
1428 if (r)
1429 return r;
1430 }
1431
1432 return 0;
1433 }
1434
amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device * adev,u32 idx)1435 int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
1436 u32 idx)
1437 {
1438 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1439 struct amdgpu_usermode_queue *queue;
1440 struct amdgpu_userq_mgr *uqm;
1441 unsigned long queue_id;
1442 int ret = 0, r;
1443
1444 /* only need to stop gfx/compute */
1445 if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE))))
1446 return 0;
1447
1448 if (adev->userq_halt_for_enforce_isolation)
1449 dev_warn(adev->dev, "userq scheduling already stopped!\n");
1450 adev->userq_halt_for_enforce_isolation = true;
1451 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1452 uqm = queue->userq_mgr;
1453 cancel_delayed_work_sync(&uqm->resume_work);
1454 mutex_lock(&uqm->userq_mutex);
1455 if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
1456 (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
1457 (queue->xcp_id == idx)) {
1458 r = amdgpu_userq_preempt_helper(queue);
1459 if (r)
1460 ret = r;
1461 }
1462 mutex_unlock(&uqm->userq_mutex);
1463 }
1464
1465 return ret;
1466 }
1467
amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device * adev,u32 idx)1468 int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,
1469 u32 idx)
1470 {
1471 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1472 struct amdgpu_usermode_queue *queue;
1473 struct amdgpu_userq_mgr *uqm;
1474 unsigned long queue_id;
1475 int ret = 0, r;
1476
1477 /* only need to stop gfx/compute */
1478 if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE))))
1479 return 0;
1480
1481 if (!adev->userq_halt_for_enforce_isolation)
1482 dev_warn(adev->dev, "userq scheduling already started!\n");
1483
1484 adev->userq_halt_for_enforce_isolation = false;
1485
1486 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1487 uqm = queue->userq_mgr;
1488 mutex_lock(&uqm->userq_mutex);
1489 if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
1490 (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
1491 (queue->xcp_id == idx)) {
1492 r = amdgpu_userq_restore_helper(queue);
1493 if (r)
1494 ret = r;
1495 }
1496 mutex_unlock(&uqm->userq_mutex);
1497 }
1498
1499 return ret;
1500 }
1501
amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device * adev,struct amdgpu_bo_va_mapping * mapping)1502 void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
1503 struct amdgpu_bo_va_mapping *mapping)
1504 {
1505 u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1506 struct amdgpu_bo_va *bo_va = mapping->bo_va;
1507 struct dma_resv *resv = bo_va->base.bo->tbo.base.resv;
1508
1509 if (!ip_mask)
1510 return;
1511
1512 /**
1513 * The userq VA mapping reservation should include the eviction fence.
1514 * Note: The eviction fence may be attached to different BOs and this
1515 * unmap is only for one kind of userq VAs, so at this point suppose
1516 * the eviction fence is always unsignaled.
1517 */
1518 dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
1519 false, MAX_SCHEDULE_TIMEOUT);
1520 }
1521
amdgpu_userq_pre_reset(struct amdgpu_device * adev)1522 void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
1523 {
1524 const struct amdgpu_userq_funcs *userq_funcs;
1525 struct amdgpu_usermode_queue *queue;
1526 unsigned long queue_id;
1527
1528 /* TODO: We probably need a new lock for the queue state */
1529 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1530 if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
1531 trace_amdgpu_userq_state_start(queue);
1532 userq_funcs = adev->userq_funcs[queue->queue_type];
1533 userq_funcs->unmap(queue);
1534 /* just mark all queues as hung at this point.
1535 * if unmap succeeds, we could map again
1536 * in amdgpu_userq_post_reset() if vram is not lost
1537 */
1538 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
1539 queue->state = AMDGPU_USERQ_STATE_HUNG;
1540 }
1541 /* Force-complete any pending fence regardless of queue state so
1542 * that eviction/suspend and queue teardown waiters don't block
1543 * forever on a fence that will never signal after the reset.
1544 */
1545 amdgpu_userq_fence_driver_force_completion(queue);
1546 }
1547 }
1548
amdgpu_userq_post_reset(struct amdgpu_device * adev,bool vram_lost)1549 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost)
1550 {
1551 /* if any queue state is AMDGPU_USERQ_STATE_UNMAPPED
1552 * at this point, we should be able to map it again
1553 * and continue if vram is not lost.
1554 */
1555 struct amdgpu_usermode_queue *queue;
1556 const struct amdgpu_userq_funcs *userq_funcs;
1557 unsigned long queue_id;
1558 int ret = 0, r;
1559
1560 xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1561 if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) {
1562 trace_amdgpu_userq_state_start(queue);
1563
1564 userq_funcs = adev->userq_funcs[queue->queue_type];
1565 /* Re-map queue */
1566 r = userq_funcs->map(queue);
1567 if (r) {
1568 dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id);
1569 ret = r;
1570 continue;
1571 }
1572 trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
1573 queue->state = AMDGPU_USERQ_STATE_MAPPED;
1574 }
1575 }
1576
1577 return ret;
1578 }
1579