xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c (revision ce5f2a25bdf6b2bccfc4ffb5e4c5f12e97eeace1)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright 2023 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <drm/drm_auth.h>
26 #include <drm/drm_exec.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/overflow.h>
29 #include <drm/drm_drv.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_reset.h"
33 #include "amdgpu_vm.h"
34 #include "amdgpu_userq.h"
35 #include "amdgpu_hmm.h"
36 #include "amdgpu_userq_fence.h"
37 #include "amdgpu_trace.h"
38 
39 u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
40 {
41 	int i;
42 	u32 userq_ip_mask = 0;
43 
44 	for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {
45 		if (adev->userq_funcs[i])
46 			userq_ip_mask |= (1 << i);
47 	}
48 
49 	return userq_ip_mask;
50 }
51 
52 static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
53 				enum amdgpu_ring_type ring_type, int reset_type)
54 {
55 
56 	if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX)
57 		return false;
58 
59 	switch (ring_type) {
60 	case AMDGPU_RING_TYPE_GFX:
61 		if (adev->gfx.gfx_supported_reset & reset_type)
62 			return true;
63 		break;
64 	case AMDGPU_RING_TYPE_COMPUTE:
65 		if (adev->gfx.compute_supported_reset & reset_type)
66 			return true;
67 		break;
68 	case AMDGPU_RING_TYPE_SDMA:
69 		if (adev->sdma.supported_reset & reset_type)
70 			return true;
71 		break;
72 	case AMDGPU_RING_TYPE_VCN_DEC:
73 	case AMDGPU_RING_TYPE_VCN_ENC:
74 		if (adev->vcn.supported_reset & reset_type)
75 			return true;
76 		break;
77 	case AMDGPU_RING_TYPE_VCN_JPEG:
78 		if (adev->jpeg.supported_reset & reset_type)
79 			return true;
80 		break;
81 	default:
82 		break;
83 	}
84 	return false;
85 }
86 
87 static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
88 {
89 	struct amdgpu_userq_mgr *uq_mgr =
90 		container_of(work, struct amdgpu_userq_mgr,
91 			     reset_work);
92 	struct amdgpu_device *adev = uq_mgr->adev;
93 	struct amdgpu_reset_context reset_context;
94 
95 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
96 		dev_err(adev->dev, "userq reset disabled by debug mask\n");
97 		return;
98 	}
99 
100 	/*
101 	 * If GPU recovery feature is disabled system-wide,
102 	 * skip all reset detection logic
103 	 */
104 	if (!amdgpu_gpu_recovery)
105 		return;
106 
107 	memset(&reset_context, 0, sizeof(reset_context));
108 
109 	reset_context.method = AMD_RESET_METHOD_NONE;
110 	reset_context.reset_req_dev = adev;
111 	reset_context.src = AMDGPU_RESET_SRC_USERQ;
112 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
113 	/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
114 
115 	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
116 }
117 
118 static void amdgpu_userq_hang_detect_work(struct work_struct *work)
119 {
120 	struct amdgpu_usermode_queue *queue =
121 		container_of(work, struct amdgpu_usermode_queue,
122 			     hang_detect_work.work);
123 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
124 	struct amdgpu_device *adev = uq_mgr->adev;
125 	const struct amdgpu_userq_funcs *userq_funcs =
126 		adev->userq_funcs[queue->queue_type];
127 	struct drm_wedge_task_info *info = NULL;
128 	struct amdgpu_task_info *ti = NULL;
129 	bool gpu_reset = false;
130 
131 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
132 		dev_err(adev->dev, "userq reset disabled by debug mask\n");
133 		return;
134 	}
135 
136 	/*
137 	 * If GPU recovery feature is disabled system-wide,
138 	 * skip all reset detection logic
139 	 */
140 	if (!amdgpu_gpu_recovery)
141 		return;
142 
143 	if (queue->vm && queue->vm->pasid) {
144 		ti = amdgpu_vm_get_task_info_pasid(adev, queue->vm->pasid);
145 		if (ti) {
146 			amdgpu_vm_print_task_info(adev, ti);
147 			info = &ti->task;
148 		}
149 	}
150 
151 	if (amdgpu_userq_is_reset_type_supported(adev, queue->queue_type,
152 						 AMDGPU_RESET_TYPE_PER_QUEUE)) {
153 		int r;
154 
155 		if (queue->queue_type == AMDGPU_HW_IP_COMPUTE)
156 			r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL,
157 							 queue, NULL, NULL);
158 		else
159 			r = userq_funcs->reset(queue);
160 		if (r) {
161 			gpu_reset = true;
162 		} else {
163 			atomic_inc(&adev->gpu_reset_counter);
164 			amdgpu_userq_fence_driver_force_completion(queue);
165 			drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);
166 		}
167 	} else {
168 		gpu_reset = true;
169 	}
170 	amdgpu_vm_put_task_info(ti);
171 
172 	/*
173 	 * Don't schedule the work here! Scheduling or queue work from one reset
174 	 * handler to another is illegal if you don't take extra precautions!
175 	 */
176 	if (gpu_reset)
177 		amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
178 }
179 
180 /*
181  * Start hang detection for a user queue fence. A delayed work will be scheduled
182  * to reset the queues when the fence doesn't signal in time.
183  */
184 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
185 {
186 	struct amdgpu_device *adev;
187 	unsigned long timeout_jiffies;
188 
189 	adev = queue->userq_mgr->adev;
190 	/* Determine timeout based on queue type */
191 	switch (queue->queue_type) {
192 	case AMDGPU_RING_TYPE_GFX:
193 		timeout_jiffies = adev->gfx_timeout;
194 		break;
195 	case AMDGPU_RING_TYPE_COMPUTE:
196 		timeout_jiffies = adev->compute_timeout;
197 		break;
198 	case AMDGPU_RING_TYPE_SDMA:
199 		timeout_jiffies = adev->sdma_timeout;
200 		break;
201 	default:
202 		timeout_jiffies = adev->gfx_timeout;
203 		break;
204 	}
205 
206 	queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
207 			   timeout_jiffies);
208 }
209 
210 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
211 {
212 	struct xarray *xa = &adev->userq_doorbell_xa;
213 	struct amdgpu_usermode_queue *queue;
214 	unsigned long flags;
215 	int r;
216 
217 	xa_lock_irqsave(xa, flags);
218 	queue = xa_load(xa, doorbell);
219 	if (queue) {
220 		r = amdgpu_userq_fence_driver_process(queue->fence_drv);
221 		/*
222 		 * We are in interrupt context here, this *can't* wait for
223 		 * reset work to finish.
224 		 */
225 		if (r >= 0)
226 			cancel_delayed_work(&queue->hang_detect_work);
227 
228 		/* Restart the timer when there are still fences pending */
229 		if (r == 1)
230 			amdgpu_userq_start_hang_detect_work(queue);
231 	}
232 	xa_unlock_irqrestore(xa, flags);
233 }
234 
235 int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
236 				   struct amdgpu_usermode_queue *queue,
237 				   u64 addr, u64 expected_size,
238 				   u64 *va_out)
239 {
240 	struct amdgpu_bo_va_mapping *va_map;
241 	struct amdgpu_vm *vm = queue->vm;
242 	u64 start_addr;
243 	u64 end_addr;
244 
245 	/* Caller must hold vm->root.bo reservation */
246 	dma_resv_assert_held(queue->vm->root.bo->tbo.base.resv);
247 
248 	if (!expected_size)
249 		return -EINVAL;
250 
251 	start_addr = addr & AMDGPU_GMC_HOLE_MASK;
252 	if (check_add_overflow(start_addr, expected_size - 1, &end_addr))
253 		return -EINVAL;
254 
255 	va_map = amdgpu_vm_bo_lookup_mapping(vm, start_addr);
256 	if (!va_map || !va_map->bo_va->base.bo)
257 		return -EINVAL;
258 
259 	/* Lookup guarantees start_addr is mapped; ensure full span is covered. */
260 	if ((end_addr >> AMDGPU_GPU_PAGE_SHIFT) <= va_map->last) {
261 		va_map->bo_va->userq_va_mapped = true;
262 		*va_out = start_addr;
263 		return 0;
264 	}
265 
266 	return -EINVAL;
267 }
268 
269 static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
270 {
271 	struct amdgpu_bo_va_mapping *mapping;
272 	bool r;
273 
274 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
275 
276 	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
277 	if (!IS_ERR_OR_NULL(mapping) && mapping->bo_va->userq_va_mapped)
278 		r = true;
279 	else
280 		r = false;
281 
282 	return r;
283 }
284 
285 static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue)
286 {
287 	int i;
288 	bool mapped;
289 
290 	for (i = 0; i < ARRAY_SIZE(queue->userq_vas.va_array); i++) {
291 		if (!queue->userq_vas.va_array[i])
292 			continue;
293 
294 		mapped = amdgpu_userq_buffer_va_mapped(queue->vm,
295 						   queue->userq_vas.va_array[i]);
296 		dev_dbg(queue->userq_mgr->adev->dev,
297 			"validate the userq mapping:%p va:%llx r:%d\n",
298 			queue, queue->userq_vas.va_array[i], mapped);
299 
300 		if (!mapped)
301 			return false;
302 	}
303 
304 	return true;
305 }
306 
307 
308 
309 static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
310 {
311 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
312 	struct amdgpu_device *adev = uq_mgr->adev;
313 	const struct amdgpu_userq_funcs *userq_funcs =
314 		adev->userq_funcs[queue->queue_type];
315 	int r;
316 
317 	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
318 		trace_amdgpu_userq_state_start(queue);
319 
320 		r = userq_funcs->preempt(queue);
321 		if (r) {
322 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
323 			queue->state = AMDGPU_USERQ_STATE_HUNG;
324 			return r;
325 		} else {
326 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_PREEMPTED);
327 			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
328 		}
329 	}
330 	return 0;
331 }
332 
333 static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
334 {
335 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
336 	struct amdgpu_device *adev = uq_mgr->adev;
337 	const struct amdgpu_userq_funcs *userq_funcs =
338 		adev->userq_funcs[queue->queue_type];
339 	int r = 0;
340 
341 	if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {
342 		trace_amdgpu_userq_state_start(queue);
343 
344 		r = userq_funcs->restore(queue);
345 		if (r) {
346 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
347 			queue->state = AMDGPU_USERQ_STATE_HUNG;
348 		} else {
349 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
350 			queue->state = AMDGPU_USERQ_STATE_MAPPED;
351 		}
352 	}
353 
354 	return r;
355 }
356 
357 static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
358 {
359 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
360 	struct amdgpu_device *adev = uq_mgr->adev;
361 	const struct amdgpu_userq_funcs *userq_funcs =
362 		adev->userq_funcs[queue->queue_type];
363 	int r;
364 
365 	/* The reset changes the queue state and the MES is unusable meanwhile */
366 	guard(rwsem_read)(&adev->reset_domain->sem);
367 	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
368 	    (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
369 		trace_amdgpu_userq_state_start(queue);
370 
371 		r = userq_funcs->unmap(queue);
372 		if (r) {
373 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
374 			queue->state = AMDGPU_USERQ_STATE_HUNG;
375 			return r;
376 		} else {
377 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED);
378 			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
379 		}
380 	}
381 
382 	return 0;
383 }
384 
385 static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
386 {
387 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
388 	struct amdgpu_device *adev = uq_mgr->adev;
389 	const struct amdgpu_userq_funcs *userq_funcs =
390 		adev->userq_funcs[queue->queue_type];
391 	int r;
392 
393 	/* The reset changes the queue state and the MES is unusable meanwhile */
394 	guard(rwsem_read)(&adev->reset_domain->sem);
395 	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
396 		trace_amdgpu_userq_state_start(queue);
397 
398 		r = userq_funcs->map(queue);
399 		if (r) {
400 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
401 			queue->state = AMDGPU_USERQ_STATE_HUNG;
402 			return r;
403 		} else {
404 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
405 			queue->state = AMDGPU_USERQ_STATE_MAPPED;
406 		}
407 	}
408 
409 	return 0;
410 }
411 
412 static void amdgpu_userq_wait_for_last_fence(struct amdgpu_usermode_queue *queue)
413 {
414 	struct dma_fence *f = queue->last_fence;
415 
416 	if (!f)
417 		return;
418 
419 	dma_fence_wait(f, false);
420 }
421 
422 static void amdgpu_userq_detach_doorbell(struct amdgpu_usermode_queue *queue)
423 {
424 	struct amdgpu_device *adev = queue->userq_mgr->adev;
425 
426 	down_read(&adev->reset_domain->sem);
427 	xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index);
428 	up_read(&adev->reset_domain->sem);
429 }
430 
431 /**
432  * amdgpu_userq_ensure_ev_fence - ensure a valid, unsignaled eviction fence exists
433  * @uq_mgr: the usermode queue manager for this process
434  * @evf_mgr: the eviction fence manager to check and rearm
435  *
436  * Ensures that a valid and not yet signaled eviction fence is attached to the
437  * usermode queue before any queue operations proceed. If it is signalled, then
438  * rearm a new eviction fence.
439  */
440 void
441 amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
442 			     struct amdgpu_eviction_fence_mgr *evf_mgr)
443 {
444 	struct dma_fence *ev_fence;
445 
446 retry:
447 	/* Flush any pending resume work to create ev_fence */
448 	flush_delayed_work(&uq_mgr->resume_work);
449 
450 	mutex_lock(&uq_mgr->userq_mutex);
451 	ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
452 	if (dma_fence_is_signaled(ev_fence)) {
453 		dma_fence_put(ev_fence);
454 		mutex_unlock(&uq_mgr->userq_mutex);
455 		/*
456 		 * Looks like there was no pending resume work,
457 		 * add one now to create a valid eviction fence
458 		 */
459 		schedule_delayed_work(&uq_mgr->resume_work, 0);
460 		goto retry;
461 	}
462 	dma_fence_put(ev_fence);
463 }
464 
465 
466 
467 static int
468 amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,
469 				struct amdgpu_db_info *db_info,
470 				struct drm_file *filp,
471 				u64 *index)
472 {
473 	u64 doorbell_index;
474 	struct drm_gem_object *gobj;
475 	struct amdgpu_userq_obj *db_obj = db_info->db_obj;
476 	struct amdgpu_bo *abo;
477 	int r, db_size;
478 
479 	gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle);
480 	if (gobj == NULL) {
481 		drm_file_err(uq_mgr->file, "Can't find GEM object for doorbell\n");
482 		return -EINVAL;
483 	}
484 
485 	/*
486 	 * Pinning a regular BO into the doorbell domain would discard its
487 	 * contents, possibly those of a buffer shared by another client.
488 	 */
489 	abo = gem_to_amdgpu_bo(gobj);
490 	if (!(abo->preferred_domains & AMDGPU_GEM_DOMAIN_DOORBELL)) {
491 		drm_gem_object_put(gobj);
492 		return -EINVAL;
493 	}
494 
495 	db_obj->obj = amdgpu_bo_ref(abo);
496 	drm_gem_object_put(gobj);
497 
498 	r = amdgpu_bo_reserve(db_obj->obj, true);
499 	if (r) {
500 		drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");
501 		goto unref_bo;
502 	}
503 
504 	/* Pin the BO before generating the index, unpin in queue destroy */
505 	r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL);
506 	if (r) {
507 		drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");
508 		goto unresv_bo;
509 	}
510 
511 	switch (db_info->queue_type) {
512 	case AMDGPU_HW_IP_GFX:
513 	case AMDGPU_HW_IP_COMPUTE:
514 	case AMDGPU_HW_IP_DMA:
515 		db_size = sizeof(u64);
516 		break;
517 	default:
518 		drm_file_err(uq_mgr->file, "[Usermode queues] IP %d not support\n",
519 			     db_info->queue_type);
520 		r = -EINVAL;
521 		goto unpin_bo;
522 	}
523 
524 	/* Validate doorbell_offset is within the doorbell BO */
525 	if ((u64)db_info->doorbell_offset * db_size + db_size >
526 	    amdgpu_bo_size(db_obj->obj)) {
527 		r = -EINVAL;
528 		goto unpin_bo;
529 	}
530 
531 	doorbell_index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj,
532 						      db_info->doorbell_offset, db_size);
533 	drm_dbg_driver(adev_to_drm(uq_mgr->adev),
534 		       "[Usermode queues] doorbell index=%lld\n", doorbell_index);
535 	amdgpu_bo_unreserve(db_obj->obj);
536 	*index = doorbell_index;
537 	return 0;
538 
539 unpin_bo:
540 	amdgpu_bo_unpin(db_obj->obj);
541 unresv_bo:
542 	amdgpu_bo_unreserve(db_obj->obj);
543 unref_bo:
544 	amdgpu_bo_unref(&db_obj->obj);
545 	return r;
546 }
547 
548 static int
549 amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue)
550 {
551 	struct amdgpu_device *adev = uq_mgr->adev;
552 	const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];
553 	int r = 0;
554 
555 	trace_amdgpu_userq_destroy_start(queue);
556 
557 	cancel_delayed_work_sync(&uq_mgr->resume_work);
558 	/* Cancel before taking userq_mutex: cancel_delayed_work_sync() waits
559 	 * for any running instance, which itself takes userq_mutex.
560 	 */
561 	cancel_delayed_work_sync(&queue->hang_detect_work);
562 
563 	mutex_lock(&uq_mgr->userq_mutex);
564 	amdgpu_userq_wait_for_last_fence(queue);
565 
566 	amdgpu_userq_detach_doorbell(queue);
567 
568 #if defined(CONFIG_DEBUG_FS)
569 	debugfs_remove_recursive(queue->debugfs_queue);
570 #endif
571 	r = amdgpu_userq_unmap_helper(queue);
572 	atomic_dec(&uq_mgr->userq_count[queue->queue_type]);
573 	amdgpu_userq_fence_driver_free(queue);
574 	queue->fence_drv = NULL;
575 	mutex_unlock(&uq_mgr->userq_mutex);
576 
577 	/*
578 	 * A failed unmap means MES could not remove the hung queue and is now
579 	 * unresponsive.  Recover the GPU here so the wedged MES does not fail
580 	 * the next, unrelated queue submission and trigger a reset attributed
581 	 * to an innocent workload.
582 	 */
583 	if (r)
584 		queue_work(adev->reset_domain->wq, &uq_mgr->reset_work);
585 
586 	uq_funcs->mqd_destroy(queue);
587 	queue->userq_mgr = NULL;
588 
589 	amdgpu_bo_reserve(queue->db_obj.obj, true);
590 	amdgpu_bo_unpin(queue->db_obj.obj);
591 	amdgpu_bo_unreserve(queue->db_obj.obj);
592 	amdgpu_bo_unref(&queue->db_obj.obj);
593 
594 	trace_amdgpu_userq_destroy_end(queue, r);
595 	kfree(queue);
596 
597 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
598 
599 	return r;
600 }
601 
602 static void amdgpu_userq_kref_destroy(struct kref *kref)
603 {
604 	int r;
605 	struct amdgpu_usermode_queue *queue =
606 		container_of(kref, struct amdgpu_usermode_queue, refcount);
607 	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
608 
609 	r = amdgpu_userq_destroy(uq_mgr, queue);
610 	if (r)
611 		drm_file_err(uq_mgr->file, "Failed to destroy usermode queue %d\n", r);
612 }
613 
614 struct amdgpu_usermode_queue *amdgpu_userq_get(struct amdgpu_userq_mgr *uq_mgr, u32 qid)
615 {
616 	struct amdgpu_usermode_queue *queue;
617 
618 	xa_lock(&uq_mgr->userq_xa);
619 	queue = xa_load(&uq_mgr->userq_xa, qid);
620 	if (queue)
621 		kref_get(&queue->refcount);
622 	xa_unlock(&uq_mgr->userq_xa);
623 
624 	return queue;
625 }
626 
627 void amdgpu_userq_put(struct amdgpu_usermode_queue *queue)
628 {
629 	if (queue)
630 		kref_put(&queue->refcount, amdgpu_userq_kref_destroy);
631 }
632 
633 static int amdgpu_userq_priority_permit(struct drm_file *filp,
634 					int priority)
635 {
636 	if (priority < AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH)
637 		return 0;
638 
639 	if (capable(CAP_SYS_NICE))
640 		return 0;
641 
642 	if (drm_is_current_master(filp))
643 		return 0;
644 
645 	return -EACCES;
646 }
647 
648 static int
649 amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
650 {
651 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
652 	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
653 	struct amdgpu_device *adev = uq_mgr->adev;
654 	const struct amdgpu_userq_funcs *uq_funcs;
655 	struct amdgpu_usermode_queue *queue;
656 	struct amdgpu_db_info db_info;
657 	uint64_t index;
658 	int priority;
659 	u32 qid;
660 	int r;
661 
662 	priority =
663 		(args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK)
664 		>> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
665 	r = amdgpu_userq_priority_permit(filp, priority);
666 	if (r)
667 		return r;
668 
669 	r = pm_runtime_resume_and_get(adev_to_drm(adev)->dev);
670 	if (r < 0) {
671 		drm_file_err(uq_mgr->file, "pm_runtime_resume_and_get() failed for userqueue create\n");
672 		return r;
673 	}
674 
675 	uq_funcs = adev->userq_funcs[args->in.ip_type];
676 	if (!uq_funcs) {
677 		switch (args->in.ip_type) {
678 		case AMDGPU_HW_IP_GFX:
679 		case AMDGPU_HW_IP_COMPUTE:
680 			dev_warn_once(adev->dev,
681 				      "Usermode queues for GFX/COMPUTE is not supported by the fw "
682 				      "on this ASIC (me: %u, pfp: %u, mec: %u, mes: %u)\n",
683 				      adev->gfx.me_fw_version, adev->gfx.pfp_fw_version,
684 				      adev->gfx.mec_fw_version, adev->mes.fw_version[0]);
685 			break;
686 		case AMDGPU_HW_IP_DMA:
687 			dev_warn_once(adev->dev,
688 				      "Usermode queues for SDMA is not supported by the fw "
689 				      "on this ASIC (sdma: %u)\n",
690 				      adev->sdma.instance[0].fw_version);
691 			break;
692 		default:
693 			break;
694 		}
695 		r = -EINVAL;
696 		goto err_pm_runtime;
697 	}
698 
699 	queue = kzalloc_obj(struct amdgpu_usermode_queue);
700 	if (!queue) {
701 		r = -ENOMEM;
702 		goto err_pm_runtime;
703 	}
704 
705 	kref_init(&queue->refcount);
706 	queue->doorbell_handle = args->in.doorbell_handle;
707 	queue->queue_type = args->in.ip_type;
708 	queue->vm = &fpriv->vm;
709 	queue->priority = priority;
710 	queue->xcp_id = (fpriv->xcp_id != AMDGPU_XCP_NO_PARTITION) ?
711 				fpriv->xcp_id : 0;
712 	queue->userq_mgr = uq_mgr;
713 	INIT_DELAYED_WORK(&queue->hang_detect_work,
714 			  amdgpu_userq_hang_detect_work);
715 
716 	r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
717 	if (r)
718 		goto free_queue;
719 
720 	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
721 	mutex_init(&queue->fence_drv_lock);
722 	/* Make sure the queue can actually run with those virtual addresses. */
723 	r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
724 	if (r)
725 		goto free_fence_drv;
726 
727 	if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va,
728 					   args->in.queue_size,
729 					   &queue->userq_vas.va.queue_rb) ||
730 	    amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
731 					   sizeof(u64),
732 					   &queue->userq_vas.va.rptr) ||
733 	    amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
734 					   sizeof(u64),
735 					   &queue->userq_vas.va.wptr)) {
736 		r = -EINVAL;
737 		amdgpu_bo_unreserve(fpriv->vm.root.bo);
738 		goto free_fence_drv;
739 	}
740 	amdgpu_bo_unreserve(fpriv->vm.root.bo);
741 
742 	/* Convert relative doorbell offset into absolute doorbell index */
743 	db_info.queue_type = queue->queue_type;
744 	db_info.doorbell_handle = queue->doorbell_handle;
745 	db_info.db_obj = &queue->db_obj;
746 	db_info.doorbell_offset = args->in.doorbell_offset;
747 	r = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp, &index);
748 	if (r) {
749 		drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
750 		goto free_fence_drv;
751 	}
752 
753 	queue->doorbell_index = index;
754 	queue->doorbell_offset = (u32)args->in.doorbell_offset;
755 	trace_amdgpu_userq_create_start(queue);
756 	r = uq_funcs->mqd_create(queue, &args->in);
757 	if (r) {
758 		drm_file_err(uq_mgr->file, "Failed to create Queue\n");
759 		goto clean_doorbell_bo;
760 	}
761 
762 	/* Update VM owner at userq submit-time for page-fault attribution. */
763 	amdgpu_vm_set_task_info(&fpriv->vm);
764 
765 	r = xa_insert_irq(&adev->userq_doorbell_xa, index, queue,
766 			  GFP_KERNEL);
767 	if (r)
768 		goto clean_mqd;
769 
770 	amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
771 
772 	/* don't map the queue if scheduling is halted */
773 	if (!adev->userq_halt_for_enforce_isolation ||
774 	    ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
775 	     (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
776 		r = amdgpu_userq_map_helper(queue);
777 		if (r) {
778 			drm_file_err(uq_mgr->file, "Failed to map Queue\n");
779 			trace_amdgpu_userq_create_end(queue, r);
780 			mutex_unlock(&uq_mgr->userq_mutex);
781 			goto erase_doorbell;
782 		}
783 	}
784 
785 	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
786 	mutex_unlock(&uq_mgr->userq_mutex);
787 
788 	r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
789 		     XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
790 		     GFP_KERNEL);
791 	if (r) {
792 		/*
793 		 * This drops the last reference which should take care of
794 		 * all cleanup.
795 		 */
796 		trace_amdgpu_userq_create_end(queue, r);
797 		amdgpu_userq_put(queue);
798 		return r;
799 	}
800 
801 	amdgpu_debugfs_userq_init(filp, queue, qid);
802 	trace_amdgpu_userq_create_end(queue, 0);
803 	args->out.queue_id = qid;
804 	return 0;
805 
806 erase_doorbell:
807 	xa_erase_irq(&adev->userq_doorbell_xa, index);
808 clean_mqd:
809 	uq_funcs->mqd_destroy(queue);
810 clean_doorbell_bo:
811 	amdgpu_bo_reserve(queue->db_obj.obj, true);
812 	amdgpu_bo_unpin(queue->db_obj.obj);
813 	amdgpu_bo_unreserve(queue->db_obj.obj);
814 	amdgpu_bo_unref(&queue->db_obj.obj);
815 free_fence_drv:
816 	amdgpu_userq_fence_driver_free(queue);
817 free_queue:
818 	trace_amdgpu_userq_create_end(queue, r);
819 	kfree(queue);
820 err_pm_runtime:
821 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
822 	return r;
823 }
824 
825 static int amdgpu_userq_input_args_validate(struct drm_device *dev,
826 					union drm_amdgpu_userq *args,
827 					struct drm_file *filp)
828 {
829 	struct amdgpu_device *adev = drm_to_adev(dev);
830 
831 	switch (args->in.op) {
832 	case AMDGPU_USERQ_OP_CREATE:
833 		if (args->in.flags & ~(AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK |
834 				       AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE))
835 			return -EINVAL;
836 		/* Usermode queues are only supported for GFX IP as of now */
837 		if (args->in.ip_type != AMDGPU_HW_IP_GFX &&
838 		    args->in.ip_type != AMDGPU_HW_IP_DMA &&
839 		    args->in.ip_type != AMDGPU_HW_IP_COMPUTE) {
840 			drm_file_err(filp, "Usermode queue doesn't support IP type %u\n",
841 				     args->in.ip_type);
842 			return -EINVAL;
843 		}
844 
845 		if ((args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE) &&
846 		    (args->in.ip_type != AMDGPU_HW_IP_GFX) &&
847 		    (args->in.ip_type != AMDGPU_HW_IP_COMPUTE) &&
848 		    !amdgpu_is_tmz(adev)) {
849 			drm_file_err(filp, "Secure only supported on GFX/Compute queues\n");
850 			return -EINVAL;
851 		}
852 
853 		if (args->in.queue_va == AMDGPU_BO_INVALID_OFFSET ||
854 		    args->in.queue_va == 0 ||
855 		    args->in.queue_size == 0) {
856 			drm_file_err(filp, "invalidate userq queue va or size\n");
857 			return -EINVAL;
858 		}
859 
860 		if (!is_power_of_2(args->in.queue_size)) {
861 			drm_file_err(filp, "Queue size must be a power of 2\n");
862 			return -EINVAL;
863 		}
864 
865 		if (args->in.queue_size < AMDGPU_GPU_PAGE_SIZE) {
866 			drm_file_err(filp, "Queue size smaller than AMDGPU_GPU_PAGE_SIZE\n");
867 			return -EINVAL;
868 		}
869 
870 		if (!args->in.wptr_va || !args->in.rptr_va) {
871 			drm_file_err(filp, "invalidate userq queue rptr or wptr\n");
872 			return -EINVAL;
873 		}
874 
875 		if (!IS_ALIGNED(args->in.wptr_va, sizeof(u64)) ||
876 		    !IS_ALIGNED(args->in.rptr_va, sizeof(u64))) {
877 			drm_file_err(filp, "user queue rptr or wptr is not 8-byte aligned\n");
878 			return -EINVAL;
879 		}
880 		break;
881 	case AMDGPU_USERQ_OP_FREE:
882 		if (args->in.ip_type ||
883 		    args->in.doorbell_handle ||
884 		    args->in.doorbell_offset ||
885 		    args->in.flags ||
886 		    args->in.queue_va ||
887 		    args->in.queue_size ||
888 		    args->in.rptr_va ||
889 		    args->in.wptr_va ||
890 		    args->in.mqd ||
891 		    args->in.mqd_size)
892 			return -EINVAL;
893 		break;
894 	default:
895 		return -EINVAL;
896 	}
897 
898 	return 0;
899 }
900 
901 bool amdgpu_userq_enabled(struct drm_device *dev)
902 {
903 	struct amdgpu_device *adev = drm_to_adev(dev);
904 	int i;
905 
906 	for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {
907 		if (adev->userq_funcs[i])
908 			return true;
909 	}
910 
911 	return false;
912 }
913 
914 int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
915 		       struct drm_file *filp)
916 {
917 	union drm_amdgpu_userq *args = data;
918 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
919 	struct amdgpu_usermode_queue *queue;
920 	int r = 0;
921 
922 	if (!amdgpu_userq_enabled(dev))
923 		return -ENOTSUPP;
924 
925 	if (amdgpu_userq_input_args_validate(dev, args, filp) < 0)
926 		return -EINVAL;
927 
928 	switch (args->in.op) {
929 	case AMDGPU_USERQ_OP_CREATE:
930 		r = amdgpu_userq_create(filp, args);
931 		if (r)
932 			drm_file_err(filp, "Failed to create usermode queue\n");
933 		break;
934 
935 	case AMDGPU_USERQ_OP_FREE: {
936 		xa_lock(&fpriv->userq_mgr.userq_xa);
937 		queue = __xa_erase(&fpriv->userq_mgr.userq_xa, args->in.queue_id);
938 		xa_unlock(&fpriv->userq_mgr.userq_xa);
939 		if (!queue)
940 			return -ENOENT;
941 
942 		amdgpu_userq_put(queue);
943 		break;
944 	}
945 
946 	default:
947 		drm_dbg_driver(dev, "Invalid user queue op specified: %d\n", args->in.op);
948 		return -EINVAL;
949 	}
950 
951 	return r;
952 }
953 
954 static int
955 amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
956 {
957 	struct amdgpu_usermode_queue *queue;
958 	unsigned long queue_id;
959 	int ret = 0, r;
960 
961 	mutex_lock(&uq_mgr->userq_mutex);
962 	/* Resume all the queues for this process */
963 	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
964 
965 		if (!amdgpu_userq_buffer_vas_mapped(queue)) {
966 			drm_file_err(uq_mgr->file,
967 				     "trying restore queue without va mapping\n");
968 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_INVALID_VA);
969 			queue->state = AMDGPU_USERQ_STATE_INVALID_VA;
970 			continue;
971 		}
972 
973 		r = amdgpu_userq_map_helper(queue);
974 		if (r)
975 			ret = r;
976 	}
977 	mutex_unlock(&uq_mgr->userq_mutex);
978 
979 	if (ret)
980 		drm_file_err(uq_mgr->file,
981 			     "Failed to map all the queues, restore failed ret=%d\n", ret);
982 	return ret;
983 }
984 
985 static int amdgpu_userq_validate_vm(void *param, struct amdgpu_bo *bo)
986 {
987 	struct ttm_operation_ctx ctx = { false, false };
988 
989 	amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
990 	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
991 }
992 
993 /*
994  * Handle all individual BOs which were moved, lock and validate them and
995  * update the PTs. An evicted individual BO lands on the needs_update list,
996  * the evicted one only holds imported DMA-bufs of compute VMs.
997  */
998 static int
999 amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
1000 			 struct amdgpu_vm *vm)
1001 {
1002 	struct ttm_operation_ctx ctx = { false, false };
1003 	struct amdgpu_bo_va *bo_va;
1004 	struct amdgpu_bo *bo, *tmp;
1005 	int ret;
1006 
1007 	spin_lock(&vm->individual_lock);
1008 	while (!list_empty(&vm->individual.needs_update)) {
1009 		bo_va = list_first_entry(&vm->individual.needs_update,
1010 					 struct amdgpu_bo_va,
1011 					 base.vm_status);
1012 		bo = bo_va->base.bo;
1013 		tmp = amdgpu_bo_ref(bo);
1014 		spin_unlock(&vm->individual_lock);
1015 
1016 		ret = drm_exec_prepare_obj(exec, &bo->tbo.base,
1017 					   TTM_NUM_MOVE_FENCES + 1);
1018 		amdgpu_bo_unref(&tmp);
1019 		if (unlikely(ret))
1020 			return ret;
1021 
1022 		amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1023 		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1024 		if (ret)
1025 			return ret;
1026 
1027 		/* This moves the bo_va to the idle list */
1028 		ret = amdgpu_vm_bo_update(adev, bo_va, false);
1029 		if (ret)
1030 			return ret;
1031 
1032 		spin_lock(&vm->individual_lock);
1033 	}
1034 	spin_unlock(&vm->individual_lock);
1035 
1036 	return 0;
1037 }
1038 
1039 /* Make sure the whole VM is ready to be used */
1040 static int
1041 amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr)
1042 {
1043 	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
1044 	bool invalidated = false, new_addition = false;
1045 	struct ttm_operation_ctx ctx = { true, false };
1046 	struct amdgpu_device *adev = uq_mgr->adev;
1047 	struct amdgpu_hmm_range *range;
1048 	struct amdgpu_vm *vm = &fpriv->vm;
1049 	unsigned long key, tmp_key;
1050 	struct amdgpu_bo_va *bo_va;
1051 	struct amdgpu_usermode_queue *queue;
1052 	struct amdgpu_bo *bo;
1053 	struct drm_exec exec;
1054 	struct xarray xa;
1055 	int ret;
1056 
1057 	xa_init(&xa);
1058 
1059 retry_lock:
1060 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
1061 	drm_exec_until_all_locked(&exec) {
1062 		/*
1063 		 * Rearm adds one BOOKKEEP fence and validation may queue move fences
1064 		 * on the root PD BO, so reserve caller-side slots accordingly.
1065 		 */
1066 		ret = amdgpu_vm_lock_pd(vm, &exec, TTM_NUM_MOVE_FENCES + 1);
1067 		drm_exec_retry_on_contention(&exec);
1068 		if (unlikely(ret))
1069 			goto unlock_all;
1070 
1071 		ret = amdgpu_vm_lock_individual(vm, &exec, TTM_NUM_MOVE_FENCES + 1);
1072 		drm_exec_retry_on_contention(&exec);
1073 		if (unlikely(ret))
1074 			goto unlock_all;
1075 
1076 		/* This validates PDs, PTs and per VM BOs */
1077 		ret = amdgpu_vm_validate(adev, vm, NULL,
1078 					 amdgpu_userq_validate_vm,
1079 					 NULL);
1080 		if (unlikely(ret))
1081 			goto unlock_all;
1082 
1083 		/* This locks and validates the remaining evicted BOs */
1084 		ret = amdgpu_userq_bo_validate(adev, &exec, vm);
1085 		drm_exec_retry_on_contention(&exec);
1086 		if (unlikely(ret))
1087 			goto unlock_all;
1088 
1089 		/*
1090 		 * WPTR BOs are VM-mapped, but each BO has its own reservation
1091 		 * object. Lock them into this drm_exec ww context so the later
1092 		 * amdgpu_bo_gpu_offset() reads are done with the BO resv locked.
1093 		 */
1094 		xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) {
1095 			struct ttm_operation_ctx wptr_ctx = { false, false };
1096 
1097 			bo = queue->wptr_obj.obj;
1098 			if (!bo)
1099 				continue;
1100 
1101 			ret = drm_exec_prepare_obj(&exec, &bo->tbo.base,
1102 						   TTM_NUM_MOVE_FENCES + 1);
1103 			drm_exec_retry_on_contention(&exec);
1104 			if (unlikely(ret))
1105 				goto unlock_all;
1106 
1107 			amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1108 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &wptr_ctx);
1109 			if (unlikely(ret))
1110 				goto unlock_all;
1111 		}
1112 	}
1113 
1114 	if (invalidated) {
1115 		xa_for_each(&xa, tmp_key, range) {
1116 			bo = range->bo;
1117 			amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
1118 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1119 			if (ret)
1120 				goto unlock_all;
1121 
1122 			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
1123 
1124 			amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
1125 			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1126 			if (ret)
1127 				goto unlock_all;
1128 		}
1129 		invalidated = false;
1130 	}
1131 
1132 	ret = amdgpu_vm_handle_moved(adev, vm, NULL);
1133 	if (ret)
1134 		goto unlock_all;
1135 
1136 	/*
1137 	 * PRT/sparse mappings are kept off the vm_bo state lists, so
1138 	 * amdgpu_vm_handle_moved() does not touch them. Refresh their PTEs
1139 	 * explicitly here (as the CS path does) so sparse mappings survive a
1140 	 * VRAM-lost reset.
1141 	 */
1142 	ret = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
1143 	if (ret)
1144 		goto unlock_all;
1145 
1146 	key = 0;
1147 	/* Validate User Ptr BOs */
1148 	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
1149 		bo = bo_va->base.bo;
1150 		if (!bo)
1151 			continue;
1152 
1153 		if (!amdgpu_ttm_tt_is_userptr(bo->tbo.ttm))
1154 			continue;
1155 
1156 		range = xa_load(&xa, key);
1157 		if (range && range->bo != bo) {
1158 			xa_erase(&xa, key);
1159 			amdgpu_hmm_range_free(range);
1160 			range = NULL;
1161 		}
1162 
1163 		if (!range) {
1164 			range = amdgpu_hmm_range_alloc(bo);
1165 			if (!range) {
1166 				ret = -ENOMEM;
1167 				goto unlock_all;
1168 			}
1169 
1170 			xa_store(&xa, key, range, GFP_KERNEL);
1171 			new_addition = true;
1172 		}
1173 		key++;
1174 	}
1175 
1176 	if (new_addition) {
1177 		drm_exec_fini(&exec);
1178 		xa_for_each(&xa, tmp_key, range) {
1179 			if (!range)
1180 				continue;
1181 			bo = range->bo;
1182 			ret = amdgpu_ttm_tt_get_user_pages(bo, range);
1183 			if (ret)
1184 				goto free_ranges;
1185 		}
1186 
1187 		invalidated = true;
1188 		new_addition = false;
1189 		goto retry_lock;
1190 	}
1191 
1192 	ret = amdgpu_vm_update_pdes(adev, vm, false);
1193 	if (ret)
1194 		goto unlock_all;
1195 
1196 	/*
1197 	 * We need to wait for all VM updates to finish before restarting the
1198 	 * queues. Using the idle list like that is now ok since everything is
1199 	 * locked in place.
1200 	 */
1201 	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
1202 		dma_fence_wait(bo_va->last_pt_update, false);
1203 	/*
1204 	 * The PRT bo_va is kept off the state lists, so its PTE update fence
1205 	 * lands in prt_va->last_pt_update rather than vm->last_update; wait on
1206 	 * it explicitly (as the CS path syncs it) before restarting queues.
1207 	 */
1208 	dma_fence_wait(fpriv->prt_va->last_pt_update, false);
1209 	dma_fence_wait(vm->last_update, false);
1210 
1211 	xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) {
1212 		bo = queue->wptr_obj.obj;
1213 		if (!bo) {
1214 			ret = -EINVAL;
1215 			goto unlock_all;
1216 		}
1217 
1218 		ret = amdgpu_ttm_alloc_gart(&bo->tbo);
1219 		if (unlikely(ret)) {
1220 			drm_file_err(uq_mgr->file,
1221 				     "failed to bind wptr bo to gart on resume, qid=%lu ret=%d\n",
1222 				     tmp_key, ret);
1223 			goto unlock_all;
1224 		}
1225 
1226 		queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(bo);
1227 	}
1228 
1229 	ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec);
1230 	if (ret) {
1231 		drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n");
1232 		goto unlock_all;
1233 	}
1234 
1235 	ret = amdgpu_userq_restore_all(uq_mgr);
1236 
1237 unlock_all:
1238 	drm_exec_fini(&exec);
1239 free_ranges:
1240 	xa_for_each(&xa, tmp_key, range) {
1241 		if (!range)
1242 			continue;
1243 		bo = range->bo;
1244 		amdgpu_hmm_range_free(range);
1245 	}
1246 	xa_destroy(&xa);
1247 	return ret;
1248 }
1249 
1250 static void amdgpu_userq_restore_worker(struct work_struct *work)
1251 {
1252 	struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work);
1253 	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
1254 	struct dma_fence *ev_fence;
1255 	int ret;
1256 
1257 	ev_fence = amdgpu_evf_mgr_get_fence(&fpriv->evf_mgr);
1258 	if (!dma_fence_is_signaled(ev_fence))
1259 		goto put_fence;
1260 
1261 	ret = amdgpu_userq_vm_validate_and_restore_queue(uq_mgr);
1262 	if (ret) {
1263 		drm_file_err(uq_mgr->file, "Failed to validate BOs to restore ret=%d\n", ret);
1264 		goto put_fence;
1265 	}
1266 
1267 put_fence:
1268 	dma_fence_put(ev_fence);
1269 }
1270 
1271 void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev,
1272 				    u32 pasid, u32 doorbell_offset)
1273 {
1274 	struct xarray *xa = &adev->userq_doorbell_xa;
1275 	struct amdgpu_usermode_queue *queue;
1276 	unsigned long flags, idx;
1277 
1278 	xa_lock_irqsave(xa, flags);
1279 	xa_for_each(xa, idx, queue) {
1280 		if (queue->vm && queue->vm->pasid == pasid &&
1281 		    queue->doorbell_offset == doorbell_offset) {
1282 			amdgpu_userq_start_hang_detect_work(queue);
1283 			break;
1284 		}
1285 	}
1286 	xa_unlock_irqrestore(xa, flags);
1287 }
1288 
1289 static int
1290 amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
1291 {
1292 	struct amdgpu_usermode_queue *queue;
1293 	unsigned long queue_id;
1294 	int ret = 0, r;
1295 
1296 	/* Try to unmap all the queues in this process ctx */
1297 	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
1298 		r = amdgpu_userq_unmap_helper(queue);
1299 		if (r)
1300 			ret = r;
1301 	}
1302 
1303 	if (ret) {
1304 		drm_file_err(uq_mgr->file,
1305 			     "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
1306 		amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
1307 					     &uq_mgr->reset_work);
1308 		flush_work(&uq_mgr->reset_work);
1309 	}
1310 	return ret;
1311 }
1312 
1313 void
1314 amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
1315 {
1316 	struct amdgpu_usermode_queue *queue;
1317 	unsigned long queue_id;
1318 
1319 	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
1320 		struct dma_fence *f = queue->last_fence;
1321 
1322 		if (!f)
1323 			continue;
1324 
1325 		dma_fence_wait(f, false);
1326 	}
1327 }
1328 
1329 void
1330 amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr)
1331 {
1332 	amdgpu_userq_evict_all(uq_mgr);
1333 }
1334 
1335 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
1336 			  struct amdgpu_device *adev)
1337 {
1338 	mutex_init(&userq_mgr->userq_mutex);
1339 	xa_init_flags(&userq_mgr->userq_xa, XA_FLAGS_ALLOC);
1340 	userq_mgr->adev = adev;
1341 	userq_mgr->file = file_priv;
1342 	userq_mgr->proc_ctx_allocated = false;
1343 	mutex_init(&userq_mgr->proc_ctx_lock);
1344 
1345 	INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
1346 	INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
1347 	return 0;
1348 }
1349 
1350 void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev)
1351 {
1352 	struct xarray *xa = &adev->userq_doorbell_xa;
1353 	struct amdgpu_usermode_queue *queue;
1354 	unsigned long flags, queue_id;
1355 
1356 	xa_lock_irqsave(xa, flags);
1357 	xa_for_each(xa, queue_id, queue) {
1358 		cancel_delayed_work(&queue->hang_detect_work);
1359 		cancel_work(&queue->userq_mgr->reset_work);
1360 	}
1361 	xa_unlock_irqrestore(xa, flags);
1362 }
1363 
1364 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
1365 {
1366 	cancel_delayed_work_sync(&userq_mgr->resume_work);
1367 }
1368 
1369 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
1370 {
1371 	struct amdgpu_mes *mes = &userq_mgr->adev->mes;
1372 	struct amdgpu_usermode_queue *queue;
1373 	unsigned long queue_id = 0;
1374 
1375 	for (;;) {
1376 		xa_lock(&userq_mgr->userq_xa);
1377 		queue = xa_find(&userq_mgr->userq_xa, &queue_id, ULONG_MAX,
1378 				XA_PRESENT);
1379 		if (queue)
1380 			__xa_erase(&userq_mgr->userq_xa, queue_id);
1381 		xa_unlock(&userq_mgr->userq_xa);
1382 
1383 		if (!queue)
1384 			break;
1385 
1386 		amdgpu_userq_put(queue);
1387 	}
1388 
1389 	xa_destroy(&userq_mgr->userq_xa);
1390 
1391 	/*
1392 	 * Drain any in-flight reset_work. By this point all queues are freed
1393 	 * and userq_count is 0, so if reset_work starts now it exits early.
1394 	 * We still need to wait in case it was already executing gpu_recover.
1395 	 */
1396 	cancel_work_sync(&userq_mgr->reset_work);
1397 
1398 	if (userq_mgr->proc_ctx_allocated) {
1399 		amdgpu_mes_free_proc_ctx_index(mes, userq_mgr->proc_ctx_array_index);
1400 		userq_mgr->proc_ctx_allocated = false;
1401 	}
1402 	amdgpu_bo_free_kernel(&userq_mgr->proc_ctx_obj.obj,
1403 			      &userq_mgr->proc_ctx_obj.gpu_addr,
1404 			      &userq_mgr->proc_ctx_obj.cpu_ptr);
1405 
1406 	mutex_destroy(&userq_mgr->proc_ctx_lock);
1407 	mutex_destroy(&userq_mgr->userq_mutex);
1408 }
1409 
1410 int amdgpu_userq_suspend(struct amdgpu_device *adev)
1411 {
1412 	struct amdgpu_usermode_queue *queue;
1413 	struct amdgpu_userq_mgr *uqm;
1414 	unsigned long queue_id;
1415 	int r;
1416 
1417 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1418 		uqm = queue->userq_mgr;
1419 		cancel_delayed_work_sync(&uqm->resume_work);
1420 		guard(mutex)(&uqm->userq_mutex);
1421 		if (adev->in_s0ix)
1422 			r = amdgpu_userq_preempt_helper(queue);
1423 		else
1424 			r = amdgpu_userq_unmap_helper(queue);
1425 		if (r)
1426 			return r;
1427 	}
1428 	return 0;
1429 }
1430 
1431 int amdgpu_userq_resume(struct amdgpu_device *adev)
1432 {
1433 	struct amdgpu_usermode_queue *queue;
1434 	struct amdgpu_userq_mgr *uqm;
1435 	unsigned long queue_id;
1436 	int r;
1437 
1438 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1439 		uqm = queue->userq_mgr;
1440 		guard(mutex)(&uqm->userq_mutex);
1441 		if (adev->in_s0ix)
1442 			r = amdgpu_userq_restore_helper(queue);
1443 		else
1444 			r = amdgpu_userq_map_helper(queue);
1445 		if (r)
1446 			return r;
1447 	}
1448 
1449 	return 0;
1450 }
1451 
1452 int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
1453 						  u32 idx)
1454 {
1455 	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1456 	struct amdgpu_usermode_queue *queue;
1457 	struct amdgpu_userq_mgr *uqm;
1458 	unsigned long queue_id;
1459 	int ret = 0, r;
1460 
1461 	/* only need to stop gfx/compute */
1462 	if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE))))
1463 		return 0;
1464 
1465 	if (adev->userq_halt_for_enforce_isolation)
1466 		dev_warn(adev->dev, "userq scheduling already stopped!\n");
1467 	adev->userq_halt_for_enforce_isolation = true;
1468 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1469 		uqm = queue->userq_mgr;
1470 		cancel_delayed_work_sync(&uqm->resume_work);
1471 		mutex_lock(&uqm->userq_mutex);
1472 		if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
1473 		     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
1474 		    (queue->xcp_id == idx)) {
1475 			r = amdgpu_userq_preempt_helper(queue);
1476 			if (r)
1477 				ret = r;
1478 		}
1479 		mutex_unlock(&uqm->userq_mutex);
1480 	}
1481 
1482 	return ret;
1483 }
1484 
1485 int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,
1486 						   u32 idx)
1487 {
1488 	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1489 	struct amdgpu_usermode_queue *queue;
1490 	struct amdgpu_userq_mgr *uqm;
1491 	unsigned long queue_id;
1492 	int ret = 0, r;
1493 
1494 	/* only need to stop gfx/compute */
1495 	if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE))))
1496 		return 0;
1497 
1498 	if (!adev->userq_halt_for_enforce_isolation)
1499 		dev_warn(adev->dev, "userq scheduling already started!\n");
1500 
1501 	adev->userq_halt_for_enforce_isolation = false;
1502 
1503 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1504 		uqm = queue->userq_mgr;
1505 		mutex_lock(&uqm->userq_mutex);
1506 		if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
1507 		     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
1508 		    (queue->xcp_id == idx)) {
1509 			r = amdgpu_userq_restore_helper(queue);
1510 			if (r)
1511 				ret = r;
1512 		}
1513 		mutex_unlock(&uqm->userq_mutex);
1514 	}
1515 
1516 	return ret;
1517 }
1518 
1519 void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
1520 					struct amdgpu_bo_va_mapping *mapping)
1521 {
1522 	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1523 	struct amdgpu_bo_va *bo_va = mapping->bo_va;
1524 	struct dma_resv *resv = bo_va->base.bo->tbo.base.resv;
1525 
1526 	if (!ip_mask)
1527 		return;
1528 
1529 	/**
1530 	 * The userq VA mapping reservation should include the eviction fence.
1531 	 * Note: The eviction fence may be attached to different BOs and this
1532 	 * unmap is only for one kind of userq VAs, so at this point suppose
1533 	 * the eviction fence is always unsignaled.
1534 	 */
1535 	dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
1536 			      false, MAX_SCHEDULE_TIMEOUT);
1537 }
1538 
1539 void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
1540 {
1541 	const struct amdgpu_userq_funcs *userq_funcs;
1542 	struct amdgpu_usermode_queue *queue;
1543 	unsigned long queue_id;
1544 
1545 	/* TODO: We probably need a new lock for the queue state */
1546 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1547 		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
1548 			trace_amdgpu_userq_state_start(queue);
1549 			userq_funcs = adev->userq_funcs[queue->queue_type];
1550 			userq_funcs->unmap(queue);
1551 			/* just mark all queues as hung at this point.
1552 			 * if unmap succeeds, we could map again
1553 			 * in amdgpu_userq_post_reset() if vram is not lost
1554 			 */
1555 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
1556 			queue->state = AMDGPU_USERQ_STATE_HUNG;
1557 		}
1558 		/* Force-complete any pending fence regardless of queue state so
1559 		 * that eviction/suspend and queue teardown waiters don't block
1560 		 * forever on a fence that will never signal after the reset.
1561 		 */
1562 		amdgpu_userq_fence_driver_force_completion(queue);
1563 	}
1564 }
1565 
1566 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost)
1567 {
1568 	/* if any queue state is AMDGPU_USERQ_STATE_UNMAPPED
1569 	 * at this point, we should be able to map it again
1570 	 * and continue if vram is not lost.
1571 	 */
1572 	struct amdgpu_usermode_queue *queue;
1573 	const struct amdgpu_userq_funcs *userq_funcs;
1574 	unsigned long queue_id;
1575 	int ret = 0, r;
1576 
1577 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1578 		if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) {
1579 			trace_amdgpu_userq_state_start(queue);
1580 
1581 			userq_funcs = adev->userq_funcs[queue->queue_type];
1582 			/* Re-map queue */
1583 			r = userq_funcs->map(queue);
1584 			if (r) {
1585 				dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id);
1586 				ret = r;
1587 				continue;
1588 			}
1589 			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
1590 			queue->state = AMDGPU_USERQ_STATE_MAPPED;
1591 		}
1592 	}
1593 
1594 	return ret;
1595 }
1596