xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: monk liu <monk.liu@amd.com>
23  */
24 
25 #include <drm/drm_auth.h>
26 #include <drm/drm_drv.h>
27 #include "amdgpu.h"
28 #include "amdgpu_sched.h"
29 #include "amdgpu_ras.h"
30 #include <linux/nospec.h>
31 #include <linux/pm_runtime.h>
32 
33 #define to_amdgpu_ctx_entity(e)	\
34 	container_of((e), struct amdgpu_ctx_entity, entity)
35 
36 const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = {
37 	[AMDGPU_HW_IP_GFX]	=	1,
38 	[AMDGPU_HW_IP_COMPUTE]	=	4,
39 	[AMDGPU_HW_IP_DMA]	=	2,
40 	[AMDGPU_HW_IP_UVD]	=	1,
41 	[AMDGPU_HW_IP_VCE]	=	1,
42 	[AMDGPU_HW_IP_UVD_ENC]	=	1,
43 	[AMDGPU_HW_IP_VCN_DEC]	=	1,
44 	[AMDGPU_HW_IP_VCN_ENC]	=	1,
45 	[AMDGPU_HW_IP_VCN_JPEG]	=	1,
46 	[AMDGPU_HW_IP_VPE]	=	1,
47 };
48 
49 bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio)
50 {
51 	switch (ctx_prio) {
52 	case AMDGPU_CTX_PRIORITY_VERY_LOW:
53 	case AMDGPU_CTX_PRIORITY_LOW:
54 	case AMDGPU_CTX_PRIORITY_NORMAL:
55 	case AMDGPU_CTX_PRIORITY_HIGH:
56 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
57 		return true;
58 	default:
59 	case AMDGPU_CTX_PRIORITY_UNSET:
60 		/* UNSET priority is not valid and we don't carry that
61 		 * around, but set it to NORMAL in the only place this
62 		 * function is called, amdgpu_ctx_ioctl().
63 		 */
64 		return false;
65 	}
66 }
67 
68 static enum drm_sched_priority
69 amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio)
70 {
71 	switch (ctx_prio) {
72 	case AMDGPU_CTX_PRIORITY_UNSET:
73 		pr_warn_once("AMD-->DRM context priority value UNSET-->NORMAL");
74 		return DRM_SCHED_PRIORITY_NORMAL;
75 
76 	case AMDGPU_CTX_PRIORITY_VERY_LOW:
77 		return DRM_SCHED_PRIORITY_LOW;
78 
79 	case AMDGPU_CTX_PRIORITY_LOW:
80 		return DRM_SCHED_PRIORITY_LOW;
81 
82 	case AMDGPU_CTX_PRIORITY_NORMAL:
83 		return DRM_SCHED_PRIORITY_NORMAL;
84 
85 	case AMDGPU_CTX_PRIORITY_HIGH:
86 		return DRM_SCHED_PRIORITY_HIGH;
87 
88 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
89 		return DRM_SCHED_PRIORITY_HIGH;
90 
91 	/* This should not happen as we sanitized userspace provided priority
92 	 * already, WARN if this happens.
93 	 */
94 	default:
95 		WARN(1, "Invalid context priority %d\n", ctx_prio);
96 		return DRM_SCHED_PRIORITY_NORMAL;
97 	}
98 
99 }
100 
101 static int amdgpu_ctx_priority_permit(struct drm_file *filp,
102 				      int32_t priority)
103 {
104 	/* NORMAL and below are accessible by everyone */
105 	if (priority <= AMDGPU_CTX_PRIORITY_NORMAL)
106 		return 0;
107 
108 	if (capable(CAP_SYS_NICE))
109 		return 0;
110 
111 	if (drm_is_current_master(filp))
112 		return 0;
113 
114 	return -EACCES;
115 }
116 
117 static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_gfx_pipe_prio(int32_t prio)
118 {
119 	switch (prio) {
120 	case AMDGPU_CTX_PRIORITY_HIGH:
121 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
122 		return AMDGPU_GFX_PIPE_PRIO_HIGH;
123 	default:
124 		return AMDGPU_GFX_PIPE_PRIO_NORMAL;
125 	}
126 }
127 
128 static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio)
129 {
130 	switch (prio) {
131 	case AMDGPU_CTX_PRIORITY_HIGH:
132 		return AMDGPU_RING_PRIO_1;
133 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
134 		return AMDGPU_RING_PRIO_2;
135 	default:
136 		return AMDGPU_RING_PRIO_0;
137 	}
138 }
139 
140 static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip)
141 {
142 	struct amdgpu_device *adev = ctx->mgr->adev;
143 	unsigned int hw_prio;
144 	int32_t ctx_prio;
145 
146 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
147 			ctx->init_priority : ctx->override_priority;
148 
149 	switch (hw_ip) {
150 	case AMDGPU_HW_IP_GFX:
151 	case AMDGPU_HW_IP_COMPUTE:
152 		hw_prio = amdgpu_ctx_prio_to_gfx_pipe_prio(ctx_prio);
153 		break;
154 	case AMDGPU_HW_IP_VCE:
155 	case AMDGPU_HW_IP_VCN_ENC:
156 		hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio);
157 		break;
158 	default:
159 		hw_prio = AMDGPU_RING_PRIO_DEFAULT;
160 		break;
161 	}
162 
163 	hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
164 	if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0)
165 		hw_prio = AMDGPU_RING_PRIO_DEFAULT;
166 
167 	return hw_prio;
168 }
169 
170 /* Calculate the time spend on the hw */
171 static ktime_t amdgpu_ctx_fence_time(struct dma_fence *fence)
172 {
173 	struct drm_sched_fence *s_fence;
174 
175 	if (!fence)
176 		return ns_to_ktime(0);
177 
178 	/* When the fence is not even scheduled it can't have spend time */
179 	s_fence = to_drm_sched_fence(fence);
180 	if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->scheduled.flags))
181 		return ns_to_ktime(0);
182 
183 	/* When it is still running account how much already spend */
184 	if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->finished.flags))
185 		return ktime_sub(ktime_get(), s_fence->scheduled.timestamp);
186 
187 	return ktime_sub(s_fence->finished.timestamp,
188 			 s_fence->scheduled.timestamp);
189 }
190 
191 static ktime_t amdgpu_ctx_entity_time(struct amdgpu_ctx *ctx,
192 				      struct amdgpu_ctx_entity *centity)
193 {
194 	ktime_t res = ns_to_ktime(0);
195 	uint32_t i;
196 
197 	spin_lock(&ctx->ring_lock);
198 	for (i = 0; i < amdgpu_sched_jobs; i++) {
199 		res = ktime_add(res, amdgpu_ctx_fence_time(centity->fences[i]));
200 	}
201 	spin_unlock(&ctx->ring_lock);
202 	return res;
203 }
204 
205 static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
206 				  const u32 ring)
207 {
208 	struct drm_gpu_scheduler **scheds = NULL, *sched = NULL;
209 	struct amdgpu_device *adev = ctx->mgr->adev;
210 	struct amdgpu_ctx_entity *entity;
211 	enum drm_sched_priority drm_prio;
212 	unsigned int hw_prio, num_scheds;
213 	int32_t ctx_prio;
214 	int r;
215 
216 	entity = kzalloc_flex(*entity, fences, amdgpu_sched_jobs);
217 	if (!entity)
218 		return  -ENOMEM;
219 
220 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
221 			ctx->init_priority : ctx->override_priority;
222 	entity->hw_ip = hw_ip;
223 	entity->sequence = 1;
224 	hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
225 	drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio);
226 
227 	hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
228 
229 	if (!(adev)->xcp_mgr) {
230 		scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
231 		num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
232 	} else {
233 		struct amdgpu_fpriv *fpriv;
234 
235 		/* TODO: Stop using fpriv here, we only need the xcp_id. */
236 		fpriv = container_of(ctx->mgr, struct amdgpu_fpriv, ctx_mgr);
237 		r = amdgpu_xcp_select_scheds(adev, hw_ip, hw_prio, fpriv,
238 						&num_scheds, &scheds);
239 		if (r)
240 			goto error_free_entity;
241 	}
242 
243 	if (num_scheds == 0) {
244 		r = -EINVAL;
245 		goto error_free_entity;
246 	}
247 
248 	/* disable load balance if the hw engine retains context among dependent jobs */
249 	if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
250 	    hw_ip == AMDGPU_HW_IP_VCN_DEC ||
251 	    hw_ip == AMDGPU_HW_IP_UVD_ENC ||
252 	    hw_ip == AMDGPU_HW_IP_UVD) {
253 		sched = drm_sched_pick_best(scheds, num_scheds);
254 		scheds = &sched;
255 		num_scheds = 1;
256 	}
257 
258 	r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds,
259 				  NULL);
260 	if (r)
261 		goto error_free_entity;
262 
263 	/* It's not an error if we fail to install the new entity */
264 	if (cmpxchg(&ctx->entities[hw_ip][ring], NULL, entity))
265 		goto cleanup_entity;
266 
267 	return 0;
268 
269 cleanup_entity:
270 	drm_sched_entity_fini(&entity->entity);
271 
272 error_free_entity:
273 	kfree(entity);
274 
275 	return r;
276 }
277 
278 static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev,
279 				  struct amdgpu_ctx_entity *entity)
280 {
281 	ktime_t res = ns_to_ktime(0);
282 	int i;
283 
284 	if (!entity)
285 		return res;
286 
287 	drm_sched_entity_destroy(&entity->entity);
288 
289 	for (i = 0; i < amdgpu_sched_jobs; ++i) {
290 		res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i]));
291 		dma_fence_put(entity->fences[i]);
292 	}
293 
294 	amdgpu_xcp_release_sched(adev, entity);
295 
296 	kfree(entity);
297 	return res;
298 }
299 
300 static u32 amdgpu_get_stable_pstate(struct amdgpu_device *adev)
301 {
302 	switch (amdgpu_dpm_get_performance_level(adev)) {
303 	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
304 		return AMDGPU_CTX_STABLE_PSTATE_STANDARD;
305 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
306 		return AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK;
307 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
308 		return AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK;
309 	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
310 		return AMDGPU_CTX_STABLE_PSTATE_PEAK;
311 	default:
312 		return AMDGPU_CTX_STABLE_PSTATE_NONE;
313 	}
314 }
315 
316 static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
317 			   struct drm_file *filp, struct amdgpu_ctx *ctx)
318 {
319 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
320 	int r;
321 
322 	r = amdgpu_ctx_priority_permit(filp, priority);
323 	if (r)
324 		return r;
325 
326 	memset(ctx, 0, sizeof(*ctx));
327 
328 	kref_init(&ctx->refcount);
329 	ctx->mgr = mgr;
330 	spin_lock_init(&ctx->ring_lock);
331 
332 	ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter);
333 	ctx->reset_counter_query = ctx->reset_counter;
334 	ctx->generation = amdgpu_vm_generation(mgr->adev, &fpriv->vm);
335 	ctx->init_priority = priority;
336 	ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET;
337 	ctx->stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE;
338 
339 	return 0;
340 }
341 
342 static int __amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
343 					  u32 stable_pstate)
344 {
345 	struct amdgpu_device *adev = ctx->mgr->adev;
346 	enum amd_dpm_forced_level level;
347 	struct amdgpu_ctx *current_ctx;
348 	u32 current_stable_pstate;
349 	int r = 0;
350 
351 	lockdep_assert_held(&adev->pm.stable_pstate_ctx_lock);
352 
353 	switch (stable_pstate) {
354 	case AMDGPU_CTX_STABLE_PSTATE_NONE:
355 		level = AMD_DPM_FORCED_LEVEL_AUTO;
356 		break;
357 	case AMDGPU_CTX_STABLE_PSTATE_STANDARD:
358 		level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD;
359 		break;
360 	case AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK:
361 		level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK;
362 		break;
363 	case AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK:
364 		level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK;
365 		break;
366 	case AMDGPU_CTX_STABLE_PSTATE_PEAK:
367 		level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
368 		break;
369 	default:
370 		return -EINVAL;
371 	}
372 
373 	current_ctx = adev->pm.stable_pstate_ctx;
374 	if (current_ctx && current_ctx != ctx)
375 		return -EBUSY;
376 
377 	current_stable_pstate = amdgpu_get_stable_pstate(adev);
378 	if (current_stable_pstate == stable_pstate)
379 		return 0;
380 
381 	r = amdgpu_dpm_force_performance_level(adev, level);
382 	if (r)
383 		return r;
384 
385 	if (!current_ctx) {
386 		adev->pm.stable_pstate_ctx = ctx;
387 		/*
388 		 * Serialized by context taking ownership for the first time
389 		 * while holding adev->pm.stable_pstate_ctx_lock).
390 		 */
391 		WRITE_ONCE(ctx->stable_pstate, current_stable_pstate);
392 	}
393 
394 	return 0;
395 }
396 
397 static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
398 					u32 stable_pstate)
399 {
400 	struct amdgpu_device *adev = ctx->mgr->adev;
401 	int r;
402 
403 	mutex_lock(&adev->pm.stable_pstate_ctx_lock);
404 	r = __amdgpu_ctx_set_stable_pstate(ctx, stable_pstate);
405 	mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
406 
407 	return r;
408 }
409 
410 void amdgpu_ctx_fini(struct kref *ref)
411 {
412 	struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
413 	struct amdgpu_ctx_mgr *mgr = ctx->mgr;
414 	struct amdgpu_device *adev = mgr->adev;
415 	unsigned i, j, idx;
416 
417 	if (!adev)
418 		return;
419 
420 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
421 		for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) {
422 			ktime_t spend;
423 
424 			spend = amdgpu_ctx_fini_entity(adev, ctx->entities[i][j]);
425 			atomic64_add(ktime_to_ns(spend), &mgr->time_spend[i]);
426 		}
427 	}
428 
429 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
430 		mutex_lock(&adev->pm.stable_pstate_ctx_lock);
431 		if (adev->pm.stable_pstate_ctx == ctx) {
432 			__amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate);
433 			adev->pm.stable_pstate_ctx = NULL;
434 		}
435 		mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
436 		drm_dev_exit(idx);
437 	}
438 
439 	kfree(ctx);
440 }
441 
442 int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance,
443 			  u32 ring, struct drm_sched_entity **entity)
444 {
445 	int r;
446 	struct drm_sched_entity *ctx_entity;
447 
448 	if (hw_ip >= AMDGPU_HW_IP_NUM) {
449 		drm_err(adev_to_drm(ctx->mgr->adev),
450 			"unknown HW IP type: %d\n", hw_ip);
451 		return -EINVAL;
452 	}
453 
454 	/* Right now all IPs have only one instance - multiple rings. */
455 	if (instance != 0) {
456 		drm_dbg(adev_to_drm(ctx->mgr->adev),
457 			"invalid ip instance: %d\n", instance);
458 		return -EINVAL;
459 	}
460 
461 	if (ring >= amdgpu_ctx_num_entities[hw_ip]) {
462 		drm_dbg(adev_to_drm(ctx->mgr->adev),
463 			"invalid ring: %d %d\n", hw_ip, ring);
464 		return -EINVAL;
465 	}
466 
467 	if (ctx->entities[hw_ip][ring] == NULL) {
468 		r = amdgpu_ctx_init_entity(ctx, hw_ip, ring);
469 		if (r)
470 			return r;
471 	}
472 
473 	ctx_entity = &ctx->entities[hw_ip][ring]->entity;
474 	r = drm_sched_entity_error(ctx_entity);
475 	if (r) {
476 		DRM_DEBUG("error entity %p\n", ctx_entity);
477 		return r;
478 	}
479 
480 	*entity = ctx_entity;
481 	return 0;
482 }
483 
484 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
485 			    struct amdgpu_fpriv *fpriv,
486 			    struct drm_file *filp,
487 			    int32_t priority,
488 			    uint32_t *id)
489 {
490 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
491 	struct amdgpu_ctx *ctx;
492 	int r;
493 
494 	ctx = kmalloc_obj(*ctx);
495 	if (!ctx)
496 		return -ENOMEM;
497 
498 	r = amdgpu_ctx_init(mgr, priority, filp, ctx);
499 	if (r) {
500 		kfree(ctx);
501 		return r;
502 	}
503 
504 	r = xa_alloc(&mgr->ctx_handles, id, ctx, xa_limit_32b, GFP_KERNEL);
505 	if (r)
506 		amdgpu_ctx_put(ctx);
507 
508 	return r;
509 }
510 
511 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
512 {
513 	struct amdgpu_ctx *ctx;
514 
515 	ctx = xa_erase(&fpriv->ctx_mgr.ctx_handles, id);
516 	amdgpu_ctx_put(ctx);
517 
518 	return ctx ? 0 : -EINVAL;
519 }
520 
521 static int amdgpu_ctx_query(struct amdgpu_device *adev,
522 			    struct amdgpu_fpriv *fpriv, uint32_t id,
523 			    union drm_amdgpu_ctx_out *out)
524 {
525 	struct amdgpu_ctx *ctx;
526 	unsigned reset_counter;
527 
528 	ctx = amdgpu_ctx_get(fpriv, id);
529 	if (!ctx)
530 		return -EINVAL;
531 
532 	/* TODO: these two are always zero */
533 	out->state.flags = 0x0;
534 	out->state.hangs = 0x0;
535 
536 	/* determine if a GPU reset has occurred since the last call */
537 	reset_counter = atomic_read(&adev->gpu_reset_counter);
538 	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
539 	if (ctx->reset_counter_query == reset_counter)
540 		out->state.reset_status = AMDGPU_CTX_NO_RESET;
541 	else
542 		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
543 	ctx->reset_counter_query = reset_counter;
544 
545 	amdgpu_ctx_put(ctx);
546 
547 	return 0;
548 }
549 
550 #define AMDGPU_RAS_COUNTE_DELAY_MS 3000
551 
552 static bool amdgpu_ctx_guilty(struct amdgpu_ctx *ctx)
553 {
554 	int i, j, r;
555 
556 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
557 		for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
558 			struct amdgpu_ctx_entity *ctx_entity;
559 
560 			ctx_entity = ctx->entities[i][j];
561 			if (!ctx_entity)
562 				continue;
563 
564 			r = drm_sched_entity_error(&ctx_entity->entity);
565 			if (r == -ETIME)
566 				return true;
567 		}
568 	}
569 
570 	return false;
571 }
572 
573 static int amdgpu_ctx_query2(struct amdgpu_device *adev,
574 			     struct amdgpu_fpriv *fpriv, uint32_t id,
575 			     union drm_amdgpu_ctx_out *out)
576 {
577 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
578 	struct amdgpu_ctx *ctx;
579 
580 	ctx = amdgpu_ctx_get(fpriv, id);
581 	if (!ctx)
582 		return -EINVAL;
583 
584 	out->state.flags = 0x0;
585 	out->state.hangs = 0x0;
586 
587 	if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
588 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
589 
590 	if (ctx->generation != amdgpu_vm_generation(adev, &fpriv->vm))
591 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
592 
593 	if (amdgpu_ctx_guilty(ctx))
594 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
595 
596 	if (amdgpu_in_reset(adev))
597 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET_IN_PROGRESS;
598 
599 	if (adev->ras_enabled && con) {
600 		/* Return the cached values in O(1),
601 		 * and schedule delayed work to cache
602 		 * new vaues.
603 		 */
604 		int ce_count, ue_count;
605 
606 		ce_count = atomic_read(&con->ras_ce_count);
607 		ue_count = atomic_read(&con->ras_ue_count);
608 
609 		if (ce_count != ctx->ras_counter_ce) {
610 			ctx->ras_counter_ce = ce_count;
611 			out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE;
612 		}
613 
614 		if (ue_count != ctx->ras_counter_ue) {
615 			ctx->ras_counter_ue = ue_count;
616 			out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE;
617 		}
618 
619 		schedule_delayed_work(&con->ras_counte_delay_work,
620 				      msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS));
621 	}
622 
623 	amdgpu_ctx_put(ctx);
624 
625 	return 0;
626 }
627 
628 static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev,
629 				    struct amdgpu_fpriv *fpriv, uint32_t id,
630 				    bool set, u32 *stable_pstate)
631 {
632 	struct amdgpu_ctx *ctx;
633 	int r = 0;
634 
635 	ctx = amdgpu_ctx_get(fpriv, id);
636 	if (!ctx)
637 		return -EINVAL;
638 
639 	/*
640 	 * The get path is odd in this uapi - it will check whether the context
641 	 * id exist, but otherwise does nothing with it. In other words, the
642 	 * uapi has historically been implemented as being able to query the
643 	 * global device state, as long as the caller supplies a random valid
644 	 * context id.
645 	 */
646 
647 	if (set) {
648 		r = pm_runtime_resume_and_get(adev->dev);
649 		if (r < 0)
650 			return r;
651 		r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate);
652 		pm_runtime_put_autosuspend(adev->dev);
653 	} else {
654 		*stable_pstate = amdgpu_get_stable_pstate(adev);
655 	}
656 
657 	amdgpu_ctx_put(ctx);
658 	return r;
659 }
660 
661 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
662 		     struct drm_file *filp)
663 {
664 	int r;
665 	uint32_t id, stable_pstate;
666 	int32_t priority;
667 
668 	union drm_amdgpu_ctx *args = data;
669 	struct amdgpu_device *adev = drm_to_adev(dev);
670 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
671 
672 	id = args->in.ctx_id;
673 	priority = args->in.priority;
674 
675 	/* For backwards compatibility, we need to accept ioctls with garbage
676 	 * in the priority field. Garbage values in the priority field, result
677 	 * in the priority being set to NORMAL.
678 	 */
679 	if (!amdgpu_ctx_priority_is_valid(priority))
680 		priority = AMDGPU_CTX_PRIORITY_NORMAL;
681 
682 	switch (args->in.op) {
683 	case AMDGPU_CTX_OP_ALLOC_CTX:
684 		if (args->in.flags)
685 			return -EINVAL;
686 		r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
687 		args->out.alloc.ctx_id = id;
688 		break;
689 	case AMDGPU_CTX_OP_FREE_CTX:
690 		if (args->in.flags)
691 			return -EINVAL;
692 		r = amdgpu_ctx_free(fpriv, id);
693 		break;
694 	case AMDGPU_CTX_OP_QUERY_STATE:
695 		if (args->in.flags)
696 			return -EINVAL;
697 		r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
698 		break;
699 	case AMDGPU_CTX_OP_QUERY_STATE2:
700 		if (args->in.flags)
701 			return -EINVAL;
702 		r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
703 		break;
704 	case AMDGPU_CTX_OP_GET_STABLE_PSTATE:
705 		if (args->in.flags)
706 			return -EINVAL;
707 		r = amdgpu_ctx_stable_pstate(adev, fpriv, id, false, &stable_pstate);
708 		if (!r)
709 			args->out.pstate.flags = stable_pstate;
710 		break;
711 	case AMDGPU_CTX_OP_SET_STABLE_PSTATE:
712 		if (args->in.flags & ~AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK)
713 			return -EINVAL;
714 		stable_pstate = args->in.flags & AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK;
715 		if (stable_pstate > AMDGPU_CTX_STABLE_PSTATE_PEAK)
716 			return -EINVAL;
717 		r = amdgpu_ctx_stable_pstate(adev, fpriv, id, true, &stable_pstate);
718 		break;
719 	default:
720 		return -EINVAL;
721 	}
722 
723 	return r;
724 }
725 
726 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
727 {
728 	struct amdgpu_ctx *ctx;
729 	struct amdgpu_ctx_mgr *mgr;
730 
731 	if (!fpriv)
732 		return NULL;
733 
734 	mgr = &fpriv->ctx_mgr;
735 
736 	xa_lock(&mgr->ctx_handles);
737 	ctx = xa_load(&mgr->ctx_handles, id);
738 	if (ctx)
739 		kref_get(&ctx->refcount);
740 	xa_unlock(&mgr->ctx_handles);
741 	return ctx;
742 }
743 
744 uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
745 			      struct drm_sched_entity *entity,
746 			      struct dma_fence *fence)
747 {
748 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
749 	uint64_t seq = centity->sequence;
750 	struct dma_fence *other = NULL;
751 	unsigned idx = 0;
752 
753 	idx = seq & (amdgpu_sched_jobs - 1);
754 	other = centity->fences[idx];
755 	WARN_ON(other && !dma_fence_is_signaled(other));
756 
757 	dma_fence_get(fence);
758 
759 	spin_lock(&ctx->ring_lock);
760 	centity->fences[idx] = fence;
761 	centity->sequence++;
762 	spin_unlock(&ctx->ring_lock);
763 
764 	atomic64_add(ktime_to_ns(amdgpu_ctx_fence_time(other)),
765 		     &ctx->mgr->time_spend[centity->hw_ip]);
766 
767 	dma_fence_put(other);
768 	return seq;
769 }
770 
771 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
772 				       struct drm_sched_entity *entity,
773 				       uint64_t seq)
774 {
775 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
776 	struct dma_fence *fence;
777 
778 	spin_lock(&ctx->ring_lock);
779 
780 	if (seq == ~0ull)
781 		seq = centity->sequence - 1;
782 
783 	if (seq >= centity->sequence) {
784 		spin_unlock(&ctx->ring_lock);
785 		return ERR_PTR(-EINVAL);
786 	}
787 
788 
789 	if (seq + amdgpu_sched_jobs < centity->sequence) {
790 		spin_unlock(&ctx->ring_lock);
791 		return NULL;
792 	}
793 
794 	fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]);
795 	spin_unlock(&ctx->ring_lock);
796 
797 	return fence;
798 }
799 
800 static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx,
801 					   struct amdgpu_ctx_entity *aentity,
802 					   int hw_ip,
803 					   int32_t priority)
804 {
805 	struct amdgpu_device *adev = ctx->mgr->adev;
806 	unsigned int hw_prio;
807 	struct drm_gpu_scheduler **scheds = NULL;
808 	unsigned num_scheds;
809 
810 	/* set sw priority */
811 	drm_sched_entity_set_priority(&aentity->entity,
812 				      amdgpu_ctx_to_drm_sched_prio(priority));
813 
814 	/* set hw priority */
815 	if (hw_ip == AMDGPU_HW_IP_COMPUTE || hw_ip == AMDGPU_HW_IP_GFX) {
816 		hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
817 		hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX);
818 		scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
819 		num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
820 		drm_sched_entity_modify_sched(&aentity->entity, scheds,
821 					      num_scheds);
822 	}
823 }
824 
825 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
826 				  int32_t priority)
827 {
828 	int32_t ctx_prio;
829 	unsigned i, j;
830 
831 	ctx->override_priority = priority;
832 
833 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
834 			ctx->init_priority : ctx->override_priority;
835 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
836 		for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
837 			if (!ctx->entities[i][j])
838 				continue;
839 
840 			amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j],
841 						       i, ctx_prio);
842 		}
843 	}
844 }
845 
846 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
847 			       struct drm_sched_entity *entity)
848 {
849 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
850 	struct dma_fence *other;
851 	unsigned idx;
852 	long r;
853 
854 	spin_lock(&ctx->ring_lock);
855 	idx = centity->sequence & (amdgpu_sched_jobs - 1);
856 	other = dma_fence_get(centity->fences[idx]);
857 	spin_unlock(&ctx->ring_lock);
858 
859 	if (!other)
860 		return 0;
861 
862 	r = dma_fence_wait(other, true);
863 	if (r < 0 && r != -ERESTARTSYS)
864 		drm_err(adev_to_drm(ctx->mgr->adev),
865 			"AMDGPU: Error waiting for fence in ctx %p\n", ctx);
866 
867 	dma_fence_put(other);
868 	return r;
869 }
870 
871 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
872 			 struct amdgpu_device *adev)
873 {
874 	unsigned int i;
875 
876 	mgr->adev = adev;
877 	xa_init_flags(&mgr->ctx_handles, XA_FLAGS_ALLOC1);
878 
879 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
880 		atomic64_set(&mgr->time_spend[i], 0);
881 }
882 
883 long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
884 {
885 	struct amdgpu_ctx *ctx;
886 	unsigned long id;
887 	int i, j;
888 
889 	xa_lock(&mgr->ctx_handles);
890 	xa_for_each(&mgr->ctx_handles, id, ctx) {
891 		kref_get(&ctx->refcount);
892 		xa_unlock(&mgr->ctx_handles);
893 		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
894 			for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
895 				struct drm_sched_entity *entity;
896 
897 				if (!ctx->entities[i][j])
898 					continue;
899 
900 				entity = &ctx->entities[i][j]->entity;
901 				timeout = drm_sched_entity_flush(entity, timeout);
902 			}
903 		}
904 		amdgpu_ctx_put(ctx);
905 		xa_lock(&mgr->ctx_handles);
906 	}
907 	xa_unlock(&mgr->ctx_handles);
908 	return timeout;
909 }
910 
911 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
912 {
913 	struct amdgpu_ctx *ctx;
914 	unsigned long id;
915 
916 	xa_for_each(&mgr->ctx_handles, id, ctx)
917 		amdgpu_ctx_put(ctx);
918 	xa_destroy(&mgr->ctx_handles);
919 }
920 
921 void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
922 			  ktime_t usage[AMDGPU_HW_IP_NUM])
923 {
924 	struct amdgpu_ctx *ctx;
925 	unsigned int hw_ip, i;
926 	unsigned long id;
927 
928 	/*
929 	 * This is a little bit racy because it can be that a ctx or a fence are
930 	 * destroyed just in the moment we try to account them. But that is ok
931 	 * since exactly that case is explicitely allowed by the interface.
932 	 */
933 	for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
934 		uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]);
935 
936 		usage[hw_ip] = ns_to_ktime(ns);
937 	}
938 
939 	xa_lock(&mgr->ctx_handles);
940 	xa_for_each(&mgr->ctx_handles, id, ctx) {
941 		for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
942 			for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) {
943 				struct amdgpu_ctx_entity *centity;
944 				ktime_t spend;
945 
946 				centity = ctx->entities[hw_ip][i];
947 				if (!centity)
948 					continue;
949 				spend = amdgpu_ctx_entity_time(ctx, centity);
950 				usage[hw_ip] = ktime_add(usage[hw_ip], spend);
951 			}
952 		}
953 	}
954 	xa_unlock(&mgr->ctx_handles);
955 }
956