xref: /linux/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/ratelimit.h>
26 #include <linux/printk.h>
27 #include <linux/slab.h>
28 #include <linux/list.h>
29 #include <linux/types.h>
30 #include <linux/bitops.h>
31 #include <linux/sched.h>
32 #include "kfd_priv.h"
33 #include "kfd_device_queue_manager.h"
34 #include "kfd_mqd_manager.h"
35 #include "cik_regs.h"
36 #include "kfd_kernel_queue.h"
37 #include "amdgpu_amdkfd.h"
38 #include "amdgpu_reset.h"
39 #include "amdgpu_sdma.h"
40 #include "amdgpu_ring.h"
41 #include "amdgpu_mes.h"
42 #include "kfd_debug.h"
43 
44 /* Size of the per-pipe EOP queue */
45 #define CIK_HPD_EOP_BYTES_LOG2 11
46 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
47 /* See unmap_queues_cpsch() */
48 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
49 
50 /* Interval for notifying MES of work on unmapped queues during oversubscription */
51 #define DQM_MES_UNMAP_NOTIFY_DELAY_US 50
52 
53 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
54 				  u32 pasid, unsigned int vmid);
55 
56 static int execute_queues_cpsch(struct device_queue_manager *dqm,
57 				enum kfd_unmap_queues_filter filter,
58 				uint32_t filter_param,
59 				uint32_t grace_period);
60 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
61 				enum kfd_unmap_queues_filter filter,
62 				uint32_t filter_param,
63 				uint32_t grace_period,
64 				bool reset);
65 
66 static int map_queues_cpsch(struct device_queue_manager *dqm);
67 
68 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
69 				struct queue *q);
70 
71 static inline void deallocate_hqd(struct device_queue_manager *dqm,
72 				struct queue *q);
73 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
74 static int allocate_sdma_queue(struct device_queue_manager *dqm,
75 				struct queue *q, const uint32_t *restore_sdma_id);
76 
77 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
78 static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm,
79 						   u32 doorbell_offset);
80 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
81 			       struct qcm_process_device *qpd, bool verbose);
82 static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q);
83 
84 static inline
85 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
86 {
87 	if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
88 		return KFD_MQD_TYPE_SDMA;
89 	return KFD_MQD_TYPE_CP;
90 }
91 
92 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
93 {
94 	int i;
95 	int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec
96 		+ pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;
97 
98 	/* queue is available for KFD usage if bit is 1 */
99 	for (i = 0; i <  dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)
100 		if (test_bit(pipe_offset + i,
101 			      dqm->dev->kfd->shared_resources.cp_queue_bitmap))
102 			return true;
103 	return false;
104 }
105 
106 unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
107 {
108 	return bitmap_weight(dqm->dev->kfd->shared_resources.cp_queue_bitmap,
109 				AMDGPU_MAX_QUEUES);
110 }
111 
112 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
113 {
114 	return dqm->dev->kfd->shared_resources.num_queue_per_pipe;
115 }
116 
117 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
118 {
119 	return dqm->dev->kfd->shared_resources.num_pipe_per_mec;
120 }
121 
122 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
123 {
124 	return kfd_get_num_sdma_engines(dqm->dev) +
125 		kfd_get_num_xgmi_sdma_engines(dqm->dev);
126 }
127 
128 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
129 {
130 	return kfd_get_num_sdma_engines(dqm->dev) *
131 		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
132 }
133 
134 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
135 {
136 	return kfd_get_num_xgmi_sdma_engines(dqm->dev) *
137 		dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
138 }
139 
140 static void init_sdma_bitmaps(struct device_queue_manager *dqm)
141 {
142 	bitmap_zero(dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);
143 	bitmap_set(dqm->sdma_bitmap, 0, get_num_sdma_queues(dqm));
144 
145 	bitmap_zero(dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
146 	bitmap_set(dqm->xgmi_sdma_bitmap, 0, get_num_xgmi_sdma_queues(dqm));
147 
148 	/* Mask out the reserved queues */
149 	bitmap_clear(dqm->sdma_bitmap, 0, kfd_get_num_sdma_engines(dqm->dev) *
150 			dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine);
151 	bitmap_clear(dqm->xgmi_sdma_bitmap, 0, kfd_get_num_xgmi_sdma_engines(dqm->dev) *
152 			dqm->dev->kfd->device_info.num_reserved_sdma_queues_per_engine);
153 }
154 
155 void program_sh_mem_settings(struct device_queue_manager *dqm,
156 					struct qcm_process_device *qpd)
157 {
158 	uint32_t xcc_mask = dqm->dev->xcc_mask;
159 	int xcc_id;
160 
161 	for_each_inst(xcc_id, xcc_mask)
162 		dqm->dev->kfd2kgd->program_sh_mem_settings(
163 			dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,
164 			qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,
165 			qpd->sh_mem_bases, xcc_id);
166 }
167 
168 static void kfd_hws_hang(struct device_queue_manager *dqm)
169 {
170 	struct device_process_node *cur;
171 	struct qcm_process_device *qpd;
172 	struct queue *q;
173 
174 	/* Mark all device queues as reset. */
175 	list_for_each_entry(cur, &dqm->queues, list) {
176 		qpd = cur->qpd;
177 		list_for_each_entry(q, &qpd->queues_list, list) {
178 			struct kfd_process_device *pdd = qpd_to_pdd(qpd);
179 
180 			pdd->has_reset_queue = true;
181 		}
182 	}
183 
184 	/*
185 	 * Issue a GPU reset if HWS is unresponsive
186 	 */
187 	amdgpu_amdkfd_gpu_reset(dqm->dev->adev);
188 }
189 
190 static int convert_to_amdgpu_ring_type(int queue_type)
191 {
192 	int amdgpu_ring_type;
193 
194 	switch (queue_type) {
195 	case KFD_QUEUE_TYPE_COMPUTE:
196 		amdgpu_ring_type = AMDGPU_RING_TYPE_COMPUTE;
197 		break;
198 	case KFD_QUEUE_TYPE_SDMA:
199 		amdgpu_ring_type = AMDGPU_RING_TYPE_SDMA;
200 		break;
201 	default:
202 		WARN(1, "Invalid queue type %d", queue_type);
203 		amdgpu_ring_type = -EINVAL;
204 		break;
205 	}
206 
207 	return amdgpu_ring_type;
208 }
209 
210 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
211 			 struct qcm_process_device *qpd)
212 {
213 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
214 	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
215 	struct mes_add_queue_input queue_input;
216 	int r, queue_type;
217 	uint64_t wptr_addr_off;
218 
219 	if (!dqm->sched_running || dqm->sched_halt)
220 		return 0;
221 	if (!down_read_trylock(&adev->reset_domain->sem))
222 		return -EIO;
223 
224 	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
225 	queue_input.process_id = pdd->pasid;
226 	queue_input.page_table_base_addr =  qpd->page_table_base;
227 	queue_input.process_va_start = 0;
228 	queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
229 	/* MES unit for quantum is 100ns */
230 	queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM;  /* Equivalent to 10ms. */
231 	queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;
232 	queue_input.process_context_array_index = pdd->proc_ctx_array_index;
233 	queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */
234 	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
235 	queue_input.gang_context_array_index = q->gang_ctx_array_index;
236 	queue_input.inprocess_gang_priority = q->properties.priority;
237 	queue_input.gang_global_priority_level =
238 					AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
239 	queue_input.doorbell_offset = q->properties.doorbell_off;
240 	queue_input.mqd_addr = q->gart_mqd_addr;
241 	queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;
242 
243 	wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);
244 	queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(q->properties.wptr_bo) + wptr_addr_off;
245 
246 	queue_input.is_kfd_process = 1;
247 	queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);
248 	queue_input.queue_size = q->properties.queue_size >> 2;
249 
250 	queue_input.paging = false;
251 	queue_input.tba_addr = qpd->tba_addr;
252 	queue_input.tma_addr = qpd->tma_addr;
253 	queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(q->device);
254 	queue_input.skip_process_ctx_clear =
255 		qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED &&
256 						(qpd->pqm->process->debug_trap_enabled ||
257 						 kfd_dbg_has_ttmps_always_setup(q->device));
258 
259 	queue_type = convert_to_amdgpu_ring_type(q->properties.type);
260 	if (queue_type < 0) {
261 		dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",
262 			q->properties.type);
263 		up_read(&adev->reset_domain->sem);
264 		return -EINVAL;
265 	}
266 	queue_input.queue_type = (uint32_t)queue_type;
267 
268 	queue_input.exclusively_scheduled = q->properties.is_gws;
269 	queue_input.sh_mem_config_data = qpd->sh_mem_config;
270 	queue_input.vm_cntx_cntl = qpd->vm_cntx_cntl;
271 	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
272 
273 	amdgpu_mes_lock(&adev->mes);
274 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
275 	amdgpu_mes_unlock(&adev->mes);
276 	up_read(&adev->reset_domain->sem);
277 	if (r) {
278 		dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n",
279 			q->properties.doorbell_off);
280 		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
281 		kfd_hws_hang(dqm);
282 		return r;
283 	}
284 
285 	/*
286 	 * GFX11: start notify timer only once when oversubscription begins.
287 	 * Skip under SR-IOV: MES round-trips are far slower there, and this
288 	 * notify call shares mes->mutex_hidden with add/remove_hw_queue, so
289 	 * a slow notify can stall remapping queues. SR-IOV guests keep MES's
290 	 * own firmware oversubscription timer instead.
291 	 */
292 	if (!amdgpu_sriov_vf(adev) &&
293 	    KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
294 	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
295 	    dqm->active_cp_queue_count > get_cp_queues_num(dqm))
296 		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
297 				   usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US));
298 
299 	return r;
300 }
301 
302 static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q,
303 					    struct qcm_process_device *qpd,
304 					    bool is_for_reset,
305 					    bool flush_mes_queue)
306 {
307 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
308 	int r;
309 	struct mes_remove_queue_input queue_input;
310 
311 	/* queue was already removed during reset */
312 	if (q->properties.is_reset)
313 		return 0;
314 
315 	if (!dqm->sched_running || dqm->sched_halt)
316 		return 0;
317 	if (!down_read_trylock(&adev->reset_domain->sem))
318 		return -EIO;
319 
320 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
321 	queue_input.doorbell_offset = q->properties.doorbell_off;
322 	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
323 	queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type);
324 	queue_input.remove_queue_after_reset = flush_mes_queue;
325 	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
326 	queue_input.gang_context_array_index = q->gang_ctx_array_index;
327 
328 	amdgpu_mes_lock(&adev->mes);
329 	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
330 	amdgpu_mes_unlock(&adev->mes);
331 	up_read(&adev->reset_domain->sem);
332 
333 	/* If is_for_reset set, it is a mes internal cleanup */
334 	if (!r || is_for_reset)
335 		return r;
336 
337 	/* remove_hw_queue failure indicates a queue hang. reset the queue */
338 	r = reset_queues_mes(dqm, q);
339 	if (r && amdgpu_gpu_recovery) {
340 		dev_err(adev->dev, "failed to remove queue from MES, doorbell=0x%x\n",
341 			q->properties.doorbell_off);
342 		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
343 		kfd_hws_hang(dqm);
344 	}
345 
346 	return r;
347 }
348 
349 static void set_perfcount(struct device_queue_manager *dqm, int enable)
350 {
351 	struct device_process_node *cur;
352 	struct qcm_process_device *qpd;
353 	struct queue *q;
354 	struct mqd_update_info minfo = { 0 };
355 
356 	if (!dqm)
357 		return;
358 
359 	minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE :
360 						 UPDATE_FLAG_PERFCOUNT_DISABLE);
361 	dqm_lock(dqm);
362 	list_for_each_entry(cur, &dqm->queues, list) {
363 		qpd = cur->qpd;
364 		list_for_each_entry(q, &qpd->queues_list, list) {
365 			pqm_update_mqd(qpd->pqm, q->properties.queue_id,
366 						&minfo);
367 		}
368 	}
369 	dqm_unlock(dqm);
370 }
371 
372 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
373 			    struct qcm_process_device *qpd)
374 {
375 	int r = remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
376 
377 	/* GFX11: stop notify timer when oversubscription clears */
378 	if (!r &&
379 	    KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
380 	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
381 	    dqm->active_cp_queue_count <= get_cp_queues_num(dqm))
382 		cancel_delayed_work(&dqm->notify_unmap_work);
383 
384 	return r;
385 }
386 
387 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
388 {
389 	struct device_process_node *cur;
390 	struct device *dev = dqm->dev->adev->dev;
391 	struct qcm_process_device *qpd;
392 	struct queue *q;
393 	int retval = 0;
394 
395 	list_for_each_entry(cur, &dqm->queues, list) {
396 		qpd = cur->qpd;
397 		list_for_each_entry(q, &qpd->queues_list, list) {
398 			if (q->properties.is_active) {
399 				retval = remove_queue_mes(dqm, q, qpd);
400 				if (retval) {
401 					dev_err(dev, "%s: Failed to remove queue %d for dev %d",
402 						__func__,
403 						q->properties.queue_id,
404 						dqm->dev->id);
405 					return retval;
406 				}
407 			}
408 		}
409 	}
410 
411 	return retval;
412 }
413 
414 static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
415 {
416 	struct device_process_node *cur;
417 	struct device *dev = dqm->dev->adev->dev;
418 	struct qcm_process_device *qpd;
419 	struct queue *q;
420 	int retval = 0;
421 
422 	list_for_each_entry(cur, &dqm->queues, list) {
423 		qpd = cur->qpd;
424 		list_for_each_entry(q, &qpd->queues_list, list) {
425 			if (!q->properties.is_active)
426 				continue;
427 			retval = add_queue_mes(dqm, q, qpd);
428 			if (retval) {
429 				dev_err(dev, "%s: Failed to add queue %d for dev %d",
430 					__func__,
431 					q->properties.queue_id,
432 					dqm->dev->id);
433 				return retval;
434 			}
435 		}
436 	}
437 
438 	return retval;
439 }
440 
441 static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q,
442 			   int queue_type, int pipe, int queue, unsigned int db)
443 {
444 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
445 	struct kfd_process_device *pdd;
446 	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
447 	int r;
448 
449 	pdd = kfd_get_process_device_data(q->device, q->process);
450 	if (!pdd)
451 		return -ENODEV;
452 
453 	if (use_mmio)
454 		r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue,
455 						ffs(dqm->dev->xcc_mask) - 1);
456 	else
457 		r = amdgpu_mes_reset_user_queue(adev, queue_type, db,
458 						ffs(dqm->dev->xcc_mask) - 1);
459 	if (r)
460 		return r;
461 	/* Proceed remove_queue with reset=true */
462 	remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true);
463 	set_queue_as_reset(dqm, q, &pdd->qpd, true);
464 	return 0;
465 }
466 
467 int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type,
468 			int pipe, int queue, unsigned int db)
469 {
470 	struct queue *q;
471 
472 	q = find_queue_by_doorbell_offset(dqm, db);
473 	if (!q)
474 		return 0;
475 	return reset_queue_mes(dqm, q, queue_type, pipe, queue, db);
476 }
477 
478 static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q)
479 {
480 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
481 	struct drm_wedge_task_info *info = NULL;
482 	struct amdgpu_task_info *ti = NULL;
483 	struct kfd_process_device *pdd;
484 	unsigned int num_hung = 0;
485 	int r = 0;
486 	struct mes_remove_queue_input queue_input;
487 
488 	if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) {
489 		r = -ENOTRECOVERABLE;
490 		goto fail;
491 	}
492 
493 	/* reset should be used only in dqm locked queue reset */
494 	if (WARN_ON(dqm->detect_hang_count > 0))
495 		return 0;
496 
497 	if (!amdgpu_gpu_recovery) {
498 		r = -ENOTRECOVERABLE;
499 		goto fail;
500 	}
501 
502 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
503 	queue_input.doorbell_offset = q->properties.doorbell_off;
504 	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
505 	queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type);
506 	queue_input.remove_queue_after_reset = false;
507 	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
508 	/* pass the known bad queue info to the reset function */
509 	r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input);
510 	if (r)
511 		goto fail;
512 	pdd = kfd_get_process_device_data(q->device, q->process);
513 	if (pdd) {
514 		ti = amdgpu_vm_get_task_info_pasid(adev, pdd->pasid);
515 		if (ti) {
516 			amdgpu_vm_print_task_info(adev, ti);
517 			info = &ti->task;
518 		}
519 	}
520 
521 	dqm->detect_hang_count = num_hung;
522 	/* When MES doesn't detect any queue hang, no reset happens. Don't signal reset
523 	 * event.
524 	 */
525 	if (dqm->detect_hang_count) {
526 		kfd_signal_reset_event(dqm->dev);
527 		if (pdd && pdd->has_reset_queue) {
528 			atomic_inc(&adev->gpu_reset_counter);
529 			drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);
530 		}
531 	}
532 	amdgpu_vm_put_task_info(ti);
533 
534 fail:
535 	dqm->detect_hang_count = 0;
536 	return r;
537 }
538 
539 static void increment_queue_count(struct device_queue_manager *dqm,
540 				  struct qcm_process_device *qpd,
541 				  struct queue *q)
542 {
543 	dqm->active_queue_count++;
544 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
545 		dqm->active_cp_queue_count++;
546 
547 	if (q->properties.is_gws) {
548 		dqm->gws_queue_count++;
549 		qpd->mapped_gws_queue = true;
550 	}
551 }
552 
553 static void decrement_queue_count(struct device_queue_manager *dqm,
554 				  struct qcm_process_device *qpd,
555 				  struct queue *q)
556 {
557 	dqm->active_queue_count--;
558 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
559 		dqm->active_cp_queue_count--;
560 
561 	if (q->properties.is_gws) {
562 		dqm->gws_queue_count--;
563 		qpd->mapped_gws_queue = false;
564 	}
565 }
566 
567 /*
568  * Allocate a doorbell ID to this queue.
569  * If doorbell_id is passed in, make sure requested ID is valid then allocate it.
570  */
571 static int allocate_doorbell(struct qcm_process_device *qpd,
572 			     struct queue *q,
573 			     uint32_t const *restore_id)
574 {
575 	struct kfd_node *dev = qpd->dqm->dev;
576 
577 	if (!KFD_IS_SOC15(dev)) {
578 		/* On pre-SOC15 chips we need to use the queue ID to
579 		 * preserve the user mode ABI.
580 		 */
581 
582 		if (restore_id && *restore_id != q->properties.queue_id)
583 			return -EINVAL;
584 
585 		q->doorbell_id = q->properties.queue_id;
586 	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
587 			q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
588 		/* For SDMA queues on SOC15 with 8-byte doorbell, use static
589 		 * doorbell assignments based on the engine and queue id.
590 		 * The doobell index distance between RLC (2*i) and (2*i+1)
591 		 * for a SDMA engine is 512.
592 		 */
593 
594 		uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;
595 
596 		/*
597 		 * q->properties.sdma_engine_id corresponds to the virtual
598 		 * sdma engine number. However, for doorbell allocation,
599 		 * we need the physical sdma engine id in order to get the
600 		 * correct doorbell offset.
601 		 */
602 		u32 engine_base = idx_offset[dev->node_id *
603 					     get_num_all_sdma_engines(qpd->dqm) +
604 					     q->properties.sdma_engine_id];
605 		u32 valid_id;
606 
607 		/*
608 		 * With the aqua_vanjaram doorbell layout, a shader doorbell
609 		 * write whose 32-byte block starts in the previous engine's
610 		 * range rings that engine instead. Engine ranges are 10
611 		 * indices apart (sdma_doorbell_range = 20 dwords, halved for
612 		 * 8-byte doorbells, in aqua_vanjaram_doorbell_index_init()),
613 		 * which puts odd engines 2 indices into a shared block.
614 		 * Aligning costs at most 2 indices, and 8 queues per engine use
615 		 * only 4 of the 10, so the aligned base stays in range. The
616 		 * 512-index mirror offset keeps the alignment.
617 		 */
618 		if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) ||
619 		    KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4) ||
620 		    KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0))
621 			engine_base = ALIGN(engine_base,
622 					    KFD_SDMA_SHADER_DOORBELL_GRANULARITY);
623 
624 		valid_id = engine_base +
625 			   (q->properties.sdma_queue_id & 1) *
626 			   KFD_QUEUE_DOORBELL_MIRROR_OFFSET +
627 			   (q->properties.sdma_queue_id >> 1);
628 
629 		if (restore_id && *restore_id != valid_id)
630 			return -EINVAL;
631 		q->doorbell_id = valid_id;
632 	} else {
633 		/* For CP queues on SOC15 */
634 		if (restore_id) {
635 			if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
636 				return -EINVAL;
637 
638 			/* make sure that ID is free  */
639 			if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
640 				return -EINVAL;
641 
642 			q->doorbell_id = *restore_id;
643 		} else {
644 			/* or reserve a free doorbell ID */
645 			unsigned int found;
646 
647 			found = find_first_zero_bit(qpd->doorbell_bitmap,
648 						    KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
649 			if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
650 				pr_debug("No doorbells available");
651 				return -EBUSY;
652 			}
653 			set_bit(found, qpd->doorbell_bitmap);
654 			q->doorbell_id = found;
655 		}
656 	}
657 
658 	q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(dev->adev,
659 								  qpd->proc_doorbells,
660 								  q->doorbell_id,
661 								  dev->kfd->device_info.doorbell_size);
662 	return 0;
663 }
664 
665 static void deallocate_doorbell(struct qcm_process_device *qpd,
666 				struct queue *q)
667 {
668 	unsigned int old;
669 	struct kfd_node *dev = qpd->dqm->dev;
670 
671 	if (!KFD_IS_SOC15(dev) ||
672 	    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
673 	    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
674 		return;
675 
676 	old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
677 	WARN_ON(!old);
678 }
679 
680 static void program_trap_handler_settings(struct device_queue_manager *dqm,
681 				struct qcm_process_device *qpd)
682 {
683 	uint32_t xcc_mask = dqm->dev->xcc_mask;
684 	int xcc_id;
685 
686 	if (dqm->dev->kfd2kgd->program_trap_handler_settings)
687 		for_each_inst(xcc_id, xcc_mask)
688 			dqm->dev->kfd2kgd->program_trap_handler_settings(
689 				dqm->dev->adev, qpd->vmid, qpd->tba_addr,
690 				qpd->tma_addr, xcc_id);
691 }
692 
693 static int allocate_vmid(struct device_queue_manager *dqm,
694 			struct qcm_process_device *qpd,
695 			struct queue *q)
696 {
697 	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
698 	struct device *dev = dqm->dev->adev->dev;
699 	int allocated_vmid = -1, i;
700 
701 	for (i = dqm->dev->vm_info.first_vmid_kfd;
702 			i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
703 		if (!dqm->vmid_pasid[i]) {
704 			allocated_vmid = i;
705 			break;
706 		}
707 	}
708 
709 	if (allocated_vmid < 0) {
710 		dev_err(dev, "no more vmid to allocate\n");
711 		return -ENOSPC;
712 	}
713 
714 	pr_debug("vmid allocated: %d\n", allocated_vmid);
715 
716 	dqm->vmid_pasid[allocated_vmid] = pdd->pasid;
717 
718 	set_pasid_vmid_mapping(dqm, pdd->pasid, allocated_vmid);
719 
720 	qpd->vmid = allocated_vmid;
721 	q->properties.vmid = allocated_vmid;
722 
723 	program_sh_mem_settings(dqm, qpd);
724 
725 	if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)
726 		program_trap_handler_settings(dqm, qpd);
727 
728 	/* qpd->page_table_base is set earlier when register_process()
729 	 * is called, i.e. when the first queue is created.
730 	 */
731 	dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
732 			qpd->vmid,
733 			qpd->page_table_base);
734 	/* invalidate the VM context after pasid and vmid mapping is set up */
735 	kfd_flush_tlb(qpd_to_pdd(qpd));
736 
737 	if (dqm->dev->kfd2kgd->set_scratch_backing_va)
738 		dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
739 				qpd->sh_hidden_private_base, qpd->vmid);
740 
741 	return 0;
742 }
743 
744 static int flush_texture_cache_nocpsch(struct kfd_node *kdev,
745 				struct qcm_process_device *qpd)
746 {
747 	const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
748 	int ret;
749 
750 	if (!qpd->ib_kaddr)
751 		return -ENOMEM;
752 
753 	ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
754 	if (ret)
755 		return ret;
756 
757 	return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid,
758 				qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
759 				pmf->release_mem_size / sizeof(uint32_t));
760 }
761 
762 static void deallocate_vmid(struct device_queue_manager *dqm,
763 				struct qcm_process_device *qpd,
764 				struct queue *q)
765 {
766 	struct device *dev = dqm->dev->adev->dev;
767 
768 	/* On GFX v7, CP doesn't flush TC at dequeue */
769 	if (q->device->adev->asic_type == CHIP_HAWAII)
770 		if (flush_texture_cache_nocpsch(q->device, qpd))
771 			dev_err(dev, "Failed to flush TC\n");
772 
773 	kfd_flush_tlb(qpd_to_pdd(qpd));
774 
775 	/* Release the vmid mapping */
776 	set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
777 	dqm->vmid_pasid[qpd->vmid] = 0;
778 
779 	qpd->vmid = 0;
780 	q->properties.vmid = 0;
781 }
782 
783 static int create_queue_nocpsch(struct device_queue_manager *dqm,
784 				struct queue *q,
785 				struct qcm_process_device *qpd,
786 				const struct kfd_criu_queue_priv_data *qd,
787 				const void *restore_mqd, const void *restore_ctl_stack)
788 {
789 	struct mqd_manager *mqd_mgr;
790 	int retval;
791 
792 	dqm_lock(dqm);
793 
794 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
795 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
796 				dqm->total_queue_count);
797 		retval = -EPERM;
798 		goto out_unlock;
799 	}
800 
801 	if (list_empty(&qpd->queues_list)) {
802 		retval = allocate_vmid(dqm, qpd, q);
803 		if (retval)
804 			goto out_unlock;
805 	}
806 	q->properties.vmid = qpd->vmid;
807 	/*
808 	 * Eviction state logic: mark all queues as evicted, even ones
809 	 * not currently active. Restoring inactive queues later only
810 	 * updates the is_evicted flag but is a no-op otherwise.
811 	 */
812 	q->properties.is_evicted = !!qpd->evicted;
813 
814 	q->properties.tba_addr = qpd->tba_addr;
815 	q->properties.tma_addr = qpd->tma_addr;
816 
817 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
818 			q->properties.type)];
819 	if (qd && !mqd_mgr->restore_mqd) {
820 		pr_debug("restore_mqd not implemented for queue type %d\n",
821 			 q->properties.type);
822 		retval = -EOPNOTSUPP;
823 		goto deallocate_vmid;
824 	}
825 
826 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
827 		retval = allocate_hqd(dqm, q);
828 		if (retval)
829 			goto deallocate_vmid;
830 		pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
831 			q->pipe, q->queue);
832 	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
833 		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
834 		retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
835 		if (retval)
836 			goto deallocate_vmid;
837 		dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
838 	}
839 
840 	retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
841 	if (retval)
842 		goto out_deallocate_hqd;
843 
844 	/* Temporarily release dqm lock to avoid a circular lock dependency */
845 	dqm_unlock(dqm);
846 	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr, &q->properties);
847 	dqm_lock(dqm);
848 
849 	if (!q->mqd_mem_obj) {
850 		retval = -ENOMEM;
851 		goto out_deallocate_doorbell;
852 	}
853 
854 	if (qd)
855 		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
856 				     &q->properties, restore_mqd, restore_ctl_stack,
857 				     qd->ctl_stack_size);
858 	else
859 		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
860 					&q->gart_mqd_addr, &q->properties);
861 
862 	if (q->properties.is_active) {
863 		if (!dqm->sched_running) {
864 			WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
865 			goto add_queue_to_list;
866 		}
867 
868 		if (WARN(q->process->mm != current->mm,
869 					"should only run in user thread"))
870 			retval = -EFAULT;
871 		else
872 			retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
873 					q->queue, &q->properties, current->mm);
874 		if (retval)
875 			goto out_free_mqd;
876 	}
877 
878 add_queue_to_list:
879 	list_add(&q->list, &qpd->queues_list);
880 	qpd->queue_count++;
881 	if (q->properties.is_active)
882 		increment_queue_count(dqm, qpd, q);
883 
884 	/*
885 	 * Unconditionally increment this counter, regardless of the queue's
886 	 * type or whether the queue is active.
887 	 */
888 	dqm->total_queue_count++;
889 	pr_debug("Total of %d queues are accountable so far\n",
890 			dqm->total_queue_count);
891 	goto out_unlock;
892 
893 out_free_mqd:
894 	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
895 out_deallocate_doorbell:
896 	deallocate_doorbell(qpd, q);
897 out_deallocate_hqd:
898 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
899 		deallocate_hqd(dqm, q);
900 	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
901 		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
902 		deallocate_sdma_queue(dqm, q);
903 deallocate_vmid:
904 	if (list_empty(&qpd->queues_list))
905 		deallocate_vmid(dqm, qpd, q);
906 out_unlock:
907 	dqm_unlock(dqm);
908 	return retval;
909 }
910 
911 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
912 {
913 	bool set;
914 	int pipe, bit, i;
915 
916 	set = false;
917 
918 	for (pipe = dqm->next_pipe_to_allocate, i = 0;
919 			i < get_pipes_per_mec(dqm);
920 			pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
921 
922 		if (!is_pipe_enabled(dqm, 0, pipe))
923 			continue;
924 
925 		if (dqm->allocated_queues[pipe] != 0) {
926 			bit = ffs(dqm->allocated_queues[pipe]) - 1;
927 			dqm->allocated_queues[pipe] &= ~(1 << bit);
928 			q->pipe = pipe;
929 			q->queue = bit;
930 			set = true;
931 			break;
932 		}
933 	}
934 
935 	if (!set)
936 		return -EBUSY;
937 
938 	pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
939 	/* horizontal hqd allocation */
940 	dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
941 
942 	return 0;
943 }
944 
945 static inline void deallocate_hqd(struct device_queue_manager *dqm,
946 				struct queue *q)
947 {
948 	dqm->allocated_queues[q->pipe] |= (1 << q->queue);
949 }
950 
951 #define SQ_IND_CMD_CMD_KILL		0x00000003
952 #define SQ_IND_CMD_MODE_BROADCAST	0x00000001
953 
954 static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)
955 {
956 	int status = 0;
957 	unsigned int vmid;
958 	uint16_t queried_pasid;
959 	union SQ_CMD_BITS reg_sq_cmd;
960 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
961 	struct kfd_process_device *pdd;
962 	int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
963 	int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
964 	uint32_t xcc_mask = dev->xcc_mask;
965 	int xcc_id;
966 
967 	reg_sq_cmd.u32All = 0;
968 	reg_gfx_index.u32All = 0;
969 
970 	pr_debug("Killing all process wavefronts\n");
971 
972 	if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) {
973 		dev_err(dev->adev->dev, "no vmid pasid mapping supported\n");
974 		return -EOPNOTSUPP;
975 	}
976 
977 	/* taking the VMID for that process on the safe way using PDD */
978 	pdd = kfd_get_process_device_data(dev, p);
979 	if (!pdd)
980 		return -EFAULT;
981 
982 	/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
983 	 * ATC_VMID15_PASID_MAPPING
984 	 * to check which VMID the current process is mapped to.
985 	 */
986 
987 	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
988 		status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
989 				(dev->adev, vmid, &queried_pasid);
990 
991 		if (status && queried_pasid == pdd->pasid) {
992 			pr_debug("Killing wave fronts of vmid %d and process pid %d\n",
993 					vmid, p->lead_thread->pid);
994 			break;
995 		}
996 	}
997 
998 	if (vmid > last_vmid_to_scan) {
999 		dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n",
1000 				p->lead_thread->pid);
1001 		return -EFAULT;
1002 	}
1003 
1004 	reg_gfx_index.bits.sh_broadcast_writes = 1;
1005 	reg_gfx_index.bits.se_broadcast_writes = 1;
1006 	reg_gfx_index.bits.instance_broadcast_writes = 1;
1007 	reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
1008 	reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
1009 	reg_sq_cmd.bits.vm_id = vmid;
1010 
1011 	for_each_inst(xcc_id, xcc_mask)
1012 		dev->kfd2kgd->wave_control_execute(
1013 			dev->adev, reg_gfx_index.u32All,
1014 			reg_sq_cmd.u32All, xcc_id);
1015 
1016 	return 0;
1017 }
1018 
1019 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
1020  * to avoid asynchronized access
1021  */
1022 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
1023 				struct qcm_process_device *qpd,
1024 				struct queue *q)
1025 {
1026 	int retval;
1027 	struct mqd_manager *mqd_mgr;
1028 
1029 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
1030 
1031 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
1032 		deallocate_hqd(dqm, q);
1033 	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1034 		deallocate_sdma_queue(dqm, q);
1035 	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1036 		deallocate_sdma_queue(dqm, q);
1037 	else {
1038 		pr_debug("q->properties.type %d is invalid\n",
1039 				q->properties.type);
1040 		return -EINVAL;
1041 	}
1042 	dqm->total_queue_count--;
1043 
1044 	deallocate_doorbell(qpd, q);
1045 
1046 	if (!dqm->sched_running) {
1047 		WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
1048 		return 0;
1049 	}
1050 
1051 	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1052 				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
1053 				KFD_UNMAP_LATENCY_MS,
1054 				q->pipe, q->queue);
1055 	if (retval == -ETIME)
1056 		qpd->reset_wavefronts = true;
1057 
1058 	list_del(&q->list);
1059 	if (list_empty(&qpd->queues_list)) {
1060 		if (qpd->reset_wavefronts) {
1061 			pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
1062 					dqm->dev);
1063 			/* dbgdev_wave_reset_wavefronts has to be called before
1064 			 * deallocate_vmid(), i.e. when vmid is still in use.
1065 			 */
1066 			dbgdev_wave_reset_wavefronts(dqm->dev,
1067 					qpd->pqm->process);
1068 			qpd->reset_wavefronts = false;
1069 		}
1070 
1071 		deallocate_vmid(dqm, qpd, q);
1072 	}
1073 	qpd->queue_count--;
1074 	if (q->properties.is_active)
1075 		decrement_queue_count(dqm, qpd, q);
1076 
1077 	return retval;
1078 }
1079 
1080 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
1081 				struct qcm_process_device *qpd,
1082 				struct queue *q)
1083 {
1084 	int retval;
1085 	uint64_t sdma_val = 0;
1086 	struct device *dev = dqm->dev->adev->dev;
1087 	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
1088 	struct mqd_manager *mqd_mgr =
1089 		dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
1090 
1091 	/* Get the SDMA queue stats */
1092 	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
1093 	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1094 		if (dqm->dev->kfd2kgd->hqd_sdma_get_counter)
1095 			retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter(
1096 					dqm->dev->adev, q->mqd,
1097 					dqm->dev->kfd->device_info.num_sdma_queues_per_engine,
1098 					&sdma_val);
1099 		else
1100 			retval = read_sdma_queue_counter(
1101 					(uint64_t __user *)q->properties.read_ptr,
1102 					&sdma_val);
1103 		if (retval)
1104 			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
1105 				q->properties.queue_id);
1106 	}
1107 
1108 	dqm_lock(dqm);
1109 	retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
1110 	if (!retval)
1111 		pdd->sdma_past_activity_counter += sdma_val;
1112 	dqm_unlock(dqm);
1113 
1114 	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
1115 
1116 	return retval;
1117 }
1118 
1119 static int update_queue(struct device_queue_manager *dqm, struct queue *q,
1120 			struct mqd_update_info *minfo)
1121 {
1122 	int retval = 0;
1123 	struct device *dev = dqm->dev->adev->dev;
1124 	struct mqd_manager *mqd_mgr;
1125 	struct kfd_process_device *pdd;
1126 	bool prev_active = false;
1127 
1128 	dqm_lock(dqm);
1129 	pdd = kfd_get_process_device_data(q->device, q->process);
1130 	if (!pdd) {
1131 		retval = -ENODEV;
1132 		goto out_unlock;
1133 	}
1134 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1135 			q->properties.type)];
1136 
1137 	/* Save previous activity state for counters */
1138 	prev_active = q->properties.is_active;
1139 
1140 	/* Make sure the queue is unmapped before updating the MQD */
1141 	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
1142 		if (!dqm->dev->kfd->shared_resources.enable_mes)
1143 			retval = unmap_queues_cpsch(dqm,
1144 						    KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD, false);
1145 		else if (prev_active)
1146 			retval = remove_queue_mes(dqm, q, &pdd->qpd);
1147 
1148 		/* queue is reset so inaccessable  */
1149 		if (pdd->has_reset_queue) {
1150 			retval = -EACCES;
1151 			goto out_unlock;
1152 		}
1153 
1154 		if (retval) {
1155 			dev_err(dev, "unmap queue failed\n");
1156 			goto out_unlock;
1157 		}
1158 	} else if (prev_active &&
1159 		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1160 		    q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1161 		    q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1162 
1163 		if (!dqm->sched_running) {
1164 			WARN_ONCE(1, "Update non-HWS queue while stopped\n");
1165 			goto out_unlock;
1166 		}
1167 
1168 		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1169 				(dqm->dev->kfd->cwsr_enabled ?
1170 				 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
1171 				 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
1172 				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
1173 		if (retval) {
1174 			dev_err(dev, "destroy mqd failed\n");
1175 			goto out_unlock;
1176 		}
1177 	}
1178 
1179 	mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);
1180 
1181 	/*
1182 	 * check active state vs. the previous state and modify
1183 	 * counter accordingly. map_queues_cpsch uses the
1184 	 * dqm->active_queue_count to determine whether a new runlist must be
1185 	 * uploaded.
1186 	 */
1187 	if (q->properties.is_active && !prev_active) {
1188 		increment_queue_count(dqm, &pdd->qpd, q);
1189 	} else if (!q->properties.is_active && prev_active) {
1190 		decrement_queue_count(dqm, &pdd->qpd, q);
1191 	} else if (q->gws && !q->properties.is_gws) {
1192 		if (q->properties.is_active) {
1193 			dqm->gws_queue_count++;
1194 			pdd->qpd.mapped_gws_queue = true;
1195 		}
1196 		q->properties.is_gws = true;
1197 	} else if (!q->gws && q->properties.is_gws) {
1198 		if (q->properties.is_active) {
1199 			dqm->gws_queue_count--;
1200 			pdd->qpd.mapped_gws_queue = false;
1201 		}
1202 		q->properties.is_gws = false;
1203 	}
1204 
1205 	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
1206 		if (!dqm->dev->kfd->shared_resources.enable_mes)
1207 			retval = map_queues_cpsch(dqm);
1208 		else if (q->properties.is_active)
1209 			retval = add_queue_mes(dqm, q, &pdd->qpd);
1210 	} else if (q->properties.is_active &&
1211 		 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1212 		  q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1213 		  q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1214 		if (WARN(q->process->mm != current->mm,
1215 			 "should only run in user thread"))
1216 			retval = -EFAULT;
1217 		else
1218 			retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
1219 						   q->pipe, q->queue,
1220 						   &q->properties, current->mm);
1221 	}
1222 
1223 out_unlock:
1224 	dqm_unlock(dqm);
1225 	return retval;
1226 }
1227 
1228 /* suspend_single_queue does not lock the dqm like the
1229  * evict_process_queues_cpsch or evict_process_queues_nocpsch. You should
1230  * lock the dqm before calling, and unlock after calling.
1231  *
1232  * The reason we don't lock the dqm is because this function may be
1233  * called on multiple queues in a loop, so rather than locking/unlocking
1234  * multiple times, we will just keep the dqm locked for all of the calls.
1235  */
1236 static int suspend_single_queue(struct device_queue_manager *dqm,
1237 				      struct kfd_process_device *pdd,
1238 				      struct queue *q)
1239 {
1240 	bool is_new;
1241 
1242 	if (q->properties.is_suspended)
1243 		return 0;
1244 
1245 	pr_debug("Suspending process pid %d queue [%i]\n",
1246 			pdd->process->lead_thread->pid,
1247 			q->properties.queue_id);
1248 
1249 	is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW);
1250 
1251 	if (is_new || q->properties.is_being_destroyed) {
1252 		pr_debug("Suspend: skip %s queue id %i\n",
1253 				is_new ? "new" : "destroyed",
1254 				q->properties.queue_id);
1255 		return -EBUSY;
1256 	}
1257 
1258 	q->properties.is_suspended = true;
1259 	if (q->properties.is_active) {
1260 		if (dqm->dev->kfd->shared_resources.enable_mes) {
1261 			int r = remove_queue_mes(dqm, q, &pdd->qpd);
1262 
1263 			if (r)
1264 				return r;
1265 		}
1266 
1267 		decrement_queue_count(dqm, &pdd->qpd, q);
1268 		q->properties.is_active = false;
1269 	}
1270 
1271 	return 0;
1272 }
1273 
1274 /* resume_single_queue does not lock the dqm like the functions
1275  * restore_process_queues_cpsch or restore_process_queues_nocpsch. You should
1276  * lock the dqm before calling, and unlock after calling.
1277  *
1278  * The reason we don't lock the dqm is because this function may be
1279  * called on multiple queues in a loop, so rather than locking/unlocking
1280  * multiple times, we will just keep the dqm locked for all of the calls.
1281  */
1282 static int resume_single_queue(struct device_queue_manager *dqm,
1283 				      struct qcm_process_device *qpd,
1284 				      struct queue *q)
1285 {
1286 	struct kfd_process_device *pdd;
1287 
1288 	if (!q->properties.is_suspended)
1289 		return 0;
1290 
1291 	pdd = qpd_to_pdd(qpd);
1292 
1293 	pr_debug("Restoring from suspend process pid %d queue [%i]\n",
1294 			    pdd->process->lead_thread->pid,
1295 			    q->properties.queue_id);
1296 
1297 	q->properties.is_suspended = false;
1298 
1299 	if (QUEUE_IS_ACTIVE(q->properties)) {
1300 		if (dqm->dev->kfd->shared_resources.enable_mes) {
1301 			int r = add_queue_mes(dqm, q, &pdd->qpd);
1302 
1303 			if (r)
1304 				return r;
1305 		}
1306 
1307 		q->properties.is_active = true;
1308 		increment_queue_count(dqm, qpd, q);
1309 	}
1310 
1311 	return 0;
1312 }
1313 
1314 /* Unpin the MQD BO at S4 suspend so it is evicted into the hibernation image;
1315  * dqm_repin_mqd_bo() pins it back on resume. Gated on adev->in_s4 so runtime
1316  * eviction is untouched.
1317  */
1318 static void dqm_evict_mqd_bo(struct device_queue_manager *dqm, struct queue *q)
1319 {
1320 	struct mqd_manager *mqd_mgr;
1321 	struct amdgpu_bo *bo;
1322 
1323 	if (!dqm->dev->adev->in_s4)
1324 		return;
1325 	if (!mqd_on_vram(dqm->dev->adev))
1326 		return;
1327 	if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE)
1328 		return;
1329 	if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem)
1330 		return;
1331 
1332 	/* Without update_mqd_gpu_addr() the MQD self-address cannot be fixed up
1333 	 * after a repin, so skip eviction (with a warning) instead of faulting.
1334 	 */
1335 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
1336 	if (!mqd_mgr->update_mqd_gpu_addr) {
1337 		dev_warn_once(dqm->dev->adev->dev,
1338 			      "MQD is in VRAM but update_mqd_gpu_addr is not implemented; skipping hibernation eviction\n");
1339 		return;
1340 	}
1341 
1342 	bo = q->mqd_mem_obj->mem;
1343 	if (amdgpu_bo_reserve(bo, false))
1344 		return;
1345 
1346 	amdgpu_bo_unpin(bo);
1347 	amdgpu_bo_unreserve(bo);
1348 	q->mqd = NULL;
1349 	q->needs_mqd_repin = true;
1350 }
1351 
1352 /* Repin the MQD BO to VRAM and refresh the cached mapping and GPU addresses.
1353  * Used both on resume and when a queue is destroyed before resume has repinned
1354  * it. A no-op unless a repin is owed (needs_mqd_repin set).
1355  */
1356 static int dqm_repin_mqd_bo(struct device_queue_manager *dqm, struct queue *q)
1357 {
1358 	struct mqd_manager *mqd_mgr;
1359 	struct amdgpu_bo *bo;
1360 	void *cpu_ptr;
1361 	int r;
1362 
1363 	if (!q->needs_mqd_repin)
1364 		return 0;
1365 	if (!q->mqd_mem_obj || !q->mqd_mem_obj->mem)
1366 		return 0;
1367 
1368 	bo = q->mqd_mem_obj->mem;
1369 	r = amdgpu_bo_reserve(bo, false);
1370 	if (r)
1371 		return r;
1372 	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM);
1373 	if (r) {
1374 		amdgpu_bo_unreserve(bo);
1375 		dev_err(dqm->dev->adev->dev,
1376 			"Failed to repin MQD of queue %d to VRAM: %d\n",
1377 			q->properties.queue_id, r);
1378 		return r;
1379 	}
1380 	/* The BO may have moved; refresh the kernel mapping and gpu address. */
1381 	amdgpu_bo_kunmap(bo);
1382 	r = amdgpu_bo_kmap(bo, &cpu_ptr);
1383 	amdgpu_bo_unreserve(bo);
1384 	if (r) {
1385 		dev_err(dqm->dev->adev->dev,
1386 			"Failed to remap MQD of queue %d: %d\n",
1387 			q->properties.queue_id, r);
1388 		return r;
1389 	}
1390 
1391 	q->mqd_mem_obj->cpu_ptr = cpu_ptr;
1392 	q->mqd_mem_obj->gpu_addr = amdgpu_bo_gpu_offset(bo);
1393 	q->gart_mqd_addr = q->mqd_mem_obj->gpu_addr;
1394 	q->mqd = cpu_ptr;
1395 
1396 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1397 			q->properties.type)];
1398 	if (mqd_mgr->update_mqd_gpu_addr)
1399 		mqd_mgr->update_mqd_gpu_addr(mqd_mgr, q->mqd,
1400 					     q->mqd_mem_obj,
1401 					     &q->properties);
1402 
1403 	q->needs_mqd_repin = false;
1404 	return 0;
1405 }
1406 
1407 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
1408 					struct qcm_process_device *qpd)
1409 {
1410 	struct queue *q;
1411 	struct mqd_manager *mqd_mgr;
1412 	struct kfd_process_device *pdd;
1413 	int retval, ret = 0;
1414 
1415 	dqm_lock(dqm);
1416 	if (qpd->evicted++ > 0) /* already evicted, do nothing */
1417 		goto out;
1418 
1419 	pdd = qpd_to_pdd(qpd);
1420 	pr_debug_ratelimited("Evicting process pid %d queues\n",
1421 			    pdd->process->lead_thread->pid);
1422 
1423 	pdd->last_evict_timestamp = get_jiffies_64();
1424 	/* Mark all queues as evicted. Deactivate all active queues on
1425 	 * the qpd.
1426 	 */
1427 	list_for_each_entry(q, &qpd->queues_list, list) {
1428 		q->properties.is_evicted = true;
1429 		if (!q->properties.is_active)
1430 			continue;
1431 
1432 		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1433 				q->properties.type)];
1434 		q->properties.is_active = false;
1435 		decrement_queue_count(dqm, qpd, q);
1436 
1437 		if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
1438 			continue;
1439 
1440 		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1441 				(dqm->dev->kfd->cwsr_enabled ?
1442 				 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
1443 				 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
1444 				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
1445 		if (retval && !ret)
1446 			/* Return the first error, but keep going to
1447 			 * maintain a consistent eviction state
1448 			 */
1449 			ret = retval;
1450 	}
1451 
1452 out:
1453 	dqm_unlock(dqm);
1454 	return ret;
1455 }
1456 
1457 static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
1458 				      struct qcm_process_device *qpd)
1459 {
1460 	struct queue *q;
1461 	struct device *dev = dqm->dev->adev->dev;
1462 	struct kfd_process_device *pdd;
1463 	int retval = 0;
1464 
1465 	dqm_lock(dqm);
1466 	if (qpd->evicted++ > 0) /* already evicted, do nothing */
1467 		goto out;
1468 
1469 	pdd = qpd_to_pdd(qpd);
1470 
1471 	/* The debugger creates processes that temporarily have not acquired
1472 	 * all VMs for all devices and has no VMs itself.
1473 	 * Skip queue eviction on process eviction.
1474 	 */
1475 	if (!pdd->drm_priv)
1476 		goto out;
1477 
1478 	pr_debug_ratelimited("Evicting process pid %d queues\n",
1479 			    pdd->process->lead_thread->pid);
1480 
1481 	if (dqm->dev->kfd->shared_resources.enable_mes)
1482 		pdd->last_evict_timestamp = get_jiffies_64();
1483 
1484 	/* Mark all queues as evicted. Deactivate all active queues on
1485 	 * the qpd.
1486 	 */
1487 	list_for_each_entry(q, &qpd->queues_list, list) {
1488 		q->properties.is_evicted = true;
1489 		if (!q->properties.is_active)
1490 			continue;
1491 
1492 		q->properties.is_active = false;
1493 		decrement_queue_count(dqm, qpd, q);
1494 
1495 		if (dqm->dev->kfd->shared_resources.enable_mes) {
1496 			retval = remove_queue_mes(dqm, q, qpd);
1497 			if (retval) {
1498 				dev_err(dev, "Failed to evict queue %d\n",
1499 					q->properties.queue_id);
1500 				goto out;
1501 			}
1502 		}
1503 
1504 		dqm_evict_mqd_bo(dqm, q);
1505 	}
1506 
1507 	/*
1508 	 * Heavy-weight TLB flush after MES removes queues to ensure
1509 	 * in-flight memory accesses complete before memory is freed/migrated.
1510 	 * HWS does this automatically, MES does not.
1511 	 */
1512 	if (dqm->dev->kfd->shared_resources.enable_mes)
1513 		kfd_flush_tlb(pdd);
1514 
1515 	if (!dqm->dev->kfd->shared_resources.enable_mes) {
1516 		pdd->last_evict_timestamp = get_jiffies_64();
1517 		retval = execute_queues_cpsch(dqm,
1518 					      qpd->is_debug ?
1519 					      KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
1520 					      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
1521 					      USE_DEFAULT_GRACE_PERIOD);
1522 	}
1523 
1524 out:
1525 	dqm_unlock(dqm);
1526 	return retval;
1527 }
1528 
1529 static int clean_process_queues_cpsch(struct device_queue_manager *dqm,
1530 				      struct qcm_process_device *qpd)
1531 {
1532 	struct queue *q;
1533 	struct kfd_process_device *pdd;
1534 	int retval = 0;
1535 
1536 	dqm_lock(dqm);
1537 	if (qpd->evicted++ > 0) /* already evicted, do nothing */
1538 		goto out;
1539 
1540 	pdd = qpd_to_pdd(qpd);
1541 
1542 	/* The debugger creates processes that temporarily have not acquired
1543 	 * all VMs for all devices and has no VMs itself.
1544 	 * Skip queue eviction on process eviction.
1545 	 */
1546 	if (!pdd->drm_priv)
1547 		goto out;
1548 
1549 	pr_debug_ratelimited("Evicting process pid %d queues\n",
1550 			    pdd->process->lead_thread->pid);
1551 
1552 	/* Mark all queues as evicted. Deactivate all active queues on
1553 	 * the qpd.
1554 	 */
1555 	list_for_each_entry(q, &qpd->queues_list, list) {
1556 		q->properties.is_evicted = true;
1557 		if (!q->properties.is_active)
1558 			continue;
1559 
1560 		q->properties.is_active = false;
1561 		decrement_queue_count(dqm, qpd, q);
1562 
1563 		dqm_evict_mqd_bo(dqm, q);
1564 	}
1565 
1566 	pdd->last_evict_timestamp = get_jiffies_64();
1567 
1568 	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) {
1569 		retval =  -EIO;
1570 		goto out;
1571 	}
1572 
1573 	retval = unmap_queues_cpsch(dqm,
1574 			    KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
1575 			    0,
1576 			    USE_DEFAULT_GRACE_PERIOD,
1577 			    false);
1578 
1579 	if (!retval) {
1580 		dqm->dev->kfd2kgd->hqd_gfx_clean_fault(dqm->dev->adev);
1581 		retval = map_queues_cpsch(dqm);
1582 	}
1583 	up_read(&dqm->dev->adev->reset_domain->sem);
1584 
1585 out:
1586 	dqm_unlock(dqm);
1587 	return retval;
1588 }
1589 
1590 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
1591 					  struct qcm_process_device *qpd)
1592 {
1593 	struct mm_struct *mm = NULL;
1594 	struct queue *q;
1595 	struct mqd_manager *mqd_mgr;
1596 	struct kfd_process_device *pdd;
1597 	uint64_t pd_base;
1598 	uint64_t eviction_duration;
1599 	int retval, ret = 0;
1600 
1601 	pdd = qpd_to_pdd(qpd);
1602 	/* Retrieve PD base */
1603 	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1604 
1605 	dqm_lock(dqm);
1606 	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1607 		goto out;
1608 	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1609 		qpd->evicted--;
1610 		goto out;
1611 	}
1612 
1613 	pr_debug_ratelimited("Restoring process pid %d queues\n",
1614 			    pdd->process->lead_thread->pid);
1615 
1616 	/* Update PD Base in QPD */
1617 	qpd->page_table_base = pd_base;
1618 	pr_debug("Updated PD address to 0x%llx\n", pd_base);
1619 
1620 	if (!list_empty(&qpd->queues_list)) {
1621 		dqm->dev->kfd2kgd->set_vm_context_page_table_base(
1622 				dqm->dev->adev,
1623 				qpd->vmid,
1624 				qpd->page_table_base);
1625 		kfd_flush_tlb(pdd);
1626 	}
1627 
1628 	/* Take a safe reference to the mm_struct, which may otherwise
1629 	 * disappear even while the kfd_process is still referenced.
1630 	 */
1631 	mm = get_task_mm(pdd->process->lead_thread);
1632 	if (!mm) {
1633 		ret = -EFAULT;
1634 		goto out;
1635 	}
1636 
1637 	/* Remove the eviction flags. Activate queues that are not
1638 	 * inactive for other reasons.
1639 	 */
1640 	list_for_each_entry(q, &qpd->queues_list, list) {
1641 		q->properties.is_evicted = false;
1642 		if (!QUEUE_IS_ACTIVE(q->properties))
1643 			continue;
1644 
1645 		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1646 				q->properties.type)];
1647 		q->properties.is_active = true;
1648 		increment_queue_count(dqm, qpd, q);
1649 
1650 		if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
1651 			continue;
1652 
1653 		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1654 				       q->queue, &q->properties, mm);
1655 		if (retval && !ret)
1656 			/* Return the first error, but keep going to
1657 			 * maintain a consistent eviction state
1658 			 */
1659 			ret = retval;
1660 	}
1661 	qpd->evicted = 0;
1662 	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1663 	atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1664 out:
1665 	if (mm)
1666 		mmput(mm);
1667 	dqm_unlock(dqm);
1668 	return ret;
1669 }
1670 
1671 static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
1672 					struct qcm_process_device *qpd)
1673 {
1674 	struct queue *q;
1675 	struct device *dev = dqm->dev->adev->dev;
1676 	struct kfd_process_device *pdd;
1677 	uint64_t eviction_duration;
1678 	int retval = 0;
1679 
1680 	pdd = qpd_to_pdd(qpd);
1681 
1682 	dqm_lock(dqm);
1683 	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1684 		goto out;
1685 	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1686 		qpd->evicted--;
1687 		goto out;
1688 	}
1689 
1690 	/* The debugger creates processes that temporarily have not acquired
1691 	 * all VMs for all devices and has no VMs itself.
1692 	 * Skip queue restore on process restore.
1693 	 */
1694 	if (!pdd->drm_priv)
1695 		goto vm_not_acquired;
1696 
1697 	pr_debug_ratelimited("Restoring process pid %d queues\n",
1698 			    pdd->process->lead_thread->pid);
1699 
1700 	/* Update PD Base in QPD */
1701 	qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1702 	pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base);
1703 
1704 	/* activate all active queues on the qpd */
1705 	list_for_each_entry(q, &qpd->queues_list, list) {
1706 		q->properties.is_evicted = false;
1707 		if (!QUEUE_IS_ACTIVE(q->properties))
1708 			continue;
1709 
1710 		q->properties.is_active = true;
1711 		increment_queue_count(dqm, &pdd->qpd, q);
1712 
1713 		retval = dqm_repin_mqd_bo(dqm, q);
1714 		if (retval) {
1715 			dev_err(dev, "Failed to repin MQD for queue %d\n",
1716 				q->properties.queue_id);
1717 			goto out;
1718 		}
1719 
1720 		if (dqm->dev->kfd->shared_resources.enable_mes) {
1721 			retval = add_queue_mes(dqm, q, qpd);
1722 			if (retval) {
1723 				dev_err(dev, "Failed to restore queue %d\n",
1724 					q->properties.queue_id);
1725 				goto out;
1726 			}
1727 		}
1728 	}
1729 	if (!dqm->dev->kfd->shared_resources.enable_mes)
1730 		retval = execute_queues_cpsch(dqm,
1731 					      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
1732 	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1733 	atomic64_add(eviction_duration, &pdd->evict_duration_counter);
1734 vm_not_acquired:
1735 	qpd->evicted = 0;
1736 out:
1737 	dqm_unlock(dqm);
1738 	return retval;
1739 }
1740 
1741 static int register_process(struct device_queue_manager *dqm,
1742 					struct qcm_process_device *qpd)
1743 {
1744 	struct device_process_node *n;
1745 	struct kfd_process_device *pdd;
1746 	uint64_t pd_base;
1747 	int retval;
1748 
1749 	n = kzalloc_obj(*n);
1750 	if (!n)
1751 		return -ENOMEM;
1752 
1753 	n->qpd = qpd;
1754 
1755 	pdd = qpd_to_pdd(qpd);
1756 	/* Retrieve PD base */
1757 	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
1758 
1759 	dqm_lock(dqm);
1760 	list_add(&n->list, &dqm->queues);
1761 
1762 	/* Update PD Base in QPD */
1763 	qpd->page_table_base = pd_base;
1764 	pr_debug("Updated PD address to 0x%llx\n", pd_base);
1765 
1766 	retval = dqm->asic_ops.update_qpd(dqm, qpd);
1767 
1768 	dqm->processes_count++;
1769 
1770 	dqm_unlock(dqm);
1771 
1772 	/* Outside the DQM lock because under the DQM lock we can't do
1773 	 * reclaim or take other locks that others hold while reclaiming.
1774 	 */
1775 	kfd_inc_compute_active(dqm->dev);
1776 
1777 	return retval;
1778 }
1779 
1780 static int unregister_process(struct device_queue_manager *dqm,
1781 					struct qcm_process_device *qpd)
1782 {
1783 	int retval = 0;
1784 	struct device_process_node *cur, *next;
1785 
1786 	pr_debug("qpd->queues_list is %s\n",
1787 			list_empty(&qpd->queues_list) ? "empty" : "not empty");
1788 
1789 	dqm_lock(dqm);
1790 
1791 	list_for_each_entry_safe(cur, next, &dqm->queues, list) {
1792 		if (qpd == cur->qpd) {
1793 			list_del(&cur->list);
1794 			kfree(cur);
1795 			dqm->processes_count--;
1796 			goto out;
1797 		}
1798 	}
1799 	/* qpd not found in dqm list */
1800 	retval = 1;
1801 out:
1802 	dqm_unlock(dqm);
1803 
1804 	/* Outside the DQM lock because under the DQM lock we can't do
1805 	 * reclaim or take other locks that others hold while reclaiming.
1806 	 */
1807 	if (!retval)
1808 		kfd_dec_compute_active(dqm->dev);
1809 
1810 	return retval;
1811 }
1812 
1813 static int
1814 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
1815 			unsigned int vmid)
1816 {
1817 	uint32_t xcc_mask = dqm->dev->xcc_mask;
1818 	int xcc_id, ret = 0;
1819 
1820 	for_each_inst(xcc_id, xcc_mask) {
1821 		ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
1822 			dqm->dev->adev, pasid, vmid, xcc_id);
1823 		if (ret)
1824 			break;
1825 	}
1826 
1827 	return ret;
1828 }
1829 
1830 static void init_interrupts(struct device_queue_manager *dqm)
1831 {
1832 	uint32_t xcc_mask = dqm->dev->xcc_mask;
1833 	unsigned int i, xcc_id;
1834 
1835 	for_each_inst(xcc_id, xcc_mask) {
1836 		for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) {
1837 			if (is_pipe_enabled(dqm, 0, i)) {
1838 				dqm->dev->kfd2kgd->init_interrupts(
1839 					dqm->dev->adev, i, xcc_id);
1840 			}
1841 		}
1842 	}
1843 }
1844 
1845 static int initialize_nocpsch(struct device_queue_manager *dqm)
1846 {
1847 	int pipe, queue;
1848 
1849 	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1850 
1851 	dqm->allocated_queues = kzalloc_objs(unsigned int,
1852 					     get_pipes_per_mec(dqm));
1853 	if (!dqm->allocated_queues)
1854 		return -ENOMEM;
1855 
1856 	mutex_init(&dqm->lock_hidden);
1857 	INIT_LIST_HEAD(&dqm->queues);
1858 	dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
1859 	dqm->active_cp_queue_count = 0;
1860 	dqm->gws_queue_count = 0;
1861 
1862 	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1863 		int pipe_offset = pipe * get_queues_per_pipe(dqm);
1864 
1865 		for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
1866 			if (test_bit(pipe_offset + queue,
1867 				     dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1868 				dqm->allocated_queues[pipe] |= 1 << queue;
1869 	}
1870 
1871 	memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
1872 
1873 	init_sdma_bitmaps(dqm);
1874 
1875 	return 0;
1876 }
1877 
1878 static void uninitialize(struct device_queue_manager *dqm)
1879 {
1880 	int i;
1881 
1882 	WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
1883 
1884 	kfree(dqm->allocated_queues);
1885 	for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
1886 		kfree(dqm->mqd_mgrs[i]);
1887 	mutex_destroy(&dqm->lock_hidden);
1888 }
1889 
1890 static int start_nocpsch(struct device_queue_manager *dqm)
1891 {
1892 	int r = 0;
1893 
1894 	pr_info("SW scheduler is used");
1895 	init_interrupts(dqm);
1896 
1897 	if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1898 		r = pm_init(&dqm->packet_mgr, dqm);
1899 	if (!r)
1900 		dqm->sched_running = true;
1901 
1902 	return r;
1903 }
1904 
1905 static int stop_nocpsch(struct device_queue_manager *dqm)
1906 {
1907 	dqm_lock(dqm);
1908 	if (!dqm->sched_running) {
1909 		dqm_unlock(dqm);
1910 		return 0;
1911 	}
1912 
1913 	if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1914 		pm_uninit(&dqm->packet_mgr);
1915 	dqm->sched_running = false;
1916 	dqm_unlock(dqm);
1917 
1918 	return 0;
1919 }
1920 
1921 static int allocate_sdma_queue(struct device_queue_manager *dqm,
1922 				struct queue *q, const uint32_t *restore_sdma_id)
1923 {
1924 	struct device *dev = dqm->dev->adev->dev;
1925 	int bit;
1926 
1927 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1928 		if (bitmap_empty(dqm->sdma_bitmap, get_num_sdma_queues(dqm))) {
1929 			dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n",
1930 				 get_num_sdma_queues(dqm));
1931 			return -ENOMEM;
1932 		}
1933 
1934 		if (restore_sdma_id) {
1935 			if (*restore_sdma_id >= get_num_sdma_queues(dqm))
1936 				return -EINVAL;
1937 
1938 			/* Re-use existing sdma_id */
1939 			if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
1940 				dev_err(dev, "SDMA queue already in use\n");
1941 				return -EBUSY;
1942 			}
1943 			clear_bit(*restore_sdma_id, dqm->sdma_bitmap);
1944 			q->sdma_id = *restore_sdma_id;
1945 		} else {
1946 			/* Find first available sdma_id */
1947 			bit = find_first_bit(dqm->sdma_bitmap,
1948 					     get_num_sdma_queues(dqm));
1949 			clear_bit(bit, dqm->sdma_bitmap);
1950 			q->sdma_id = bit;
1951 		}
1952 
1953 		q->properties.sdma_engine_id =
1954 			q->sdma_id % kfd_get_num_sdma_engines(dqm->dev);
1955 		q->properties.sdma_queue_id = q->sdma_id /
1956 				kfd_get_num_sdma_engines(dqm->dev);
1957 	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1958 		if (bitmap_empty(dqm->xgmi_sdma_bitmap, get_num_xgmi_sdma_queues(dqm))) {
1959 			dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n",
1960 				 get_num_xgmi_sdma_queues(dqm));
1961 			return -ENOMEM;
1962 		}
1963 		if (restore_sdma_id) {
1964 			if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm))
1965 				return -EINVAL;
1966 
1967 			/* Re-use existing sdma_id */
1968 			if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
1969 				dev_err(dev, "SDMA queue already in use\n");
1970 				return -EBUSY;
1971 			}
1972 			clear_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap);
1973 			q->sdma_id = *restore_sdma_id;
1974 		} else {
1975 			bit = find_first_bit(dqm->xgmi_sdma_bitmap,
1976 					     get_num_xgmi_sdma_queues(dqm));
1977 			clear_bit(bit, dqm->xgmi_sdma_bitmap);
1978 			q->sdma_id = bit;
1979 		}
1980 		/* sdma_engine_id is sdma id including
1981 		 * both PCIe-optimized SDMAs and XGMI-
1982 		 * optimized SDMAs. The calculation below
1983 		 * assumes the first N engines are always
1984 		 * PCIe-optimized ones
1985 		 */
1986 		q->properties.sdma_engine_id =
1987 			kfd_get_num_sdma_engines(dqm->dev) +
1988 			q->sdma_id % kfd_get_num_xgmi_sdma_engines(dqm->dev);
1989 		q->properties.sdma_queue_id = q->sdma_id /
1990 			kfd_get_num_xgmi_sdma_engines(dqm->dev);
1991 	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1992 		int i, num_queues, num_engines, eng_offset = 0, start_engine;
1993 		bool free_bit_found = false, is_xgmi = false;
1994 
1995 		if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(dqm->dev)) {
1996 			num_queues = get_num_sdma_queues(dqm);
1997 			num_engines = kfd_get_num_sdma_engines(dqm->dev);
1998 			q->properties.type = KFD_QUEUE_TYPE_SDMA;
1999 		} else {
2000 			num_queues = get_num_xgmi_sdma_queues(dqm);
2001 			num_engines = kfd_get_num_xgmi_sdma_engines(dqm->dev);
2002 			eng_offset = kfd_get_num_sdma_engines(dqm->dev);
2003 			q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI;
2004 			is_xgmi = true;
2005 		}
2006 
2007 		/* Scan available bit based on target engine ID. */
2008 		start_engine = q->properties.sdma_engine_id - eng_offset;
2009 		for (i = start_engine; i < num_queues; i += num_engines) {
2010 
2011 			if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap))
2012 				continue;
2013 
2014 			clear_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap);
2015 			q->sdma_id = i;
2016 			q->properties.sdma_queue_id = q->sdma_id / num_engines;
2017 			free_bit_found = true;
2018 			break;
2019 		}
2020 
2021 		if (!free_bit_found) {
2022 			dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n",
2023 				 q->properties.sdma_engine_id, num_queues);
2024 			return -ENOMEM;
2025 		}
2026 	}
2027 
2028 	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
2029 	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
2030 
2031 	return 0;
2032 }
2033 
2034 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
2035 				struct queue *q)
2036 {
2037 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
2038 		if (q->sdma_id >= get_num_sdma_queues(dqm))
2039 			return;
2040 		set_bit(q->sdma_id, dqm->sdma_bitmap);
2041 	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
2042 		if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
2043 			return;
2044 		set_bit(q->sdma_id, dqm->xgmi_sdma_bitmap);
2045 	}
2046 }
2047 
2048 /*
2049  * Device Queue Manager implementation for cp scheduler
2050  */
2051 
2052 static int set_sched_resources(struct device_queue_manager *dqm)
2053 {
2054 	int i, mec;
2055 	struct scheduling_resources res;
2056 	struct device *dev = dqm->dev->adev->dev;
2057 
2058 	res.vmid_mask = dqm->dev->compute_vmid_bitmap;
2059 
2060 	res.queue_mask = 0;
2061 	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
2062 		mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
2063 			/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;
2064 
2065 		if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
2066 			continue;
2067 
2068 		/* only acquire queues from the first MEC */
2069 		if (mec > 0)
2070 			continue;
2071 
2072 		/* This situation may be hit in the future if a new HW
2073 		 * generation exposes more than 64 queues. If so, the
2074 		 * definition of res.queue_mask needs updating
2075 		 */
2076 		if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
2077 			dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i);
2078 			break;
2079 		}
2080 
2081 		res.queue_mask |= 1ull
2082 			<< amdgpu_queue_mask_bit_to_set_resource_bit(
2083 				dqm->dev->adev, i);
2084 	}
2085 	res.gws_mask = ~0ull;
2086 	res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
2087 
2088 	pr_debug("Scheduling resources:\n"
2089 			"vmid mask: 0x%8X\n"
2090 			"queue mask: 0x%8llX\n",
2091 			res.vmid_mask, res.queue_mask);
2092 
2093 	return pm_send_set_resources(&dqm->packet_mgr, &res);
2094 }
2095 
2096 static int initialize_cpsch(struct device_queue_manager *dqm)
2097 {
2098 	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
2099 
2100 	mutex_init(&dqm->lock_hidden);
2101 	INIT_LIST_HEAD(&dqm->queues);
2102 	dqm->active_queue_count = dqm->processes_count = 0;
2103 	dqm->active_cp_queue_count = 0;
2104 	dqm->gws_queue_count = 0;
2105 	dqm->active_runlist = false;
2106 	dqm->trap_debug_vmid = 0;
2107 
2108 	init_sdma_bitmaps(dqm);
2109 
2110 	update_dqm_wait_times(dqm);
2111 	return 0;
2112 }
2113 
2114 /* halt_cpsch:
2115  * Unmap queues so the schedule doesn't continue remaining jobs in the queue.
2116  * Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch
2117  * is called.
2118  */
2119 static int halt_cpsch(struct device_queue_manager *dqm)
2120 {
2121 	int ret = 0;
2122 
2123 	dqm_lock(dqm);
2124 	if (!dqm->sched_running) {
2125 		dqm_unlock(dqm);
2126 		return 0;
2127 	}
2128 
2129 	WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n");
2130 
2131 	if (!dqm->is_hws_hang) {
2132 		if (!dqm->dev->kfd->shared_resources.enable_mes)
2133 			ret = unmap_queues_cpsch(dqm,
2134 						 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
2135 				USE_DEFAULT_GRACE_PERIOD, false);
2136 		else
2137 			ret = remove_all_kfd_queues_mes(dqm);
2138 	}
2139 	dqm->sched_halt = true;
2140 	dqm_unlock(dqm);
2141 
2142 	return ret;
2143 }
2144 
2145 /* unhalt_cpsch
2146  * Unset dqm->sched_halt and map queues back to runlist
2147  */
2148 static int unhalt_cpsch(struct device_queue_manager *dqm)
2149 {
2150 	int ret = 0;
2151 	struct amdgpu_device *adev = dqm->dev->adev;
2152 
2153 	dqm_lock(dqm);
2154 	if (!dqm->sched_running || !dqm->sched_halt) {
2155 		dev_dbg(adev->dev, "Scheduling is not on halt.\n");
2156 		dqm_unlock(dqm);
2157 		return 0;
2158 	}
2159 	dqm->sched_halt = false;
2160 	if (!dqm->dev->kfd->shared_resources.enable_mes)
2161 		ret = execute_queues_cpsch(dqm,
2162 					   KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
2163 			0, USE_DEFAULT_GRACE_PERIOD);
2164 	else
2165 		ret = add_all_kfd_queues_mes(dqm);
2166 
2167 	dqm_unlock(dqm);
2168 
2169 	return ret;
2170 }
2171 
2172 /* Program CP_IQ_WAIT_TIME2 via the MES WRITE_REG op (MES has no HIQ). */
2173 static int init_dequeue_wait_counts_mes(struct device_queue_manager *dqm)
2174 {
2175 	struct kfd_node *dev = dqm->dev;
2176 	struct amdgpu_device *adev = dev->adev;
2177 	uint32_t sch_wave = 0, que_sleep = 1;
2178 	int inst, ret;
2179 
2180 	if (!dev->kfd2kgd->build_dequeue_wait_counts_packet_info)
2181 		return 0;
2182 
2183 	if (KFD_GC_VERSION(dev) != IP_VERSION(12, 1, 0))
2184 		return 0;
2185 
2186 	/* CP_IQ_WAIT_TIME2 is per-XCC; the offset must encode the target
2187 	 * XCC so MES routes the write to that XCC's local register.
2188 	 */
2189 	for_each_inst(inst, dev->xcc_mask) {
2190 		uint32_t reg_offset = 0, reg_data = 0;
2191 
2192 		dev->kfd2kgd->build_dequeue_wait_counts_packet_info(
2193 				adev, dqm->wait_times, sch_wave, que_sleep,
2194 				&reg_offset, &reg_data, inst);
2195 
2196 		ret = amdgpu_mes_wreg(adev, reg_offset, reg_data, inst);
2197 		if (ret) {
2198 			dev_err(adev->dev,
2199 				"Failed to set optimized dequeue wait via MES on xcc %d\n",
2200 				inst);
2201 			return ret;
2202 		}
2203 	}
2204 
2205 	update_dqm_wait_times(dqm);
2206 	return 0;
2207 }
2208 
2209 static int start_cpsch(struct device_queue_manager *dqm)
2210 {
2211 	struct device *dev = dqm->dev->adev->dev;
2212 	int retval, num_hw_queue_slots;
2213 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
2214 	int hung_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
2215 	int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
2216 
2217 	dqm_lock(dqm);
2218 
2219 	if (!dqm->dev->kfd->shared_resources.enable_mes) {
2220 		retval = pm_init(&dqm->packet_mgr, dqm);
2221 		if (retval)
2222 			goto fail_packet_manager_init;
2223 
2224 		retval = set_sched_resources(dqm);
2225 		if (retval)
2226 			goto fail_set_sched_resources;
2227 	}
2228 	pr_debug("Allocating fence memory\n");
2229 
2230 	/* allocate fence memory on the gart */
2231 	retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
2232 					&dqm->fence_mem);
2233 
2234 	if (retval)
2235 		goto fail_allocate_vidmem;
2236 
2237 	dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;
2238 	dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
2239 
2240 	init_interrupts(dqm);
2241 
2242 	/* clear hang status when driver try to start the hw scheduler */
2243 	dqm->sched_running = true;
2244 
2245 	if (!dqm->dev->kfd->shared_resources.enable_mes) {
2246 		if (pm_config_dequeue_wait_counts(&dqm->packet_mgr,
2247 				KFD_DEQUEUE_WAIT_INIT, 0 /* unused */))
2248 			dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
2249 		execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
2250 	} else {
2251 		if (init_dequeue_wait_counts_mes(dqm))
2252 			dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
2253 	}
2254 
2255 	/* setup per-queue reset detection buffer  */
2256 	num_hw_queue_slots =  dqm->dev->kfd->shared_resources.num_queue_per_pipe *
2257 			      dqm->dev->kfd->shared_resources.num_pipe_per_mec *
2258 			      NUM_XCC(dqm->dev->xcc_mask);
2259 
2260 	dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info);
2261 	dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL);
2262 
2263 	if (!dqm->detect_hang_info) {
2264 		retval = -ENOMEM;
2265 		goto fail_detect_hang_buffer;
2266 	}
2267 
2268 	dqm->hung_db_array = kzalloc(hung_array_size * sizeof(u32), GFP_KERNEL);
2269 	dqm->hqd_info = kzalloc(
2270 		hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info),
2271 		GFP_KERNEL);
2272 
2273 	dqm_unlock(dqm);
2274 
2275 	return 0;
2276 fail_detect_hang_buffer:
2277 	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
2278 fail_allocate_vidmem:
2279 fail_set_sched_resources:
2280 	if (!dqm->dev->kfd->shared_resources.enable_mes)
2281 		pm_uninit(&dqm->packet_mgr);
2282 fail_packet_manager_init:
2283 	dqm_unlock(dqm);
2284 	return retval;
2285 }
2286 
2287 static int stop_cpsch(struct device_queue_manager *dqm)
2288 {
2289 	int ret = 0;
2290 
2291 	dqm_lock(dqm);
2292 	if (!dqm->sched_running) {
2293 		dqm_unlock(dqm);
2294 		return 0;
2295 	}
2296 
2297 	if (!dqm->dev->kfd->shared_resources.enable_mes)
2298 		ret = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
2299 								0, USE_DEFAULT_GRACE_PERIOD, false);
2300 	else
2301 		ret = remove_all_kfd_queues_mes(dqm);
2302 
2303 	dqm->sched_running = false;
2304 
2305 	if (!dqm->dev->kfd->shared_resources.enable_mes)
2306 		pm_release_ib(&dqm->packet_mgr);
2307 
2308 	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
2309 	if (!dqm->dev->kfd->shared_resources.enable_mes)
2310 		pm_uninit(&dqm->packet_mgr);
2311 	kfree(dqm->detect_hang_info);
2312 	dqm->detect_hang_info = NULL;
2313 	kfree(dqm->hung_db_array);
2314 	kfree(dqm->hqd_info);
2315 
2316 	dqm_unlock(dqm);
2317 
2318 	return ret;
2319 }
2320 
2321 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
2322 					struct kernel_queue *kq,
2323 					struct qcm_process_device *qpd)
2324 {
2325 	dqm_lock(dqm);
2326 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
2327 		pr_warn("Can't create new kernel queue because %d queues were already created\n",
2328 				dqm->total_queue_count);
2329 		dqm_unlock(dqm);
2330 		return -EPERM;
2331 	}
2332 
2333 	/*
2334 	 * Unconditionally increment this counter, regardless of the queue's
2335 	 * type or whether the queue is active.
2336 	 */
2337 	dqm->total_queue_count++;
2338 	pr_debug("Total of %d queues are accountable so far\n",
2339 			dqm->total_queue_count);
2340 
2341 	list_add(&kq->list, &qpd->priv_queue_list);
2342 	increment_queue_count(dqm, qpd, kq->queue);
2343 	qpd->is_debug = true;
2344 	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
2345 			USE_DEFAULT_GRACE_PERIOD);
2346 	dqm_unlock(dqm);
2347 
2348 	return 0;
2349 }
2350 
2351 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
2352 					struct kernel_queue *kq,
2353 					struct qcm_process_device *qpd)
2354 {
2355 	dqm_lock(dqm);
2356 	list_del(&kq->list);
2357 	decrement_queue_count(dqm, qpd, kq->queue);
2358 	qpd->is_debug = false;
2359 	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
2360 			USE_DEFAULT_GRACE_PERIOD);
2361 	/*
2362 	 * Unconditionally decrement this counter, regardless of the queue's
2363 	 * type.
2364 	 */
2365 	dqm->total_queue_count--;
2366 	pr_debug("Total of %d queues are accountable so far\n",
2367 			dqm->total_queue_count);
2368 	dqm_unlock(dqm);
2369 }
2370 
2371 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
2372 			struct qcm_process_device *qpd,
2373 			const struct kfd_criu_queue_priv_data *qd,
2374 			const void *restore_mqd, const void *restore_ctl_stack)
2375 {
2376 	int retval;
2377 	struct mqd_manager *mqd_mgr;
2378 
2379 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
2380 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
2381 				dqm->total_queue_count);
2382 		retval = -EPERM;
2383 		goto out;
2384 	}
2385 
2386 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2387 		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI ||
2388 		q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
2389 		dqm_lock(dqm);
2390 		retval = allocate_sdma_queue(dqm, q, qd ? &qd->sdma_id : NULL);
2391 		dqm_unlock(dqm);
2392 		if (retval)
2393 			goto out;
2394 	}
2395 
2396 	retval = allocate_doorbell(qpd, q, qd ? &qd->doorbell_id : NULL);
2397 	if (retval)
2398 		goto out_deallocate_sdma_queue;
2399 
2400 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2401 			q->properties.type)];
2402 	if (qd && !mqd_mgr->restore_mqd) {
2403 		pr_debug("restore_mqd not implemented for queue type %d\n",
2404 			 q->properties.type);
2405 		retval = -EOPNOTSUPP;
2406 		goto out_deallocate_doorbell;
2407 	}
2408 
2409 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2410 		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2411 		dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
2412 	q->properties.tba_addr = qpd->tba_addr;
2413 	q->properties.tma_addr = qpd->tma_addr;
2414 	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr, &q->properties);
2415 	if (!q->mqd_mem_obj) {
2416 		retval = -ENOMEM;
2417 		goto out_deallocate_doorbell;
2418 	}
2419 
2420 	dqm_lock(dqm);
2421 	/*
2422 	 * Eviction state logic: mark all queues as evicted, even ones
2423 	 * not currently active. Restoring inactive queues later only
2424 	 * updates the is_evicted flag but is a no-op otherwise.
2425 	 */
2426 	q->properties.is_evicted = !!qpd->evicted;
2427 	q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled &&
2428 				  kfd_dbg_has_cwsr_workaround(q->device);
2429 
2430 	if (qd)
2431 		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
2432 				     &q->properties, restore_mqd, restore_ctl_stack,
2433 				     qd->ctl_stack_size);
2434 	else
2435 		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
2436 					&q->gart_mqd_addr, &q->properties);
2437 
2438 	list_add(&q->list, &qpd->queues_list);
2439 	qpd->queue_count++;
2440 
2441 	if (q->properties.is_active) {
2442 		increment_queue_count(dqm, qpd, q);
2443 
2444 		if (!dqm->dev->kfd->shared_resources.enable_mes)
2445 			retval = execute_queues_cpsch(dqm,
2446 					KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
2447 		else
2448 			retval = add_queue_mes(dqm, q, qpd);
2449 		if (retval)
2450 			goto cleanup_queue;
2451 	}
2452 
2453 	/*
2454 	 * Unconditionally increment this counter, regardless of the queue's
2455 	 * type or whether the queue is active.
2456 	 */
2457 	dqm->total_queue_count++;
2458 
2459 	pr_debug("Total of %d queues are accountable so far\n",
2460 			dqm->total_queue_count);
2461 
2462 	dqm_unlock(dqm);
2463 	return retval;
2464 
2465 cleanup_queue:
2466 	qpd->queue_count--;
2467 	list_del(&q->list);
2468 	if (q->properties.is_active)
2469 		decrement_queue_count(dqm, qpd, q);
2470 	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2471 	dqm_unlock(dqm);
2472 out_deallocate_doorbell:
2473 	deallocate_doorbell(qpd, q);
2474 out_deallocate_sdma_queue:
2475 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2476 		q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
2477 		dqm_lock(dqm);
2478 		deallocate_sdma_queue(dqm, q);
2479 		dqm_unlock(dqm);
2480 	}
2481 out:
2482 	return retval;
2483 }
2484 
2485 int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm,
2486 			      uint64_t fence_value,
2487 			      unsigned int timeout_ms)
2488 {
2489 	unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
2490 	struct device *dev = dqm->dev->adev->dev;
2491 	uint64_t *fence_addr = dqm->fence_addr;
2492 
2493 	while (*fence_addr != fence_value) {
2494 		/* Fatal err detected, this response won't come */
2495 		if (amdgpu_amdkfd_is_fed(dqm->dev->adev) ||
2496 		    amdgpu_in_reset(dqm->dev->adev))
2497 			return -EIO;
2498 
2499 		if (time_after(jiffies, end_jiffies)) {
2500 			dev_err(dev, "qcm fence wait loop timeout expired\n");
2501 			/* In HWS case, this is used to halt the driver thread
2502 			 * in order not to mess up CP states before doing
2503 			 * scandumps for FW debugging.
2504 			 */
2505 			while (halt_if_hws_hang)
2506 				schedule();
2507 
2508 			return -ETIME;
2509 		}
2510 		schedule();
2511 	}
2512 
2513 	return 0;
2514 }
2515 
2516 /* dqm->lock mutex has to be locked before calling this function */
2517 static int map_queues_cpsch(struct device_queue_manager *dqm)
2518 {
2519 	struct device *dev = dqm->dev->adev->dev;
2520 	int retval;
2521 
2522 	if (!dqm->sched_running || dqm->sched_halt)
2523 		return 0;
2524 	if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
2525 		return 0;
2526 	if (dqm->active_runlist)
2527 		return 0;
2528 
2529 	retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues);
2530 	pr_debug("%s sent runlist\n", __func__);
2531 	if (retval) {
2532 		dev_err(dev, "failed to execute runlist\n");
2533 		return retval;
2534 	}
2535 	dqm->active_runlist = true;
2536 
2537 	return retval;
2538 }
2539 
2540 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
2541 			       struct qcm_process_device *qpd, bool verbose)
2542 {
2543 	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2544 
2545 	if (verbose)
2546 		dev_err(dqm->dev->adev->dev,
2547 			"queue id 0x%0x at pasid %d is reset\n",
2548 			q->properties.queue_id, pdd->process->lead_thread->pid);
2549 
2550 	pdd->has_reset_queue = true;
2551 	q->properties.is_reset = true;
2552 	if (q->properties.is_active) {
2553 		q->properties.is_active = false;
2554 		decrement_queue_count(dqm, qpd, q);
2555 	}
2556 }
2557 
2558 void kfd_dqm_set_queues_as_reset(struct device_queue_manager *dqm)
2559 {
2560 	struct device_process_node *cur;
2561 	struct qcm_process_device *qpd;
2562 	struct queue *q;
2563 
2564 	dqm_lock(dqm);
2565 	if (dqm->processes_count)
2566 		dev_warn(dqm->dev->adev->dev,
2567 			 "GPU reset: queues of %d process(es) are no longer usable\n",
2568 			 dqm->processes_count);
2569 
2570 	list_for_each_entry(cur, &dqm->queues, list) {
2571 		qpd = cur->qpd;
2572 		list_for_each_entry(q, &qpd->queues_list, list)
2573 			set_queue_as_reset(dqm, q, qpd, false);
2574 	}
2575 	dqm_unlock(dqm);
2576 }
2577 
2578 static int detect_queue_hang(struct device_queue_manager *dqm)
2579 {
2580 	int i;
2581 
2582 	/* detect should be used only in dqm locked queue reset */
2583 	if (WARN_ON(dqm->detect_hang_count > 0))
2584 		return 0;
2585 
2586 	memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size);
2587 
2588 	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
2589 		uint32_t mec, pipe, queue;
2590 		int xcc_id;
2591 
2592 		mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
2593 			/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;
2594 
2595 		if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
2596 			continue;
2597 
2598 		amdgpu_queue_mask_bit_to_mec_queue(dqm->dev->adev, i, &mec, &pipe, &queue);
2599 
2600 		for_each_inst(xcc_id, dqm->dev->xcc_mask) {
2601 			uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr(
2602 						dqm->dev->adev, pipe, queue, xcc_id);
2603 			struct dqm_detect_hang_info hang_info;
2604 
2605 			if (!queue_addr)
2606 				continue;
2607 
2608 			hang_info.pipe_id = pipe;
2609 			hang_info.queue_id = queue;
2610 			hang_info.xcc_id = xcc_id;
2611 			hang_info.queue_address = queue_addr;
2612 
2613 			dqm->detect_hang_info[dqm->detect_hang_count] = hang_info;
2614 			dqm->detect_hang_count++;
2615 		}
2616 	}
2617 
2618 	return dqm->detect_hang_count;
2619 }
2620 
2621 static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address)
2622 {
2623 	struct device_process_node *cur;
2624 	struct qcm_process_device *qpd;
2625 	struct queue *q;
2626 
2627 	list_for_each_entry(cur, &dqm->queues, list) {
2628 		qpd = cur->qpd;
2629 		list_for_each_entry(q, &qpd->queues_list, list) {
2630 			if (queue_address == q->properties.queue_address)
2631 				return q;
2632 		}
2633 	}
2634 
2635 	return NULL;
2636 }
2637 
2638 static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset)
2639 {
2640 	struct device_process_node *cur;
2641 	struct qcm_process_device *qpd;
2642 	struct queue *q;
2643 
2644 	list_for_each_entry(cur, &dqm->queues, list) {
2645 		qpd = cur->qpd;
2646 		list_for_each_entry(q, &qpd->queues_list, list) {
2647 			if (doorbell_offset == q->properties.doorbell_off)
2648 				return q;
2649 		}
2650 	}
2651 
2652 	return NULL;
2653 }
2654 
2655 static int reset_hung_queues(struct device_queue_manager *dqm)
2656 {
2657 	int r = 0, reset_count = 0, i;
2658 
2659 	if (!dqm->detect_hang_info || dqm->is_hws_hang)
2660 		return -EIO;
2661 
2662 	/* assume dqm locked. */
2663 	if (!detect_queue_hang(dqm))
2664 		return -ENOTRECOVERABLE;
2665 
2666 	for (i = 0; i < dqm->detect_hang_count; i++) {
2667 		struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i];
2668 		struct queue *q = find_queue_by_address(dqm, hang_info.queue_address);
2669 		struct kfd_process_device *pdd;
2670 		uint64_t queue_addr = 0;
2671 
2672 		if (!q) {
2673 			r = -ENOTRECOVERABLE;
2674 			goto reset_fail;
2675 		}
2676 
2677 		pdd = kfd_get_process_device_data(dqm->dev, q->process);
2678 		if (!pdd) {
2679 			r = -ENOTRECOVERABLE;
2680 			goto reset_fail;
2681 		}
2682 
2683 		queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev,
2684 				hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id,
2685 				KFD_UNMAP_LATENCY_MS);
2686 
2687 		/* either reset failed or we reset an unexpected queue. */
2688 		if (queue_addr != q->properties.queue_address) {
2689 			r = -ENOTRECOVERABLE;
2690 			goto reset_fail;
2691 		}
2692 
2693 		set_queue_as_reset(dqm, q, &pdd->qpd, true);
2694 		reset_count++;
2695 	}
2696 
2697 	if (reset_count == dqm->detect_hang_count)
2698 		kfd_signal_reset_event(dqm->dev);
2699 	else
2700 		r = -ENOTRECOVERABLE;
2701 
2702 reset_fail:
2703 	dqm->detect_hang_count = 0;
2704 
2705 	return r;
2706 }
2707 
2708 static bool sdma_has_hang(struct device_queue_manager *dqm)
2709 {
2710 	int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2711 	int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2712 	int num_queues_per_eng =  dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2713 	int i, j;
2714 
2715 	for (i = engine_start; i < engine_end; i++) {
2716 		for (j = 0; j < num_queues_per_eng; j++) {
2717 			if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j))
2718 				continue;
2719 
2720 			return true;
2721 		}
2722 	}
2723 
2724 	return false;
2725 }
2726 
2727 static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm,
2728 				    uint32_t doorbell_off)
2729 {
2730 	struct device_process_node *cur;
2731 	struct qcm_process_device *qpd;
2732 	struct queue *q;
2733 
2734 	list_for_each_entry(cur, &dqm->queues, list) {
2735 		qpd = cur->qpd;
2736 		list_for_each_entry(q, &qpd->queues_list, list) {
2737 			if ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2738 			     q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
2739 			     q->properties.doorbell_off == doorbell_off) {
2740 				set_queue_as_reset(dqm, q, qpd, true);
2741 				return true;
2742 			}
2743 		}
2744 	}
2745 
2746 	return false;
2747 }
2748 
2749 static int reset_hung_queues_sdma(struct device_queue_manager *dqm)
2750 {
2751 	int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2752 	int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2753 	int num_queues_per_eng =  dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2754 	int r = 0, i, j;
2755 
2756 	if (dqm->is_hws_hang)
2757 		return -EIO;
2758 
2759 	/* Scan for hung HW queues and reset engine. */
2760 	dqm->detect_hang_count = 0;
2761 	for (i = engine_start; i < engine_end; i++) {
2762 		for (j = 0; j < num_queues_per_eng; j++) {
2763 			uint32_t doorbell_off =
2764 				dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j);
2765 
2766 			if (!doorbell_off)
2767 				continue;
2768 
2769 			/* Reset engine and check. */
2770 			if (amdgpu_sdma_reset_engine(dqm->dev->adev, i) ||
2771 			    dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) ||
2772 			    !set_sdma_queue_as_reset(dqm, doorbell_off)) {
2773 				r = -ENOTRECOVERABLE;
2774 				goto reset_fail;
2775 			}
2776 
2777 			/* Should only expect one queue active per engine */
2778 			dqm->detect_hang_count++;
2779 			break;
2780 		}
2781 	}
2782 
2783 	/* Signal process reset */
2784 	if (dqm->detect_hang_count)
2785 		kfd_signal_reset_event(dqm->dev);
2786 	else
2787 		r = -ENOTRECOVERABLE;
2788 
2789 reset_fail:
2790 	dqm->detect_hang_count = 0;
2791 
2792 	return r;
2793 }
2794 
2795 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma)
2796 {
2797 	struct amdgpu_device *adev = dqm->dev->adev;
2798 
2799 	while (halt_if_hws_hang)
2800 		schedule();
2801 
2802 	if (adev->debug_disable_gpu_ring_reset) {
2803 		dev_info_once(adev->dev,
2804 			      "%s queue hung, but ring reset disabled",
2805 			      is_sdma ? "sdma" : "compute");
2806 
2807 		return -EPERM;
2808 	}
2809 	if (!amdgpu_gpu_recovery)
2810 		return -ENOTRECOVERABLE;
2811 
2812 	return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm);
2813 }
2814 
2815 /* dqm->lock mutex has to be locked before calling this function
2816  *
2817  * @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time
2818  *   for context switch latency. Lower values are used by debugger
2819  *   since context switching are triggered at high frequency.
2820  *   This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE
2821  *
2822  */
2823 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
2824 				enum kfd_unmap_queues_filter filter,
2825 				uint32_t filter_param,
2826 				uint32_t grace_period,
2827 				bool reset)
2828 {
2829 	struct device *dev = dqm->dev->adev->dev;
2830 	struct mqd_manager *mqd_mgr;
2831 	int retval;
2832 
2833 	if (!dqm->sched_running)
2834 		return 0;
2835 	if (!dqm->active_runlist)
2836 		return 0;
2837 	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))
2838 		return -EIO;
2839 
2840 	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2841 		retval = pm_config_dequeue_wait_counts(&dqm->packet_mgr,
2842 				KFD_DEQUEUE_WAIT_SET_SCH_WAVE, grace_period);
2843 		if (retval)
2844 			goto out;
2845 	}
2846 
2847 	retval = pm_send_unmap_queue(&dqm->packet_mgr, filter, filter_param, reset);
2848 	if (retval)
2849 		goto out;
2850 
2851 	*dqm->fence_addr = KFD_FENCE_INIT;
2852 	mb();
2853 	pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr,
2854 				KFD_FENCE_COMPLETED);
2855 	/* should be timed out */
2856 	retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED,
2857 					   queue_preemption_timeout_ms);
2858 	if (retval) {
2859 		dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
2860 		kfd_hws_hang(dqm);
2861 		goto out;
2862 	}
2863 
2864 	/* In the current MEC firmware implementation, if compute queue
2865 	 * doesn't response to the preemption request in time, HIQ will
2866 	 * abandon the unmap request without returning any timeout error
2867 	 * to driver. Instead, MEC firmware will log the doorbell of the
2868 	 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.
2869 	 * To make sure the queue unmap was successful, driver need to
2870 	 * check those fields
2871 	 */
2872 	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
2873 	if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) &&
2874 	    reset_queues_on_hws_hang(dqm, false))
2875 		goto reset_fail;
2876 
2877 	/* Check for SDMA hang and attempt SDMA reset */
2878 	if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, true))
2879 		goto reset_fail;
2880 
2881 	/* We need to reset the grace period value for this device */
2882 	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2883 		if (pm_config_dequeue_wait_counts(&dqm->packet_mgr,
2884 				KFD_DEQUEUE_WAIT_RESET, 0 /* unused */))
2885 			dev_err(dev, "Failed to reset grace period\n");
2886 	}
2887 
2888 	pm_release_ib(&dqm->packet_mgr);
2889 	dqm->active_runlist = false;
2890 out:
2891 	up_read(&dqm->dev->adev->reset_domain->sem);
2892 	return retval;
2893 
2894 reset_fail:
2895 	dqm->is_hws_hang = true;
2896 	kfd_hws_hang(dqm);
2897 	up_read(&dqm->dev->adev->reset_domain->sem);
2898 	return -ETIME;
2899 }
2900 
2901 /* only for compute queue */
2902 static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid)
2903 {
2904 	int retval;
2905 
2906 	dqm_lock(dqm);
2907 
2908 	retval = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_BY_PASID,
2909 			pasid, USE_DEFAULT_GRACE_PERIOD, true);
2910 
2911 	dqm_unlock(dqm);
2912 	return retval;
2913 }
2914 
2915 /* dqm->lock mutex has to be locked before calling this function */
2916 static int execute_queues_cpsch(struct device_queue_manager *dqm,
2917 				enum kfd_unmap_queues_filter filter,
2918 				uint32_t filter_param,
2919 				uint32_t grace_period)
2920 {
2921 	int retval;
2922 
2923 	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem))
2924 		return -EIO;
2925 	retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, false);
2926 	if (!retval)
2927 		retval = map_queues_cpsch(dqm);
2928 	up_read(&dqm->dev->adev->reset_domain->sem);
2929 	return retval;
2930 }
2931 
2932 static int wait_on_destroy_queue(struct device_queue_manager *dqm,
2933 				 struct queue *q)
2934 {
2935 	struct kfd_process_device *pdd = kfd_get_process_device_data(q->device,
2936 								q->process);
2937 	int ret = 0;
2938 
2939 	if (WARN_ON(!pdd))
2940 		return ret;
2941 
2942 	if (pdd->qpd.is_debug)
2943 		return ret;
2944 
2945 	if (q->properties.is_being_destroyed)
2946 		return -EBUSY;
2947 
2948 	q->properties.is_being_destroyed = true;
2949 
2950 	if (pdd->process->debug_trap_enabled && q->properties.is_suspended) {
2951 		dqm_unlock(dqm);
2952 		mutex_unlock(&q->process->mutex);
2953 		ret = wait_event_interruptible(dqm->destroy_wait,
2954 						!q->properties.is_suspended);
2955 
2956 		mutex_lock(&q->process->mutex);
2957 		dqm_lock(dqm);
2958 	}
2959 
2960 	if (ret)
2961 		q->properties.is_being_destroyed = false;
2962 
2963 	return ret;
2964 }
2965 
2966 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
2967 				struct qcm_process_device *qpd,
2968 				struct queue *q)
2969 {
2970 	int retval;
2971 	struct mqd_manager *mqd_mgr;
2972 	uint64_t sdma_val = 0;
2973 	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2974 	struct device *dev = dqm->dev->adev->dev;
2975 
2976 	/* Get the SDMA queue stats */
2977 	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2978 	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2979 		if (dqm->dev->kfd2kgd->hqd_sdma_get_counter)
2980 			retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter(
2981 					dqm->dev->adev, q->mqd,
2982 					dqm->dev->kfd->device_info.num_sdma_queues_per_engine,
2983 					&sdma_val);
2984 		else
2985 			retval = read_sdma_queue_counter(
2986 					(uint64_t __user *)q->properties.read_ptr,
2987 					&sdma_val);
2988 
2989 		if (retval)
2990 			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
2991 				q->properties.queue_id);
2992 	}
2993 
2994 	/* remove queue from list to prevent rescheduling after preemption */
2995 	dqm_lock(dqm);
2996 
2997 	retval = wait_on_destroy_queue(dqm, q);
2998 
2999 	if (retval) {
3000 		dqm_unlock(dqm);
3001 		return retval;
3002 	}
3003 
3004 	if (qpd->is_debug) {
3005 		/*
3006 		 * error, currently we do not allow to destroy a queue
3007 		 * of a currently debugged process
3008 		 */
3009 		retval = -EBUSY;
3010 		goto failed_try_destroy_debugged_queue;
3011 
3012 	}
3013 
3014 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
3015 			q->properties.type)];
3016 
3017 	deallocate_doorbell(qpd, q);
3018 
3019 	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
3020 	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
3021 		deallocate_sdma_queue(dqm, q);
3022 		pdd->sdma_past_activity_counter += sdma_val;
3023 	}
3024 
3025 	if (q->properties.is_active) {
3026 		decrement_queue_count(dqm, qpd, q);
3027 		q->properties.is_active = false;
3028 		if (!dqm->dev->kfd->shared_resources.enable_mes) {
3029 			retval = execute_queues_cpsch(dqm,
3030 						      KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
3031 						      USE_DEFAULT_GRACE_PERIOD);
3032 			if (retval == -ETIME)
3033 				qpd->reset_wavefronts = true;
3034 		} else {
3035 			retval = remove_queue_mes(dqm, q, qpd);
3036 		}
3037 	}
3038 	list_del(&q->list);
3039 	qpd->queue_count--;
3040 
3041 	/*
3042 	 * Unconditionally decrement this counter, regardless of the queue's
3043 	 * type
3044 	 */
3045 	dqm->total_queue_count--;
3046 	pr_debug("Total of %d queues are accountable so far\n",
3047 			dqm->total_queue_count);
3048 
3049 	dqm_unlock(dqm);
3050 
3051 	/*
3052 	 * Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid
3053 	 * circular locking
3054 	 */
3055 	kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE),
3056 				qpd->pqm->process, q->device,
3057 				-1, false, NULL, 0);
3058 
3059 	/* Repin the MQD BO if still evicted for hibernation, before it is freed. */
3060 	dqm_repin_mqd_bo(dqm, q);
3061 	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
3062 
3063 	return retval;
3064 
3065 failed_try_destroy_debugged_queue:
3066 	q->properties.is_being_destroyed = false;
3067 	dqm_unlock(dqm);
3068 	return retval;
3069 }
3070 
3071 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
3072 				   struct qcm_process_device *qpd,
3073 				   enum cache_policy default_policy,
3074 				   enum cache_policy alternate_policy,
3075 				   void __user *alternate_aperture_base,
3076 				   uint64_t alternate_aperture_size,
3077 				   u32 misc_process_properties)
3078 {
3079 	bool retval = true;
3080 
3081 	if (!dqm->asic_ops.set_cache_memory_policy)
3082 		return retval;
3083 
3084 	dqm_lock(dqm);
3085 
3086 	retval = dqm->asic_ops.set_cache_memory_policy(
3087 			dqm,
3088 			qpd,
3089 			default_policy,
3090 			alternate_policy,
3091 			alternate_aperture_base,
3092 			alternate_aperture_size,
3093 			misc_process_properties);
3094 
3095 	if (retval)
3096 		goto out;
3097 
3098 	if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
3099 		program_sh_mem_settings(dqm, qpd);
3100 
3101 	pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
3102 		qpd->sh_mem_config, qpd->sh_mem_ape1_base,
3103 		qpd->sh_mem_ape1_limit);
3104 
3105 out:
3106 	dqm_unlock(dqm);
3107 	return retval;
3108 }
3109 
3110 static int process_termination_nocpsch(struct device_queue_manager *dqm,
3111 		struct qcm_process_device *qpd)
3112 {
3113 	struct queue *q;
3114 	struct device_process_node *cur, *next_dpn;
3115 	int retval = 0;
3116 	bool found = false;
3117 
3118 	dqm_lock(dqm);
3119 
3120 	/* Clear all user mode queues */
3121 	while (!list_empty(&qpd->queues_list)) {
3122 		struct mqd_manager *mqd_mgr;
3123 		int ret;
3124 
3125 		q = list_first_entry(&qpd->queues_list, struct queue, list);
3126 		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
3127 				q->properties.type)];
3128 		ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
3129 		if (ret)
3130 			retval = ret;
3131 		dqm_unlock(dqm);
3132 		mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
3133 		dqm_lock(dqm);
3134 	}
3135 
3136 	/* Unregister process */
3137 	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
3138 		if (qpd == cur->qpd) {
3139 			list_del(&cur->list);
3140 			kfree(cur);
3141 			dqm->processes_count--;
3142 			found = true;
3143 			break;
3144 		}
3145 	}
3146 
3147 	dqm_unlock(dqm);
3148 
3149 	/* Outside the DQM lock because under the DQM lock we can't do
3150 	 * reclaim or take other locks that others hold while reclaiming.
3151 	 */
3152 	if (found)
3153 		kfd_dec_compute_active(dqm->dev);
3154 
3155 	return retval;
3156 }
3157 
3158 static int get_wave_state(struct device_queue_manager *dqm,
3159 			  struct queue *q,
3160 			  void __user *ctl_stack,
3161 			  u32 *ctl_stack_used_size,
3162 			  u32 *save_area_used_size)
3163 {
3164 	struct mqd_manager *mqd_mgr;
3165 
3166 	dqm_lock(dqm);
3167 
3168 	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
3169 
3170 	if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
3171 	    q->properties.is_active || !q->device->kfd->cwsr_enabled ||
3172 	    !mqd_mgr->get_wave_state) {
3173 		dqm_unlock(dqm);
3174 		return -EINVAL;
3175 	}
3176 
3177 	dqm_unlock(dqm);
3178 
3179 	/*
3180 	 * get_wave_state is outside the dqm lock to prevent circular locking
3181 	 * and the queue should be protected against destruction by the process
3182 	 * lock.
3183 	 */
3184 	return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties,
3185 			ctl_stack, ctl_stack_used_size, save_area_used_size);
3186 }
3187 
3188 static int get_queue_checkpoint_info(struct device_queue_manager *dqm,
3189 			const struct queue *q,
3190 			u32 *mqd_size,
3191 			u32 *ctl_stack_size)
3192 {
3193 	struct mqd_manager *mqd_mgr;
3194 	enum KFD_MQD_TYPE mqd_type =
3195 			get_mqd_type_from_queue_type(q->properties.type);
3196 	int ret = 0;
3197 
3198 	dqm_lock(dqm);
3199 	mqd_mgr = dqm->mqd_mgrs[mqd_type];
3200 	*mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask);
3201 	*ctl_stack_size = 0;
3202 
3203 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
3204 		ret = mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
3205 
3206 	dqm_unlock(dqm);
3207 
3208 	return ret;
3209 }
3210 
3211 static int checkpoint_mqd(struct device_queue_manager *dqm,
3212 			  const struct queue *q,
3213 			  void *mqd,
3214 			  void *ctl_stack)
3215 {
3216 	struct mqd_manager *mqd_mgr;
3217 	int r = 0;
3218 	enum KFD_MQD_TYPE mqd_type =
3219 			get_mqd_type_from_queue_type(q->properties.type);
3220 
3221 	dqm_lock(dqm);
3222 
3223 	if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {
3224 		r = -EINVAL;
3225 		goto dqm_unlock;
3226 	}
3227 
3228 	mqd_mgr = dqm->mqd_mgrs[mqd_type];
3229 	if (!mqd_mgr->checkpoint_mqd) {
3230 		r = -EOPNOTSUPP;
3231 		goto dqm_unlock;
3232 	}
3233 
3234 	mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
3235 
3236 dqm_unlock:
3237 	dqm_unlock(dqm);
3238 	return r;
3239 }
3240 
3241 static int process_termination_cpsch(struct device_queue_manager *dqm,
3242 		struct qcm_process_device *qpd)
3243 {
3244 	int retval = 0;
3245 	struct queue *q;
3246 	struct device *dev = dqm->dev->adev->dev;
3247 	struct kernel_queue *kq, *kq_next;
3248 	struct mqd_manager *mqd_mgr;
3249 	struct device_process_node *cur, *next_dpn;
3250 	enum kfd_unmap_queues_filter filter =
3251 		KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
3252 	bool found = false;
3253 
3254 	dqm_lock(dqm);
3255 
3256 	/* Clean all kernel queues */
3257 	list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
3258 		list_del(&kq->list);
3259 		decrement_queue_count(dqm, qpd, kq->queue);
3260 		qpd->is_debug = false;
3261 		dqm->total_queue_count--;
3262 		filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
3263 	}
3264 
3265 	/* Clear all user mode queues */
3266 	list_for_each_entry(q, &qpd->queues_list, list) {
3267 		if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
3268 			deallocate_sdma_queue(dqm, q);
3269 		else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
3270 			deallocate_sdma_queue(dqm, q);
3271 
3272 		if (q->properties.is_active) {
3273 			decrement_queue_count(dqm, qpd, q);
3274 
3275 			if (dqm->dev->kfd->shared_resources.enable_mes) {
3276 				retval = remove_queue_mes(dqm, q, qpd);
3277 				if (retval)
3278 					dev_err(dev, "Failed to remove queue %d\n",
3279 						q->properties.queue_id);
3280 			}
3281 		}
3282 
3283 		dqm->total_queue_count--;
3284 	}
3285 
3286 	/* Unregister process */
3287 	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
3288 		if (qpd == cur->qpd) {
3289 			list_del(&cur->list);
3290 			kfree(cur);
3291 			dqm->processes_count--;
3292 			found = true;
3293 			break;
3294 		}
3295 	}
3296 
3297 	if (!dqm->dev->kfd->shared_resources.enable_mes)
3298 		retval = execute_queues_cpsch(dqm, filter, 0, USE_DEFAULT_GRACE_PERIOD);
3299 
3300 	if ((retval || qpd->reset_wavefronts) &&
3301 	    down_read_trylock(&dqm->dev->adev->reset_domain->sem)) {
3302 		pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
3303 		dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
3304 		qpd->reset_wavefronts = false;
3305 		up_read(&dqm->dev->adev->reset_domain->sem);
3306 	}
3307 
3308 	/* Lastly, free mqd resources.
3309 	 * Do free_mqd() after dqm_unlock to avoid circular locking.
3310 	 */
3311 	while (!list_empty(&qpd->queues_list)) {
3312 		q = list_first_entry(&qpd->queues_list, struct queue, list);
3313 		mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
3314 				q->properties.type)];
3315 		list_del(&q->list);
3316 		qpd->queue_count--;
3317 		dqm_unlock(dqm);
3318 		/* Repin the MQD BO if still evicted for hibernation, before free. */
3319 		dqm_repin_mqd_bo(dqm, q);
3320 		mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
3321 		dqm_lock(dqm);
3322 	}
3323 	dqm_unlock(dqm);
3324 
3325 	/* Outside the DQM lock because under the DQM lock we can't do
3326 	 * reclaim or take other locks that others hold while reclaiming.
3327 	 */
3328 	if (found)
3329 		kfd_dec_compute_active(dqm->dev);
3330 
3331 	return retval;
3332 }
3333 
3334 static int init_mqd_managers(struct device_queue_manager *dqm)
3335 {
3336 	int i, j;
3337 	struct device *dev = dqm->dev->adev->dev;
3338 	struct mqd_manager *mqd_mgr;
3339 
3340 	for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
3341 		mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
3342 		if (!mqd_mgr) {
3343 			dev_err(dev, "mqd manager [%d] initialization failed\n", i);
3344 			goto out_free;
3345 		}
3346 		dqm->mqd_mgrs[i] = mqd_mgr;
3347 	}
3348 
3349 	return 0;
3350 
3351 out_free:
3352 	for (j = 0; j < i; j++) {
3353 		kfree(dqm->mqd_mgrs[j]);
3354 		dqm->mqd_mgrs[j] = NULL;
3355 	}
3356 
3357 	return -ENOMEM;
3358 }
3359 
3360 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
3361 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
3362 {
3363 	int retval;
3364 	struct kfd_node *dev = dqm->dev;
3365 	struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
3366 	uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
3367 		get_num_all_sdma_engines(dqm) *
3368 		dev->kfd->device_info.num_sdma_queues_per_engine +
3369 		(dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size *
3370 		NUM_XCC(dqm->dev->xcc_mask));
3371 
3372 	retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev, size,
3373 		AMDGPU_GEM_DOMAIN_GTT,
3374 		&(mem_obj->mem), &(mem_obj->gpu_addr),
3375 		(void *)&(mem_obj->cpu_ptr), false);
3376 
3377 	return retval;
3378 }
3379 
3380 static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
3381 				    struct kfd_mem_obj *mqd)
3382 {
3383 	WARN(!mqd, "No hiq sdma mqd trunk to free");
3384 
3385 	amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem);
3386 }
3387 
3388 static void mes_notify_unmap_work_handler(struct work_struct *work)
3389 {
3390 	struct device_queue_manager *dqm =
3391 		container_of(work, struct device_queue_manager,
3392 			     notify_unmap_work.work);
3393 
3394 	amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev);
3395 
3396 	/* Re-arm if still oversubscribed */
3397 	if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm))
3398 		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
3399 				   usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US));
3400 }
3401 
3402 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
3403 {
3404 	struct device_queue_manager *dqm;
3405 	int i;
3406 
3407 	pr_debug("Loading device queue manager\n");
3408 
3409 	dqm = kzalloc_obj(*dqm);
3410 	if (!dqm)
3411 		return NULL;
3412 
3413 	switch (dev->adev->asic_type) {
3414 	/* HWS is not available on Hawaii. */
3415 	case CHIP_HAWAII:
3416 	/* HWS depends on CWSR for timely dequeue. CWSR is not
3417 	 * available on Tonga.
3418 	 *
3419 	 * FIXME: This argument also applies to Kaveri.
3420 	 */
3421 	case CHIP_TONGA:
3422 		dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
3423 		break;
3424 	default:
3425 		dqm->sched_policy = sched_policy;
3426 		break;
3427 	}
3428 
3429 	dqm->dev = dev;
3430 	switch (dqm->sched_policy) {
3431 	case KFD_SCHED_POLICY_HWS:
3432 	case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
3433 		/* initialize dqm for cp scheduling */
3434 		dqm->ops.create_queue = create_queue_cpsch;
3435 		dqm->ops.initialize = initialize_cpsch;
3436 		dqm->ops.start = start_cpsch;
3437 		dqm->ops.stop = stop_cpsch;
3438 		dqm->ops.halt = halt_cpsch;
3439 		dqm->ops.unhalt = unhalt_cpsch;
3440 		dqm->ops.destroy_queue = destroy_queue_cpsch;
3441 		dqm->ops.update_queue = update_queue;
3442 		dqm->ops.register_process = register_process;
3443 		dqm->ops.unregister_process = unregister_process;
3444 		dqm->ops.uninitialize = uninitialize;
3445 		dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
3446 		dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
3447 		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
3448 		dqm->ops.process_termination = process_termination_cpsch;
3449 		dqm->ops.evict_process_queues = evict_process_queues_cpsch;
3450 		dqm->ops.restore_process_queues = restore_process_queues_cpsch;
3451 		dqm->ops.get_wave_state = get_wave_state;
3452 		dqm->ops.reset_queues = reset_queues_cpsch;
3453 		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
3454 		dqm->ops.checkpoint_mqd = checkpoint_mqd;
3455 		dqm->ops.set_perfcount = set_perfcount;
3456 		dqm->ops.clean_process_queues_cpsch = clean_process_queues_cpsch;
3457 		break;
3458 	case KFD_SCHED_POLICY_NO_HWS:
3459 		/* initialize dqm for no cp scheduling */
3460 		dqm->ops.start = start_nocpsch;
3461 		dqm->ops.stop = stop_nocpsch;
3462 		dqm->ops.create_queue = create_queue_nocpsch;
3463 		dqm->ops.destroy_queue = destroy_queue_nocpsch;
3464 		dqm->ops.update_queue = update_queue;
3465 		dqm->ops.register_process = register_process;
3466 		dqm->ops.unregister_process = unregister_process;
3467 		dqm->ops.initialize = initialize_nocpsch;
3468 		dqm->ops.uninitialize = uninitialize;
3469 		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
3470 		dqm->ops.process_termination = process_termination_nocpsch;
3471 		dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
3472 		dqm->ops.restore_process_queues =
3473 			restore_process_queues_nocpsch;
3474 		dqm->ops.get_wave_state = get_wave_state;
3475 		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
3476 		dqm->ops.checkpoint_mqd = checkpoint_mqd;
3477 		dqm->ops.set_perfcount = set_perfcount;
3478 		dqm->ops.clean_process_queues_cpsch = clean_process_queues_cpsch;
3479 		break;
3480 	default:
3481 		dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
3482 		goto out_free;
3483 	}
3484 
3485 	switch (dev->adev->asic_type) {
3486 	case CHIP_KAVERI:
3487 	case CHIP_HAWAII:
3488 		device_queue_manager_init_cik(&dqm->asic_ops);
3489 		break;
3490 
3491 	case CHIP_CARRIZO:
3492 	case CHIP_TONGA:
3493 	case CHIP_FIJI:
3494 	case CHIP_POLARIS10:
3495 	case CHIP_POLARIS11:
3496 	case CHIP_POLARIS12:
3497 	case CHIP_VEGAM:
3498 		device_queue_manager_init_vi(&dqm->asic_ops);
3499 		break;
3500 
3501 	default:
3502 		if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 1, 0))
3503 			device_queue_manager_init_v12_1(&dqm->asic_ops);
3504 		else if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))
3505 			device_queue_manager_init_v12(&dqm->asic_ops);
3506 		else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))
3507 			device_queue_manager_init_v11(&dqm->asic_ops);
3508 		else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
3509 			device_queue_manager_init_v10(&dqm->asic_ops);
3510 		else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
3511 			device_queue_manager_init_v9(&dqm->asic_ops);
3512 		else {
3513 			WARN(1, "Unexpected ASIC family %u",
3514 			     dev->adev->asic_type);
3515 			goto out_free;
3516 		}
3517 	}
3518 
3519 	if (init_mqd_managers(dqm))
3520 		goto out_free;
3521 
3522 	if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
3523 		dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n");
3524 		goto out_free;
3525 	}
3526 
3527 	if (!dqm->ops.initialize(dqm)) {
3528 		init_waitqueue_head(&dqm->destroy_wait);
3529 		INIT_DELAYED_WORK(&dqm->notify_unmap_work,
3530 				  mes_notify_unmap_work_handler);
3531 		return dqm;
3532 	}
3533 
3534 	if (!dev->kfd->shared_resources.enable_mes)
3535 		deallocate_hiq_sdma_mqd(dev, &dqm->hiq_sdma_mqd);
3536 
3537 out_free:
3538 	for (i = 0; i < KFD_MQD_TYPE_MAX; i++)
3539 		kfree(dqm->mqd_mgrs[i]);
3540 
3541 	kfree(dqm);
3542 	return NULL;
3543 }
3544 
3545 void device_queue_manager_uninit(struct device_queue_manager *dqm)
3546 {
3547 	cancel_delayed_work_sync(&dqm->notify_unmap_work);
3548 	dqm->ops.stop(dqm);
3549 	dqm->ops.uninitialize(dqm);
3550 	if (!dqm->dev->kfd->shared_resources.enable_mes)
3551 		deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
3552 	kfree(dqm);
3553 }
3554 
3555 /* bad queue notified by interrupt from CP */
3556 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id)
3557 {
3558 	struct kfd_process_device *pdd = NULL;
3559 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd);
3560 	struct device_queue_manager *dqm = knode->dqm;
3561 	struct qcm_process_device *qpd;
3562 	struct queue *q = NULL;
3563 	int ret = 0;
3564 
3565 	if (!pdd)
3566 		return -EINVAL;
3567 
3568 	dqm_lock(dqm);
3569 
3570 	if (pdd) {
3571 		qpd = &pdd->qpd;
3572 
3573 		list_for_each_entry(q, &qpd->queues_list, list) {
3574 			if (q->doorbell_id == doorbell_id && q->properties.is_active) {
3575 				reset_queues_mes(dqm, q);
3576 				q->properties.is_evicted = true;
3577 				q->properties.is_active = false;
3578 				decrement_queue_count(dqm, qpd, q);
3579 				break;
3580 			}
3581 		}
3582 	}
3583 
3584 	dqm_unlock(dqm);
3585 	kfd_unref_process(p);
3586 	return ret;
3587 }
3588 
3589 int kfd_evict_process_device(struct kfd_process_device *pdd)
3590 {
3591 	struct device_queue_manager *dqm;
3592 	struct kfd_process *p;
3593 
3594 	p = pdd->process;
3595 	dqm = pdd->dev->dqm;
3596 
3597 	WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
3598 
3599 	return dqm->ops.evict_process_queues(dqm, &pdd->qpd);
3600 }
3601 
3602 int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
3603 				struct qcm_process_device *qpd)
3604 {
3605 	int r;
3606 	struct device *dev = dqm->dev->adev->dev;
3607 	int updated_vmid_mask;
3608 
3609 	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3610 		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3611 		return -EINVAL;
3612 	}
3613 
3614 	dqm_lock(dqm);
3615 
3616 	if (dqm->trap_debug_vmid != 0) {
3617 		dev_err(dev, "Trap debug id already reserved\n");
3618 		r = -EBUSY;
3619 		goto out_unlock;
3620 	}
3621 
3622 	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
3623 			USE_DEFAULT_GRACE_PERIOD, false);
3624 	if (r)
3625 		goto out_unlock;
3626 
3627 	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3628 	updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd);
3629 
3630 	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3631 	dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd;
3632 	r = set_sched_resources(dqm);
3633 	if (r)
3634 		goto out_unlock;
3635 
3636 	r = map_queues_cpsch(dqm);
3637 	if (r)
3638 		goto out_unlock;
3639 
3640 	pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid);
3641 
3642 out_unlock:
3643 	dqm_unlock(dqm);
3644 	return r;
3645 }
3646 
3647 /*
3648  * Releases vmid for the trap debugger
3649  */
3650 int release_debug_trap_vmid(struct device_queue_manager *dqm,
3651 			struct qcm_process_device *qpd)
3652 {
3653 	struct device *dev = dqm->dev->adev->dev;
3654 	int r;
3655 	int updated_vmid_mask;
3656 	uint32_t trap_debug_vmid;
3657 
3658 	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3659 		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3660 		return -EINVAL;
3661 	}
3662 
3663 	dqm_lock(dqm);
3664 	trap_debug_vmid = dqm->trap_debug_vmid;
3665 	if (dqm->trap_debug_vmid == 0) {
3666 		dev_err(dev, "Trap debug id is not reserved\n");
3667 		r = -EINVAL;
3668 		goto out_unlock;
3669 	}
3670 
3671 	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0,
3672 			USE_DEFAULT_GRACE_PERIOD, false);
3673 	if (r)
3674 		goto out_unlock;
3675 
3676 	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3677 	updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd);
3678 
3679 	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3680 	dqm->trap_debug_vmid = 0;
3681 	r = set_sched_resources(dqm);
3682 	if (r)
3683 		goto out_unlock;
3684 
3685 	r = map_queues_cpsch(dqm);
3686 	if (r)
3687 		goto out_unlock;
3688 
3689 	pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid);
3690 
3691 out_unlock:
3692 	dqm_unlock(dqm);
3693 	return r;
3694 }
3695 
3696 #define QUEUE_NOT_FOUND		-1
3697 /* invalidate queue operation in array */
3698 static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids)
3699 {
3700 	int i;
3701 
3702 	for (i = 0; i < num_queues; i++)
3703 		queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK;
3704 }
3705 
3706 /* find queue index in array */
3707 static int q_array_get_index(unsigned int queue_id,
3708 		uint32_t num_queues,
3709 		uint32_t *queue_ids)
3710 {
3711 	int i;
3712 
3713 	for (i = 0; i < num_queues; i++)
3714 		if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK))
3715 			return i;
3716 
3717 	return QUEUE_NOT_FOUND;
3718 }
3719 
3720 struct copy_context_work_handler_workarea {
3721 	struct work_struct copy_context_work;
3722 	struct kfd_process *p;
3723 };
3724 
3725 static void copy_context_work_handler(struct work_struct *work)
3726 {
3727 	struct copy_context_work_handler_workarea *workarea;
3728 	struct mqd_manager *mqd_mgr;
3729 	struct queue *q;
3730 	struct mm_struct *mm;
3731 	struct kfd_process *p;
3732 	uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size;
3733 	int i;
3734 
3735 	workarea = container_of(work,
3736 			struct copy_context_work_handler_workarea,
3737 			copy_context_work);
3738 
3739 	p = workarea->p;
3740 	mm = get_task_mm(p->lead_thread);
3741 
3742 	if (!mm)
3743 		return;
3744 
3745 	kthread_use_mm(mm);
3746 	for (i = 0; i < p->n_pdds; i++) {
3747 		struct kfd_process_device *pdd = p->pdds[i];
3748 		struct device_queue_manager *dqm = pdd->dev->dqm;
3749 		struct qcm_process_device *qpd = &pdd->qpd;
3750 
3751 		list_for_each_entry(q, &qpd->queues_list, list) {
3752 			if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE)
3753 				continue;
3754 
3755 			mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
3756 
3757 			/* We ignore the return value from get_wave_state
3758 			 * because
3759 			 * i) right now, it always returns 0, and
3760 			 * ii) if we hit an error, we would continue to the
3761 			 *      next queue anyway.
3762 			 */
3763 			mqd_mgr->get_wave_state(mqd_mgr,
3764 					q->mqd,
3765 					&q->properties,
3766 					(void __user *)	q->properties.ctx_save_restore_area_address,
3767 					&tmp_ctl_stack_used_size,
3768 					&tmp_save_area_used_size);
3769 		}
3770 	}
3771 	kthread_unuse_mm(mm);
3772 	mmput(mm);
3773 }
3774 
3775 static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)
3776 {
3777 	if (!usr_queue_id_array)
3778 		return num_queues ? ERR_PTR(-EINVAL) : NULL;
3779 
3780 	if (num_queues > KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
3781 		return ERR_PTR(-EINVAL);
3782 
3783 	return memdup_user(usr_queue_id_array,
3784 			   array_size(num_queues, sizeof(uint32_t)));
3785 }
3786 
3787 int resume_queues(struct kfd_process *p,
3788 		uint32_t num_queues,
3789 		uint32_t *usr_queue_id_array)
3790 {
3791 	uint32_t *queue_ids = NULL;
3792 	int total_resumed = 0;
3793 	int i;
3794 
3795 	if (usr_queue_id_array) {
3796 		queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3797 
3798 		if (IS_ERR(queue_ids))
3799 			return PTR_ERR(queue_ids);
3800 
3801 		/* mask all queues as invalid.  unmask per successful request */
3802 		q_array_invalidate(num_queues, queue_ids);
3803 	}
3804 
3805 	for (i = 0; i < p->n_pdds; i++) {
3806 		struct kfd_process_device *pdd = p->pdds[i];
3807 		struct device_queue_manager *dqm = pdd->dev->dqm;
3808 		struct device *dev = dqm->dev->adev->dev;
3809 		struct qcm_process_device *qpd = &pdd->qpd;
3810 		struct queue *q;
3811 		int r, per_device_resumed = 0;
3812 
3813 		dqm_lock(dqm);
3814 
3815 		/* unmask queues that resume or already resumed as valid */
3816 		list_for_each_entry(q, &qpd->queues_list, list) {
3817 			int q_idx = QUEUE_NOT_FOUND;
3818 
3819 			if (queue_ids)
3820 				q_idx = q_array_get_index(
3821 						q->properties.queue_id,
3822 						num_queues,
3823 						queue_ids);
3824 
3825 			if (!queue_ids || q_idx != QUEUE_NOT_FOUND) {
3826 				int err = resume_single_queue(dqm, &pdd->qpd, q);
3827 
3828 				if (queue_ids) {
3829 					if (!err) {
3830 						queue_ids[q_idx] &=
3831 							~KFD_DBG_QUEUE_INVALID_MASK;
3832 					} else {
3833 						queue_ids[q_idx] |=
3834 							KFD_DBG_QUEUE_ERROR_MASK;
3835 						break;
3836 					}
3837 				}
3838 
3839 				if (dqm->dev->kfd->shared_resources.enable_mes) {
3840 					wake_up_all(&dqm->destroy_wait);
3841 					if (!err)
3842 						total_resumed++;
3843 				} else {
3844 					per_device_resumed++;
3845 				}
3846 			}
3847 		}
3848 
3849 		if (!per_device_resumed) {
3850 			dqm_unlock(dqm);
3851 			continue;
3852 		}
3853 
3854 		r = execute_queues_cpsch(dqm,
3855 					KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
3856 					0,
3857 					USE_DEFAULT_GRACE_PERIOD);
3858 		if (r) {
3859 			dev_err(dev, "Failed to resume process queues\n");
3860 			if (queue_ids) {
3861 				list_for_each_entry(q, &qpd->queues_list, list) {
3862 					int q_idx = q_array_get_index(
3863 							q->properties.queue_id,
3864 							num_queues,
3865 							queue_ids);
3866 
3867 					/* mask queue as error on resume fail */
3868 					if (q_idx != QUEUE_NOT_FOUND)
3869 						queue_ids[q_idx] |=
3870 							KFD_DBG_QUEUE_ERROR_MASK;
3871 				}
3872 			}
3873 		} else {
3874 			wake_up_all(&dqm->destroy_wait);
3875 			total_resumed += per_device_resumed;
3876 		}
3877 
3878 		dqm_unlock(dqm);
3879 	}
3880 
3881 	if (queue_ids) {
3882 		if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
3883 				num_queues * sizeof(uint32_t)))
3884 			pr_err("copy_to_user failed on queue resume\n");
3885 
3886 		kfree(queue_ids);
3887 	}
3888 
3889 	return total_resumed;
3890 }
3891 
3892 int suspend_queues(struct kfd_process *p,
3893 			uint32_t num_queues,
3894 			uint32_t grace_period,
3895 			uint64_t exception_clear_mask,
3896 			uint32_t *usr_queue_id_array)
3897 {
3898 	uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3899 	int total_suspended = 0;
3900 	int i;
3901 
3902 	if (IS_ERR(queue_ids))
3903 		return PTR_ERR(queue_ids);
3904 
3905 	/* mask all queues as invalid.  umask on successful request */
3906 	q_array_invalidate(num_queues, queue_ids);
3907 
3908 	for (i = 0; i < p->n_pdds; i++) {
3909 		struct kfd_process_device *pdd = p->pdds[i];
3910 		struct device_queue_manager *dqm = pdd->dev->dqm;
3911 		struct device *dev = dqm->dev->adev->dev;
3912 		struct qcm_process_device *qpd = &pdd->qpd;
3913 		struct queue *q;
3914 		int r, per_device_suspended = 0;
3915 
3916 		mutex_lock(&p->event_mutex);
3917 		dqm_lock(dqm);
3918 
3919 		/* unmask queues that suspend or already suspended */
3920 		list_for_each_entry(q, &qpd->queues_list, list) {
3921 			int q_idx = q_array_get_index(q->properties.queue_id,
3922 							num_queues,
3923 							queue_ids);
3924 
3925 			if (q_idx != QUEUE_NOT_FOUND) {
3926 				int err = suspend_single_queue(dqm, pdd, q);
3927 				bool is_mes = dqm->dev->kfd->shared_resources.enable_mes;
3928 
3929 				if (!err) {
3930 					queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK;
3931 					if (exception_clear_mask && is_mes)
3932 						q->properties.exception_status &=
3933 							~exception_clear_mask;
3934 
3935 					if (is_mes)
3936 						total_suspended++;
3937 					else
3938 						per_device_suspended++;
3939 				} else if (err != -EBUSY) {
3940 					queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3941 					break;
3942 				}
3943 			}
3944 		}
3945 
3946 		if (!per_device_suspended) {
3947 			dqm_unlock(dqm);
3948 			mutex_unlock(&p->event_mutex);
3949 			if (total_suspended) {
3950 				amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev);
3951 				/* Heavy-weight TLB flush after MES suspends queues */
3952 				kfd_flush_tlb(pdd);
3953 			}
3954 			continue;
3955 		}
3956 
3957 		r = execute_queues_cpsch(dqm,
3958 			KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0,
3959 			grace_period);
3960 
3961 		if (r)
3962 			dev_err(dev, "Failed to suspend process queues.\n");
3963 		else
3964 			total_suspended += per_device_suspended;
3965 
3966 		list_for_each_entry(q, &qpd->queues_list, list) {
3967 			int q_idx = q_array_get_index(q->properties.queue_id,
3968 						num_queues, queue_ids);
3969 
3970 			if (q_idx == QUEUE_NOT_FOUND)
3971 				continue;
3972 
3973 			/* mask queue as error on suspend fail */
3974 			if (r)
3975 				queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3976 			else if (exception_clear_mask)
3977 				q->properties.exception_status &=
3978 							~exception_clear_mask;
3979 		}
3980 
3981 		dqm_unlock(dqm);
3982 		mutex_unlock(&p->event_mutex);
3983 		amdgpu_device_flush_hdp(dqm->dev->adev, NULL);
3984 	}
3985 
3986 	if (total_suspended) {
3987 		struct copy_context_work_handler_workarea copy_context_worker;
3988 
3989 		INIT_WORK_ONSTACK(
3990 				&copy_context_worker.copy_context_work,
3991 				copy_context_work_handler);
3992 
3993 		copy_context_worker.p = p;
3994 
3995 		schedule_work(&copy_context_worker.copy_context_work);
3996 
3997 
3998 		flush_work(&copy_context_worker.copy_context_work);
3999 		destroy_work_on_stack(&copy_context_worker.copy_context_work);
4000 	}
4001 
4002 	if (copy_to_user((void __user *)usr_queue_id_array, queue_ids,
4003 			num_queues * sizeof(uint32_t)))
4004 		pr_err("copy_to_user failed on queue suspend\n");
4005 
4006 	kfree(queue_ids);
4007 
4008 	return total_suspended;
4009 }
4010 
4011 static uint32_t set_queue_type_for_user(struct queue_properties *q_props)
4012 {
4013 	switch (q_props->type) {
4014 	case KFD_QUEUE_TYPE_COMPUTE:
4015 		return q_props->format == KFD_QUEUE_FORMAT_PM4
4016 					? KFD_IOC_QUEUE_TYPE_COMPUTE
4017 					: KFD_IOC_QUEUE_TYPE_COMPUTE_AQL;
4018 	case KFD_QUEUE_TYPE_SDMA:
4019 		return KFD_IOC_QUEUE_TYPE_SDMA;
4020 	case KFD_QUEUE_TYPE_SDMA_XGMI:
4021 		return KFD_IOC_QUEUE_TYPE_SDMA_XGMI;
4022 	default:
4023 		WARN_ONCE(true, "queue type not recognized!");
4024 		return 0xffffffff;
4025 	};
4026 }
4027 
4028 void set_queue_snapshot_entry(struct queue *q,
4029 			      uint64_t exception_clear_mask,
4030 			      struct kfd_queue_snapshot_entry *qss_entry)
4031 {
4032 	qss_entry->ring_base_address = q->properties.queue_address;
4033 	qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr;
4034 	qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr;
4035 	qss_entry->ctx_save_restore_address =
4036 				q->properties.ctx_save_restore_area_address;
4037 	qss_entry->ctx_save_restore_area_size =
4038 				q->properties.ctx_save_restore_area_size;
4039 	qss_entry->exception_status = q->properties.exception_status;
4040 	qss_entry->queue_id = q->properties.queue_id;
4041 	qss_entry->gpu_id = q->device->id;
4042 	qss_entry->ring_size = (uint32_t)q->properties.queue_size;
4043 	qss_entry->queue_type = set_queue_type_for_user(&q->properties);
4044 	q->properties.exception_status &= ~exception_clear_mask;
4045 }
4046 
4047 int debug_lock_and_unmap(struct device_queue_manager *dqm)
4048 {
4049 	struct device *dev = dqm->dev->adev->dev;
4050 	int r;
4051 
4052 	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
4053 		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
4054 		return -EINVAL;
4055 	}
4056 
4057 	if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
4058 		return 0;
4059 
4060 	dqm_lock(dqm);
4061 
4062 	r = unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0, 0, false);
4063 	if (r)
4064 		dqm_unlock(dqm);
4065 
4066 	return r;
4067 }
4068 
4069 int debug_map_and_unlock(struct device_queue_manager *dqm)
4070 {
4071 	struct device *dev = dqm->dev->adev->dev;
4072 	int r;
4073 
4074 	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
4075 		dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
4076 		return -EINVAL;
4077 	}
4078 
4079 	if (!kfd_dbg_is_per_vmid_supported(dqm->dev))
4080 		return 0;
4081 
4082 	r = map_queues_cpsch(dqm);
4083 
4084 	dqm_unlock(dqm);
4085 
4086 	return r;
4087 }
4088 
4089 int debug_refresh_runlist(struct device_queue_manager *dqm)
4090 {
4091 	int r = debug_lock_and_unmap(dqm);
4092 
4093 	if (r)
4094 		return r;
4095 
4096 	return debug_map_and_unlock(dqm);
4097 }
4098 
4099 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
4100 				 struct qcm_process_device *qpd,
4101 				 int doorbell_off, u32 *queue_format)
4102 {
4103 	struct queue *q;
4104 	bool r = false;
4105 
4106 	if (!queue_format)
4107 		return r;
4108 
4109 	dqm_lock(dqm);
4110 
4111 	list_for_each_entry(q, &qpd->queues_list, list) {
4112 		if (q->properties.doorbell_off == doorbell_off) {
4113 			*queue_format = q->properties.format;
4114 			r = true;
4115 			goto out;
4116 		}
4117 	}
4118 
4119 out:
4120 	dqm_unlock(dqm);
4121 	return r;
4122 }
4123 
4124 size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, enum kfd_queue_type type)
4125 {
4126 	return dqm->mqd_mgrs[get_mqd_type_from_queue_type(type)]->mqd_size;
4127 }
4128 
4129 #if defined(CONFIG_DEBUG_FS)
4130 
4131 static void seq_reg_dump(struct seq_file *m,
4132 			 uint32_t (*dump)[2], uint32_t n_regs)
4133 {
4134 	uint32_t i, count;
4135 
4136 	for (i = 0, count = 0; i < n_regs; i++) {
4137 		if (count == 0 ||
4138 		    dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
4139 			seq_printf(m, "%s    %08x: %08x",
4140 				   i ? "\n" : "",
4141 				   dump[i][0], dump[i][1]);
4142 			count = 7;
4143 		} else {
4144 			seq_printf(m, " %08x", dump[i][1]);
4145 			count--;
4146 		}
4147 	}
4148 
4149 	seq_puts(m, "\n");
4150 }
4151 
4152 int dqm_debugfs_hqds(struct seq_file *m, void *data)
4153 {
4154 	struct device_queue_manager *dqm = data;
4155 	uint32_t xcc_mask = dqm->dev->xcc_mask;
4156 	uint32_t (*dump)[2], n_regs;
4157 	int pipe, queue;
4158 	int r = 0, xcc_id;
4159 	uint32_t sdma_engine_start;
4160 
4161 	if (!dqm->sched_running) {
4162 		seq_puts(m, " Device is stopped\n");
4163 		return 0;
4164 	}
4165 
4166 	for_each_inst(xcc_id, xcc_mask) {
4167 		r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
4168 						KFD_CIK_HIQ_PIPE,
4169 						KFD_CIK_HIQ_QUEUE, &dump,
4170 						&n_regs, xcc_id);
4171 		if (!r) {
4172 			seq_printf(
4173 				m,
4174 				"   Inst %d, HIQ on MEC %d Pipe %d Queue %d\n",
4175 				xcc_id,
4176 				KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1,
4177 				KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm),
4178 				KFD_CIK_HIQ_QUEUE);
4179 			seq_reg_dump(m, dump, n_regs);
4180 
4181 			kfree(dump);
4182 		}
4183 
4184 		for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
4185 			int pipe_offset = pipe * get_queues_per_pipe(dqm);
4186 
4187 			for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
4188 				if (!test_bit(pipe_offset + queue,
4189 				      dqm->dev->kfd->shared_resources.cp_queue_bitmap))
4190 					continue;
4191 
4192 				r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
4193 								pipe, queue,
4194 								&dump, &n_regs,
4195 								xcc_id);
4196 				if (r)
4197 					break;
4198 
4199 				seq_printf(m,
4200 					   " Inst %d,  CP Pipe %d, Queue %d\n",
4201 					   xcc_id, pipe, queue);
4202 				seq_reg_dump(m, dump, n_regs);
4203 
4204 				kfree(dump);
4205 			}
4206 		}
4207 	}
4208 
4209 	sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
4210 	for (pipe = sdma_engine_start;
4211 	     pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm));
4212 	     pipe++) {
4213 		for (queue = 0;
4214 		     queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
4215 		     queue++) {
4216 			r = dqm->dev->kfd2kgd->hqd_sdma_dump(
4217 				dqm->dev->adev, pipe, queue, &dump, &n_regs);
4218 			if (r == -EINVAL) {
4219 				/*
4220 				 * SDMA engine not present/mapped on this
4221 				 * device; skip it instead of aborting the
4222 				 * whole dump.
4223 				 */
4224 				r = 0;
4225 				break;
4226 			}
4227 			if (r)
4228 				break;
4229 
4230 			seq_printf(m, "  SDMA Engine %d, RLC %d\n",
4231 				  pipe, queue);
4232 			seq_reg_dump(m, dump, n_regs);
4233 
4234 			kfree(dump);
4235 		}
4236 	}
4237 
4238 	return r;
4239 }
4240 
4241 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)
4242 {
4243 	int r = 0;
4244 
4245 	dqm_lock(dqm);
4246 	r = pm_debugfs_hang_hws(&dqm->packet_mgr);
4247 	if (r) {
4248 		dqm_unlock(dqm);
4249 		return r;
4250 	}
4251 	dqm->active_runlist = true;
4252 	r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
4253 				0, USE_DEFAULT_GRACE_PERIOD);
4254 	dqm_unlock(dqm);
4255 
4256 	return r;
4257 }
4258 
4259 #endif
4260