xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/stdarg.h>
26 #include "amdgpu.h"
27 #include "amdgpu_sdma.h"
28 #include "amdgpu_ras.h"
29 #include "amdgpu_reset.h"
30 #include "gc/gc_10_1_0_offset.h"
31 #include "gc/gc_10_3_0_sh_mask.h"
32 
33 #define AMDGPU_CSA_SDMA_SIZE 64
34 /* SDMA CSA reside in the 3rd page of CSA */
35 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
36 
37 /*
38  * GPU SDMA IP block helpers function.
39  */
40 
41 int __printf(4, 5)
42 amdgpu_sdma_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
43 		      unsigned int instance, const char *fmt, ...)
44 {
45 	struct amdgpu_sdma_instance *sdma;
46 	va_list args;
47 	int r;
48 
49 	ring->ring_obj = NULL;
50 
51 	va_start(args, fmt);
52 	vsnprintf(ring->name, sizeof(ring->name), fmt, args);
53 	va_end(args);
54 
55 	r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
56 			     AMDGPU_SDMA_IRQ_INSTANCE0 + instance,
57 			     AMDGPU_RING_PRIO_DEFAULT, NULL);
58 	if (r)
59 		return r;
60 
61 	sdma = amdgpu_sdma_get_instance_from_ring(ring);
62 	if (amdgpu_sriov_vf(adev) || !adev->gfx.mcbp)
63 		sdma->csa_addr = 0;
64 	else
65 		sdma->csa_addr = amdgpu_csa_vaddr(adev) +
66 				 AMDGPU_CSA_SDMA_OFFSET +
67 				 ring->me * AMDGPU_CSA_SDMA_SIZE;
68 
69 	return r;
70 }
71 
72 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
73 			      struct ras_common_if *ras_block)
74 {
75 	int r, i;
76 
77 	r = amdgpu_ras_block_late_init(adev, ras_block);
78 	if (r)
79 		return r;
80 
81 	if (amdgpu_ras_is_supported(adev, ras_block->block)) {
82 		for (i = 0; i < adev->sdma.num_instances; i++) {
83 			r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,
84 				AMDGPU_SDMA_IRQ_INSTANCE0 + i);
85 			if (r)
86 				goto late_fini;
87 		}
88 	}
89 
90 	return 0;
91 
92 late_fini:
93 	amdgpu_ras_block_late_fini(adev, ras_block);
94 	return r;
95 }
96 
97 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
98 		void *err_data,
99 		struct amdgpu_iv_entry *entry)
100 {
101 	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
102 
103 	if (amdgpu_sriov_vf(adev))
104 		return AMDGPU_RAS_SUCCESS;
105 
106 	amdgpu_ras_reset_gpu(adev);
107 
108 	return AMDGPU_RAS_SUCCESS;
109 }
110 
111 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
112 				      struct amdgpu_irq_src *source,
113 				      struct amdgpu_iv_entry *entry)
114 {
115 	struct ras_common_if *ras_if = adev->sdma.ras_if;
116 	struct ras_dispatch_if ih_data = {
117 		.entry = entry,
118 	};
119 
120 	if (!ras_if)
121 		return 0;
122 
123 	ih_data.head = *ras_if;
124 
125 	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
126 	return 0;
127 }
128 
129 static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
130 {
131 	uint16_t version_major;
132 	const struct common_firmware_header *header = NULL;
133 	const struct sdma_firmware_header_v1_0 *hdr;
134 	const struct sdma_firmware_header_v2_0 *hdr_v2;
135 	const struct sdma_firmware_header_v3_0 *hdr_v3;
136 
137 	header = (const struct common_firmware_header *)
138 		sdma_inst->fw->data;
139 	version_major = le16_to_cpu(header->header_version_major);
140 
141 	switch (version_major) {
142 	case 1:
143 		hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
144 		sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
145 		sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
146 		break;
147 	case 2:
148 		hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
149 		sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
150 		sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
151 		break;
152 	case 3:
153 		hdr_v3 = (const struct sdma_firmware_header_v3_0 *)sdma_inst->fw->data;
154 		sdma_inst->fw_version = le32_to_cpu(hdr_v3->header.ucode_version);
155 		sdma_inst->feature_version = le32_to_cpu(hdr_v3->ucode_feature_version);
156 		break;
157 	default:
158 		return -EINVAL;
159 	}
160 
161 	if (sdma_inst->feature_version >= 20)
162 		sdma_inst->burst_nop = true;
163 
164 	return 0;
165 }
166 
167 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
168 				  bool duplicate)
169 {
170 	int i;
171 
172 	for (i = 0; i < adev->sdma.num_instances; i++) {
173 		amdgpu_ucode_release(&adev->sdma.instance[i].fw);
174 		if (duplicate)
175 			break;
176 	}
177 
178 	memset((void *)adev->sdma.instance, 0,
179 	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
180 }
181 
182 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
183 			       u32 instance, bool duplicate)
184 {
185 	struct amdgpu_firmware_info *info = NULL;
186 	const struct common_firmware_header *header = NULL;
187 	int err, i;
188 	const struct sdma_firmware_header_v2_0 *sdma_hdr;
189 	const struct sdma_firmware_header_v3_0 *sdma_hv3;
190 	uint16_t version_major;
191 	char ucode_prefix[30];
192 
193 	amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix));
194 	if (instance == 0)
195 		err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
196 					   AMDGPU_UCODE_REQUIRED,
197 					   "amdgpu/%s.bin", ucode_prefix);
198 	else
199 		err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
200 					   AMDGPU_UCODE_REQUIRED,
201 					   "amdgpu/%s%d.bin", ucode_prefix, instance);
202 	if (err)
203 		goto out;
204 
205 	header = (const struct common_firmware_header *)
206 		adev->sdma.instance[instance].fw->data;
207 	version_major = le16_to_cpu(header->header_version_major);
208 
209 	if ((duplicate && instance) || (!duplicate && version_major > 1)) {
210 		err = -EINVAL;
211 		goto out;
212 	}
213 
214 	err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
215 	if (err)
216 		goto out;
217 
218 	if (duplicate) {
219 		for (i = 1; i < adev->sdma.num_instances; i++)
220 			memcpy((void *)&adev->sdma.instance[i],
221 			       (void *)&adev->sdma.instance[0],
222 			       sizeof(struct amdgpu_sdma_instance));
223 	}
224 
225 	DRM_DEBUG("psp_load == '%s'\n",
226 		  adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
227 
228 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
229 		switch (version_major) {
230 		case 1:
231 			for (i = 0; i < adev->sdma.num_instances; i++) {
232 				if (!duplicate && (instance != i))
233 					continue;
234 				else {
235 					/* Use a single copy per SDMA firmware type. PSP uses the same instance for all
236 					 * groups of SDMAs */
237 					if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
238 						IP_VERSION(4, 4, 2) ||
239 					     amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
240 						IP_VERSION(4, 4, 4) ||
241 					     amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
242 						IP_VERSION(4, 4, 5)) &&
243 					    adev->firmware.load_type ==
244 						AMDGPU_FW_LOAD_PSP &&
245 					    adev->sdma.num_inst_per_aid == i) {
246 						break;
247 					}
248 					info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
249 					info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
250 					info->fw = adev->sdma.instance[i].fw;
251 					adev->firmware.fw_size +=
252 						ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
253 				}
254 			}
255 			break;
256 		case 2:
257 			sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
258 				adev->sdma.instance[0].fw->data;
259 			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
260 			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
261 			info->fw = adev->sdma.instance[0].fw;
262 			adev->firmware.fw_size +=
263 				ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
264 			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
265 			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
266 			info->fw = adev->sdma.instance[0].fw;
267 			adev->firmware.fw_size +=
268 				ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
269 			break;
270 		case 3:
271 			sdma_hv3 = (const struct sdma_firmware_header_v3_0 *)
272 				adev->sdma.instance[0].fw->data;
273 			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_RS64];
274 			info->ucode_id = AMDGPU_UCODE_ID_SDMA_RS64;
275 			info->fw = adev->sdma.instance[0].fw;
276 			adev->firmware.fw_size +=
277 				ALIGN(le32_to_cpu(sdma_hv3->ucode_size_bytes), PAGE_SIZE);
278 			break;
279 		default:
280 			err = -EINVAL;
281 		}
282 	}
283 
284 out:
285 	if (err)
286 		amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
287 	return err;
288 }
289 
290 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev)
291 {
292 	int err = 0;
293 	struct amdgpu_sdma_ras *ras = NULL;
294 
295 	/* adev->sdma.ras is NULL, which means sdma does not
296 	 * support ras function, then do nothing here.
297 	 */
298 	if (!adev->sdma.ras)
299 		return 0;
300 
301 	ras = adev->sdma.ras;
302 
303 	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
304 	if (err) {
305 		dev_err(adev->dev, "Failed to register sdma ras block!\n");
306 		return err;
307 	}
308 
309 	strcpy(ras->ras_block.ras_comm.name, "sdma");
310 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA;
311 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
312 	adev->sdma.ras_if = &ras->ras_block.ras_comm;
313 
314 	/* If not define special ras_late_init function, use default ras_late_init */
315 	if (!ras->ras_block.ras_late_init)
316 		ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init;
317 
318 	/* If not defined special ras_cb function, use default ras_cb */
319 	if (!ras->ras_block.ras_cb)
320 		ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb;
321 
322 	return 0;
323 }
324 
325 /*
326  * debugfs for to enable/disable sdma job submission to specific core.
327  */
328 #if defined(CONFIG_DEBUG_FS)
329 static int amdgpu_debugfs_sdma_sched_mask_set(void *data, u64 val)
330 {
331 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
332 	u64 i, num_ring;
333 	u64 mask = 0;
334 	struct amdgpu_ring *ring, *page = NULL;
335 
336 	if (!adev)
337 		return -ENODEV;
338 
339 	/* Determine the number of rings per SDMA instance
340 	 * (1 for sdma gfx ring, 2 if page queue exists)
341 	 */
342 	if (adev->sdma.has_page_queue)
343 		num_ring = 2;
344 	else
345 		num_ring = 1;
346 
347 	/* Calculate the maximum possible mask value
348 	 * based on the number of SDMA instances and rings
349 	*/
350 	mask = BIT_ULL(adev->sdma.num_instances * num_ring) - 1;
351 
352 	if ((val & mask) == 0)
353 		return -EINVAL;
354 
355 	for (i = 0; i < adev->sdma.num_instances; ++i) {
356 		ring = &adev->sdma.instance[i].ring;
357 		if (adev->sdma.has_page_queue)
358 			page = &adev->sdma.instance[i].page;
359 		if (val & BIT_ULL(i * num_ring))
360 			ring->sched.ready = true;
361 		else
362 			ring->sched.ready = false;
363 
364 		if (page) {
365 			if (val & BIT_ULL(i * num_ring + 1))
366 				page->sched.ready = true;
367 			else
368 				page->sched.ready = false;
369 		}
370 	}
371 	/* publish sched.ready flag update effective immediately across smp */
372 	smp_rmb();
373 	return 0;
374 }
375 
376 static int amdgpu_debugfs_sdma_sched_mask_get(void *data, u64 *val)
377 {
378 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
379 	u64 i, num_ring;
380 	u64 mask = 0;
381 	struct amdgpu_ring *ring, *page = NULL;
382 
383 	if (!adev)
384 		return -ENODEV;
385 
386 	/* Determine the number of rings per SDMA instance
387 	 * (1 for sdma gfx ring, 2 if page queue exists)
388 	 */
389 	if (adev->sdma.has_page_queue)
390 		num_ring = 2;
391 	else
392 		num_ring = 1;
393 
394 	for (i = 0; i < adev->sdma.num_instances; ++i) {
395 		ring = &adev->sdma.instance[i].ring;
396 		if (adev->sdma.has_page_queue)
397 			page = &adev->sdma.instance[i].page;
398 
399 		if (ring->sched.ready)
400 			mask |= BIT_ULL(i * num_ring);
401 		else
402 			mask &= ~BIT_ULL(i * num_ring);
403 
404 		if (page) {
405 			if (page->sched.ready)
406 				mask |= BIT_ULL(i * num_ring + 1);
407 			else
408 				mask &= ~BIT_ULL(i * num_ring + 1);
409 		}
410 	}
411 
412 	*val = mask;
413 	return 0;
414 }
415 
416 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_sdma_sched_mask_fops,
417 			 amdgpu_debugfs_sdma_sched_mask_get,
418 			 amdgpu_debugfs_sdma_sched_mask_set, "%llx\n");
419 
420 #endif
421 
422 void amdgpu_debugfs_sdma_sched_mask_init(struct amdgpu_device *adev)
423 {
424 #if defined(CONFIG_DEBUG_FS)
425 	struct drm_minor *minor = adev_to_drm(adev)->primary;
426 	struct dentry *root = minor->debugfs_root;
427 	char name[32];
428 
429 	if (!(adev->sdma.num_instances > 1))
430 		return;
431 	sprintf(name, "amdgpu_sdma_sched_mask");
432 	debugfs_create_file(name, 0600, root, adev,
433 			    &amdgpu_debugfs_sdma_sched_mask_fops);
434 #endif
435 }
436 
437 static ssize_t amdgpu_get_sdma_reset_mask(struct device *dev,
438 						struct device_attribute *attr,
439 						char *buf)
440 {
441 	struct drm_device *ddev = dev_get_drvdata(dev);
442 	struct amdgpu_device *adev = drm_to_adev(ddev);
443 
444 	if (!adev)
445 		return -ENODEV;
446 
447 	return amdgpu_show_reset_mask(buf, adev->sdma.supported_reset);
448 }
449 
450 static DEVICE_ATTR(sdma_reset_mask, 0444,
451 		   amdgpu_get_sdma_reset_mask, NULL);
452 
453 int amdgpu_sdma_sysfs_reset_mask_init(struct amdgpu_device *adev)
454 {
455 	int r = 0;
456 
457 	if (!amdgpu_gpu_recovery)
458 		return r;
459 
460 	if (adev->sdma.num_instances) {
461 		r = device_create_file(adev->dev, &dev_attr_sdma_reset_mask);
462 		if (r)
463 			return r;
464 	}
465 
466 	return r;
467 }
468 
469 void amdgpu_sdma_sysfs_reset_mask_fini(struct amdgpu_device *adev)
470 {
471 	if (!amdgpu_gpu_recovery)
472 		return;
473 
474 	if (adev->dev->kobj.sd) {
475 		if (adev->sdma.num_instances)
476 			device_remove_file(adev->dev, &dev_attr_sdma_reset_mask);
477 	}
478 }
479 
480 struct amdgpu_ring *amdgpu_sdma_get_shared_ring(struct amdgpu_device *adev, struct amdgpu_ring *ring)
481 {
482 	if (adev->sdma.has_page_queue &&
483 	    (ring->me < adev->sdma.num_instances) &&
484 	    (ring == &adev->sdma.instance[ring->me].ring))
485 		return &adev->sdma.instance[ring->me].page;
486 	else
487 		return NULL;
488 }
489 
490 /**
491 * amdgpu_sdma_is_shared_inv_eng - Check if a ring is an SDMA ring that shares a VM invalidation engine
492 * @adev: Pointer to the AMDGPU device structure
493 * @ring: Pointer to the ring structure to check
494 *
495 * This function checks if the given ring is an SDMA ring that shares a VM invalidation engine.
496 * It returns true if the ring is such an SDMA ring, false otherwise.
497 */
498 bool amdgpu_sdma_is_shared_inv_eng(struct amdgpu_device *adev, struct amdgpu_ring *ring)
499 {
500 	int i = ring->me;
501 
502 	if (!adev->sdma.has_page_queue || i >= adev->sdma.num_instances)
503 		return false;
504 
505 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
506 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
507 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0))
508 		return (ring == &adev->sdma.instance[i].page);
509 	else
510 		return false;
511 }
512 
513 static int amdgpu_sdma_soft_reset(struct amdgpu_device *adev, u32 instance_id)
514 {
515 	struct amdgpu_sdma_instance *sdma_instance = &adev->sdma.instance[instance_id];
516 
517 	if (sdma_instance->funcs->soft_reset_kernel_queue)
518 		return sdma_instance->funcs->soft_reset_kernel_queue(adev, instance_id);
519 
520 	return -EOPNOTSUPP;
521 }
522 
523 /**
524  * amdgpu_sdma_reset_engine() - Reset a specific SDMA engine instance.
525  *
526  * @adev: Pointer to the AMDGPU device
527  * @instance_id: Logical ID of the SDMA engine instance to reset
528  *
529  * Returns: 0 on success, or a negative error code on failure.
530  */
531 int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id)
532 {
533 	struct amdgpu_sdma_instance *sdma_instance = &adev->sdma.instance[instance_id];
534 	struct amdgpu_ring *gfx_ring = &sdma_instance->ring;
535 	struct amdgpu_ring *page_ring = &sdma_instance->page;
536 	struct amdgpu_fence *gfx_fence, *page_fence;
537 	int ret = 0;
538 
539 	if (amdgpu_sriov_vf(adev))
540 		return -EOPNOTSUPP;
541 
542 	mutex_lock(&sdma_instance->engine_reset_mutex);
543 
544 	/*
545 	 * Stop the scheduler's work queue for the GFX and page rings if they are running.
546 	 * This ensures that no new tasks are submitted to the queues while
547 	 * the reset is in progress.
548 	 */
549 	if (amdgpu_ring_sched_ready(gfx_ring) && !drm_sched_is_stopped(&gfx_ring->sched))
550 		drm_sched_wqueue_stop(&gfx_ring->sched);
551 
552 	gfx_fence = amdgpu_ring_find_guilty_fence(gfx_ring);
553 	amdgpu_ring_reset_helper_begin(gfx_ring, gfx_fence);
554 
555 	if (adev->sdma.has_page_queue) {
556 		if (amdgpu_ring_sched_ready(page_ring) && !drm_sched_is_stopped(&page_ring->sched))
557 			drm_sched_wqueue_stop(&page_ring->sched);
558 
559 		page_fence = amdgpu_ring_find_guilty_fence(page_ring);
560 		amdgpu_ring_reset_helper_begin(page_ring, page_fence);
561 	}
562 
563 	if (sdma_instance->funcs->stop_kernel_queue) {
564 		sdma_instance->funcs->stop_kernel_queue(gfx_ring);
565 		if (adev->sdma.has_page_queue)
566 			sdma_instance->funcs->stop_kernel_queue(page_ring);
567 	}
568 
569 	/* Perform the SDMA reset for the specified instance */
570 	ret = amdgpu_sdma_soft_reset(adev, instance_id);
571 	if (ret) {
572 		dev_err(adev->dev, "Failed to reset SDMA logical instance %u\n", instance_id);
573 		goto exit;
574 	}
575 
576 	amdgpu_ring_clear_ring_and_ptrs(gfx_ring);
577 
578 	if (adev->sdma.has_page_queue)
579 		amdgpu_ring_clear_ring_and_ptrs(page_ring);
580 
581 	if (sdma_instance->funcs->start_kernel_queue) {
582 		sdma_instance->funcs->start_kernel_queue(gfx_ring);
583 		if (adev->sdma.has_page_queue)
584 			sdma_instance->funcs->start_kernel_queue(page_ring);
585 	}
586 
587 exit:
588 	/* Restart the scheduler's work queue for the GFX and page rings
589 	 * if they were stopped by this function. This allows new tasks
590 	 * to be submitted to the queues after the reset is complete.
591 	 */
592 	if (!ret) {
593 		ret = amdgpu_ring_reset_helper_end(gfx_ring, gfx_fence);
594 		if (ret)
595 			goto unlock;
596 
597 		if (amdgpu_ring_sched_ready(gfx_ring))
598 			drm_sched_wqueue_start(&gfx_ring->sched);
599 
600 		if (adev->sdma.has_page_queue) {
601 			ret = amdgpu_ring_reset_helper_end(page_ring, page_fence);
602 			if (ret)
603 				goto unlock;
604 
605 			if (amdgpu_ring_sched_ready(page_ring))
606 				drm_sched_wqueue_start(&page_ring->sched);
607 		}
608 	}
609 unlock:
610 	mutex_unlock(&sdma_instance->engine_reset_mutex);
611 
612 	return ret;
613 }
614 
615 /**
616  * amdgpu_sdma_reset_queue_legacy() - Reset legacy SDMA queue after timeout (without MES)
617  *
618  * @ring: Pointer to the ring of the SDMA queue
619  * @vmid: VMID of the timed out job
620  * @timedout_fence: Fence of the timed out job
621  *
622  * Common implementation for resetting SDMA queues without MES (legacy).
623  * This relies on the proper amdgpu_sdma_funcs to be set up
624  * for the given ring.
625  *
626  * Applicable to SDMA versions that don't rely on the MES yet,
627  * that is all versions up to SDMA v5.x and older.
628  */
629 int amdgpu_sdma_reset_queue_legacy(struct amdgpu_ring *ring,
630 				   unsigned int vmid,
631 				   struct amdgpu_fence *timedout_fence)
632 {
633 	struct amdgpu_device *adev = ring->adev;
634 	int r;
635 
636 	if (ring->me >= adev->sdma.num_instances) {
637 		dev_err(adev->dev, "sdma instance not found\n");
638 		return -EINVAL;
639 	}
640 
641 	amdgpu_amdkfd_suspend(adev, true);
642 	r = amdgpu_sdma_reset_engine(adev, ring->me);
643 	amdgpu_amdkfd_resume(adev, true);
644 	if (r)
645 		return r;
646 
647 	return 0;
648 }
649