xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  *
23  */
24 #include <linux/kthread.h>
25 #include <linux/wait.h>
26 #include <linux/sched.h>
27 
28 #include <drm/drm_drv.h>
29 
30 #include "amdgpu.h"
31 #include "amdgpu_trace.h"
32 #include "amdgpu_reset.h"
33 #include "amdgpu_dev_coredump.h"
34 #include "amdgpu_xgmi.h"
35 
36 static void amdgpu_job_do_core_dump(struct amdgpu_device *adev,
37 				    struct amdgpu_job *job)
38 {
39 	int i;
40 
41 	dev_info(adev->dev, "Dumping IP State\n");
42 	for (i = 0; i < adev->num_ip_blocks; i++)
43 		if (adev->ip_blocks[i].version->funcs->dump_ip_state)
44 			adev->ip_blocks[i].version->funcs
45 				->dump_ip_state((void *)&adev->ip_blocks[i]);
46 	dev_info(adev->dev, "Dumping IP State Completed\n");
47 
48 	amdgpu_coredump(adev, true, false, job);
49 }
50 
51 static void amdgpu_job_core_dump(struct amdgpu_device *adev,
52 				 struct amdgpu_job *job)
53 {
54 	struct list_head device_list, *device_list_handle =  NULL;
55 	struct amdgpu_device *tmp_adev = NULL;
56 	struct amdgpu_hive_info *hive = NULL;
57 
58 	if (!amdgpu_sriov_vf(adev))
59 		hive = amdgpu_get_xgmi_hive(adev);
60 	if (hive)
61 		mutex_lock(&hive->hive_lock);
62 	/*
63 	 * Reuse the logic in amdgpu_device_gpu_recover() to build list of
64 	 * devices for code dump
65 	 */
66 	INIT_LIST_HEAD(&device_list);
67 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {
68 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
69 			list_add_tail(&tmp_adev->reset_list, &device_list);
70 		if (!list_is_first(&adev->reset_list, &device_list))
71 			list_rotate_to_front(&adev->reset_list, &device_list);
72 		device_list_handle = &device_list;
73 	} else {
74 		list_add_tail(&adev->reset_list, &device_list);
75 		device_list_handle = &device_list;
76 	}
77 
78 	/* Do the coredump for each device */
79 	list_for_each_entry(tmp_adev, device_list_handle, reset_list)
80 		amdgpu_job_do_core_dump(tmp_adev, job);
81 
82 	if (hive) {
83 		mutex_unlock(&hive->hive_lock);
84 		amdgpu_put_xgmi_hive(hive);
85 	}
86 }
87 
88 static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
89 {
90 	struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched);
91 	struct amdgpu_job *job = to_amdgpu_job(s_job);
92 	struct drm_wedge_task_info *info = NULL;
93 	struct amdgpu_task_info *ti = NULL;
94 	struct amdgpu_device *adev = ring->adev;
95 	int idx, r;
96 
97 	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
98 		dev_info(adev->dev, "%s - device unplugged skipping recovery on scheduler:%s",
99 			 __func__, s_job->sched->name);
100 
101 		/* Effectively the job is aborted as the device is gone */
102 		return DRM_GPU_SCHED_STAT_ENODEV;
103 	}
104 
105 	/*
106 	 * Do the coredump immediately after a job timeout to get a very
107 	 * close dump/snapshot/representation of GPU's current error status
108 	 * Skip it for SRIOV, since VF FLR will be triggered by host driver
109 	 * before job timeout
110 	 */
111 	if (!amdgpu_sriov_vf(adev))
112 		amdgpu_job_core_dump(adev, job);
113 
114 	if (amdgpu_gpu_recovery &&
115 	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_SOFT_RECOVERY) &&
116 	    amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) {
117 		dev_err(adev->dev, "ring %s timeout, but soft recovered\n",
118 			s_job->sched->name);
119 		goto exit;
120 	}
121 
122 	dev_err(adev->dev, "ring %s timeout, signaled seq=%u, emitted seq=%u\n",
123 		job->base.sched->name, atomic_read(&ring->fence_drv.last_seq),
124 		ring->fence_drv.sync_seq);
125 
126 	ti = amdgpu_vm_get_task_info_pasid(ring->adev, job->pasid);
127 	if (ti) {
128 		amdgpu_vm_print_task_info(adev, ti);
129 		info = &ti->task;
130 	}
131 
132 	/* attempt a per ring reset */
133 	if (amdgpu_gpu_recovery &&
134 	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_PER_QUEUE) &&
135 	    ring->funcs->reset) {
136 		dev_err(adev->dev, "Starting %s ring reset\n",
137 			s_job->sched->name);
138 		/* Stop the scheduler to prevent anybody else from touching the ring buffer. */
139 		drm_sched_wqueue_stop(&ring->sched);
140 		r = amdgpu_ring_reset(ring, job->vmid, job->hw_fence);
141 		if (!r) {
142 			/* Start the scheduler again */
143 			drm_sched_wqueue_start(&ring->sched);
144 			atomic_inc(&ring->adev->gpu_reset_counter);
145 			dev_err(adev->dev, "Ring %s reset succeeded\n",
146 				ring->sched.name);
147 			drm_dev_wedged_event(adev_to_drm(adev),
148 					     DRM_WEDGE_RECOVERY_NONE, info);
149 			goto exit;
150 		}
151 		dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name);
152 	}
153 
154 	/* Attempt an IP block soft reset, if supported. */
155 	if (amdgpu_gpu_recovery &&
156 	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET)) {
157 		r = amdgpu_device_ip_soft_reset(ring, job->hw_fence);
158 		if (!r) {
159 			atomic_inc(&ring->adev->gpu_reset_counter);
160 			drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);
161 			goto exit;
162 		}
163 	}
164 
165 	if (dma_fence_get_status(&s_job->s_fence->finished) == 0)
166 		dma_fence_set_error(&s_job->s_fence->finished, -ETIME);
167 
168 	if (amdgpu_device_should_recover_gpu(ring->adev)) {
169 		struct amdgpu_reset_context reset_context;
170 		memset(&reset_context, 0, sizeof(reset_context));
171 
172 		reset_context.method = AMD_RESET_METHOD_NONE;
173 		reset_context.reset_req_dev = adev;
174 		reset_context.src = AMDGPU_RESET_SRC_JOB;
175 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
176 
177 		/*
178 		 * To avoid an unnecessary extra coredump, as we have already
179 		 * got the very close representation of GPU's error status
180 		 */
181 		set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
182 
183 		r = amdgpu_device_gpu_recover(ring->adev, job, &reset_context);
184 		if (r)
185 			dev_err(adev->dev, "GPU Recovery Failed: %d\n", r);
186 	} else {
187 		drm_sched_suspend_timeout(&ring->sched);
188 		if (amdgpu_sriov_vf(adev))
189 			adev->virt.tdr_debug = true;
190 	}
191 
192 exit:
193 	amdgpu_vm_put_task_info(ti);
194 	drm_dev_exit(idx);
195 	/* This is needed to add the job back to the pending list */
196 	return DRM_GPU_SCHED_STAT_NO_HANG;
197 }
198 
199 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
200 		     struct drm_sched_entity *entity, void *owner,
201 		     unsigned int num_ibs, u64 drm_client_id,
202 		     gfp_t gfp_flags, struct amdgpu_job **job)
203 {
204 	struct amdgpu_fence *af;
205 	int r;
206 
207 	if (num_ibs == 0)
208 		return -EINVAL;
209 
210 	*job = kzalloc_flex(**job, ibs, num_ibs, gfp_flags);
211 	if (!*job)
212 		return -ENOMEM;
213 
214 	af = kzalloc_obj(struct amdgpu_fence, gfp_flags);
215 	if (!af) {
216 		r = -ENOMEM;
217 		goto err_job;
218 	}
219 	(*job)->hw_fence = af;
220 
221 	af = kzalloc_obj(struct amdgpu_fence, gfp_flags);
222 	if (!af) {
223 		r = -ENOMEM;
224 		goto err_fence;
225 	}
226 	(*job)->hw_vm_fence = af;
227 
228 	(*job)->vm = vm;
229 
230 	amdgpu_sync_create(&(*job)->explicit_sync);
231 	(*job)->generation = amdgpu_vm_generation(adev, vm);
232 	(*job)->vm_pd_addr = AMDGPU_BO_INVALID_OFFSET;
233 
234 	if (!entity)
235 		return 0;
236 
237 	r = drm_sched_job_init(&(*job)->base, entity, 1, owner, drm_client_id);
238 	if (!r)
239 		return 0;
240 
241 	kfree((*job)->hw_vm_fence);
242 
243 err_fence:
244 	kfree((*job)->hw_fence);
245 err_job:
246 	kfree(*job);
247 	*job = NULL;
248 
249 	return r;
250 }
251 
252 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
253 			     struct drm_sched_entity *entity, void *owner,
254 			     size_t size, enum amdgpu_ib_pool_type pool_type,
255 			     u64 k_job_id, struct amdgpu_job **job)
256 {
257 	int r;
258 
259 	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, k_job_id,
260 			     amdgpu_ib_pool_gfp_flags(adev, pool_type),
261 			     job);
262 	if (r)
263 		return r;
264 
265 	(*job)->num_ibs = 1;
266 	r = amdgpu_ib_get(adev, NULL, size, pool_type, &(*job)->ibs[0]);
267 	if (r) {
268 		if (entity)
269 			drm_sched_job_cleanup(&(*job)->base);
270 		kfree((*job)->hw_vm_fence);
271 		kfree((*job)->hw_fence);
272 		kfree(*job);
273 		*job = NULL;
274 	}
275 
276 	return r;
277 }
278 
279 void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds,
280 			      struct amdgpu_bo *gws, struct amdgpu_bo *oa)
281 {
282 	if (gds) {
283 		job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT;
284 		job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT;
285 	}
286 	if (gws) {
287 		job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT;
288 		job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT;
289 	}
290 	if (oa) {
291 		job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT;
292 		job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT;
293 	}
294 }
295 
296 void amdgpu_job_free_resources(struct amdgpu_job *job)
297 {
298 	struct dma_fence *f;
299 	unsigned i;
300 
301 	/* Check if any fences were initialized */
302 	if (job->base.s_fence &&
303 	    dma_fence_was_initialized(&job->base.s_fence->finished))
304 		f = &job->base.s_fence->finished;
305 	else if (dma_fence_was_initialized(&job->hw_fence->base))
306 		f = &job->hw_fence->base;
307 	else
308 		f = NULL;
309 
310 	for (i = 0; i < job->num_ibs; ++i)
311 		amdgpu_ib_free(&job->ibs[i], f);
312 }
313 
314 static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
315 {
316 	amdgpu_job_free(to_amdgpu_job(s_job));
317 }
318 
319 void amdgpu_job_set_gang_leader(struct amdgpu_job *job,
320 				struct amdgpu_job *leader)
321 {
322 	struct dma_fence *fence = &leader->base.s_fence->scheduled;
323 
324 	WARN_ON(job->gang_submit);
325 
326 	/*
327 	 * Don't add a reference when we are the gang leader to avoid circle
328 	 * dependency.
329 	 */
330 	if (job != leader)
331 		dma_fence_get(fence);
332 	job->gang_submit = fence;
333 }
334 
335 void amdgpu_job_free(struct amdgpu_job *job)
336 {
337 	if (job->gang_submit != &job->base.s_fence->scheduled)
338 		dma_fence_put(job->gang_submit);
339 
340 	if (job->base.s_fence)
341 		drm_sched_job_cleanup(&job->base);
342 
343 	amdgpu_job_free_resources(job);
344 	amdgpu_sync_free(&job->explicit_sync);
345 
346 	if (dma_fence_was_initialized(&job->hw_fence->base))
347 		dma_fence_put(&job->hw_fence->base);
348 	else
349 		kfree(job->hw_fence);
350 	if (dma_fence_was_initialized(&job->hw_vm_fence->base))
351 		dma_fence_put(&job->hw_vm_fence->base);
352 	else
353 		kfree(job->hw_vm_fence);
354 
355 	kfree(job);
356 }
357 
358 struct dma_fence *amdgpu_job_submit(struct amdgpu_job *job)
359 {
360 	struct dma_fence *f;
361 
362 	drm_sched_job_arm(&job->base);
363 	f = dma_fence_get(&job->base.s_fence->finished);
364 	amdgpu_job_free_resources(job);
365 	drm_sched_entity_push_job(&job->base);
366 
367 	return f;
368 }
369 
370 int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
371 			     struct dma_fence **fence)
372 {
373 	int r;
374 
375 	job->base.sched = &ring->sched;
376 	r = amdgpu_ib_schedule(ring, job, fence);
377 
378 	if (r)
379 		return r;
380 
381 	amdgpu_job_free(job);
382 	return 0;
383 }
384 
385 static struct dma_fence *
386 amdgpu_job_prepare_job(struct drm_sched_job *sched_job,
387 		      struct drm_sched_entity *s_entity)
388 {
389 	struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched);
390 	struct amdgpu_job *job = to_amdgpu_job(sched_job);
391 	struct dma_fence *fence;
392 	int r;
393 
394 	r = drm_sched_entity_error(s_entity);
395 	if (r)
396 		goto error;
397 
398 	if (job->gang_submit) {
399 		fence = amdgpu_device_switch_gang(ring->adev, job->gang_submit);
400 		if (fence)
401 			return fence;
402 	}
403 
404 	fence = amdgpu_device_enforce_isolation(ring->adev, ring, job);
405 	if (fence)
406 		return fence;
407 
408 	if (job->vm && !job->vmid) {
409 		r = amdgpu_vmid_grab(job->vm, ring, job, &fence);
410 		if (r) {
411 			dev_err(ring->adev->dev, "Error getting VM ID (%d)\n", r);
412 			goto error;
413 		}
414 		return fence;
415 	}
416 
417 	return NULL;
418 
419 error:
420 	dma_fence_set_error(&job->base.s_fence->finished, r);
421 	return NULL;
422 }
423 
424 static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
425 {
426 	struct amdgpu_ring *ring = to_amdgpu_ring(sched_job->sched);
427 	struct amdgpu_device *adev = ring->adev;
428 	struct dma_fence *fence = NULL, *finished;
429 	struct amdgpu_job *job;
430 	int r = 0;
431 
432 	job = to_amdgpu_job(sched_job);
433 	finished = &job->base.s_fence->finished;
434 
435 	trace_amdgpu_sched_run_job(job);
436 
437 	/* Skip job if VRAM is lost and never resubmit gangs */
438 	if (job->generation != amdgpu_vm_generation(adev, job->vm) ||
439 	    (job->job_run_counter && job->gang_submit))
440 		dma_fence_set_error(finished, -ECANCELED);
441 
442 	if (finished->error < 0) {
443 		dev_dbg(adev->dev, "Skip scheduling IBs in ring(%s)",
444 			ring->name);
445 	} else {
446 		r = amdgpu_ib_schedule(ring, job, &fence);
447 		if (r)
448 			dev_err(adev->dev,
449 				"Error scheduling IBs (%d) in ring(%s)", r,
450 				ring->name);
451 	}
452 
453 	job->job_run_counter++;
454 	amdgpu_job_free_resources(job);
455 
456 	fence = r ? ERR_PTR(r) : fence;
457 	return fence;
458 }
459 
460 /*
461  * This is a duplicate function from DRM scheduler sched_internal.h.
462  * Plan is to remove it when amdgpu_job_stop_all_jobs_on_sched is removed, due
463  * latter being incorrect and racy.
464  *
465  * See https://lore.kernel.org/amd-gfx/44edde63-7181-44fb-a4f7-94e50514f539@amd.com/
466  */
467 static struct drm_sched_job *
468 drm_sched_entity_queue_pop(struct drm_sched_entity *entity)
469 {
470 	struct spsc_node *node;
471 
472 	node = spsc_queue_pop(&entity->job_queue);
473 	if (!node)
474 		return NULL;
475 
476 	return container_of(node, struct drm_sched_job, queue_node);
477 }
478 
479 void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched)
480 {
481 	struct drm_sched_job *s_job;
482 	struct drm_sched_entity *s_entity = NULL;
483 	int i;
484 
485 	/* Signal all jobs not yet scheduled */
486 	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) {
487 		struct drm_sched_rq *rq = sched->sched_rq[i];
488 		spin_lock(&rq->lock);
489 		list_for_each_entry(s_entity, &rq->entities, list) {
490 			while ((s_job = drm_sched_entity_queue_pop(s_entity))) {
491 				struct drm_sched_fence *s_fence = s_job->s_fence;
492 
493 				dma_fence_signal(&s_fence->scheduled);
494 				dma_fence_set_error(&s_fence->finished, -EHWPOISON);
495 				dma_fence_signal(&s_fence->finished);
496 			}
497 		}
498 		spin_unlock(&rq->lock);
499 	}
500 
501 	/* Signal all jobs already scheduled to HW */
502 	list_for_each_entry(s_job, &sched->pending_list, list) {
503 		struct drm_sched_fence *s_fence = s_job->s_fence;
504 
505 		dma_fence_set_error(&s_fence->finished, -EHWPOISON);
506 		dma_fence_signal(&s_fence->finished);
507 	}
508 }
509 
510 const struct drm_sched_backend_ops amdgpu_sched_ops = {
511 	.prepare_job = amdgpu_job_prepare_job,
512 	.run_job = amdgpu_job_run,
513 	.timedout_job = amdgpu_job_timedout,
514 	.free_job = amdgpu_job_free_cb
515 };
516