Lines Matching +full:dt +full:- +full:check +full:- +full:base
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
5 #include <linux/iosys-map.h>
21 struct dma_fence base; member
35 return -ENOMEM; in lima_sched_slab_init()
44 if (!--lima_fence_slab_refcnt) { in lima_sched_slab_fini()
52 return container_of(fence, struct lima_fence, base); in to_lima_fence()
64 return f->pipe->base.name; in lima_fence_get_timeline_name()
79 call_rcu(&f->base.rcu, lima_fence_release_rcu); in lima_fence_release()
96 fence->pipe = pipe; in lima_fence_create()
97 dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock, in lima_fence_create()
98 pipe->fence_context, ++pipe->fence_seqno); in lima_fence_create()
105 return container_of(job, struct lima_sched_task, base); in to_lima_task()
110 return container_of(sched, struct lima_sched_pipe, base); in to_lima_pipe()
121 task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL); in lima_sched_task_init()
122 if (!task->bos) in lima_sched_task_init()
123 return -ENOMEM; in lima_sched_task_init()
126 drm_gem_object_get(&bos[i]->base.base); in lima_sched_task_init()
128 err = drm_sched_job_init(&task->base, &context->base, 1, vm, in lima_sched_task_init()
131 kfree(task->bos); in lima_sched_task_init()
135 drm_sched_job_arm(&task->base); in lima_sched_task_init()
137 task->num_bos = num_bos; in lima_sched_task_init()
138 task->vm = lima_vm_get(vm); in lima_sched_task_init()
147 drm_sched_job_cleanup(&task->base); in lima_sched_task_fini()
149 if (task->bos) { in lima_sched_task_fini()
150 for (i = 0; i < task->num_bos; i++) in lima_sched_task_fini()
151 drm_gem_object_put(&task->bos[i]->base.base); in lima_sched_task_fini()
152 kfree(task->bos); in lima_sched_task_fini()
155 lima_vm_put(task->vm); in lima_sched_task_fini()
161 struct drm_gpu_scheduler *sched = &pipe->base; in lima_sched_context_init()
163 return drm_sched_entity_init(&context->base, DRM_SCHED_PRIORITY_NORMAL, in lima_sched_context_init()
170 drm_sched_entity_destroy(&context->base); in lima_sched_context_fini()
175 struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished); in lima_sched_context_queue_task()
178 drm_sched_entity_push_job(&task->base); in lima_sched_context_queue_task()
187 ret = pm_runtime_resume_and_get(ldev->dev); in lima_pm_busy()
191 lima_devfreq_record_busy(&ldev->devfreq); in lima_pm_busy()
197 lima_devfreq_record_idle(&ldev->devfreq); in lima_pm_idle()
200 pm_runtime_mark_last_busy(ldev->dev); in lima_pm_idle()
201 pm_runtime_put_autosuspend(ldev->dev); in lima_pm_idle()
207 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched); in lima_sched_run_job()
208 struct lima_device *ldev = pipe->ldev; in lima_sched_run_job()
213 if (job->s_fence->finished.error < 0) in lima_sched_run_job()
222 dma_fence_put(&fence->base); in lima_sched_run_job()
226 task->fence = &fence->base; in lima_sched_run_job()
231 dma_fence_get(task->fence); in lima_sched_run_job()
233 pipe->current_task = task; in lima_sched_run_job()
250 for (i = 0; i < pipe->num_l2_cache; i++) in lima_sched_run_job()
251 lima_l2_cache_flush(pipe->l2_cache[i]); in lima_sched_run_job()
253 lima_vm_put(pipe->current_vm); in lima_sched_run_job()
254 pipe->current_vm = lima_vm_get(task->vm); in lima_sched_run_job()
256 if (pipe->bcast_mmu) in lima_sched_run_job()
257 lima_mmu_switch_vm(pipe->bcast_mmu, pipe->current_vm); in lima_sched_run_job()
259 for (i = 0; i < pipe->num_mmu; i++) in lima_sched_run_job()
260 lima_mmu_switch_vm(pipe->mmu[i], pipe->current_vm); in lima_sched_run_job()
265 pipe->error = false; in lima_sched_run_job()
266 pipe->task_run(pipe, task); in lima_sched_run_job()
268 return task->fence; in lima_sched_run_job()
274 struct lima_sched_pipe *pipe = to_lima_pipe(task->base.sched); in lima_sched_build_error_task_list()
275 struct lima_ip *ip = pipe->processor[0]; in lima_sched_build_error_task_list()
276 int pipe_id = ip->id == lima_ip_gp ? lima_pipe_gp : lima_pipe_pp; in lima_sched_build_error_task_list()
277 struct lima_device *dev = ip->dev; in lima_sched_build_error_task_list()
279 container_of(task->base.entity, in lima_sched_build_error_task_list()
280 struct lima_sched_context, base); in lima_sched_build_error_task_list()
283 struct lima_dump_task *dt; in lima_sched_build_error_task_list() local
292 mutex_lock(&dev->error_task_list_lock); in lima_sched_build_error_task_list()
294 if (dev->dump.num_tasks >= lima_max_error_tasks) { in lima_sched_build_error_task_list()
295 dev_info(dev->dev, "fail to save task state from %s pid %d: " in lima_sched_build_error_task_list()
296 "error task list is full\n", ctx->pname, ctx->pid); in lima_sched_build_error_task_list()
301 size = sizeof(struct lima_dump_chunk) + pipe->frame_size; in lima_sched_build_error_task_list()
303 size += sizeof(struct lima_dump_chunk) + sizeof(ctx->pname); in lima_sched_build_error_task_list()
307 for (i = 0; i < task->num_bos; i++) { in lima_sched_build_error_task_list()
308 struct lima_bo *bo = task->bos[i]; in lima_sched_build_error_task_list()
311 size += bo->heap_size ? bo->heap_size : lima_bo_size(bo); in lima_sched_build_error_task_list()
318 dev_err(dev->dev, "fail to alloc task dump buffer of size %x\n", in lima_sched_build_error_task_list()
323 et->data = et + 1; in lima_sched_build_error_task_list()
324 et->size = task_size; in lima_sched_build_error_task_list()
326 dt = et->data; in lima_sched_build_error_task_list()
327 memset(dt, 0, sizeof(*dt)); in lima_sched_build_error_task_list()
328 dt->id = pipe_id; in lima_sched_build_error_task_list()
329 dt->size = size; in lima_sched_build_error_task_list()
331 chunk = (struct lima_dump_chunk *)(dt + 1); in lima_sched_build_error_task_list()
333 chunk->id = LIMA_DUMP_CHUNK_FRAME; in lima_sched_build_error_task_list()
334 chunk->size = pipe->frame_size; in lima_sched_build_error_task_list()
335 memcpy(chunk + 1, task->frame, pipe->frame_size); in lima_sched_build_error_task_list()
336 dt->num_chunks++; in lima_sched_build_error_task_list()
338 chunk = (void *)(chunk + 1) + chunk->size; in lima_sched_build_error_task_list()
340 chunk->id = LIMA_DUMP_CHUNK_PROCESS_NAME; in lima_sched_build_error_task_list()
341 chunk->size = sizeof(ctx->pname); in lima_sched_build_error_task_list()
342 memcpy(chunk + 1, ctx->pname, sizeof(ctx->pname)); in lima_sched_build_error_task_list()
343 dt->num_chunks++; in lima_sched_build_error_task_list()
345 pid_chunk = (void *)(chunk + 1) + chunk->size; in lima_sched_build_error_task_list()
347 pid_chunk->id = LIMA_DUMP_CHUNK_PROCESS_ID; in lima_sched_build_error_task_list()
348 pid_chunk->pid = ctx->pid; in lima_sched_build_error_task_list()
349 dt->num_chunks++; in lima_sched_build_error_task_list()
351 buffer_chunk = (void *)(pid_chunk + 1) + pid_chunk->size; in lima_sched_build_error_task_list()
352 for (i = 0; i < task->num_bos; i++) { in lima_sched_build_error_task_list()
353 struct lima_bo *bo = task->bos[i]; in lima_sched_build_error_task_list()
357 buffer_chunk->id = LIMA_DUMP_CHUNK_BUFFER; in lima_sched_build_error_task_list()
358 buffer_chunk->va = lima_vm_get_va(task->vm, bo); in lima_sched_build_error_task_list()
360 if (bo->heap_size) { in lima_sched_build_error_task_list()
361 buffer_chunk->size = bo->heap_size; in lima_sched_build_error_task_list()
363 data = vmap(bo->base.pages, bo->heap_size >> PAGE_SHIFT, in lima_sched_build_error_task_list()
370 memcpy(buffer_chunk + 1, data, buffer_chunk->size); in lima_sched_build_error_task_list()
374 buffer_chunk->size = lima_bo_size(bo); in lima_sched_build_error_task_list()
376 ret = drm_gem_vmap(&bo->base.base, &map); in lima_sched_build_error_task_list()
382 memcpy(buffer_chunk + 1, map.vaddr, buffer_chunk->size); in lima_sched_build_error_task_list()
384 drm_gem_vunmap(&bo->base.base, &map); in lima_sched_build_error_task_list()
387 buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size; in lima_sched_build_error_task_list()
388 dt->num_chunks++; in lima_sched_build_error_task_list()
391 list_add(&et->list, &dev->error_task_list); in lima_sched_build_error_task_list()
392 dev->dump.size += et->size; in lima_sched_build_error_task_list()
393 dev->dump.num_tasks++; in lima_sched_build_error_task_list()
395 dev_info(dev->dev, "save error task state success\n"); in lima_sched_build_error_task_list()
398 mutex_unlock(&dev->error_task_list_lock); in lima_sched_build_error_task_list()
403 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched); in lima_sched_timedout_job()
405 struct lima_device *ldev = pipe->ldev; in lima_sched_timedout_job()
406 struct lima_ip *ip = pipe->processor[0]; in lima_sched_timedout_job()
413 if (dma_fence_is_signaled(task->fence)) { in lima_sched_timedout_job()
422 * timeouts, synchronize the IRQ handler and re-check the fence in lima_sched_timedout_job()
425 for (i = 0; i < pipe->num_processor; i++) in lima_sched_timedout_job()
426 synchronize_irq(pipe->processor[i]->irq); in lima_sched_timedout_job()
427 if (pipe->bcast_processor) in lima_sched_timedout_job()
428 synchronize_irq(pipe->bcast_processor->irq); in lima_sched_timedout_job()
430 if (dma_fence_is_signaled(task->fence)) { in lima_sched_timedout_job()
440 pipe->task_mask_irq(pipe); in lima_sched_timedout_job()
442 if (!pipe->error) in lima_sched_timedout_job()
445 drm_sched_stop(&pipe->base, &task->base); in lima_sched_timedout_job()
447 drm_sched_increase_karma(&task->base); in lima_sched_timedout_job()
452 pipe->task_error(pipe); in lima_sched_timedout_job()
454 if (pipe->bcast_mmu) in lima_sched_timedout_job()
455 lima_mmu_page_fault_resume(pipe->bcast_mmu); in lima_sched_timedout_job()
457 for (i = 0; i < pipe->num_mmu; i++) in lima_sched_timedout_job()
458 lima_mmu_page_fault_resume(pipe->mmu[i]); in lima_sched_timedout_job()
461 lima_vm_put(pipe->current_vm); in lima_sched_timedout_job()
462 pipe->current_vm = NULL; in lima_sched_timedout_job()
463 pipe->current_task = NULL; in lima_sched_timedout_job()
467 drm_sched_resubmit_jobs(&pipe->base); in lima_sched_timedout_job()
468 drm_sched_start(&pipe->base, 0); in lima_sched_timedout_job()
476 struct lima_sched_pipe *pipe = to_lima_pipe(job->sched); in lima_sched_free_job()
477 struct lima_vm *vm = task->vm; in lima_sched_free_job()
478 struct lima_bo **bos = task->bos; in lima_sched_free_job()
481 dma_fence_put(task->fence); in lima_sched_free_job()
483 for (i = 0; i < task->num_bos; i++) in lima_sched_free_job()
487 kmem_cache_free(pipe->task_slab, task); in lima_sched_free_job()
502 for (i = 0; i < pipe->num_l2_cache; i++) in lima_sched_recover_work()
503 lima_l2_cache_flush(pipe->l2_cache[i]); in lima_sched_recover_work()
505 if (pipe->bcast_mmu) { in lima_sched_recover_work()
506 lima_mmu_flush_tlb(pipe->bcast_mmu); in lima_sched_recover_work()
508 for (i = 0; i < pipe->num_mmu; i++) in lima_sched_recover_work()
509 lima_mmu_flush_tlb(pipe->mmu[i]); in lima_sched_recover_work()
512 if (pipe->task_recover(pipe)) in lima_sched_recover_work()
513 drm_sched_fault(&pipe->base); in lima_sched_recover_work()
527 .dev = pipe->ldev->dev, in lima_sched_pipe_init()
530 pipe->fence_context = dma_fence_context_alloc(1); in lima_sched_pipe_init()
531 spin_lock_init(&pipe->fence_lock); in lima_sched_pipe_init()
533 INIT_WORK(&pipe->recover_work, lima_sched_recover_work); in lima_sched_pipe_init()
535 return drm_sched_init(&pipe->base, &args); in lima_sched_pipe_init()
540 drm_sched_fini(&pipe->base); in lima_sched_pipe_fini()
545 struct lima_sched_task *task = pipe->current_task; in lima_sched_pipe_task_done()
546 struct lima_device *ldev = pipe->ldev; in lima_sched_pipe_task_done()
548 if (pipe->error) { in lima_sched_pipe_task_done()
549 if (task && task->recoverable) in lima_sched_pipe_task_done()
550 schedule_work(&pipe->recover_work); in lima_sched_pipe_task_done()
552 drm_sched_fault(&pipe->base); in lima_sched_pipe_task_done()
554 pipe->task_fini(pipe); in lima_sched_pipe_task_done()
555 dma_fence_signal(task->fence); in lima_sched_pipe_task_done()