Lines Matching +full:panfrost +full:- +full:gpu
1 // SPDX-License-Identifier: GPL-2.0
10 #include <linux/dma-resv.h>
27 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
28 #define job_read(dev, reg) readl(dev->iomem + (reg))
51 /* panfrost seqno for signaled() test */
64 return "panfrost";
71 switch (f->queue) {
73 return "panfrost-js-0";
75 return "panfrost-js-1";
77 return "panfrost-js-2";
91 struct panfrost_job_slot *js = pfdev->js;
95 return ERR_PTR(-ENOMEM);
97 fence->dev = pfdev->ddev;
98 fence->queue = js_num;
99 fence->seqno = ++js->queue[js_num].emit_seqno;
100 dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,
101 js->queue[js_num].fence_context, fence->seqno);
103 return &fence->base;
112 if (job->requirements & PANFROST_JD_REQ_FS)
117 if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {
118 if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&
119 (job->pfdev->features.nr_core_groups == 2))
121 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))
139 affinity = pfdev->features.shader_present;
148 struct panfrost_fence *f = to_panfrost_fence(job->done_fence);
150 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
153 return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0;
159 struct panfrost_job *job = pfdev->jobs[slot][0];
163 if (job->is_profiled && job->engine_usage) {
164 job->engine_usage->elapsed_ns[slot] +=
165 ktime_to_ns(ktime_sub(ktime_get(), job->start_time));
166 job->engine_usage->cycles[slot] +=
167 panfrost_cycle_counter_read(pfdev) - job->start_cycles;
170 if (job->requirements & PANFROST_JD_REQ_CYCLE_COUNT || job->is_profiled)
173 pfdev->jobs[slot][0] = pfdev->jobs[slot][1];
174 pfdev->jobs[slot][1] = NULL;
186 if (!pfdev->jobs[slot][0]) {
187 pfdev->jobs[slot][0] = job;
191 WARN_ON(pfdev->jobs[slot][1]);
192 pfdev->jobs[slot][1] = job;
194 panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));
200 struct panfrost_device *pfdev = job->pfdev;
203 u64 jc_head = job->jc;
206 panfrost_devfreq_record_busy(&pfdev->pfdevfreq);
208 ret = pm_runtime_get_sync(pfdev->dev);
216 cfg = panfrost_mmu_as_get(pfdev, job->mmu);
221 panfrost_job_write_affinity(pfdev, job->requirements, js);
239 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
243 spin_lock(&pfdev->js->job_lock);
246 if (!atomic_read(&pfdev->reset.pending)) {
247 job->is_profiled = pfdev->profile_mode;
249 if (job->requirements & PANFROST_JD_REQ_CYCLE_COUNT ||
250 job->is_profiled)
253 if (job->is_profiled) {
254 job->start_time = ktime_get();
255 job->start_cycles = panfrost_cycle_counter_read(pfdev);
259 dev_dbg(pfdev->dev,
263 spin_unlock(&pfdev->js->job_lock);
273 ret = dma_resv_reserve_fences(bos[i]->resv, 1);
277 /* panfrost always uses write mode in its current uapi */
294 dma_resv_add_fence(bos[i]->resv, fence, DMA_RESV_USAGE_WRITE);
299 struct panfrost_device *pfdev = job->pfdev;
303 ret = drm_gem_lock_reservations(job->bos, job->bo_count,
308 mutex_lock(&pfdev->sched_lock);
309 drm_sched_job_arm(&job->base);
311 job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);
313 ret = panfrost_acquire_object_fences(job->bos, job->bo_count,
314 &job->base);
316 mutex_unlock(&pfdev->sched_lock);
320 kref_get(&job->refcount); /* put by scheduler job completion */
322 drm_sched_entity_push_job(&job->base);
324 mutex_unlock(&pfdev->sched_lock);
326 panfrost_attach_object_fences(job->bos, job->bo_count,
327 job->render_done_fence);
330 drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);
341 dma_fence_put(job->done_fence);
342 dma_fence_put(job->render_done_fence);
344 if (job->mappings) {
345 for (i = 0; i < job->bo_count; i++) {
346 if (!job->mappings[i])
349 atomic_dec(&job->mappings[i]->obj->gpu_usecount);
350 panfrost_gem_mapping_put(job->mappings[i]);
352 kvfree(job->mappings);
355 if (job->bos) {
356 for (i = 0; i < job->bo_count; i++)
357 drm_gem_object_put(job->bos[i]);
359 kvfree(job->bos);
367 kref_put(&job->refcount, panfrost_job_cleanup);
382 struct panfrost_device *pfdev = job->pfdev;
386 if (unlikely(job->base.s_fence->finished.error))
390 * we were resetting the GPU.
392 if (!job->jc)
399 if (job->done_fence)
400 dma_fence_put(job->done_fence);
401 job->done_fence = dma_fence_get(fence);
413 clear_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended);
425 set_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended);
428 synchronize_irq(pfdev->js->irq);
440 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",
445 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
453 job->jc = job_read(pfdev, JS_TAIL_LO(js)) |
459 /* Job has been hard-stopped, flag it as canceled */
460 dma_fence_set_error(job->done_fence, -ECANCELED);
461 job->jc = 0;
463 /* We might want to provide finer-grained error code based on
467 dma_fence_set_error(job->done_fence, -EINVAL);
468 job->jc = 0;
471 panfrost_mmu_as_put(pfdev, job->mmu);
472 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
475 dma_fence_signal_locked(job->done_fence);
477 pm_runtime_put_autosuspend(pfdev->dev);
480 atomic_set(&pfdev->reset.pending, 1);
481 drm_sched_fault(&pfdev->js->queue[js].sched);
488 /* Set ->jc to 0 to avoid re-submitting an already finished job (can
489 * happen when we receive the DONE interrupt while doing a GPU reset).
491 job->jc = 0;
492 panfrost_mmu_as_put(pfdev, job->mmu);
493 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
495 dma_fence_signal_locked(job->done_fence);
496 pm_runtime_put_autosuspend(pfdev->dev);
557 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {
562 * are inactive, but one job remains in pfdev->jobs[j],
584 if (!failed[j] || !pfdev->jobs[j][0])
587 if (pfdev->jobs[j][0]->jc == 0) {
591 dma_fence_set_error(canceled->done_fence, -ECANCELED);
593 } else if (!atomic_read(&pfdev->reset.pending)) {
605 pm_runtime_mark_last_busy(pfdev->dev);
607 spin_lock(&pfdev->js->job_lock);
609 spin_unlock(&pfdev->js->job_lock);
644 if (!atomic_read(&pfdev->reset.pending))
661 drm_sched_stop(&pfdev->js->queue[i].sched, bad);
672 synchronize_irq(pfdev->js->irq);
675 /* Cancel the next job and soft-stop the running job. */
680 /* Wait at most 10ms for soft-stops to complete */
681 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,
686 dev_err(pfdev->dev, "Soft-stop failed\n");
698 spin_lock(&pfdev->js->job_lock);
700 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) {
701 if (pfdev->jobs[i][j]->requirements & PANFROST_JD_REQ_CYCLE_COUNT ||
702 pfdev->jobs[i][j]->is_profiled)
703 panfrost_cycle_counter_put(pfdev->jobs[i][j]->pfdev);
704 pm_runtime_put_noidle(pfdev->dev);
705 panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
708 memset(pfdev->jobs, 0, sizeof(pfdev->jobs));
709 spin_unlock(&pfdev->js->job_lock);
719 /* GPU has been reset, we can clear the reset pending bit. */
720 atomic_set(&pfdev->reset.pending, 0);
731 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
736 drm_sched_start(&pfdev->js->queue[i].sched, 0);
738 /* Re-enable job interrupts now that everything has been restarted. */
740 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
741 GENMASK(NUM_JOB_SLOTS - 1, 0));
750 struct panfrost_device *pfdev = job->pfdev;
754 * If the GPU managed to complete this jobs fence, the timeout is
757 if (dma_fence_is_signaled(job->done_fence))
761 * Panfrost IRQ handler may take a long time to process an interrupt
765 * In order to catch such cases and not report spurious Panfrost
766 * job timeouts, synchronize the IRQ handler and re-check the fence
769 synchronize_irq(pfdev->js->irq);
771 if (dma_fence_is_signaled(job->done_fence)) {
772 dev_warn(pfdev->dev, "unexpectedly high interrupt latency\n");
776 dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",
786 atomic_set(&pfdev->reset.pending, 1);
813 if (!test_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended))
815 GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |
816 GENMASK(NUM_JOB_SLOTS - 1, 0));
826 if (test_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended))
844 .timeout_wq = pfdev->reset.wq,
846 .dev = pfdev->dev,
858 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
860 return -ENOMEM;
862 INIT_WORK(&pfdev->reset.work, panfrost_reset_work);
863 spin_lock_init(&js->job_lock);
865 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
866 if (js->irq < 0)
867 return js->irq;
869 ret = devm_request_threaded_irq(pfdev->dev, js->irq,
872 IRQF_SHARED, KBUILD_MODNAME "-job",
875 dev_err(pfdev->dev, "failed to request job irq");
879 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0);
880 if (!pfdev->reset.wq)
881 return -ENOMEM;
884 js->queue[j].fence_context = dma_fence_context_alloc(1);
886 ret = drm_sched_init(&js->queue[j].sched, &args);
888 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
898 for (j--; j >= 0; j--)
899 drm_sched_fini(&js->queue[j].sched);
901 destroy_workqueue(pfdev->reset.wq);
907 struct panfrost_job_slot *js = pfdev->js;
913 drm_sched_fini(&js->queue[j].sched);
916 cancel_work_sync(&pfdev->reset.work);
917 destroy_workqueue(pfdev->reset.wq);
922 struct panfrost_device *pfdev = panfrost_priv->pfdev;
923 struct panfrost_job_slot *js = pfdev->js;
928 sched = &js->queue[i].sched;
929 ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i],
940 struct panfrost_device *pfdev = panfrost_priv->pfdev;
944 drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);
946 /* Kill in-flight jobs */
947 spin_lock(&pfdev->js->job_lock);
949 struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i];
952 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {
953 struct panfrost_job *job = pfdev->jobs[i][j];
956 if (!job || job->base.entity != entity)
965 job->jc = 0;
979 job->engine_usage = NULL;
982 spin_unlock(&pfdev->js->job_lock);
987 struct panfrost_job_slot *js = pfdev->js;
992 if (atomic_read(&js->queue[i].sched.credit_count))