44 static struct dma_fence *rocket_fence_create(struct rocket_core *core)  in rocket_fence_create()  argument
52 	dma_fence_init(fence, &rocket_fence_ops, &core->fence_lock,  in rocket_fence_create()
53 		       core->fence_context, ++core->emit_seqno);  in rocket_fence_create()
108 static void rocket_job_hw_submit(struct rocket_core *core, struct rocket_job *job)  in rocket_job_hw_submit()  argument
114 	if (atomic_read(&core->reset.pending))  in rocket_job_hw_submit()
122 	rocket_pc_writel(core, BASE_ADDRESS, 0x1);  in rocket_job_hw_submit()
125 	extra_bit = 0x10000000 * core->index;  in rocket_job_hw_submit()
126 	rocket_cna_writel(core, S_POINTER, CNA_S_POINTER_POINTER_PP_EN(1) |  in rocket_job_hw_submit()
131 	rocket_core_writel(core, S_POINTER, CORE_S_POINTER_POINTER_PP_EN(1) |  in rocket_job_hw_submit()
136 	rocket_pc_writel(core, BASE_ADDRESS, task->regcmd);  in rocket_job_hw_submit()
137 	rocket_pc_writel(core, REGISTER_AMOUNTS,  in rocket_job_hw_submit()
140 	rocket_pc_writel(core, INTERRUPT_MASK, PC_INTERRUPT_MASK_DPU_0 | PC_INTERRUPT_MASK_DPU_1);  in rocket_job_hw_submit()
141 	rocket_pc_writel(core, INTERRUPT_CLEAR, PC_INTERRUPT_CLEAR_DPU_0 | PC_INTERRUPT_CLEAR_DPU_1);  in rocket_job_hw_submit()
143 	rocket_pc_writel(core, TASK_CON, PC_TASK_CON_RESERVED_0(1) |  in rocket_job_hw_submit()
148 	rocket_pc_writel(core, TASK_DMA_BASE_ADDR, PC_TASK_DMA_BASE_ADDR_DMA_BASE_ADDR(0x0));  in rocket_job_hw_submit()
150 	rocket_pc_writel(core, OPERATION_ENABLE, PC_OPERATION_ENABLE_OP_EN(1));  in rocket_job_hw_submit()
152 	dev_dbg(core->dev, "Submitted regcmd at 0x%llx to core %d", task->regcmd, core->index);  in rocket_job_hw_submit()
277 	unsigned int core;  in sched_to_core()  local
279 	for (core = 0; core < rdev->num_cores; core++) {  in sched_to_core()
280 		if (&rdev->cores[core].sched == sched)  in sched_to_core()
281 			return &rdev->cores[core];  in sched_to_core()
291 	struct rocket_core *core = sched_to_core(rdev, sched_job->sched);  in rocket_job_run()  local
305 	fence = rocket_fence_create(core);  in rocket_job_run()
313 	ret = pm_runtime_get_sync(core->dev);  in rocket_job_run()
317 	ret = iommu_attach_group(job->domain->domain, core->iommu_group);  in rocket_job_run()
321 	scoped_guard(mutex, &core->job_lock) {  in rocket_job_run()
322 		core->in_flight_job = job;  in rocket_job_run()
323 		rocket_job_hw_submit(core, job);  in rocket_job_run()
329 static void rocket_job_handle_irq(struct rocket_core *core)  in rocket_job_handle_irq()  argument
331 	pm_runtime_mark_last_busy(core->dev);  in rocket_job_handle_irq()
333 	rocket_pc_writel(core, OPERATION_ENABLE, 0x0);  in rocket_job_handle_irq()
334 	rocket_pc_writel(core, INTERRUPT_CLEAR, 0x1ffff);  in rocket_job_handle_irq()
336 	scoped_guard(mutex, &core->job_lock)  in rocket_job_handle_irq()
337 		if (core->in_flight_job) {  in rocket_job_handle_irq()
338 			if (core->in_flight_job->next_task_idx < core->in_flight_job->task_count) {  in rocket_job_handle_irq()
339 				rocket_job_hw_submit(core, core->in_flight_job);  in rocket_job_handle_irq()
343 			iommu_detach_group(NULL, iommu_group_get(core->dev));  in rocket_job_handle_irq()
344 			dma_fence_signal(core->in_flight_job->done_fence);  in rocket_job_handle_irq()
345 			pm_runtime_put_autosuspend(core->dev);  in rocket_job_handle_irq()
346 			core->in_flight_job = NULL;  in rocket_job_handle_irq()
351 rocket_reset(struct rocket_core *core, struct drm_sched_job *bad)  in rocket_reset()  argument
353 	if (!atomic_read(&core->reset.pending))  in rocket_reset()
356 	drm_sched_stop(&core->sched, bad);  in rocket_reset()
363 	scoped_guard(mutex, &core->job_lock) {  in rocket_reset()
364 		if (core->in_flight_job)  in rocket_reset()
365 			pm_runtime_put_noidle(core->dev);  in rocket_reset()
367 		iommu_detach_group(NULL, core->iommu_group);  in rocket_reset()
369 		core->in_flight_job = NULL;  in rocket_reset()
373 	rocket_core_reset(core);  in rocket_reset()
376 	atomic_set(&core->reset.pending, 0);  in rocket_reset()
379 	drm_sched_start(&core->sched, 0);  in rocket_reset()
386 	struct rocket_core *core = sched_to_core(rdev, sched_job->sched);  in rocket_job_timedout()  local
388 	dev_err(core->dev, "NPU job timed out");  in rocket_job_timedout()
390 	atomic_set(&core->reset.pending, 1);  in rocket_job_timedout()
391 	rocket_reset(core, sched_job);  in rocket_job_timedout()
398 	struct rocket_core *core;  in rocket_reset_work()  local
400 	core = container_of(work, struct rocket_core, reset.work);  in rocket_reset_work()
401 	rocket_reset(core, NULL);  in rocket_reset_work()
412 	struct rocket_core *core = data;  in rocket_job_irq_handler_thread()  local
414 	rocket_job_handle_irq(core);  in rocket_job_irq_handler_thread()
421 	struct rocket_core *core = data;  in rocket_job_irq_handler()  local
422 	u32 raw_status = rocket_pc_readl(core, INTERRUPT_RAW_STATUS);  in rocket_job_irq_handler()
431 	rocket_pc_writel(core, INTERRUPT_MASK, 0x0);  in rocket_job_irq_handler()
436 int rocket_job_init(struct rocket_core *core)  in rocket_job_init()  argument
443 		.name = dev_name(core->dev),  in rocket_job_init()
444 		.dev = core->dev,  in rocket_job_init()
448 	INIT_WORK(&core->reset.work, rocket_reset_work);  in rocket_job_init()
449 	spin_lock_init(&core->fence_lock);  in rocket_job_init()
450 	mutex_init(&core->job_lock);  in rocket_job_init()
452 	core->irq = platform_get_irq(to_platform_device(core->dev), 0);  in rocket_job_init()
453 	if (core->irq < 0)  in rocket_job_init()
454 		return core->irq;  in rocket_job_init()
456 	ret = devm_request_threaded_irq(core->dev, core->irq,  in rocket_job_init()
459 					IRQF_SHARED, dev_name(core->dev),  in rocket_job_init()
460 					core);  in rocket_job_init()
462 		dev_err(core->dev, "failed to request job irq");  in rocket_job_init()
466 	core->reset.wq = alloc_ordered_workqueue("rocket-reset-%d", 0, core->index);  in rocket_job_init()
467 	if (!core->reset.wq)  in rocket_job_init()
470 	core->fence_context = dma_fence_context_alloc(1);  in rocket_job_init()
472 	args.timeout_wq = core->reset.wq;  in rocket_job_init()
473 	ret = drm_sched_init(&core->sched, &args);  in rocket_job_init()
475 		dev_err(core->dev, "Failed to create scheduler: %d.", ret);  in rocket_job_init()
482 	drm_sched_fini(&core->sched);  in rocket_job_init()
484 	destroy_workqueue(core->reset.wq);  in rocket_job_init()
488 void rocket_job_fini(struct rocket_core *core)  in rocket_job_fini()  argument
490 	drm_sched_fini(&core->sched);  in rocket_job_fini()
492 	cancel_work_sync(&core->reset.work);  in rocket_job_fini()
493 	destroy_workqueue(core->reset.wq);  in rocket_job_fini()
501 	unsigned int core;  in rocket_job_open()  local
504 	for (core = 0; core < rdev->num_cores; core++)  in rocket_job_open()
505 		scheds[core] = &rdev->cores[core].sched;  in rocket_job_open()
525 int rocket_job_is_idle(struct rocket_core *core)  in rocket_job_is_idle()  argument
528 	if (atomic_read(&core->sched.credit_count))  in rocket_job_is_idle()