72 	struct ivpu_device *vdev = file_priv->vdev;  in ivpu_file_priv_get()  local
76 	ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n",  in ivpu_file_priv_get()
82 static void file_priv_unbind(struct ivpu_device *vdev, struct ivpu_file_priv *file_priv)  in file_priv_unbind()  argument
86 		ivpu_dbg(vdev, FILE, "file_priv unbind: ctx %u\n", file_priv->ctx.id);  in file_priv_unbind()
89 		ivpu_bo_unbind_all_bos_from_context(vdev, &file_priv->ctx);  in file_priv_unbind()
90 		ivpu_mmu_context_fini(vdev, &file_priv->ctx);  in file_priv_unbind()
92 		drm_WARN_ON(&vdev->drm, !xa_erase_irq(&vdev->context_xa, file_priv->ctx.id));  in file_priv_unbind()
100 	struct ivpu_device *vdev = file_priv->vdev;  in file_priv_release()  local
102 	ivpu_dbg(vdev, FILE, "file_priv release: ctx %u bound %d\n",  in file_priv_release()
105 	pm_runtime_get_sync(vdev->drm.dev);  in file_priv_release()
106 	mutex_lock(&vdev->context_list_lock);  in file_priv_release()
107 	file_priv_unbind(vdev, file_priv);  in file_priv_release()
108 	drm_WARN_ON(&vdev->drm, !xa_empty(&file_priv->cmdq_xa));  in file_priv_release()
110 	mutex_unlock(&vdev->context_list_lock);  in file_priv_release()
111 	pm_runtime_put_autosuspend(vdev->drm.dev);  in file_priv_release()
121 	struct ivpu_device *vdev = file_priv->vdev;  in ivpu_file_priv_put()  local
123 	ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",  in ivpu_file_priv_put()
130 bool ivpu_is_capable(struct ivpu_device *vdev, u32 capability)  in ivpu_is_capable()  argument
140 		return vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW;  in ivpu_is_capable()
149 	struct ivpu_device *vdev = file_priv->vdev;  in ivpu_get_param_ioctl()  local
150 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);  in ivpu_get_param_ioctl()
166 		args->value = vdev->platform;  in ivpu_get_param_ioctl()
169 		args->value = ivpu_hw_dpu_max_freq_get(vdev);  in ivpu_get_param_ioctl()
172 		args->value = ivpu_get_context_count(vdev);  in ivpu_get_param_ioctl()
175 		args->value = vdev->hw->ranges.user.start;  in ivpu_get_param_ioctl()
184 			fw_hdr = (struct vpu_firmware_header *)vdev->fw->file->data;  in ivpu_get_param_ioctl()
191 		ret = ivpu_jsm_get_heartbeat(vdev, args->index, &args->value);  in ivpu_get_param_ioctl()
194 		args->value = (u64)atomic64_inc_return(&vdev->unique_id_counter);  in ivpu_get_param_ioctl()
197 		args->value = vdev->hw->tile_fuse;  in ivpu_get_param_ioctl()
200 		args->value = vdev->hw->sku;  in ivpu_get_param_ioctl()
203 		args->value = ivpu_is_capable(vdev, args->index);  in ivpu_get_param_ioctl()
206 		args->value = ivpu_fw_preempt_buf_size(vdev);  in ivpu_get_param_ioctl()
232 	struct ivpu_device *vdev = to_ivpu_device(dev);  in ivpu_open()  local
248 	file_priv->vdev = vdev;  in ivpu_open()
254 	mutex_lock(&vdev->context_list_lock);  in ivpu_open()
256 	ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, file_priv,  in ivpu_open()
257 			   vdev->context_xa_limit, GFP_KERNEL);  in ivpu_open()
259 		ivpu_err(vdev, "Failed to allocate context id: %d\n", ret);  in ivpu_open()
263 	ivpu_mmu_context_init(vdev, &file_priv->ctx, ctx_id);  in ivpu_open()
272 	mutex_unlock(&vdev->context_list_lock);  in ivpu_open()
277 	ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n",  in ivpu_open()
283 	mutex_unlock(&vdev->context_list_lock);  in ivpu_open()
295 	struct ivpu_device *vdev = to_ivpu_device(dev);  in ivpu_postclose()  local
297 	ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n",  in ivpu_postclose()
321 static int ivpu_wait_for_ready(struct ivpu_device *vdev)  in ivpu_wait_for_ready()  argument
331 	ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG, NULL);  in ivpu_wait_for_ready()
333 	timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot);  in ivpu_wait_for_ready()
335 		ivpu_ipc_irq_handler(vdev);  in ivpu_wait_for_ready()
336 		ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0);  in ivpu_wait_for_ready()
343 	ivpu_ipc_consumer_del(vdev, &cons);  in ivpu_wait_for_ready()
346 		ivpu_err(vdev, "Invalid NPU ready message: 0x%x\n",  in ivpu_wait_for_ready()
352 		ivpu_dbg(vdev, PM, "NPU ready message received successfully\n");  in ivpu_wait_for_ready()
357 static int ivpu_hw_sched_init(struct ivpu_device *vdev)  in ivpu_hw_sched_init()  argument
361 	if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) {  in ivpu_hw_sched_init()
362 		ret = ivpu_jsm_hws_setup_priority_bands(vdev);  in ivpu_hw_sched_init()
364 			ivpu_err(vdev, "Failed to enable hw scheduler: %d", ret);  in ivpu_hw_sched_init()
379 int ivpu_boot(struct ivpu_device *vdev)  in ivpu_boot()  argument
383 	drm_WARN_ON(&vdev->drm, atomic_read(&vdev->job_timeout_counter));  in ivpu_boot()
384 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));  in ivpu_boot()
386 	ivpu_fw_boot_params_setup(vdev, ivpu_bo_vaddr(vdev->fw->mem_bp));  in ivpu_boot()
387 	vdev->fw->last_boot_mode = vdev->fw->next_boot_mode;  in ivpu_boot()
389 	ret = ivpu_hw_boot_fw(vdev);  in ivpu_boot()
391 		ivpu_err(vdev, "Failed to start the firmware: %d\n", ret);  in ivpu_boot()
395 	ret = ivpu_wait_for_ready(vdev);  in ivpu_boot()
397 		ivpu_err(vdev, "Failed to boot the firmware: %d\n", ret);  in ivpu_boot()
400 	ivpu_hw_irq_clear(vdev);  in ivpu_boot()
401 	enable_irq(vdev->irq);  in ivpu_boot()
402 	ivpu_hw_irq_enable(vdev);  in ivpu_boot()
403 	ivpu_ipc_enable(vdev);  in ivpu_boot()
405 	if (!ivpu_fw_is_warm_boot(vdev)) {  in ivpu_boot()
406 		ret = ivpu_pm_dct_init(vdev);  in ivpu_boot()
410 		ret = ivpu_hw_sched_init(vdev);  in ivpu_boot()
418 	ivpu_ipc_disable(vdev);  in ivpu_boot()
419 	ivpu_hw_irq_disable(vdev);  in ivpu_boot()
420 	disable_irq(vdev->irq);  in ivpu_boot()
422 	ivpu_hw_diagnose_failure(vdev);  in ivpu_boot()
423 	ivpu_mmu_evtq_dump(vdev);  in ivpu_boot()
424 	ivpu_dev_coredump(vdev);  in ivpu_boot()
428 void ivpu_prepare_for_reset(struct ivpu_device *vdev)  in ivpu_prepare_for_reset()  argument
430 	ivpu_hw_irq_disable(vdev);  in ivpu_prepare_for_reset()
431 	disable_irq(vdev->irq);  in ivpu_prepare_for_reset()
432 	flush_work(&vdev->irq_ipc_work);  in ivpu_prepare_for_reset()
433 	flush_work(&vdev->irq_dct_work);  in ivpu_prepare_for_reset()
434 	flush_work(&vdev->context_abort_work);  in ivpu_prepare_for_reset()
435 	ivpu_ipc_disable(vdev);  in ivpu_prepare_for_reset()
436 	ivpu_mmu_disable(vdev);  in ivpu_prepare_for_reset()
439 int ivpu_shutdown(struct ivpu_device *vdev)  in ivpu_shutdown()  argument
444 	pci_save_state(to_pci_dev(vdev->drm.dev));  in ivpu_shutdown()
446 	ret = ivpu_hw_power_down(vdev);  in ivpu_shutdown()
448 		ivpu_warn(vdev, "Failed to power down HW: %d\n", ret);  in ivpu_shutdown()
450 	pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);  in ivpu_shutdown()
485 static int ivpu_irq_init(struct ivpu_device *vdev)  in ivpu_irq_init()  argument
487 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);  in ivpu_irq_init()
492 		ivpu_err(vdev, "Failed to allocate a MSI IRQ: %d\n", ret);  in ivpu_irq_init()
496 	INIT_WORK(&vdev->irq_ipc_work, ivpu_ipc_irq_work_fn);  in ivpu_irq_init()
497 	INIT_WORK(&vdev->irq_dct_work, ivpu_pm_irq_dct_work_fn);  in ivpu_irq_init()
498 	INIT_WORK(&vdev->context_abort_work, ivpu_context_abort_work_fn);  in ivpu_irq_init()
500 	ivpu_irq_handlers_init(vdev);  in ivpu_irq_init()
502 	vdev->irq = pci_irq_vector(pdev, 0);  in ivpu_irq_init()
504 	ret = devm_request_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler,  in ivpu_irq_init()
505 			       IRQF_NO_AUTOEN, DRIVER_NAME, vdev);  in ivpu_irq_init()
507 		ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);  in ivpu_irq_init()
512 static int ivpu_pci_init(struct ivpu_device *vdev)  in ivpu_pci_init()  argument
514 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);  in ivpu_pci_init()
519 	ivpu_dbg(vdev, MISC, "Mapping BAR0 (RegV) %pR\n", bar0);  in ivpu_pci_init()
520 	vdev->regv = devm_ioremap_resource(vdev->drm.dev, bar0);  in ivpu_pci_init()
521 	if (IS_ERR(vdev->regv)) {  in ivpu_pci_init()
522 		ivpu_err(vdev, "Failed to map bar 0: %pe\n", vdev->regv);  in ivpu_pci_init()
523 		return PTR_ERR(vdev->regv);  in ivpu_pci_init()
526 	ivpu_dbg(vdev, MISC, "Mapping BAR4 (RegB) %pR\n", bar4);  in ivpu_pci_init()
527 	vdev->regb = devm_ioremap_resource(vdev->drm.dev, bar4);  in ivpu_pci_init()
528 	if (IS_ERR(vdev->regb)) {  in ivpu_pci_init()
529 		ivpu_err(vdev, "Failed to map bar 4: %pe\n", vdev->regb);  in ivpu_pci_init()
530 		return PTR_ERR(vdev->regb);  in ivpu_pci_init()
533 	ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(vdev->hw->dma_bits));  in ivpu_pci_init()
535 		ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret);  in ivpu_pci_init()
538 	dma_set_max_seg_size(vdev->drm.dev, UINT_MAX);  in ivpu_pci_init()
548 		ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret);  in ivpu_pci_init()
557 static int ivpu_dev_init(struct ivpu_device *vdev)  in ivpu_dev_init()  argument
561 	vdev->hw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->hw), GFP_KERNEL);  in ivpu_dev_init()
562 	if (!vdev->hw)  in ivpu_dev_init()
565 	vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL);  in ivpu_dev_init()
566 	if (!vdev->mmu)  in ivpu_dev_init()
569 	vdev->fw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->fw), GFP_KERNEL);  in ivpu_dev_init()
570 	if (!vdev->fw)  in ivpu_dev_init()
573 	vdev->ipc = drmm_kzalloc(&vdev->drm, sizeof(*vdev->ipc), GFP_KERNEL);  in ivpu_dev_init()
574 	if (!vdev->ipc)  in ivpu_dev_init()
577 	vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL);  in ivpu_dev_init()
578 	if (!vdev->pm)  in ivpu_dev_init()
581 	if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)  in ivpu_dev_init()
582 		vdev->hw->dma_bits = 48;  in ivpu_dev_init()
584 		vdev->hw->dma_bits = 38;  in ivpu_dev_init()
586 	vdev->platform = IVPU_PLATFORM_INVALID;  in ivpu_dev_init()
587 	vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID;  in ivpu_dev_init()
588 	vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID;  in ivpu_dev_init()
589 	atomic64_set(&vdev->unique_id_counter, 0);  in ivpu_dev_init()
590 	atomic_set(&vdev->job_timeout_counter, 0);  in ivpu_dev_init()
591 	xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);  in ivpu_dev_init()
592 	xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);  in ivpu_dev_init()
593 	xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1);  in ivpu_dev_init()
594 	INIT_LIST_HEAD(&vdev->bo_list);  in ivpu_dev_init()
596 	vdev->db_limit.min = IVPU_MIN_DB;  in ivpu_dev_init()
597 	vdev->db_limit.max = IVPU_MAX_DB;  in ivpu_dev_init()
599 	ret = drmm_mutex_init(&vdev->drm, &vdev->context_list_lock);  in ivpu_dev_init()
603 	ret = drmm_mutex_init(&vdev->drm, &vdev->submitted_jobs_lock);  in ivpu_dev_init()
607 	ret = drmm_mutex_init(&vdev->drm, &vdev->bo_list_lock);  in ivpu_dev_init()
611 	ret = ivpu_pci_init(vdev);  in ivpu_dev_init()
615 	ret = ivpu_irq_init(vdev);  in ivpu_dev_init()
620 	ret = ivpu_hw_init(vdev);  in ivpu_dev_init()
625 	ret = ivpu_hw_power_up(vdev);  in ivpu_dev_init()
629 	ivpu_mmu_global_context_init(vdev);  in ivpu_dev_init()
631 	ret = ivpu_mmu_init(vdev);  in ivpu_dev_init()
635 	ret = ivpu_mmu_reserved_context_init(vdev);  in ivpu_dev_init()
639 	ret = ivpu_fw_init(vdev);  in ivpu_dev_init()
643 	ret = ivpu_ipc_init(vdev);  in ivpu_dev_init()
647 	ivpu_pm_init(vdev);  in ivpu_dev_init()
649 	ret = ivpu_boot(vdev);  in ivpu_dev_init()
653 	ivpu_job_done_consumer_init(vdev);  in ivpu_dev_init()
654 	ivpu_pm_enable(vdev);  in ivpu_dev_init()
659 	ivpu_ipc_fini(vdev);  in ivpu_dev_init()
661 	ivpu_fw_fini(vdev);  in ivpu_dev_init()
663 	ivpu_mmu_reserved_context_fini(vdev);  in ivpu_dev_init()
665 	ivpu_mmu_global_context_fini(vdev);  in ivpu_dev_init()
667 	ivpu_shutdown(vdev);  in ivpu_dev_init()
669 	xa_destroy(&vdev->db_xa);  in ivpu_dev_init()
670 	xa_destroy(&vdev->submitted_jobs_xa);  in ivpu_dev_init()
671 	xa_destroy(&vdev->context_xa);  in ivpu_dev_init()
675 static void ivpu_bo_unbind_all_user_contexts(struct ivpu_device *vdev)  in ivpu_bo_unbind_all_user_contexts()  argument
680 	mutex_lock(&vdev->context_list_lock);  in ivpu_bo_unbind_all_user_contexts()
682 	xa_for_each(&vdev->context_xa, ctx_id, file_priv)  in ivpu_bo_unbind_all_user_contexts()
683 		file_priv_unbind(vdev, file_priv);  in ivpu_bo_unbind_all_user_contexts()
685 	mutex_unlock(&vdev->context_list_lock);  in ivpu_bo_unbind_all_user_contexts()
688 static void ivpu_dev_fini(struct ivpu_device *vdev)  in ivpu_dev_fini()  argument
690 	ivpu_jobs_abort_all(vdev);  in ivpu_dev_fini()
691 	ivpu_pm_disable_recovery(vdev);  in ivpu_dev_fini()
692 	ivpu_pm_disable(vdev);  in ivpu_dev_fini()
693 	ivpu_prepare_for_reset(vdev);  in ivpu_dev_fini()
694 	ivpu_shutdown(vdev);  in ivpu_dev_fini()
696 	ivpu_ms_cleanup_all(vdev);  in ivpu_dev_fini()
697 	ivpu_job_done_consumer_fini(vdev);  in ivpu_dev_fini()
698 	ivpu_bo_unbind_all_user_contexts(vdev);  in ivpu_dev_fini()
700 	ivpu_ipc_fini(vdev);  in ivpu_dev_fini()
701 	ivpu_fw_fini(vdev);  in ivpu_dev_fini()
702 	ivpu_mmu_reserved_context_fini(vdev);  in ivpu_dev_fini()
703 	ivpu_mmu_global_context_fini(vdev);  in ivpu_dev_fini()
705 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->db_xa));  in ivpu_dev_fini()
706 	xa_destroy(&vdev->db_xa);  in ivpu_dev_fini()
707 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));  in ivpu_dev_fini()
708 	xa_destroy(&vdev->submitted_jobs_xa);  in ivpu_dev_fini()
709 	drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa));  in ivpu_dev_fini()
710 	xa_destroy(&vdev->context_xa);  in ivpu_dev_fini()
726 	struct ivpu_device *vdev;  in ivpu_probe()  local
729 	vdev = devm_drm_dev_alloc(&pdev->dev, &driver, struct ivpu_device, drm);  in ivpu_probe()
730 	if (IS_ERR(vdev))  in ivpu_probe()
731 		return PTR_ERR(vdev);  in ivpu_probe()
733 	pci_set_drvdata(pdev, vdev);  in ivpu_probe()
735 	ret = ivpu_dev_init(vdev);  in ivpu_probe()
739 	ivpu_debugfs_init(vdev);  in ivpu_probe()
740 	ivpu_sysfs_init(vdev);  in ivpu_probe()
742 	ret = drm_dev_register(&vdev->drm, 0);  in ivpu_probe()
745 		ivpu_dev_fini(vdev);  in ivpu_probe()
753 	struct ivpu_device *vdev = pci_get_drvdata(pdev);  in ivpu_remove()  local
755 	drm_dev_unplug(&vdev->drm);  in ivpu_remove()
756 	ivpu_dev_fini(vdev);  in ivpu_remove()