1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (C) 2013--2024 Intel Corporation
8 #include <linux/dma-mapping.h>
16 #include <linux/pci-ats.h>
23 #include <media/ipu-bridge.h>
24 #include <media/ipu6-pci-table.h>
27 #include "ipu6-bus.h"
28 #include "ipu6-buttress.h"
29 #include "ipu6-cpd.h"
30 #include "ipu6-isys.h"
31 #include "ipu6-mmu.h"
32 #include "ipu6-platform-buttress-regs.h"
33 #include "ipu6-platform-isys-csi2-reg.h"
34 #include "ipu6-platform-regs.h"
36 #define IPU6_PCI_BAR		0
99 				.info_bits = 0,
118 				.info_bits = 0,
119 				.nr_l1streams = 0,
120 				.nr_l2streams = 0,
159 				.info_bits = 0,
182 				.info_bits = 0,
201 				.info_bits = 0,
202 				.nr_l1streams = 0,
203 				.nr_l2streams = 0,
232 ipu6_pkg_dir_configure_spc(struct ipu6_device *isp,  in ipu6_pkg_dir_configure_spc()  argument
245 		dma_addr = sg_dma_address(isp->isys->fw_sgt.sgl);  in ipu6_pkg_dir_configure_spc()
247 		dma_addr = sg_dma_address(isp->psys->fw_sgt.sgl);  in ipu6_pkg_dir_configure_spc()
249 	pg_offset = server_fw_addr - dma_addr;  in ipu6_pkg_dir_configure_spc()
250 	prog = (struct ipu6_cell_program *)((uintptr_t)isp->cpd_fw->data +  in ipu6_pkg_dir_configure_spc()
252 	spc_base = base + prog->regs_addr;  in ipu6_pkg_dir_configure_spc()
253 	if (spc_base != (base + hw_variant->spc_offset))  in ipu6_pkg_dir_configure_spc()
254 		dev_warn(&isp->pdev->dev,  in ipu6_pkg_dir_configure_spc()
256 			 base + hw_variant->spc_offset, spc_base);  in ipu6_pkg_dir_configure_spc()
257 	writel(server_fw_addr + prog->blob_offset +  in ipu6_pkg_dir_configure_spc()
258 	       prog->icache_source, spc_base + IPU6_PSYS_REG_SPC_ICACHE_BASE);  in ipu6_pkg_dir_configure_spc()
261 	writel(prog->start[1], spc_base + IPU6_PSYS_REG_SPC_START_PC);  in ipu6_pkg_dir_configure_spc()
262 	writel(pkg_dir_vied_address, base + hw_variant->dmem_offset);  in ipu6_pkg_dir_configure_spc()
265 void ipu6_configure_spc(struct ipu6_device *isp,  in ipu6_configure_spc()  argument
270 	void __iomem *dmem_base = base + hw_variant->dmem_offset;  in ipu6_configure_spc()
271 	void __iomem *spc_regs_base = base + hw_variant->spc_offset;  in ipu6_configure_spc()
278 	if (isp->secure_mode)  in ipu6_configure_spc()
281 		ipu6_pkg_dir_configure_spc(isp, hw_variant, pkg_dir_idx, base,  in ipu6_configure_spc()
291 static void ipu6_internal_pdata_init(struct ipu6_device *isp)  in ipu6_internal_pdata_init()  argument
293 	u8 hw_ver = isp->hw_ver;  in ipu6_internal_pdata_init()
373 	struct device *dev = &pdev->dev;  in ipu6_isys_init()
386 		return ERR_PTR(-ENOMEM);  in ipu6_isys_init()
388 	pdata->base = base;  in ipu6_isys_init()
389 	pdata->ipdata = ipdata;  in ipu6_isys_init()
399 	isys_adev->mmu = ipu6_mmu_init(dev, base, ISYS_MMID,  in ipu6_isys_init()
400 				       &ipdata->hw_variant);  in ipu6_isys_init()
401 	if (IS_ERR(isys_adev->mmu)) {  in ipu6_isys_init()
402 		put_device(&isys_adev->auxdev.dev);  in ipu6_isys_init()
404 		return dev_err_cast_probe(dev, isys_adev->mmu,  in ipu6_isys_init()
405 				"ipu6_mmu_init(isys_adev->mmu) failed\n");  in ipu6_isys_init()
408 	isys_adev->mmu->dev = &isys_adev->auxdev.dev;  in ipu6_isys_init()
430 		return ERR_PTR(-ENOMEM);  in ipu6_psys_init()
432 	pdata->base = base;  in ipu6_psys_init()
433 	pdata->ipdata = ipdata;  in ipu6_psys_init()
439 		return dev_err_cast_probe(&pdev->dev, psys_adev,  in ipu6_psys_init()
443 	psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, PSYS_MMID,  in ipu6_psys_init()
444 				       &ipdata->hw_variant);  in ipu6_psys_init()
445 	if (IS_ERR(psys_adev->mmu)) {  in ipu6_psys_init()
446 		put_device(&psys_adev->auxdev.dev);  in ipu6_psys_init()
448 		return dev_err_cast_probe(&pdev->dev, psys_adev->mmu,  in ipu6_psys_init()
449 				"ipu6_mmu_init(psys_adev->mmu) failed\n");  in ipu6_psys_init()
452 	psys_adev->mmu->dev = &psys_adev->auxdev.dev;  in ipu6_psys_init()
468 	if (is_ipu6ep_mtl(hw_ver) && boot_cpu_data.x86_stepping == 0x2 &&  in ipu6_pci_config_setup()
476 		return 0;  in ipu6_pci_config_setup()
480 	if (ret < 0)  in ipu6_pci_config_setup()
481 		return dev_err_probe(&dev->dev, ret, "Request msi failed");  in ipu6_pci_config_setup()
483 	return 0;  in ipu6_pci_config_setup()
486 static void ipu6_configure_vc_mechanism(struct ipu6_device *isp)  in ipu6_configure_vc_mechanism()  argument
488 	u32 val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL);  in ipu6_configure_vc_mechanism()
500 	writel(val, isp->base + BUTTRESS_REG_BTRS_CTRL);  in ipu6_configure_vc_mechanism()
506 	struct device *dev = &pdev->dev;  in ipu6_pci_probe()
509 	struct ipu6_device *isp;  in ipu6_pci_probe()  local
514 	isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL);  in ipu6_pci_probe()
515 	if (!isp)  in ipu6_pci_probe()
516 		return -ENOMEM;  in ipu6_pci_probe()
518 	isp->pdev = pdev;  in ipu6_pci_probe()
519 	INIT_LIST_HEAD(&isp->devices);  in ipu6_pci_probe()
532 	isp->base = pcim_iomap_table(pdev)[IPU6_PCI_BAR];  in ipu6_pci_probe()
533 	pci_set_drvdata(pdev, isp);  in ipu6_pci_probe()
536 	isp->cpd_metadata_cmpnt_size = sizeof(struct ipu6_cpd_metadata_cmpnt);  in ipu6_pci_probe()
537 	switch (id->device) {  in ipu6_pci_probe()
539 		isp->hw_ver = IPU6_VER_6;  in ipu6_pci_probe()
540 		isp->cpd_fw_name = IPU6_FIRMWARE_NAME;  in ipu6_pci_probe()
543 		isp->hw_ver = IPU6_VER_6SE;  in ipu6_pci_probe()
544 		isp->cpd_fw_name = IPU6SE_FIRMWARE_NAME;  in ipu6_pci_probe()
545 		isp->cpd_metadata_cmpnt_size =  in ipu6_pci_probe()
550 		isp->hw_ver = IPU6_VER_6EP;  in ipu6_pci_probe()
551 		isp->cpd_fw_name = IPU6EP_FIRMWARE_NAME;  in ipu6_pci_probe()
554 		isp->hw_ver = IPU6_VER_6EP;  in ipu6_pci_probe()
555 		isp->cpd_fw_name = IPU6EPADLN_FIRMWARE_NAME;  in ipu6_pci_probe()
558 		isp->hw_ver = IPU6_VER_6EP_MTL;  in ipu6_pci_probe()
559 		isp->cpd_fw_name = IPU6EPMTL_FIRMWARE_NAME;  in ipu6_pci_probe()
562 		return dev_err_probe(dev, -ENODEV,  in ipu6_pci_probe()
564 				     id->device);  in ipu6_pci_probe()
567 	ipu6_internal_pdata_init(isp);  in ipu6_pci_probe()
569 	isys_base = isp->base + isys_ipdata.hw_variant.offset;  in ipu6_pci_probe()
570 	psys_base = isp->base + psys_ipdata.hw_variant.offset;  in ipu6_pci_probe()
578 	ret = ipu6_pci_config_setup(pdev, isp->hw_ver);  in ipu6_pci_probe()
582 	ret = ipu6_buttress_init(isp);  in ipu6_pci_probe()
586 	ret = request_firmware(&isp->cpd_fw, isp->cpd_fw_name, dev);  in ipu6_pci_probe()
588 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
590 			      isp->cpd_fw_name);  in ipu6_pci_probe()
594 	ret = ipu6_cpd_validate_cpd_file(isp, isp->cpd_fw->data,  in ipu6_pci_probe()
595 					 isp->cpd_fw->size);  in ipu6_pci_probe()
597 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
605 		ret = -ENOMEM;  in ipu6_pci_probe()
609 	isp->isys = ipu6_isys_init(pdev, dev, isys_ctrl, isys_base,  in ipu6_pci_probe()
611 	if (IS_ERR(isp->isys)) {  in ipu6_pci_probe()
612 		ret = PTR_ERR(isp->isys);  in ipu6_pci_probe()
619 		ret = -ENOMEM;  in ipu6_pci_probe()
623 	isp->psys = ipu6_psys_init(pdev, &isp->isys->auxdev.dev, psys_ctrl,  in ipu6_pci_probe()
625 	if (IS_ERR(isp->psys)) {  in ipu6_pci_probe()
626 		ret = PTR_ERR(isp->psys);  in ipu6_pci_probe()
630 	ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);  in ipu6_pci_probe()
631 	if (ret < 0)  in ipu6_pci_probe()
634 	ret = ipu6_mmu_hw_init(isp->psys->mmu);  in ipu6_pci_probe()
636 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
641 	ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw,  in ipu6_pci_probe()
642 					 &isp->psys->fw_sgt);  in ipu6_pci_probe()
644 		dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n");  in ipu6_pci_probe()
648 	ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data);  in ipu6_pci_probe()
650 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
655 	ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr,  in ipu6_pci_probe()
657 					IRQF_SHARED, IPU6_NAME, isp);  in ipu6_pci_probe()
663 	ret = ipu6_buttress_authenticate(isp);  in ipu6_pci_probe()
665 		dev_err_probe(&isp->pdev->dev, ret,  in ipu6_pci_probe()
670 	ipu6_mmu_hw_cleanup(isp->psys->mmu);  in ipu6_pci_probe()
671 	pm_runtime_put(&isp->psys->auxdev.dev);  in ipu6_pci_probe()
674 	ipu6_configure_vc_mechanism(isp);  in ipu6_pci_probe()
676 	val = readl(isp->base + BUTTRESS_REG_SKU);  in ipu6_pci_probe()
678 	version = FIELD_GET(GENMASK(3, 0), val);  in ipu6_pci_probe()
679 	dev_info(dev, "IPU%u-v%u[%x] hardware version %d\n", version, sku_id,  in ipu6_pci_probe()
680 		 pdev->device, isp->hw_ver);  in ipu6_pci_probe()
685 	isp->bus_ready_to_probe = true;  in ipu6_pci_probe()
687 	return 0;  in ipu6_pci_probe()
690 	devm_free_irq(dev, pdev->irq, isp);  in ipu6_pci_probe()
692 	if (isp->psys) {  in ipu6_pci_probe()
693 		ipu6_cpd_free_pkg_dir(isp->psys);  in ipu6_pci_probe()
694 		ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);  in ipu6_pci_probe()
696 	if (!IS_ERR_OR_NULL(isp->psys) && !IS_ERR_OR_NULL(isp->psys->mmu))  in ipu6_pci_probe()
697 		ipu6_mmu_cleanup(isp->psys->mmu);  in ipu6_pci_probe()
698 	if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu))  in ipu6_pci_probe()
699 		ipu6_mmu_cleanup(isp->isys->mmu);  in ipu6_pci_probe()
701 	release_firmware(isp->cpd_fw);  in ipu6_pci_probe()
703 	ipu6_buttress_exit(isp);  in ipu6_pci_probe()
710 	struct ipu6_device *isp = pci_get_drvdata(pdev);  in ipu6_pci_remove()  local
711 	struct ipu6_mmu *isys_mmu = isp->isys->mmu;  in ipu6_pci_remove()
712 	struct ipu6_mmu *psys_mmu = isp->psys->mmu;  in ipu6_pci_remove()
714 	devm_free_irq(&pdev->dev, pdev->irq, isp);  in ipu6_pci_remove()
715 	ipu6_cpd_free_pkg_dir(isp->psys);  in ipu6_pci_remove()
717 	ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);  in ipu6_pci_remove()
718 	ipu6_buttress_exit(isp);  in ipu6_pci_remove()
722 	pm_runtime_forbid(&pdev->dev);  in ipu6_pci_remove()
723 	pm_runtime_get_noresume(&pdev->dev);  in ipu6_pci_remove()
725 	release_firmware(isp->cpd_fw);  in ipu6_pci_remove()
733 	struct ipu6_device *isp = pci_get_drvdata(pdev);  in ipu6_pci_reset_prepare()  local
735 	pm_runtime_forbid(&isp->pdev->dev);  in ipu6_pci_reset_prepare()
740 	struct ipu6_device *isp = pci_get_drvdata(pdev);  in ipu6_pci_reset_done()  local
742 	ipu6_buttress_restore(isp);  in ipu6_pci_reset_done()
743 	if (isp->secure_mode)  in ipu6_pci_reset_done()
744 		ipu6_buttress_reset_authentication(isp);  in ipu6_pci_reset_done()
746 	isp->need_ipc_reset = true;  in ipu6_pci_reset_done()
747 	pm_runtime_allow(&isp->pdev->dev);  in ipu6_pci_reset_done()
752  * PCI device level PM state transitions (D0<->D3)
758 	synchronize_irq(pdev->irq);  in ipu6_suspend()
759 	return 0;  in ipu6_suspend()
765 	struct ipu6_device *isp = pci_get_drvdata(pdev);  in ipu6_resume()  local
766 	struct ipu6_buttress *b = &isp->buttress;  in ipu6_resume()
770 	ipu6_configure_vc_mechanism(isp);  in ipu6_resume()
772 	isp->secure_mode = ipu6_buttress_get_secure_mode(isp);  in ipu6_resume()
774 		 isp->secure_mode ? "secure" : "non-secure");  in ipu6_resume()
776 	ipu6_buttress_restore(isp);  in ipu6_resume()
778 	ret = ipu6_buttress_ipc_reset(isp, &b->cse);  in ipu6_resume()
780 		dev_err(&isp->pdev->dev, "IPC reset protocol failed!\n");  in ipu6_resume()
782 	ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);  in ipu6_resume()
783 	if (ret < 0) {  in ipu6_resume()
784 		dev_err(&isp->psys->auxdev.dev, "Failed to get runtime PM\n");  in ipu6_resume()
785 		return 0;  in ipu6_resume()
788 	ret = ipu6_buttress_authenticate(isp);  in ipu6_resume()
790 		dev_err(&isp->pdev->dev, "FW authentication failed(%d)\n", ret);  in ipu6_resume()
792 	pm_runtime_put(&isp->psys->auxdev.dev);  in ipu6_resume()
794 	return 0;  in ipu6_resume()
800 	struct ipu6_device *isp = pci_get_drvdata(pdev);  in ipu6_runtime_resume()  local
803 	ipu6_configure_vc_mechanism(isp);  in ipu6_runtime_resume()
804 	ipu6_buttress_restore(isp);  in ipu6_runtime_resume()
806 	if (isp->need_ipc_reset) {  in ipu6_runtime_resume()
807 		struct ipu6_buttress *b = &isp->buttress;  in ipu6_runtime_resume()
809 		isp->need_ipc_reset = false;  in ipu6_runtime_resume()
810 		ret = ipu6_buttress_ipc_reset(isp, &b->cse);  in ipu6_runtime_resume()
812 			dev_err(&isp->pdev->dev, "IPC reset protocol failed\n");  in ipu6_runtime_resume()
815 	return 0;  in ipu6_runtime_resume()