1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
13 #include <linux/io-pgtable.h>
18 #include <linux/iommu.h>
25 #include "msm_iommu_hw-8xxx.h"
54 static int __enable_clocks(struct msm_iommu_dev *iommu)  in __enable_clocks()  argument
58 	ret = clk_enable(iommu->pclk);  in __enable_clocks()
62 	if (iommu->clk) {  in __enable_clocks()
63 		ret = clk_enable(iommu->clk);  in __enable_clocks()
65 			clk_disable(iommu->pclk);  in __enable_clocks()
71 static void __disable_clocks(struct msm_iommu_dev *iommu)  in __disable_clocks()  argument
73 	if (iommu->clk)  in __disable_clocks()
74 		clk_disable(iommu->clk);  in __disable_clocks()
75 	clk_disable(iommu->pclk);  in __disable_clocks()
80 	int ctx;  in msm_iommu_reset()  local
94 	for (ctx = 0; ctx < ncb; ctx++) {  in msm_iommu_reset()
95 		SET_BPRCOSH(base, ctx, 0);  in msm_iommu_reset()
96 		SET_BPRCISH(base, ctx, 0);  in msm_iommu_reset()
97 		SET_BPRCNSH(base, ctx, 0);  in msm_iommu_reset()
98 		SET_BPSHCFG(base, ctx, 0);  in msm_iommu_reset()
99 		SET_BPMTCFG(base, ctx, 0);  in msm_iommu_reset()
100 		SET_ACTLR(base, ctx, 0);  in msm_iommu_reset()
101 		SET_SCTLR(base, ctx, 0);  in msm_iommu_reset()
102 		SET_FSRRESTORE(base, ctx, 0);  in msm_iommu_reset()
103 		SET_TTBR0(base, ctx, 0);  in msm_iommu_reset()
104 		SET_TTBR1(base, ctx, 0);  in msm_iommu_reset()
105 		SET_TTBCR(base, ctx, 0);  in msm_iommu_reset()
106 		SET_BFBCR(base, ctx, 0);  in msm_iommu_reset()
107 		SET_PAR(base, ctx, 0);  in msm_iommu_reset()
108 		SET_FAR(base, ctx, 0);  in msm_iommu_reset()
109 		SET_CTX_TLBIALL(base, ctx, 0);  in msm_iommu_reset()
110 		SET_TLBFLPTER(base, ctx, 0);  in msm_iommu_reset()
111 		SET_TLBSLPTER(base, ctx, 0);  in msm_iommu_reset()
112 		SET_TLBLKCR(base, ctx, 0);  in msm_iommu_reset()
113 		SET_CONTEXTIDR(base, ctx, 0);  in msm_iommu_reset()
120 	struct msm_iommu_dev *iommu = NULL;  in __flush_iotlb()  local
124 	list_for_each_entry(iommu, &priv->list_attached, dom_node) {  in __flush_iotlb()
125 		ret = __enable_clocks(iommu);  in __flush_iotlb()
129 		list_for_each_entry(master, &iommu->ctx_list, list)  in __flush_iotlb()
130 			SET_CTX_TLBIALL(iommu->base, master->num, 0);  in __flush_iotlb()
132 		__disable_clocks(iommu);  in __flush_iotlb()
142 	struct msm_iommu_dev *iommu = NULL;  in __flush_iotlb_range()  local
147 	list_for_each_entry(iommu, &priv->list_attached, dom_node) {  in __flush_iotlb_range()
148 		ret = __enable_clocks(iommu);  in __flush_iotlb_range()
152 		list_for_each_entry(master, &iommu->ctx_list, list) {  in __flush_iotlb_range()
156 				iova |= GET_CONTEXTIDR_ASID(iommu->base,  in __flush_iotlb_range()
157 							    master->num);  in __flush_iotlb_range()
158 				SET_TLBIVA(iommu->base, master->num, iova);  in __flush_iotlb_range()
160 			} while (temp_size -= granule);  in __flush_iotlb_range()
163 		__disable_clocks(iommu);  in __flush_iotlb_range()
195 			return -ENOSPC;  in msm_iommu_alloc_ctx()
206 static void config_mids(struct msm_iommu_dev *iommu,  in config_mids()  argument
209 	int mid, ctx, i;  in config_mids()  local
211 	for (i = 0; i < master->num_mids; i++) {  in config_mids()
212 		mid = master->mids[i];  in config_mids()
213 		ctx = master->num;  in config_mids()
215 		SET_M2VCBR_N(iommu->base, mid, 0);  in config_mids()
216 		SET_CBACR_N(iommu->base, ctx, 0);  in config_mids()
219 		SET_VMID(iommu->base, mid, 0);  in config_mids()
222 		SET_CBNDX(iommu->base, mid, ctx);  in config_mids()
225 		SET_CBVMID(iommu->base, ctx, 0);  in config_mids()
228 		SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx);  in config_mids()
230 		/* Set security bit override to be Non-secure */  in config_mids()
231 		SET_NSCFG(iommu->base, mid, 3);  in config_mids()
235 static void __reset_context(void __iomem *base, int ctx)  in __reset_context()  argument
237 	SET_BPRCOSH(base, ctx, 0);  in __reset_context()
238 	SET_BPRCISH(base, ctx, 0);  in __reset_context()
239 	SET_BPRCNSH(base, ctx, 0);  in __reset_context()
240 	SET_BPSHCFG(base, ctx, 0);  in __reset_context()
241 	SET_BPMTCFG(base, ctx, 0);  in __reset_context()
242 	SET_ACTLR(base, ctx, 0);  in __reset_context()
243 	SET_SCTLR(base, ctx, 0);  in __reset_context()
244 	SET_FSRRESTORE(base, ctx, 0);  in __reset_context()
245 	SET_TTBR0(base, ctx, 0);  in __reset_context()
246 	SET_TTBR1(base, ctx, 0);  in __reset_context()
247 	SET_TTBCR(base, ctx, 0);  in __reset_context()
248 	SET_BFBCR(base, ctx, 0);  in __reset_context()
249 	SET_PAR(base, ctx, 0);  in __reset_context()
250 	SET_FAR(base, ctx, 0);  in __reset_context()
251 	SET_CTX_TLBIALL(base, ctx, 0);  in __reset_context()
252 	SET_TLBFLPTER(base, ctx, 0);  in __reset_context()
253 	SET_TLBSLPTER(base, ctx, 0);  in __reset_context()
254 	SET_TLBLKCR(base, ctx, 0);  in __reset_context()
257 static void __program_context(void __iomem *base, int ctx,  in __program_context()  argument
260 	__reset_context(base, ctx);  in __program_context()
263 	SET_TRE(base, ctx, 1);  in __program_context()
264 	SET_AFE(base, ctx, 1);  in __program_context()
268 	SET_TLBMCFG(base, ctx, 0x3);  in __program_context()
271 	SET_V2PCFG(base, ctx, 0x3);  in __program_context()
273 	SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr);  in __program_context()
274 	SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr);  in __program_context()
275 	SET_TTBR1(base, ctx, 0);  in __program_context()
278 	SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr);  in __program_context()
279 	SET_NMRR(base, ctx, priv->cfg.arm_v7s_cfg.nmrr);  in __program_context()
282 	SET_CTX_TLBIALL(base, ctx, 0);  in __program_context()
285 	SET_IRPTNDX(base, ctx, 0);  in __program_context()
288 	SET_CFEIE(base, ctx, 1);  in __program_context()
291 	SET_CFCFG(base, ctx, 1);  in __program_context()
294 	SET_RCISH(base, ctx, 1);  in __program_context()
295 	SET_RCOSH(base, ctx, 1);  in __program_context()
296 	SET_RCNSH(base, ctx, 1);  in __program_context()
299 	SET_BFBDFE(base, ctx, 1);  in __program_context()
302 	SET_M(base, ctx, 1);  in __program_context()
313 	INIT_LIST_HEAD(&priv->list_attached);  in msm_iommu_domain_alloc_paging()
315 	priv->domain.geometry.aperture_start = 0;  in msm_iommu_domain_alloc_paging()
316 	priv->domain.geometry.aperture_end   = (1ULL << 32) - 1;  in msm_iommu_domain_alloc_paging()
317 	priv->domain.geometry.force_aperture = true;  in msm_iommu_domain_alloc_paging()
319 	return &priv->domain;  in msm_iommu_domain_alloc_paging()
339 	spin_lock_init(&priv->pgtlock);  in msm_iommu_domain_config()
341 	priv->cfg = (struct io_pgtable_cfg) {  in msm_iommu_domain_config()
346 		.iommu_dev = priv->dev,  in msm_iommu_domain_config()
349 	priv->iop = alloc_io_pgtable_ops(ARM_V7S, &priv->cfg, priv);  in msm_iommu_domain_config()
350 	if (!priv->iop) {  in msm_iommu_domain_config()
351 		dev_err(priv->dev, "Failed to allocate pgtable\n");  in msm_iommu_domain_config()
352 		return -EINVAL;  in msm_iommu_domain_config()
355 	msm_iommu_ops.pgsize_bitmap = priv->cfg.pgsize_bitmap;  in msm_iommu_domain_config()
363 	struct msm_iommu_dev *iommu, *ret = NULL;  in find_iommu_for_dev()  local
366 	list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {  in find_iommu_for_dev()
367 		master = list_first_entry(&iommu->ctx_list,  in find_iommu_for_dev()
370 		if (master->of_node == dev->of_node) {  in find_iommu_for_dev()
371 			ret = iommu;  in find_iommu_for_dev()
381 	struct msm_iommu_dev *iommu;  in msm_iommu_probe_device()  local
385 	iommu = find_iommu_for_dev(dev);  in msm_iommu_probe_device()
388 	if (!iommu)  in msm_iommu_probe_device()
389 		return ERR_PTR(-ENODEV);  in msm_iommu_probe_device()
391 	return &iommu->iommu;  in msm_iommu_probe_device()
398 	struct msm_iommu_dev *iommu;  in msm_iommu_attach_dev()  local
402 	priv->dev = dev;  in msm_iommu_attach_dev()
406 	list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {  in msm_iommu_attach_dev()
407 		master = list_first_entry(&iommu->ctx_list,  in msm_iommu_attach_dev()
410 		if (master->of_node == dev->of_node) {  in msm_iommu_attach_dev()
411 			ret = __enable_clocks(iommu);  in msm_iommu_attach_dev()
415 			list_for_each_entry(master, &iommu->ctx_list, list) {  in msm_iommu_attach_dev()
416 				if (master->num) {  in msm_iommu_attach_dev()
418 					ret = -EEXIST;  in msm_iommu_attach_dev()
421 				master->num =  in msm_iommu_attach_dev()
422 					msm_iommu_alloc_ctx(iommu->context_map,  in msm_iommu_attach_dev()
423 							    0, iommu->ncb);  in msm_iommu_attach_dev()
424 				if (IS_ERR_VALUE(master->num)) {  in msm_iommu_attach_dev()
425 					ret = -ENODEV;  in msm_iommu_attach_dev()
428 				config_mids(iommu, master);  in msm_iommu_attach_dev()
429 				__program_context(iommu->base, master->num,  in msm_iommu_attach_dev()
432 			__disable_clocks(iommu);  in msm_iommu_attach_dev()
433 			list_add(&iommu->dom_node, &priv->list_attached);  in msm_iommu_attach_dev()
449 	struct msm_iommu_dev *iommu;  in msm_iommu_identity_attach()  local
457 	free_io_pgtable_ops(priv->iop);  in msm_iommu_identity_attach()
460 	list_for_each_entry(iommu, &priv->list_attached, dom_node) {  in msm_iommu_identity_attach()
461 		ret = __enable_clocks(iommu);  in msm_iommu_identity_attach()
465 		list_for_each_entry(master, &iommu->ctx_list, list) {  in msm_iommu_identity_attach()
466 			msm_iommu_free_ctx(iommu->context_map, master->num);  in msm_iommu_identity_attach()
467 			__reset_context(iommu->base, master->num);  in msm_iommu_identity_attach()
469 		__disable_clocks(iommu);  in msm_iommu_identity_attach()
493 	spin_lock_irqsave(&priv->pgtlock, flags);  in msm_iommu_map()
494 	ret = priv->iop->map_pages(priv->iop, iova, pa, pgsize, pgcount, prot,  in msm_iommu_map()
496 	spin_unlock_irqrestore(&priv->pgtlock, flags);  in msm_iommu_map()
518 	spin_lock_irqsave(&priv->pgtlock, flags);  in msm_iommu_unmap()
519 	ret = priv->iop->unmap_pages(priv->iop, iova, pgsize, pgcount, gather);  in msm_iommu_unmap()
520 	spin_unlock_irqrestore(&priv->pgtlock, flags);  in msm_iommu_unmap()
529 	struct msm_iommu_dev *iommu;  in msm_iommu_iova_to_phys()  local
538 	iommu = list_first_entry(&priv->list_attached,  in msm_iommu_iova_to_phys()
541 	if (list_empty(&iommu->ctx_list))  in msm_iommu_iova_to_phys()
544 	master = list_first_entry(&iommu->ctx_list,  in msm_iommu_iova_to_phys()
549 	ret = __enable_clocks(iommu);  in msm_iommu_iova_to_phys()
554 	SET_CTX_TLBIALL(iommu->base, master->num, 0);  in msm_iommu_iova_to_phys()
555 	SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA);  in msm_iommu_iova_to_phys()
557 	par = GET_PAR(iommu->base, master->num);  in msm_iommu_iova_to_phys()
560 	if (GET_NOFAULT_SS(iommu->base, master->num))  in msm_iommu_iova_to_phys()
565 	if (GET_FAULT(iommu->base, master->num))  in msm_iommu_iova_to_phys()
568 	__disable_clocks(iommu);  in msm_iommu_iova_to_phys()
574 static void print_ctx_regs(void __iomem *base, int ctx)  in print_ctx_regs()  argument
576 	unsigned int fsr = GET_FSR(base, ctx);  in print_ctx_regs()
578 	       GET_FAR(base, ctx), GET_PAR(base, ctx));  in print_ctx_regs()
592 	       GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx));  in print_ctx_regs()
594 	       GET_TTBR0(base, ctx), GET_TTBR1(base, ctx));  in print_ctx_regs()
596 	       GET_SCTLR(base, ctx), GET_ACTLR(base, ctx));  in print_ctx_regs()
600 				struct msm_iommu_dev **iommu,  in insert_iommu_master()  argument
606 	if (list_empty(&(*iommu)->ctx_list)) {  in insert_iommu_master()
610 			return -ENOMEM;  in insert_iommu_master()
612 		master->of_node = dev->of_node;  in insert_iommu_master()
613 		list_add(&master->list, &(*iommu)->ctx_list);  in insert_iommu_master()
617 	for (sid = 0; sid < master->num_mids; sid++)  in insert_iommu_master()
618 		if (master->mids[sid] == spec->args[0]) {  in insert_iommu_master()
624 	master->mids[master->num_mids++] = spec->args[0];  in insert_iommu_master()
631 	struct msm_iommu_dev *iommu = NULL, *iter;  in qcom_iommu_of_xlate()  local
637 		if (iter->dev->of_node == spec->np) {  in qcom_iommu_of_xlate()
638 			iommu = iter;  in qcom_iommu_of_xlate()
643 	if (!iommu) {  in qcom_iommu_of_xlate()
644 		ret = -ENODEV;  in qcom_iommu_of_xlate()
648 	ret = insert_iommu_master(dev, &iommu, spec);  in qcom_iommu_of_xlate()
657 	struct msm_iommu_dev *iommu = dev_id;  in msm_iommu_fault_handler()  local
663 	if (!iommu) {  in msm_iommu_fault_handler()
668 	pr_err("Unexpected IOMMU page fault!\n");  in msm_iommu_fault_handler()
669 	pr_err("base = %08x\n", (unsigned int)iommu->base);  in msm_iommu_fault_handler()
671 	ret = __enable_clocks(iommu);  in msm_iommu_fault_handler()
675 	for (i = 0; i < iommu->ncb; i++) {  in msm_iommu_fault_handler()
676 		fsr = GET_FSR(iommu->base, i);  in msm_iommu_fault_handler()
680 			print_ctx_regs(iommu->base, i);  in msm_iommu_fault_handler()
681 			SET_FSR(iommu->base, i, 0x4000000F);  in msm_iommu_fault_handler()
684 	__disable_clocks(iommu);  in msm_iommu_fault_handler()
704 		 * taken care when the iommu client does a writel before
718 	struct msm_iommu_dev *iommu;  in msm_iommu_probe()  local
721 	iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);  in msm_iommu_probe()
722 	if (!iommu)  in msm_iommu_probe()
723 		return -ENODEV;  in msm_iommu_probe()
725 	iommu->dev = &pdev->dev;  in msm_iommu_probe()
726 	INIT_LIST_HEAD(&iommu->ctx_list);  in msm_iommu_probe()
728 	iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk");  in msm_iommu_probe()
729 	if (IS_ERR(iommu->pclk))  in msm_iommu_probe()
730 		return dev_err_probe(iommu->dev, PTR_ERR(iommu->pclk),  in msm_iommu_probe()
733 	ret = clk_prepare(iommu->pclk);  in msm_iommu_probe()
735 		return dev_err_probe(iommu->dev, ret,  in msm_iommu_probe()
738 	iommu->clk = devm_clk_get(iommu->dev, "iommu_clk");  in msm_iommu_probe()
739 	if (IS_ERR(iommu->clk)) {  in msm_iommu_probe()
740 		clk_unprepare(iommu->pclk);  in msm_iommu_probe()
741 		return dev_err_probe(iommu->dev, PTR_ERR(iommu->clk),  in msm_iommu_probe()
745 	ret = clk_prepare(iommu->clk);  in msm_iommu_probe()
747 		clk_unprepare(iommu->pclk);  in msm_iommu_probe()
748 		return dev_err_probe(iommu->dev, ret, "could not prepare iommu_clk\n");  in msm_iommu_probe()
752 	iommu->base = devm_ioremap_resource(iommu->dev, r);  in msm_iommu_probe()
753 	if (IS_ERR(iommu->base)) {  in msm_iommu_probe()
754 		ret = dev_err_probe(iommu->dev, PTR_ERR(iommu->base), "could not get iommu base\n");  in msm_iommu_probe()
757 	ioaddr = r->start;  in msm_iommu_probe()
759 	iommu->irq = platform_get_irq(pdev, 0);  in msm_iommu_probe()
760 	if (iommu->irq < 0) {  in msm_iommu_probe()
761 		ret = -ENODEV;  in msm_iommu_probe()
765 	ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val);  in msm_iommu_probe()
767 		dev_err(iommu->dev, "could not get ncb\n");  in msm_iommu_probe()
770 	iommu->ncb = val;  in msm_iommu_probe()
772 	msm_iommu_reset(iommu->base, iommu->ncb);  in msm_iommu_probe()
773 	SET_M(iommu->base, 0, 1);  in msm_iommu_probe()
774 	SET_PAR(iommu->base, 0, 0);  in msm_iommu_probe()
775 	SET_V2PCFG(iommu->base, 0, 1);  in msm_iommu_probe()
776 	SET_V2PPR(iommu->base, 0, 0);  in msm_iommu_probe()
777 	par = GET_PAR(iommu->base, 0);  in msm_iommu_probe()
778 	SET_V2PCFG(iommu->base, 0, 0);  in msm_iommu_probe()
779 	SET_M(iommu->base, 0, 0);  in msm_iommu_probe()
783 		ret = -ENODEV;  in msm_iommu_probe()
787 	ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL,  in msm_iommu_probe()
791 					iommu);  in msm_iommu_probe()
793 		pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret);  in msm_iommu_probe()
797 	list_add(&iommu->dev_node, &qcom_iommu_devices);  in msm_iommu_probe()
799 	ret = iommu_device_sysfs_add(&iommu->iommu, iommu->dev, NULL,  in msm_iommu_probe()
800 				     "msm-smmu.%pa", &ioaddr);  in msm_iommu_probe()
802 		pr_err("Could not add msm-smmu at %pa to sysfs\n", &ioaddr);  in msm_iommu_probe()
806 	ret = iommu_device_register(&iommu->iommu, &msm_iommu_ops, &pdev->dev);  in msm_iommu_probe()
808 		pr_err("Could not register msm-smmu at %pa\n", &ioaddr);  in msm_iommu_probe()
812 	pr_info("device mapped at %p, irq %d with %d ctx banks\n",  in msm_iommu_probe()
813 		iommu->base, iommu->irq, iommu->ncb);  in msm_iommu_probe()
817 	clk_unprepare(iommu->clk);  in msm_iommu_probe()
818 	clk_unprepare(iommu->pclk);  in msm_iommu_probe()
823 	{ .compatible = "qcom,apq8064-iommu" },
829 	struct msm_iommu_dev *iommu = platform_get_drvdata(pdev);  in msm_iommu_remove()  local
831 	clk_unprepare(iommu->clk);  in msm_iommu_remove()
832 	clk_unprepare(iommu->pclk);  in msm_iommu_remove()