Lines Matching +full:iommu +full:- +full:base
1 // SPDX-License-Identifier: GPL-2.0-only
3 * IOMMU API for QCOM secure IOMMUs. Somewhat based on arm-smmu.c
13 #include <linux/dma-mapping.h>
17 #include <linux/io-64-nonatomic-hi-lo.h>
18 #include <linux/io-pgtable.h>
19 #include <linux/iommu.h>
33 #include "arm-smmu.h"
47 /* IOMMU core code handle */
48 struct iommu_device iommu; member
59 void __iomem *base; member
69 struct mutex init_mutex; /* Protects iommu pointer */
71 struct qcom_iommu_dev *iommu; member
84 struct qcom_iommu_dev *qcom_iommu = d->iommu; in to_ctx()
87 return qcom_iommu->ctxs[asid]; in to_ctx()
93 writel_relaxed(val, ctx->base + reg); in iommu_writel()
99 writeq_relaxed(val, ctx->base + reg); in iommu_writeq()
105 return readl_relaxed(ctx->base + reg); in iommu_readl()
111 return readq_relaxed(ctx->base + reg); in iommu_readq()
117 struct iommu_fwspec *fwspec = qcom_domain->fwspec; in qcom_iommu_tlb_sync()
120 for (i = 0; i < fwspec->num_ids; i++) { in qcom_iommu_tlb_sync()
121 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]); in qcom_iommu_tlb_sync()
126 ret = readl_poll_timeout(ctx->base + ARM_SMMU_CB_TLBSTATUS, val, in qcom_iommu_tlb_sync()
129 dev_err(ctx->dev, "timeout waiting for TLB SYNC\n"); in qcom_iommu_tlb_sync()
136 struct iommu_fwspec *fwspec = qcom_domain->fwspec; in qcom_iommu_tlb_inv_context()
139 for (i = 0; i < fwspec->num_ids; i++) { in qcom_iommu_tlb_inv_context()
140 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]); in qcom_iommu_tlb_inv_context()
141 iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid); in qcom_iommu_tlb_inv_context()
151 struct iommu_fwspec *fwspec = qcom_domain->fwspec; in qcom_iommu_tlb_inv_range_nosync()
156 for (i = 0; i < fwspec->num_ids; i++) { in qcom_iommu_tlb_inv_range_nosync()
157 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]); in qcom_iommu_tlb_inv_range_nosync()
161 iova |= ctx->asid; in qcom_iommu_tlb_inv_range_nosync()
165 } while (s -= granule); in qcom_iommu_tlb_inv_range_nosync()
203 if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { in qcom_iommu_fault()
204 dev_err_ratelimited(ctx->dev, in qcom_iommu_fault()
207 fsr, iova, fsynr, ctx->asid); in qcom_iommu_fault()
227 mutex_lock(&qcom_domain->init_mutex); in qcom_iommu_init_domain()
228 if (qcom_domain->iommu) in qcom_iommu_init_domain()
232 .pgsize_bitmap = domain->pgsize_bitmap, in qcom_iommu_init_domain()
236 .iommu_dev = qcom_iommu->dev, in qcom_iommu_init_domain()
239 qcom_domain->iommu = qcom_iommu; in qcom_iommu_init_domain()
240 qcom_domain->fwspec = fwspec; in qcom_iommu_init_domain()
244 dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n"); in qcom_iommu_init_domain()
245 ret = -ENOMEM; in qcom_iommu_init_domain()
249 domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1; in qcom_iommu_init_domain()
250 domain->geometry.force_aperture = true; in qcom_iommu_init_domain()
252 for (i = 0; i < fwspec->num_ids; i++) { in qcom_iommu_init_domain()
253 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]); in qcom_iommu_init_domain()
255 if (!ctx->secure_init) { in qcom_iommu_init_domain()
256 ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid); in qcom_iommu_init_domain()
258 dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret); in qcom_iommu_init_domain()
261 ctx->secure_init = true; in qcom_iommu_init_domain()
264 /* Secured QSMMU-500/QSMMU-v2 contexts cannot be programmed */ in qcom_iommu_init_domain()
265 if (ctx->secured_ctx) { in qcom_iommu_init_domain()
266 ctx->domain = domain; in qcom_iommu_init_domain()
280 FIELD_PREP(ARM_SMMU_TTBRn_ASID, ctx->asid)); in qcom_iommu_init_domain()
289 /* MAIRs (stage-1 only) */ in qcom_iommu_init_domain()
306 ctx->domain = domain; in qcom_iommu_init_domain()
309 mutex_unlock(&qcom_domain->init_mutex); in qcom_iommu_init_domain()
312 qcom_domain->pgtbl_ops = pgtbl_ops; in qcom_iommu_init_domain()
317 qcom_domain->iommu = NULL; in qcom_iommu_init_domain()
319 mutex_unlock(&qcom_domain->init_mutex); in qcom_iommu_init_domain()
336 mutex_init(&qcom_domain->init_mutex); in qcom_iommu_domain_alloc_paging()
337 spin_lock_init(&qcom_domain->pgtbl_lock); in qcom_iommu_domain_alloc_paging()
338 qcom_domain->domain.pgsize_bitmap = SZ_4K; in qcom_iommu_domain_alloc_paging()
340 return &qcom_domain->domain; in qcom_iommu_domain_alloc_paging()
347 if (qcom_domain->iommu) { in qcom_iommu_domain_free()
350 * off, for example, with GPUs or anything involving dma-buf. in qcom_iommu_domain_free()
351 * So we cannot rely on the device_link. Make sure the IOMMU in qcom_iommu_domain_free()
354 pm_runtime_get_sync(qcom_domain->iommu->dev); in qcom_iommu_domain_free()
355 free_io_pgtable_ops(qcom_domain->pgtbl_ops); in qcom_iommu_domain_free()
356 pm_runtime_put_sync(qcom_domain->iommu->dev); in qcom_iommu_domain_free()
369 dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n"); in qcom_iommu_attach_dev()
370 return -ENXIO; in qcom_iommu_attach_dev()
374 pm_runtime_get_sync(qcom_iommu->dev); in qcom_iommu_attach_dev()
376 pm_runtime_put_sync(qcom_iommu->dev); in qcom_iommu_attach_dev()
384 if (qcom_domain->iommu != qcom_iommu) in qcom_iommu_attach_dev()
385 return -EINVAL; in qcom_iommu_attach_dev()
403 if (WARN_ON(!qcom_domain->iommu)) in qcom_iommu_identity_attach()
404 return -EINVAL; in qcom_iommu_identity_attach()
406 pm_runtime_get_sync(qcom_iommu->dev); in qcom_iommu_identity_attach()
407 for (i = 0; i < fwspec->num_ids; i++) { in qcom_iommu_identity_attach()
408 struct qcom_iommu_ctx *ctx = to_ctx(qcom_domain, fwspec->ids[i]); in qcom_iommu_identity_attach()
413 ctx->domain = NULL; in qcom_iommu_identity_attach()
415 pm_runtime_put_sync(qcom_iommu->dev); in qcom_iommu_identity_attach()
435 struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops; in qcom_iommu_map()
438 return -ENODEV; in qcom_iommu_map()
440 spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags); in qcom_iommu_map()
441 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, GFP_ATOMIC, mapped); in qcom_iommu_map()
442 spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags); in qcom_iommu_map()
453 struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops; in qcom_iommu_unmap()
459 * for example, with GPUs or anything involving dma-buf. So we in qcom_iommu_unmap()
460 * cannot rely on the device_link. Make sure the IOMMU is on to in qcom_iommu_unmap()
463 pm_runtime_get_sync(qcom_domain->iommu->dev); in qcom_iommu_unmap()
464 spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags); in qcom_iommu_unmap()
465 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, gather); in qcom_iommu_unmap()
466 spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags); in qcom_iommu_unmap()
467 pm_runtime_put_sync(qcom_domain->iommu->dev); in qcom_iommu_unmap()
475 struct io_pgtable *pgtable = container_of(qcom_domain->pgtbl_ops, in qcom_iommu_flush_iotlb_all()
477 if (!qcom_domain->pgtbl_ops) in qcom_iommu_flush_iotlb_all()
480 pm_runtime_get_sync(qcom_domain->iommu->dev); in qcom_iommu_flush_iotlb_all()
481 qcom_iommu_tlb_sync(pgtable->cookie); in qcom_iommu_flush_iotlb_all()
482 pm_runtime_put_sync(qcom_domain->iommu->dev); in qcom_iommu_flush_iotlb_all()
497 struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops; in qcom_iommu_iova_to_phys()
502 spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags); in qcom_iommu_iova_to_phys()
503 ret = ops->iova_to_phys(ops, iova); in qcom_iommu_iova_to_phys()
504 spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags); in qcom_iommu_iova_to_phys()
531 return ERR_PTR(-ENODEV); in qcom_iommu_probe_device()
534 * Establish the link between iommu and master, so that the in qcom_iommu_probe_device()
535 * iommu gets runtime enabled/disabled as per the master's in qcom_iommu_probe_device()
538 link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME); in qcom_iommu_probe_device()
540 dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n", in qcom_iommu_probe_device()
541 dev_name(qcom_iommu->dev), dev_name(dev)); in qcom_iommu_probe_device()
542 return ERR_PTR(-ENODEV); in qcom_iommu_probe_device()
545 return &qcom_iommu->iommu; in qcom_iommu_probe_device()
553 unsigned asid = args->args[0]; in qcom_iommu_of_xlate()
555 if (args->args_count != 1) { in qcom_iommu_of_xlate()
556 dev_err(dev, "incorrect number of iommu params found for %s " in qcom_iommu_of_xlate()
558 args->np->full_name, args->args_count); in qcom_iommu_of_xlate()
559 return -EINVAL; in qcom_iommu_of_xlate()
562 iommu_pdev = of_find_device_by_node(args->np); in qcom_iommu_of_xlate()
564 return -EINVAL; in qcom_iommu_of_xlate()
571 if (WARN_ON(asid > qcom_iommu->max_asid) || in qcom_iommu_of_xlate()
572 WARN_ON(qcom_iommu->ctxs[asid] == NULL)) { in qcom_iommu_of_xlate()
573 put_device(&iommu_pdev->dev); in qcom_iommu_of_xlate()
574 return -EINVAL; in qcom_iommu_of_xlate()
581 * multiple different iommu devices. Multiple context in qcom_iommu_of_xlate()
585 put_device(&iommu_pdev->dev); in qcom_iommu_of_xlate()
586 return -EINVAL; in qcom_iommu_of_xlate()
626 dev_err(dev, "failed to get iommu secure pgtable size (%d)\n", in qcom_iommu_sec_ptbl_init()
631 dev_info(dev, "iommu sec: pgtable size: %zu\n", psize); in qcom_iommu_sec_ptbl_init()
639 return -ENOMEM; in qcom_iommu_sec_ptbl_init()
644 dev_err(dev, "failed to init iommu pgtable (%d)\n", ret); in qcom_iommu_sec_ptbl_init()
665 return -ENODEV; in get_asid()
672 if (!of_property_read_u32(np, "qcom,ctx-asid", &val)) in get_asid()
683 struct device *dev = &pdev->dev; in qcom_iommu_ctx_probe()
684 struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent); in qcom_iommu_ctx_probe()
689 return -ENOMEM; in qcom_iommu_ctx_probe()
691 ctx->dev = dev; in qcom_iommu_ctx_probe()
694 ctx->base = devm_platform_ioremap_resource(pdev, 0); in qcom_iommu_ctx_probe()
695 if (IS_ERR(ctx->base)) in qcom_iommu_ctx_probe()
696 return PTR_ERR(ctx->base); in qcom_iommu_ctx_probe()
702 if (of_device_is_compatible(dev->of_node, "qcom,msm-iommu-v2-sec")) in qcom_iommu_ctx_probe()
703 ctx->secured_ctx = true; in qcom_iommu_ctx_probe()
706 * boot-loader left us a surprise: in qcom_iommu_ctx_probe()
708 if (!ctx->secured_ctx) in qcom_iommu_ctx_probe()
714 "qcom-iommu-fault", in qcom_iommu_ctx_probe()
721 ret = get_asid(dev->of_node); in qcom_iommu_ctx_probe()
727 ctx->asid = ret; in qcom_iommu_ctx_probe()
729 dev_dbg(dev, "found asid %u\n", ctx->asid); in qcom_iommu_ctx_probe()
731 qcom_iommu->ctxs[ctx->asid] = ctx; in qcom_iommu_ctx_probe()
738 struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(pdev->dev.parent); in qcom_iommu_ctx_remove()
743 qcom_iommu->ctxs[ctx->asid] = NULL; in qcom_iommu_ctx_remove()
747 { .compatible = "qcom,msm-iommu-v1-ns" },
748 { .compatible = "qcom,msm-iommu-v1-sec" },
749 { .compatible = "qcom,msm-iommu-v2-ns" },
750 { .compatible = "qcom,msm-iommu-v2-sec" },
756 .name = "qcom-iommu-ctx",
767 for_each_child_of_node(qcom_iommu->dev->of_node, child) { in qcom_iommu_has_secure_context()
768 if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec") || in qcom_iommu_has_secure_context()
769 of_device_is_compatible(child, "qcom,msm-iommu-v2-sec")) { in qcom_iommu_has_secure_context()
782 struct device *dev = &pdev->dev; in qcom_iommu_device_probe()
790 for_each_child_of_node(dev->of_node, child) in qcom_iommu_device_probe()
796 return -ENOMEM; in qcom_iommu_device_probe()
797 qcom_iommu->max_asid = max_asid; in qcom_iommu_device_probe()
798 qcom_iommu->dev = dev; in qcom_iommu_device_probe()
802 qcom_iommu->local_base = devm_ioremap_resource(dev, res); in qcom_iommu_device_probe()
803 if (IS_ERR(qcom_iommu->local_base)) in qcom_iommu_device_probe()
804 return PTR_ERR(qcom_iommu->local_base); in qcom_iommu_device_probe()
812 qcom_iommu->clks[CLK_IFACE].clk = clk; in qcom_iommu_device_probe()
819 qcom_iommu->clks[CLK_BUS].clk = clk; in qcom_iommu_device_probe()
826 qcom_iommu->clks[CLK_TBU].clk = clk; in qcom_iommu_device_probe()
828 if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id", in qcom_iommu_device_probe()
829 &qcom_iommu->sec_id)) { in qcom_iommu_device_probe()
830 dev_err(dev, "missing qcom,iommu-secure-id property\n"); in qcom_iommu_device_probe()
831 return -ENODEV; in qcom_iommu_device_probe()
849 dev_err(dev, "Failed to populate iommu contexts\n"); in qcom_iommu_device_probe()
853 ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL, in qcom_iommu_device_probe()
856 dev_err(dev, "Failed to register iommu in sysfs\n"); in qcom_iommu_device_probe()
860 ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev); in qcom_iommu_device_probe()
862 dev_err(dev, "Failed to register iommu\n"); in qcom_iommu_device_probe()
866 if (qcom_iommu->local_base) { in qcom_iommu_device_probe()
868 writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS); in qcom_iommu_device_probe()
883 pm_runtime_force_suspend(&pdev->dev); in qcom_iommu_device_remove()
885 iommu_device_sysfs_remove(&qcom_iommu->iommu); in qcom_iommu_device_remove()
886 iommu_device_unregister(&qcom_iommu->iommu); in qcom_iommu_device_remove()
894 ret = clk_bulk_prepare_enable(CLK_NUM, qcom_iommu->clks); in qcom_iommu_resume()
898 if (dev->pm_domain) in qcom_iommu_resume()
899 return qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, 0); in qcom_iommu_resume()
908 clk_bulk_disable_unprepare(CLK_NUM, qcom_iommu->clks); in qcom_iommu_suspend()
920 { .compatible = "qcom,msm-iommu-v1" },
921 { .compatible = "qcom,msm-iommu-v2" },
927 .name = "qcom-iommu",