14 #include <linux/iommu.h>
29 #include "iommu-pages.h"
101 	struct iommu_device iommu;  member
103 	/* Lock to modify the IOMMU registers */
125 	struct sun50i_iommu *iommu;  member
138 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)  in iommu_read()  argument
140 	return readl(iommu->base + offset);  in iommu_read()
143 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)  in iommu_write()  argument
145 	writel(value, iommu->base + offset);  in iommu_write()
149  * The Allwinner H6 IOMMU uses a 2-level page table.
158  * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
225  * The way permissions work is that the IOMMU has 16 "domains" that
236  * In order to make it work with the IOMMU framework, we will be using
239  * have each master setup in the same way, since the IOMMU framework
294 	struct sun50i_iommu *iommu = sun50i_domain->iommu;  in sun50i_table_flush()  local
298 	dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);  in sun50i_table_flush()
301 static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,  in sun50i_iommu_zap_iova()  argument
307 	iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);  in sun50i_iommu_zap_iova()
308 	iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));  in sun50i_iommu_zap_iova()
309 	iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,  in sun50i_iommu_zap_iova()
312 	ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,  in sun50i_iommu_zap_iova()
315 		dev_warn(iommu->dev, "TLB invalidation timed out!\n");  in sun50i_iommu_zap_iova()
318 static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,  in sun50i_iommu_zap_ptw_cache()  argument
324 	iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);  in sun50i_iommu_zap_ptw_cache()
325 	iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,  in sun50i_iommu_zap_ptw_cache()
328 	ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,  in sun50i_iommu_zap_ptw_cache()
331 		dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");  in sun50i_iommu_zap_ptw_cache()
334 static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,  in sun50i_iommu_zap_range()  argument
337 	assert_spin_locked(&iommu->iommu_lock);  in sun50i_iommu_zap_range()
339 	iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);  in sun50i_iommu_zap_range()
341 	sun50i_iommu_zap_iova(iommu, iova);  in sun50i_iommu_zap_range()
342 	sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);  in sun50i_iommu_zap_range()
344 		sun50i_iommu_zap_iova(iommu, iova + size);  in sun50i_iommu_zap_range()
345 		sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);  in sun50i_iommu_zap_range()
347 	sun50i_iommu_zap_ptw_cache(iommu, iova);  in sun50i_iommu_zap_range()
348 	sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);  in sun50i_iommu_zap_range()
350 		sun50i_iommu_zap_ptw_cache(iommu, iova + size);  in sun50i_iommu_zap_range()
351 		sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);  in sun50i_iommu_zap_range()
354 	iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);  in sun50i_iommu_zap_range()
357 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)  in sun50i_iommu_flush_all_tlb()  argument
362 	assert_spin_locked(&iommu->iommu_lock);  in sun50i_iommu_flush_all_tlb()
364 	iommu_write(iommu,  in sun50i_iommu_flush_all_tlb()
375 	ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,  in sun50i_iommu_flush_all_tlb()
379 		dev_warn(iommu->dev, "TLB Flush timed out!\n");  in sun50i_iommu_flush_all_tlb()
387 	struct sun50i_iommu *iommu = sun50i_domain->iommu;  in sun50i_iommu_flush_iotlb_all()  local
392 	 * .probe_device, and since we link our (single) domain to our iommu in  in sun50i_iommu_flush_iotlb_all()
398 	if (!iommu)  in sun50i_iommu_flush_iotlb_all()
401 	spin_lock_irqsave(&iommu->iommu_lock, flags);  in sun50i_iommu_flush_iotlb_all()
402 	sun50i_iommu_flush_all_tlb(iommu);  in sun50i_iommu_flush_iotlb_all()
403 	spin_unlock_irqrestore(&iommu->iommu_lock, flags);  in sun50i_iommu_flush_iotlb_all()
410 	struct sun50i_iommu *iommu = sun50i_domain->iommu;  in sun50i_iommu_iotlb_sync_map()  local
413 	spin_lock_irqsave(&iommu->iommu_lock, flags);  in sun50i_iommu_iotlb_sync_map()
414 	sun50i_iommu_zap_range(iommu, iova, size);  in sun50i_iommu_iotlb_sync_map()
415 	spin_unlock_irqrestore(&iommu->iommu_lock, flags);  in sun50i_iommu_iotlb_sync_map()
426 static int sun50i_iommu_enable(struct sun50i_iommu *iommu)  in sun50i_iommu_enable()  argument
432 	if (!iommu->domain)  in sun50i_iommu_enable()
435 	sun50i_domain = to_sun50i_domain(iommu->domain);  in sun50i_iommu_enable()
437 	ret = reset_control_deassert(iommu->reset);  in sun50i_iommu_enable()
441 	ret = clk_prepare_enable(iommu->clk);  in sun50i_iommu_enable()
445 	spin_lock_irqsave(&iommu->iommu_lock, flags);  in sun50i_iommu_enable()
447 	iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);  in sun50i_iommu_enable()
448 	iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,  in sun50i_iommu_enable()
455 	iommu_write(iommu, IOMMU_BYPASS_REG, 0);  in sun50i_iommu_enable()
456 	iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);  in sun50i_iommu_enable()
457 	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),  in sun50i_iommu_enable()
471 	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),  in sun50i_iommu_enable()
479 	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),  in sun50i_iommu_enable()
487 	ret = sun50i_iommu_flush_all_tlb(iommu);  in sun50i_iommu_enable()
489 		spin_unlock_irqrestore(&iommu->iommu_lock, flags);  in sun50i_iommu_enable()
493 	iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);  in sun50i_iommu_enable()
494 	iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);  in sun50i_iommu_enable()
496 	spin_unlock_irqrestore(&iommu->iommu_lock, flags);  in sun50i_iommu_enable()
501 	clk_disable_unprepare(iommu->clk);  in sun50i_iommu_enable()
504 	reset_control_assert(iommu->reset);  in sun50i_iommu_enable()
509 static void sun50i_iommu_disable(struct sun50i_iommu *iommu)  in sun50i_iommu_disable()  argument
513 	spin_lock_irqsave(&iommu->iommu_lock, flags);  in sun50i_iommu_disable()
515 	iommu_write(iommu, IOMMU_ENABLE_REG, 0);  in sun50i_iommu_disable()
516 	iommu_write(iommu, IOMMU_TTB_REG, 0);  in sun50i_iommu_disable()
518 	spin_unlock_irqrestore(&iommu->iommu_lock, flags);  in sun50i_iommu_disable()
520 	clk_disable_unprepare(iommu->clk);  in sun50i_iommu_disable()
521 	reset_control_assert(iommu->reset);  in sun50i_iommu_disable()
524 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,  in sun50i_iommu_alloc_page_table()  argument
530 	page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);  in sun50i_iommu_alloc_page_table()
534 	pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);  in sun50i_iommu_alloc_page_table()
535 	if (dma_mapping_error(iommu->dev, pt_dma)) {  in sun50i_iommu_alloc_page_table()
536 		dev_err(iommu->dev, "Couldn't map L2 Page Table\n");  in sun50i_iommu_alloc_page_table()
537 		kmem_cache_free(iommu->pt_pool, page_table);  in sun50i_iommu_alloc_page_table()
547 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,  in sun50i_iommu_free_page_table()  argument
552 	dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);  in sun50i_iommu_free_page_table()
553 	kmem_cache_free(iommu->pt_pool, page_table);  in sun50i_iommu_free_page_table()
559 	struct sun50i_iommu *iommu = sun50i_domain->iommu;  in sun50i_dte_get_page_table()  local
572 	page_table = sun50i_iommu_alloc_page_table(iommu, gfp);  in sun50i_dte_get_page_table()
586 		sun50i_iommu_free_page_table(iommu, drop_pt);  in sun50i_dte_get_page_table()
600 	struct sun50i_iommu *iommu = sun50i_domain->iommu;  in sun50i_iommu_map()  local
605 	/* the IOMMU can only handle 32-bit addresses, both input and output */  in sun50i_iommu_map()
608 		dev_warn_once(iommu->dev,  in sun50i_iommu_map()
623 		dev_err(iommu->dev,  in sun50i_iommu_map()
722 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,  in sun50i_iommu_attach_domain()  argument
725 	iommu->domain = &sun50i_domain->domain;  in sun50i_iommu_attach_domain()
726 	sun50i_domain->iommu = iommu;  in sun50i_iommu_attach_domain()
728 	sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,  in sun50i_iommu_attach_domain()
730 	if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {  in sun50i_iommu_attach_domain()
731 		dev_err(iommu->dev, "Couldn't map L1 Page Table\n");  in sun50i_iommu_attach_domain()
735 	return sun50i_iommu_enable(iommu);  in sun50i_iommu_attach_domain()
738 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,  in sun50i_iommu_detach_domain()  argument
759 		sun50i_iommu_free_page_table(iommu, page_table);  in sun50i_iommu_detach_domain()
763 	sun50i_iommu_disable(iommu);  in sun50i_iommu_detach_domain()
765 	dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),  in sun50i_iommu_detach_domain()
768 	iommu->domain = NULL;  in sun50i_iommu_detach_domain()
774 	struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);  in sun50i_iommu_identity_attach()  local
777 	dev_dbg(dev, "Detaching from IOMMU domain\n");  in sun50i_iommu_identity_attach()
779 	if (iommu->domain == identity_domain)  in sun50i_iommu_identity_attach()
782 	sun50i_domain = to_sun50i_domain(iommu->domain);  in sun50i_iommu_identity_attach()
784 		sun50i_iommu_detach_domain(iommu, sun50i_domain);  in sun50i_iommu_identity_attach()
801 	struct sun50i_iommu *iommu;  in sun50i_iommu_attach_device()  local
803 	iommu = sun50i_iommu_from_dev(dev);  in sun50i_iommu_attach_device()
804 	if (!iommu)  in sun50i_iommu_attach_device()
807 	dev_dbg(dev, "Attaching to IOMMU domain\n");  in sun50i_iommu_attach_device()
811 	if (iommu->domain == domain)  in sun50i_iommu_attach_device()
816 	sun50i_iommu_attach_domain(iommu, sun50i_domain);  in sun50i_iommu_attach_device()
823 	struct sun50i_iommu *iommu;  in sun50i_iommu_probe_device()  local
825 	iommu = sun50i_iommu_from_dev(dev);  in sun50i_iommu_probe_device()
826 	if (!iommu)  in sun50i_iommu_probe_device()
829 	return &iommu->iommu;  in sun50i_iommu_probe_device()
862 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,  in sun50i_iommu_report_fault()  argument
866 	dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",  in sun50i_iommu_report_fault()
869 	if (iommu->domain)  in sun50i_iommu_report_fault()
870 		report_iommu_fault(iommu->domain, iommu->dev, iova, prot);  in sun50i_iommu_report_fault()
872 		dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");  in sun50i_iommu_report_fault()
874 	sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);  in sun50i_iommu_report_fault()
877 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,  in sun50i_iommu_handle_pt_irq()  argument
885 	assert_spin_locked(&iommu->iommu_lock);  in sun50i_iommu_handle_pt_irq()
887 	iova = iommu_read(iommu, addr_reg);  in sun50i_iommu_handle_pt_irq()
888 	blame = iommu_read(iommu, blame_reg);  in sun50i_iommu_handle_pt_irq()
896 	sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);  in sun50i_iommu_handle_pt_irq()
901 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)  in sun50i_iommu_handle_perm_irq()  argument
909 	assert_spin_locked(&iommu->iommu_lock);  in sun50i_iommu_handle_perm_irq()
911 	blame = iommu_read(iommu, IOMMU_INT_STA_REG);  in sun50i_iommu_handle_perm_irq()
913 	iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));  in sun50i_iommu_handle_perm_irq()
914 	aci = sun50i_get_pte_aci(iommu_read(iommu,  in sun50i_iommu_handle_perm_irq()
951 	sun50i_iommu_report_fault(iommu, master, iova, dir);  in sun50i_iommu_handle_perm_irq()
959 	struct sun50i_iommu *iommu = dev_id;  in sun50i_iommu_irq()  local
961 	spin_lock(&iommu->iommu_lock);  in sun50i_iommu_irq()
963 	status = iommu_read(iommu, IOMMU_INT_STA_REG);  in sun50i_iommu_irq()
965 		spin_unlock(&iommu->iommu_lock);  in sun50i_iommu_irq()
969 	l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);  in sun50i_iommu_irq()
970 	l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);  in sun50i_iommu_irq()
973 		sun50i_iommu_handle_pt_irq(iommu,  in sun50i_iommu_irq()
977 		sun50i_iommu_handle_pt_irq(iommu,  in sun50i_iommu_irq()
981 		sun50i_iommu_handle_perm_irq(iommu);  in sun50i_iommu_irq()
983 	iommu_write(iommu, IOMMU_INT_CLR_REG, status);  in sun50i_iommu_irq()
986 	iommu_write(iommu, IOMMU_RESET_REG, ~resets);  in sun50i_iommu_irq()
987 	iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);  in sun50i_iommu_irq()
989 	spin_unlock(&iommu->iommu_lock);  in sun50i_iommu_irq()
996 	struct sun50i_iommu *iommu;  in sun50i_iommu_probe()  local
999 	iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);  in sun50i_iommu_probe()
1000 	if (!iommu)  in sun50i_iommu_probe()
1002 	spin_lock_init(&iommu->iommu_lock);  in sun50i_iommu_probe()
1003 	iommu->domain = &sun50i_iommu_identity_domain;  in sun50i_iommu_probe()
1004 	platform_set_drvdata(pdev, iommu);  in sun50i_iommu_probe()
1005 	iommu->dev = &pdev->dev;  in sun50i_iommu_probe()
1007 	iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),  in sun50i_iommu_probe()
1011 	if (!iommu->pt_pool)  in sun50i_iommu_probe()
1014 	iommu->base = devm_platform_ioremap_resource(pdev, 0);  in sun50i_iommu_probe()
1015 	if (IS_ERR(iommu->base)) {  in sun50i_iommu_probe()
1016 		ret = PTR_ERR(iommu->base);  in sun50i_iommu_probe()
1026 	iommu->clk = devm_clk_get(&pdev->dev, NULL);  in sun50i_iommu_probe()
1027 	if (IS_ERR(iommu->clk)) {  in sun50i_iommu_probe()
1029 		ret = PTR_ERR(iommu->clk);  in sun50i_iommu_probe()
1033 	iommu->reset = devm_reset_control_get(&pdev->dev, NULL);  in sun50i_iommu_probe()
1034 	if (IS_ERR(iommu->reset)) {  in sun50i_iommu_probe()
1036 		ret = PTR_ERR(iommu->reset);  in sun50i_iommu_probe()
1040 	ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,  in sun50i_iommu_probe()
1045 	ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev);  in sun50i_iommu_probe()
1050 			       dev_name(&pdev->dev), iommu);  in sun50i_iommu_probe()
1057 	iommu_device_unregister(&iommu->iommu);  in sun50i_iommu_probe()
1060 	iommu_device_sysfs_remove(&iommu->iommu);  in sun50i_iommu_probe()
1063 	kmem_cache_destroy(iommu->pt_pool);  in sun50i_iommu_probe()
1069 	{ .compatible = "allwinner,sun50i-h6-iommu", },
1070 	{ .compatible = "allwinner,sun50i-h616-iommu", },
1077 		.name			= "sun50i-iommu",
1084 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");