Lines Matching +full:sun50i +full:- +full:h6 +full:- +full:iommu
1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
3 // Copyright (C) 2019-2020, Cerno
9 #include <linux/dma-direction.h>
10 #include <linux/dma-mapping.h>
14 #include <linux/iommu.h>
29 #include "iommu-pages.h"
101 struct iommu_device iommu; member
103 /* Lock to modify the IOMMU registers */
125 struct sun50i_iommu *iommu; member
138 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) in iommu_read() argument
140 return readl(iommu->base + offset); in iommu_read()
143 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) in iommu_write() argument
145 writel(value, iommu->base + offset); in iommu_write()
149 * The Allwinner H6 IOMMU uses a 2-level page table.
152 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
155 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
158 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
185 * +---------------------+-----------+-+
187 * +---------------------+-----------+-+
188 * 31:10 - Page Table address
189 * 9:2 - Reserved
190 * 1:0 - 1 if the entry is valid
215 * +----------------+-----+-----+-----+---+-----+
217 * +----------------+-----+-----+-----+---+-----+
218 * 31:12 - Page address
219 * 11:8 - Reserved
220 * 7:4 - Authority Control Index
221 * 3:2 - Reserved
222 * 1 - 1 if the entry is valid
223 * 0 - Reserved
225 * The way permissions work is that the IOMMU has 16 "domains" that
229 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
233 * affected to, so that we can actually enforce them on a per-page
236 * In order to make it work with the IOMMU framework, we will be using
239 * have each master setup in the same way, since the IOMMU framework
240 * doesn't seem to restrict page access on a per-device basis. And
294 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_table_flush() local
298 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); in sun50i_table_flush()
301 static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu, in sun50i_iommu_zap_iova() argument
307 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova); in sun50i_iommu_zap_iova()
308 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12)); in sun50i_iommu_zap_iova()
309 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, in sun50i_iommu_zap_iova()
312 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG, in sun50i_iommu_zap_iova()
315 dev_warn(iommu->dev, "TLB invalidation timed out!\n"); in sun50i_iommu_zap_iova()
318 static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu, in sun50i_iommu_zap_ptw_cache() argument
324 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova); in sun50i_iommu_zap_ptw_cache()
325 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, in sun50i_iommu_zap_ptw_cache()
328 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG, in sun50i_iommu_zap_ptw_cache()
331 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n"); in sun50i_iommu_zap_ptw_cache()
334 static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu, in sun50i_iommu_zap_range() argument
337 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_zap_range()
339 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0); in sun50i_iommu_zap_range()
341 sun50i_iommu_zap_iova(iommu, iova); in sun50i_iommu_zap_range()
342 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE); in sun50i_iommu_zap_range()
344 sun50i_iommu_zap_iova(iommu, iova + size); in sun50i_iommu_zap_range()
345 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE); in sun50i_iommu_zap_range()
347 sun50i_iommu_zap_ptw_cache(iommu, iova); in sun50i_iommu_zap_range()
348 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M); in sun50i_iommu_zap_range()
350 sun50i_iommu_zap_ptw_cache(iommu, iova + size); in sun50i_iommu_zap_range()
351 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M); in sun50i_iommu_zap_range()
354 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); in sun50i_iommu_zap_range()
357 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) in sun50i_iommu_flush_all_tlb() argument
362 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_flush_all_tlb()
364 iommu_write(iommu, in sun50i_iommu_flush_all_tlb()
375 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG, in sun50i_iommu_flush_all_tlb()
379 dev_warn(iommu->dev, "TLB Flush timed out!\n"); in sun50i_iommu_flush_all_tlb()
387 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_iommu_flush_iotlb_all() local
392 * .probe_device, and since we link our (single) domain to our iommu in in sun50i_iommu_flush_iotlb_all()
398 if (!iommu) in sun50i_iommu_flush_iotlb_all()
401 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_flush_iotlb_all()
402 sun50i_iommu_flush_all_tlb(iommu); in sun50i_iommu_flush_iotlb_all()
403 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_flush_iotlb_all()
410 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_iommu_iotlb_sync_map() local
413 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_iotlb_sync_map()
414 sun50i_iommu_zap_range(iommu, iova, size); in sun50i_iommu_iotlb_sync_map()
415 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_iotlb_sync_map()
426 static int sun50i_iommu_enable(struct sun50i_iommu *iommu) in sun50i_iommu_enable() argument
432 if (!iommu->domain) in sun50i_iommu_enable()
435 sun50i_domain = to_sun50i_domain(iommu->domain); in sun50i_iommu_enable()
437 ret = reset_control_deassert(iommu->reset); in sun50i_iommu_enable()
441 ret = clk_prepare_enable(iommu->clk); in sun50i_iommu_enable()
445 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_enable()
447 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); in sun50i_iommu_enable()
448 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, in sun50i_iommu_enable()
455 iommu_write(iommu, IOMMU_BYPASS_REG, 0); in sun50i_iommu_enable()
456 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); in sun50i_iommu_enable()
457 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), in sun50i_iommu_enable()
471 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), in sun50i_iommu_enable()
479 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), in sun50i_iommu_enable()
487 ret = sun50i_iommu_flush_all_tlb(iommu); in sun50i_iommu_enable()
489 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_enable()
493 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); in sun50i_iommu_enable()
494 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); in sun50i_iommu_enable()
496 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_enable()
501 clk_disable_unprepare(iommu->clk); in sun50i_iommu_enable()
504 reset_control_assert(iommu->reset); in sun50i_iommu_enable()
509 static void sun50i_iommu_disable(struct sun50i_iommu *iommu) in sun50i_iommu_disable() argument
513 spin_lock_irqsave(&iommu->iommu_lock, flags); in sun50i_iommu_disable()
515 iommu_write(iommu, IOMMU_ENABLE_REG, 0); in sun50i_iommu_disable()
516 iommu_write(iommu, IOMMU_TTB_REG, 0); in sun50i_iommu_disable()
518 spin_unlock_irqrestore(&iommu->iommu_lock, flags); in sun50i_iommu_disable()
520 clk_disable_unprepare(iommu->clk); in sun50i_iommu_disable()
521 reset_control_assert(iommu->reset); in sun50i_iommu_disable()
524 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, in sun50i_iommu_alloc_page_table() argument
530 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); in sun50i_iommu_alloc_page_table()
532 return ERR_PTR(-ENOMEM); in sun50i_iommu_alloc_page_table()
534 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); in sun50i_iommu_alloc_page_table()
535 if (dma_mapping_error(iommu->dev, pt_dma)) { in sun50i_iommu_alloc_page_table()
536 dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); in sun50i_iommu_alloc_page_table()
537 kmem_cache_free(iommu->pt_pool, page_table); in sun50i_iommu_alloc_page_table()
538 return ERR_PTR(-ENOMEM); in sun50i_iommu_alloc_page_table()
547 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, in sun50i_iommu_free_page_table() argument
552 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); in sun50i_iommu_free_page_table()
553 kmem_cache_free(iommu->pt_pool, page_table); in sun50i_iommu_free_page_table()
559 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_dte_get_page_table() local
565 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; in sun50i_dte_get_page_table()
572 page_table = sun50i_iommu_alloc_page_table(iommu, gfp); in sun50i_dte_get_page_table()
586 sun50i_iommu_free_page_table(iommu, drop_pt); in sun50i_dte_get_page_table()
600 struct sun50i_iommu *iommu = sun50i_domain->iommu; in sun50i_iommu_map() local
605 /* the IOMMU can only handle 32-bit addresses, both input and output */ in sun50i_iommu_map()
607 ret = -EINVAL; in sun50i_iommu_map()
608 dev_warn_once(iommu->dev, in sun50i_iommu_map()
623 dev_err(iommu->dev, in sun50i_iommu_map()
626 ret = -EBUSY; in sun50i_iommu_map()
646 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; in sun50i_iommu_unmap()
670 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; in sun50i_iommu_iova_to_phys()
693 sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, in sun50i_iommu_domain_alloc_paging()
695 if (!sun50i_domain->dt) in sun50i_iommu_domain_alloc_paging()
698 refcount_set(&sun50i_domain->refcnt, 1); in sun50i_iommu_domain_alloc_paging()
700 sun50i_domain->domain.geometry.aperture_start = 0; in sun50i_iommu_domain_alloc_paging()
701 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); in sun50i_iommu_domain_alloc_paging()
702 sun50i_domain->domain.geometry.force_aperture = true; in sun50i_iommu_domain_alloc_paging()
704 return &sun50i_domain->domain; in sun50i_iommu_domain_alloc_paging()
716 iommu_free_pages(sun50i_domain->dt, get_order(DT_SIZE)); in sun50i_iommu_domain_free()
717 sun50i_domain->dt = NULL; in sun50i_iommu_domain_free()
722 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, in sun50i_iommu_attach_domain() argument
725 iommu->domain = &sun50i_domain->domain; in sun50i_iommu_attach_domain()
726 sun50i_domain->iommu = iommu; in sun50i_iommu_attach_domain()
728 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, in sun50i_iommu_attach_domain()
730 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { in sun50i_iommu_attach_domain()
731 dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); in sun50i_iommu_attach_domain()
732 return -ENOMEM; in sun50i_iommu_attach_domain()
735 return sun50i_iommu_enable(iommu); in sun50i_iommu_attach_domain()
738 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, in sun50i_iommu_detach_domain() argument
749 dte_addr = &sun50i_domain->dt[i]; in sun50i_iommu_detach_domain()
759 sun50i_iommu_free_page_table(iommu, page_table); in sun50i_iommu_detach_domain()
763 sun50i_iommu_disable(iommu); in sun50i_iommu_detach_domain()
765 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), in sun50i_iommu_detach_domain()
768 iommu->domain = NULL; in sun50i_iommu_detach_domain()
774 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); in sun50i_iommu_identity_attach() local
777 dev_dbg(dev, "Detaching from IOMMU domain\n"); in sun50i_iommu_identity_attach()
779 if (iommu->domain == identity_domain) in sun50i_iommu_identity_attach()
782 sun50i_domain = to_sun50i_domain(iommu->domain); in sun50i_iommu_identity_attach()
783 if (refcount_dec_and_test(&sun50i_domain->refcnt)) in sun50i_iommu_identity_attach()
784 sun50i_iommu_detach_domain(iommu, sun50i_domain); in sun50i_iommu_identity_attach()
801 struct sun50i_iommu *iommu; in sun50i_iommu_attach_device() local
803 iommu = sun50i_iommu_from_dev(dev); in sun50i_iommu_attach_device()
804 if (!iommu) in sun50i_iommu_attach_device()
805 return -ENODEV; in sun50i_iommu_attach_device()
807 dev_dbg(dev, "Attaching to IOMMU domain\n"); in sun50i_iommu_attach_device()
809 refcount_inc(&sun50i_domain->refcnt); in sun50i_iommu_attach_device()
811 if (iommu->domain == domain) in sun50i_iommu_attach_device()
816 sun50i_iommu_attach_domain(iommu, sun50i_domain); in sun50i_iommu_attach_device()
823 struct sun50i_iommu *iommu; in sun50i_iommu_probe_device() local
825 iommu = sun50i_iommu_from_dev(dev); in sun50i_iommu_probe_device()
826 if (!iommu) in sun50i_iommu_probe_device()
827 return ERR_PTR(-ENODEV); in sun50i_iommu_probe_device()
829 return &iommu->iommu; in sun50i_iommu_probe_device()
835 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); in sun50i_iommu_of_xlate()
836 unsigned id = args->args[0]; in sun50i_iommu_of_xlate()
862 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, in sun50i_iommu_report_fault() argument
866 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", in sun50i_iommu_report_fault()
869 if (iommu->domain) in sun50i_iommu_report_fault()
870 report_iommu_fault(iommu->domain, iommu->dev, iova, prot); in sun50i_iommu_report_fault()
872 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); in sun50i_iommu_report_fault()
874 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE); in sun50i_iommu_report_fault()
877 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, in sun50i_iommu_handle_pt_irq() argument
885 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_handle_pt_irq()
887 iova = iommu_read(iommu, addr_reg); in sun50i_iommu_handle_pt_irq()
888 blame = iommu_read(iommu, blame_reg); in sun50i_iommu_handle_pt_irq()
896 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); in sun50i_iommu_handle_pt_irq()
901 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) in sun50i_iommu_handle_perm_irq() argument
909 assert_spin_locked(&iommu->iommu_lock); in sun50i_iommu_handle_perm_irq()
911 blame = iommu_read(iommu, IOMMU_INT_STA_REG); in sun50i_iommu_handle_perm_irq()
913 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); in sun50i_iommu_handle_perm_irq()
914 aci = sun50i_get_pte_aci(iommu_read(iommu, in sun50i_iommu_handle_perm_irq()
919 * If we are in the read-only domain, then it means we in sun50i_iommu_handle_perm_irq()
927 * If we are in the write-only domain, then it means in sun50i_iommu_handle_perm_irq()
951 sun50i_iommu_report_fault(iommu, master, iova, dir); in sun50i_iommu_handle_perm_irq()
959 struct sun50i_iommu *iommu = dev_id; in sun50i_iommu_irq() local
961 spin_lock(&iommu->iommu_lock); in sun50i_iommu_irq()
963 status = iommu_read(iommu, IOMMU_INT_STA_REG); in sun50i_iommu_irq()
965 spin_unlock(&iommu->iommu_lock); in sun50i_iommu_irq()
969 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG); in sun50i_iommu_irq()
970 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG); in sun50i_iommu_irq()
973 sun50i_iommu_handle_pt_irq(iommu, in sun50i_iommu_irq()
977 sun50i_iommu_handle_pt_irq(iommu, in sun50i_iommu_irq()
981 sun50i_iommu_handle_perm_irq(iommu); in sun50i_iommu_irq()
983 iommu_write(iommu, IOMMU_INT_CLR_REG, status); in sun50i_iommu_irq()
986 iommu_write(iommu, IOMMU_RESET_REG, ~resets); in sun50i_iommu_irq()
987 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL); in sun50i_iommu_irq()
989 spin_unlock(&iommu->iommu_lock); in sun50i_iommu_irq()
996 struct sun50i_iommu *iommu; in sun50i_iommu_probe() local
999 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); in sun50i_iommu_probe()
1000 if (!iommu) in sun50i_iommu_probe()
1001 return -ENOMEM; in sun50i_iommu_probe()
1002 spin_lock_init(&iommu->iommu_lock); in sun50i_iommu_probe()
1003 iommu->domain = &sun50i_iommu_identity_domain; in sun50i_iommu_probe()
1004 platform_set_drvdata(pdev, iommu); in sun50i_iommu_probe()
1005 iommu->dev = &pdev->dev; in sun50i_iommu_probe()
1007 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), in sun50i_iommu_probe()
1011 if (!iommu->pt_pool) in sun50i_iommu_probe()
1012 return -ENOMEM; in sun50i_iommu_probe()
1014 iommu->base = devm_platform_ioremap_resource(pdev, 0); in sun50i_iommu_probe()
1015 if (IS_ERR(iommu->base)) { in sun50i_iommu_probe()
1016 ret = PTR_ERR(iommu->base); in sun50i_iommu_probe()
1026 iommu->clk = devm_clk_get(&pdev->dev, NULL); in sun50i_iommu_probe()
1027 if (IS_ERR(iommu->clk)) { in sun50i_iommu_probe()
1028 dev_err(&pdev->dev, "Couldn't get our clock.\n"); in sun50i_iommu_probe()
1029 ret = PTR_ERR(iommu->clk); in sun50i_iommu_probe()
1033 iommu->reset = devm_reset_control_get(&pdev->dev, NULL); in sun50i_iommu_probe()
1034 if (IS_ERR(iommu->reset)) { in sun50i_iommu_probe()
1035 dev_err(&pdev->dev, "Couldn't get our reset line.\n"); in sun50i_iommu_probe()
1036 ret = PTR_ERR(iommu->reset); in sun50i_iommu_probe()
1040 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, in sun50i_iommu_probe()
1041 NULL, dev_name(&pdev->dev)); in sun50i_iommu_probe()
1045 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev); in sun50i_iommu_probe()
1049 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0, in sun50i_iommu_probe()
1050 dev_name(&pdev->dev), iommu); in sun50i_iommu_probe()
1057 iommu_device_unregister(&iommu->iommu); in sun50i_iommu_probe()
1060 iommu_device_sysfs_remove(&iommu->iommu); in sun50i_iommu_probe()
1063 kmem_cache_destroy(iommu->pt_pool); in sun50i_iommu_probe()
1069 { .compatible = "allwinner,sun50i-h6-iommu", },
1070 { .compatible = "allwinner,sun50i-h616-iommu", },
1077 .name = "sun50i-iommu",
1084 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");