1c68a2921SDaniel Kurtz /* 2c68a2921SDaniel Kurtz * This program is free software; you can redistribute it and/or modify 3c68a2921SDaniel Kurtz * it under the terms of the GNU General Public License version 2 as 4c68a2921SDaniel Kurtz * published by the Free Software Foundation. 5c68a2921SDaniel Kurtz */ 6c68a2921SDaniel Kurtz 7c68a2921SDaniel Kurtz #include <asm/cacheflush.h> 8c68a2921SDaniel Kurtz #include <asm/pgtable.h> 9c68a2921SDaniel Kurtz #include <linux/compiler.h> 10c68a2921SDaniel Kurtz #include <linux/delay.h> 11c68a2921SDaniel Kurtz #include <linux/device.h> 12c68a2921SDaniel Kurtz #include <linux/errno.h> 13c68a2921SDaniel Kurtz #include <linux/interrupt.h> 14c68a2921SDaniel Kurtz #include <linux/io.h> 15c68a2921SDaniel Kurtz #include <linux/iommu.h> 16c68a2921SDaniel Kurtz #include <linux/jiffies.h> 17c68a2921SDaniel Kurtz #include <linux/list.h> 18c68a2921SDaniel Kurtz #include <linux/mm.h> 19c68a2921SDaniel Kurtz #include <linux/module.h> 20c68a2921SDaniel Kurtz #include <linux/of.h> 21c68a2921SDaniel Kurtz #include <linux/of_platform.h> 22c68a2921SDaniel Kurtz #include <linux/platform_device.h> 23c68a2921SDaniel Kurtz #include <linux/slab.h> 24c68a2921SDaniel Kurtz #include <linux/spinlock.h> 25c68a2921SDaniel Kurtz 26c68a2921SDaniel Kurtz /** MMU register offsets */ 27c68a2921SDaniel Kurtz #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ 28c68a2921SDaniel Kurtz #define RK_MMU_STATUS 0x04 29c68a2921SDaniel Kurtz #define RK_MMU_COMMAND 0x08 30c68a2921SDaniel Kurtz #define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */ 31c68a2921SDaniel Kurtz #define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */ 32c68a2921SDaniel Kurtz #define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */ 33c68a2921SDaniel Kurtz #define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */ 34c68a2921SDaniel Kurtz #define RK_MMU_INT_MASK 0x1C /* IRQ enable */ 35c68a2921SDaniel Kurtz #define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */ 36c68a2921SDaniel Kurtz #define RK_MMU_AUTO_GATING 0x24 37c68a2921SDaniel Kurtz 38c68a2921SDaniel Kurtz #define DTE_ADDR_DUMMY 0xCAFEBABE 39c68a2921SDaniel Kurtz #define FORCE_RESET_TIMEOUT 100 /* ms */ 40c68a2921SDaniel Kurtz 41c68a2921SDaniel Kurtz /* RK_MMU_STATUS fields */ 42c68a2921SDaniel Kurtz #define RK_MMU_STATUS_PAGING_ENABLED BIT(0) 43c68a2921SDaniel Kurtz #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1) 44c68a2921SDaniel Kurtz #define RK_MMU_STATUS_STALL_ACTIVE BIT(2) 45c68a2921SDaniel Kurtz #define RK_MMU_STATUS_IDLE BIT(3) 46c68a2921SDaniel Kurtz #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4) 47c68a2921SDaniel Kurtz #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5) 48c68a2921SDaniel Kurtz #define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31) 49c68a2921SDaniel Kurtz 50c68a2921SDaniel Kurtz /* RK_MMU_COMMAND command values */ 51c68a2921SDaniel Kurtz #define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */ 52c68a2921SDaniel Kurtz #define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */ 53c68a2921SDaniel Kurtz #define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */ 54c68a2921SDaniel Kurtz #define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */ 55c68a2921SDaniel Kurtz #define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */ 56c68a2921SDaniel Kurtz #define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */ 57c68a2921SDaniel Kurtz #define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */ 58c68a2921SDaniel Kurtz 59c68a2921SDaniel Kurtz /* RK_MMU_INT_* register fields */ 60c68a2921SDaniel Kurtz #define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */ 61c68a2921SDaniel Kurtz #define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */ 62c68a2921SDaniel Kurtz #define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR) 63c68a2921SDaniel Kurtz 64c68a2921SDaniel Kurtz #define NUM_DT_ENTRIES 1024 65c68a2921SDaniel Kurtz #define NUM_PT_ENTRIES 1024 66c68a2921SDaniel Kurtz 67c68a2921SDaniel Kurtz #define SPAGE_ORDER 12 68c68a2921SDaniel Kurtz #define SPAGE_SIZE (1 << SPAGE_ORDER) 69c68a2921SDaniel Kurtz 70c68a2921SDaniel Kurtz /* 71c68a2921SDaniel Kurtz * Support mapping any size that fits in one page table: 72c68a2921SDaniel Kurtz * 4 KiB to 4 MiB 73c68a2921SDaniel Kurtz */ 74c68a2921SDaniel Kurtz #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 75c68a2921SDaniel Kurtz 76c68a2921SDaniel Kurtz #define IOMMU_REG_POLL_COUNT_FAST 1000 77c68a2921SDaniel Kurtz 78c68a2921SDaniel Kurtz struct rk_iommu_domain { 79c68a2921SDaniel Kurtz struct list_head iommus; 80c68a2921SDaniel Kurtz u32 *dt; /* page directory table */ 81c68a2921SDaniel Kurtz spinlock_t iommus_lock; /* lock for iommus list */ 82c68a2921SDaniel Kurtz spinlock_t dt_lock; /* lock for modifying page directory table */ 83*bcd516a3SJoerg Roedel 84*bcd516a3SJoerg Roedel struct iommu_domain domain; 85c68a2921SDaniel Kurtz }; 86c68a2921SDaniel Kurtz 87c68a2921SDaniel Kurtz struct rk_iommu { 88c68a2921SDaniel Kurtz struct device *dev; 89c68a2921SDaniel Kurtz void __iomem *base; 90c68a2921SDaniel Kurtz int irq; 91c68a2921SDaniel Kurtz struct list_head node; /* entry in rk_iommu_domain.iommus */ 92c68a2921SDaniel Kurtz struct iommu_domain *domain; /* domain to which iommu is attached */ 93c68a2921SDaniel Kurtz }; 94c68a2921SDaniel Kurtz 95c68a2921SDaniel Kurtz static inline void rk_table_flush(u32 *va, unsigned int count) 96c68a2921SDaniel Kurtz { 97c68a2921SDaniel Kurtz phys_addr_t pa_start = virt_to_phys(va); 98c68a2921SDaniel Kurtz phys_addr_t pa_end = virt_to_phys(va + count); 99c68a2921SDaniel Kurtz size_t size = pa_end - pa_start; 100c68a2921SDaniel Kurtz 101c68a2921SDaniel Kurtz __cpuc_flush_dcache_area(va, size); 102c68a2921SDaniel Kurtz outer_flush_range(pa_start, pa_end); 103c68a2921SDaniel Kurtz } 104c68a2921SDaniel Kurtz 105*bcd516a3SJoerg Roedel static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) 106*bcd516a3SJoerg Roedel { 107*bcd516a3SJoerg Roedel return container_of(dom, struct rk_iommu_domain, domain); 108*bcd516a3SJoerg Roedel } 109*bcd516a3SJoerg Roedel 110c68a2921SDaniel Kurtz /** 111c68a2921SDaniel Kurtz * Inspired by _wait_for in intel_drv.h 112c68a2921SDaniel Kurtz * This is NOT safe for use in interrupt context. 113c68a2921SDaniel Kurtz * 114c68a2921SDaniel Kurtz * Note that it's important that we check the condition again after having 115c68a2921SDaniel Kurtz * timed out, since the timeout could be due to preemption or similar and 116c68a2921SDaniel Kurtz * we've never had a chance to check the condition before the timeout. 117c68a2921SDaniel Kurtz */ 118c68a2921SDaniel Kurtz #define rk_wait_for(COND, MS) ({ \ 119c68a2921SDaniel Kurtz unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ 120c68a2921SDaniel Kurtz int ret__ = 0; \ 121c68a2921SDaniel Kurtz while (!(COND)) { \ 122c68a2921SDaniel Kurtz if (time_after(jiffies, timeout__)) { \ 123c68a2921SDaniel Kurtz ret__ = (COND) ? 0 : -ETIMEDOUT; \ 124c68a2921SDaniel Kurtz break; \ 125c68a2921SDaniel Kurtz } \ 126c68a2921SDaniel Kurtz usleep_range(50, 100); \ 127c68a2921SDaniel Kurtz } \ 128c68a2921SDaniel Kurtz ret__; \ 129c68a2921SDaniel Kurtz }) 130c68a2921SDaniel Kurtz 131c68a2921SDaniel Kurtz /* 132c68a2921SDaniel Kurtz * The Rockchip rk3288 iommu uses a 2-level page table. 133c68a2921SDaniel Kurtz * The first level is the "Directory Table" (DT). 134c68a2921SDaniel Kurtz * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing 135c68a2921SDaniel Kurtz * to a "Page Table". 136c68a2921SDaniel Kurtz * The second level is the 1024 Page Tables (PT). 137c68a2921SDaniel Kurtz * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to 138c68a2921SDaniel Kurtz * a 4 KB page of physical memory. 139c68a2921SDaniel Kurtz * 140c68a2921SDaniel Kurtz * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). 141c68a2921SDaniel Kurtz * Each iommu device has a MMU_DTE_ADDR register that contains the physical 142c68a2921SDaniel Kurtz * address of the start of the DT page. 143c68a2921SDaniel Kurtz * 144c68a2921SDaniel Kurtz * The structure of the page table is as follows: 145c68a2921SDaniel Kurtz * 146c68a2921SDaniel Kurtz * DT 147c68a2921SDaniel Kurtz * MMU_DTE_ADDR -> +-----+ 148c68a2921SDaniel Kurtz * | | 149c68a2921SDaniel Kurtz * +-----+ PT 150c68a2921SDaniel Kurtz * | DTE | -> +-----+ 151c68a2921SDaniel Kurtz * +-----+ | | Memory 152c68a2921SDaniel Kurtz * | | +-----+ Page 153c68a2921SDaniel Kurtz * | | | PTE | -> +-----+ 154c68a2921SDaniel Kurtz * +-----+ +-----+ | | 155c68a2921SDaniel Kurtz * | | | | 156c68a2921SDaniel Kurtz * | | | | 157c68a2921SDaniel Kurtz * +-----+ | | 158c68a2921SDaniel Kurtz * | | 159c68a2921SDaniel Kurtz * | | 160c68a2921SDaniel Kurtz * +-----+ 161c68a2921SDaniel Kurtz */ 162c68a2921SDaniel Kurtz 163c68a2921SDaniel Kurtz /* 164c68a2921SDaniel Kurtz * Each DTE has a PT address and a valid bit: 165c68a2921SDaniel Kurtz * +---------------------+-----------+-+ 166c68a2921SDaniel Kurtz * | PT address | Reserved |V| 167c68a2921SDaniel Kurtz * +---------------------+-----------+-+ 168c68a2921SDaniel Kurtz * 31:12 - PT address (PTs always starts on a 4 KB boundary) 169c68a2921SDaniel Kurtz * 11: 1 - Reserved 170c68a2921SDaniel Kurtz * 0 - 1 if PT @ PT address is valid 171c68a2921SDaniel Kurtz */ 172c68a2921SDaniel Kurtz #define RK_DTE_PT_ADDRESS_MASK 0xfffff000 173c68a2921SDaniel Kurtz #define RK_DTE_PT_VALID BIT(0) 174c68a2921SDaniel Kurtz 175c68a2921SDaniel Kurtz static inline phys_addr_t rk_dte_pt_address(u32 dte) 176c68a2921SDaniel Kurtz { 177c68a2921SDaniel Kurtz return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK; 178c68a2921SDaniel Kurtz } 179c68a2921SDaniel Kurtz 180c68a2921SDaniel Kurtz static inline bool rk_dte_is_pt_valid(u32 dte) 181c68a2921SDaniel Kurtz { 182c68a2921SDaniel Kurtz return dte & RK_DTE_PT_VALID; 183c68a2921SDaniel Kurtz } 184c68a2921SDaniel Kurtz 185c68a2921SDaniel Kurtz static u32 rk_mk_dte(u32 *pt) 186c68a2921SDaniel Kurtz { 187c68a2921SDaniel Kurtz phys_addr_t pt_phys = virt_to_phys(pt); 188c68a2921SDaniel Kurtz return (pt_phys & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; 189c68a2921SDaniel Kurtz } 190c68a2921SDaniel Kurtz 191c68a2921SDaniel Kurtz /* 192c68a2921SDaniel Kurtz * Each PTE has a Page address, some flags and a valid bit: 193c68a2921SDaniel Kurtz * +---------------------+---+-------+-+ 194c68a2921SDaniel Kurtz * | Page address |Rsv| Flags |V| 195c68a2921SDaniel Kurtz * +---------------------+---+-------+-+ 196c68a2921SDaniel Kurtz * 31:12 - Page address (Pages always start on a 4 KB boundary) 197c68a2921SDaniel Kurtz * 11: 9 - Reserved 198c68a2921SDaniel Kurtz * 8: 1 - Flags 199c68a2921SDaniel Kurtz * 8 - Read allocate - allocate cache space on read misses 200c68a2921SDaniel Kurtz * 7 - Read cache - enable cache & prefetch of data 201c68a2921SDaniel Kurtz * 6 - Write buffer - enable delaying writes on their way to memory 202c68a2921SDaniel Kurtz * 5 - Write allocate - allocate cache space on write misses 203c68a2921SDaniel Kurtz * 4 - Write cache - different writes can be merged together 204c68a2921SDaniel Kurtz * 3 - Override cache attributes 205c68a2921SDaniel Kurtz * if 1, bits 4-8 control cache attributes 206c68a2921SDaniel Kurtz * if 0, the system bus defaults are used 207c68a2921SDaniel Kurtz * 2 - Writable 208c68a2921SDaniel Kurtz * 1 - Readable 209c68a2921SDaniel Kurtz * 0 - 1 if Page @ Page address is valid 210c68a2921SDaniel Kurtz */ 211c68a2921SDaniel Kurtz #define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000 212c68a2921SDaniel Kurtz #define RK_PTE_PAGE_FLAGS_MASK 0x000001fe 213c68a2921SDaniel Kurtz #define RK_PTE_PAGE_WRITABLE BIT(2) 214c68a2921SDaniel Kurtz #define RK_PTE_PAGE_READABLE BIT(1) 215c68a2921SDaniel Kurtz #define RK_PTE_PAGE_VALID BIT(0) 216c68a2921SDaniel Kurtz 217c68a2921SDaniel Kurtz static inline phys_addr_t rk_pte_page_address(u32 pte) 218c68a2921SDaniel Kurtz { 219c68a2921SDaniel Kurtz return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK; 220c68a2921SDaniel Kurtz } 221c68a2921SDaniel Kurtz 222c68a2921SDaniel Kurtz static inline bool rk_pte_is_page_valid(u32 pte) 223c68a2921SDaniel Kurtz { 224c68a2921SDaniel Kurtz return pte & RK_PTE_PAGE_VALID; 225c68a2921SDaniel Kurtz } 226c68a2921SDaniel Kurtz 227c68a2921SDaniel Kurtz /* TODO: set cache flags per prot IOMMU_CACHE */ 228c68a2921SDaniel Kurtz static u32 rk_mk_pte(phys_addr_t page, int prot) 229c68a2921SDaniel Kurtz { 230c68a2921SDaniel Kurtz u32 flags = 0; 231c68a2921SDaniel Kurtz flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; 232c68a2921SDaniel Kurtz flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; 233c68a2921SDaniel Kurtz page &= RK_PTE_PAGE_ADDRESS_MASK; 234c68a2921SDaniel Kurtz return page | flags | RK_PTE_PAGE_VALID; 235c68a2921SDaniel Kurtz } 236c68a2921SDaniel Kurtz 237c68a2921SDaniel Kurtz static u32 rk_mk_pte_invalid(u32 pte) 238c68a2921SDaniel Kurtz { 239c68a2921SDaniel Kurtz return pte & ~RK_PTE_PAGE_VALID; 240c68a2921SDaniel Kurtz } 241c68a2921SDaniel Kurtz 242c68a2921SDaniel Kurtz /* 243c68a2921SDaniel Kurtz * rk3288 iova (IOMMU Virtual Address) format 244c68a2921SDaniel Kurtz * 31 22.21 12.11 0 245c68a2921SDaniel Kurtz * +-----------+-----------+-------------+ 246c68a2921SDaniel Kurtz * | DTE index | PTE index | Page offset | 247c68a2921SDaniel Kurtz * +-----------+-----------+-------------+ 248c68a2921SDaniel Kurtz * 31:22 - DTE index - index of DTE in DT 249c68a2921SDaniel Kurtz * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address 250c68a2921SDaniel Kurtz * 11: 0 - Page offset - offset into page @ PTE.page_address 251c68a2921SDaniel Kurtz */ 252c68a2921SDaniel Kurtz #define RK_IOVA_DTE_MASK 0xffc00000 253c68a2921SDaniel Kurtz #define RK_IOVA_DTE_SHIFT 22 254c68a2921SDaniel Kurtz #define RK_IOVA_PTE_MASK 0x003ff000 255c68a2921SDaniel Kurtz #define RK_IOVA_PTE_SHIFT 12 256c68a2921SDaniel Kurtz #define RK_IOVA_PAGE_MASK 0x00000fff 257c68a2921SDaniel Kurtz #define RK_IOVA_PAGE_SHIFT 0 258c68a2921SDaniel Kurtz 259c68a2921SDaniel Kurtz static u32 rk_iova_dte_index(dma_addr_t iova) 260c68a2921SDaniel Kurtz { 261c68a2921SDaniel Kurtz return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT; 262c68a2921SDaniel Kurtz } 263c68a2921SDaniel Kurtz 264c68a2921SDaniel Kurtz static u32 rk_iova_pte_index(dma_addr_t iova) 265c68a2921SDaniel Kurtz { 266c68a2921SDaniel Kurtz return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT; 267c68a2921SDaniel Kurtz } 268c68a2921SDaniel Kurtz 269c68a2921SDaniel Kurtz static u32 rk_iova_page_offset(dma_addr_t iova) 270c68a2921SDaniel Kurtz { 271c68a2921SDaniel Kurtz return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; 272c68a2921SDaniel Kurtz } 273c68a2921SDaniel Kurtz 274c68a2921SDaniel Kurtz static u32 rk_iommu_read(struct rk_iommu *iommu, u32 offset) 275c68a2921SDaniel Kurtz { 276c68a2921SDaniel Kurtz return readl(iommu->base + offset); 277c68a2921SDaniel Kurtz } 278c68a2921SDaniel Kurtz 279c68a2921SDaniel Kurtz static void rk_iommu_write(struct rk_iommu *iommu, u32 offset, u32 value) 280c68a2921SDaniel Kurtz { 281c68a2921SDaniel Kurtz writel(value, iommu->base + offset); 282c68a2921SDaniel Kurtz } 283c68a2921SDaniel Kurtz 284c68a2921SDaniel Kurtz static void rk_iommu_command(struct rk_iommu *iommu, u32 command) 285c68a2921SDaniel Kurtz { 286c68a2921SDaniel Kurtz writel(command, iommu->base + RK_MMU_COMMAND); 287c68a2921SDaniel Kurtz } 288c68a2921SDaniel Kurtz 289c68a2921SDaniel Kurtz static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, 290c68a2921SDaniel Kurtz size_t size) 291c68a2921SDaniel Kurtz { 292c68a2921SDaniel Kurtz dma_addr_t iova_end = iova + size; 293c68a2921SDaniel Kurtz /* 294c68a2921SDaniel Kurtz * TODO(djkurtz): Figure out when it is more efficient to shootdown the 295c68a2921SDaniel Kurtz * entire iotlb rather than iterate over individual iovas. 296c68a2921SDaniel Kurtz */ 297c68a2921SDaniel Kurtz for (; iova < iova_end; iova += SPAGE_SIZE) 298c68a2921SDaniel Kurtz rk_iommu_write(iommu, RK_MMU_ZAP_ONE_LINE, iova); 299c68a2921SDaniel Kurtz } 300c68a2921SDaniel Kurtz 301c68a2921SDaniel Kurtz static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) 302c68a2921SDaniel Kurtz { 303c68a2921SDaniel Kurtz return rk_iommu_read(iommu, RK_MMU_STATUS) & RK_MMU_STATUS_STALL_ACTIVE; 304c68a2921SDaniel Kurtz } 305c68a2921SDaniel Kurtz 306c68a2921SDaniel Kurtz static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) 307c68a2921SDaniel Kurtz { 308c68a2921SDaniel Kurtz return rk_iommu_read(iommu, RK_MMU_STATUS) & 309c68a2921SDaniel Kurtz RK_MMU_STATUS_PAGING_ENABLED; 310c68a2921SDaniel Kurtz } 311c68a2921SDaniel Kurtz 312c68a2921SDaniel Kurtz static int rk_iommu_enable_stall(struct rk_iommu *iommu) 313c68a2921SDaniel Kurtz { 314c68a2921SDaniel Kurtz int ret; 315c68a2921SDaniel Kurtz 316c68a2921SDaniel Kurtz if (rk_iommu_is_stall_active(iommu)) 317c68a2921SDaniel Kurtz return 0; 318c68a2921SDaniel Kurtz 319c68a2921SDaniel Kurtz /* Stall can only be enabled if paging is enabled */ 320c68a2921SDaniel Kurtz if (!rk_iommu_is_paging_enabled(iommu)) 321c68a2921SDaniel Kurtz return 0; 322c68a2921SDaniel Kurtz 323c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); 324c68a2921SDaniel Kurtz 325c68a2921SDaniel Kurtz ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); 326c68a2921SDaniel Kurtz if (ret) 327c68a2921SDaniel Kurtz dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", 328c68a2921SDaniel Kurtz rk_iommu_read(iommu, RK_MMU_STATUS)); 329c68a2921SDaniel Kurtz 330c68a2921SDaniel Kurtz return ret; 331c68a2921SDaniel Kurtz } 332c68a2921SDaniel Kurtz 333c68a2921SDaniel Kurtz static int rk_iommu_disable_stall(struct rk_iommu *iommu) 334c68a2921SDaniel Kurtz { 335c68a2921SDaniel Kurtz int ret; 336c68a2921SDaniel Kurtz 337c68a2921SDaniel Kurtz if (!rk_iommu_is_stall_active(iommu)) 338c68a2921SDaniel Kurtz return 0; 339c68a2921SDaniel Kurtz 340c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); 341c68a2921SDaniel Kurtz 342c68a2921SDaniel Kurtz ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); 343c68a2921SDaniel Kurtz if (ret) 344c68a2921SDaniel Kurtz dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", 345c68a2921SDaniel Kurtz rk_iommu_read(iommu, RK_MMU_STATUS)); 346c68a2921SDaniel Kurtz 347c68a2921SDaniel Kurtz return ret; 348c68a2921SDaniel Kurtz } 349c68a2921SDaniel Kurtz 350c68a2921SDaniel Kurtz static int rk_iommu_enable_paging(struct rk_iommu *iommu) 351c68a2921SDaniel Kurtz { 352c68a2921SDaniel Kurtz int ret; 353c68a2921SDaniel Kurtz 354c68a2921SDaniel Kurtz if (rk_iommu_is_paging_enabled(iommu)) 355c68a2921SDaniel Kurtz return 0; 356c68a2921SDaniel Kurtz 357c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); 358c68a2921SDaniel Kurtz 359c68a2921SDaniel Kurtz ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); 360c68a2921SDaniel Kurtz if (ret) 361c68a2921SDaniel Kurtz dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", 362c68a2921SDaniel Kurtz rk_iommu_read(iommu, RK_MMU_STATUS)); 363c68a2921SDaniel Kurtz 364c68a2921SDaniel Kurtz return ret; 365c68a2921SDaniel Kurtz } 366c68a2921SDaniel Kurtz 367c68a2921SDaniel Kurtz static int rk_iommu_disable_paging(struct rk_iommu *iommu) 368c68a2921SDaniel Kurtz { 369c68a2921SDaniel Kurtz int ret; 370c68a2921SDaniel Kurtz 371c68a2921SDaniel Kurtz if (!rk_iommu_is_paging_enabled(iommu)) 372c68a2921SDaniel Kurtz return 0; 373c68a2921SDaniel Kurtz 374c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); 375c68a2921SDaniel Kurtz 376c68a2921SDaniel Kurtz ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); 377c68a2921SDaniel Kurtz if (ret) 378c68a2921SDaniel Kurtz dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", 379c68a2921SDaniel Kurtz rk_iommu_read(iommu, RK_MMU_STATUS)); 380c68a2921SDaniel Kurtz 381c68a2921SDaniel Kurtz return ret; 382c68a2921SDaniel Kurtz } 383c68a2921SDaniel Kurtz 384c68a2921SDaniel Kurtz static int rk_iommu_force_reset(struct rk_iommu *iommu) 385c68a2921SDaniel Kurtz { 386c68a2921SDaniel Kurtz int ret; 387c68a2921SDaniel Kurtz u32 dte_addr; 388c68a2921SDaniel Kurtz 389c68a2921SDaniel Kurtz /* 390c68a2921SDaniel Kurtz * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY 391c68a2921SDaniel Kurtz * and verifying that upper 5 nybbles are read back. 392c68a2921SDaniel Kurtz */ 393c68a2921SDaniel Kurtz rk_iommu_write(iommu, RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); 394c68a2921SDaniel Kurtz 395c68a2921SDaniel Kurtz dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 396c68a2921SDaniel Kurtz if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { 397c68a2921SDaniel Kurtz dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); 398c68a2921SDaniel Kurtz return -EFAULT; 399c68a2921SDaniel Kurtz } 400c68a2921SDaniel Kurtz 401c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); 402c68a2921SDaniel Kurtz 403c68a2921SDaniel Kurtz ret = rk_wait_for(rk_iommu_read(iommu, RK_MMU_DTE_ADDR) == 0x00000000, 404c68a2921SDaniel Kurtz FORCE_RESET_TIMEOUT); 405c68a2921SDaniel Kurtz if (ret) 406c68a2921SDaniel Kurtz dev_err(iommu->dev, "FORCE_RESET command timed out\n"); 407c68a2921SDaniel Kurtz 408c68a2921SDaniel Kurtz return ret; 409c68a2921SDaniel Kurtz } 410c68a2921SDaniel Kurtz 411c68a2921SDaniel Kurtz static void log_iova(struct rk_iommu *iommu, dma_addr_t iova) 412c68a2921SDaniel Kurtz { 413c68a2921SDaniel Kurtz u32 dte_index, pte_index, page_offset; 414c68a2921SDaniel Kurtz u32 mmu_dte_addr; 415c68a2921SDaniel Kurtz phys_addr_t mmu_dte_addr_phys, dte_addr_phys; 416c68a2921SDaniel Kurtz u32 *dte_addr; 417c68a2921SDaniel Kurtz u32 dte; 418c68a2921SDaniel Kurtz phys_addr_t pte_addr_phys = 0; 419c68a2921SDaniel Kurtz u32 *pte_addr = NULL; 420c68a2921SDaniel Kurtz u32 pte = 0; 421c68a2921SDaniel Kurtz phys_addr_t page_addr_phys = 0; 422c68a2921SDaniel Kurtz u32 page_flags = 0; 423c68a2921SDaniel Kurtz 424c68a2921SDaniel Kurtz dte_index = rk_iova_dte_index(iova); 425c68a2921SDaniel Kurtz pte_index = rk_iova_pte_index(iova); 426c68a2921SDaniel Kurtz page_offset = rk_iova_page_offset(iova); 427c68a2921SDaniel Kurtz 428c68a2921SDaniel Kurtz mmu_dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 429c68a2921SDaniel Kurtz mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr; 430c68a2921SDaniel Kurtz 431c68a2921SDaniel Kurtz dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); 432c68a2921SDaniel Kurtz dte_addr = phys_to_virt(dte_addr_phys); 433c68a2921SDaniel Kurtz dte = *dte_addr; 434c68a2921SDaniel Kurtz 435c68a2921SDaniel Kurtz if (!rk_dte_is_pt_valid(dte)) 436c68a2921SDaniel Kurtz goto print_it; 437c68a2921SDaniel Kurtz 438c68a2921SDaniel Kurtz pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4); 439c68a2921SDaniel Kurtz pte_addr = phys_to_virt(pte_addr_phys); 440c68a2921SDaniel Kurtz pte = *pte_addr; 441c68a2921SDaniel Kurtz 442c68a2921SDaniel Kurtz if (!rk_pte_is_page_valid(pte)) 443c68a2921SDaniel Kurtz goto print_it; 444c68a2921SDaniel Kurtz 445c68a2921SDaniel Kurtz page_addr_phys = rk_pte_page_address(pte) + page_offset; 446c68a2921SDaniel Kurtz page_flags = pte & RK_PTE_PAGE_FLAGS_MASK; 447c68a2921SDaniel Kurtz 448c68a2921SDaniel Kurtz print_it: 449c68a2921SDaniel Kurtz dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n", 450c68a2921SDaniel Kurtz &iova, dte_index, pte_index, page_offset); 451c68a2921SDaniel Kurtz dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n", 452c68a2921SDaniel Kurtz &mmu_dte_addr_phys, &dte_addr_phys, dte, 453c68a2921SDaniel Kurtz rk_dte_is_pt_valid(dte), &pte_addr_phys, pte, 454c68a2921SDaniel Kurtz rk_pte_is_page_valid(pte), &page_addr_phys, page_flags); 455c68a2921SDaniel Kurtz } 456c68a2921SDaniel Kurtz 457c68a2921SDaniel Kurtz static irqreturn_t rk_iommu_irq(int irq, void *dev_id) 458c68a2921SDaniel Kurtz { 459c68a2921SDaniel Kurtz struct rk_iommu *iommu = dev_id; 460c68a2921SDaniel Kurtz u32 status; 461c68a2921SDaniel Kurtz u32 int_status; 462c68a2921SDaniel Kurtz dma_addr_t iova; 463c68a2921SDaniel Kurtz 464c68a2921SDaniel Kurtz int_status = rk_iommu_read(iommu, RK_MMU_INT_STATUS); 465c68a2921SDaniel Kurtz if (int_status == 0) 466c68a2921SDaniel Kurtz return IRQ_NONE; 467c68a2921SDaniel Kurtz 468c68a2921SDaniel Kurtz iova = rk_iommu_read(iommu, RK_MMU_PAGE_FAULT_ADDR); 469c68a2921SDaniel Kurtz 470c68a2921SDaniel Kurtz if (int_status & RK_MMU_IRQ_PAGE_FAULT) { 471c68a2921SDaniel Kurtz int flags; 472c68a2921SDaniel Kurtz 473c68a2921SDaniel Kurtz status = rk_iommu_read(iommu, RK_MMU_STATUS); 474c68a2921SDaniel Kurtz flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? 475c68a2921SDaniel Kurtz IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 476c68a2921SDaniel Kurtz 477c68a2921SDaniel Kurtz dev_err(iommu->dev, "Page fault at %pad of type %s\n", 478c68a2921SDaniel Kurtz &iova, 479c68a2921SDaniel Kurtz (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); 480c68a2921SDaniel Kurtz 481c68a2921SDaniel Kurtz log_iova(iommu, iova); 482c68a2921SDaniel Kurtz 483c68a2921SDaniel Kurtz /* 484c68a2921SDaniel Kurtz * Report page fault to any installed handlers. 485c68a2921SDaniel Kurtz * Ignore the return code, though, since we always zap cache 486c68a2921SDaniel Kurtz * and clear the page fault anyway. 487c68a2921SDaniel Kurtz */ 488c68a2921SDaniel Kurtz if (iommu->domain) 489c68a2921SDaniel Kurtz report_iommu_fault(iommu->domain, iommu->dev, iova, 490c68a2921SDaniel Kurtz flags); 491c68a2921SDaniel Kurtz else 492c68a2921SDaniel Kurtz dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); 493c68a2921SDaniel Kurtz 494c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 495c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_PAGE_FAULT_DONE); 496c68a2921SDaniel Kurtz } 497c68a2921SDaniel Kurtz 498c68a2921SDaniel Kurtz if (int_status & RK_MMU_IRQ_BUS_ERROR) 499c68a2921SDaniel Kurtz dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); 500c68a2921SDaniel Kurtz 501c68a2921SDaniel Kurtz if (int_status & ~RK_MMU_IRQ_MASK) 502c68a2921SDaniel Kurtz dev_err(iommu->dev, "unexpected int_status: %#08x\n", 503c68a2921SDaniel Kurtz int_status); 504c68a2921SDaniel Kurtz 505c68a2921SDaniel Kurtz rk_iommu_write(iommu, RK_MMU_INT_CLEAR, int_status); 506c68a2921SDaniel Kurtz 507c68a2921SDaniel Kurtz return IRQ_HANDLED; 508c68a2921SDaniel Kurtz } 509c68a2921SDaniel Kurtz 510c68a2921SDaniel Kurtz static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, 511c68a2921SDaniel Kurtz dma_addr_t iova) 512c68a2921SDaniel Kurtz { 513*bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 514c68a2921SDaniel Kurtz unsigned long flags; 515c68a2921SDaniel Kurtz phys_addr_t pt_phys, phys = 0; 516c68a2921SDaniel Kurtz u32 dte, pte; 517c68a2921SDaniel Kurtz u32 *page_table; 518c68a2921SDaniel Kurtz 519c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->dt_lock, flags); 520c68a2921SDaniel Kurtz 521c68a2921SDaniel Kurtz dte = rk_domain->dt[rk_iova_dte_index(iova)]; 522c68a2921SDaniel Kurtz if (!rk_dte_is_pt_valid(dte)) 523c68a2921SDaniel Kurtz goto out; 524c68a2921SDaniel Kurtz 525c68a2921SDaniel Kurtz pt_phys = rk_dte_pt_address(dte); 526c68a2921SDaniel Kurtz page_table = (u32 *)phys_to_virt(pt_phys); 527c68a2921SDaniel Kurtz pte = page_table[rk_iova_pte_index(iova)]; 528c68a2921SDaniel Kurtz if (!rk_pte_is_page_valid(pte)) 529c68a2921SDaniel Kurtz goto out; 530c68a2921SDaniel Kurtz 531c68a2921SDaniel Kurtz phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova); 532c68a2921SDaniel Kurtz out: 533c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 534c68a2921SDaniel Kurtz 535c68a2921SDaniel Kurtz return phys; 536c68a2921SDaniel Kurtz } 537c68a2921SDaniel Kurtz 538c68a2921SDaniel Kurtz static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, 539c68a2921SDaniel Kurtz dma_addr_t iova, size_t size) 540c68a2921SDaniel Kurtz { 541c68a2921SDaniel Kurtz struct list_head *pos; 542c68a2921SDaniel Kurtz unsigned long flags; 543c68a2921SDaniel Kurtz 544c68a2921SDaniel Kurtz /* shootdown these iova from all iommus using this domain */ 545c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->iommus_lock, flags); 546c68a2921SDaniel Kurtz list_for_each(pos, &rk_domain->iommus) { 547c68a2921SDaniel Kurtz struct rk_iommu *iommu; 548c68a2921SDaniel Kurtz iommu = list_entry(pos, struct rk_iommu, node); 549c68a2921SDaniel Kurtz rk_iommu_zap_lines(iommu, iova, size); 550c68a2921SDaniel Kurtz } 551c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 552c68a2921SDaniel Kurtz } 553c68a2921SDaniel Kurtz 554c68a2921SDaniel Kurtz static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 555c68a2921SDaniel Kurtz dma_addr_t iova) 556c68a2921SDaniel Kurtz { 557c68a2921SDaniel Kurtz u32 *page_table, *dte_addr; 558c68a2921SDaniel Kurtz u32 dte; 559c68a2921SDaniel Kurtz phys_addr_t pt_phys; 560c68a2921SDaniel Kurtz 561c68a2921SDaniel Kurtz assert_spin_locked(&rk_domain->dt_lock); 562c68a2921SDaniel Kurtz 563c68a2921SDaniel Kurtz dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)]; 564c68a2921SDaniel Kurtz dte = *dte_addr; 565c68a2921SDaniel Kurtz if (rk_dte_is_pt_valid(dte)) 566c68a2921SDaniel Kurtz goto done; 567c68a2921SDaniel Kurtz 568c68a2921SDaniel Kurtz page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 569c68a2921SDaniel Kurtz if (!page_table) 570c68a2921SDaniel Kurtz return ERR_PTR(-ENOMEM); 571c68a2921SDaniel Kurtz 572c68a2921SDaniel Kurtz dte = rk_mk_dte(page_table); 573c68a2921SDaniel Kurtz *dte_addr = dte; 574c68a2921SDaniel Kurtz 575c68a2921SDaniel Kurtz rk_table_flush(page_table, NUM_PT_ENTRIES); 576c68a2921SDaniel Kurtz rk_table_flush(dte_addr, 1); 577c68a2921SDaniel Kurtz 578c68a2921SDaniel Kurtz /* 579c68a2921SDaniel Kurtz * Zap the first iova of newly allocated page table so iommu evicts 580c68a2921SDaniel Kurtz * old cached value of new dte from the iotlb. 581c68a2921SDaniel Kurtz */ 582c68a2921SDaniel Kurtz rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); 583c68a2921SDaniel Kurtz 584c68a2921SDaniel Kurtz done: 585c68a2921SDaniel Kurtz pt_phys = rk_dte_pt_address(dte); 586c68a2921SDaniel Kurtz return (u32 *)phys_to_virt(pt_phys); 587c68a2921SDaniel Kurtz } 588c68a2921SDaniel Kurtz 589c68a2921SDaniel Kurtz static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, 590c68a2921SDaniel Kurtz u32 *pte_addr, dma_addr_t iova, size_t size) 591c68a2921SDaniel Kurtz { 592c68a2921SDaniel Kurtz unsigned int pte_count; 593c68a2921SDaniel Kurtz unsigned int pte_total = size / SPAGE_SIZE; 594c68a2921SDaniel Kurtz 595c68a2921SDaniel Kurtz assert_spin_locked(&rk_domain->dt_lock); 596c68a2921SDaniel Kurtz 597c68a2921SDaniel Kurtz for (pte_count = 0; pte_count < pte_total; pte_count++) { 598c68a2921SDaniel Kurtz u32 pte = pte_addr[pte_count]; 599c68a2921SDaniel Kurtz if (!rk_pte_is_page_valid(pte)) 600c68a2921SDaniel Kurtz break; 601c68a2921SDaniel Kurtz 602c68a2921SDaniel Kurtz pte_addr[pte_count] = rk_mk_pte_invalid(pte); 603c68a2921SDaniel Kurtz } 604c68a2921SDaniel Kurtz 605c68a2921SDaniel Kurtz rk_table_flush(pte_addr, pte_count); 606c68a2921SDaniel Kurtz 607c68a2921SDaniel Kurtz return pte_count * SPAGE_SIZE; 608c68a2921SDaniel Kurtz } 609c68a2921SDaniel Kurtz 610c68a2921SDaniel Kurtz static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, 611c68a2921SDaniel Kurtz dma_addr_t iova, phys_addr_t paddr, size_t size, 612c68a2921SDaniel Kurtz int prot) 613c68a2921SDaniel Kurtz { 614c68a2921SDaniel Kurtz unsigned int pte_count; 615c68a2921SDaniel Kurtz unsigned int pte_total = size / SPAGE_SIZE; 616c68a2921SDaniel Kurtz phys_addr_t page_phys; 617c68a2921SDaniel Kurtz 618c68a2921SDaniel Kurtz assert_spin_locked(&rk_domain->dt_lock); 619c68a2921SDaniel Kurtz 620c68a2921SDaniel Kurtz for (pte_count = 0; pte_count < pte_total; pte_count++) { 621c68a2921SDaniel Kurtz u32 pte = pte_addr[pte_count]; 622c68a2921SDaniel Kurtz 623c68a2921SDaniel Kurtz if (rk_pte_is_page_valid(pte)) 624c68a2921SDaniel Kurtz goto unwind; 625c68a2921SDaniel Kurtz 626c68a2921SDaniel Kurtz pte_addr[pte_count] = rk_mk_pte(paddr, prot); 627c68a2921SDaniel Kurtz 628c68a2921SDaniel Kurtz paddr += SPAGE_SIZE; 629c68a2921SDaniel Kurtz } 630c68a2921SDaniel Kurtz 631c68a2921SDaniel Kurtz rk_table_flush(pte_addr, pte_count); 632c68a2921SDaniel Kurtz 633c68a2921SDaniel Kurtz return 0; 634c68a2921SDaniel Kurtz unwind: 635c68a2921SDaniel Kurtz /* Unmap the range of iovas that we just mapped */ 636c68a2921SDaniel Kurtz rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE); 637c68a2921SDaniel Kurtz 638c68a2921SDaniel Kurtz iova += pte_count * SPAGE_SIZE; 639c68a2921SDaniel Kurtz page_phys = rk_pte_page_address(pte_addr[pte_count]); 640c68a2921SDaniel Kurtz pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", 641c68a2921SDaniel Kurtz &iova, &page_phys, &paddr, prot); 642c68a2921SDaniel Kurtz 643c68a2921SDaniel Kurtz return -EADDRINUSE; 644c68a2921SDaniel Kurtz } 645c68a2921SDaniel Kurtz 646c68a2921SDaniel Kurtz static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, 647c68a2921SDaniel Kurtz phys_addr_t paddr, size_t size, int prot) 648c68a2921SDaniel Kurtz { 649*bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 650c68a2921SDaniel Kurtz unsigned long flags; 651c68a2921SDaniel Kurtz dma_addr_t iova = (dma_addr_t)_iova; 652c68a2921SDaniel Kurtz u32 *page_table, *pte_addr; 653c68a2921SDaniel Kurtz int ret; 654c68a2921SDaniel Kurtz 655c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->dt_lock, flags); 656c68a2921SDaniel Kurtz 657c68a2921SDaniel Kurtz /* 658c68a2921SDaniel Kurtz * pgsize_bitmap specifies iova sizes that fit in one page table 659c68a2921SDaniel Kurtz * (1024 4-KiB pages = 4 MiB). 660c68a2921SDaniel Kurtz * So, size will always be 4096 <= size <= 4194304. 661c68a2921SDaniel Kurtz * Since iommu_map() guarantees that both iova and size will be 662c68a2921SDaniel Kurtz * aligned, we will always only be mapping from a single dte here. 663c68a2921SDaniel Kurtz */ 664c68a2921SDaniel Kurtz page_table = rk_dte_get_page_table(rk_domain, iova); 665c68a2921SDaniel Kurtz if (IS_ERR(page_table)) { 666c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 667c68a2921SDaniel Kurtz return PTR_ERR(page_table); 668c68a2921SDaniel Kurtz } 669c68a2921SDaniel Kurtz 670c68a2921SDaniel Kurtz pte_addr = &page_table[rk_iova_pte_index(iova)]; 671c68a2921SDaniel Kurtz ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot); 672c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 673c68a2921SDaniel Kurtz 674c68a2921SDaniel Kurtz return ret; 675c68a2921SDaniel Kurtz } 676c68a2921SDaniel Kurtz 677c68a2921SDaniel Kurtz static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, 678c68a2921SDaniel Kurtz size_t size) 679c68a2921SDaniel Kurtz { 680*bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 681c68a2921SDaniel Kurtz unsigned long flags; 682c68a2921SDaniel Kurtz dma_addr_t iova = (dma_addr_t)_iova; 683c68a2921SDaniel Kurtz phys_addr_t pt_phys; 684c68a2921SDaniel Kurtz u32 dte; 685c68a2921SDaniel Kurtz u32 *pte_addr; 686c68a2921SDaniel Kurtz size_t unmap_size; 687c68a2921SDaniel Kurtz 688c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->dt_lock, flags); 689c68a2921SDaniel Kurtz 690c68a2921SDaniel Kurtz /* 691c68a2921SDaniel Kurtz * pgsize_bitmap specifies iova sizes that fit in one page table 692c68a2921SDaniel Kurtz * (1024 4-KiB pages = 4 MiB). 693c68a2921SDaniel Kurtz * So, size will always be 4096 <= size <= 4194304. 694c68a2921SDaniel Kurtz * Since iommu_unmap() guarantees that both iova and size will be 695c68a2921SDaniel Kurtz * aligned, we will always only be unmapping from a single dte here. 696c68a2921SDaniel Kurtz */ 697c68a2921SDaniel Kurtz dte = rk_domain->dt[rk_iova_dte_index(iova)]; 698c68a2921SDaniel Kurtz /* Just return 0 if iova is unmapped */ 699c68a2921SDaniel Kurtz if (!rk_dte_is_pt_valid(dte)) { 700c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 701c68a2921SDaniel Kurtz return 0; 702c68a2921SDaniel Kurtz } 703c68a2921SDaniel Kurtz 704c68a2921SDaniel Kurtz pt_phys = rk_dte_pt_address(dte); 705c68a2921SDaniel Kurtz pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); 706c68a2921SDaniel Kurtz unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size); 707c68a2921SDaniel Kurtz 708c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 709c68a2921SDaniel Kurtz 710c68a2921SDaniel Kurtz /* Shootdown iotlb entries for iova range that was just unmapped */ 711c68a2921SDaniel Kurtz rk_iommu_zap_iova(rk_domain, iova, unmap_size); 712c68a2921SDaniel Kurtz 713c68a2921SDaniel Kurtz return unmap_size; 714c68a2921SDaniel Kurtz } 715c68a2921SDaniel Kurtz 716c68a2921SDaniel Kurtz static struct rk_iommu *rk_iommu_from_dev(struct device *dev) 717c68a2921SDaniel Kurtz { 718c68a2921SDaniel Kurtz struct iommu_group *group; 719c68a2921SDaniel Kurtz struct device *iommu_dev; 720c68a2921SDaniel Kurtz struct rk_iommu *rk_iommu; 721c68a2921SDaniel Kurtz 722c68a2921SDaniel Kurtz group = iommu_group_get(dev); 723c68a2921SDaniel Kurtz if (!group) 724c68a2921SDaniel Kurtz return NULL; 725c68a2921SDaniel Kurtz iommu_dev = iommu_group_get_iommudata(group); 726c68a2921SDaniel Kurtz rk_iommu = dev_get_drvdata(iommu_dev); 727c68a2921SDaniel Kurtz iommu_group_put(group); 728c68a2921SDaniel Kurtz 729c68a2921SDaniel Kurtz return rk_iommu; 730c68a2921SDaniel Kurtz } 731c68a2921SDaniel Kurtz 732c68a2921SDaniel Kurtz static int rk_iommu_attach_device(struct iommu_domain *domain, 733c68a2921SDaniel Kurtz struct device *dev) 734c68a2921SDaniel Kurtz { 735c68a2921SDaniel Kurtz struct rk_iommu *iommu; 736*bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 737c68a2921SDaniel Kurtz unsigned long flags; 738c68a2921SDaniel Kurtz int ret; 739c68a2921SDaniel Kurtz phys_addr_t dte_addr; 740c68a2921SDaniel Kurtz 741c68a2921SDaniel Kurtz /* 742c68a2921SDaniel Kurtz * Allow 'virtual devices' (e.g., drm) to attach to domain. 743c68a2921SDaniel Kurtz * Such a device does not belong to an iommu group. 744c68a2921SDaniel Kurtz */ 745c68a2921SDaniel Kurtz iommu = rk_iommu_from_dev(dev); 746c68a2921SDaniel Kurtz if (!iommu) 747c68a2921SDaniel Kurtz return 0; 748c68a2921SDaniel Kurtz 749c68a2921SDaniel Kurtz ret = rk_iommu_enable_stall(iommu); 750c68a2921SDaniel Kurtz if (ret) 751c68a2921SDaniel Kurtz return ret; 752c68a2921SDaniel Kurtz 753c68a2921SDaniel Kurtz ret = rk_iommu_force_reset(iommu); 754c68a2921SDaniel Kurtz if (ret) 755c68a2921SDaniel Kurtz return ret; 756c68a2921SDaniel Kurtz 757c68a2921SDaniel Kurtz iommu->domain = domain; 758c68a2921SDaniel Kurtz 759c68a2921SDaniel Kurtz ret = devm_request_irq(dev, iommu->irq, rk_iommu_irq, 760c68a2921SDaniel Kurtz IRQF_SHARED, dev_name(dev), iommu); 761c68a2921SDaniel Kurtz if (ret) 762c68a2921SDaniel Kurtz return ret; 763c68a2921SDaniel Kurtz 764c68a2921SDaniel Kurtz dte_addr = virt_to_phys(rk_domain->dt); 765c68a2921SDaniel Kurtz rk_iommu_write(iommu, RK_MMU_DTE_ADDR, dte_addr); 766c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 767c68a2921SDaniel Kurtz rk_iommu_write(iommu, RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); 768c68a2921SDaniel Kurtz 769c68a2921SDaniel Kurtz ret = rk_iommu_enable_paging(iommu); 770c68a2921SDaniel Kurtz if (ret) 771c68a2921SDaniel Kurtz return ret; 772c68a2921SDaniel Kurtz 773c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->iommus_lock, flags); 774c68a2921SDaniel Kurtz list_add_tail(&iommu->node, &rk_domain->iommus); 775c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 776c68a2921SDaniel Kurtz 777c68a2921SDaniel Kurtz dev_info(dev, "Attached to iommu domain\n"); 778c68a2921SDaniel Kurtz 779c68a2921SDaniel Kurtz rk_iommu_disable_stall(iommu); 780c68a2921SDaniel Kurtz 781c68a2921SDaniel Kurtz return 0; 782c68a2921SDaniel Kurtz } 783c68a2921SDaniel Kurtz 784c68a2921SDaniel Kurtz static void rk_iommu_detach_device(struct iommu_domain *domain, 785c68a2921SDaniel Kurtz struct device *dev) 786c68a2921SDaniel Kurtz { 787c68a2921SDaniel Kurtz struct rk_iommu *iommu; 788*bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 789c68a2921SDaniel Kurtz unsigned long flags; 790c68a2921SDaniel Kurtz 791c68a2921SDaniel Kurtz /* Allow 'virtual devices' (eg drm) to detach from domain */ 792c68a2921SDaniel Kurtz iommu = rk_iommu_from_dev(dev); 793c68a2921SDaniel Kurtz if (!iommu) 794c68a2921SDaniel Kurtz return; 795c68a2921SDaniel Kurtz 796c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->iommus_lock, flags); 797c68a2921SDaniel Kurtz list_del_init(&iommu->node); 798c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 799c68a2921SDaniel Kurtz 800c68a2921SDaniel Kurtz /* Ignore error while disabling, just keep going */ 801c68a2921SDaniel Kurtz rk_iommu_enable_stall(iommu); 802c68a2921SDaniel Kurtz rk_iommu_disable_paging(iommu); 803c68a2921SDaniel Kurtz rk_iommu_write(iommu, RK_MMU_INT_MASK, 0); 804c68a2921SDaniel Kurtz rk_iommu_write(iommu, RK_MMU_DTE_ADDR, 0); 805c68a2921SDaniel Kurtz rk_iommu_disable_stall(iommu); 806c68a2921SDaniel Kurtz 807c68a2921SDaniel Kurtz devm_free_irq(dev, iommu->irq, iommu); 808c68a2921SDaniel Kurtz 809c68a2921SDaniel Kurtz iommu->domain = NULL; 810c68a2921SDaniel Kurtz 811c68a2921SDaniel Kurtz dev_info(dev, "Detached from iommu domain\n"); 812c68a2921SDaniel Kurtz } 813c68a2921SDaniel Kurtz 814*bcd516a3SJoerg Roedel static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 815c68a2921SDaniel Kurtz { 816c68a2921SDaniel Kurtz struct rk_iommu_domain *rk_domain; 817c68a2921SDaniel Kurtz 818*bcd516a3SJoerg Roedel if (type != IOMMU_DOMAIN_UNMANAGED) 819*bcd516a3SJoerg Roedel return NULL; 820*bcd516a3SJoerg Roedel 821c68a2921SDaniel Kurtz rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL); 822c68a2921SDaniel Kurtz if (!rk_domain) 823*bcd516a3SJoerg Roedel return NULL; 824c68a2921SDaniel Kurtz 825c68a2921SDaniel Kurtz /* 826c68a2921SDaniel Kurtz * rk32xx iommus use a 2 level pagetable. 827c68a2921SDaniel Kurtz * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 828c68a2921SDaniel Kurtz * Allocate one 4 KiB page for each table. 829c68a2921SDaniel Kurtz */ 830c68a2921SDaniel Kurtz rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); 831c68a2921SDaniel Kurtz if (!rk_domain->dt) 832c68a2921SDaniel Kurtz goto err_dt; 833c68a2921SDaniel Kurtz 834c68a2921SDaniel Kurtz rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES); 835c68a2921SDaniel Kurtz 836c68a2921SDaniel Kurtz spin_lock_init(&rk_domain->iommus_lock); 837c68a2921SDaniel Kurtz spin_lock_init(&rk_domain->dt_lock); 838c68a2921SDaniel Kurtz INIT_LIST_HEAD(&rk_domain->iommus); 839c68a2921SDaniel Kurtz 840*bcd516a3SJoerg Roedel return &rk_domain->domain; 841c68a2921SDaniel Kurtz 842c68a2921SDaniel Kurtz err_dt: 843c68a2921SDaniel Kurtz kfree(rk_domain); 844*bcd516a3SJoerg Roedel return NULL; 845c68a2921SDaniel Kurtz } 846c68a2921SDaniel Kurtz 847*bcd516a3SJoerg Roedel static void rk_iommu_domain_free(struct iommu_domain *domain) 848c68a2921SDaniel Kurtz { 849*bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 850c68a2921SDaniel Kurtz int i; 851c68a2921SDaniel Kurtz 852c68a2921SDaniel Kurtz WARN_ON(!list_empty(&rk_domain->iommus)); 853c68a2921SDaniel Kurtz 854c68a2921SDaniel Kurtz for (i = 0; i < NUM_DT_ENTRIES; i++) { 855c68a2921SDaniel Kurtz u32 dte = rk_domain->dt[i]; 856c68a2921SDaniel Kurtz if (rk_dte_is_pt_valid(dte)) { 857c68a2921SDaniel Kurtz phys_addr_t pt_phys = rk_dte_pt_address(dte); 858c68a2921SDaniel Kurtz u32 *page_table = phys_to_virt(pt_phys); 859c68a2921SDaniel Kurtz free_page((unsigned long)page_table); 860c68a2921SDaniel Kurtz } 861c68a2921SDaniel Kurtz } 862c68a2921SDaniel Kurtz 863c68a2921SDaniel Kurtz free_page((unsigned long)rk_domain->dt); 864*bcd516a3SJoerg Roedel kfree(rk_domain); 865c68a2921SDaniel Kurtz } 866c68a2921SDaniel Kurtz 867c68a2921SDaniel Kurtz static bool rk_iommu_is_dev_iommu_master(struct device *dev) 868c68a2921SDaniel Kurtz { 869c68a2921SDaniel Kurtz struct device_node *np = dev->of_node; 870c68a2921SDaniel Kurtz int ret; 871c68a2921SDaniel Kurtz 872c68a2921SDaniel Kurtz /* 873c68a2921SDaniel Kurtz * An iommu master has an iommus property containing a list of phandles 874c68a2921SDaniel Kurtz * to iommu nodes, each with an #iommu-cells property with value 0. 875c68a2921SDaniel Kurtz */ 876c68a2921SDaniel Kurtz ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); 877c68a2921SDaniel Kurtz return (ret > 0); 878c68a2921SDaniel Kurtz } 879c68a2921SDaniel Kurtz 880c68a2921SDaniel Kurtz static int rk_iommu_group_set_iommudata(struct iommu_group *group, 881c68a2921SDaniel Kurtz struct device *dev) 882c68a2921SDaniel Kurtz { 883c68a2921SDaniel Kurtz struct device_node *np = dev->of_node; 884c68a2921SDaniel Kurtz struct platform_device *pd; 885c68a2921SDaniel Kurtz int ret; 886c68a2921SDaniel Kurtz struct of_phandle_args args; 887c68a2921SDaniel Kurtz 888c68a2921SDaniel Kurtz /* 889c68a2921SDaniel Kurtz * An iommu master has an iommus property containing a list of phandles 890c68a2921SDaniel Kurtz * to iommu nodes, each with an #iommu-cells property with value 0. 891c68a2921SDaniel Kurtz */ 892c68a2921SDaniel Kurtz ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0, 893c68a2921SDaniel Kurtz &args); 894c68a2921SDaniel Kurtz if (ret) { 895c68a2921SDaniel Kurtz dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n", 896c68a2921SDaniel Kurtz np->full_name, ret); 897c68a2921SDaniel Kurtz return ret; 898c68a2921SDaniel Kurtz } 899c68a2921SDaniel Kurtz if (args.args_count != 0) { 900c68a2921SDaniel Kurtz dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n", 901c68a2921SDaniel Kurtz args.np->full_name, args.args_count); 902c68a2921SDaniel Kurtz return -EINVAL; 903c68a2921SDaniel Kurtz } 904c68a2921SDaniel Kurtz 905c68a2921SDaniel Kurtz pd = of_find_device_by_node(args.np); 906c68a2921SDaniel Kurtz of_node_put(args.np); 907c68a2921SDaniel Kurtz if (!pd) { 908c68a2921SDaniel Kurtz dev_err(dev, "iommu %s not found\n", args.np->full_name); 909c68a2921SDaniel Kurtz return -EPROBE_DEFER; 910c68a2921SDaniel Kurtz } 911c68a2921SDaniel Kurtz 912c68a2921SDaniel Kurtz /* TODO(djkurtz): handle multiple slave iommus for a single master */ 913c68a2921SDaniel Kurtz iommu_group_set_iommudata(group, &pd->dev, NULL); 914c68a2921SDaniel Kurtz 915c68a2921SDaniel Kurtz return 0; 916c68a2921SDaniel Kurtz } 917c68a2921SDaniel Kurtz 918c68a2921SDaniel Kurtz static int rk_iommu_add_device(struct device *dev) 919c68a2921SDaniel Kurtz { 920c68a2921SDaniel Kurtz struct iommu_group *group; 921c68a2921SDaniel Kurtz int ret; 922c68a2921SDaniel Kurtz 923c68a2921SDaniel Kurtz if (!rk_iommu_is_dev_iommu_master(dev)) 924c68a2921SDaniel Kurtz return -ENODEV; 925c68a2921SDaniel Kurtz 926c68a2921SDaniel Kurtz group = iommu_group_get(dev); 927c68a2921SDaniel Kurtz if (!group) { 928c68a2921SDaniel Kurtz group = iommu_group_alloc(); 929c68a2921SDaniel Kurtz if (IS_ERR(group)) { 930c68a2921SDaniel Kurtz dev_err(dev, "Failed to allocate IOMMU group\n"); 931c68a2921SDaniel Kurtz return PTR_ERR(group); 932c68a2921SDaniel Kurtz } 933c68a2921SDaniel Kurtz } 934c68a2921SDaniel Kurtz 935c68a2921SDaniel Kurtz ret = iommu_group_add_device(group, dev); 936c68a2921SDaniel Kurtz if (ret) 937c68a2921SDaniel Kurtz goto err_put_group; 938c68a2921SDaniel Kurtz 939c68a2921SDaniel Kurtz ret = rk_iommu_group_set_iommudata(group, dev); 940c68a2921SDaniel Kurtz if (ret) 941c68a2921SDaniel Kurtz goto err_remove_device; 942c68a2921SDaniel Kurtz 943c68a2921SDaniel Kurtz iommu_group_put(group); 944c68a2921SDaniel Kurtz 945c68a2921SDaniel Kurtz return 0; 946c68a2921SDaniel Kurtz 947c68a2921SDaniel Kurtz err_remove_device: 948c68a2921SDaniel Kurtz iommu_group_remove_device(dev); 949c68a2921SDaniel Kurtz err_put_group: 950c68a2921SDaniel Kurtz iommu_group_put(group); 951c68a2921SDaniel Kurtz return ret; 952c68a2921SDaniel Kurtz } 953c68a2921SDaniel Kurtz 954c68a2921SDaniel Kurtz static void rk_iommu_remove_device(struct device *dev) 955c68a2921SDaniel Kurtz { 956c68a2921SDaniel Kurtz if (!rk_iommu_is_dev_iommu_master(dev)) 957c68a2921SDaniel Kurtz return; 958c68a2921SDaniel Kurtz 959c68a2921SDaniel Kurtz iommu_group_remove_device(dev); 960c68a2921SDaniel Kurtz } 961c68a2921SDaniel Kurtz 962c68a2921SDaniel Kurtz static const struct iommu_ops rk_iommu_ops = { 963*bcd516a3SJoerg Roedel .domain_alloc = rk_iommu_domain_alloc, 964*bcd516a3SJoerg Roedel .domain_free = rk_iommu_domain_free, 965c68a2921SDaniel Kurtz .attach_dev = rk_iommu_attach_device, 966c68a2921SDaniel Kurtz .detach_dev = rk_iommu_detach_device, 967c68a2921SDaniel Kurtz .map = rk_iommu_map, 968c68a2921SDaniel Kurtz .unmap = rk_iommu_unmap, 969c68a2921SDaniel Kurtz .add_device = rk_iommu_add_device, 970c68a2921SDaniel Kurtz .remove_device = rk_iommu_remove_device, 971c68a2921SDaniel Kurtz .iova_to_phys = rk_iommu_iova_to_phys, 972c68a2921SDaniel Kurtz .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, 973c68a2921SDaniel Kurtz }; 974c68a2921SDaniel Kurtz 975c68a2921SDaniel Kurtz static int rk_iommu_probe(struct platform_device *pdev) 976c68a2921SDaniel Kurtz { 977c68a2921SDaniel Kurtz struct device *dev = &pdev->dev; 978c68a2921SDaniel Kurtz struct rk_iommu *iommu; 979c68a2921SDaniel Kurtz struct resource *res; 980c68a2921SDaniel Kurtz 981c68a2921SDaniel Kurtz iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); 982c68a2921SDaniel Kurtz if (!iommu) 983c68a2921SDaniel Kurtz return -ENOMEM; 984c68a2921SDaniel Kurtz 985c68a2921SDaniel Kurtz platform_set_drvdata(pdev, iommu); 986c68a2921SDaniel Kurtz iommu->dev = dev; 987c68a2921SDaniel Kurtz 988c68a2921SDaniel Kurtz res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 989c68a2921SDaniel Kurtz iommu->base = devm_ioremap_resource(&pdev->dev, res); 990c68a2921SDaniel Kurtz if (IS_ERR(iommu->base)) 991c68a2921SDaniel Kurtz return PTR_ERR(iommu->base); 992c68a2921SDaniel Kurtz 993c68a2921SDaniel Kurtz iommu->irq = platform_get_irq(pdev, 0); 994c68a2921SDaniel Kurtz if (iommu->irq < 0) { 995c68a2921SDaniel Kurtz dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq); 996c68a2921SDaniel Kurtz return -ENXIO; 997c68a2921SDaniel Kurtz } 998c68a2921SDaniel Kurtz 999c68a2921SDaniel Kurtz return 0; 1000c68a2921SDaniel Kurtz } 1001c68a2921SDaniel Kurtz 1002c68a2921SDaniel Kurtz static int rk_iommu_remove(struct platform_device *pdev) 1003c68a2921SDaniel Kurtz { 1004c68a2921SDaniel Kurtz return 0; 1005c68a2921SDaniel Kurtz } 1006c68a2921SDaniel Kurtz 1007c68a2921SDaniel Kurtz #ifdef CONFIG_OF 1008c68a2921SDaniel Kurtz static const struct of_device_id rk_iommu_dt_ids[] = { 1009c68a2921SDaniel Kurtz { .compatible = "rockchip,iommu" }, 1010c68a2921SDaniel Kurtz { /* sentinel */ } 1011c68a2921SDaniel Kurtz }; 1012c68a2921SDaniel Kurtz MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); 1013c68a2921SDaniel Kurtz #endif 1014c68a2921SDaniel Kurtz 1015c68a2921SDaniel Kurtz static struct platform_driver rk_iommu_driver = { 1016c68a2921SDaniel Kurtz .probe = rk_iommu_probe, 1017c68a2921SDaniel Kurtz .remove = rk_iommu_remove, 1018c68a2921SDaniel Kurtz .driver = { 1019c68a2921SDaniel Kurtz .name = "rk_iommu", 1020c68a2921SDaniel Kurtz .of_match_table = of_match_ptr(rk_iommu_dt_ids), 1021c68a2921SDaniel Kurtz }, 1022c68a2921SDaniel Kurtz }; 1023c68a2921SDaniel Kurtz 1024c68a2921SDaniel Kurtz static int __init rk_iommu_init(void) 1025c68a2921SDaniel Kurtz { 1026425061b0SThierry Reding struct device_node *np; 1027c68a2921SDaniel Kurtz int ret; 1028c68a2921SDaniel Kurtz 1029425061b0SThierry Reding np = of_find_matching_node(NULL, rk_iommu_dt_ids); 1030425061b0SThierry Reding if (!np) 1031425061b0SThierry Reding return 0; 1032425061b0SThierry Reding 1033425061b0SThierry Reding of_node_put(np); 1034425061b0SThierry Reding 1035c68a2921SDaniel Kurtz ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops); 1036c68a2921SDaniel Kurtz if (ret) 1037c68a2921SDaniel Kurtz return ret; 1038c68a2921SDaniel Kurtz 1039c68a2921SDaniel Kurtz return platform_driver_register(&rk_iommu_driver); 1040c68a2921SDaniel Kurtz } 1041c68a2921SDaniel Kurtz static void __exit rk_iommu_exit(void) 1042c68a2921SDaniel Kurtz { 1043c68a2921SDaniel Kurtz platform_driver_unregister(&rk_iommu_driver); 1044c68a2921SDaniel Kurtz } 1045c68a2921SDaniel Kurtz 1046c68a2921SDaniel Kurtz subsys_initcall(rk_iommu_init); 1047c68a2921SDaniel Kurtz module_exit(rk_iommu_exit); 1048c68a2921SDaniel Kurtz 1049c68a2921SDaniel Kurtz MODULE_DESCRIPTION("IOMMU API for Rockchip"); 1050c68a2921SDaniel Kurtz MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com> and Daniel Kurtz <djkurtz@chromium.org>"); 1051c68a2921SDaniel Kurtz MODULE_ALIAS("platform:rockchip-iommu"); 1052c68a2921SDaniel Kurtz MODULE_LICENSE("GPL v2"); 1053