1c68a2921SDaniel Kurtz /* 2c68a2921SDaniel Kurtz * This program is free software; you can redistribute it and/or modify 3c68a2921SDaniel Kurtz * it under the terms of the GNU General Public License version 2 as 4c68a2921SDaniel Kurtz * published by the Free Software Foundation. 5c68a2921SDaniel Kurtz */ 6c68a2921SDaniel Kurtz 7c68a2921SDaniel Kurtz #include <linux/compiler.h> 8c68a2921SDaniel Kurtz #include <linux/delay.h> 9c68a2921SDaniel Kurtz #include <linux/device.h> 104f0aba67SShunqian Zheng #include <linux/dma-iommu.h> 11c68a2921SDaniel Kurtz #include <linux/errno.h> 12c68a2921SDaniel Kurtz #include <linux/interrupt.h> 13c68a2921SDaniel Kurtz #include <linux/io.h> 14c68a2921SDaniel Kurtz #include <linux/iommu.h> 15c68a2921SDaniel Kurtz #include <linux/jiffies.h> 16c68a2921SDaniel Kurtz #include <linux/list.h> 17c68a2921SDaniel Kurtz #include <linux/mm.h> 18c68a2921SDaniel Kurtz #include <linux/module.h> 19c68a2921SDaniel Kurtz #include <linux/of.h> 20c68a2921SDaniel Kurtz #include <linux/of_platform.h> 21c68a2921SDaniel Kurtz #include <linux/platform_device.h> 22c68a2921SDaniel Kurtz #include <linux/slab.h> 23c68a2921SDaniel Kurtz #include <linux/spinlock.h> 24c68a2921SDaniel Kurtz 25c68a2921SDaniel Kurtz /** MMU register offsets */ 26c68a2921SDaniel Kurtz #define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ 27c68a2921SDaniel Kurtz #define RK_MMU_STATUS 0x04 28c68a2921SDaniel Kurtz #define RK_MMU_COMMAND 0x08 29c68a2921SDaniel Kurtz #define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */ 30c68a2921SDaniel Kurtz #define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */ 31c68a2921SDaniel Kurtz #define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */ 32c68a2921SDaniel Kurtz #define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */ 33c68a2921SDaniel Kurtz #define RK_MMU_INT_MASK 0x1C /* IRQ enable */ 34c68a2921SDaniel Kurtz #define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */ 35c68a2921SDaniel Kurtz #define RK_MMU_AUTO_GATING 0x24 36c68a2921SDaniel Kurtz 37c68a2921SDaniel Kurtz #define DTE_ADDR_DUMMY 0xCAFEBABE 38c68a2921SDaniel Kurtz #define FORCE_RESET_TIMEOUT 100 /* ms */ 39c68a2921SDaniel Kurtz 40c68a2921SDaniel Kurtz /* RK_MMU_STATUS fields */ 41c68a2921SDaniel Kurtz #define RK_MMU_STATUS_PAGING_ENABLED BIT(0) 42c68a2921SDaniel Kurtz #define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1) 43c68a2921SDaniel Kurtz #define RK_MMU_STATUS_STALL_ACTIVE BIT(2) 44c68a2921SDaniel Kurtz #define RK_MMU_STATUS_IDLE BIT(3) 45c68a2921SDaniel Kurtz #define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4) 46c68a2921SDaniel Kurtz #define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5) 47c68a2921SDaniel Kurtz #define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31) 48c68a2921SDaniel Kurtz 49c68a2921SDaniel Kurtz /* RK_MMU_COMMAND command values */ 50c68a2921SDaniel Kurtz #define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */ 51c68a2921SDaniel Kurtz #define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */ 52c68a2921SDaniel Kurtz #define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */ 53c68a2921SDaniel Kurtz #define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */ 54c68a2921SDaniel Kurtz #define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */ 55c68a2921SDaniel Kurtz #define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */ 56c68a2921SDaniel Kurtz #define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */ 57c68a2921SDaniel Kurtz 58c68a2921SDaniel Kurtz /* RK_MMU_INT_* register fields */ 59c68a2921SDaniel Kurtz #define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */ 60c68a2921SDaniel Kurtz #define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */ 61c68a2921SDaniel Kurtz #define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR) 62c68a2921SDaniel Kurtz 63c68a2921SDaniel Kurtz #define NUM_DT_ENTRIES 1024 64c68a2921SDaniel Kurtz #define NUM_PT_ENTRIES 1024 65c68a2921SDaniel Kurtz 66c68a2921SDaniel Kurtz #define SPAGE_ORDER 12 67c68a2921SDaniel Kurtz #define SPAGE_SIZE (1 << SPAGE_ORDER) 68c68a2921SDaniel Kurtz 69c68a2921SDaniel Kurtz /* 70c68a2921SDaniel Kurtz * Support mapping any size that fits in one page table: 71c68a2921SDaniel Kurtz * 4 KiB to 4 MiB 72c68a2921SDaniel Kurtz */ 73c68a2921SDaniel Kurtz #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 74c68a2921SDaniel Kurtz 75c68a2921SDaniel Kurtz #define IOMMU_REG_POLL_COUNT_FAST 1000 76c68a2921SDaniel Kurtz 77c68a2921SDaniel Kurtz struct rk_iommu_domain { 78c68a2921SDaniel Kurtz struct list_head iommus; 794f0aba67SShunqian Zheng struct platform_device *pdev; 80c68a2921SDaniel Kurtz u32 *dt; /* page directory table */ 814f0aba67SShunqian Zheng dma_addr_t dt_dma; 82c68a2921SDaniel Kurtz spinlock_t iommus_lock; /* lock for iommus list */ 83c68a2921SDaniel Kurtz spinlock_t dt_lock; /* lock for modifying page directory table */ 84bcd516a3SJoerg Roedel 85bcd516a3SJoerg Roedel struct iommu_domain domain; 86c68a2921SDaniel Kurtz }; 87c68a2921SDaniel Kurtz 88c68a2921SDaniel Kurtz struct rk_iommu { 89c68a2921SDaniel Kurtz struct device *dev; 90cd6438c5SZhengShunQian void __iomem **bases; 91cd6438c5SZhengShunQian int num_mmu; 92c68a2921SDaniel Kurtz int irq; 93c68a2921SDaniel Kurtz struct list_head node; /* entry in rk_iommu_domain.iommus */ 94c68a2921SDaniel Kurtz struct iommu_domain *domain; /* domain to which iommu is attached */ 95c68a2921SDaniel Kurtz }; 96c68a2921SDaniel Kurtz 974f0aba67SShunqian Zheng static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, 984f0aba67SShunqian Zheng unsigned int count) 99c68a2921SDaniel Kurtz { 1004f0aba67SShunqian Zheng size_t size = count * sizeof(u32); /* count of u32 entry */ 101c68a2921SDaniel Kurtz 1024f0aba67SShunqian Zheng dma_sync_single_for_device(&dom->pdev->dev, dma, size, DMA_TO_DEVICE); 103c68a2921SDaniel Kurtz } 104c68a2921SDaniel Kurtz 105bcd516a3SJoerg Roedel static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) 106bcd516a3SJoerg Roedel { 107bcd516a3SJoerg Roedel return container_of(dom, struct rk_iommu_domain, domain); 108bcd516a3SJoerg Roedel } 109bcd516a3SJoerg Roedel 110c68a2921SDaniel Kurtz /** 111c68a2921SDaniel Kurtz * Inspired by _wait_for in intel_drv.h 112c68a2921SDaniel Kurtz * This is NOT safe for use in interrupt context. 113c68a2921SDaniel Kurtz * 114c68a2921SDaniel Kurtz * Note that it's important that we check the condition again after having 115c68a2921SDaniel Kurtz * timed out, since the timeout could be due to preemption or similar and 116c68a2921SDaniel Kurtz * we've never had a chance to check the condition before the timeout. 117c68a2921SDaniel Kurtz */ 118c68a2921SDaniel Kurtz #define rk_wait_for(COND, MS) ({ \ 119c68a2921SDaniel Kurtz unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ 120c68a2921SDaniel Kurtz int ret__ = 0; \ 121c68a2921SDaniel Kurtz while (!(COND)) { \ 122c68a2921SDaniel Kurtz if (time_after(jiffies, timeout__)) { \ 123c68a2921SDaniel Kurtz ret__ = (COND) ? 0 : -ETIMEDOUT; \ 124c68a2921SDaniel Kurtz break; \ 125c68a2921SDaniel Kurtz } \ 126c68a2921SDaniel Kurtz usleep_range(50, 100); \ 127c68a2921SDaniel Kurtz } \ 128c68a2921SDaniel Kurtz ret__; \ 129c68a2921SDaniel Kurtz }) 130c68a2921SDaniel Kurtz 131c68a2921SDaniel Kurtz /* 132c68a2921SDaniel Kurtz * The Rockchip rk3288 iommu uses a 2-level page table. 133c68a2921SDaniel Kurtz * The first level is the "Directory Table" (DT). 134c68a2921SDaniel Kurtz * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing 135c68a2921SDaniel Kurtz * to a "Page Table". 136c68a2921SDaniel Kurtz * The second level is the 1024 Page Tables (PT). 137c68a2921SDaniel Kurtz * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to 138c68a2921SDaniel Kurtz * a 4 KB page of physical memory. 139c68a2921SDaniel Kurtz * 140c68a2921SDaniel Kurtz * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). 141c68a2921SDaniel Kurtz * Each iommu device has a MMU_DTE_ADDR register that contains the physical 142c68a2921SDaniel Kurtz * address of the start of the DT page. 143c68a2921SDaniel Kurtz * 144c68a2921SDaniel Kurtz * The structure of the page table is as follows: 145c68a2921SDaniel Kurtz * 146c68a2921SDaniel Kurtz * DT 147c68a2921SDaniel Kurtz * MMU_DTE_ADDR -> +-----+ 148c68a2921SDaniel Kurtz * | | 149c68a2921SDaniel Kurtz * +-----+ PT 150c68a2921SDaniel Kurtz * | DTE | -> +-----+ 151c68a2921SDaniel Kurtz * +-----+ | | Memory 152c68a2921SDaniel Kurtz * | | +-----+ Page 153c68a2921SDaniel Kurtz * | | | PTE | -> +-----+ 154c68a2921SDaniel Kurtz * +-----+ +-----+ | | 155c68a2921SDaniel Kurtz * | | | | 156c68a2921SDaniel Kurtz * | | | | 157c68a2921SDaniel Kurtz * +-----+ | | 158c68a2921SDaniel Kurtz * | | 159c68a2921SDaniel Kurtz * | | 160c68a2921SDaniel Kurtz * +-----+ 161c68a2921SDaniel Kurtz */ 162c68a2921SDaniel Kurtz 163c68a2921SDaniel Kurtz /* 164c68a2921SDaniel Kurtz * Each DTE has a PT address and a valid bit: 165c68a2921SDaniel Kurtz * +---------------------+-----------+-+ 166c68a2921SDaniel Kurtz * | PT address | Reserved |V| 167c68a2921SDaniel Kurtz * +---------------------+-----------+-+ 168c68a2921SDaniel Kurtz * 31:12 - PT address (PTs always starts on a 4 KB boundary) 169c68a2921SDaniel Kurtz * 11: 1 - Reserved 170c68a2921SDaniel Kurtz * 0 - 1 if PT @ PT address is valid 171c68a2921SDaniel Kurtz */ 172c68a2921SDaniel Kurtz #define RK_DTE_PT_ADDRESS_MASK 0xfffff000 173c68a2921SDaniel Kurtz #define RK_DTE_PT_VALID BIT(0) 174c68a2921SDaniel Kurtz 175c68a2921SDaniel Kurtz static inline phys_addr_t rk_dte_pt_address(u32 dte) 176c68a2921SDaniel Kurtz { 177c68a2921SDaniel Kurtz return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK; 178c68a2921SDaniel Kurtz } 179c68a2921SDaniel Kurtz 180c68a2921SDaniel Kurtz static inline bool rk_dte_is_pt_valid(u32 dte) 181c68a2921SDaniel Kurtz { 182c68a2921SDaniel Kurtz return dte & RK_DTE_PT_VALID; 183c68a2921SDaniel Kurtz } 184c68a2921SDaniel Kurtz 1854f0aba67SShunqian Zheng static inline u32 rk_mk_dte(dma_addr_t pt_dma) 186c68a2921SDaniel Kurtz { 1874f0aba67SShunqian Zheng return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; 188c68a2921SDaniel Kurtz } 189c68a2921SDaniel Kurtz 190c68a2921SDaniel Kurtz /* 191c68a2921SDaniel Kurtz * Each PTE has a Page address, some flags and a valid bit: 192c68a2921SDaniel Kurtz * +---------------------+---+-------+-+ 193c68a2921SDaniel Kurtz * | Page address |Rsv| Flags |V| 194c68a2921SDaniel Kurtz * +---------------------+---+-------+-+ 195c68a2921SDaniel Kurtz * 31:12 - Page address (Pages always start on a 4 KB boundary) 196c68a2921SDaniel Kurtz * 11: 9 - Reserved 197c68a2921SDaniel Kurtz * 8: 1 - Flags 198c68a2921SDaniel Kurtz * 8 - Read allocate - allocate cache space on read misses 199c68a2921SDaniel Kurtz * 7 - Read cache - enable cache & prefetch of data 200c68a2921SDaniel Kurtz * 6 - Write buffer - enable delaying writes on their way to memory 201c68a2921SDaniel Kurtz * 5 - Write allocate - allocate cache space on write misses 202c68a2921SDaniel Kurtz * 4 - Write cache - different writes can be merged together 203c68a2921SDaniel Kurtz * 3 - Override cache attributes 204c68a2921SDaniel Kurtz * if 1, bits 4-8 control cache attributes 205c68a2921SDaniel Kurtz * if 0, the system bus defaults are used 206c68a2921SDaniel Kurtz * 2 - Writable 207c68a2921SDaniel Kurtz * 1 - Readable 208c68a2921SDaniel Kurtz * 0 - 1 if Page @ Page address is valid 209c68a2921SDaniel Kurtz */ 210c68a2921SDaniel Kurtz #define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000 211c68a2921SDaniel Kurtz #define RK_PTE_PAGE_FLAGS_MASK 0x000001fe 212c68a2921SDaniel Kurtz #define RK_PTE_PAGE_WRITABLE BIT(2) 213c68a2921SDaniel Kurtz #define RK_PTE_PAGE_READABLE BIT(1) 214c68a2921SDaniel Kurtz #define RK_PTE_PAGE_VALID BIT(0) 215c68a2921SDaniel Kurtz 216c68a2921SDaniel Kurtz static inline phys_addr_t rk_pte_page_address(u32 pte) 217c68a2921SDaniel Kurtz { 218c68a2921SDaniel Kurtz return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK; 219c68a2921SDaniel Kurtz } 220c68a2921SDaniel Kurtz 221c68a2921SDaniel Kurtz static inline bool rk_pte_is_page_valid(u32 pte) 222c68a2921SDaniel Kurtz { 223c68a2921SDaniel Kurtz return pte & RK_PTE_PAGE_VALID; 224c68a2921SDaniel Kurtz } 225c68a2921SDaniel Kurtz 226c68a2921SDaniel Kurtz /* TODO: set cache flags per prot IOMMU_CACHE */ 227c68a2921SDaniel Kurtz static u32 rk_mk_pte(phys_addr_t page, int prot) 228c68a2921SDaniel Kurtz { 229c68a2921SDaniel Kurtz u32 flags = 0; 230c68a2921SDaniel Kurtz flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; 231c68a2921SDaniel Kurtz flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; 232c68a2921SDaniel Kurtz page &= RK_PTE_PAGE_ADDRESS_MASK; 233c68a2921SDaniel Kurtz return page | flags | RK_PTE_PAGE_VALID; 234c68a2921SDaniel Kurtz } 235c68a2921SDaniel Kurtz 236c68a2921SDaniel Kurtz static u32 rk_mk_pte_invalid(u32 pte) 237c68a2921SDaniel Kurtz { 238c68a2921SDaniel Kurtz return pte & ~RK_PTE_PAGE_VALID; 239c68a2921SDaniel Kurtz } 240c68a2921SDaniel Kurtz 241c68a2921SDaniel Kurtz /* 242c68a2921SDaniel Kurtz * rk3288 iova (IOMMU Virtual Address) format 243c68a2921SDaniel Kurtz * 31 22.21 12.11 0 244c68a2921SDaniel Kurtz * +-----------+-----------+-------------+ 245c68a2921SDaniel Kurtz * | DTE index | PTE index | Page offset | 246c68a2921SDaniel Kurtz * +-----------+-----------+-------------+ 247c68a2921SDaniel Kurtz * 31:22 - DTE index - index of DTE in DT 248c68a2921SDaniel Kurtz * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address 249c68a2921SDaniel Kurtz * 11: 0 - Page offset - offset into page @ PTE.page_address 250c68a2921SDaniel Kurtz */ 251c68a2921SDaniel Kurtz #define RK_IOVA_DTE_MASK 0xffc00000 252c68a2921SDaniel Kurtz #define RK_IOVA_DTE_SHIFT 22 253c68a2921SDaniel Kurtz #define RK_IOVA_PTE_MASK 0x003ff000 254c68a2921SDaniel Kurtz #define RK_IOVA_PTE_SHIFT 12 255c68a2921SDaniel Kurtz #define RK_IOVA_PAGE_MASK 0x00000fff 256c68a2921SDaniel Kurtz #define RK_IOVA_PAGE_SHIFT 0 257c68a2921SDaniel Kurtz 258c68a2921SDaniel Kurtz static u32 rk_iova_dte_index(dma_addr_t iova) 259c68a2921SDaniel Kurtz { 260c68a2921SDaniel Kurtz return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT; 261c68a2921SDaniel Kurtz } 262c68a2921SDaniel Kurtz 263c68a2921SDaniel Kurtz static u32 rk_iova_pte_index(dma_addr_t iova) 264c68a2921SDaniel Kurtz { 265c68a2921SDaniel Kurtz return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT; 266c68a2921SDaniel Kurtz } 267c68a2921SDaniel Kurtz 268c68a2921SDaniel Kurtz static u32 rk_iova_page_offset(dma_addr_t iova) 269c68a2921SDaniel Kurtz { 270c68a2921SDaniel Kurtz return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; 271c68a2921SDaniel Kurtz } 272c68a2921SDaniel Kurtz 273cd6438c5SZhengShunQian static u32 rk_iommu_read(void __iomem *base, u32 offset) 274c68a2921SDaniel Kurtz { 275cd6438c5SZhengShunQian return readl(base + offset); 276c68a2921SDaniel Kurtz } 277c68a2921SDaniel Kurtz 278cd6438c5SZhengShunQian static void rk_iommu_write(void __iomem *base, u32 offset, u32 value) 279c68a2921SDaniel Kurtz { 280cd6438c5SZhengShunQian writel(value, base + offset); 281c68a2921SDaniel Kurtz } 282c68a2921SDaniel Kurtz 283c68a2921SDaniel Kurtz static void rk_iommu_command(struct rk_iommu *iommu, u32 command) 284c68a2921SDaniel Kurtz { 285cd6438c5SZhengShunQian int i; 286cd6438c5SZhengShunQian 287cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 288cd6438c5SZhengShunQian writel(command, iommu->bases[i] + RK_MMU_COMMAND); 289c68a2921SDaniel Kurtz } 290c68a2921SDaniel Kurtz 291cd6438c5SZhengShunQian static void rk_iommu_base_command(void __iomem *base, u32 command) 292cd6438c5SZhengShunQian { 293cd6438c5SZhengShunQian writel(command, base + RK_MMU_COMMAND); 294cd6438c5SZhengShunQian } 295c68a2921SDaniel Kurtz static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, 296c68a2921SDaniel Kurtz size_t size) 297c68a2921SDaniel Kurtz { 298cd6438c5SZhengShunQian int i; 299cd6438c5SZhengShunQian 300c68a2921SDaniel Kurtz dma_addr_t iova_end = iova + size; 301c68a2921SDaniel Kurtz /* 302c68a2921SDaniel Kurtz * TODO(djkurtz): Figure out when it is more efficient to shootdown the 303c68a2921SDaniel Kurtz * entire iotlb rather than iterate over individual iovas. 304c68a2921SDaniel Kurtz */ 305cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 306c68a2921SDaniel Kurtz for (; iova < iova_end; iova += SPAGE_SIZE) 307cd6438c5SZhengShunQian rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova); 308c68a2921SDaniel Kurtz } 309c68a2921SDaniel Kurtz 310c68a2921SDaniel Kurtz static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) 311c68a2921SDaniel Kurtz { 312cd6438c5SZhengShunQian bool active = true; 313cd6438c5SZhengShunQian int i; 314cd6438c5SZhengShunQian 315cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 316fbedd9b9SJohn Keeping active &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) & 317fbedd9b9SJohn Keeping RK_MMU_STATUS_STALL_ACTIVE); 318cd6438c5SZhengShunQian 319cd6438c5SZhengShunQian return active; 320c68a2921SDaniel Kurtz } 321c68a2921SDaniel Kurtz 322c68a2921SDaniel Kurtz static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) 323c68a2921SDaniel Kurtz { 324cd6438c5SZhengShunQian bool enable = true; 325cd6438c5SZhengShunQian int i; 326cd6438c5SZhengShunQian 327cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 328fbedd9b9SJohn Keeping enable &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) & 329fbedd9b9SJohn Keeping RK_MMU_STATUS_PAGING_ENABLED); 330cd6438c5SZhengShunQian 331cd6438c5SZhengShunQian return enable; 332c68a2921SDaniel Kurtz } 333c68a2921SDaniel Kurtz 334c68a2921SDaniel Kurtz static int rk_iommu_enable_stall(struct rk_iommu *iommu) 335c68a2921SDaniel Kurtz { 336cd6438c5SZhengShunQian int ret, i; 337c68a2921SDaniel Kurtz 338c68a2921SDaniel Kurtz if (rk_iommu_is_stall_active(iommu)) 339c68a2921SDaniel Kurtz return 0; 340c68a2921SDaniel Kurtz 341c68a2921SDaniel Kurtz /* Stall can only be enabled if paging is enabled */ 342c68a2921SDaniel Kurtz if (!rk_iommu_is_paging_enabled(iommu)) 343c68a2921SDaniel Kurtz return 0; 344c68a2921SDaniel Kurtz 345c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); 346c68a2921SDaniel Kurtz 347c68a2921SDaniel Kurtz ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); 348c68a2921SDaniel Kurtz if (ret) 349cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 350c68a2921SDaniel Kurtz dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", 351cd6438c5SZhengShunQian rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 352c68a2921SDaniel Kurtz 353c68a2921SDaniel Kurtz return ret; 354c68a2921SDaniel Kurtz } 355c68a2921SDaniel Kurtz 356c68a2921SDaniel Kurtz static int rk_iommu_disable_stall(struct rk_iommu *iommu) 357c68a2921SDaniel Kurtz { 358cd6438c5SZhengShunQian int ret, i; 359c68a2921SDaniel Kurtz 360c68a2921SDaniel Kurtz if (!rk_iommu_is_stall_active(iommu)) 361c68a2921SDaniel Kurtz return 0; 362c68a2921SDaniel Kurtz 363c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); 364c68a2921SDaniel Kurtz 365c68a2921SDaniel Kurtz ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); 366c68a2921SDaniel Kurtz if (ret) 367cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 368c68a2921SDaniel Kurtz dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", 369cd6438c5SZhengShunQian rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 370c68a2921SDaniel Kurtz 371c68a2921SDaniel Kurtz return ret; 372c68a2921SDaniel Kurtz } 373c68a2921SDaniel Kurtz 374c68a2921SDaniel Kurtz static int rk_iommu_enable_paging(struct rk_iommu *iommu) 375c68a2921SDaniel Kurtz { 376cd6438c5SZhengShunQian int ret, i; 377c68a2921SDaniel Kurtz 378c68a2921SDaniel Kurtz if (rk_iommu_is_paging_enabled(iommu)) 379c68a2921SDaniel Kurtz return 0; 380c68a2921SDaniel Kurtz 381c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); 382c68a2921SDaniel Kurtz 383c68a2921SDaniel Kurtz ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); 384c68a2921SDaniel Kurtz if (ret) 385cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 386c68a2921SDaniel Kurtz dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", 387cd6438c5SZhengShunQian rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 388c68a2921SDaniel Kurtz 389c68a2921SDaniel Kurtz return ret; 390c68a2921SDaniel Kurtz } 391c68a2921SDaniel Kurtz 392c68a2921SDaniel Kurtz static int rk_iommu_disable_paging(struct rk_iommu *iommu) 393c68a2921SDaniel Kurtz { 394cd6438c5SZhengShunQian int ret, i; 395c68a2921SDaniel Kurtz 396c68a2921SDaniel Kurtz if (!rk_iommu_is_paging_enabled(iommu)) 397c68a2921SDaniel Kurtz return 0; 398c68a2921SDaniel Kurtz 399c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); 400c68a2921SDaniel Kurtz 401c68a2921SDaniel Kurtz ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); 402c68a2921SDaniel Kurtz if (ret) 403cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) 404c68a2921SDaniel Kurtz dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", 405cd6438c5SZhengShunQian rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); 406c68a2921SDaniel Kurtz 407c68a2921SDaniel Kurtz return ret; 408c68a2921SDaniel Kurtz } 409c68a2921SDaniel Kurtz 410c68a2921SDaniel Kurtz static int rk_iommu_force_reset(struct rk_iommu *iommu) 411c68a2921SDaniel Kurtz { 412cd6438c5SZhengShunQian int ret, i; 413c68a2921SDaniel Kurtz u32 dte_addr; 414c68a2921SDaniel Kurtz 415c68a2921SDaniel Kurtz /* 416c68a2921SDaniel Kurtz * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY 417c68a2921SDaniel Kurtz * and verifying that upper 5 nybbles are read back. 418c68a2921SDaniel Kurtz */ 419cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) { 420cd6438c5SZhengShunQian rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); 421c68a2921SDaniel Kurtz 422cd6438c5SZhengShunQian dte_addr = rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR); 423c68a2921SDaniel Kurtz if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { 424c68a2921SDaniel Kurtz dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); 425c68a2921SDaniel Kurtz return -EFAULT; 426c68a2921SDaniel Kurtz } 427cd6438c5SZhengShunQian } 428c68a2921SDaniel Kurtz 429c68a2921SDaniel Kurtz rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); 430c68a2921SDaniel Kurtz 431cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) { 432cd6438c5SZhengShunQian ret = rk_wait_for(rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0x00000000, 433c68a2921SDaniel Kurtz FORCE_RESET_TIMEOUT); 434cd6438c5SZhengShunQian if (ret) { 435c68a2921SDaniel Kurtz dev_err(iommu->dev, "FORCE_RESET command timed out\n"); 436c68a2921SDaniel Kurtz return ret; 437c68a2921SDaniel Kurtz } 438cd6438c5SZhengShunQian } 439c68a2921SDaniel Kurtz 440cd6438c5SZhengShunQian return 0; 441cd6438c5SZhengShunQian } 442cd6438c5SZhengShunQian 443cd6438c5SZhengShunQian static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova) 444c68a2921SDaniel Kurtz { 445cd6438c5SZhengShunQian void __iomem *base = iommu->bases[index]; 446c68a2921SDaniel Kurtz u32 dte_index, pte_index, page_offset; 447c68a2921SDaniel Kurtz u32 mmu_dte_addr; 448c68a2921SDaniel Kurtz phys_addr_t mmu_dte_addr_phys, dte_addr_phys; 449c68a2921SDaniel Kurtz u32 *dte_addr; 450c68a2921SDaniel Kurtz u32 dte; 451c68a2921SDaniel Kurtz phys_addr_t pte_addr_phys = 0; 452c68a2921SDaniel Kurtz u32 *pte_addr = NULL; 453c68a2921SDaniel Kurtz u32 pte = 0; 454c68a2921SDaniel Kurtz phys_addr_t page_addr_phys = 0; 455c68a2921SDaniel Kurtz u32 page_flags = 0; 456c68a2921SDaniel Kurtz 457c68a2921SDaniel Kurtz dte_index = rk_iova_dte_index(iova); 458c68a2921SDaniel Kurtz pte_index = rk_iova_pte_index(iova); 459c68a2921SDaniel Kurtz page_offset = rk_iova_page_offset(iova); 460c68a2921SDaniel Kurtz 461cd6438c5SZhengShunQian mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR); 462c68a2921SDaniel Kurtz mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr; 463c68a2921SDaniel Kurtz 464c68a2921SDaniel Kurtz dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); 465c68a2921SDaniel Kurtz dte_addr = phys_to_virt(dte_addr_phys); 466c68a2921SDaniel Kurtz dte = *dte_addr; 467c68a2921SDaniel Kurtz 468c68a2921SDaniel Kurtz if (!rk_dte_is_pt_valid(dte)) 469c68a2921SDaniel Kurtz goto print_it; 470c68a2921SDaniel Kurtz 471c68a2921SDaniel Kurtz pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4); 472c68a2921SDaniel Kurtz pte_addr = phys_to_virt(pte_addr_phys); 473c68a2921SDaniel Kurtz pte = *pte_addr; 474c68a2921SDaniel Kurtz 475c68a2921SDaniel Kurtz if (!rk_pte_is_page_valid(pte)) 476c68a2921SDaniel Kurtz goto print_it; 477c68a2921SDaniel Kurtz 478c68a2921SDaniel Kurtz page_addr_phys = rk_pte_page_address(pte) + page_offset; 479c68a2921SDaniel Kurtz page_flags = pte & RK_PTE_PAGE_FLAGS_MASK; 480c68a2921SDaniel Kurtz 481c68a2921SDaniel Kurtz print_it: 482c68a2921SDaniel Kurtz dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n", 483c68a2921SDaniel Kurtz &iova, dte_index, pte_index, page_offset); 484c68a2921SDaniel Kurtz dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n", 485c68a2921SDaniel Kurtz &mmu_dte_addr_phys, &dte_addr_phys, dte, 486c68a2921SDaniel Kurtz rk_dte_is_pt_valid(dte), &pte_addr_phys, pte, 487c68a2921SDaniel Kurtz rk_pte_is_page_valid(pte), &page_addr_phys, page_flags); 488c68a2921SDaniel Kurtz } 489c68a2921SDaniel Kurtz 490c68a2921SDaniel Kurtz static irqreturn_t rk_iommu_irq(int irq, void *dev_id) 491c68a2921SDaniel Kurtz { 492c68a2921SDaniel Kurtz struct rk_iommu *iommu = dev_id; 493c68a2921SDaniel Kurtz u32 status; 494c68a2921SDaniel Kurtz u32 int_status; 495c68a2921SDaniel Kurtz dma_addr_t iova; 496cd6438c5SZhengShunQian irqreturn_t ret = IRQ_NONE; 497cd6438c5SZhengShunQian int i; 498c68a2921SDaniel Kurtz 499cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) { 500cd6438c5SZhengShunQian int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS); 501c68a2921SDaniel Kurtz if (int_status == 0) 502cd6438c5SZhengShunQian continue; 503c68a2921SDaniel Kurtz 504cd6438c5SZhengShunQian ret = IRQ_HANDLED; 505cd6438c5SZhengShunQian iova = rk_iommu_read(iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR); 506c68a2921SDaniel Kurtz 507c68a2921SDaniel Kurtz if (int_status & RK_MMU_IRQ_PAGE_FAULT) { 508c68a2921SDaniel Kurtz int flags; 509c68a2921SDaniel Kurtz 510cd6438c5SZhengShunQian status = rk_iommu_read(iommu->bases[i], RK_MMU_STATUS); 511c68a2921SDaniel Kurtz flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? 512c68a2921SDaniel Kurtz IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 513c68a2921SDaniel Kurtz 514c68a2921SDaniel Kurtz dev_err(iommu->dev, "Page fault at %pad of type %s\n", 515c68a2921SDaniel Kurtz &iova, 516c68a2921SDaniel Kurtz (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); 517c68a2921SDaniel Kurtz 518cd6438c5SZhengShunQian log_iova(iommu, i, iova); 519c68a2921SDaniel Kurtz 520c68a2921SDaniel Kurtz /* 521c68a2921SDaniel Kurtz * Report page fault to any installed handlers. 522c68a2921SDaniel Kurtz * Ignore the return code, though, since we always zap cache 523c68a2921SDaniel Kurtz * and clear the page fault anyway. 524c68a2921SDaniel Kurtz */ 525c68a2921SDaniel Kurtz if (iommu->domain) 526c68a2921SDaniel Kurtz report_iommu_fault(iommu->domain, iommu->dev, iova, 527c68a2921SDaniel Kurtz flags); 528c68a2921SDaniel Kurtz else 529c68a2921SDaniel Kurtz dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); 530c68a2921SDaniel Kurtz 531cd6438c5SZhengShunQian rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); 532cd6438c5SZhengShunQian rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE); 533c68a2921SDaniel Kurtz } 534c68a2921SDaniel Kurtz 535c68a2921SDaniel Kurtz if (int_status & RK_MMU_IRQ_BUS_ERROR) 536c68a2921SDaniel Kurtz dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); 537c68a2921SDaniel Kurtz 538c68a2921SDaniel Kurtz if (int_status & ~RK_MMU_IRQ_MASK) 539c68a2921SDaniel Kurtz dev_err(iommu->dev, "unexpected int_status: %#08x\n", 540c68a2921SDaniel Kurtz int_status); 541c68a2921SDaniel Kurtz 542cd6438c5SZhengShunQian rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status); 543cd6438c5SZhengShunQian } 544c68a2921SDaniel Kurtz 545cd6438c5SZhengShunQian return ret; 546c68a2921SDaniel Kurtz } 547c68a2921SDaniel Kurtz 548c68a2921SDaniel Kurtz static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, 549c68a2921SDaniel Kurtz dma_addr_t iova) 550c68a2921SDaniel Kurtz { 551bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 552c68a2921SDaniel Kurtz unsigned long flags; 553c68a2921SDaniel Kurtz phys_addr_t pt_phys, phys = 0; 554c68a2921SDaniel Kurtz u32 dte, pte; 555c68a2921SDaniel Kurtz u32 *page_table; 556c68a2921SDaniel Kurtz 557c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->dt_lock, flags); 558c68a2921SDaniel Kurtz 559c68a2921SDaniel Kurtz dte = rk_domain->dt[rk_iova_dte_index(iova)]; 560c68a2921SDaniel Kurtz if (!rk_dte_is_pt_valid(dte)) 561c68a2921SDaniel Kurtz goto out; 562c68a2921SDaniel Kurtz 563c68a2921SDaniel Kurtz pt_phys = rk_dte_pt_address(dte); 564c68a2921SDaniel Kurtz page_table = (u32 *)phys_to_virt(pt_phys); 565c68a2921SDaniel Kurtz pte = page_table[rk_iova_pte_index(iova)]; 566c68a2921SDaniel Kurtz if (!rk_pte_is_page_valid(pte)) 567c68a2921SDaniel Kurtz goto out; 568c68a2921SDaniel Kurtz 569c68a2921SDaniel Kurtz phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova); 570c68a2921SDaniel Kurtz out: 571c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 572c68a2921SDaniel Kurtz 573c68a2921SDaniel Kurtz return phys; 574c68a2921SDaniel Kurtz } 575c68a2921SDaniel Kurtz 576c68a2921SDaniel Kurtz static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, 577c68a2921SDaniel Kurtz dma_addr_t iova, size_t size) 578c68a2921SDaniel Kurtz { 579c68a2921SDaniel Kurtz struct list_head *pos; 580c68a2921SDaniel Kurtz unsigned long flags; 581c68a2921SDaniel Kurtz 582c68a2921SDaniel Kurtz /* shootdown these iova from all iommus using this domain */ 583c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->iommus_lock, flags); 584c68a2921SDaniel Kurtz list_for_each(pos, &rk_domain->iommus) { 585c68a2921SDaniel Kurtz struct rk_iommu *iommu; 586c68a2921SDaniel Kurtz iommu = list_entry(pos, struct rk_iommu, node); 587c68a2921SDaniel Kurtz rk_iommu_zap_lines(iommu, iova, size); 588c68a2921SDaniel Kurtz } 589c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 590c68a2921SDaniel Kurtz } 591c68a2921SDaniel Kurtz 592d4dd920cSTomasz Figa static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, 593d4dd920cSTomasz Figa dma_addr_t iova, size_t size) 594d4dd920cSTomasz Figa { 595d4dd920cSTomasz Figa rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); 596d4dd920cSTomasz Figa if (size > SPAGE_SIZE) 597d4dd920cSTomasz Figa rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE, 598d4dd920cSTomasz Figa SPAGE_SIZE); 599d4dd920cSTomasz Figa } 600d4dd920cSTomasz Figa 601c68a2921SDaniel Kurtz static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 602c68a2921SDaniel Kurtz dma_addr_t iova) 603c68a2921SDaniel Kurtz { 6044f0aba67SShunqian Zheng struct device *dev = &rk_domain->pdev->dev; 605c68a2921SDaniel Kurtz u32 *page_table, *dte_addr; 6064f0aba67SShunqian Zheng u32 dte_index, dte; 607c68a2921SDaniel Kurtz phys_addr_t pt_phys; 6084f0aba67SShunqian Zheng dma_addr_t pt_dma; 609c68a2921SDaniel Kurtz 610c68a2921SDaniel Kurtz assert_spin_locked(&rk_domain->dt_lock); 611c68a2921SDaniel Kurtz 6124f0aba67SShunqian Zheng dte_index = rk_iova_dte_index(iova); 6134f0aba67SShunqian Zheng dte_addr = &rk_domain->dt[dte_index]; 614c68a2921SDaniel Kurtz dte = *dte_addr; 615c68a2921SDaniel Kurtz if (rk_dte_is_pt_valid(dte)) 616c68a2921SDaniel Kurtz goto done; 617c68a2921SDaniel Kurtz 618c68a2921SDaniel Kurtz page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 619c68a2921SDaniel Kurtz if (!page_table) 620c68a2921SDaniel Kurtz return ERR_PTR(-ENOMEM); 621c68a2921SDaniel Kurtz 6224f0aba67SShunqian Zheng pt_dma = dma_map_single(dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); 6234f0aba67SShunqian Zheng if (dma_mapping_error(dev, pt_dma)) { 6244f0aba67SShunqian Zheng dev_err(dev, "DMA mapping error while allocating page table\n"); 6254f0aba67SShunqian Zheng free_page((unsigned long)page_table); 6264f0aba67SShunqian Zheng return ERR_PTR(-ENOMEM); 6274f0aba67SShunqian Zheng } 6284f0aba67SShunqian Zheng 6294f0aba67SShunqian Zheng dte = rk_mk_dte(pt_dma); 630c68a2921SDaniel Kurtz *dte_addr = dte; 631c68a2921SDaniel Kurtz 6324f0aba67SShunqian Zheng rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES); 6334f0aba67SShunqian Zheng rk_table_flush(rk_domain, 6344f0aba67SShunqian Zheng rk_domain->dt_dma + dte_index * sizeof(u32), 1); 635c68a2921SDaniel Kurtz done: 636c68a2921SDaniel Kurtz pt_phys = rk_dte_pt_address(dte); 637c68a2921SDaniel Kurtz return (u32 *)phys_to_virt(pt_phys); 638c68a2921SDaniel Kurtz } 639c68a2921SDaniel Kurtz 640c68a2921SDaniel Kurtz static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, 6414f0aba67SShunqian Zheng u32 *pte_addr, dma_addr_t pte_dma, 6424f0aba67SShunqian Zheng size_t size) 643c68a2921SDaniel Kurtz { 644c68a2921SDaniel Kurtz unsigned int pte_count; 645c68a2921SDaniel Kurtz unsigned int pte_total = size / SPAGE_SIZE; 646c68a2921SDaniel Kurtz 647c68a2921SDaniel Kurtz assert_spin_locked(&rk_domain->dt_lock); 648c68a2921SDaniel Kurtz 649c68a2921SDaniel Kurtz for (pte_count = 0; pte_count < pte_total; pte_count++) { 650c68a2921SDaniel Kurtz u32 pte = pte_addr[pte_count]; 651c68a2921SDaniel Kurtz if (!rk_pte_is_page_valid(pte)) 652c68a2921SDaniel Kurtz break; 653c68a2921SDaniel Kurtz 654c68a2921SDaniel Kurtz pte_addr[pte_count] = rk_mk_pte_invalid(pte); 655c68a2921SDaniel Kurtz } 656c68a2921SDaniel Kurtz 6574f0aba67SShunqian Zheng rk_table_flush(rk_domain, pte_dma, pte_count); 658c68a2921SDaniel Kurtz 659c68a2921SDaniel Kurtz return pte_count * SPAGE_SIZE; 660c68a2921SDaniel Kurtz } 661c68a2921SDaniel Kurtz 662c68a2921SDaniel Kurtz static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, 6634f0aba67SShunqian Zheng dma_addr_t pte_dma, dma_addr_t iova, 6644f0aba67SShunqian Zheng phys_addr_t paddr, size_t size, int prot) 665c68a2921SDaniel Kurtz { 666c68a2921SDaniel Kurtz unsigned int pte_count; 667c68a2921SDaniel Kurtz unsigned int pte_total = size / SPAGE_SIZE; 668c68a2921SDaniel Kurtz phys_addr_t page_phys; 669c68a2921SDaniel Kurtz 670c68a2921SDaniel Kurtz assert_spin_locked(&rk_domain->dt_lock); 671c68a2921SDaniel Kurtz 672c68a2921SDaniel Kurtz for (pte_count = 0; pte_count < pte_total; pte_count++) { 673c68a2921SDaniel Kurtz u32 pte = pte_addr[pte_count]; 674c68a2921SDaniel Kurtz 675c68a2921SDaniel Kurtz if (rk_pte_is_page_valid(pte)) 676c68a2921SDaniel Kurtz goto unwind; 677c68a2921SDaniel Kurtz 678c68a2921SDaniel Kurtz pte_addr[pte_count] = rk_mk_pte(paddr, prot); 679c68a2921SDaniel Kurtz 680c68a2921SDaniel Kurtz paddr += SPAGE_SIZE; 681c68a2921SDaniel Kurtz } 682c68a2921SDaniel Kurtz 6834f0aba67SShunqian Zheng rk_table_flush(rk_domain, pte_dma, pte_total); 684c68a2921SDaniel Kurtz 685d4dd920cSTomasz Figa /* 686d4dd920cSTomasz Figa * Zap the first and last iova to evict from iotlb any previously 687d4dd920cSTomasz Figa * mapped cachelines holding stale values for its dte and pte. 688d4dd920cSTomasz Figa * We only zap the first and last iova, since only they could have 689d4dd920cSTomasz Figa * dte or pte shared with an existing mapping. 690d4dd920cSTomasz Figa */ 691d4dd920cSTomasz Figa rk_iommu_zap_iova_first_last(rk_domain, iova, size); 692d4dd920cSTomasz Figa 693c68a2921SDaniel Kurtz return 0; 694c68a2921SDaniel Kurtz unwind: 695c68a2921SDaniel Kurtz /* Unmap the range of iovas that we just mapped */ 6964f0aba67SShunqian Zheng rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, 6974f0aba67SShunqian Zheng pte_count * SPAGE_SIZE); 698c68a2921SDaniel Kurtz 699c68a2921SDaniel Kurtz iova += pte_count * SPAGE_SIZE; 700c68a2921SDaniel Kurtz page_phys = rk_pte_page_address(pte_addr[pte_count]); 701c68a2921SDaniel Kurtz pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", 702c68a2921SDaniel Kurtz &iova, &page_phys, &paddr, prot); 703c68a2921SDaniel Kurtz 704c68a2921SDaniel Kurtz return -EADDRINUSE; 705c68a2921SDaniel Kurtz } 706c68a2921SDaniel Kurtz 707c68a2921SDaniel Kurtz static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, 708c68a2921SDaniel Kurtz phys_addr_t paddr, size_t size, int prot) 709c68a2921SDaniel Kurtz { 710bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 711c68a2921SDaniel Kurtz unsigned long flags; 7124f0aba67SShunqian Zheng dma_addr_t pte_dma, iova = (dma_addr_t)_iova; 713c68a2921SDaniel Kurtz u32 *page_table, *pte_addr; 7144f0aba67SShunqian Zheng u32 dte_index, pte_index; 715c68a2921SDaniel Kurtz int ret; 716c68a2921SDaniel Kurtz 717c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->dt_lock, flags); 718c68a2921SDaniel Kurtz 719c68a2921SDaniel Kurtz /* 720c68a2921SDaniel Kurtz * pgsize_bitmap specifies iova sizes that fit in one page table 721c68a2921SDaniel Kurtz * (1024 4-KiB pages = 4 MiB). 722c68a2921SDaniel Kurtz * So, size will always be 4096 <= size <= 4194304. 723c68a2921SDaniel Kurtz * Since iommu_map() guarantees that both iova and size will be 724c68a2921SDaniel Kurtz * aligned, we will always only be mapping from a single dte here. 725c68a2921SDaniel Kurtz */ 726c68a2921SDaniel Kurtz page_table = rk_dte_get_page_table(rk_domain, iova); 727c68a2921SDaniel Kurtz if (IS_ERR(page_table)) { 728c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 729c68a2921SDaniel Kurtz return PTR_ERR(page_table); 730c68a2921SDaniel Kurtz } 731c68a2921SDaniel Kurtz 7324f0aba67SShunqian Zheng dte_index = rk_domain->dt[rk_iova_dte_index(iova)]; 7334f0aba67SShunqian Zheng pte_index = rk_iova_pte_index(iova); 7344f0aba67SShunqian Zheng pte_addr = &page_table[pte_index]; 7354f0aba67SShunqian Zheng pte_dma = rk_dte_pt_address(dte_index) + pte_index * sizeof(u32); 7364f0aba67SShunqian Zheng ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova, 7374f0aba67SShunqian Zheng paddr, size, prot); 7384f0aba67SShunqian Zheng 739c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 740c68a2921SDaniel Kurtz 741c68a2921SDaniel Kurtz return ret; 742c68a2921SDaniel Kurtz } 743c68a2921SDaniel Kurtz 744c68a2921SDaniel Kurtz static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, 745c68a2921SDaniel Kurtz size_t size) 746c68a2921SDaniel Kurtz { 747bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 748c68a2921SDaniel Kurtz unsigned long flags; 7494f0aba67SShunqian Zheng dma_addr_t pte_dma, iova = (dma_addr_t)_iova; 750c68a2921SDaniel Kurtz phys_addr_t pt_phys; 751c68a2921SDaniel Kurtz u32 dte; 752c68a2921SDaniel Kurtz u32 *pte_addr; 753c68a2921SDaniel Kurtz size_t unmap_size; 754c68a2921SDaniel Kurtz 755c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->dt_lock, flags); 756c68a2921SDaniel Kurtz 757c68a2921SDaniel Kurtz /* 758c68a2921SDaniel Kurtz * pgsize_bitmap specifies iova sizes that fit in one page table 759c68a2921SDaniel Kurtz * (1024 4-KiB pages = 4 MiB). 760c68a2921SDaniel Kurtz * So, size will always be 4096 <= size <= 4194304. 761c68a2921SDaniel Kurtz * Since iommu_unmap() guarantees that both iova and size will be 762c68a2921SDaniel Kurtz * aligned, we will always only be unmapping from a single dte here. 763c68a2921SDaniel Kurtz */ 764c68a2921SDaniel Kurtz dte = rk_domain->dt[rk_iova_dte_index(iova)]; 765c68a2921SDaniel Kurtz /* Just return 0 if iova is unmapped */ 766c68a2921SDaniel Kurtz if (!rk_dte_is_pt_valid(dte)) { 767c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 768c68a2921SDaniel Kurtz return 0; 769c68a2921SDaniel Kurtz } 770c68a2921SDaniel Kurtz 771c68a2921SDaniel Kurtz pt_phys = rk_dte_pt_address(dte); 772c68a2921SDaniel Kurtz pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); 7734f0aba67SShunqian Zheng pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32); 7744f0aba67SShunqian Zheng unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size); 775c68a2921SDaniel Kurtz 776c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 777c68a2921SDaniel Kurtz 778c68a2921SDaniel Kurtz /* Shootdown iotlb entries for iova range that was just unmapped */ 779c68a2921SDaniel Kurtz rk_iommu_zap_iova(rk_domain, iova, unmap_size); 780c68a2921SDaniel Kurtz 781c68a2921SDaniel Kurtz return unmap_size; 782c68a2921SDaniel Kurtz } 783c68a2921SDaniel Kurtz 784c68a2921SDaniel Kurtz static struct rk_iommu *rk_iommu_from_dev(struct device *dev) 785c68a2921SDaniel Kurtz { 786c68a2921SDaniel Kurtz struct iommu_group *group; 787c68a2921SDaniel Kurtz struct device *iommu_dev; 788c68a2921SDaniel Kurtz struct rk_iommu *rk_iommu; 789c68a2921SDaniel Kurtz 790c68a2921SDaniel Kurtz group = iommu_group_get(dev); 791c68a2921SDaniel Kurtz if (!group) 792c68a2921SDaniel Kurtz return NULL; 793c68a2921SDaniel Kurtz iommu_dev = iommu_group_get_iommudata(group); 794c68a2921SDaniel Kurtz rk_iommu = dev_get_drvdata(iommu_dev); 795c68a2921SDaniel Kurtz iommu_group_put(group); 796c68a2921SDaniel Kurtz 797c68a2921SDaniel Kurtz return rk_iommu; 798c68a2921SDaniel Kurtz } 799c68a2921SDaniel Kurtz 800c68a2921SDaniel Kurtz static int rk_iommu_attach_device(struct iommu_domain *domain, 801c68a2921SDaniel Kurtz struct device *dev) 802c68a2921SDaniel Kurtz { 803c68a2921SDaniel Kurtz struct rk_iommu *iommu; 804bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 805c68a2921SDaniel Kurtz unsigned long flags; 806cd6438c5SZhengShunQian int ret, i; 807c68a2921SDaniel Kurtz 808c68a2921SDaniel Kurtz /* 809c68a2921SDaniel Kurtz * Allow 'virtual devices' (e.g., drm) to attach to domain. 810c68a2921SDaniel Kurtz * Such a device does not belong to an iommu group. 811c68a2921SDaniel Kurtz */ 812c68a2921SDaniel Kurtz iommu = rk_iommu_from_dev(dev); 813c68a2921SDaniel Kurtz if (!iommu) 814c68a2921SDaniel Kurtz return 0; 815c68a2921SDaniel Kurtz 816c68a2921SDaniel Kurtz ret = rk_iommu_enable_stall(iommu); 817c68a2921SDaniel Kurtz if (ret) 818c68a2921SDaniel Kurtz return ret; 819c68a2921SDaniel Kurtz 820c68a2921SDaniel Kurtz ret = rk_iommu_force_reset(iommu); 821c68a2921SDaniel Kurtz if (ret) 822c68a2921SDaniel Kurtz return ret; 823c68a2921SDaniel Kurtz 824c68a2921SDaniel Kurtz iommu->domain = domain; 825c68a2921SDaniel Kurtz 826fec3b217SSimon Xue ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq, 827c68a2921SDaniel Kurtz IRQF_SHARED, dev_name(dev), iommu); 828c68a2921SDaniel Kurtz if (ret) 829c68a2921SDaniel Kurtz return ret; 830c68a2921SDaniel Kurtz 831cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) { 8324f0aba67SShunqian Zheng rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 8334f0aba67SShunqian Zheng rk_domain->dt_dma); 834ae8a7910SJohn Keeping rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); 835cd6438c5SZhengShunQian rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); 836cd6438c5SZhengShunQian } 837c68a2921SDaniel Kurtz 838c68a2921SDaniel Kurtz ret = rk_iommu_enable_paging(iommu); 839c68a2921SDaniel Kurtz if (ret) 840c68a2921SDaniel Kurtz return ret; 841c68a2921SDaniel Kurtz 842c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->iommus_lock, flags); 843c68a2921SDaniel Kurtz list_add_tail(&iommu->node, &rk_domain->iommus); 844c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 845c68a2921SDaniel Kurtz 846ec4292deSHeiko Stuebner dev_dbg(dev, "Attached to iommu domain\n"); 847c68a2921SDaniel Kurtz 848c68a2921SDaniel Kurtz rk_iommu_disable_stall(iommu); 849c68a2921SDaniel Kurtz 850c68a2921SDaniel Kurtz return 0; 851c68a2921SDaniel Kurtz } 852c68a2921SDaniel Kurtz 853c68a2921SDaniel Kurtz static void rk_iommu_detach_device(struct iommu_domain *domain, 854c68a2921SDaniel Kurtz struct device *dev) 855c68a2921SDaniel Kurtz { 856c68a2921SDaniel Kurtz struct rk_iommu *iommu; 857bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 858c68a2921SDaniel Kurtz unsigned long flags; 859cd6438c5SZhengShunQian int i; 860c68a2921SDaniel Kurtz 861c68a2921SDaniel Kurtz /* Allow 'virtual devices' (eg drm) to detach from domain */ 862c68a2921SDaniel Kurtz iommu = rk_iommu_from_dev(dev); 863c68a2921SDaniel Kurtz if (!iommu) 864c68a2921SDaniel Kurtz return; 865c68a2921SDaniel Kurtz 866c68a2921SDaniel Kurtz spin_lock_irqsave(&rk_domain->iommus_lock, flags); 867c68a2921SDaniel Kurtz list_del_init(&iommu->node); 868c68a2921SDaniel Kurtz spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 869c68a2921SDaniel Kurtz 870c68a2921SDaniel Kurtz /* Ignore error while disabling, just keep going */ 871c68a2921SDaniel Kurtz rk_iommu_enable_stall(iommu); 872c68a2921SDaniel Kurtz rk_iommu_disable_paging(iommu); 873cd6438c5SZhengShunQian for (i = 0; i < iommu->num_mmu; i++) { 874cd6438c5SZhengShunQian rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0); 875cd6438c5SZhengShunQian rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0); 876cd6438c5SZhengShunQian } 877c68a2921SDaniel Kurtz rk_iommu_disable_stall(iommu); 878c68a2921SDaniel Kurtz 879fec3b217SSimon Xue devm_free_irq(iommu->dev, iommu->irq, iommu); 880c68a2921SDaniel Kurtz 881c68a2921SDaniel Kurtz iommu->domain = NULL; 882c68a2921SDaniel Kurtz 883ec4292deSHeiko Stuebner dev_dbg(dev, "Detached from iommu domain\n"); 884c68a2921SDaniel Kurtz } 885c68a2921SDaniel Kurtz 886bcd516a3SJoerg Roedel static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 887c68a2921SDaniel Kurtz { 888c68a2921SDaniel Kurtz struct rk_iommu_domain *rk_domain; 8894f0aba67SShunqian Zheng struct platform_device *pdev; 8904f0aba67SShunqian Zheng struct device *iommu_dev; 891c68a2921SDaniel Kurtz 892*a93db2f2SShunqian Zheng if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) 893bcd516a3SJoerg Roedel return NULL; 894bcd516a3SJoerg Roedel 8954f0aba67SShunqian Zheng /* Register a pdev per domain, so DMA API can base on this *dev 8964f0aba67SShunqian Zheng * even some virtual master doesn't have an iommu slave 8974f0aba67SShunqian Zheng */ 8984f0aba67SShunqian Zheng pdev = platform_device_register_simple("rk_iommu_domain", 8994f0aba67SShunqian Zheng PLATFORM_DEVID_AUTO, NULL, 0); 9004f0aba67SShunqian Zheng if (IS_ERR(pdev)) 901bcd516a3SJoerg Roedel return NULL; 902c68a2921SDaniel Kurtz 9034f0aba67SShunqian Zheng rk_domain = devm_kzalloc(&pdev->dev, sizeof(*rk_domain), GFP_KERNEL); 9044f0aba67SShunqian Zheng if (!rk_domain) 9054f0aba67SShunqian Zheng goto err_unreg_pdev; 9064f0aba67SShunqian Zheng 9074f0aba67SShunqian Zheng rk_domain->pdev = pdev; 9084f0aba67SShunqian Zheng 909*a93db2f2SShunqian Zheng if (type == IOMMU_DOMAIN_DMA && 910*a93db2f2SShunqian Zheng iommu_get_dma_cookie(&rk_domain->domain)) 9114f0aba67SShunqian Zheng goto err_unreg_pdev; 9124f0aba67SShunqian Zheng 913c68a2921SDaniel Kurtz /* 914c68a2921SDaniel Kurtz * rk32xx iommus use a 2 level pagetable. 915c68a2921SDaniel Kurtz * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 916c68a2921SDaniel Kurtz * Allocate one 4 KiB page for each table. 917c68a2921SDaniel Kurtz */ 918c68a2921SDaniel Kurtz rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); 919c68a2921SDaniel Kurtz if (!rk_domain->dt) 9204f0aba67SShunqian Zheng goto err_put_cookie; 921c68a2921SDaniel Kurtz 9224f0aba67SShunqian Zheng iommu_dev = &pdev->dev; 9234f0aba67SShunqian Zheng rk_domain->dt_dma = dma_map_single(iommu_dev, rk_domain->dt, 9244f0aba67SShunqian Zheng SPAGE_SIZE, DMA_TO_DEVICE); 9254f0aba67SShunqian Zheng if (dma_mapping_error(iommu_dev, rk_domain->dt_dma)) { 9264f0aba67SShunqian Zheng dev_err(iommu_dev, "DMA map error for DT\n"); 9274f0aba67SShunqian Zheng goto err_free_dt; 9284f0aba67SShunqian Zheng } 9294f0aba67SShunqian Zheng 9304f0aba67SShunqian Zheng rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES); 931c68a2921SDaniel Kurtz 932c68a2921SDaniel Kurtz spin_lock_init(&rk_domain->iommus_lock); 933c68a2921SDaniel Kurtz spin_lock_init(&rk_domain->dt_lock); 934c68a2921SDaniel Kurtz INIT_LIST_HEAD(&rk_domain->iommus); 935c68a2921SDaniel Kurtz 936*a93db2f2SShunqian Zheng rk_domain->domain.geometry.aperture_start = 0; 937*a93db2f2SShunqian Zheng rk_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); 938*a93db2f2SShunqian Zheng rk_domain->domain.geometry.force_aperture = true; 939*a93db2f2SShunqian Zheng 940bcd516a3SJoerg Roedel return &rk_domain->domain; 941c68a2921SDaniel Kurtz 9424f0aba67SShunqian Zheng err_free_dt: 9434f0aba67SShunqian Zheng free_page((unsigned long)rk_domain->dt); 9444f0aba67SShunqian Zheng err_put_cookie: 945*a93db2f2SShunqian Zheng if (type == IOMMU_DOMAIN_DMA) 9464f0aba67SShunqian Zheng iommu_put_dma_cookie(&rk_domain->domain); 9474f0aba67SShunqian Zheng err_unreg_pdev: 9484f0aba67SShunqian Zheng platform_device_unregister(pdev); 9494f0aba67SShunqian Zheng 950bcd516a3SJoerg Roedel return NULL; 951c68a2921SDaniel Kurtz } 952c68a2921SDaniel Kurtz 953bcd516a3SJoerg Roedel static void rk_iommu_domain_free(struct iommu_domain *domain) 954c68a2921SDaniel Kurtz { 955bcd516a3SJoerg Roedel struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 956c68a2921SDaniel Kurtz int i; 957c68a2921SDaniel Kurtz 958c68a2921SDaniel Kurtz WARN_ON(!list_empty(&rk_domain->iommus)); 959c68a2921SDaniel Kurtz 960c68a2921SDaniel Kurtz for (i = 0; i < NUM_DT_ENTRIES; i++) { 961c68a2921SDaniel Kurtz u32 dte = rk_domain->dt[i]; 962c68a2921SDaniel Kurtz if (rk_dte_is_pt_valid(dte)) { 963c68a2921SDaniel Kurtz phys_addr_t pt_phys = rk_dte_pt_address(dte); 964c68a2921SDaniel Kurtz u32 *page_table = phys_to_virt(pt_phys); 9654f0aba67SShunqian Zheng dma_unmap_single(&rk_domain->pdev->dev, pt_phys, 9664f0aba67SShunqian Zheng SPAGE_SIZE, DMA_TO_DEVICE); 967c68a2921SDaniel Kurtz free_page((unsigned long)page_table); 968c68a2921SDaniel Kurtz } 969c68a2921SDaniel Kurtz } 970c68a2921SDaniel Kurtz 9714f0aba67SShunqian Zheng dma_unmap_single(&rk_domain->pdev->dev, rk_domain->dt_dma, 9724f0aba67SShunqian Zheng SPAGE_SIZE, DMA_TO_DEVICE); 973c68a2921SDaniel Kurtz free_page((unsigned long)rk_domain->dt); 9744f0aba67SShunqian Zheng 975*a93db2f2SShunqian Zheng if (domain->type == IOMMU_DOMAIN_DMA) 9764f0aba67SShunqian Zheng iommu_put_dma_cookie(&rk_domain->domain); 9774f0aba67SShunqian Zheng 9784f0aba67SShunqian Zheng platform_device_unregister(rk_domain->pdev); 979c68a2921SDaniel Kurtz } 980c68a2921SDaniel Kurtz 981c68a2921SDaniel Kurtz static bool rk_iommu_is_dev_iommu_master(struct device *dev) 982c68a2921SDaniel Kurtz { 983c68a2921SDaniel Kurtz struct device_node *np = dev->of_node; 984c68a2921SDaniel Kurtz int ret; 985c68a2921SDaniel Kurtz 986c68a2921SDaniel Kurtz /* 987c68a2921SDaniel Kurtz * An iommu master has an iommus property containing a list of phandles 988c68a2921SDaniel Kurtz * to iommu nodes, each with an #iommu-cells property with value 0. 989c68a2921SDaniel Kurtz */ 990c68a2921SDaniel Kurtz ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); 991c68a2921SDaniel Kurtz return (ret > 0); 992c68a2921SDaniel Kurtz } 993c68a2921SDaniel Kurtz 994c68a2921SDaniel Kurtz static int rk_iommu_group_set_iommudata(struct iommu_group *group, 995c68a2921SDaniel Kurtz struct device *dev) 996c68a2921SDaniel Kurtz { 997c68a2921SDaniel Kurtz struct device_node *np = dev->of_node; 998c68a2921SDaniel Kurtz struct platform_device *pd; 999c68a2921SDaniel Kurtz int ret; 1000c68a2921SDaniel Kurtz struct of_phandle_args args; 1001c68a2921SDaniel Kurtz 1002c68a2921SDaniel Kurtz /* 1003c68a2921SDaniel Kurtz * An iommu master has an iommus property containing a list of phandles 1004c68a2921SDaniel Kurtz * to iommu nodes, each with an #iommu-cells property with value 0. 1005c68a2921SDaniel Kurtz */ 1006c68a2921SDaniel Kurtz ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0, 1007c68a2921SDaniel Kurtz &args); 1008c68a2921SDaniel Kurtz if (ret) { 1009c68a2921SDaniel Kurtz dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n", 1010c68a2921SDaniel Kurtz np->full_name, ret); 1011c68a2921SDaniel Kurtz return ret; 1012c68a2921SDaniel Kurtz } 1013c68a2921SDaniel Kurtz if (args.args_count != 0) { 1014c68a2921SDaniel Kurtz dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n", 1015c68a2921SDaniel Kurtz args.np->full_name, args.args_count); 1016c68a2921SDaniel Kurtz return -EINVAL; 1017c68a2921SDaniel Kurtz } 1018c68a2921SDaniel Kurtz 1019c68a2921SDaniel Kurtz pd = of_find_device_by_node(args.np); 1020c68a2921SDaniel Kurtz of_node_put(args.np); 1021c68a2921SDaniel Kurtz if (!pd) { 1022c68a2921SDaniel Kurtz dev_err(dev, "iommu %s not found\n", args.np->full_name); 1023c68a2921SDaniel Kurtz return -EPROBE_DEFER; 1024c68a2921SDaniel Kurtz } 1025c68a2921SDaniel Kurtz 1026c68a2921SDaniel Kurtz /* TODO(djkurtz): handle multiple slave iommus for a single master */ 1027c68a2921SDaniel Kurtz iommu_group_set_iommudata(group, &pd->dev, NULL); 1028c68a2921SDaniel Kurtz 1029c68a2921SDaniel Kurtz return 0; 1030c68a2921SDaniel Kurtz } 1031c68a2921SDaniel Kurtz 1032c68a2921SDaniel Kurtz static int rk_iommu_add_device(struct device *dev) 1033c68a2921SDaniel Kurtz { 1034c68a2921SDaniel Kurtz struct iommu_group *group; 1035c68a2921SDaniel Kurtz int ret; 1036c68a2921SDaniel Kurtz 1037c68a2921SDaniel Kurtz if (!rk_iommu_is_dev_iommu_master(dev)) 1038c68a2921SDaniel Kurtz return -ENODEV; 1039c68a2921SDaniel Kurtz 1040c68a2921SDaniel Kurtz group = iommu_group_get(dev); 1041c68a2921SDaniel Kurtz if (!group) { 1042c68a2921SDaniel Kurtz group = iommu_group_alloc(); 1043c68a2921SDaniel Kurtz if (IS_ERR(group)) { 1044c68a2921SDaniel Kurtz dev_err(dev, "Failed to allocate IOMMU group\n"); 1045c68a2921SDaniel Kurtz return PTR_ERR(group); 1046c68a2921SDaniel Kurtz } 1047c68a2921SDaniel Kurtz } 1048c68a2921SDaniel Kurtz 1049c68a2921SDaniel Kurtz ret = iommu_group_add_device(group, dev); 1050c68a2921SDaniel Kurtz if (ret) 1051c68a2921SDaniel Kurtz goto err_put_group; 1052c68a2921SDaniel Kurtz 1053c68a2921SDaniel Kurtz ret = rk_iommu_group_set_iommudata(group, dev); 1054c68a2921SDaniel Kurtz if (ret) 1055c68a2921SDaniel Kurtz goto err_remove_device; 1056c68a2921SDaniel Kurtz 1057c68a2921SDaniel Kurtz iommu_group_put(group); 1058c68a2921SDaniel Kurtz 1059c68a2921SDaniel Kurtz return 0; 1060c68a2921SDaniel Kurtz 1061c68a2921SDaniel Kurtz err_remove_device: 1062c68a2921SDaniel Kurtz iommu_group_remove_device(dev); 1063c68a2921SDaniel Kurtz err_put_group: 1064c68a2921SDaniel Kurtz iommu_group_put(group); 1065c68a2921SDaniel Kurtz return ret; 1066c68a2921SDaniel Kurtz } 1067c68a2921SDaniel Kurtz 1068c68a2921SDaniel Kurtz static void rk_iommu_remove_device(struct device *dev) 1069c68a2921SDaniel Kurtz { 1070c68a2921SDaniel Kurtz if (!rk_iommu_is_dev_iommu_master(dev)) 1071c68a2921SDaniel Kurtz return; 1072c68a2921SDaniel Kurtz 1073c68a2921SDaniel Kurtz iommu_group_remove_device(dev); 1074c68a2921SDaniel Kurtz } 1075c68a2921SDaniel Kurtz 1076c68a2921SDaniel Kurtz static const struct iommu_ops rk_iommu_ops = { 1077bcd516a3SJoerg Roedel .domain_alloc = rk_iommu_domain_alloc, 1078bcd516a3SJoerg Roedel .domain_free = rk_iommu_domain_free, 1079c68a2921SDaniel Kurtz .attach_dev = rk_iommu_attach_device, 1080c68a2921SDaniel Kurtz .detach_dev = rk_iommu_detach_device, 1081c68a2921SDaniel Kurtz .map = rk_iommu_map, 1082c68a2921SDaniel Kurtz .unmap = rk_iommu_unmap, 1083e6d0f473SSimon Xue .map_sg = default_iommu_map_sg, 1084c68a2921SDaniel Kurtz .add_device = rk_iommu_add_device, 1085c68a2921SDaniel Kurtz .remove_device = rk_iommu_remove_device, 1086c68a2921SDaniel Kurtz .iova_to_phys = rk_iommu_iova_to_phys, 1087c68a2921SDaniel Kurtz .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, 1088c68a2921SDaniel Kurtz }; 1089c68a2921SDaniel Kurtz 10904f0aba67SShunqian Zheng static int rk_iommu_domain_probe(struct platform_device *pdev) 10914f0aba67SShunqian Zheng { 10924f0aba67SShunqian Zheng struct device *dev = &pdev->dev; 10934f0aba67SShunqian Zheng 10944f0aba67SShunqian Zheng dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL); 10954f0aba67SShunqian Zheng if (!dev->dma_parms) 10964f0aba67SShunqian Zheng return -ENOMEM; 10974f0aba67SShunqian Zheng 10984f0aba67SShunqian Zheng /* Set dma_ops for dev, otherwise it would be dummy_dma_ops */ 10994f0aba67SShunqian Zheng arch_setup_dma_ops(dev, 0, DMA_BIT_MASK(32), NULL, false); 11004f0aba67SShunqian Zheng 11014f0aba67SShunqian Zheng dma_set_max_seg_size(dev, DMA_BIT_MASK(32)); 11024f0aba67SShunqian Zheng dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32)); 11034f0aba67SShunqian Zheng 11044f0aba67SShunqian Zheng return 0; 11054f0aba67SShunqian Zheng } 11064f0aba67SShunqian Zheng 11074f0aba67SShunqian Zheng static struct platform_driver rk_iommu_domain_driver = { 11084f0aba67SShunqian Zheng .probe = rk_iommu_domain_probe, 11094f0aba67SShunqian Zheng .driver = { 11104f0aba67SShunqian Zheng .name = "rk_iommu_domain", 11114f0aba67SShunqian Zheng }, 11124f0aba67SShunqian Zheng }; 11134f0aba67SShunqian Zheng 1114c68a2921SDaniel Kurtz static int rk_iommu_probe(struct platform_device *pdev) 1115c68a2921SDaniel Kurtz { 1116c68a2921SDaniel Kurtz struct device *dev = &pdev->dev; 1117c68a2921SDaniel Kurtz struct rk_iommu *iommu; 1118c68a2921SDaniel Kurtz struct resource *res; 11193d08f434SShunqian Zheng int num_res = pdev->num_resources; 1120cd6438c5SZhengShunQian int i; 1121c68a2921SDaniel Kurtz 1122c68a2921SDaniel Kurtz iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); 1123c68a2921SDaniel Kurtz if (!iommu) 1124c68a2921SDaniel Kurtz return -ENOMEM; 1125c68a2921SDaniel Kurtz 1126c68a2921SDaniel Kurtz platform_set_drvdata(pdev, iommu); 1127c68a2921SDaniel Kurtz iommu->dev = dev; 1128cd6438c5SZhengShunQian iommu->num_mmu = 0; 11293d08f434SShunqian Zheng 11303d08f434SShunqian Zheng iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * num_res, 1131cd6438c5SZhengShunQian GFP_KERNEL); 1132cd6438c5SZhengShunQian if (!iommu->bases) 1133cd6438c5SZhengShunQian return -ENOMEM; 1134c68a2921SDaniel Kurtz 11353d08f434SShunqian Zheng for (i = 0; i < num_res; i++) { 1136cd6438c5SZhengShunQian res = platform_get_resource(pdev, IORESOURCE_MEM, i); 11378d7f2d84STomeu Vizoso if (!res) 11388d7f2d84STomeu Vizoso continue; 1139cd6438c5SZhengShunQian iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res); 1140cd6438c5SZhengShunQian if (IS_ERR(iommu->bases[i])) 1141cd6438c5SZhengShunQian continue; 1142cd6438c5SZhengShunQian iommu->num_mmu++; 1143cd6438c5SZhengShunQian } 1144cd6438c5SZhengShunQian if (iommu->num_mmu == 0) 1145cd6438c5SZhengShunQian return PTR_ERR(iommu->bases[0]); 1146c68a2921SDaniel Kurtz 1147c68a2921SDaniel Kurtz iommu->irq = platform_get_irq(pdev, 0); 1148c68a2921SDaniel Kurtz if (iommu->irq < 0) { 1149c68a2921SDaniel Kurtz dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq); 1150c68a2921SDaniel Kurtz return -ENXIO; 1151c68a2921SDaniel Kurtz } 1152c68a2921SDaniel Kurtz 1153c68a2921SDaniel Kurtz return 0; 1154c68a2921SDaniel Kurtz } 1155c68a2921SDaniel Kurtz 1156c68a2921SDaniel Kurtz static int rk_iommu_remove(struct platform_device *pdev) 1157c68a2921SDaniel Kurtz { 1158c68a2921SDaniel Kurtz return 0; 1159c68a2921SDaniel Kurtz } 1160c68a2921SDaniel Kurtz 1161c68a2921SDaniel Kurtz static const struct of_device_id rk_iommu_dt_ids[] = { 1162c68a2921SDaniel Kurtz { .compatible = "rockchip,iommu" }, 1163c68a2921SDaniel Kurtz { /* sentinel */ } 1164c68a2921SDaniel Kurtz }; 1165c68a2921SDaniel Kurtz MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); 1166c68a2921SDaniel Kurtz 1167c68a2921SDaniel Kurtz static struct platform_driver rk_iommu_driver = { 1168c68a2921SDaniel Kurtz .probe = rk_iommu_probe, 1169c68a2921SDaniel Kurtz .remove = rk_iommu_remove, 1170c68a2921SDaniel Kurtz .driver = { 1171c68a2921SDaniel Kurtz .name = "rk_iommu", 1172d9e7eb15SArnd Bergmann .of_match_table = rk_iommu_dt_ids, 1173c68a2921SDaniel Kurtz }, 1174c68a2921SDaniel Kurtz }; 1175c68a2921SDaniel Kurtz 1176c68a2921SDaniel Kurtz static int __init rk_iommu_init(void) 1177c68a2921SDaniel Kurtz { 1178425061b0SThierry Reding struct device_node *np; 1179c68a2921SDaniel Kurtz int ret; 1180c68a2921SDaniel Kurtz 1181425061b0SThierry Reding np = of_find_matching_node(NULL, rk_iommu_dt_ids); 1182425061b0SThierry Reding if (!np) 1183425061b0SThierry Reding return 0; 1184425061b0SThierry Reding 1185425061b0SThierry Reding of_node_put(np); 1186425061b0SThierry Reding 1187c68a2921SDaniel Kurtz ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops); 1188c68a2921SDaniel Kurtz if (ret) 1189c68a2921SDaniel Kurtz return ret; 1190c68a2921SDaniel Kurtz 11914f0aba67SShunqian Zheng ret = platform_driver_register(&rk_iommu_domain_driver); 11924f0aba67SShunqian Zheng if (ret) 11934f0aba67SShunqian Zheng return ret; 11944f0aba67SShunqian Zheng 11954f0aba67SShunqian Zheng ret = platform_driver_register(&rk_iommu_driver); 11964f0aba67SShunqian Zheng if (ret) 11974f0aba67SShunqian Zheng platform_driver_unregister(&rk_iommu_domain_driver); 11984f0aba67SShunqian Zheng return ret; 1199c68a2921SDaniel Kurtz } 1200c68a2921SDaniel Kurtz static void __exit rk_iommu_exit(void) 1201c68a2921SDaniel Kurtz { 1202c68a2921SDaniel Kurtz platform_driver_unregister(&rk_iommu_driver); 12034f0aba67SShunqian Zheng platform_driver_unregister(&rk_iommu_domain_driver); 1204c68a2921SDaniel Kurtz } 1205c68a2921SDaniel Kurtz 1206c68a2921SDaniel Kurtz subsys_initcall(rk_iommu_init); 1207c68a2921SDaniel Kurtz module_exit(rk_iommu_exit); 1208c68a2921SDaniel Kurtz 1209c68a2921SDaniel Kurtz MODULE_DESCRIPTION("IOMMU API for Rockchip"); 1210c68a2921SDaniel Kurtz MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com> and Daniel Kurtz <djkurtz@chromium.org>"); 1211c68a2921SDaniel Kurtz MODULE_ALIAS("platform:rockchip-iommu"); 1212c68a2921SDaniel Kurtz MODULE_LICENSE("GPL v2"); 1213