1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD. 3 // Copyright (C) 2019-2020, Cerno 4 5 #include <linux/bitfield.h> 6 #include <linux/bug.h> 7 #include <linux/clk.h> 8 #include <linux/device.h> 9 #include <linux/dma-direction.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/err.h> 12 #include <linux/errno.h> 13 #include <linux/interrupt.h> 14 #include <linux/iommu.h> 15 #include <linux/iopoll.h> 16 #include <linux/ioport.h> 17 #include <linux/log2.h> 18 #include <linux/module.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/reset.h> 24 #include <linux/sizes.h> 25 #include <linux/slab.h> 26 #include <linux/spinlock.h> 27 #include <linux/types.h> 28 29 #include "iommu-pages.h" 30 31 #define IOMMU_RESET_REG 0x010 32 #define IOMMU_RESET_RELEASE_ALL 0xffffffff 33 #define IOMMU_ENABLE_REG 0x020 34 #define IOMMU_ENABLE_ENABLE BIT(0) 35 36 #define IOMMU_BYPASS_REG 0x030 37 #define IOMMU_AUTO_GATING_REG 0x040 38 #define IOMMU_AUTO_GATING_ENABLE BIT(0) 39 40 #define IOMMU_WBUF_CTRL_REG 0x044 41 #define IOMMU_OOO_CTRL_REG 0x048 42 #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c 43 #define IOMMU_TTB_REG 0x050 44 #define IOMMU_TLB_ENABLE_REG 0x060 45 #define IOMMU_TLB_PREFETCH_REG 0x070 46 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m) 47 48 #define IOMMU_TLB_FLUSH_REG 0x080 49 #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17) 50 #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16) 51 #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0)) 52 53 #define IOMMU_TLB_IVLD_ADDR_REG 0x090 54 #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094 55 #define IOMMU_TLB_IVLD_ENABLE_REG 0x098 56 #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0) 57 58 #define IOMMU_PC_IVLD_ADDR_REG 0x0a0 59 #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8 60 #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0) 61 62 #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4) 63 #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2))) 64 #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1)) 65 66 #define IOMMU_DM_AUT_OVWT_REG 0x0d0 67 #define IOMMU_INT_ENABLE_REG 0x100 68 #define IOMMU_INT_CLR_REG 0x104 69 #define IOMMU_INT_STA_REG 0x108 70 #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4) 71 #define IOMMU_INT_ERR_ADDR_L1_REG 0x130 72 #define IOMMU_INT_ERR_ADDR_L2_REG 0x134 73 #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4) 74 #define IOMMU_L1PG_INT_REG 0x0180 75 #define IOMMU_L2PG_INT_REG 0x0184 76 77 #define IOMMU_INT_INVALID_L2PG BIT(17) 78 #define IOMMU_INT_INVALID_L1PG BIT(16) 79 #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m) 80 #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \ 81 IOMMU_INT_MASTER_PERMISSION(1) | \ 82 IOMMU_INT_MASTER_PERMISSION(2) | \ 83 IOMMU_INT_MASTER_PERMISSION(3) | \ 84 IOMMU_INT_MASTER_PERMISSION(4) | \ 85 IOMMU_INT_MASTER_PERMISSION(5)) 86 #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \ 87 IOMMU_INT_INVALID_L2PG | \ 88 IOMMU_INT_MASTER_MASK) 89 90 #define PT_ENTRY_SIZE sizeof(u32) 91 92 #define NUM_DT_ENTRIES 4096 93 #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE) 94 95 #define NUM_PT_ENTRIES 256 96 #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE) 97 98 #define SPAGE_SIZE 4096 99 100 struct sun50i_iommu { 101 struct iommu_device iommu; 102 103 /* Lock to modify the IOMMU registers */ 104 spinlock_t iommu_lock; 105 106 struct device *dev; 107 void __iomem *base; 108 struct reset_control *reset; 109 struct clk *clk; 110 111 struct iommu_domain *domain; 112 struct kmem_cache *pt_pool; 113 }; 114 115 struct sun50i_iommu_domain { 116 struct iommu_domain domain; 117 118 /* Number of devices attached to the domain */ 119 refcount_t refcnt; 120 121 /* L1 Page Table */ 122 u32 *dt; 123 dma_addr_t dt_dma; 124 125 struct sun50i_iommu *iommu; 126 }; 127 128 static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain) 129 { 130 return container_of(domain, struct sun50i_iommu_domain, domain); 131 } 132 133 static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev) 134 { 135 return dev_iommu_priv_get(dev); 136 } 137 138 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) 139 { 140 return readl(iommu->base + offset); 141 } 142 143 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) 144 { 145 writel(value, iommu->base + offset); 146 } 147 148 /* 149 * The Allwinner H6 IOMMU uses a 2-level page table. 150 * 151 * The first level is the usual Directory Table (DT), that consists of 152 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page 153 * Table (PT). 154 * 155 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each 156 * pointing to a 4kB page of physical memory. 157 * 158 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG 159 * register that contains its physical address. 160 */ 161 162 #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20) 163 #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12) 164 #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0) 165 166 static u32 sun50i_iova_get_dte_index(dma_addr_t iova) 167 { 168 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova); 169 } 170 171 static u32 sun50i_iova_get_pte_index(dma_addr_t iova) 172 { 173 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova); 174 } 175 176 static u32 sun50i_iova_get_page_offset(dma_addr_t iova) 177 { 178 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova); 179 } 180 181 /* 182 * Each Directory Table Entry has a Page Table address and a valid 183 * bit: 184 185 * +---------------------+-----------+-+ 186 * | PT address | Reserved |V| 187 * +---------------------+-----------+-+ 188 * 31:10 - Page Table address 189 * 9:2 - Reserved 190 * 1:0 - 1 if the entry is valid 191 */ 192 193 #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10) 194 #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0) 195 #define SUN50I_DTE_PT_VALID 1 196 197 static phys_addr_t sun50i_dte_get_pt_address(u32 dte) 198 { 199 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK; 200 } 201 202 static bool sun50i_dte_is_pt_valid(u32 dte) 203 { 204 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID; 205 } 206 207 static u32 sun50i_mk_dte(dma_addr_t pt_dma) 208 { 209 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID; 210 } 211 212 /* 213 * Each PTE has a Page address, an authority index and a valid bit: 214 * 215 * +----------------+-----+-----+-----+---+-----+ 216 * | Page address | Rsv | ACI | Rsv | V | Rsv | 217 * +----------------+-----+-----+-----+---+-----+ 218 * 31:12 - Page address 219 * 11:8 - Reserved 220 * 7:4 - Authority Control Index 221 * 3:2 - Reserved 222 * 1 - 1 if the entry is valid 223 * 0 - Reserved 224 * 225 * The way permissions work is that the IOMMU has 16 "domains" that 226 * can be configured to give each masters either read or write 227 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain 228 * 0 seems like the default domain, and its permissions in the 229 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really 230 * useful to enforce any particular permission. 231 * 232 * Each page entry will then have a reference to the domain they are 233 * affected to, so that we can actually enforce them on a per-page 234 * basis. 235 * 236 * In order to make it work with the IOMMU framework, we will be using 237 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE 238 * depending on the permission we want to enforce. Each domain will 239 * have each master setup in the same way, since the IOMMU framework 240 * doesn't seem to restrict page access on a per-device basis. And 241 * then we will use the relevant domain index when generating the page 242 * table entry depending on the permissions we want to be enforced. 243 */ 244 245 enum sun50i_iommu_aci { 246 SUN50I_IOMMU_ACI_DO_NOT_USE = 0, 247 SUN50I_IOMMU_ACI_NONE, 248 SUN50I_IOMMU_ACI_RD, 249 SUN50I_IOMMU_ACI_WR, 250 SUN50I_IOMMU_ACI_RD_WR, 251 }; 252 253 #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12) 254 #define SUN50I_PTE_ACI_MASK GENMASK(7, 4) 255 #define SUN50I_PTE_PAGE_VALID BIT(1) 256 257 static phys_addr_t sun50i_pte_get_page_address(u32 pte) 258 { 259 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK; 260 } 261 262 static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte) 263 { 264 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte); 265 } 266 267 static bool sun50i_pte_is_page_valid(u32 pte) 268 { 269 return pte & SUN50I_PTE_PAGE_VALID; 270 } 271 272 static u32 sun50i_mk_pte(phys_addr_t page, int prot) 273 { 274 enum sun50i_iommu_aci aci; 275 u32 flags = 0; 276 277 if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE)) 278 aci = SUN50I_IOMMU_ACI_RD_WR; 279 else if (prot & IOMMU_READ) 280 aci = SUN50I_IOMMU_ACI_RD; 281 else if (prot & IOMMU_WRITE) 282 aci = SUN50I_IOMMU_ACI_WR; 283 else 284 aci = SUN50I_IOMMU_ACI_NONE; 285 286 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci); 287 page &= SUN50I_PTE_PAGE_ADDRESS_MASK; 288 return page | flags | SUN50I_PTE_PAGE_VALID; 289 } 290 291 static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain, 292 void *vaddr, unsigned int count) 293 { 294 struct sun50i_iommu *iommu = sun50i_domain->iommu; 295 dma_addr_t dma = virt_to_phys(vaddr); 296 size_t size = count * PT_ENTRY_SIZE; 297 298 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); 299 } 300 301 static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu, 302 unsigned long iova) 303 { 304 u32 reg; 305 int ret; 306 307 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova); 308 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12)); 309 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 310 IOMMU_TLB_IVLD_ENABLE_ENABLE); 311 312 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG, 313 reg, !reg, 1, 2000); 314 if (ret) 315 dev_warn(iommu->dev, "TLB invalidation timed out!\n"); 316 } 317 318 static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu, 319 unsigned long iova) 320 { 321 u32 reg; 322 int ret; 323 324 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova); 325 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 326 IOMMU_PC_IVLD_ENABLE_ENABLE); 327 328 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG, 329 reg, !reg, 1, 2000); 330 if (ret) 331 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n"); 332 } 333 334 static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu, 335 unsigned long iova, size_t size) 336 { 337 assert_spin_locked(&iommu->iommu_lock); 338 339 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0); 340 341 sun50i_iommu_zap_iova(iommu, iova); 342 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE); 343 if (size > SPAGE_SIZE) { 344 sun50i_iommu_zap_iova(iommu, iova + size); 345 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE); 346 } 347 sun50i_iommu_zap_ptw_cache(iommu, iova); 348 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M); 349 if (size > SZ_1M) { 350 sun50i_iommu_zap_ptw_cache(iommu, iova + size); 351 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M); 352 } 353 354 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 355 } 356 357 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) 358 { 359 u32 reg; 360 int ret; 361 362 assert_spin_locked(&iommu->iommu_lock); 363 364 iommu_write(iommu, 365 IOMMU_TLB_FLUSH_REG, 366 IOMMU_TLB_FLUSH_PTW_CACHE | 367 IOMMU_TLB_FLUSH_MACRO_TLB | 368 IOMMU_TLB_FLUSH_MICRO_TLB(5) | 369 IOMMU_TLB_FLUSH_MICRO_TLB(4) | 370 IOMMU_TLB_FLUSH_MICRO_TLB(3) | 371 IOMMU_TLB_FLUSH_MICRO_TLB(2) | 372 IOMMU_TLB_FLUSH_MICRO_TLB(1) | 373 IOMMU_TLB_FLUSH_MICRO_TLB(0)); 374 375 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG, 376 reg, !reg, 377 1, 2000); 378 if (ret) 379 dev_warn(iommu->dev, "TLB Flush timed out!\n"); 380 381 return ret; 382 } 383 384 static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain) 385 { 386 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 387 struct sun50i_iommu *iommu = sun50i_domain->iommu; 388 unsigned long flags; 389 390 /* 391 * At boot, we'll have a first call into .flush_iotlb_all right after 392 * .probe_device, and since we link our (single) domain to our iommu in 393 * the .attach_device callback, we don't have that pointer set. 394 * 395 * It shouldn't really be any trouble to ignore it though since we flush 396 * all caches as part of the device powerup. 397 */ 398 if (!iommu) 399 return; 400 401 spin_lock_irqsave(&iommu->iommu_lock, flags); 402 sun50i_iommu_flush_all_tlb(iommu); 403 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 404 } 405 406 static int sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain, 407 unsigned long iova, size_t size) 408 { 409 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 410 struct sun50i_iommu *iommu = sun50i_domain->iommu; 411 unsigned long flags; 412 413 spin_lock_irqsave(&iommu->iommu_lock, flags); 414 sun50i_iommu_zap_range(iommu, iova, size); 415 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 416 417 return 0; 418 } 419 420 static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain, 421 struct iommu_iotlb_gather *gather) 422 { 423 sun50i_iommu_flush_iotlb_all(domain); 424 } 425 426 static int sun50i_iommu_enable(struct sun50i_iommu *iommu) 427 { 428 struct sun50i_iommu_domain *sun50i_domain; 429 unsigned long flags; 430 int ret; 431 432 if (!iommu->domain) 433 return 0; 434 435 sun50i_domain = to_sun50i_domain(iommu->domain); 436 437 ret = reset_control_deassert(iommu->reset); 438 if (ret) 439 return ret; 440 441 ret = clk_prepare_enable(iommu->clk); 442 if (ret) 443 goto err_reset_assert; 444 445 spin_lock_irqsave(&iommu->iommu_lock, flags); 446 447 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); 448 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, 449 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) | 450 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) | 451 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) | 452 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) | 453 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) | 454 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5)); 455 iommu_write(iommu, IOMMU_BYPASS_REG, 0); 456 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); 457 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), 458 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 459 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 460 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 461 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 462 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 463 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 464 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 465 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 466 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 467 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 468 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) | 469 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5)); 470 471 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), 472 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) | 473 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) | 474 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) | 475 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) | 476 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) | 477 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5)); 478 479 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), 480 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) | 481 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) | 482 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) | 483 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) | 484 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) | 485 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5)); 486 487 ret = sun50i_iommu_flush_all_tlb(iommu); 488 if (ret) { 489 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 490 goto err_clk_disable; 491 } 492 493 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 494 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); 495 496 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 497 498 return 0; 499 500 err_clk_disable: 501 clk_disable_unprepare(iommu->clk); 502 503 err_reset_assert: 504 reset_control_assert(iommu->reset); 505 506 return ret; 507 } 508 509 static void sun50i_iommu_disable(struct sun50i_iommu *iommu) 510 { 511 unsigned long flags; 512 513 spin_lock_irqsave(&iommu->iommu_lock, flags); 514 515 iommu_write(iommu, IOMMU_ENABLE_REG, 0); 516 iommu_write(iommu, IOMMU_TTB_REG, 0); 517 518 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 519 520 clk_disable_unprepare(iommu->clk); 521 reset_control_assert(iommu->reset); 522 } 523 524 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, 525 gfp_t gfp) 526 { 527 dma_addr_t pt_dma; 528 u32 *page_table; 529 530 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); 531 if (!page_table) 532 return ERR_PTR(-ENOMEM); 533 534 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); 535 if (dma_mapping_error(iommu->dev, pt_dma)) { 536 dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); 537 kmem_cache_free(iommu->pt_pool, page_table); 538 return ERR_PTR(-ENOMEM); 539 } 540 541 /* We rely on the physical address and DMA address being the same */ 542 WARN_ON(pt_dma != virt_to_phys(page_table)); 543 544 return page_table; 545 } 546 547 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, 548 u32 *page_table) 549 { 550 phys_addr_t pt_phys = virt_to_phys(page_table); 551 552 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); 553 kmem_cache_free(iommu->pt_pool, page_table); 554 } 555 556 static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, 557 dma_addr_t iova, gfp_t gfp) 558 { 559 struct sun50i_iommu *iommu = sun50i_domain->iommu; 560 u32 *page_table; 561 u32 *dte_addr; 562 u32 old_dte; 563 u32 dte; 564 565 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 566 dte = *dte_addr; 567 if (sun50i_dte_is_pt_valid(dte)) { 568 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte); 569 return (u32 *)phys_to_virt(pt_phys); 570 } 571 572 page_table = sun50i_iommu_alloc_page_table(iommu, gfp); 573 if (IS_ERR(page_table)) 574 return page_table; 575 576 dte = sun50i_mk_dte(virt_to_phys(page_table)); 577 old_dte = cmpxchg(dte_addr, 0, dte); 578 if (old_dte) { 579 phys_addr_t installed_pt_phys = 580 sun50i_dte_get_pt_address(old_dte); 581 u32 *installed_pt = phys_to_virt(installed_pt_phys); 582 u32 *drop_pt = page_table; 583 584 page_table = installed_pt; 585 dte = old_dte; 586 sun50i_iommu_free_page_table(iommu, drop_pt); 587 } 588 589 sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES); 590 sun50i_table_flush(sun50i_domain, dte_addr, 1); 591 592 return page_table; 593 } 594 595 static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, 596 phys_addr_t paddr, size_t size, size_t count, 597 int prot, gfp_t gfp, size_t *mapped) 598 { 599 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 600 struct sun50i_iommu *iommu = sun50i_domain->iommu; 601 u32 pte_index; 602 u32 *page_table, *pte_addr; 603 int ret = 0; 604 605 /* the IOMMU can only handle 32-bit addresses, both input and output */ 606 if ((uint64_t)paddr >> 32) { 607 ret = -EINVAL; 608 dev_warn_once(iommu->dev, 609 "attempt to map address beyond 4GB\n"); 610 goto out; 611 } 612 613 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp); 614 if (IS_ERR(page_table)) { 615 ret = PTR_ERR(page_table); 616 goto out; 617 } 618 619 pte_index = sun50i_iova_get_pte_index(iova); 620 pte_addr = &page_table[pte_index]; 621 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) { 622 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr); 623 dev_err(iommu->dev, 624 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n", 625 &iova, &page_phys, &paddr, prot); 626 ret = -EBUSY; 627 goto out; 628 } 629 630 *pte_addr = sun50i_mk_pte(paddr, prot); 631 sun50i_table_flush(sun50i_domain, pte_addr, 1); 632 *mapped = size; 633 634 out: 635 return ret; 636 } 637 638 static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, 639 size_t size, size_t count, struct iommu_iotlb_gather *gather) 640 { 641 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 642 phys_addr_t pt_phys; 643 u32 *pte_addr; 644 u32 dte; 645 646 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 647 if (!sun50i_dte_is_pt_valid(dte)) 648 return 0; 649 650 pt_phys = sun50i_dte_get_pt_address(dte); 651 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova); 652 653 if (!sun50i_pte_is_page_valid(*pte_addr)) 654 return 0; 655 656 memset(pte_addr, 0, sizeof(*pte_addr)); 657 sun50i_table_flush(sun50i_domain, pte_addr, 1); 658 659 return SZ_4K; 660 } 661 662 static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain, 663 dma_addr_t iova) 664 { 665 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 666 phys_addr_t pt_phys; 667 u32 *page_table; 668 u32 dte, pte; 669 670 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 671 if (!sun50i_dte_is_pt_valid(dte)) 672 return 0; 673 674 pt_phys = sun50i_dte_get_pt_address(dte); 675 page_table = (u32 *)phys_to_virt(pt_phys); 676 pte = page_table[sun50i_iova_get_pte_index(iova)]; 677 if (!sun50i_pte_is_page_valid(pte)) 678 return 0; 679 680 return sun50i_pte_get_page_address(pte) + 681 sun50i_iova_get_page_offset(iova); 682 } 683 684 static struct iommu_domain * 685 sun50i_iommu_domain_alloc_paging(struct device *dev) 686 { 687 struct sun50i_iommu_domain *sun50i_domain; 688 689 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL); 690 if (!sun50i_domain) 691 return NULL; 692 693 sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL | GFP_DMA32, 694 get_order(DT_SIZE)); 695 if (!sun50i_domain->dt) 696 goto err_free_domain; 697 698 refcount_set(&sun50i_domain->refcnt, 1); 699 700 sun50i_domain->domain.geometry.aperture_start = 0; 701 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); 702 sun50i_domain->domain.geometry.force_aperture = true; 703 704 return &sun50i_domain->domain; 705 706 err_free_domain: 707 kfree(sun50i_domain); 708 709 return NULL; 710 } 711 712 static void sun50i_iommu_domain_free(struct iommu_domain *domain) 713 { 714 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 715 716 iommu_free_pages(sun50i_domain->dt, get_order(DT_SIZE)); 717 sun50i_domain->dt = NULL; 718 719 kfree(sun50i_domain); 720 } 721 722 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, 723 struct sun50i_iommu_domain *sun50i_domain) 724 { 725 iommu->domain = &sun50i_domain->domain; 726 sun50i_domain->iommu = iommu; 727 728 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, 729 DT_SIZE, DMA_TO_DEVICE); 730 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { 731 dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); 732 return -ENOMEM; 733 } 734 735 return sun50i_iommu_enable(iommu); 736 } 737 738 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, 739 struct sun50i_iommu_domain *sun50i_domain) 740 { 741 unsigned int i; 742 743 for (i = 0; i < NUM_DT_ENTRIES; i++) { 744 phys_addr_t pt_phys; 745 u32 *page_table; 746 u32 *dte_addr; 747 u32 dte; 748 749 dte_addr = &sun50i_domain->dt[i]; 750 dte = *dte_addr; 751 if (!sun50i_dte_is_pt_valid(dte)) 752 continue; 753 754 memset(dte_addr, 0, sizeof(*dte_addr)); 755 sun50i_table_flush(sun50i_domain, dte_addr, 1); 756 757 pt_phys = sun50i_dte_get_pt_address(dte); 758 page_table = phys_to_virt(pt_phys); 759 sun50i_iommu_free_page_table(iommu, page_table); 760 } 761 762 763 sun50i_iommu_disable(iommu); 764 765 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), 766 DT_SIZE, DMA_TO_DEVICE); 767 768 iommu->domain = NULL; 769 } 770 771 static int sun50i_iommu_identity_attach(struct iommu_domain *identity_domain, 772 struct device *dev) 773 { 774 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); 775 struct sun50i_iommu_domain *sun50i_domain; 776 777 dev_dbg(dev, "Detaching from IOMMU domain\n"); 778 779 if (iommu->domain == identity_domain) 780 return 0; 781 782 sun50i_domain = to_sun50i_domain(iommu->domain); 783 if (refcount_dec_and_test(&sun50i_domain->refcnt)) 784 sun50i_iommu_detach_domain(iommu, sun50i_domain); 785 return 0; 786 } 787 788 static struct iommu_domain_ops sun50i_iommu_identity_ops = { 789 .attach_dev = sun50i_iommu_identity_attach, 790 }; 791 792 static struct iommu_domain sun50i_iommu_identity_domain = { 793 .type = IOMMU_DOMAIN_IDENTITY, 794 .ops = &sun50i_iommu_identity_ops, 795 }; 796 797 static int sun50i_iommu_attach_device(struct iommu_domain *domain, 798 struct device *dev) 799 { 800 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 801 struct sun50i_iommu *iommu; 802 803 iommu = sun50i_iommu_from_dev(dev); 804 if (!iommu) 805 return -ENODEV; 806 807 dev_dbg(dev, "Attaching to IOMMU domain\n"); 808 809 refcount_inc(&sun50i_domain->refcnt); 810 811 if (iommu->domain == domain) 812 return 0; 813 814 sun50i_iommu_identity_attach(&sun50i_iommu_identity_domain, dev); 815 816 sun50i_iommu_attach_domain(iommu, sun50i_domain); 817 818 return 0; 819 } 820 821 static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) 822 { 823 struct sun50i_iommu *iommu; 824 825 iommu = sun50i_iommu_from_dev(dev); 826 if (!iommu) 827 return ERR_PTR(-ENODEV); 828 829 return &iommu->iommu; 830 } 831 832 static int sun50i_iommu_of_xlate(struct device *dev, 833 const struct of_phandle_args *args) 834 { 835 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); 836 unsigned id = args->args[0]; 837 838 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev)); 839 840 return iommu_fwspec_add_ids(dev, &id, 1); 841 } 842 843 static const struct iommu_ops sun50i_iommu_ops = { 844 .identity_domain = &sun50i_iommu_identity_domain, 845 .pgsize_bitmap = SZ_4K, 846 .device_group = generic_single_device_group, 847 .domain_alloc_paging = sun50i_iommu_domain_alloc_paging, 848 .of_xlate = sun50i_iommu_of_xlate, 849 .probe_device = sun50i_iommu_probe_device, 850 .default_domain_ops = &(const struct iommu_domain_ops) { 851 .attach_dev = sun50i_iommu_attach_device, 852 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all, 853 .iotlb_sync_map = sun50i_iommu_iotlb_sync_map, 854 .iotlb_sync = sun50i_iommu_iotlb_sync, 855 .iova_to_phys = sun50i_iommu_iova_to_phys, 856 .map_pages = sun50i_iommu_map, 857 .unmap_pages = sun50i_iommu_unmap, 858 .free = sun50i_iommu_domain_free, 859 } 860 }; 861 862 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, 863 unsigned master, phys_addr_t iova, 864 unsigned prot) 865 { 866 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", 867 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd"); 868 869 if (iommu->domain) 870 report_iommu_fault(iommu->domain, iommu->dev, iova, prot); 871 else 872 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); 873 874 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE); 875 } 876 877 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, 878 unsigned addr_reg, 879 unsigned blame_reg) 880 { 881 phys_addr_t iova; 882 unsigned master; 883 u32 blame; 884 885 assert_spin_locked(&iommu->iommu_lock); 886 887 iova = iommu_read(iommu, addr_reg); 888 blame = iommu_read(iommu, blame_reg); 889 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 890 891 /* 892 * If the address is not in the page table, we can't get what 893 * operation triggered the fault. Assume it's a read 894 * operation. 895 */ 896 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); 897 898 return iova; 899 } 900 901 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) 902 { 903 enum sun50i_iommu_aci aci; 904 phys_addr_t iova; 905 unsigned master; 906 unsigned dir; 907 u32 blame; 908 909 assert_spin_locked(&iommu->iommu_lock); 910 911 blame = iommu_read(iommu, IOMMU_INT_STA_REG); 912 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 913 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); 914 aci = sun50i_get_pte_aci(iommu_read(iommu, 915 IOMMU_INT_ERR_DATA_REG(master))); 916 917 switch (aci) { 918 /* 919 * If we are in the read-only domain, then it means we 920 * tried to write. 921 */ 922 case SUN50I_IOMMU_ACI_RD: 923 dir = IOMMU_FAULT_WRITE; 924 break; 925 926 /* 927 * If we are in the write-only domain, then it means 928 * we tried to read. 929 */ 930 case SUN50I_IOMMU_ACI_WR: 931 932 /* 933 * If we are in the domain without any permission, we 934 * can't really tell. Let's default to a read 935 * operation. 936 */ 937 case SUN50I_IOMMU_ACI_NONE: 938 939 /* WTF? */ 940 case SUN50I_IOMMU_ACI_RD_WR: 941 default: 942 dir = IOMMU_FAULT_READ; 943 break; 944 } 945 946 /* 947 * If the address is not in the page table, we can't get what 948 * operation triggered the fault. Assume it's a read 949 * operation. 950 */ 951 sun50i_iommu_report_fault(iommu, master, iova, dir); 952 953 return iova; 954 } 955 956 static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id) 957 { 958 u32 status, l1_status, l2_status, resets; 959 struct sun50i_iommu *iommu = dev_id; 960 961 spin_lock(&iommu->iommu_lock); 962 963 status = iommu_read(iommu, IOMMU_INT_STA_REG); 964 if (!(status & IOMMU_INT_MASK)) { 965 spin_unlock(&iommu->iommu_lock); 966 return IRQ_NONE; 967 } 968 969 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG); 970 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG); 971 972 if (status & IOMMU_INT_INVALID_L2PG) 973 sun50i_iommu_handle_pt_irq(iommu, 974 IOMMU_INT_ERR_ADDR_L2_REG, 975 IOMMU_L2PG_INT_REG); 976 else if (status & IOMMU_INT_INVALID_L1PG) 977 sun50i_iommu_handle_pt_irq(iommu, 978 IOMMU_INT_ERR_ADDR_L1_REG, 979 IOMMU_L1PG_INT_REG); 980 else 981 sun50i_iommu_handle_perm_irq(iommu); 982 983 iommu_write(iommu, IOMMU_INT_CLR_REG, status); 984 985 resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK; 986 iommu_write(iommu, IOMMU_RESET_REG, ~resets); 987 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL); 988 989 spin_unlock(&iommu->iommu_lock); 990 991 return IRQ_HANDLED; 992 } 993 994 static int sun50i_iommu_probe(struct platform_device *pdev) 995 { 996 struct sun50i_iommu *iommu; 997 int ret, irq; 998 999 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); 1000 if (!iommu) 1001 return -ENOMEM; 1002 spin_lock_init(&iommu->iommu_lock); 1003 iommu->domain = &sun50i_iommu_identity_domain; 1004 platform_set_drvdata(pdev, iommu); 1005 iommu->dev = &pdev->dev; 1006 1007 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), 1008 PT_SIZE, PT_SIZE, 1009 SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA32, 1010 NULL); 1011 if (!iommu->pt_pool) 1012 return -ENOMEM; 1013 1014 iommu->base = devm_platform_ioremap_resource(pdev, 0); 1015 if (IS_ERR(iommu->base)) { 1016 ret = PTR_ERR(iommu->base); 1017 goto err_free_cache; 1018 } 1019 1020 irq = platform_get_irq(pdev, 0); 1021 if (irq < 0) { 1022 ret = irq; 1023 goto err_free_cache; 1024 } 1025 1026 iommu->clk = devm_clk_get(&pdev->dev, NULL); 1027 if (IS_ERR(iommu->clk)) { 1028 dev_err(&pdev->dev, "Couldn't get our clock.\n"); 1029 ret = PTR_ERR(iommu->clk); 1030 goto err_free_cache; 1031 } 1032 1033 iommu->reset = devm_reset_control_get(&pdev->dev, NULL); 1034 if (IS_ERR(iommu->reset)) { 1035 dev_err(&pdev->dev, "Couldn't get our reset line.\n"); 1036 ret = PTR_ERR(iommu->reset); 1037 goto err_free_cache; 1038 } 1039 1040 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, 1041 NULL, dev_name(&pdev->dev)); 1042 if (ret) 1043 goto err_free_cache; 1044 1045 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev); 1046 if (ret) 1047 goto err_remove_sysfs; 1048 1049 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0, 1050 dev_name(&pdev->dev), iommu); 1051 if (ret < 0) 1052 goto err_unregister; 1053 1054 return 0; 1055 1056 err_unregister: 1057 iommu_device_unregister(&iommu->iommu); 1058 1059 err_remove_sysfs: 1060 iommu_device_sysfs_remove(&iommu->iommu); 1061 1062 err_free_cache: 1063 kmem_cache_destroy(iommu->pt_pool); 1064 1065 return ret; 1066 } 1067 1068 static const struct of_device_id sun50i_iommu_dt[] = { 1069 { .compatible = "allwinner,sun50i-h6-iommu", }, 1070 { .compatible = "allwinner,sun50i-h616-iommu", }, 1071 { /* sentinel */ }, 1072 }; 1073 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt); 1074 1075 static struct platform_driver sun50i_iommu_driver = { 1076 .driver = { 1077 .name = "sun50i-iommu", 1078 .of_match_table = sun50i_iommu_dt, 1079 .suppress_bind_attrs = true, 1080 } 1081 }; 1082 builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); 1083 1084 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); 1085 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); 1086 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); 1087