1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (C) 2016-2018, Allwinner Technology CO., LTD. 3 // Copyright (C) 2019-2020, Cerno 4 5 #include <linux/bitfield.h> 6 #include <linux/bug.h> 7 #include <linux/clk.h> 8 #include <linux/device.h> 9 #include <linux/dma-direction.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/err.h> 12 #include <linux/errno.h> 13 #include <linux/interrupt.h> 14 #include <linux/iommu.h> 15 #include <linux/iopoll.h> 16 #include <linux/ioport.h> 17 #include <linux/log2.h> 18 #include <linux/module.h> 19 #include <linux/of_platform.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm.h> 22 #include <linux/pm_runtime.h> 23 #include <linux/reset.h> 24 #include <linux/sizes.h> 25 #include <linux/slab.h> 26 #include <linux/spinlock.h> 27 #include <linux/types.h> 28 29 #include "iommu-pages.h" 30 31 #define IOMMU_RESET_REG 0x010 32 #define IOMMU_RESET_RELEASE_ALL 0xffffffff 33 #define IOMMU_ENABLE_REG 0x020 34 #define IOMMU_ENABLE_ENABLE BIT(0) 35 36 #define IOMMU_BYPASS_REG 0x030 37 #define IOMMU_AUTO_GATING_REG 0x040 38 #define IOMMU_AUTO_GATING_ENABLE BIT(0) 39 40 #define IOMMU_WBUF_CTRL_REG 0x044 41 #define IOMMU_OOO_CTRL_REG 0x048 42 #define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c 43 #define IOMMU_TTB_REG 0x050 44 #define IOMMU_TLB_ENABLE_REG 0x060 45 #define IOMMU_TLB_PREFETCH_REG 0x070 46 #define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m) 47 48 #define IOMMU_TLB_FLUSH_REG 0x080 49 #define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17) 50 #define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16) 51 #define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0)) 52 53 #define IOMMU_TLB_IVLD_ADDR_REG 0x090 54 #define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094 55 #define IOMMU_TLB_IVLD_ENABLE_REG 0x098 56 #define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0) 57 58 #define IOMMU_PC_IVLD_ADDR_REG 0x0a0 59 #define IOMMU_PC_IVLD_ENABLE_REG 0x0a8 60 #define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0) 61 62 #define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4) 63 #define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2))) 64 #define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1)) 65 66 #define IOMMU_DM_AUT_OVWT_REG 0x0d0 67 #define IOMMU_INT_ENABLE_REG 0x100 68 #define IOMMU_INT_CLR_REG 0x104 69 #define IOMMU_INT_STA_REG 0x108 70 #define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4) 71 #define IOMMU_INT_ERR_ADDR_L1_REG 0x130 72 #define IOMMU_INT_ERR_ADDR_L2_REG 0x134 73 #define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4) 74 #define IOMMU_L1PG_INT_REG 0x0180 75 #define IOMMU_L2PG_INT_REG 0x0184 76 77 #define IOMMU_INT_INVALID_L2PG BIT(17) 78 #define IOMMU_INT_INVALID_L1PG BIT(16) 79 #define IOMMU_INT_MASTER_PERMISSION(m) BIT(m) 80 #define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \ 81 IOMMU_INT_MASTER_PERMISSION(1) | \ 82 IOMMU_INT_MASTER_PERMISSION(2) | \ 83 IOMMU_INT_MASTER_PERMISSION(3) | \ 84 IOMMU_INT_MASTER_PERMISSION(4) | \ 85 IOMMU_INT_MASTER_PERMISSION(5)) 86 #define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \ 87 IOMMU_INT_INVALID_L2PG | \ 88 IOMMU_INT_MASTER_MASK) 89 90 #define PT_ENTRY_SIZE sizeof(u32) 91 92 #define NUM_DT_ENTRIES 4096 93 #define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE) 94 95 #define NUM_PT_ENTRIES 256 96 #define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE) 97 98 #define SPAGE_SIZE 4096 99 100 struct sun50i_iommu { 101 struct iommu_device iommu; 102 103 /* Lock to modify the IOMMU registers */ 104 spinlock_t iommu_lock; 105 106 struct device *dev; 107 void __iomem *base; 108 struct reset_control *reset; 109 struct clk *clk; 110 111 struct iommu_domain *domain; 112 struct kmem_cache *pt_pool; 113 }; 114 115 struct sun50i_iommu_domain { 116 struct iommu_domain domain; 117 118 /* Number of devices attached to the domain */ 119 refcount_t refcnt; 120 121 /* L1 Page Table */ 122 u32 *dt; 123 dma_addr_t dt_dma; 124 125 struct sun50i_iommu *iommu; 126 }; 127 128 static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain) 129 { 130 return container_of(domain, struct sun50i_iommu_domain, domain); 131 } 132 133 static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev) 134 { 135 return dev_iommu_priv_get(dev); 136 } 137 138 static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset) 139 { 140 return readl(iommu->base + offset); 141 } 142 143 static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) 144 { 145 writel(value, iommu->base + offset); 146 } 147 148 /* 149 * The Allwinner H6 IOMMU uses a 2-level page table. 150 * 151 * The first level is the usual Directory Table (DT), that consists of 152 * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page 153 * Table (PT). 154 * 155 * Each PT consits of 256 4-bytes Page Table Entries (PTE), each 156 * pointing to a 4kB page of physical memory. 157 * 158 * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG 159 * register that contains its physical address. 160 */ 161 162 #define SUN50I_IOVA_DTE_MASK GENMASK(31, 20) 163 #define SUN50I_IOVA_PTE_MASK GENMASK(19, 12) 164 #define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0) 165 166 static u32 sun50i_iova_get_dte_index(dma_addr_t iova) 167 { 168 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova); 169 } 170 171 static u32 sun50i_iova_get_pte_index(dma_addr_t iova) 172 { 173 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova); 174 } 175 176 static u32 sun50i_iova_get_page_offset(dma_addr_t iova) 177 { 178 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova); 179 } 180 181 /* 182 * Each Directory Table Entry has a Page Table address and a valid 183 * bit: 184 185 * +---------------------+-----------+-+ 186 * | PT address | Reserved |V| 187 * +---------------------+-----------+-+ 188 * 31:10 - Page Table address 189 * 9:2 - Reserved 190 * 1:0 - 1 if the entry is valid 191 */ 192 193 #define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10) 194 #define SUN50I_DTE_PT_ATTRS GENMASK(1, 0) 195 #define SUN50I_DTE_PT_VALID 1 196 197 static phys_addr_t sun50i_dte_get_pt_address(u32 dte) 198 { 199 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK; 200 } 201 202 static bool sun50i_dte_is_pt_valid(u32 dte) 203 { 204 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID; 205 } 206 207 static u32 sun50i_mk_dte(dma_addr_t pt_dma) 208 { 209 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID; 210 } 211 212 /* 213 * Each PTE has a Page address, an authority index and a valid bit: 214 * 215 * +----------------+-----+-----+-----+---+-----+ 216 * | Page address | Rsv | ACI | Rsv | V | Rsv | 217 * +----------------+-----+-----+-----+---+-----+ 218 * 31:12 - Page address 219 * 11:8 - Reserved 220 * 7:4 - Authority Control Index 221 * 3:2 - Reserved 222 * 1 - 1 if the entry is valid 223 * 0 - Reserved 224 * 225 * The way permissions work is that the IOMMU has 16 "domains" that 226 * can be configured to give each masters either read or write 227 * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain 228 * 0 seems like the default domain, and its permissions in the 229 * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really 230 * useful to enforce any particular permission. 231 * 232 * Each page entry will then have a reference to the domain they are 233 * affected to, so that we can actually enforce them on a per-page 234 * basis. 235 * 236 * In order to make it work with the IOMMU framework, we will be using 237 * 4 different domains, starting at 1: RD_WR, RD, WR and NONE 238 * depending on the permission we want to enforce. Each domain will 239 * have each master setup in the same way, since the IOMMU framework 240 * doesn't seem to restrict page access on a per-device basis. And 241 * then we will use the relevant domain index when generating the page 242 * table entry depending on the permissions we want to be enforced. 243 */ 244 245 enum sun50i_iommu_aci { 246 SUN50I_IOMMU_ACI_DO_NOT_USE = 0, 247 SUN50I_IOMMU_ACI_NONE, 248 SUN50I_IOMMU_ACI_RD, 249 SUN50I_IOMMU_ACI_WR, 250 SUN50I_IOMMU_ACI_RD_WR, 251 }; 252 253 #define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12) 254 #define SUN50I_PTE_ACI_MASK GENMASK(7, 4) 255 #define SUN50I_PTE_PAGE_VALID BIT(1) 256 257 static phys_addr_t sun50i_pte_get_page_address(u32 pte) 258 { 259 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK; 260 } 261 262 static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte) 263 { 264 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte); 265 } 266 267 static bool sun50i_pte_is_page_valid(u32 pte) 268 { 269 return pte & SUN50I_PTE_PAGE_VALID; 270 } 271 272 static u32 sun50i_mk_pte(phys_addr_t page, int prot) 273 { 274 enum sun50i_iommu_aci aci; 275 u32 flags = 0; 276 277 if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE)) 278 aci = SUN50I_IOMMU_ACI_RD_WR; 279 else if (prot & IOMMU_READ) 280 aci = SUN50I_IOMMU_ACI_RD; 281 else if (prot & IOMMU_WRITE) 282 aci = SUN50I_IOMMU_ACI_WR; 283 else 284 aci = SUN50I_IOMMU_ACI_NONE; 285 286 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci); 287 page &= SUN50I_PTE_PAGE_ADDRESS_MASK; 288 return page | flags | SUN50I_PTE_PAGE_VALID; 289 } 290 291 static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain, 292 void *vaddr, unsigned int count) 293 { 294 struct sun50i_iommu *iommu = sun50i_domain->iommu; 295 dma_addr_t dma = virt_to_phys(vaddr); 296 size_t size = count * PT_ENTRY_SIZE; 297 298 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE); 299 } 300 301 static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu, 302 unsigned long iova) 303 { 304 u32 reg; 305 int ret; 306 307 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova); 308 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12)); 309 iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG, 310 IOMMU_TLB_IVLD_ENABLE_ENABLE); 311 312 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG, 313 reg, !reg, 1, 2000); 314 if (ret) 315 dev_warn(iommu->dev, "TLB invalidation timed out!\n"); 316 } 317 318 static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu, 319 unsigned long iova) 320 { 321 u32 reg; 322 int ret; 323 324 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova); 325 iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG, 326 IOMMU_PC_IVLD_ENABLE_ENABLE); 327 328 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG, 329 reg, !reg, 1, 2000); 330 if (ret) 331 dev_warn(iommu->dev, "PTW cache invalidation timed out!\n"); 332 } 333 334 static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu, 335 unsigned long iova, size_t size) 336 { 337 assert_spin_locked(&iommu->iommu_lock); 338 339 iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0); 340 341 sun50i_iommu_zap_iova(iommu, iova); 342 sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE); 343 if (size > SPAGE_SIZE) { 344 sun50i_iommu_zap_iova(iommu, iova + size); 345 sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE); 346 } 347 sun50i_iommu_zap_ptw_cache(iommu, iova); 348 sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M); 349 if (size > SZ_1M) { 350 sun50i_iommu_zap_ptw_cache(iommu, iova + size); 351 sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M); 352 } 353 354 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 355 } 356 357 static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu) 358 { 359 u32 reg; 360 int ret; 361 362 assert_spin_locked(&iommu->iommu_lock); 363 364 iommu_write(iommu, 365 IOMMU_TLB_FLUSH_REG, 366 IOMMU_TLB_FLUSH_PTW_CACHE | 367 IOMMU_TLB_FLUSH_MACRO_TLB | 368 IOMMU_TLB_FLUSH_MICRO_TLB(5) | 369 IOMMU_TLB_FLUSH_MICRO_TLB(4) | 370 IOMMU_TLB_FLUSH_MICRO_TLB(3) | 371 IOMMU_TLB_FLUSH_MICRO_TLB(2) | 372 IOMMU_TLB_FLUSH_MICRO_TLB(1) | 373 IOMMU_TLB_FLUSH_MICRO_TLB(0)); 374 375 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG, 376 reg, !reg, 377 1, 2000); 378 if (ret) 379 dev_warn(iommu->dev, "TLB Flush timed out!\n"); 380 381 return ret; 382 } 383 384 static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain) 385 { 386 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 387 struct sun50i_iommu *iommu = sun50i_domain->iommu; 388 unsigned long flags; 389 390 /* 391 * At boot, we'll have a first call into .flush_iotlb_all right after 392 * .probe_device, and since we link our (single) domain to our iommu in 393 * the .attach_device callback, we don't have that pointer set. 394 * 395 * It shouldn't really be any trouble to ignore it though since we flush 396 * all caches as part of the device powerup. 397 */ 398 if (!iommu) 399 return; 400 401 spin_lock_irqsave(&iommu->iommu_lock, flags); 402 sun50i_iommu_flush_all_tlb(iommu); 403 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 404 } 405 406 static int sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain, 407 unsigned long iova, size_t size) 408 { 409 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 410 struct sun50i_iommu *iommu = sun50i_domain->iommu; 411 unsigned long flags; 412 413 spin_lock_irqsave(&iommu->iommu_lock, flags); 414 sun50i_iommu_zap_range(iommu, iova, size); 415 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 416 417 return 0; 418 } 419 420 static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain, 421 struct iommu_iotlb_gather *gather) 422 { 423 sun50i_iommu_flush_iotlb_all(domain); 424 } 425 426 static int sun50i_iommu_enable(struct sun50i_iommu *iommu) 427 { 428 struct sun50i_iommu_domain *sun50i_domain; 429 unsigned long flags; 430 int ret; 431 432 if (!iommu->domain) 433 return 0; 434 435 sun50i_domain = to_sun50i_domain(iommu->domain); 436 437 ret = reset_control_deassert(iommu->reset); 438 if (ret) 439 return ret; 440 441 ret = clk_prepare_enable(iommu->clk); 442 if (ret) 443 goto err_reset_assert; 444 445 spin_lock_irqsave(&iommu->iommu_lock, flags); 446 447 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma); 448 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG, 449 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) | 450 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) | 451 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) | 452 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) | 453 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) | 454 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5)); 455 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK); 456 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE), 457 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 458 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) | 459 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 460 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) | 461 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 462 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) | 463 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 464 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) | 465 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 466 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) | 467 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) | 468 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5)); 469 470 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD), 471 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) | 472 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) | 473 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) | 474 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) | 475 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) | 476 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5)); 477 478 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR), 479 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) | 480 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) | 481 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) | 482 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) | 483 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) | 484 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5)); 485 486 ret = sun50i_iommu_flush_all_tlb(iommu); 487 if (ret) { 488 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 489 goto err_clk_disable; 490 } 491 492 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE); 493 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE); 494 495 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 496 497 return 0; 498 499 err_clk_disable: 500 clk_disable_unprepare(iommu->clk); 501 502 err_reset_assert: 503 reset_control_assert(iommu->reset); 504 505 return ret; 506 } 507 508 static void sun50i_iommu_disable(struct sun50i_iommu *iommu) 509 { 510 unsigned long flags; 511 512 spin_lock_irqsave(&iommu->iommu_lock, flags); 513 514 iommu_write(iommu, IOMMU_ENABLE_REG, 0); 515 iommu_write(iommu, IOMMU_TTB_REG, 0); 516 517 spin_unlock_irqrestore(&iommu->iommu_lock, flags); 518 519 clk_disable_unprepare(iommu->clk); 520 reset_control_assert(iommu->reset); 521 } 522 523 static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu, 524 gfp_t gfp) 525 { 526 dma_addr_t pt_dma; 527 u32 *page_table; 528 529 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp); 530 if (!page_table) 531 return ERR_PTR(-ENOMEM); 532 533 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE); 534 if (dma_mapping_error(iommu->dev, pt_dma)) { 535 dev_err(iommu->dev, "Couldn't map L2 Page Table\n"); 536 kmem_cache_free(iommu->pt_pool, page_table); 537 return ERR_PTR(-ENOMEM); 538 } 539 540 /* We rely on the physical address and DMA address being the same */ 541 WARN_ON(pt_dma != virt_to_phys(page_table)); 542 543 return page_table; 544 } 545 546 static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu, 547 u32 *page_table) 548 { 549 phys_addr_t pt_phys = virt_to_phys(page_table); 550 551 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE); 552 kmem_cache_free(iommu->pt_pool, page_table); 553 } 554 555 static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, 556 dma_addr_t iova, gfp_t gfp) 557 { 558 struct sun50i_iommu *iommu = sun50i_domain->iommu; 559 u32 *page_table; 560 u32 *dte_addr; 561 u32 old_dte; 562 u32 dte; 563 564 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 565 dte = *dte_addr; 566 if (sun50i_dte_is_pt_valid(dte)) { 567 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte); 568 return (u32 *)phys_to_virt(pt_phys); 569 } 570 571 page_table = sun50i_iommu_alloc_page_table(iommu, gfp); 572 if (IS_ERR(page_table)) 573 return page_table; 574 575 dte = sun50i_mk_dte(virt_to_phys(page_table)); 576 old_dte = cmpxchg(dte_addr, 0, dte); 577 if (old_dte) { 578 phys_addr_t installed_pt_phys = 579 sun50i_dte_get_pt_address(old_dte); 580 u32 *installed_pt = phys_to_virt(installed_pt_phys); 581 u32 *drop_pt = page_table; 582 583 page_table = installed_pt; 584 dte = old_dte; 585 sun50i_iommu_free_page_table(iommu, drop_pt); 586 } 587 588 sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES); 589 sun50i_table_flush(sun50i_domain, dte_addr, 1); 590 591 return page_table; 592 } 593 594 static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, 595 phys_addr_t paddr, size_t size, size_t count, 596 int prot, gfp_t gfp, size_t *mapped) 597 { 598 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 599 struct sun50i_iommu *iommu = sun50i_domain->iommu; 600 u32 pte_index; 601 u32 *page_table, *pte_addr; 602 int ret = 0; 603 604 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp); 605 if (IS_ERR(page_table)) { 606 ret = PTR_ERR(page_table); 607 goto out; 608 } 609 610 pte_index = sun50i_iova_get_pte_index(iova); 611 pte_addr = &page_table[pte_index]; 612 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) { 613 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr); 614 dev_err(iommu->dev, 615 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n", 616 &iova, &page_phys, &paddr, prot); 617 ret = -EBUSY; 618 goto out; 619 } 620 621 *pte_addr = sun50i_mk_pte(paddr, prot); 622 sun50i_table_flush(sun50i_domain, pte_addr, 1); 623 *mapped = size; 624 625 out: 626 return ret; 627 } 628 629 static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, 630 size_t size, size_t count, struct iommu_iotlb_gather *gather) 631 { 632 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 633 phys_addr_t pt_phys; 634 u32 *pte_addr; 635 u32 dte; 636 637 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 638 if (!sun50i_dte_is_pt_valid(dte)) 639 return 0; 640 641 pt_phys = sun50i_dte_get_pt_address(dte); 642 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova); 643 644 if (!sun50i_pte_is_page_valid(*pte_addr)) 645 return 0; 646 647 memset(pte_addr, 0, sizeof(*pte_addr)); 648 sun50i_table_flush(sun50i_domain, pte_addr, 1); 649 650 return SZ_4K; 651 } 652 653 static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain, 654 dma_addr_t iova) 655 { 656 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 657 phys_addr_t pt_phys; 658 u32 *page_table; 659 u32 dte, pte; 660 661 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)]; 662 if (!sun50i_dte_is_pt_valid(dte)) 663 return 0; 664 665 pt_phys = sun50i_dte_get_pt_address(dte); 666 page_table = (u32 *)phys_to_virt(pt_phys); 667 pte = page_table[sun50i_iova_get_pte_index(iova)]; 668 if (!sun50i_pte_is_page_valid(pte)) 669 return 0; 670 671 return sun50i_pte_get_page_address(pte) + 672 sun50i_iova_get_page_offset(iova); 673 } 674 675 static struct iommu_domain * 676 sun50i_iommu_domain_alloc_paging(struct device *dev) 677 { 678 struct sun50i_iommu_domain *sun50i_domain; 679 680 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL); 681 if (!sun50i_domain) 682 return NULL; 683 684 sun50i_domain->dt = iommu_alloc_pages(GFP_KERNEL, get_order(DT_SIZE)); 685 if (!sun50i_domain->dt) 686 goto err_free_domain; 687 688 refcount_set(&sun50i_domain->refcnt, 1); 689 690 sun50i_domain->domain.geometry.aperture_start = 0; 691 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); 692 sun50i_domain->domain.geometry.force_aperture = true; 693 694 return &sun50i_domain->domain; 695 696 err_free_domain: 697 kfree(sun50i_domain); 698 699 return NULL; 700 } 701 702 static void sun50i_iommu_domain_free(struct iommu_domain *domain) 703 { 704 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 705 706 iommu_free_pages(sun50i_domain->dt, get_order(DT_SIZE)); 707 sun50i_domain->dt = NULL; 708 709 kfree(sun50i_domain); 710 } 711 712 static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu, 713 struct sun50i_iommu_domain *sun50i_domain) 714 { 715 iommu->domain = &sun50i_domain->domain; 716 sun50i_domain->iommu = iommu; 717 718 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt, 719 DT_SIZE, DMA_TO_DEVICE); 720 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) { 721 dev_err(iommu->dev, "Couldn't map L1 Page Table\n"); 722 return -ENOMEM; 723 } 724 725 return sun50i_iommu_enable(iommu); 726 } 727 728 static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu, 729 struct sun50i_iommu_domain *sun50i_domain) 730 { 731 unsigned int i; 732 733 for (i = 0; i < NUM_DT_ENTRIES; i++) { 734 phys_addr_t pt_phys; 735 u32 *page_table; 736 u32 *dte_addr; 737 u32 dte; 738 739 dte_addr = &sun50i_domain->dt[i]; 740 dte = *dte_addr; 741 if (!sun50i_dte_is_pt_valid(dte)) 742 continue; 743 744 memset(dte_addr, 0, sizeof(*dte_addr)); 745 sun50i_table_flush(sun50i_domain, dte_addr, 1); 746 747 pt_phys = sun50i_dte_get_pt_address(dte); 748 page_table = phys_to_virt(pt_phys); 749 sun50i_iommu_free_page_table(iommu, page_table); 750 } 751 752 753 sun50i_iommu_disable(iommu); 754 755 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt), 756 DT_SIZE, DMA_TO_DEVICE); 757 758 iommu->domain = NULL; 759 } 760 761 static int sun50i_iommu_identity_attach(struct iommu_domain *identity_domain, 762 struct device *dev) 763 { 764 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev); 765 struct sun50i_iommu_domain *sun50i_domain; 766 767 dev_dbg(dev, "Detaching from IOMMU domain\n"); 768 769 if (iommu->domain == identity_domain) 770 return 0; 771 772 sun50i_domain = to_sun50i_domain(iommu->domain); 773 if (refcount_dec_and_test(&sun50i_domain->refcnt)) 774 sun50i_iommu_detach_domain(iommu, sun50i_domain); 775 return 0; 776 } 777 778 static struct iommu_domain_ops sun50i_iommu_identity_ops = { 779 .attach_dev = sun50i_iommu_identity_attach, 780 }; 781 782 static struct iommu_domain sun50i_iommu_identity_domain = { 783 .type = IOMMU_DOMAIN_IDENTITY, 784 .ops = &sun50i_iommu_identity_ops, 785 }; 786 787 static int sun50i_iommu_attach_device(struct iommu_domain *domain, 788 struct device *dev) 789 { 790 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain); 791 struct sun50i_iommu *iommu; 792 793 iommu = sun50i_iommu_from_dev(dev); 794 if (!iommu) 795 return -ENODEV; 796 797 dev_dbg(dev, "Attaching to IOMMU domain\n"); 798 799 refcount_inc(&sun50i_domain->refcnt); 800 801 if (iommu->domain == domain) 802 return 0; 803 804 sun50i_iommu_identity_attach(&sun50i_iommu_identity_domain, dev); 805 806 sun50i_iommu_attach_domain(iommu, sun50i_domain); 807 808 return 0; 809 } 810 811 static struct iommu_device *sun50i_iommu_probe_device(struct device *dev) 812 { 813 struct sun50i_iommu *iommu; 814 815 iommu = sun50i_iommu_from_dev(dev); 816 if (!iommu) 817 return ERR_PTR(-ENODEV); 818 819 return &iommu->iommu; 820 } 821 822 static int sun50i_iommu_of_xlate(struct device *dev, 823 const struct of_phandle_args *args) 824 { 825 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); 826 unsigned id = args->args[0]; 827 828 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev)); 829 830 return iommu_fwspec_add_ids(dev, &id, 1); 831 } 832 833 static const struct iommu_ops sun50i_iommu_ops = { 834 .identity_domain = &sun50i_iommu_identity_domain, 835 .pgsize_bitmap = SZ_4K, 836 .device_group = generic_single_device_group, 837 .domain_alloc_paging = sun50i_iommu_domain_alloc_paging, 838 .of_xlate = sun50i_iommu_of_xlate, 839 .probe_device = sun50i_iommu_probe_device, 840 .default_domain_ops = &(const struct iommu_domain_ops) { 841 .attach_dev = sun50i_iommu_attach_device, 842 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all, 843 .iotlb_sync_map = sun50i_iommu_iotlb_sync_map, 844 .iotlb_sync = sun50i_iommu_iotlb_sync, 845 .iova_to_phys = sun50i_iommu_iova_to_phys, 846 .map_pages = sun50i_iommu_map, 847 .unmap_pages = sun50i_iommu_unmap, 848 .free = sun50i_iommu_domain_free, 849 } 850 }; 851 852 static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu, 853 unsigned master, phys_addr_t iova, 854 unsigned prot) 855 { 856 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n", 857 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd"); 858 859 if (iommu->domain) 860 report_iommu_fault(iommu->domain, iommu->dev, iova, prot); 861 else 862 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n"); 863 864 sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE); 865 } 866 867 static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu, 868 unsigned addr_reg, 869 unsigned blame_reg) 870 { 871 phys_addr_t iova; 872 unsigned master; 873 u32 blame; 874 875 assert_spin_locked(&iommu->iommu_lock); 876 877 iova = iommu_read(iommu, addr_reg); 878 blame = iommu_read(iommu, blame_reg); 879 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 880 881 /* 882 * If the address is not in the page table, we can't get what 883 * operation triggered the fault. Assume it's a read 884 * operation. 885 */ 886 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ); 887 888 return iova; 889 } 890 891 static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu) 892 { 893 enum sun50i_iommu_aci aci; 894 phys_addr_t iova; 895 unsigned master; 896 unsigned dir; 897 u32 blame; 898 899 assert_spin_locked(&iommu->iommu_lock); 900 901 blame = iommu_read(iommu, IOMMU_INT_STA_REG); 902 master = ilog2(blame & IOMMU_INT_MASTER_MASK); 903 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master)); 904 aci = sun50i_get_pte_aci(iommu_read(iommu, 905 IOMMU_INT_ERR_DATA_REG(master))); 906 907 switch (aci) { 908 /* 909 * If we are in the read-only domain, then it means we 910 * tried to write. 911 */ 912 case SUN50I_IOMMU_ACI_RD: 913 dir = IOMMU_FAULT_WRITE; 914 break; 915 916 /* 917 * If we are in the write-only domain, then it means 918 * we tried to read. 919 */ 920 case SUN50I_IOMMU_ACI_WR: 921 922 /* 923 * If we are in the domain without any permission, we 924 * can't really tell. Let's default to a read 925 * operation. 926 */ 927 case SUN50I_IOMMU_ACI_NONE: 928 929 /* WTF? */ 930 case SUN50I_IOMMU_ACI_RD_WR: 931 default: 932 dir = IOMMU_FAULT_READ; 933 break; 934 } 935 936 /* 937 * If the address is not in the page table, we can't get what 938 * operation triggered the fault. Assume it's a read 939 * operation. 940 */ 941 sun50i_iommu_report_fault(iommu, master, iova, dir); 942 943 return iova; 944 } 945 946 static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id) 947 { 948 u32 status, l1_status, l2_status, resets; 949 struct sun50i_iommu *iommu = dev_id; 950 951 spin_lock(&iommu->iommu_lock); 952 953 status = iommu_read(iommu, IOMMU_INT_STA_REG); 954 if (!(status & IOMMU_INT_MASK)) { 955 spin_unlock(&iommu->iommu_lock); 956 return IRQ_NONE; 957 } 958 959 l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG); 960 l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG); 961 962 if (status & IOMMU_INT_INVALID_L2PG) 963 sun50i_iommu_handle_pt_irq(iommu, 964 IOMMU_INT_ERR_ADDR_L2_REG, 965 IOMMU_L2PG_INT_REG); 966 else if (status & IOMMU_INT_INVALID_L1PG) 967 sun50i_iommu_handle_pt_irq(iommu, 968 IOMMU_INT_ERR_ADDR_L1_REG, 969 IOMMU_L1PG_INT_REG); 970 else 971 sun50i_iommu_handle_perm_irq(iommu); 972 973 iommu_write(iommu, IOMMU_INT_CLR_REG, status); 974 975 resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK; 976 iommu_write(iommu, IOMMU_RESET_REG, ~resets); 977 iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL); 978 979 spin_unlock(&iommu->iommu_lock); 980 981 return IRQ_HANDLED; 982 } 983 984 static int sun50i_iommu_probe(struct platform_device *pdev) 985 { 986 struct sun50i_iommu *iommu; 987 int ret, irq; 988 989 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); 990 if (!iommu) 991 return -ENOMEM; 992 spin_lock_init(&iommu->iommu_lock); 993 iommu->domain = &sun50i_iommu_identity_domain; 994 platform_set_drvdata(pdev, iommu); 995 iommu->dev = &pdev->dev; 996 997 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev), 998 PT_SIZE, PT_SIZE, 999 SLAB_HWCACHE_ALIGN, 1000 NULL); 1001 if (!iommu->pt_pool) 1002 return -ENOMEM; 1003 1004 iommu->base = devm_platform_ioremap_resource(pdev, 0); 1005 if (IS_ERR(iommu->base)) { 1006 ret = PTR_ERR(iommu->base); 1007 goto err_free_cache; 1008 } 1009 1010 irq = platform_get_irq(pdev, 0); 1011 if (irq < 0) { 1012 ret = irq; 1013 goto err_free_cache; 1014 } 1015 1016 iommu->clk = devm_clk_get(&pdev->dev, NULL); 1017 if (IS_ERR(iommu->clk)) { 1018 dev_err(&pdev->dev, "Couldn't get our clock.\n"); 1019 ret = PTR_ERR(iommu->clk); 1020 goto err_free_cache; 1021 } 1022 1023 iommu->reset = devm_reset_control_get(&pdev->dev, NULL); 1024 if (IS_ERR(iommu->reset)) { 1025 dev_err(&pdev->dev, "Couldn't get our reset line.\n"); 1026 ret = PTR_ERR(iommu->reset); 1027 goto err_free_cache; 1028 } 1029 1030 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev, 1031 NULL, dev_name(&pdev->dev)); 1032 if (ret) 1033 goto err_free_cache; 1034 1035 ret = iommu_device_register(&iommu->iommu, &sun50i_iommu_ops, &pdev->dev); 1036 if (ret) 1037 goto err_remove_sysfs; 1038 1039 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0, 1040 dev_name(&pdev->dev), iommu); 1041 if (ret < 0) 1042 goto err_unregister; 1043 1044 return 0; 1045 1046 err_unregister: 1047 iommu_device_unregister(&iommu->iommu); 1048 1049 err_remove_sysfs: 1050 iommu_device_sysfs_remove(&iommu->iommu); 1051 1052 err_free_cache: 1053 kmem_cache_destroy(iommu->pt_pool); 1054 1055 return ret; 1056 } 1057 1058 static const struct of_device_id sun50i_iommu_dt[] = { 1059 { .compatible = "allwinner,sun50i-h6-iommu", }, 1060 { /* sentinel */ }, 1061 }; 1062 MODULE_DEVICE_TABLE(of, sun50i_iommu_dt); 1063 1064 static struct platform_driver sun50i_iommu_driver = { 1065 .driver = { 1066 .name = "sun50i-iommu", 1067 .of_match_table = sun50i_iommu_dt, 1068 .suppress_bind_attrs = true, 1069 } 1070 }; 1071 builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); 1072 1073 MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); 1074 MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); 1075 MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); 1076