1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Unisoc IOMMU driver 4 * 5 * Copyright (C) 2020 Unisoc, Inc. 6 * Author: Chunyan Zhang <chunyan.zhang@unisoc.com> 7 */ 8 9 #include <linux/clk.h> 10 #include <linux/device.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/errno.h> 13 #include <linux/iommu.h> 14 #include <linux/mfd/syscon.h> 15 #include <linux/module.h> 16 #include <linux/of_platform.h> 17 #include <linux/platform_device.h> 18 #include <linux/regmap.h> 19 #include <linux/slab.h> 20 21 #define SPRD_IOMMU_PAGE_SHIFT 12 22 #define SPRD_IOMMU_PAGE_SIZE SZ_4K 23 24 #define SPRD_EX_CFG 0x0 25 #define SPRD_IOMMU_VAOR_BYPASS BIT(4) 26 #define SPRD_IOMMU_GATE_EN BIT(1) 27 #define SPRD_IOMMU_EN BIT(0) 28 #define SPRD_EX_UPDATE 0x4 29 #define SPRD_EX_FIRST_VPN 0x8 30 #define SPRD_EX_VPN_RANGE 0xc 31 #define SPRD_EX_FIRST_PPN 0x10 32 #define SPRD_EX_DEFAULT_PPN 0x14 33 34 #define SPRD_IOMMU_VERSION 0x0 35 #define SPRD_VERSION_MASK GENMASK(15, 8) 36 #define SPRD_VERSION_SHIFT 0x8 37 #define SPRD_VAU_CFG 0x4 38 #define SPRD_VAU_UPDATE 0x8 39 #define SPRD_VAU_AUTH_CFG 0xc 40 #define SPRD_VAU_FIRST_PPN 0x10 41 #define SPRD_VAU_DEFAULT_PPN_RD 0x14 42 #define SPRD_VAU_DEFAULT_PPN_WR 0x18 43 #define SPRD_VAU_FIRST_VPN 0x1c 44 #define SPRD_VAU_VPN_RANGE 0x20 45 46 enum sprd_iommu_version { 47 SPRD_IOMMU_EX, 48 SPRD_IOMMU_VAU, 49 }; 50 51 /* 52 * struct sprd_iommu_device - high-level sprd IOMMU device representation, 53 * including hardware information and configuration, also driver data, etc 54 * 55 * @ver: sprd IOMMU IP version 56 * @prot_page_va: protect page base virtual address 57 * @prot_page_pa: protect page base physical address, data would be 58 * written to here while translation fault 59 * @base: mapped base address for accessing registers 60 * @dev: pointer to basic device structure 61 * @iommu: IOMMU core representation 62 * @group: IOMMU group 63 * @eb: gate clock which controls IOMMU access 64 */ 65 struct sprd_iommu_device { 66 struct sprd_iommu_domain *dom; 67 enum sprd_iommu_version ver; 68 u32 *prot_page_va; 69 dma_addr_t prot_page_pa; 70 void __iomem *base; 71 struct device *dev; 72 struct iommu_device iommu; 73 struct clk *eb; 74 }; 75 76 struct sprd_iommu_domain { 77 spinlock_t pgtlock; /* lock for page table */ 78 struct iommu_domain domain; 79 u32 *pgt_va; /* page table virtual address base */ 80 dma_addr_t pgt_pa; /* page table physical address base */ 81 struct sprd_iommu_device *sdev; 82 }; 83 84 static const struct iommu_ops sprd_iommu_ops; 85 86 static struct sprd_iommu_domain *to_sprd_domain(struct iommu_domain *dom) 87 { 88 return container_of(dom, struct sprd_iommu_domain, domain); 89 } 90 91 static inline void 92 sprd_iommu_write(struct sprd_iommu_device *sdev, unsigned int reg, u32 val) 93 { 94 writel_relaxed(val, sdev->base + reg); 95 } 96 97 static inline u32 98 sprd_iommu_read(struct sprd_iommu_device *sdev, unsigned int reg) 99 { 100 return readl_relaxed(sdev->base + reg); 101 } 102 103 static inline void 104 sprd_iommu_update_bits(struct sprd_iommu_device *sdev, unsigned int reg, 105 u32 mask, u32 shift, u32 val) 106 { 107 u32 t = sprd_iommu_read(sdev, reg); 108 109 t = (t & (~(mask << shift))) | ((val & mask) << shift); 110 sprd_iommu_write(sdev, reg, t); 111 } 112 113 static inline int 114 sprd_iommu_get_version(struct sprd_iommu_device *sdev) 115 { 116 int ver = (sprd_iommu_read(sdev, SPRD_IOMMU_VERSION) & 117 SPRD_VERSION_MASK) >> SPRD_VERSION_SHIFT; 118 119 switch (ver) { 120 case SPRD_IOMMU_EX: 121 case SPRD_IOMMU_VAU: 122 return ver; 123 default: 124 return -EINVAL; 125 } 126 } 127 128 static size_t 129 sprd_iommu_pgt_size(struct iommu_domain *domain) 130 { 131 return ((domain->geometry.aperture_end - 132 domain->geometry.aperture_start + 1) >> 133 SPRD_IOMMU_PAGE_SHIFT) * sizeof(u32); 134 } 135 136 static struct iommu_domain *sprd_iommu_domain_alloc_paging(struct device *dev) 137 { 138 struct sprd_iommu_domain *dom; 139 140 dom = kzalloc(sizeof(*dom), GFP_KERNEL); 141 if (!dom) 142 return NULL; 143 144 spin_lock_init(&dom->pgtlock); 145 146 dom->domain.geometry.aperture_start = 0; 147 dom->domain.geometry.aperture_end = SZ_256M - 1; 148 dom->domain.geometry.force_aperture = true; 149 150 return &dom->domain; 151 } 152 153 static void sprd_iommu_first_vpn(struct sprd_iommu_domain *dom) 154 { 155 struct sprd_iommu_device *sdev = dom->sdev; 156 u32 val; 157 unsigned int reg; 158 159 if (sdev->ver == SPRD_IOMMU_EX) 160 reg = SPRD_EX_FIRST_VPN; 161 else 162 reg = SPRD_VAU_FIRST_VPN; 163 164 val = dom->domain.geometry.aperture_start >> SPRD_IOMMU_PAGE_SHIFT; 165 sprd_iommu_write(sdev, reg, val); 166 } 167 168 static void sprd_iommu_vpn_range(struct sprd_iommu_domain *dom) 169 { 170 struct sprd_iommu_device *sdev = dom->sdev; 171 u32 val; 172 unsigned int reg; 173 174 if (sdev->ver == SPRD_IOMMU_EX) 175 reg = SPRD_EX_VPN_RANGE; 176 else 177 reg = SPRD_VAU_VPN_RANGE; 178 179 val = (dom->domain.geometry.aperture_end - 180 dom->domain.geometry.aperture_start) >> SPRD_IOMMU_PAGE_SHIFT; 181 sprd_iommu_write(sdev, reg, val); 182 } 183 184 static void sprd_iommu_first_ppn(struct sprd_iommu_domain *dom) 185 { 186 u32 val = dom->pgt_pa >> SPRD_IOMMU_PAGE_SHIFT; 187 struct sprd_iommu_device *sdev = dom->sdev; 188 unsigned int reg; 189 190 if (sdev->ver == SPRD_IOMMU_EX) 191 reg = SPRD_EX_FIRST_PPN; 192 else 193 reg = SPRD_VAU_FIRST_PPN; 194 195 sprd_iommu_write(sdev, reg, val); 196 } 197 198 static void sprd_iommu_default_ppn(struct sprd_iommu_device *sdev) 199 { 200 u32 val = sdev->prot_page_pa >> SPRD_IOMMU_PAGE_SHIFT; 201 202 if (sdev->ver == SPRD_IOMMU_EX) { 203 sprd_iommu_write(sdev, SPRD_EX_DEFAULT_PPN, val); 204 } else if (sdev->ver == SPRD_IOMMU_VAU) { 205 sprd_iommu_write(sdev, SPRD_VAU_DEFAULT_PPN_RD, val); 206 sprd_iommu_write(sdev, SPRD_VAU_DEFAULT_PPN_WR, val); 207 } 208 } 209 210 static void sprd_iommu_hw_en(struct sprd_iommu_device *sdev, bool en) 211 { 212 unsigned int reg_cfg; 213 u32 mask, val; 214 215 if (sdev->ver == SPRD_IOMMU_EX) 216 reg_cfg = SPRD_EX_CFG; 217 else 218 reg_cfg = SPRD_VAU_CFG; 219 220 mask = SPRD_IOMMU_EN | SPRD_IOMMU_GATE_EN; 221 val = en ? mask : 0; 222 sprd_iommu_update_bits(sdev, reg_cfg, mask, 0, val); 223 } 224 225 static void sprd_iommu_cleanup(struct sprd_iommu_domain *dom) 226 { 227 size_t pgt_size; 228 229 /* Nothing need to do if the domain hasn't been attached */ 230 if (!dom->sdev) 231 return; 232 233 pgt_size = sprd_iommu_pgt_size(&dom->domain); 234 dma_free_coherent(dom->sdev->dev, pgt_size, dom->pgt_va, dom->pgt_pa); 235 sprd_iommu_hw_en(dom->sdev, false); 236 dom->sdev = NULL; 237 } 238 239 static void sprd_iommu_domain_free(struct iommu_domain *domain) 240 { 241 struct sprd_iommu_domain *dom = to_sprd_domain(domain); 242 243 sprd_iommu_cleanup(dom); 244 kfree(dom); 245 } 246 247 static int sprd_iommu_attach_device(struct iommu_domain *domain, 248 struct device *dev) 249 { 250 struct sprd_iommu_device *sdev = dev_iommu_priv_get(dev); 251 struct sprd_iommu_domain *dom = to_sprd_domain(domain); 252 size_t pgt_size = sprd_iommu_pgt_size(domain); 253 254 /* The device is attached to this domain */ 255 if (sdev->dom == dom) 256 return 0; 257 258 /* The first time that domain is attaching to a device */ 259 if (!dom->pgt_va) { 260 dom->pgt_va = dma_alloc_coherent(sdev->dev, pgt_size, &dom->pgt_pa, GFP_KERNEL); 261 if (!dom->pgt_va) 262 return -ENOMEM; 263 264 dom->sdev = sdev; 265 } 266 267 sdev->dom = dom; 268 269 /* 270 * One sprd IOMMU serves one client device only, disabled it before 271 * configure mapping table to avoid access conflict in case other 272 * mapping table is stored in. 273 */ 274 sprd_iommu_hw_en(sdev, false); 275 sprd_iommu_first_ppn(dom); 276 sprd_iommu_first_vpn(dom); 277 sprd_iommu_vpn_range(dom); 278 sprd_iommu_default_ppn(sdev); 279 sprd_iommu_hw_en(sdev, true); 280 281 return 0; 282 } 283 284 static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova, 285 phys_addr_t paddr, size_t pgsize, size_t pgcount, 286 int prot, gfp_t gfp, size_t *mapped) 287 { 288 struct sprd_iommu_domain *dom = to_sprd_domain(domain); 289 size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE; 290 unsigned long flags; 291 unsigned int i; 292 u32 *pgt_base_iova; 293 u32 pabase = (u32)paddr; 294 unsigned long start = domain->geometry.aperture_start; 295 unsigned long end = domain->geometry.aperture_end; 296 297 if (!dom->sdev) { 298 pr_err("No sprd_iommu_device attached to the domain\n"); 299 return -EINVAL; 300 } 301 302 if (iova < start || (iova + size) > (end + 1)) { 303 dev_err(dom->sdev->dev, "(iova(0x%lx) + size(%zx)) are not in the range!\n", 304 iova, size); 305 return -EINVAL; 306 } 307 308 pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT); 309 310 spin_lock_irqsave(&dom->pgtlock, flags); 311 for (i = 0; i < pgcount; i++) { 312 pgt_base_iova[i] = pabase >> SPRD_IOMMU_PAGE_SHIFT; 313 pabase += SPRD_IOMMU_PAGE_SIZE; 314 } 315 spin_unlock_irqrestore(&dom->pgtlock, flags); 316 317 *mapped = size; 318 return 0; 319 } 320 321 static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova, 322 size_t pgsize, size_t pgcount, 323 struct iommu_iotlb_gather *iotlb_gather) 324 { 325 struct sprd_iommu_domain *dom = to_sprd_domain(domain); 326 unsigned long flags; 327 u32 *pgt_base_iova; 328 size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE; 329 unsigned long start = domain->geometry.aperture_start; 330 unsigned long end = domain->geometry.aperture_end; 331 332 if (iova < start || (iova + size) > (end + 1)) 333 return 0; 334 335 pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT); 336 337 spin_lock_irqsave(&dom->pgtlock, flags); 338 memset(pgt_base_iova, 0, pgcount * sizeof(u32)); 339 spin_unlock_irqrestore(&dom->pgtlock, flags); 340 341 return size; 342 } 343 344 static int sprd_iommu_sync_map(struct iommu_domain *domain, 345 unsigned long iova, size_t size) 346 { 347 struct sprd_iommu_domain *dom = to_sprd_domain(domain); 348 unsigned int reg; 349 350 if (dom->sdev->ver == SPRD_IOMMU_EX) 351 reg = SPRD_EX_UPDATE; 352 else 353 reg = SPRD_VAU_UPDATE; 354 355 /* clear IOMMU TLB buffer after page table updated */ 356 sprd_iommu_write(dom->sdev, reg, 0xffffffff); 357 return 0; 358 } 359 360 static void sprd_iommu_sync(struct iommu_domain *domain, 361 struct iommu_iotlb_gather *iotlb_gather) 362 { 363 sprd_iommu_sync_map(domain, 0, 0); 364 } 365 366 static phys_addr_t sprd_iommu_iova_to_phys(struct iommu_domain *domain, 367 dma_addr_t iova) 368 { 369 struct sprd_iommu_domain *dom = to_sprd_domain(domain); 370 unsigned long flags; 371 phys_addr_t pa; 372 unsigned long start = domain->geometry.aperture_start; 373 unsigned long end = domain->geometry.aperture_end; 374 375 if (WARN_ON(iova < start || iova > end)) 376 return 0; 377 378 spin_lock_irqsave(&dom->pgtlock, flags); 379 pa = *(dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT)); 380 pa = (pa << SPRD_IOMMU_PAGE_SHIFT) + ((iova - start) & (SPRD_IOMMU_PAGE_SIZE - 1)); 381 spin_unlock_irqrestore(&dom->pgtlock, flags); 382 383 return pa; 384 } 385 386 static struct iommu_device *sprd_iommu_probe_device(struct device *dev) 387 { 388 struct sprd_iommu_device *sdev = dev_iommu_priv_get(dev); 389 390 return &sdev->iommu; 391 } 392 393 static int sprd_iommu_of_xlate(struct device *dev, 394 const struct of_phandle_args *args) 395 { 396 struct platform_device *pdev; 397 398 if (!dev_iommu_priv_get(dev)) { 399 pdev = of_find_device_by_node(args->np); 400 dev_iommu_priv_set(dev, platform_get_drvdata(pdev)); 401 platform_device_put(pdev); 402 } 403 404 return 0; 405 } 406 407 408 static const struct iommu_ops sprd_iommu_ops = { 409 .domain_alloc_paging = sprd_iommu_domain_alloc_paging, 410 .probe_device = sprd_iommu_probe_device, 411 .device_group = generic_single_device_group, 412 .of_xlate = sprd_iommu_of_xlate, 413 .pgsize_bitmap = SPRD_IOMMU_PAGE_SIZE, 414 .owner = THIS_MODULE, 415 .default_domain_ops = &(const struct iommu_domain_ops) { 416 .attach_dev = sprd_iommu_attach_device, 417 .map_pages = sprd_iommu_map, 418 .unmap_pages = sprd_iommu_unmap, 419 .iotlb_sync_map = sprd_iommu_sync_map, 420 .iotlb_sync = sprd_iommu_sync, 421 .iova_to_phys = sprd_iommu_iova_to_phys, 422 .free = sprd_iommu_domain_free, 423 } 424 }; 425 426 static const struct of_device_id sprd_iommu_of_match[] = { 427 { .compatible = "sprd,iommu-v1" }, 428 { }, 429 }; 430 MODULE_DEVICE_TABLE(of, sprd_iommu_of_match); 431 432 /* 433 * Clock is not required, access to some of IOMMUs is controlled by gate 434 * clk, enabled clocks for that kind of IOMMUs before accessing. 435 * Return 0 for success or no clocks found. 436 */ 437 static int sprd_iommu_clk_enable(struct sprd_iommu_device *sdev) 438 { 439 struct clk *eb; 440 441 eb = devm_clk_get_optional(sdev->dev, NULL); 442 if (!eb) 443 return 0; 444 445 if (IS_ERR(eb)) 446 return PTR_ERR(eb); 447 448 sdev->eb = eb; 449 return clk_prepare_enable(eb); 450 } 451 452 static void sprd_iommu_clk_disable(struct sprd_iommu_device *sdev) 453 { 454 if (sdev->eb) 455 clk_disable_unprepare(sdev->eb); 456 } 457 458 static int sprd_iommu_probe(struct platform_device *pdev) 459 { 460 struct sprd_iommu_device *sdev; 461 struct device *dev = &pdev->dev; 462 void __iomem *base; 463 int ret; 464 465 sdev = devm_kzalloc(dev, sizeof(*sdev), GFP_KERNEL); 466 if (!sdev) 467 return -ENOMEM; 468 469 base = devm_platform_ioremap_resource(pdev, 0); 470 if (IS_ERR(base)) { 471 dev_err(dev, "Failed to get ioremap resource.\n"); 472 return PTR_ERR(base); 473 } 474 sdev->base = base; 475 476 sdev->prot_page_va = dma_alloc_coherent(dev, SPRD_IOMMU_PAGE_SIZE, 477 &sdev->prot_page_pa, GFP_KERNEL); 478 if (!sdev->prot_page_va) 479 return -ENOMEM; 480 481 platform_set_drvdata(pdev, sdev); 482 sdev->dev = dev; 483 484 ret = iommu_device_sysfs_add(&sdev->iommu, dev, NULL, dev_name(dev)); 485 if (ret) 486 goto free_page; 487 488 ret = iommu_device_register(&sdev->iommu, &sprd_iommu_ops, dev); 489 if (ret) 490 goto remove_sysfs; 491 492 ret = sprd_iommu_clk_enable(sdev); 493 if (ret) 494 goto unregister_iommu; 495 496 ret = sprd_iommu_get_version(sdev); 497 if (ret < 0) { 498 dev_err(dev, "IOMMU version(%d) is invalid.\n", ret); 499 goto disable_clk; 500 } 501 sdev->ver = ret; 502 503 return 0; 504 505 disable_clk: 506 sprd_iommu_clk_disable(sdev); 507 unregister_iommu: 508 iommu_device_unregister(&sdev->iommu); 509 remove_sysfs: 510 iommu_device_sysfs_remove(&sdev->iommu); 511 free_page: 512 dma_free_coherent(sdev->dev, SPRD_IOMMU_PAGE_SIZE, sdev->prot_page_va, sdev->prot_page_pa); 513 return ret; 514 } 515 516 static void sprd_iommu_remove(struct platform_device *pdev) 517 { 518 struct sprd_iommu_device *sdev = platform_get_drvdata(pdev); 519 520 dma_free_coherent(sdev->dev, SPRD_IOMMU_PAGE_SIZE, sdev->prot_page_va, sdev->prot_page_pa); 521 522 platform_set_drvdata(pdev, NULL); 523 iommu_device_sysfs_remove(&sdev->iommu); 524 iommu_device_unregister(&sdev->iommu); 525 } 526 527 static struct platform_driver sprd_iommu_driver = { 528 .driver = { 529 .name = "sprd-iommu", 530 .of_match_table = sprd_iommu_of_match, 531 .suppress_bind_attrs = true, 532 }, 533 .probe = sprd_iommu_probe, 534 .remove_new = sprd_iommu_remove, 535 }; 536 module_platform_driver(sprd_iommu_driver); 537 538 MODULE_DESCRIPTION("IOMMU driver for Unisoc SoCs"); 539 MODULE_ALIAS("platform:sprd-iommu"); 540 MODULE_LICENSE("GPL"); 541