1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPU-agnostic ARM page table allocator. 4 * 5 * ARMv7 Short-descriptor format, supporting 6 * - Basic memory attributes 7 * - Simplified access permissions (AP[2:1] model) 8 * - Backwards-compatible TEX remap 9 * - Large pages/supersections (if indicated by the caller) 10 * 11 * Not supporting: 12 * - Legacy access permissions (AP[2:0] model) 13 * 14 * Almost certainly never supporting: 15 * - PXN 16 * - Domains 17 * 18 * Copyright (C) 2014-2015 ARM Limited 19 * Copyright (c) 2014-2015 MediaTek Inc. 20 */ 21 22 #define pr_fmt(fmt) "arm-v7s io-pgtable: " fmt 23 24 #include <linux/atomic.h> 25 #include <linux/dma-mapping.h> 26 #include <linux/gfp.h> 27 #include <linux/io-pgtable.h> 28 #include <linux/iommu.h> 29 #include <linux/kernel.h> 30 #include <linux/kmemleak.h> 31 #include <linux/sizes.h> 32 #include <linux/slab.h> 33 #include <linux/spinlock.h> 34 #include <linux/types.h> 35 36 #include <asm/barrier.h> 37 38 /* Struct accessors */ 39 #define io_pgtable_to_data(x) \ 40 container_of((x), struct arm_v7s_io_pgtable, iop) 41 42 #define io_pgtable_ops_to_data(x) \ 43 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) 44 45 /* 46 * We have 32 bits total; 12 bits resolved at level 1, 8 bits at level 2, 47 * and 12 bits in a page. 48 * MediaTek extend 2 bits to reach 34bits, 14 bits at lvl1 and 8 bits at lvl2. 49 */ 50 #define ARM_V7S_ADDR_BITS 32 51 #define _ARM_V7S_LVL_BITS(lvl, cfg) ((lvl) == 1 ? ((cfg)->ias - 20) : 8) 52 #define ARM_V7S_LVL_SHIFT(lvl) ((lvl) == 1 ? 20 : 12) 53 #define ARM_V7S_TABLE_SHIFT 10 54 55 #define ARM_V7S_PTES_PER_LVL(lvl, cfg) (1 << _ARM_V7S_LVL_BITS(lvl, cfg)) 56 #define ARM_V7S_TABLE_SIZE(lvl, cfg) \ 57 (ARM_V7S_PTES_PER_LVL(lvl, cfg) * sizeof(arm_v7s_iopte)) 58 59 #define ARM_V7S_BLOCK_SIZE(lvl) (1UL << ARM_V7S_LVL_SHIFT(lvl)) 60 #define ARM_V7S_LVL_MASK(lvl) ((u32)(~0U << ARM_V7S_LVL_SHIFT(lvl))) 61 #define ARM_V7S_TABLE_MASK ((u32)(~0U << ARM_V7S_TABLE_SHIFT)) 62 #define _ARM_V7S_IDX_MASK(lvl, cfg) (ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1) 63 #define ARM_V7S_LVL_IDX(addr, lvl, cfg) ({ \ 64 int _l = lvl; \ 65 ((addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l, cfg); \ 66 }) 67 68 /* 69 * Large page/supersection entries are effectively a block of 16 page/section 70 * entries, along the lines of the LPAE contiguous hint, but all with the 71 * same output address. For want of a better common name we'll call them 72 * "contiguous" versions of their respective page/section entries here, but 73 * noting the distinction (WRT to TLB maintenance) that they represent *one* 74 * entry repeated 16 times, not 16 separate entries (as in the LPAE case). 75 */ 76 #define ARM_V7S_CONT_PAGES 16 77 78 /* PTE type bits: these are all mixed up with XN/PXN bits in most cases */ 79 #define ARM_V7S_PTE_TYPE_TABLE 0x1 80 #define ARM_V7S_PTE_TYPE_PAGE 0x2 81 #define ARM_V7S_PTE_TYPE_CONT_PAGE 0x1 82 83 #define ARM_V7S_PTE_IS_VALID(pte) (((pte) & 0x3) != 0) 84 #define ARM_V7S_PTE_IS_TABLE(pte, lvl) \ 85 ((lvl) == 1 && (((pte) & 0x3) == ARM_V7S_PTE_TYPE_TABLE)) 86 87 /* Page table bits */ 88 #define ARM_V7S_ATTR_XN(lvl) BIT(4 * (2 - (lvl))) 89 #define ARM_V7S_ATTR_B BIT(2) 90 #define ARM_V7S_ATTR_C BIT(3) 91 #define ARM_V7S_ATTR_NS_TABLE BIT(3) 92 #define ARM_V7S_ATTR_NS_SECTION BIT(19) 93 94 #define ARM_V7S_CONT_SECTION BIT(18) 95 #define ARM_V7S_CONT_PAGE_XN_SHIFT 15 96 97 /* 98 * The attribute bits are consistently ordered*, but occupy bits [17:10] of 99 * a level 1 PTE vs. bits [11:4] at level 2. Thus we define the individual 100 * fields relative to that 8-bit block, plus a total shift relative to the PTE. 101 */ 102 #define ARM_V7S_ATTR_SHIFT(lvl) (16 - (lvl) * 6) 103 104 #define ARM_V7S_ATTR_MASK 0xff 105 #define ARM_V7S_ATTR_AP0 BIT(0) 106 #define ARM_V7S_ATTR_AP1 BIT(1) 107 #define ARM_V7S_ATTR_AP2 BIT(5) 108 #define ARM_V7S_ATTR_S BIT(6) 109 #define ARM_V7S_ATTR_NG BIT(7) 110 #define ARM_V7S_TEX_SHIFT 2 111 #define ARM_V7S_TEX_MASK 0x7 112 #define ARM_V7S_ATTR_TEX(val) (((val) & ARM_V7S_TEX_MASK) << ARM_V7S_TEX_SHIFT) 113 114 /* MediaTek extend the bits below for PA 32bit/33bit/34bit */ 115 #define ARM_V7S_ATTR_MTK_PA_BIT32 BIT(9) 116 #define ARM_V7S_ATTR_MTK_PA_BIT33 BIT(4) 117 #define ARM_V7S_ATTR_MTK_PA_BIT34 BIT(5) 118 119 /* *well, except for TEX on level 2 large pages, of course :( */ 120 #define ARM_V7S_CONT_PAGE_TEX_SHIFT 6 121 #define ARM_V7S_CONT_PAGE_TEX_MASK (ARM_V7S_TEX_MASK << ARM_V7S_CONT_PAGE_TEX_SHIFT) 122 123 /* Simplified access permissions */ 124 #define ARM_V7S_PTE_AF ARM_V7S_ATTR_AP0 125 #define ARM_V7S_PTE_AP_UNPRIV ARM_V7S_ATTR_AP1 126 #define ARM_V7S_PTE_AP_RDONLY ARM_V7S_ATTR_AP2 127 128 /* Register bits */ 129 #define ARM_V7S_RGN_NC 0 130 #define ARM_V7S_RGN_WBWA 1 131 #define ARM_V7S_RGN_WT 2 132 #define ARM_V7S_RGN_WB 3 133 134 #define ARM_V7S_PRRR_TYPE_DEVICE 1 135 #define ARM_V7S_PRRR_TYPE_NORMAL 2 136 #define ARM_V7S_PRRR_TR(n, type) (((type) & 0x3) << ((n) * 2)) 137 #define ARM_V7S_PRRR_DS0 BIT(16) 138 #define ARM_V7S_PRRR_DS1 BIT(17) 139 #define ARM_V7S_PRRR_NS0 BIT(18) 140 #define ARM_V7S_PRRR_NS1 BIT(19) 141 #define ARM_V7S_PRRR_NOS(n) BIT((n) + 24) 142 143 #define ARM_V7S_NMRR_IR(n, attr) (((attr) & 0x3) << ((n) * 2)) 144 #define ARM_V7S_NMRR_OR(n, attr) (((attr) & 0x3) << ((n) * 2 + 16)) 145 146 #define ARM_V7S_TTBR_S BIT(1) 147 #define ARM_V7S_TTBR_NOS BIT(5) 148 #define ARM_V7S_TTBR_ORGN_ATTR(attr) (((attr) & 0x3) << 3) 149 #define ARM_V7S_TTBR_IRGN_ATTR(attr) \ 150 ((((attr) & 0x1) << 6) | (((attr) & 0x2) >> 1)) 151 152 #ifdef CONFIG_ZONE_DMA32 153 #define ARM_V7S_TABLE_GFP_DMA GFP_DMA32 154 #define ARM_V7S_TABLE_SLAB_FLAGS SLAB_CACHE_DMA32 155 #else 156 #define ARM_V7S_TABLE_GFP_DMA GFP_DMA 157 #define ARM_V7S_TABLE_SLAB_FLAGS SLAB_CACHE_DMA 158 #endif 159 160 typedef u32 arm_v7s_iopte; 161 162 static bool selftest_running; 163 164 struct arm_v7s_io_pgtable { 165 struct io_pgtable iop; 166 167 arm_v7s_iopte *pgd; 168 struct kmem_cache *l2_tables; 169 }; 170 171 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl); 172 173 static dma_addr_t __arm_v7s_dma_addr(void *pages) 174 { 175 return (dma_addr_t)virt_to_phys(pages); 176 } 177 178 static bool arm_v7s_is_mtk_enabled(struct io_pgtable_cfg *cfg) 179 { 180 return IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && 181 (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT); 182 } 183 184 static arm_v7s_iopte to_mtk_iopte(phys_addr_t paddr, arm_v7s_iopte pte) 185 { 186 if (paddr & BIT_ULL(32)) 187 pte |= ARM_V7S_ATTR_MTK_PA_BIT32; 188 if (paddr & BIT_ULL(33)) 189 pte |= ARM_V7S_ATTR_MTK_PA_BIT33; 190 if (paddr & BIT_ULL(34)) 191 pte |= ARM_V7S_ATTR_MTK_PA_BIT34; 192 return pte; 193 } 194 195 static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl, 196 struct io_pgtable_cfg *cfg) 197 { 198 arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl); 199 200 if (arm_v7s_is_mtk_enabled(cfg)) 201 return to_mtk_iopte(paddr, pte); 202 203 return pte; 204 } 205 206 static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl, 207 struct io_pgtable_cfg *cfg) 208 { 209 arm_v7s_iopte mask; 210 phys_addr_t paddr; 211 212 if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) 213 mask = ARM_V7S_TABLE_MASK; 214 else if (arm_v7s_pte_is_cont(pte, lvl)) 215 mask = ARM_V7S_LVL_MASK(lvl) * ARM_V7S_CONT_PAGES; 216 else 217 mask = ARM_V7S_LVL_MASK(lvl); 218 219 paddr = pte & mask; 220 if (!arm_v7s_is_mtk_enabled(cfg)) 221 return paddr; 222 223 if (pte & ARM_V7S_ATTR_MTK_PA_BIT32) 224 paddr |= BIT_ULL(32); 225 if (pte & ARM_V7S_ATTR_MTK_PA_BIT33) 226 paddr |= BIT_ULL(33); 227 if (pte & ARM_V7S_ATTR_MTK_PA_BIT34) 228 paddr |= BIT_ULL(34); 229 return paddr; 230 } 231 232 static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl, 233 struct arm_v7s_io_pgtable *data) 234 { 235 return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg)); 236 } 237 238 static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, 239 struct arm_v7s_io_pgtable *data) 240 { 241 struct io_pgtable_cfg *cfg = &data->iop.cfg; 242 struct device *dev = cfg->iommu_dev; 243 phys_addr_t phys; 244 dma_addr_t dma; 245 size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg); 246 void *table = NULL; 247 gfp_t gfp_l1; 248 249 /* 250 * ARM_MTK_TTBR_EXT extend the translation table base support larger 251 * memory address. 252 */ 253 gfp_l1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? 254 GFP_KERNEL : ARM_V7S_TABLE_GFP_DMA; 255 256 if (lvl == 1) 257 table = (void *)__get_free_pages(gfp_l1 | __GFP_ZERO, get_order(size)); 258 else if (lvl == 2) 259 table = kmem_cache_zalloc(data->l2_tables, gfp); 260 261 if (!table) 262 return NULL; 263 264 phys = virt_to_phys(table); 265 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? 266 phys >= (1ULL << cfg->oas) : phys != (arm_v7s_iopte)phys) { 267 /* Doesn't fit in PTE */ 268 dev_err(dev, "Page table does not fit in PTE: %pa", &phys); 269 goto out_free; 270 } 271 if (!cfg->coherent_walk) { 272 dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); 273 if (dma_mapping_error(dev, dma)) 274 goto out_free; 275 /* 276 * We depend on the IOMMU being able to work with any physical 277 * address directly, so if the DMA layer suggests otherwise by 278 * translating or truncating them, that bodes very badly... 279 */ 280 if (dma != phys) 281 goto out_unmap; 282 } 283 if (lvl == 2) 284 kmemleak_ignore(table); 285 return table; 286 287 out_unmap: 288 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); 289 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); 290 out_free: 291 if (lvl == 1) 292 free_pages((unsigned long)table, get_order(size)); 293 else 294 kmem_cache_free(data->l2_tables, table); 295 return NULL; 296 } 297 298 static void __arm_v7s_free_table(void *table, int lvl, 299 struct arm_v7s_io_pgtable *data) 300 { 301 struct io_pgtable_cfg *cfg = &data->iop.cfg; 302 struct device *dev = cfg->iommu_dev; 303 size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg); 304 305 if (!cfg->coherent_walk) 306 dma_unmap_single(dev, __arm_v7s_dma_addr(table), size, 307 DMA_TO_DEVICE); 308 if (lvl == 1) 309 free_pages((unsigned long)table, get_order(size)); 310 else 311 kmem_cache_free(data->l2_tables, table); 312 } 313 314 static void __arm_v7s_pte_sync(arm_v7s_iopte *ptep, int num_entries, 315 struct io_pgtable_cfg *cfg) 316 { 317 if (cfg->coherent_walk) 318 return; 319 320 dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep), 321 num_entries * sizeof(*ptep), DMA_TO_DEVICE); 322 } 323 static void __arm_v7s_set_pte(arm_v7s_iopte *ptep, arm_v7s_iopte pte, 324 int num_entries, struct io_pgtable_cfg *cfg) 325 { 326 int i; 327 328 for (i = 0; i < num_entries; i++) 329 ptep[i] = pte; 330 331 __arm_v7s_pte_sync(ptep, num_entries, cfg); 332 } 333 334 static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, 335 struct io_pgtable_cfg *cfg) 336 { 337 bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS); 338 arm_v7s_iopte pte = ARM_V7S_ATTR_NG | ARM_V7S_ATTR_S; 339 340 if (!(prot & IOMMU_MMIO)) 341 pte |= ARM_V7S_ATTR_TEX(1); 342 if (ap) { 343 pte |= ARM_V7S_PTE_AF; 344 if (!(prot & IOMMU_PRIV)) 345 pte |= ARM_V7S_PTE_AP_UNPRIV; 346 if (!(prot & IOMMU_WRITE)) 347 pte |= ARM_V7S_PTE_AP_RDONLY; 348 } 349 pte <<= ARM_V7S_ATTR_SHIFT(lvl); 350 351 if ((prot & IOMMU_NOEXEC) && ap) 352 pte |= ARM_V7S_ATTR_XN(lvl); 353 if (prot & IOMMU_MMIO) 354 pte |= ARM_V7S_ATTR_B; 355 else if (prot & IOMMU_CACHE) 356 pte |= ARM_V7S_ATTR_B | ARM_V7S_ATTR_C; 357 358 pte |= ARM_V7S_PTE_TYPE_PAGE; 359 if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)) 360 pte |= ARM_V7S_ATTR_NS_SECTION; 361 362 return pte; 363 } 364 365 static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl) 366 { 367 if (lvl == 1) { 368 pte |= ARM_V7S_CONT_SECTION; 369 } else if (lvl == 2) { 370 arm_v7s_iopte xn = pte & ARM_V7S_ATTR_XN(lvl); 371 arm_v7s_iopte tex = pte & ARM_V7S_CONT_PAGE_TEX_MASK; 372 373 pte ^= xn | tex | ARM_V7S_PTE_TYPE_PAGE; 374 pte |= (xn << ARM_V7S_CONT_PAGE_XN_SHIFT) | 375 (tex << ARM_V7S_CONT_PAGE_TEX_SHIFT) | 376 ARM_V7S_PTE_TYPE_CONT_PAGE; 377 } 378 return pte; 379 } 380 381 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl) 382 { 383 if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl)) 384 return pte & ARM_V7S_CONT_SECTION; 385 else if (lvl == 2) 386 return !(pte & ARM_V7S_PTE_TYPE_PAGE); 387 return false; 388 } 389 390 static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *, 391 struct iommu_iotlb_gather *, unsigned long, 392 size_t, int, arm_v7s_iopte *); 393 394 static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, 395 unsigned long iova, phys_addr_t paddr, int prot, 396 int lvl, int num_entries, arm_v7s_iopte *ptep) 397 { 398 struct io_pgtable_cfg *cfg = &data->iop.cfg; 399 arm_v7s_iopte pte; 400 int i; 401 402 for (i = 0; i < num_entries; i++) 403 if (ARM_V7S_PTE_IS_TABLE(ptep[i], lvl)) { 404 /* 405 * We need to unmap and free the old table before 406 * overwriting it with a block entry. 407 */ 408 arm_v7s_iopte *tblp; 409 size_t sz = ARM_V7S_BLOCK_SIZE(lvl); 410 411 tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg); 412 if (WARN_ON(__arm_v7s_unmap(data, NULL, iova + i * sz, 413 sz, lvl, tblp) != sz)) 414 return -EINVAL; 415 } else if (ptep[i]) { 416 /* We require an unmap first */ 417 WARN_ON(!selftest_running); 418 return -EEXIST; 419 } 420 421 pte = arm_v7s_prot_to_pte(prot, lvl, cfg); 422 if (num_entries > 1) 423 pte = arm_v7s_pte_to_cont(pte, lvl); 424 425 pte |= paddr_to_iopte(paddr, lvl, cfg); 426 427 __arm_v7s_set_pte(ptep, pte, num_entries, cfg); 428 return 0; 429 } 430 431 static arm_v7s_iopte arm_v7s_install_table(arm_v7s_iopte *table, 432 arm_v7s_iopte *ptep, 433 arm_v7s_iopte curr, 434 struct io_pgtable_cfg *cfg) 435 { 436 phys_addr_t phys = virt_to_phys(table); 437 arm_v7s_iopte old, new; 438 439 new = phys | ARM_V7S_PTE_TYPE_TABLE; 440 441 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) 442 new = to_mtk_iopte(phys, new); 443 444 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) 445 new |= ARM_V7S_ATTR_NS_TABLE; 446 447 /* 448 * Ensure the table itself is visible before its PTE can be. 449 * Whilst we could get away with cmpxchg64_release below, this 450 * doesn't have any ordering semantics when !CONFIG_SMP. 451 */ 452 dma_wmb(); 453 454 old = cmpxchg_relaxed(ptep, curr, new); 455 __arm_v7s_pte_sync(ptep, 1, cfg); 456 457 return old; 458 } 459 460 static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, 461 phys_addr_t paddr, size_t size, int prot, 462 int lvl, arm_v7s_iopte *ptep, gfp_t gfp) 463 { 464 struct io_pgtable_cfg *cfg = &data->iop.cfg; 465 arm_v7s_iopte pte, *cptep; 466 int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); 467 468 /* Find our entry at the current level */ 469 ptep += ARM_V7S_LVL_IDX(iova, lvl, cfg); 470 471 /* If we can install a leaf entry at this level, then do so */ 472 if (num_entries) 473 return arm_v7s_init_pte(data, iova, paddr, prot, 474 lvl, num_entries, ptep); 475 476 /* We can't allocate tables at the final level */ 477 if (WARN_ON(lvl == 2)) 478 return -EINVAL; 479 480 /* Grab a pointer to the next level */ 481 pte = READ_ONCE(*ptep); 482 if (!pte) { 483 cptep = __arm_v7s_alloc_table(lvl + 1, gfp, data); 484 if (!cptep) 485 return -ENOMEM; 486 487 pte = arm_v7s_install_table(cptep, ptep, 0, cfg); 488 if (pte) 489 __arm_v7s_free_table(cptep, lvl + 1, data); 490 } else { 491 /* We've no easy way of knowing if it's synced yet, so... */ 492 __arm_v7s_pte_sync(ptep, 1, cfg); 493 } 494 495 if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) { 496 cptep = iopte_deref(pte, lvl, data); 497 } else if (pte) { 498 /* We require an unmap first */ 499 WARN_ON(!selftest_running); 500 return -EEXIST; 501 } 502 503 /* Rinse, repeat */ 504 return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp); 505 } 506 507 static int arm_v7s_map_pages(struct io_pgtable_ops *ops, unsigned long iova, 508 phys_addr_t paddr, size_t pgsize, size_t pgcount, 509 int prot, gfp_t gfp, size_t *mapped) 510 { 511 struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); 512 int ret = -EINVAL; 513 514 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || 515 paddr >= (1ULL << data->iop.cfg.oas))) 516 return -ERANGE; 517 518 if (!(prot & (IOMMU_READ | IOMMU_WRITE))) 519 return -EINVAL; 520 521 while (pgcount--) { 522 ret = __arm_v7s_map(data, iova, paddr, pgsize, prot, 1, data->pgd, 523 gfp); 524 if (ret) 525 break; 526 527 iova += pgsize; 528 paddr += pgsize; 529 *mapped += pgsize; 530 } 531 /* 532 * Synchronise all PTE updates for the new mapping before there's 533 * a chance for anything to kick off a table walk for the new iova. 534 */ 535 wmb(); 536 537 return ret; 538 } 539 540 static void arm_v7s_free_pgtable(struct io_pgtable *iop) 541 { 542 struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop); 543 int i; 544 545 for (i = 0; i < ARM_V7S_PTES_PER_LVL(1, &data->iop.cfg); i++) { 546 arm_v7s_iopte pte = data->pgd[i]; 547 548 if (ARM_V7S_PTE_IS_TABLE(pte, 1)) 549 __arm_v7s_free_table(iopte_deref(pte, 1, data), 550 2, data); 551 } 552 __arm_v7s_free_table(data->pgd, 1, data); 553 kmem_cache_destroy(data->l2_tables); 554 kfree(data); 555 } 556 557 static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, 558 struct iommu_iotlb_gather *gather, 559 unsigned long iova, size_t size, int lvl, 560 arm_v7s_iopte *ptep) 561 { 562 arm_v7s_iopte pte[ARM_V7S_CONT_PAGES]; 563 struct io_pgtable *iop = &data->iop; 564 int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); 565 566 /* Something went horribly wrong and we ran out of page table */ 567 if (WARN_ON(lvl > 2)) 568 return 0; 569 570 idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg); 571 ptep += idx; 572 do { 573 pte[i] = READ_ONCE(ptep[i]); 574 if (WARN_ON(!ARM_V7S_PTE_IS_VALID(pte[i]))) 575 return 0; 576 } while (++i < num_entries); 577 578 /* 579 * If we've hit a contiguous 'large page' entry at this level, it 580 * needs splitting first, unless we're unmapping the whole lot. 581 * 582 * For splitting, we can't rewrite 16 PTEs atomically, and since we 583 * can't necessarily assume TEX remap we don't have a software bit to 584 * mark live entries being split. In practice (i.e. DMA API code), we 585 * will never be splitting large pages anyway, so just wrap this edge 586 * case in a lock for the sake of correctness and be done with it. 587 */ 588 if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) { 589 WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed"); 590 return 0; 591 } 592 593 /* If the size matches this level, we're in the right place */ 594 if (num_entries) { 595 size_t blk_size = ARM_V7S_BLOCK_SIZE(lvl); 596 597 __arm_v7s_set_pte(ptep, 0, num_entries, &iop->cfg); 598 599 for (i = 0; i < num_entries; i++) { 600 if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) { 601 /* Also flush any partial walks */ 602 io_pgtable_tlb_flush_walk(iop, iova, blk_size, 603 ARM_V7S_BLOCK_SIZE(lvl + 1)); 604 ptep = iopte_deref(pte[i], lvl, data); 605 __arm_v7s_free_table(ptep, lvl + 1, data); 606 } else if (!iommu_iotlb_gather_queued(gather)) { 607 io_pgtable_tlb_add_page(iop, gather, iova, blk_size); 608 } 609 iova += blk_size; 610 } 611 return size; 612 } else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) { 613 WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed"); 614 return 0; 615 } 616 617 /* Keep on walkin' */ 618 ptep = iopte_deref(pte[0], lvl, data); 619 return __arm_v7s_unmap(data, gather, iova, size, lvl + 1, ptep); 620 } 621 622 static size_t arm_v7s_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, 623 size_t pgsize, size_t pgcount, 624 struct iommu_iotlb_gather *gather) 625 { 626 struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); 627 size_t unmapped = 0, ret; 628 629 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) 630 return 0; 631 632 while (pgcount--) { 633 ret = __arm_v7s_unmap(data, gather, iova, pgsize, 1, data->pgd); 634 if (!ret) 635 break; 636 637 unmapped += pgsize; 638 iova += pgsize; 639 } 640 641 return unmapped; 642 } 643 644 static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, 645 unsigned long iova) 646 { 647 struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); 648 arm_v7s_iopte *ptep = data->pgd, pte; 649 int lvl = 0; 650 u32 mask; 651 652 do { 653 ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg); 654 pte = READ_ONCE(*ptep); 655 ptep = iopte_deref(pte, lvl, data); 656 } while (ARM_V7S_PTE_IS_TABLE(pte, lvl)); 657 658 if (!ARM_V7S_PTE_IS_VALID(pte)) 659 return 0; 660 661 mask = ARM_V7S_LVL_MASK(lvl); 662 if (arm_v7s_pte_is_cont(pte, lvl)) 663 mask *= ARM_V7S_CONT_PAGES; 664 return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask); 665 } 666 667 static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, 668 void *cookie) 669 { 670 struct arm_v7s_io_pgtable *data; 671 slab_flags_t slab_flag; 672 phys_addr_t paddr; 673 674 if (cfg->ias > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS)) 675 return NULL; 676 677 if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 35 : ARM_V7S_ADDR_BITS)) 678 return NULL; 679 680 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | 681 IO_PGTABLE_QUIRK_NO_PERMS | 682 IO_PGTABLE_QUIRK_ARM_MTK_EXT | 683 IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT)) 684 return NULL; 685 686 /* If ARM_MTK_4GB is enabled, the NO_PERMS is also expected. */ 687 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT && 688 !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS)) 689 return NULL; 690 691 if ((cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT) && 692 !arm_v7s_is_mtk_enabled(cfg)) 693 return NULL; 694 695 data = kmalloc(sizeof(*data), GFP_KERNEL); 696 if (!data) 697 return NULL; 698 699 /* 700 * ARM_MTK_TTBR_EXT extend the translation table base support larger 701 * memory address. 702 */ 703 slab_flag = cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT ? 704 0 : ARM_V7S_TABLE_SLAB_FLAGS; 705 706 data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2", 707 ARM_V7S_TABLE_SIZE(2, cfg), 708 ARM_V7S_TABLE_SIZE(2, cfg), 709 slab_flag, NULL); 710 if (!data->l2_tables) 711 goto out_free_data; 712 713 data->iop.ops = (struct io_pgtable_ops) { 714 .map_pages = arm_v7s_map_pages, 715 .unmap_pages = arm_v7s_unmap_pages, 716 .iova_to_phys = arm_v7s_iova_to_phys, 717 }; 718 719 /* We have to do this early for __arm_v7s_alloc_table to work... */ 720 data->iop.cfg = *cfg; 721 722 /* 723 * Unless the IOMMU driver indicates supersection support by 724 * having SZ_16M set in the initial bitmap, they won't be used. 725 */ 726 cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M; 727 728 /* TCR: T0SZ=0, EAE=0 (if applicable) */ 729 cfg->arm_v7s_cfg.tcr = 0; 730 731 /* 732 * TEX remap: the indices used map to the closest equivalent types 733 * under the non-TEX-remap interpretation of those attribute bits, 734 * excepting various implementation-defined aspects of shareability. 735 */ 736 cfg->arm_v7s_cfg.prrr = ARM_V7S_PRRR_TR(1, ARM_V7S_PRRR_TYPE_DEVICE) | 737 ARM_V7S_PRRR_TR(4, ARM_V7S_PRRR_TYPE_NORMAL) | 738 ARM_V7S_PRRR_TR(7, ARM_V7S_PRRR_TYPE_NORMAL) | 739 ARM_V7S_PRRR_DS0 | ARM_V7S_PRRR_DS1 | 740 ARM_V7S_PRRR_NS1 | ARM_V7S_PRRR_NOS(7); 741 cfg->arm_v7s_cfg.nmrr = ARM_V7S_NMRR_IR(7, ARM_V7S_RGN_WBWA) | 742 ARM_V7S_NMRR_OR(7, ARM_V7S_RGN_WBWA); 743 744 /* Looking good; allocate a pgd */ 745 data->pgd = __arm_v7s_alloc_table(1, GFP_KERNEL, data); 746 if (!data->pgd) 747 goto out_free_data; 748 749 /* Ensure the empty pgd is visible before any actual TTBR write */ 750 wmb(); 751 752 /* TTBR */ 753 paddr = virt_to_phys(data->pgd); 754 if (arm_v7s_is_mtk_enabled(cfg)) 755 cfg->arm_v7s_cfg.ttbr = paddr | upper_32_bits(paddr); 756 else 757 cfg->arm_v7s_cfg.ttbr = paddr | ARM_V7S_TTBR_S | 758 (cfg->coherent_walk ? (ARM_V7S_TTBR_NOS | 759 ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | 760 ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA)) : 761 (ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_NC) | 762 ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_NC))); 763 return &data->iop; 764 765 out_free_data: 766 kmem_cache_destroy(data->l2_tables); 767 kfree(data); 768 return NULL; 769 } 770 771 struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns = { 772 .alloc = arm_v7s_alloc_pgtable, 773 .free = arm_v7s_free_pgtable, 774 }; 775 776 #ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S_SELFTEST 777 778 static struct io_pgtable_cfg *cfg_cookie __initdata; 779 780 static void __init dummy_tlb_flush_all(void *cookie) 781 { 782 WARN_ON(cookie != cfg_cookie); 783 } 784 785 static void __init dummy_tlb_flush(unsigned long iova, size_t size, 786 size_t granule, void *cookie) 787 { 788 WARN_ON(cookie != cfg_cookie); 789 WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); 790 } 791 792 static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather, 793 unsigned long iova, size_t granule, 794 void *cookie) 795 { 796 dummy_tlb_flush(iova, granule, granule, cookie); 797 } 798 799 static const struct iommu_flush_ops dummy_tlb_ops __initconst = { 800 .tlb_flush_all = dummy_tlb_flush_all, 801 .tlb_flush_walk = dummy_tlb_flush, 802 .tlb_add_page = dummy_tlb_add_page, 803 }; 804 805 #define __FAIL(ops) ({ \ 806 WARN(1, "selftest: test failed\n"); \ 807 selftest_running = false; \ 808 -EFAULT; \ 809 }) 810 811 static int __init arm_v7s_do_selftests(void) 812 { 813 struct io_pgtable_ops *ops; 814 struct io_pgtable_cfg cfg = { 815 .tlb = &dummy_tlb_ops, 816 .oas = 32, 817 .ias = 32, 818 .coherent_walk = true, 819 .quirks = IO_PGTABLE_QUIRK_ARM_NS, 820 .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, 821 }; 822 unsigned int iova, size; 823 unsigned int i; 824 size_t mapped; 825 826 selftest_running = true; 827 828 cfg_cookie = &cfg; 829 830 ops = alloc_io_pgtable_ops(ARM_V7S, &cfg, &cfg); 831 if (!ops) { 832 pr_err("selftest: failed to allocate io pgtable ops\n"); 833 return -EINVAL; 834 } 835 836 /* 837 * Initial sanity checks. 838 * Empty page tables shouldn't provide any translations. 839 */ 840 if (ops->iova_to_phys(ops, 42)) 841 return __FAIL(ops); 842 843 if (ops->iova_to_phys(ops, SZ_1G + 42)) 844 return __FAIL(ops); 845 846 if (ops->iova_to_phys(ops, SZ_2G + 42)) 847 return __FAIL(ops); 848 849 /* 850 * Distinct mappings of different granule sizes. 851 */ 852 iova = 0; 853 for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) { 854 size = 1UL << i; 855 if (ops->map_pages(ops, iova, iova, size, 1, 856 IOMMU_READ | IOMMU_WRITE | 857 IOMMU_NOEXEC | IOMMU_CACHE, 858 GFP_KERNEL, &mapped)) 859 return __FAIL(ops); 860 861 /* Overlapping mappings */ 862 if (!ops->map_pages(ops, iova, iova + size, size, 1, 863 IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL, 864 &mapped)) 865 return __FAIL(ops); 866 867 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) 868 return __FAIL(ops); 869 870 iova += SZ_16M; 871 } 872 873 /* Full unmap */ 874 iova = 0; 875 for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) { 876 size = 1UL << i; 877 878 if (ops->unmap_pages(ops, iova, size, 1, NULL) != size) 879 return __FAIL(ops); 880 881 if (ops->iova_to_phys(ops, iova + 42)) 882 return __FAIL(ops); 883 884 /* Remap full block */ 885 if (ops->map_pages(ops, iova, iova, size, 1, IOMMU_WRITE, 886 GFP_KERNEL, &mapped)) 887 return __FAIL(ops); 888 889 if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) 890 return __FAIL(ops); 891 892 iova += SZ_16M; 893 } 894 895 free_io_pgtable_ops(ops); 896 897 selftest_running = false; 898 899 pr_info("self test ok\n"); 900 return 0; 901 } 902 subsys_initcall(arm_v7s_do_selftests); 903 #endif 904