1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPU-agnostic ARM page table allocator. 4 * 5 * Copyright (C) 2014 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 */ 9 10 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt 11 12 #include <linux/atomic.h> 13 #include <linux/bitops.h> 14 #include <linux/io-pgtable.h> 15 #include <linux/sizes.h> 16 #include <linux/slab.h> 17 #include <linux/types.h> 18 #include <linux/dma-mapping.h> 19 20 #include <asm/barrier.h> 21 22 #include "io-pgtable-arm.h" 23 #include "iommu-pages.h" 24 25 #define ARM_LPAE_MAX_ADDR_BITS 52 26 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 27 #define ARM_LPAE_MAX_LEVELS 4 28 29 /* Struct accessors */ 30 #define io_pgtable_to_data(x) \ 31 container_of((x), struct arm_lpae_io_pgtable, iop) 32 33 #define io_pgtable_ops_to_data(x) \ 34 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) 35 36 /* 37 * Calculate the right shift amount to get to the portion describing level l 38 * in a virtual address mapped by the pagetable in d. 39 */ 40 #define ARM_LPAE_LVL_SHIFT(l,d) \ 41 (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \ 42 ilog2(sizeof(arm_lpae_iopte))) 43 44 #define ARM_LPAE_GRANULE(d) \ 45 (sizeof(arm_lpae_iopte) << (d)->bits_per_level) 46 #define ARM_LPAE_PGD_SIZE(d) \ 47 (sizeof(arm_lpae_iopte) << (d)->pgd_bits) 48 49 #define ARM_LPAE_PTES_PER_TABLE(d) \ 50 (ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte))) 51 52 /* 53 * Calculate the index at level l used to map virtual address a using the 54 * pagetable in d. 55 */ 56 #define ARM_LPAE_PGD_IDX(l,d) \ 57 ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0) 58 59 #define ARM_LPAE_LVL_IDX(a,l,d) \ 60 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ 61 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) 62 63 /* Calculate the block/page mapping size at level l for pagetable in d. */ 64 #define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d)) 65 66 /* Page table bits */ 67 #define ARM_LPAE_PTE_TYPE_SHIFT 0 68 #define ARM_LPAE_PTE_TYPE_MASK 0x3 69 70 #define ARM_LPAE_PTE_TYPE_BLOCK 1 71 #define ARM_LPAE_PTE_TYPE_TABLE 3 72 #define ARM_LPAE_PTE_TYPE_PAGE 3 73 74 #define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12) 75 76 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) 77 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) 78 #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51) 79 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) 80 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) 81 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) 82 #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) 83 #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5) 84 #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) 85 86 /* Software bit for solving coherency races */ 87 #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) 88 89 /* Stage-1 PTE */ 90 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) 91 #define ARM_LPAE_PTE_AP_RDONLY_BIT 7 92 #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)1) << \ 93 ARM_LPAE_PTE_AP_RDONLY_BIT) 94 #define ARM_LPAE_PTE_AP_WR_CLEAN_MASK (ARM_LPAE_PTE_AP_RDONLY | \ 95 ARM_LPAE_PTE_DBM) 96 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 97 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) 98 99 /* Stage-2 PTE */ 100 #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6) 101 #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6) 102 #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6) 103 /* 104 * For !FWB these code to: 105 * 1111 = Normal outer write back cachable / Inner Write Back Cachable 106 * Permit S1 to override 107 * 0101 = Normal Non-cachable / Inner Non-cachable 108 * 0001 = Device / Device-nGnRE 109 * For S2FWB these code: 110 * 0110 Force Normal Write Back 111 * 0101 Normal* is forced Normal-NC, Device unchanged 112 * 0001 Force Device-nGnRE 113 */ 114 #define ARM_LPAE_PTE_MEMATTR_FWB_WB (((arm_lpae_iopte)0x6) << 2) 115 #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2) 116 #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2) 117 #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) 118 119 /* Register bits */ 120 #define ARM_LPAE_VTCR_SL0_MASK 0x3 121 122 #define ARM_LPAE_TCR_T0SZ_SHIFT 0 123 124 #define ARM_LPAE_VTCR_PS_SHIFT 16 125 #define ARM_LPAE_VTCR_PS_MASK 0x7 126 127 #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) 128 #define ARM_LPAE_MAIR_ATTR_MASK 0xff 129 #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 130 #define ARM_LPAE_MAIR_ATTR_NC 0x44 131 #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4 132 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff 133 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 134 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 135 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 136 #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3 137 138 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0) 139 #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2) 140 #define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4) 141 142 #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL 143 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL 144 145 /* IOPTE accessors */ 146 #define iopte_deref(pte, d) phys_to_virt(iopte_to_paddr(pte, d)) 147 148 #define iopte_type(pte) \ 149 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) 150 151 #define iopte_writeable_dirty(pte) \ 152 (((pte) & ARM_LPAE_PTE_AP_WR_CLEAN_MASK) == ARM_LPAE_PTE_DBM) 153 154 #define iopte_set_writeable_clean(ptep) \ 155 set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep)) 156 157 struct arm_lpae_io_pgtable { 158 struct io_pgtable iop; 159 160 int pgd_bits; 161 int start_level; 162 int bits_per_level; 163 164 void *pgd; 165 }; 166 167 typedef u64 arm_lpae_iopte; 168 169 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl, 170 enum io_pgtable_fmt fmt) 171 { 172 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) 173 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE; 174 175 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK; 176 } 177 178 static inline bool iopte_table(arm_lpae_iopte pte, int lvl) 179 { 180 if (lvl == (ARM_LPAE_MAX_LEVELS - 1)) 181 return false; 182 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE; 183 } 184 185 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, 186 struct arm_lpae_io_pgtable *data) 187 { 188 arm_lpae_iopte pte = paddr; 189 190 /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ 191 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; 192 } 193 194 static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, 195 struct arm_lpae_io_pgtable *data) 196 { 197 u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 198 199 if (ARM_LPAE_GRANULE(data) < SZ_64K) 200 return paddr; 201 202 /* Rotate the packed high-order bits back to the top */ 203 return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 204 } 205 206 /* 207 * Convert an index returned by ARM_LPAE_PGD_IDX(), which can point into 208 * a concatenated PGD, into the maximum number of entries that can be 209 * mapped in the same table page. 210 */ 211 static inline int arm_lpae_max_entries(int i, struct arm_lpae_io_pgtable *data) 212 { 213 int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data); 214 215 return ptes_per_table - (i & (ptes_per_table - 1)); 216 } 217 218 /* 219 * Check if concatenated PGDs are mandatory according to Arm DDI0487 (K.a) 220 * 1) R_DXBSH: For 16KB, and 48-bit input size, use level 1 instead of 0. 221 * 2) R_SRKBC: After de-ciphering the table for PA size and valid initial lookup 222 * a) 40 bits PA size with 4K: use level 1 instead of level 0 (2 tables for ias = oas) 223 * b) 40 bits PA size with 16K: use level 2 instead of level 1 (16 tables for ias = oas) 224 * c) 42 bits PA size with 4K: use level 1 instead of level 0 (8 tables for ias = oas) 225 * d) 48 bits PA size with 16K: use level 1 instead of level 0 (2 tables for ias = oas) 226 */ 227 static inline bool arm_lpae_concat_mandatory(struct io_pgtable_cfg *cfg, 228 struct arm_lpae_io_pgtable *data) 229 { 230 unsigned int ias = cfg->ias; 231 unsigned int oas = cfg->oas; 232 233 /* Covers 1 and 2.d */ 234 if ((ARM_LPAE_GRANULE(data) == SZ_16K) && (data->start_level == 0)) 235 return (oas == 48) || (ias == 48); 236 237 /* Covers 2.a and 2.c */ 238 if ((ARM_LPAE_GRANULE(data) == SZ_4K) && (data->start_level == 0)) 239 return (oas == 40) || (oas == 42); 240 241 /* Case 2.b */ 242 return (ARM_LPAE_GRANULE(data) == SZ_16K) && 243 (data->start_level == 1) && (oas == 40); 244 } 245 246 static dma_addr_t __arm_lpae_dma_addr(void *pages) 247 { 248 return (dma_addr_t)virt_to_phys(pages); 249 } 250 251 static void *__arm_lpae_cfg_alloc(size_t size, gfp_t gfp, 252 struct io_pgtable_cfg *cfg, 253 void *cookie) 254 { 255 struct device *dev = cfg->iommu_dev; 256 dma_addr_t dma; 257 void *pages; 258 259 pages = cfg->alloc(cookie, size, gfp); 260 if (!pages) 261 return NULL; 262 263 if (!cfg->coherent_walk) { 264 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); 265 if (dma_mapping_error(dev, dma)) 266 goto out_free; 267 /* 268 * We depend on the IOMMU being able to work with any physical 269 * address directly, so if the DMA layer suggests otherwise by 270 * translating or truncating them, that bodes very badly... 271 */ 272 if (dma != virt_to_phys(pages)) 273 goto out_unmap; 274 } 275 276 return pages; 277 278 out_unmap: 279 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); 280 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); 281 282 out_free: 283 cfg->free(cookie, pages, size); 284 return NULL; 285 } 286 287 static void __arm_lpae_cfg_free(void *pages, size_t size, 288 struct io_pgtable_cfg *cfg, 289 void *cookie) 290 { 291 if (!cfg->coherent_walk) 292 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), 293 size, DMA_TO_DEVICE); 294 295 cfg->free(cookie, pages, size); 296 } 297 298 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, 299 struct io_pgtable_cfg *cfg, 300 void *cookie) 301 { 302 struct device *dev = cfg->iommu_dev; 303 void *pages; 304 305 /* 306 * For very small starting-level translation tables the HW requires a 307 * minimum alignment of at least 64 to cover all cases. 308 */ 309 size = max(size, 64); 310 311 if (cfg->alloc) 312 return __arm_lpae_cfg_alloc(size, gfp, cfg, cookie); 313 314 pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, size); 315 if (!pages) 316 return NULL; 317 318 if (!cfg->coherent_walk) { 319 int ret = iommu_pages_start_incoherent(pages, dev); 320 321 if (ret) { 322 if (ret == -EOPNOTSUPP) 323 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); 324 iommu_free_pages(pages); 325 return NULL; 326 } 327 } 328 329 return pages; 330 } 331 332 static void __arm_lpae_free_pages(void *pages, size_t size, 333 struct io_pgtable_cfg *cfg, 334 void *cookie) 335 { 336 /* See __arm_lpae_alloc_pages(). */ 337 size = max(size, 64); 338 339 if (cfg->free) { 340 __arm_lpae_cfg_free(pages, size, cfg, cookie); 341 return; 342 } 343 344 if (!cfg->coherent_walk) 345 iommu_pages_free_incoherent(pages, cfg->iommu_dev); 346 else 347 iommu_free_pages(pages); 348 } 349 350 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, 351 struct io_pgtable_cfg *cfg) 352 { 353 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), 354 sizeof(*ptep) * num_entries, DMA_TO_DEVICE); 355 } 356 357 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries) 358 { 359 for (int i = 0; i < num_entries; i++) 360 ptep[i] = 0; 361 362 if (!cfg->coherent_walk && num_entries) 363 __arm_lpae_sync_pte(ptep, num_entries, cfg); 364 } 365 366 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 367 struct iommu_iotlb_gather *gather, 368 unsigned long iova, size_t size, size_t pgcount, 369 int lvl, arm_lpae_iopte *ptep); 370 371 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 372 phys_addr_t paddr, arm_lpae_iopte prot, 373 int lvl, int num_entries, arm_lpae_iopte *ptep) 374 { 375 arm_lpae_iopte pte = prot; 376 struct io_pgtable_cfg *cfg = &data->iop.cfg; 377 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 378 int i; 379 380 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) 381 pte |= ARM_LPAE_PTE_TYPE_PAGE; 382 else 383 pte |= ARM_LPAE_PTE_TYPE_BLOCK; 384 385 for (i = 0; i < num_entries; i++) 386 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); 387 388 if (!cfg->coherent_walk) 389 __arm_lpae_sync_pte(ptep, num_entries, cfg); 390 } 391 392 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 393 unsigned long iova, phys_addr_t paddr, 394 arm_lpae_iopte prot, int lvl, int num_entries, 395 arm_lpae_iopte *ptep) 396 { 397 int i; 398 399 for (i = 0; i < num_entries; i++) 400 if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) { 401 /* We require an unmap first */ 402 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 403 return -EEXIST; 404 } else if (iopte_type(ptep[i]) == ARM_LPAE_PTE_TYPE_TABLE) { 405 /* 406 * We need to unmap and free the old table before 407 * overwriting it with a block entry. 408 */ 409 arm_lpae_iopte *tblp; 410 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 411 412 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 413 if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1, 414 lvl, tblp) != sz) { 415 WARN_ON(1); 416 return -EINVAL; 417 } 418 } 419 420 __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep); 421 return 0; 422 } 423 424 static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, 425 arm_lpae_iopte *ptep, 426 arm_lpae_iopte curr, 427 struct arm_lpae_io_pgtable *data) 428 { 429 arm_lpae_iopte old, new; 430 struct io_pgtable_cfg *cfg = &data->iop.cfg; 431 432 new = paddr_to_iopte(virt_to_phys(table), data) | ARM_LPAE_PTE_TYPE_TABLE; 433 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) 434 new |= ARM_LPAE_PTE_NSTABLE; 435 436 /* 437 * Ensure the table itself is visible before its PTE can be. 438 * Whilst we could get away with cmpxchg64_release below, this 439 * doesn't have any ordering semantics when !CONFIG_SMP. 440 */ 441 dma_wmb(); 442 443 old = cmpxchg64_relaxed(ptep, curr, new); 444 445 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) 446 return old; 447 448 /* Even if it's not ours, there's no point waiting; just kick it */ 449 __arm_lpae_sync_pte(ptep, 1, cfg); 450 if (old == curr) 451 WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC); 452 453 return old; 454 } 455 456 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, 457 phys_addr_t paddr, size_t size, size_t pgcount, 458 arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep, 459 gfp_t gfp, size_t *mapped) 460 { 461 arm_lpae_iopte *cptep, pte; 462 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 463 size_t tblsz = ARM_LPAE_GRANULE(data); 464 struct io_pgtable_cfg *cfg = &data->iop.cfg; 465 int ret = 0, num_entries, max_entries, map_idx_start; 466 467 /* Find our entry at the current level */ 468 map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); 469 ptep += map_idx_start; 470 471 /* If we can install a leaf entry at this level, then do so */ 472 if (size == block_size) { 473 max_entries = arm_lpae_max_entries(map_idx_start, data); 474 num_entries = min_t(int, pgcount, max_entries); 475 ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); 476 if (!ret) 477 *mapped += num_entries * size; 478 479 return ret; 480 } 481 482 /* We can't allocate tables at the final level */ 483 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) 484 return -EINVAL; 485 486 /* Grab a pointer to the next level */ 487 pte = READ_ONCE(*ptep); 488 if (!pte) { 489 cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie); 490 if (!cptep) 491 return -ENOMEM; 492 493 pte = arm_lpae_install_table(cptep, ptep, 0, data); 494 if (pte) 495 __arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie); 496 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { 497 __arm_lpae_sync_pte(ptep, 1, cfg); 498 } 499 500 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { 501 cptep = iopte_deref(pte, data); 502 } else if (pte) { 503 /* We require an unmap first */ 504 WARN_ON(!(cfg->quirks & IO_PGTABLE_QUIRK_NO_WARN)); 505 return -EEXIST; 506 } 507 508 /* Rinse, repeat */ 509 return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1, 510 cptep, gfp, mapped); 511 } 512 513 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, 514 int prot) 515 { 516 arm_lpae_iopte pte; 517 518 if (data->iop.fmt == ARM_64_LPAE_S1 || 519 data->iop.fmt == ARM_32_LPAE_S1) { 520 pte = ARM_LPAE_PTE_nG; 521 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) 522 pte |= ARM_LPAE_PTE_AP_RDONLY; 523 else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD) 524 pte |= ARM_LPAE_PTE_DBM; 525 if (!(prot & IOMMU_PRIV)) 526 pte |= ARM_LPAE_PTE_AP_UNPRIV; 527 } else { 528 pte = ARM_LPAE_PTE_HAP_FAULT; 529 if (prot & IOMMU_READ) 530 pte |= ARM_LPAE_PTE_HAP_READ; 531 if (prot & IOMMU_WRITE) 532 pte |= ARM_LPAE_PTE_HAP_WRITE; 533 } 534 535 /* 536 * Note that this logic is structured to accommodate Mali LPAE 537 * having stage-1-like attributes but stage-2-like permissions. 538 */ 539 if (data->iop.fmt == ARM_64_LPAE_S2 || 540 data->iop.fmt == ARM_32_LPAE_S2) { 541 if (prot & IOMMU_MMIO) { 542 pte |= ARM_LPAE_PTE_MEMATTR_DEV; 543 } else if (prot & IOMMU_CACHE) { 544 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB) 545 pte |= ARM_LPAE_PTE_MEMATTR_FWB_WB; 546 else 547 pte |= ARM_LPAE_PTE_MEMATTR_OIWB; 548 } else { 549 pte |= ARM_LPAE_PTE_MEMATTR_NC; 550 } 551 } else { 552 if (prot & IOMMU_MMIO) 553 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV 554 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 555 else if (prot & IOMMU_CACHE) 556 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE 557 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 558 } 559 560 /* 561 * Also Mali has its own notions of shareability wherein its Inner 562 * domain covers the cores within the GPU, and its Outer domain is 563 * "outside the GPU" (i.e. either the Inner or System domain in CPU 564 * terms, depending on coherency). 565 */ 566 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE) 567 pte |= ARM_LPAE_PTE_SH_IS; 568 else 569 pte |= ARM_LPAE_PTE_SH_OS; 570 571 if (prot & IOMMU_NOEXEC) 572 pte |= ARM_LPAE_PTE_XN; 573 574 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) 575 pte |= ARM_LPAE_PTE_NS; 576 577 if (data->iop.fmt != ARM_MALI_LPAE) 578 pte |= ARM_LPAE_PTE_AF; 579 580 return pte; 581 } 582 583 static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova, 584 phys_addr_t paddr, size_t pgsize, size_t pgcount, 585 int iommu_prot, gfp_t gfp, size_t *mapped) 586 { 587 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 588 struct io_pgtable_cfg *cfg = &data->iop.cfg; 589 arm_lpae_iopte *ptep = data->pgd; 590 int ret, lvl = data->start_level; 591 arm_lpae_iopte prot; 592 long iaext = (s64)iova >> cfg->ias; 593 594 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) 595 return -EINVAL; 596 597 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 598 iaext = ~iaext; 599 if (WARN_ON(iaext || paddr >> cfg->oas)) 600 return -ERANGE; 601 602 if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) 603 return -EINVAL; 604 605 prot = arm_lpae_prot_to_pte(data, iommu_prot); 606 ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl, 607 ptep, gfp, mapped); 608 /* 609 * Synchronise all PTE updates for the new mapping before there's 610 * a chance for anything to kick off a table walk for the new iova. 611 */ 612 wmb(); 613 614 return ret; 615 } 616 617 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, 618 arm_lpae_iopte *ptep) 619 { 620 arm_lpae_iopte *start, *end; 621 unsigned long table_size; 622 623 if (lvl == data->start_level) 624 table_size = ARM_LPAE_PGD_SIZE(data); 625 else 626 table_size = ARM_LPAE_GRANULE(data); 627 628 start = ptep; 629 630 /* Only leaf entries at the last level */ 631 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 632 end = ptep; 633 else 634 end = (void *)ptep + table_size; 635 636 while (ptep != end) { 637 arm_lpae_iopte pte = *ptep++; 638 639 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) 640 continue; 641 642 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 643 } 644 645 __arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie); 646 } 647 648 static void arm_lpae_free_pgtable(struct io_pgtable *iop) 649 { 650 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); 651 652 __arm_lpae_free_pgtable(data, data->start_level, data->pgd); 653 kfree(data); 654 } 655 656 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 657 struct iommu_iotlb_gather *gather, 658 unsigned long iova, size_t size, size_t pgcount, 659 int lvl, arm_lpae_iopte *ptep) 660 { 661 arm_lpae_iopte pte; 662 struct io_pgtable *iop = &data->iop; 663 int i = 0, num_entries, max_entries, unmap_idx_start; 664 665 /* Something went horribly wrong and we ran out of page table */ 666 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 667 return 0; 668 669 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); 670 ptep += unmap_idx_start; 671 pte = READ_ONCE(*ptep); 672 if (!pte) { 673 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 674 return 0; 675 } 676 677 /* If the size matches this level, we're in the right place */ 678 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { 679 max_entries = arm_lpae_max_entries(unmap_idx_start, data); 680 num_entries = min_t(int, pgcount, max_entries); 681 682 /* Find and handle non-leaf entries */ 683 for (i = 0; i < num_entries; i++) { 684 pte = READ_ONCE(ptep[i]); 685 if (!pte) { 686 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 687 break; 688 } 689 690 if (!iopte_leaf(pte, lvl, iop->fmt)) { 691 __arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1); 692 693 /* Also flush any partial walks */ 694 io_pgtable_tlb_flush_walk(iop, iova + i * size, size, 695 ARM_LPAE_GRANULE(data)); 696 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 697 } 698 } 699 700 /* Clear the remaining entries */ 701 __arm_lpae_clear_pte(ptep, &iop->cfg, i); 702 703 if (gather && !iommu_iotlb_gather_queued(gather)) 704 for (int j = 0; j < i; j++) 705 io_pgtable_tlb_add_page(iop, gather, iova + j * size, size); 706 707 return i * size; 708 } else if (iopte_leaf(pte, lvl, iop->fmt)) { 709 WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed"); 710 return 0; 711 } 712 713 /* Keep on walkin' */ 714 ptep = iopte_deref(pte, data); 715 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep); 716 } 717 718 static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, 719 size_t pgsize, size_t pgcount, 720 struct iommu_iotlb_gather *gather) 721 { 722 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 723 struct io_pgtable_cfg *cfg = &data->iop.cfg; 724 arm_lpae_iopte *ptep = data->pgd; 725 long iaext = (s64)iova >> cfg->ias; 726 727 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) 728 return 0; 729 730 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 731 iaext = ~iaext; 732 if (WARN_ON(iaext)) 733 return 0; 734 735 return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount, 736 data->start_level, ptep); 737 } 738 739 struct io_pgtable_walk_data { 740 struct io_pgtable *iop; 741 void *data; 742 int (*visit)(struct io_pgtable_walk_data *walk_data, int lvl, 743 arm_lpae_iopte *ptep, size_t size); 744 unsigned long flags; 745 u64 addr; 746 const u64 end; 747 }; 748 749 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data, 750 struct io_pgtable_walk_data *walk_data, 751 arm_lpae_iopte *ptep, 752 int lvl); 753 754 struct iova_to_phys_data { 755 arm_lpae_iopte pte; 756 int lvl; 757 }; 758 759 static int visit_iova_to_phys(struct io_pgtable_walk_data *walk_data, int lvl, 760 arm_lpae_iopte *ptep, size_t size) 761 { 762 struct iova_to_phys_data *data = walk_data->data; 763 data->pte = *ptep; 764 data->lvl = lvl; 765 return 0; 766 } 767 768 static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, 769 unsigned long iova) 770 { 771 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 772 struct iova_to_phys_data d; 773 struct io_pgtable_walk_data walk_data = { 774 .data = &d, 775 .visit = visit_iova_to_phys, 776 .addr = iova, 777 .end = iova + 1, 778 }; 779 int ret; 780 781 ret = __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level); 782 if (ret) 783 return 0; 784 785 iova &= (ARM_LPAE_BLOCK_SIZE(d.lvl, data) - 1); 786 return iopte_to_paddr(d.pte, data) | iova; 787 } 788 789 static int visit_pgtable_walk(struct io_pgtable_walk_data *walk_data, int lvl, 790 arm_lpae_iopte *ptep, size_t size) 791 { 792 struct arm_lpae_io_pgtable_walk_data *data = walk_data->data; 793 data->ptes[lvl] = *ptep; 794 return 0; 795 } 796 797 static int arm_lpae_pgtable_walk(struct io_pgtable_ops *ops, unsigned long iova, 798 void *wd) 799 { 800 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 801 struct io_pgtable_walk_data walk_data = { 802 .data = wd, 803 .visit = visit_pgtable_walk, 804 .addr = iova, 805 .end = iova + 1, 806 }; 807 808 return __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level); 809 } 810 811 static int io_pgtable_visit(struct arm_lpae_io_pgtable *data, 812 struct io_pgtable_walk_data *walk_data, 813 arm_lpae_iopte *ptep, int lvl) 814 { 815 struct io_pgtable *iop = &data->iop; 816 arm_lpae_iopte pte = READ_ONCE(*ptep); 817 818 size_t size = ARM_LPAE_BLOCK_SIZE(lvl, data); 819 int ret = walk_data->visit(walk_data, lvl, ptep, size); 820 if (ret) 821 return ret; 822 823 if (iopte_leaf(pte, lvl, iop->fmt)) { 824 walk_data->addr += size; 825 return 0; 826 } 827 828 if (!iopte_table(pte, lvl)) { 829 return -EINVAL; 830 } 831 832 ptep = iopte_deref(pte, data); 833 return __arm_lpae_iopte_walk(data, walk_data, ptep, lvl + 1); 834 } 835 836 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data, 837 struct io_pgtable_walk_data *walk_data, 838 arm_lpae_iopte *ptep, 839 int lvl) 840 { 841 u32 idx; 842 int max_entries, ret; 843 844 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 845 return -EINVAL; 846 847 if (lvl == data->start_level) 848 max_entries = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte); 849 else 850 max_entries = ARM_LPAE_PTES_PER_TABLE(data); 851 852 for (idx = ARM_LPAE_LVL_IDX(walk_data->addr, lvl, data); 853 (idx < max_entries) && (walk_data->addr < walk_data->end); ++idx) { 854 ret = io_pgtable_visit(data, walk_data, ptep + idx, lvl); 855 if (ret) 856 return ret; 857 } 858 859 return 0; 860 } 861 862 static int visit_dirty(struct io_pgtable_walk_data *walk_data, int lvl, 863 arm_lpae_iopte *ptep, size_t size) 864 { 865 struct iommu_dirty_bitmap *dirty = walk_data->data; 866 867 if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt)) 868 return 0; 869 870 if (iopte_writeable_dirty(*ptep)) { 871 iommu_dirty_bitmap_record(dirty, walk_data->addr, size); 872 if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR)) 873 iopte_set_writeable_clean(ptep); 874 } 875 876 return 0; 877 } 878 879 static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops, 880 unsigned long iova, size_t size, 881 unsigned long flags, 882 struct iommu_dirty_bitmap *dirty) 883 { 884 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 885 struct io_pgtable_cfg *cfg = &data->iop.cfg; 886 struct io_pgtable_walk_data walk_data = { 887 .iop = &data->iop, 888 .data = dirty, 889 .visit = visit_dirty, 890 .flags = flags, 891 .addr = iova, 892 .end = iova + size, 893 }; 894 arm_lpae_iopte *ptep = data->pgd; 895 int lvl = data->start_level; 896 897 if (WARN_ON(!size)) 898 return -EINVAL; 899 if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1))) 900 return -EINVAL; 901 if (data->iop.fmt != ARM_64_LPAE_S1) 902 return -EINVAL; 903 904 return __arm_lpae_iopte_walk(data, &walk_data, ptep, lvl); 905 } 906 907 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) 908 { 909 unsigned long granule, page_sizes; 910 unsigned int max_addr_bits = 48; 911 912 /* 913 * We need to restrict the supported page sizes to match the 914 * translation regime for a particular granule. Aim to match 915 * the CPU page size if possible, otherwise prefer smaller sizes. 916 * While we're at it, restrict the block sizes to match the 917 * chosen granule. 918 */ 919 if (cfg->pgsize_bitmap & PAGE_SIZE) 920 granule = PAGE_SIZE; 921 else if (cfg->pgsize_bitmap & ~PAGE_MASK) 922 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); 923 else if (cfg->pgsize_bitmap & PAGE_MASK) 924 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); 925 else 926 granule = 0; 927 928 switch (granule) { 929 case SZ_4K: 930 page_sizes = (SZ_4K | SZ_2M | SZ_1G); 931 break; 932 case SZ_16K: 933 page_sizes = (SZ_16K | SZ_32M); 934 break; 935 case SZ_64K: 936 max_addr_bits = 52; 937 page_sizes = (SZ_64K | SZ_512M); 938 if (cfg->oas > 48) 939 page_sizes |= 1ULL << 42; /* 4TB */ 940 break; 941 default: 942 page_sizes = 0; 943 } 944 945 cfg->pgsize_bitmap &= page_sizes; 946 cfg->ias = min(cfg->ias, max_addr_bits); 947 cfg->oas = min(cfg->oas, max_addr_bits); 948 } 949 950 static struct arm_lpae_io_pgtable * 951 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) 952 { 953 struct arm_lpae_io_pgtable *data; 954 int levels, va_bits, pg_shift; 955 956 arm_lpae_restrict_pgsizes(cfg); 957 958 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) 959 return NULL; 960 961 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) 962 return NULL; 963 964 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) 965 return NULL; 966 967 data = kmalloc_obj(*data); 968 if (!data) 969 return NULL; 970 971 pg_shift = __ffs(cfg->pgsize_bitmap); 972 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte)); 973 974 va_bits = cfg->ias - pg_shift; 975 levels = DIV_ROUND_UP(va_bits, data->bits_per_level); 976 data->start_level = ARM_LPAE_MAX_LEVELS - levels; 977 978 /* Calculate the actual size of our pgd (without concatenation) */ 979 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); 980 981 data->iop.ops = (struct io_pgtable_ops) { 982 .map_pages = arm_lpae_map_pages, 983 .unmap_pages = arm_lpae_unmap_pages, 984 .iova_to_phys = arm_lpae_iova_to_phys, 985 .read_and_clear_dirty = arm_lpae_read_and_clear_dirty, 986 .pgtable_walk = arm_lpae_pgtable_walk, 987 }; 988 989 return data; 990 } 991 992 static struct io_pgtable * 993 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 994 { 995 u64 reg; 996 struct arm_lpae_io_pgtable *data; 997 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; 998 bool tg1; 999 1000 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | 1001 IO_PGTABLE_QUIRK_ARM_TTBR1 | 1002 IO_PGTABLE_QUIRK_ARM_OUTER_WBWA | 1003 IO_PGTABLE_QUIRK_ARM_HD | 1004 IO_PGTABLE_QUIRK_NO_WARN)) 1005 return NULL; 1006 1007 data = arm_lpae_alloc_pgtable(cfg); 1008 if (!data) 1009 return NULL; 1010 1011 /* TCR */ 1012 if (cfg->coherent_walk) { 1013 tcr->sh = ARM_LPAE_TCR_SH_IS; 1014 tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; 1015 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; 1016 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA) 1017 goto out_free_data; 1018 } else { 1019 tcr->sh = ARM_LPAE_TCR_SH_OS; 1020 tcr->irgn = ARM_LPAE_TCR_RGN_NC; 1021 if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) 1022 tcr->orgn = ARM_LPAE_TCR_RGN_NC; 1023 else 1024 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; 1025 } 1026 1027 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; 1028 switch (ARM_LPAE_GRANULE(data)) { 1029 case SZ_4K: 1030 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K; 1031 break; 1032 case SZ_16K: 1033 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K; 1034 break; 1035 case SZ_64K: 1036 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K; 1037 break; 1038 } 1039 1040 switch (cfg->oas) { 1041 case 32: 1042 tcr->ips = ARM_LPAE_TCR_PS_32_BIT; 1043 break; 1044 case 36: 1045 tcr->ips = ARM_LPAE_TCR_PS_36_BIT; 1046 break; 1047 case 40: 1048 tcr->ips = ARM_LPAE_TCR_PS_40_BIT; 1049 break; 1050 case 42: 1051 tcr->ips = ARM_LPAE_TCR_PS_42_BIT; 1052 break; 1053 case 44: 1054 tcr->ips = ARM_LPAE_TCR_PS_44_BIT; 1055 break; 1056 case 48: 1057 tcr->ips = ARM_LPAE_TCR_PS_48_BIT; 1058 break; 1059 case 52: 1060 tcr->ips = ARM_LPAE_TCR_PS_52_BIT; 1061 break; 1062 default: 1063 goto out_free_data; 1064 } 1065 1066 tcr->tsz = 64ULL - cfg->ias; 1067 1068 /* MAIRs */ 1069 reg = (ARM_LPAE_MAIR_ATTR_NC 1070 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | 1071 (ARM_LPAE_MAIR_ATTR_WBRWA 1072 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | 1073 (ARM_LPAE_MAIR_ATTR_DEVICE 1074 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) | 1075 (ARM_LPAE_MAIR_ATTR_INC_OWBRWA 1076 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE)); 1077 1078 cfg->arm_lpae_s1_cfg.mair = reg; 1079 1080 /* Looking good; allocate a pgd */ 1081 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), 1082 GFP_KERNEL, cfg, cookie); 1083 if (!data->pgd) 1084 goto out_free_data; 1085 1086 /* Ensure the empty pgd is visible before any actual TTBR write */ 1087 wmb(); 1088 1089 /* TTBR */ 1090 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); 1091 return &data->iop; 1092 1093 out_free_data: 1094 kfree(data); 1095 return NULL; 1096 } 1097 1098 static struct io_pgtable * 1099 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 1100 { 1101 u64 sl; 1102 struct arm_lpae_io_pgtable *data; 1103 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; 1104 1105 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB | 1106 IO_PGTABLE_QUIRK_NO_WARN)) 1107 return NULL; 1108 1109 data = arm_lpae_alloc_pgtable(cfg); 1110 if (!data) 1111 return NULL; 1112 1113 if (arm_lpae_concat_mandatory(cfg, data)) { 1114 if (WARN_ON((ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte)) > 1115 ARM_LPAE_S2_MAX_CONCAT_PAGES)) 1116 return NULL; 1117 data->pgd_bits += data->bits_per_level; 1118 data->start_level++; 1119 } 1120 1121 /* VTCR */ 1122 if (cfg->coherent_walk) { 1123 vtcr->sh = ARM_LPAE_TCR_SH_IS; 1124 vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA; 1125 vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA; 1126 } else { 1127 vtcr->sh = ARM_LPAE_TCR_SH_OS; 1128 vtcr->irgn = ARM_LPAE_TCR_RGN_NC; 1129 vtcr->orgn = ARM_LPAE_TCR_RGN_NC; 1130 } 1131 1132 sl = data->start_level; 1133 1134 switch (ARM_LPAE_GRANULE(data)) { 1135 case SZ_4K: 1136 vtcr->tg = ARM_LPAE_TCR_TG0_4K; 1137 sl++; /* SL0 format is different for 4K granule size */ 1138 break; 1139 case SZ_16K: 1140 vtcr->tg = ARM_LPAE_TCR_TG0_16K; 1141 break; 1142 case SZ_64K: 1143 vtcr->tg = ARM_LPAE_TCR_TG0_64K; 1144 break; 1145 } 1146 1147 switch (cfg->oas) { 1148 case 32: 1149 vtcr->ps = ARM_LPAE_TCR_PS_32_BIT; 1150 break; 1151 case 36: 1152 vtcr->ps = ARM_LPAE_TCR_PS_36_BIT; 1153 break; 1154 case 40: 1155 vtcr->ps = ARM_LPAE_TCR_PS_40_BIT; 1156 break; 1157 case 42: 1158 vtcr->ps = ARM_LPAE_TCR_PS_42_BIT; 1159 break; 1160 case 44: 1161 vtcr->ps = ARM_LPAE_TCR_PS_44_BIT; 1162 break; 1163 case 48: 1164 vtcr->ps = ARM_LPAE_TCR_PS_48_BIT; 1165 break; 1166 case 52: 1167 vtcr->ps = ARM_LPAE_TCR_PS_52_BIT; 1168 break; 1169 default: 1170 goto out_free_data; 1171 } 1172 1173 vtcr->tsz = 64ULL - cfg->ias; 1174 vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK; 1175 1176 /* Allocate pgd pages */ 1177 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), 1178 GFP_KERNEL, cfg, cookie); 1179 if (!data->pgd) 1180 goto out_free_data; 1181 1182 /* Ensure the empty pgd is visible before any actual TTBR write */ 1183 wmb(); 1184 1185 /* VTTBR */ 1186 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); 1187 return &data->iop; 1188 1189 out_free_data: 1190 kfree(data); 1191 return NULL; 1192 } 1193 1194 static struct io_pgtable * 1195 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 1196 { 1197 if (cfg->ias > 32 || cfg->oas > 40) 1198 return NULL; 1199 1200 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 1201 return arm_64_lpae_alloc_pgtable_s1(cfg, cookie); 1202 } 1203 1204 static struct io_pgtable * 1205 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 1206 { 1207 if (cfg->ias > 40 || cfg->oas > 40) 1208 return NULL; 1209 1210 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 1211 return arm_64_lpae_alloc_pgtable_s2(cfg, cookie); 1212 } 1213 1214 static struct io_pgtable * 1215 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) 1216 { 1217 struct arm_lpae_io_pgtable *data; 1218 1219 /* No quirks for Mali (hopefully) */ 1220 if (cfg->quirks) 1221 return NULL; 1222 1223 if (cfg->ias > 48 || cfg->oas > 40) 1224 return NULL; 1225 1226 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 1227 1228 data = arm_lpae_alloc_pgtable(cfg); 1229 if (!data) 1230 return NULL; 1231 1232 /* Mali seems to need a full 4-level table regardless of IAS */ 1233 if (data->start_level > 0) { 1234 data->start_level = 0; 1235 data->pgd_bits = 0; 1236 } 1237 /* 1238 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the 1239 * best we can do is mimic the out-of-tree driver and hope that the 1240 * "implementation-defined caching policy" is good enough. Similarly, 1241 * we'll use it for the sake of a valid attribute for our 'device' 1242 * index, although callers should never request that in practice. 1243 */ 1244 cfg->arm_mali_lpae_cfg.memattr = 1245 (ARM_MALI_LPAE_MEMATTR_IMP_DEF 1246 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | 1247 (ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 1248 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | 1249 (ARM_MALI_LPAE_MEMATTR_IMP_DEF 1250 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); 1251 1252 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, 1253 cfg, cookie); 1254 if (!data->pgd) 1255 goto out_free_data; 1256 1257 /* Ensure the empty pgd is visible before TRANSTAB can be written */ 1258 wmb(); 1259 1260 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | 1261 ARM_MALI_LPAE_TTBR_READ_INNER | 1262 ARM_MALI_LPAE_TTBR_ADRMODE_TABLE; 1263 if (cfg->coherent_walk) 1264 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER; 1265 1266 return &data->iop; 1267 1268 out_free_data: 1269 kfree(data); 1270 return NULL; 1271 } 1272 1273 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { 1274 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1275 .alloc = arm_64_lpae_alloc_pgtable_s1, 1276 .free = arm_lpae_free_pgtable, 1277 }; 1278 1279 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = { 1280 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1281 .alloc = arm_64_lpae_alloc_pgtable_s2, 1282 .free = arm_lpae_free_pgtable, 1283 }; 1284 1285 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = { 1286 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1287 .alloc = arm_32_lpae_alloc_pgtable_s1, 1288 .free = arm_lpae_free_pgtable, 1289 }; 1290 1291 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { 1292 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1293 .alloc = arm_32_lpae_alloc_pgtable_s2, 1294 .free = arm_lpae_free_pgtable, 1295 }; 1296 1297 struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = { 1298 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1299 .alloc = arm_mali_lpae_alloc_pgtable, 1300 .free = arm_lpae_free_pgtable, 1301 }; 1302