1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * CPU-agnostic ARM page table allocator. 4 * 5 * Copyright (C) 2014 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 */ 9 10 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt 11 12 #include <linux/atomic.h> 13 #include <linux/bitops.h> 14 #include <linux/io-pgtable.h> 15 #include <linux/sizes.h> 16 #include <linux/slab.h> 17 #include <linux/types.h> 18 #include <linux/dma-mapping.h> 19 20 #include <asm/barrier.h> 21 22 #include "io-pgtable-arm.h" 23 #include "iommu-pages.h" 24 25 #define ARM_LPAE_MAX_ADDR_BITS 52 26 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 27 #define ARM_LPAE_MAX_LEVELS 4 28 29 /* Struct accessors */ 30 #define io_pgtable_to_data(x) \ 31 container_of((x), struct arm_lpae_io_pgtable, iop) 32 33 #define io_pgtable_ops_to_data(x) \ 34 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) 35 36 /* 37 * Calculate the right shift amount to get to the portion describing level l 38 * in a virtual address mapped by the pagetable in d. 39 */ 40 #define ARM_LPAE_LVL_SHIFT(l,d) \ 41 (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \ 42 ilog2(sizeof(arm_lpae_iopte))) 43 44 #define ARM_LPAE_GRANULE(d) \ 45 (sizeof(arm_lpae_iopte) << (d)->bits_per_level) 46 #define ARM_LPAE_PGD_SIZE(d) \ 47 (sizeof(arm_lpae_iopte) << (d)->pgd_bits) 48 49 #define ARM_LPAE_PTES_PER_TABLE(d) \ 50 (ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte))) 51 52 /* 53 * Calculate the index at level l used to map virtual address a using the 54 * pagetable in d. 55 */ 56 #define ARM_LPAE_PGD_IDX(l,d) \ 57 ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0) 58 59 #define ARM_LPAE_LVL_IDX(a,l,d) \ 60 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ 61 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) 62 63 /* Calculate the block/page mapping size at level l for pagetable in d. */ 64 #define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d)) 65 66 /* Page table bits */ 67 #define ARM_LPAE_PTE_TYPE_SHIFT 0 68 #define ARM_LPAE_PTE_TYPE_MASK 0x3 69 70 #define ARM_LPAE_PTE_TYPE_BLOCK 1 71 #define ARM_LPAE_PTE_TYPE_TABLE 3 72 #define ARM_LPAE_PTE_TYPE_PAGE 3 73 74 #define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12) 75 76 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) 77 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) 78 #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51) 79 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) 80 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) 81 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) 82 #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) 83 #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5) 84 #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) 85 86 /* Software bit for solving coherency races */ 87 #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) 88 89 /* Stage-1 PTE */ 90 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) 91 #define ARM_LPAE_PTE_AP_RDONLY_BIT 7 92 #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)1) << \ 93 ARM_LPAE_PTE_AP_RDONLY_BIT) 94 #define ARM_LPAE_PTE_AP_WR_CLEAN_MASK (ARM_LPAE_PTE_AP_RDONLY | \ 95 ARM_LPAE_PTE_DBM) 96 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 97 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) 98 99 /* Stage-2 PTE */ 100 #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6) 101 #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6) 102 #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6) 103 /* 104 * For !FWB these code to: 105 * 1111 = Normal outer write back cachable / Inner Write Back Cachable 106 * Permit S1 to override 107 * 0101 = Normal Non-cachable / Inner Non-cachable 108 * 0001 = Device / Device-nGnRE 109 * For S2FWB these code: 110 * 0110 Force Normal Write Back 111 * 0101 Normal* is forced Normal-NC, Device unchanged 112 * 0001 Force Device-nGnRE 113 */ 114 #define ARM_LPAE_PTE_MEMATTR_FWB_WB (((arm_lpae_iopte)0x6) << 2) 115 #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2) 116 #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2) 117 #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) 118 119 /* Register bits */ 120 #define ARM_LPAE_VTCR_SL0_MASK 0x3 121 122 #define ARM_LPAE_TCR_T0SZ_SHIFT 0 123 124 #define ARM_LPAE_VTCR_PS_SHIFT 16 125 #define ARM_LPAE_VTCR_PS_MASK 0x7 126 127 #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) 128 #define ARM_LPAE_MAIR_ATTR_MASK 0xff 129 #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 130 #define ARM_LPAE_MAIR_ATTR_NC 0x44 131 #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4 132 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff 133 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 134 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 135 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 136 #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3 137 138 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0) 139 #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2) 140 #define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4) 141 142 #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL 143 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL 144 145 /* IOPTE accessors */ 146 #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) 147 148 #define iopte_type(pte) \ 149 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) 150 151 #define iopte_writeable_dirty(pte) \ 152 (((pte) & ARM_LPAE_PTE_AP_WR_CLEAN_MASK) == ARM_LPAE_PTE_DBM) 153 154 #define iopte_set_writeable_clean(ptep) \ 155 set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep)) 156 157 struct arm_lpae_io_pgtable { 158 struct io_pgtable iop; 159 160 int pgd_bits; 161 int start_level; 162 int bits_per_level; 163 164 void *pgd; 165 }; 166 167 typedef u64 arm_lpae_iopte; 168 169 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl, 170 enum io_pgtable_fmt fmt) 171 { 172 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE) 173 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE; 174 175 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK; 176 } 177 178 static inline bool iopte_table(arm_lpae_iopte pte, int lvl) 179 { 180 if (lvl == (ARM_LPAE_MAX_LEVELS - 1)) 181 return false; 182 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE; 183 } 184 185 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, 186 struct arm_lpae_io_pgtable *data) 187 { 188 arm_lpae_iopte pte = paddr; 189 190 /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ 191 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; 192 } 193 194 static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, 195 struct arm_lpae_io_pgtable *data) 196 { 197 u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; 198 199 if (ARM_LPAE_GRANULE(data) < SZ_64K) 200 return paddr; 201 202 /* Rotate the packed high-order bits back to the top */ 203 return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); 204 } 205 206 /* 207 * Convert an index returned by ARM_LPAE_PGD_IDX(), which can point into 208 * a concatenated PGD, into the maximum number of entries that can be 209 * mapped in the same table page. 210 */ 211 static inline int arm_lpae_max_entries(int i, struct arm_lpae_io_pgtable *data) 212 { 213 int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data); 214 215 return ptes_per_table - (i & (ptes_per_table - 1)); 216 } 217 218 /* 219 * Check if concatenated PGDs are mandatory according to Arm DDI0487 (K.a) 220 * 1) R_DXBSH: For 16KB, and 48-bit input size, use level 1 instead of 0. 221 * 2) R_SRKBC: After de-ciphering the table for PA size and valid initial lookup 222 * a) 40 bits PA size with 4K: use level 1 instead of level 0 (2 tables for ias = oas) 223 * b) 40 bits PA size with 16K: use level 2 instead of level 1 (16 tables for ias = oas) 224 * c) 42 bits PA size with 4K: use level 1 instead of level 0 (8 tables for ias = oas) 225 * d) 48 bits PA size with 16K: use level 1 instead of level 0 (2 tables for ias = oas) 226 */ 227 static inline bool arm_lpae_concat_mandatory(struct io_pgtable_cfg *cfg, 228 struct arm_lpae_io_pgtable *data) 229 { 230 unsigned int ias = cfg->ias; 231 unsigned int oas = cfg->oas; 232 233 /* Covers 1 and 2.d */ 234 if ((ARM_LPAE_GRANULE(data) == SZ_16K) && (data->start_level == 0)) 235 return (oas == 48) || (ias == 48); 236 237 /* Covers 2.a and 2.c */ 238 if ((ARM_LPAE_GRANULE(data) == SZ_4K) && (data->start_level == 0)) 239 return (oas == 40) || (oas == 42); 240 241 /* Case 2.b */ 242 return (ARM_LPAE_GRANULE(data) == SZ_16K) && 243 (data->start_level == 1) && (oas == 40); 244 } 245 246 static dma_addr_t __arm_lpae_dma_addr(void *pages) 247 { 248 return (dma_addr_t)virt_to_phys(pages); 249 } 250 251 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, 252 struct io_pgtable_cfg *cfg, 253 void *cookie) 254 { 255 struct device *dev = cfg->iommu_dev; 256 size_t alloc_size; 257 dma_addr_t dma; 258 void *pages; 259 260 /* 261 * For very small starting-level translation tables the HW requires a 262 * minimum alignment of at least 64 to cover all cases. 263 */ 264 alloc_size = max(size, 64); 265 if (cfg->alloc) 266 pages = cfg->alloc(cookie, alloc_size, gfp); 267 else 268 pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, 269 alloc_size); 270 271 if (!pages) 272 return NULL; 273 274 if (!cfg->coherent_walk) { 275 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); 276 if (dma_mapping_error(dev, dma)) 277 goto out_free; 278 /* 279 * We depend on the IOMMU being able to work with any physical 280 * address directly, so if the DMA layer suggests otherwise by 281 * translating or truncating them, that bodes very badly... 282 */ 283 if (dma != virt_to_phys(pages)) 284 goto out_unmap; 285 } 286 287 return pages; 288 289 out_unmap: 290 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); 291 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); 292 293 out_free: 294 if (cfg->free) 295 cfg->free(cookie, pages, size); 296 else 297 iommu_free_pages(pages); 298 299 return NULL; 300 } 301 302 static void __arm_lpae_free_pages(void *pages, size_t size, 303 struct io_pgtable_cfg *cfg, 304 void *cookie) 305 { 306 if (!cfg->coherent_walk) 307 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), 308 size, DMA_TO_DEVICE); 309 310 if (cfg->free) 311 cfg->free(cookie, pages, size); 312 else 313 iommu_free_pages(pages); 314 } 315 316 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, 317 struct io_pgtable_cfg *cfg) 318 { 319 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), 320 sizeof(*ptep) * num_entries, DMA_TO_DEVICE); 321 } 322 323 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries) 324 { 325 for (int i = 0; i < num_entries; i++) 326 ptep[i] = 0; 327 328 if (!cfg->coherent_walk && num_entries) 329 __arm_lpae_sync_pte(ptep, num_entries, cfg); 330 } 331 332 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 333 struct iommu_iotlb_gather *gather, 334 unsigned long iova, size_t size, size_t pgcount, 335 int lvl, arm_lpae_iopte *ptep); 336 337 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 338 phys_addr_t paddr, arm_lpae_iopte prot, 339 int lvl, int num_entries, arm_lpae_iopte *ptep) 340 { 341 arm_lpae_iopte pte = prot; 342 struct io_pgtable_cfg *cfg = &data->iop.cfg; 343 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 344 int i; 345 346 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1) 347 pte |= ARM_LPAE_PTE_TYPE_PAGE; 348 else 349 pte |= ARM_LPAE_PTE_TYPE_BLOCK; 350 351 for (i = 0; i < num_entries; i++) 352 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); 353 354 if (!cfg->coherent_walk) 355 __arm_lpae_sync_pte(ptep, num_entries, cfg); 356 } 357 358 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, 359 unsigned long iova, phys_addr_t paddr, 360 arm_lpae_iopte prot, int lvl, int num_entries, 361 arm_lpae_iopte *ptep) 362 { 363 int i; 364 365 for (i = 0; i < num_entries; i++) 366 if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) { 367 /* We require an unmap first */ 368 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 369 return -EEXIST; 370 } else if (iopte_type(ptep[i]) == ARM_LPAE_PTE_TYPE_TABLE) { 371 /* 372 * We need to unmap and free the old table before 373 * overwriting it with a block entry. 374 */ 375 arm_lpae_iopte *tblp; 376 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); 377 378 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); 379 if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1, 380 lvl, tblp) != sz) { 381 WARN_ON(1); 382 return -EINVAL; 383 } 384 } 385 386 __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep); 387 return 0; 388 } 389 390 static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, 391 arm_lpae_iopte *ptep, 392 arm_lpae_iopte curr, 393 struct arm_lpae_io_pgtable *data) 394 { 395 arm_lpae_iopte old, new; 396 struct io_pgtable_cfg *cfg = &data->iop.cfg; 397 398 new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE; 399 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) 400 new |= ARM_LPAE_PTE_NSTABLE; 401 402 /* 403 * Ensure the table itself is visible before its PTE can be. 404 * Whilst we could get away with cmpxchg64_release below, this 405 * doesn't have any ordering semantics when !CONFIG_SMP. 406 */ 407 dma_wmb(); 408 409 old = cmpxchg64_relaxed(ptep, curr, new); 410 411 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) 412 return old; 413 414 /* Even if it's not ours, there's no point waiting; just kick it */ 415 __arm_lpae_sync_pte(ptep, 1, cfg); 416 if (old == curr) 417 WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC); 418 419 return old; 420 } 421 422 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, 423 phys_addr_t paddr, size_t size, size_t pgcount, 424 arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep, 425 gfp_t gfp, size_t *mapped) 426 { 427 arm_lpae_iopte *cptep, pte; 428 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); 429 size_t tblsz = ARM_LPAE_GRANULE(data); 430 struct io_pgtable_cfg *cfg = &data->iop.cfg; 431 int ret = 0, num_entries, max_entries, map_idx_start; 432 433 /* Find our entry at the current level */ 434 map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); 435 ptep += map_idx_start; 436 437 /* If we can install a leaf entry at this level, then do so */ 438 if (size == block_size) { 439 max_entries = arm_lpae_max_entries(map_idx_start, data); 440 num_entries = min_t(int, pgcount, max_entries); 441 ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); 442 if (!ret) 443 *mapped += num_entries * size; 444 445 return ret; 446 } 447 448 /* We can't allocate tables at the final level */ 449 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) 450 return -EINVAL; 451 452 /* Grab a pointer to the next level */ 453 pte = READ_ONCE(*ptep); 454 if (!pte) { 455 cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie); 456 if (!cptep) 457 return -ENOMEM; 458 459 pte = arm_lpae_install_table(cptep, ptep, 0, data); 460 if (pte) 461 __arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie); 462 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { 463 __arm_lpae_sync_pte(ptep, 1, cfg); 464 } 465 466 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { 467 cptep = iopte_deref(pte, data); 468 } else if (pte) { 469 /* We require an unmap first */ 470 WARN_ON(!(cfg->quirks & IO_PGTABLE_QUIRK_NO_WARN)); 471 return -EEXIST; 472 } 473 474 /* Rinse, repeat */ 475 return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1, 476 cptep, gfp, mapped); 477 } 478 479 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, 480 int prot) 481 { 482 arm_lpae_iopte pte; 483 484 if (data->iop.fmt == ARM_64_LPAE_S1 || 485 data->iop.fmt == ARM_32_LPAE_S1) { 486 pte = ARM_LPAE_PTE_nG; 487 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) 488 pte |= ARM_LPAE_PTE_AP_RDONLY; 489 else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD) 490 pte |= ARM_LPAE_PTE_DBM; 491 if (!(prot & IOMMU_PRIV)) 492 pte |= ARM_LPAE_PTE_AP_UNPRIV; 493 } else { 494 pte = ARM_LPAE_PTE_HAP_FAULT; 495 if (prot & IOMMU_READ) 496 pte |= ARM_LPAE_PTE_HAP_READ; 497 if (prot & IOMMU_WRITE) 498 pte |= ARM_LPAE_PTE_HAP_WRITE; 499 } 500 501 /* 502 * Note that this logic is structured to accommodate Mali LPAE 503 * having stage-1-like attributes but stage-2-like permissions. 504 */ 505 if (data->iop.fmt == ARM_64_LPAE_S2 || 506 data->iop.fmt == ARM_32_LPAE_S2) { 507 if (prot & IOMMU_MMIO) { 508 pte |= ARM_LPAE_PTE_MEMATTR_DEV; 509 } else if (prot & IOMMU_CACHE) { 510 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB) 511 pte |= ARM_LPAE_PTE_MEMATTR_FWB_WB; 512 else 513 pte |= ARM_LPAE_PTE_MEMATTR_OIWB; 514 } else { 515 pte |= ARM_LPAE_PTE_MEMATTR_NC; 516 } 517 } else { 518 if (prot & IOMMU_MMIO) 519 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV 520 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 521 else if (prot & IOMMU_CACHE) 522 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE 523 << ARM_LPAE_PTE_ATTRINDX_SHIFT); 524 } 525 526 /* 527 * Also Mali has its own notions of shareability wherein its Inner 528 * domain covers the cores within the GPU, and its Outer domain is 529 * "outside the GPU" (i.e. either the Inner or System domain in CPU 530 * terms, depending on coherency). 531 */ 532 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE) 533 pte |= ARM_LPAE_PTE_SH_IS; 534 else 535 pte |= ARM_LPAE_PTE_SH_OS; 536 537 if (prot & IOMMU_NOEXEC) 538 pte |= ARM_LPAE_PTE_XN; 539 540 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) 541 pte |= ARM_LPAE_PTE_NS; 542 543 if (data->iop.fmt != ARM_MALI_LPAE) 544 pte |= ARM_LPAE_PTE_AF; 545 546 return pte; 547 } 548 549 static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova, 550 phys_addr_t paddr, size_t pgsize, size_t pgcount, 551 int iommu_prot, gfp_t gfp, size_t *mapped) 552 { 553 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 554 struct io_pgtable_cfg *cfg = &data->iop.cfg; 555 arm_lpae_iopte *ptep = data->pgd; 556 int ret, lvl = data->start_level; 557 arm_lpae_iopte prot; 558 long iaext = (s64)iova >> cfg->ias; 559 560 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) 561 return -EINVAL; 562 563 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 564 iaext = ~iaext; 565 if (WARN_ON(iaext || paddr >> cfg->oas)) 566 return -ERANGE; 567 568 if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) 569 return -EINVAL; 570 571 prot = arm_lpae_prot_to_pte(data, iommu_prot); 572 ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl, 573 ptep, gfp, mapped); 574 /* 575 * Synchronise all PTE updates for the new mapping before there's 576 * a chance for anything to kick off a table walk for the new iova. 577 */ 578 wmb(); 579 580 return ret; 581 } 582 583 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, 584 arm_lpae_iopte *ptep) 585 { 586 arm_lpae_iopte *start, *end; 587 unsigned long table_size; 588 589 if (lvl == data->start_level) 590 table_size = ARM_LPAE_PGD_SIZE(data); 591 else 592 table_size = ARM_LPAE_GRANULE(data); 593 594 start = ptep; 595 596 /* Only leaf entries at the last level */ 597 if (lvl == ARM_LPAE_MAX_LEVELS - 1) 598 end = ptep; 599 else 600 end = (void *)ptep + table_size; 601 602 while (ptep != end) { 603 arm_lpae_iopte pte = *ptep++; 604 605 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) 606 continue; 607 608 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 609 } 610 611 __arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie); 612 } 613 614 static void arm_lpae_free_pgtable(struct io_pgtable *iop) 615 { 616 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); 617 618 __arm_lpae_free_pgtable(data, data->start_level, data->pgd); 619 kfree(data); 620 } 621 622 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, 623 struct iommu_iotlb_gather *gather, 624 unsigned long iova, size_t size, size_t pgcount, 625 int lvl, arm_lpae_iopte *ptep) 626 { 627 arm_lpae_iopte pte; 628 struct io_pgtable *iop = &data->iop; 629 int i = 0, num_entries, max_entries, unmap_idx_start; 630 631 /* Something went horribly wrong and we ran out of page table */ 632 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 633 return 0; 634 635 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data); 636 ptep += unmap_idx_start; 637 pte = READ_ONCE(*ptep); 638 if (!pte) { 639 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 640 return -ENOENT; 641 } 642 643 /* If the size matches this level, we're in the right place */ 644 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) { 645 max_entries = arm_lpae_max_entries(unmap_idx_start, data); 646 num_entries = min_t(int, pgcount, max_entries); 647 648 /* Find and handle non-leaf entries */ 649 for (i = 0; i < num_entries; i++) { 650 pte = READ_ONCE(ptep[i]); 651 if (!pte) { 652 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); 653 break; 654 } 655 656 if (!iopte_leaf(pte, lvl, iop->fmt)) { 657 __arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1); 658 659 /* Also flush any partial walks */ 660 io_pgtable_tlb_flush_walk(iop, iova + i * size, size, 661 ARM_LPAE_GRANULE(data)); 662 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); 663 } 664 } 665 666 /* Clear the remaining entries */ 667 __arm_lpae_clear_pte(ptep, &iop->cfg, i); 668 669 if (gather && !iommu_iotlb_gather_queued(gather)) 670 for (int j = 0; j < i; j++) 671 io_pgtable_tlb_add_page(iop, gather, iova + j * size, size); 672 673 return i * size; 674 } else if (iopte_leaf(pte, lvl, iop->fmt)) { 675 WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed"); 676 return 0; 677 } 678 679 /* Keep on walkin' */ 680 ptep = iopte_deref(pte, data); 681 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep); 682 } 683 684 static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, 685 size_t pgsize, size_t pgcount, 686 struct iommu_iotlb_gather *gather) 687 { 688 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 689 struct io_pgtable_cfg *cfg = &data->iop.cfg; 690 arm_lpae_iopte *ptep = data->pgd; 691 long iaext = (s64)iova >> cfg->ias; 692 693 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) 694 return 0; 695 696 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 697 iaext = ~iaext; 698 if (WARN_ON(iaext)) 699 return 0; 700 701 return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount, 702 data->start_level, ptep); 703 } 704 705 struct io_pgtable_walk_data { 706 struct io_pgtable *iop; 707 void *data; 708 int (*visit)(struct io_pgtable_walk_data *walk_data, int lvl, 709 arm_lpae_iopte *ptep, size_t size); 710 unsigned long flags; 711 u64 addr; 712 const u64 end; 713 }; 714 715 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data, 716 struct io_pgtable_walk_data *walk_data, 717 arm_lpae_iopte *ptep, 718 int lvl); 719 720 struct iova_to_phys_data { 721 arm_lpae_iopte pte; 722 int lvl; 723 }; 724 725 static int visit_iova_to_phys(struct io_pgtable_walk_data *walk_data, int lvl, 726 arm_lpae_iopte *ptep, size_t size) 727 { 728 struct iova_to_phys_data *data = walk_data->data; 729 data->pte = *ptep; 730 data->lvl = lvl; 731 return 0; 732 } 733 734 static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, 735 unsigned long iova) 736 { 737 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 738 struct iova_to_phys_data d; 739 struct io_pgtable_walk_data walk_data = { 740 .data = &d, 741 .visit = visit_iova_to_phys, 742 .addr = iova, 743 .end = iova + 1, 744 }; 745 int ret; 746 747 ret = __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level); 748 if (ret) 749 return 0; 750 751 iova &= (ARM_LPAE_BLOCK_SIZE(d.lvl, data) - 1); 752 return iopte_to_paddr(d.pte, data) | iova; 753 } 754 755 static int visit_pgtable_walk(struct io_pgtable_walk_data *walk_data, int lvl, 756 arm_lpae_iopte *ptep, size_t size) 757 { 758 struct arm_lpae_io_pgtable_walk_data *data = walk_data->data; 759 data->ptes[lvl] = *ptep; 760 return 0; 761 } 762 763 static int arm_lpae_pgtable_walk(struct io_pgtable_ops *ops, unsigned long iova, 764 void *wd) 765 { 766 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 767 struct io_pgtable_walk_data walk_data = { 768 .data = wd, 769 .visit = visit_pgtable_walk, 770 .addr = iova, 771 .end = iova + 1, 772 }; 773 774 return __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level); 775 } 776 777 static int io_pgtable_visit(struct arm_lpae_io_pgtable *data, 778 struct io_pgtable_walk_data *walk_data, 779 arm_lpae_iopte *ptep, int lvl) 780 { 781 struct io_pgtable *iop = &data->iop; 782 arm_lpae_iopte pte = READ_ONCE(*ptep); 783 784 size_t size = ARM_LPAE_BLOCK_SIZE(lvl, data); 785 int ret = walk_data->visit(walk_data, lvl, ptep, size); 786 if (ret) 787 return ret; 788 789 if (iopte_leaf(pte, lvl, iop->fmt)) { 790 walk_data->addr += size; 791 return 0; 792 } 793 794 if (!iopte_table(pte, lvl)) { 795 return -EINVAL; 796 } 797 798 ptep = iopte_deref(pte, data); 799 return __arm_lpae_iopte_walk(data, walk_data, ptep, lvl + 1); 800 } 801 802 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data, 803 struct io_pgtable_walk_data *walk_data, 804 arm_lpae_iopte *ptep, 805 int lvl) 806 { 807 u32 idx; 808 int max_entries, ret; 809 810 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) 811 return -EINVAL; 812 813 if (lvl == data->start_level) 814 max_entries = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte); 815 else 816 max_entries = ARM_LPAE_PTES_PER_TABLE(data); 817 818 for (idx = ARM_LPAE_LVL_IDX(walk_data->addr, lvl, data); 819 (idx < max_entries) && (walk_data->addr < walk_data->end); ++idx) { 820 ret = io_pgtable_visit(data, walk_data, ptep + idx, lvl); 821 if (ret) 822 return ret; 823 } 824 825 return 0; 826 } 827 828 static int visit_dirty(struct io_pgtable_walk_data *walk_data, int lvl, 829 arm_lpae_iopte *ptep, size_t size) 830 { 831 struct iommu_dirty_bitmap *dirty = walk_data->data; 832 833 if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt)) 834 return 0; 835 836 if (iopte_writeable_dirty(*ptep)) { 837 iommu_dirty_bitmap_record(dirty, walk_data->addr, size); 838 if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR)) 839 iopte_set_writeable_clean(ptep); 840 } 841 842 return 0; 843 } 844 845 static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops, 846 unsigned long iova, size_t size, 847 unsigned long flags, 848 struct iommu_dirty_bitmap *dirty) 849 { 850 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); 851 struct io_pgtable_cfg *cfg = &data->iop.cfg; 852 struct io_pgtable_walk_data walk_data = { 853 .iop = &data->iop, 854 .data = dirty, 855 .visit = visit_dirty, 856 .flags = flags, 857 .addr = iova, 858 .end = iova + size, 859 }; 860 arm_lpae_iopte *ptep = data->pgd; 861 int lvl = data->start_level; 862 863 if (WARN_ON(!size)) 864 return -EINVAL; 865 if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1))) 866 return -EINVAL; 867 if (data->iop.fmt != ARM_64_LPAE_S1) 868 return -EINVAL; 869 870 return __arm_lpae_iopte_walk(data, &walk_data, ptep, lvl); 871 } 872 873 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) 874 { 875 unsigned long granule, page_sizes; 876 unsigned int max_addr_bits = 48; 877 878 /* 879 * We need to restrict the supported page sizes to match the 880 * translation regime for a particular granule. Aim to match 881 * the CPU page size if possible, otherwise prefer smaller sizes. 882 * While we're at it, restrict the block sizes to match the 883 * chosen granule. 884 */ 885 if (cfg->pgsize_bitmap & PAGE_SIZE) 886 granule = PAGE_SIZE; 887 else if (cfg->pgsize_bitmap & ~PAGE_MASK) 888 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); 889 else if (cfg->pgsize_bitmap & PAGE_MASK) 890 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); 891 else 892 granule = 0; 893 894 switch (granule) { 895 case SZ_4K: 896 page_sizes = (SZ_4K | SZ_2M | SZ_1G); 897 break; 898 case SZ_16K: 899 page_sizes = (SZ_16K | SZ_32M); 900 break; 901 case SZ_64K: 902 max_addr_bits = 52; 903 page_sizes = (SZ_64K | SZ_512M); 904 if (cfg->oas > 48) 905 page_sizes |= 1ULL << 42; /* 4TB */ 906 break; 907 default: 908 page_sizes = 0; 909 } 910 911 cfg->pgsize_bitmap &= page_sizes; 912 cfg->ias = min(cfg->ias, max_addr_bits); 913 cfg->oas = min(cfg->oas, max_addr_bits); 914 } 915 916 static struct arm_lpae_io_pgtable * 917 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) 918 { 919 struct arm_lpae_io_pgtable *data; 920 int levels, va_bits, pg_shift; 921 922 arm_lpae_restrict_pgsizes(cfg); 923 924 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) 925 return NULL; 926 927 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) 928 return NULL; 929 930 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) 931 return NULL; 932 933 data = kmalloc(sizeof(*data), GFP_KERNEL); 934 if (!data) 935 return NULL; 936 937 pg_shift = __ffs(cfg->pgsize_bitmap); 938 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte)); 939 940 va_bits = cfg->ias - pg_shift; 941 levels = DIV_ROUND_UP(va_bits, data->bits_per_level); 942 data->start_level = ARM_LPAE_MAX_LEVELS - levels; 943 944 /* Calculate the actual size of our pgd (without concatenation) */ 945 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); 946 947 data->iop.ops = (struct io_pgtable_ops) { 948 .map_pages = arm_lpae_map_pages, 949 .unmap_pages = arm_lpae_unmap_pages, 950 .iova_to_phys = arm_lpae_iova_to_phys, 951 .read_and_clear_dirty = arm_lpae_read_and_clear_dirty, 952 .pgtable_walk = arm_lpae_pgtable_walk, 953 }; 954 955 return data; 956 } 957 958 static struct io_pgtable * 959 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 960 { 961 u64 reg; 962 struct arm_lpae_io_pgtable *data; 963 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; 964 bool tg1; 965 966 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | 967 IO_PGTABLE_QUIRK_ARM_TTBR1 | 968 IO_PGTABLE_QUIRK_ARM_OUTER_WBWA | 969 IO_PGTABLE_QUIRK_ARM_HD | 970 IO_PGTABLE_QUIRK_NO_WARN)) 971 return NULL; 972 973 data = arm_lpae_alloc_pgtable(cfg); 974 if (!data) 975 return NULL; 976 977 /* TCR */ 978 if (cfg->coherent_walk) { 979 tcr->sh = ARM_LPAE_TCR_SH_IS; 980 tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; 981 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; 982 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA) 983 goto out_free_data; 984 } else { 985 tcr->sh = ARM_LPAE_TCR_SH_OS; 986 tcr->irgn = ARM_LPAE_TCR_RGN_NC; 987 if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) 988 tcr->orgn = ARM_LPAE_TCR_RGN_NC; 989 else 990 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; 991 } 992 993 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; 994 switch (ARM_LPAE_GRANULE(data)) { 995 case SZ_4K: 996 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K; 997 break; 998 case SZ_16K: 999 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K; 1000 break; 1001 case SZ_64K: 1002 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K; 1003 break; 1004 } 1005 1006 switch (cfg->oas) { 1007 case 32: 1008 tcr->ips = ARM_LPAE_TCR_PS_32_BIT; 1009 break; 1010 case 36: 1011 tcr->ips = ARM_LPAE_TCR_PS_36_BIT; 1012 break; 1013 case 40: 1014 tcr->ips = ARM_LPAE_TCR_PS_40_BIT; 1015 break; 1016 case 42: 1017 tcr->ips = ARM_LPAE_TCR_PS_42_BIT; 1018 break; 1019 case 44: 1020 tcr->ips = ARM_LPAE_TCR_PS_44_BIT; 1021 break; 1022 case 48: 1023 tcr->ips = ARM_LPAE_TCR_PS_48_BIT; 1024 break; 1025 case 52: 1026 tcr->ips = ARM_LPAE_TCR_PS_52_BIT; 1027 break; 1028 default: 1029 goto out_free_data; 1030 } 1031 1032 tcr->tsz = 64ULL - cfg->ias; 1033 1034 /* MAIRs */ 1035 reg = (ARM_LPAE_MAIR_ATTR_NC 1036 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | 1037 (ARM_LPAE_MAIR_ATTR_WBRWA 1038 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | 1039 (ARM_LPAE_MAIR_ATTR_DEVICE 1040 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) | 1041 (ARM_LPAE_MAIR_ATTR_INC_OWBRWA 1042 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE)); 1043 1044 cfg->arm_lpae_s1_cfg.mair = reg; 1045 1046 /* Looking good; allocate a pgd */ 1047 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), 1048 GFP_KERNEL, cfg, cookie); 1049 if (!data->pgd) 1050 goto out_free_data; 1051 1052 /* Ensure the empty pgd is visible before any actual TTBR write */ 1053 wmb(); 1054 1055 /* TTBR */ 1056 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); 1057 return &data->iop; 1058 1059 out_free_data: 1060 kfree(data); 1061 return NULL; 1062 } 1063 1064 static struct io_pgtable * 1065 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 1066 { 1067 u64 sl; 1068 struct arm_lpae_io_pgtable *data; 1069 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; 1070 1071 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB | 1072 IO_PGTABLE_QUIRK_NO_WARN)) 1073 return NULL; 1074 1075 data = arm_lpae_alloc_pgtable(cfg); 1076 if (!data) 1077 return NULL; 1078 1079 if (arm_lpae_concat_mandatory(cfg, data)) { 1080 if (WARN_ON((ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte)) > 1081 ARM_LPAE_S2_MAX_CONCAT_PAGES)) 1082 return NULL; 1083 data->pgd_bits += data->bits_per_level; 1084 data->start_level++; 1085 } 1086 1087 /* VTCR */ 1088 if (cfg->coherent_walk) { 1089 vtcr->sh = ARM_LPAE_TCR_SH_IS; 1090 vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA; 1091 vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA; 1092 } else { 1093 vtcr->sh = ARM_LPAE_TCR_SH_OS; 1094 vtcr->irgn = ARM_LPAE_TCR_RGN_NC; 1095 vtcr->orgn = ARM_LPAE_TCR_RGN_NC; 1096 } 1097 1098 sl = data->start_level; 1099 1100 switch (ARM_LPAE_GRANULE(data)) { 1101 case SZ_4K: 1102 vtcr->tg = ARM_LPAE_TCR_TG0_4K; 1103 sl++; /* SL0 format is different for 4K granule size */ 1104 break; 1105 case SZ_16K: 1106 vtcr->tg = ARM_LPAE_TCR_TG0_16K; 1107 break; 1108 case SZ_64K: 1109 vtcr->tg = ARM_LPAE_TCR_TG0_64K; 1110 break; 1111 } 1112 1113 switch (cfg->oas) { 1114 case 32: 1115 vtcr->ps = ARM_LPAE_TCR_PS_32_BIT; 1116 break; 1117 case 36: 1118 vtcr->ps = ARM_LPAE_TCR_PS_36_BIT; 1119 break; 1120 case 40: 1121 vtcr->ps = ARM_LPAE_TCR_PS_40_BIT; 1122 break; 1123 case 42: 1124 vtcr->ps = ARM_LPAE_TCR_PS_42_BIT; 1125 break; 1126 case 44: 1127 vtcr->ps = ARM_LPAE_TCR_PS_44_BIT; 1128 break; 1129 case 48: 1130 vtcr->ps = ARM_LPAE_TCR_PS_48_BIT; 1131 break; 1132 case 52: 1133 vtcr->ps = ARM_LPAE_TCR_PS_52_BIT; 1134 break; 1135 default: 1136 goto out_free_data; 1137 } 1138 1139 vtcr->tsz = 64ULL - cfg->ias; 1140 vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK; 1141 1142 /* Allocate pgd pages */ 1143 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), 1144 GFP_KERNEL, cfg, cookie); 1145 if (!data->pgd) 1146 goto out_free_data; 1147 1148 /* Ensure the empty pgd is visible before any actual TTBR write */ 1149 wmb(); 1150 1151 /* VTTBR */ 1152 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); 1153 return &data->iop; 1154 1155 out_free_data: 1156 kfree(data); 1157 return NULL; 1158 } 1159 1160 static struct io_pgtable * 1161 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 1162 { 1163 if (cfg->ias > 32 || cfg->oas > 40) 1164 return NULL; 1165 1166 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 1167 return arm_64_lpae_alloc_pgtable_s1(cfg, cookie); 1168 } 1169 1170 static struct io_pgtable * 1171 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 1172 { 1173 if (cfg->ias > 40 || cfg->oas > 40) 1174 return NULL; 1175 1176 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 1177 return arm_64_lpae_alloc_pgtable_s2(cfg, cookie); 1178 } 1179 1180 static struct io_pgtable * 1181 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) 1182 { 1183 struct arm_lpae_io_pgtable *data; 1184 1185 /* No quirks for Mali (hopefully) */ 1186 if (cfg->quirks) 1187 return NULL; 1188 1189 if (cfg->ias > 48 || cfg->oas > 40) 1190 return NULL; 1191 1192 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); 1193 1194 data = arm_lpae_alloc_pgtable(cfg); 1195 if (!data) 1196 return NULL; 1197 1198 /* Mali seems to need a full 4-level table regardless of IAS */ 1199 if (data->start_level > 0) { 1200 data->start_level = 0; 1201 data->pgd_bits = 0; 1202 } 1203 /* 1204 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the 1205 * best we can do is mimic the out-of-tree driver and hope that the 1206 * "implementation-defined caching policy" is good enough. Similarly, 1207 * we'll use it for the sake of a valid attribute for our 'device' 1208 * index, although callers should never request that in practice. 1209 */ 1210 cfg->arm_mali_lpae_cfg.memattr = 1211 (ARM_MALI_LPAE_MEMATTR_IMP_DEF 1212 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | 1213 (ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 1214 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | 1215 (ARM_MALI_LPAE_MEMATTR_IMP_DEF 1216 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); 1217 1218 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, 1219 cfg, cookie); 1220 if (!data->pgd) 1221 goto out_free_data; 1222 1223 /* Ensure the empty pgd is visible before TRANSTAB can be written */ 1224 wmb(); 1225 1226 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | 1227 ARM_MALI_LPAE_TTBR_READ_INNER | 1228 ARM_MALI_LPAE_TTBR_ADRMODE_TABLE; 1229 if (cfg->coherent_walk) 1230 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER; 1231 1232 return &data->iop; 1233 1234 out_free_data: 1235 kfree(data); 1236 return NULL; 1237 } 1238 1239 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { 1240 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1241 .alloc = arm_64_lpae_alloc_pgtable_s1, 1242 .free = arm_lpae_free_pgtable, 1243 }; 1244 1245 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = { 1246 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1247 .alloc = arm_64_lpae_alloc_pgtable_s2, 1248 .free = arm_lpae_free_pgtable, 1249 }; 1250 1251 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = { 1252 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1253 .alloc = arm_32_lpae_alloc_pgtable_s1, 1254 .free = arm_lpae_free_pgtable, 1255 }; 1256 1257 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { 1258 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1259 .alloc = arm_32_lpae_alloc_pgtable_s2, 1260 .free = arm_lpae_free_pgtable, 1261 }; 1262 1263 struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = { 1264 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, 1265 .alloc = arm_mali_lpae_alloc_pgtable, 1266 .free = arm_lpae_free_pgtable, 1267 }; 1268