1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * CPU-agnostic ARM page table allocator.
4 *
5 * Copyright (C) 2014 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 */
9
10 #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt
11
12 #include <linux/atomic.h>
13 #include <linux/bitops.h>
14 #include <linux/io-pgtable.h>
15 #include <linux/sizes.h>
16 #include <linux/slab.h>
17 #include <linux/types.h>
18 #include <linux/dma-mapping.h>
19
20 #include <asm/barrier.h>
21
22 #include "io-pgtable-arm.h"
23 #include "iommu-pages.h"
24
25 #define ARM_LPAE_MAX_ADDR_BITS 52
26 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16
27 #define ARM_LPAE_MAX_LEVELS 4
28
29 /* Struct accessors */
30 #define io_pgtable_to_data(x) \
31 container_of((x), struct arm_lpae_io_pgtable, iop)
32
33 #define io_pgtable_ops_to_data(x) \
34 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
35
36 /*
37 * Calculate the right shift amount to get to the portion describing level l
38 * in a virtual address mapped by the pagetable in d.
39 */
40 #define ARM_LPAE_LVL_SHIFT(l,d) \
41 (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
42 ilog2(sizeof(arm_lpae_iopte)))
43
44 #define ARM_LPAE_GRANULE(d) \
45 (sizeof(arm_lpae_iopte) << (d)->bits_per_level)
46 #define ARM_LPAE_PGD_SIZE(d) \
47 (sizeof(arm_lpae_iopte) << (d)->pgd_bits)
48
49 #define ARM_LPAE_PTES_PER_TABLE(d) \
50 (ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))
51
52 /*
53 * Calculate the index at level l used to map virtual address a using the
54 * pagetable in d.
55 */
56 #define ARM_LPAE_PGD_IDX(l,d) \
57 ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
58
59 #define ARM_LPAE_LVL_IDX(a,l,d) \
60 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
61 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
62
63 /* Calculate the block/page mapping size at level l for pagetable in d. */
64 #define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d))
65
66 /* Page table bits */
67 #define ARM_LPAE_PTE_TYPE_SHIFT 0
68 #define ARM_LPAE_PTE_TYPE_MASK 0x3
69
70 #define ARM_LPAE_PTE_TYPE_BLOCK 1
71 #define ARM_LPAE_PTE_TYPE_TABLE 3
72 #define ARM_LPAE_PTE_TYPE_PAGE 3
73
74 #define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12)
75
76 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
77 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
78 #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51)
79 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
80 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
81 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
82 #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
83 #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
84 #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)
85
86 /* Software bit for solving coherency races */
87 #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)
88
89 /* Stage-1 PTE */
90 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
91 #define ARM_LPAE_PTE_AP_RDONLY_BIT 7
92 #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)1) << \
93 ARM_LPAE_PTE_AP_RDONLY_BIT)
94 #define ARM_LPAE_PTE_AP_WR_CLEAN_MASK (ARM_LPAE_PTE_AP_RDONLY | \
95 ARM_LPAE_PTE_DBM)
96 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
97 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)
98
99 /* Stage-2 PTE */
100 #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
101 #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
102 #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
103 /*
104 * For !FWB these code to:
105 * 1111 = Normal outer write back cachable / Inner Write Back Cachable
106 * Permit S1 to override
107 * 0101 = Normal Non-cachable / Inner Non-cachable
108 * 0001 = Device / Device-nGnRE
109 * For S2FWB these code:
110 * 0110 Force Normal Write Back
111 * 0101 Normal* is forced Normal-NC, Device unchanged
112 * 0001 Force Device-nGnRE
113 */
114 #define ARM_LPAE_PTE_MEMATTR_FWB_WB (((arm_lpae_iopte)0x6) << 2)
115 #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
116 #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
117 #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
118
119 /* Register bits */
120 #define ARM_LPAE_VTCR_SL0_MASK 0x3
121
122 #define ARM_LPAE_TCR_T0SZ_SHIFT 0
123
124 #define ARM_LPAE_VTCR_PS_SHIFT 16
125 #define ARM_LPAE_VTCR_PS_MASK 0x7
126
127 #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
128 #define ARM_LPAE_MAIR_ATTR_MASK 0xff
129 #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
130 #define ARM_LPAE_MAIR_ATTR_NC 0x44
131 #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4
132 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff
133 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0
134 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
135 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
136 #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3
137
138 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
139 #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
140 #define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)
141
142 #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL
143 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
144
145 /* IOPTE accessors */
146 #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
147
148 #define iopte_type(pte) \
149 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
150
151 #define iopte_writeable_dirty(pte) \
152 (((pte) & ARM_LPAE_PTE_AP_WR_CLEAN_MASK) == ARM_LPAE_PTE_DBM)
153
154 #define iopte_set_writeable_clean(ptep) \
155 set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep))
156
157 struct arm_lpae_io_pgtable {
158 struct io_pgtable iop;
159
160 int pgd_bits;
161 int start_level;
162 int bits_per_level;
163
164 void *pgd;
165 };
166
167 typedef u64 arm_lpae_iopte;
168
iopte_leaf(arm_lpae_iopte pte,int lvl,enum io_pgtable_fmt fmt)169 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
170 enum io_pgtable_fmt fmt)
171 {
172 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
173 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;
174
175 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
176 }
177
iopte_table(arm_lpae_iopte pte,int lvl)178 static inline bool iopte_table(arm_lpae_iopte pte, int lvl)
179 {
180 if (lvl == (ARM_LPAE_MAX_LEVELS - 1))
181 return false;
182 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE;
183 }
184
paddr_to_iopte(phys_addr_t paddr,struct arm_lpae_io_pgtable * data)185 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
186 struct arm_lpae_io_pgtable *data)
187 {
188 arm_lpae_iopte pte = paddr;
189
190 /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */
191 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK;
192 }
193
iopte_to_paddr(arm_lpae_iopte pte,struct arm_lpae_io_pgtable * data)194 static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
195 struct arm_lpae_io_pgtable *data)
196 {
197 u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK;
198
199 if (ARM_LPAE_GRANULE(data) < SZ_64K)
200 return paddr;
201
202 /* Rotate the packed high-order bits back to the top */
203 return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
204 }
205
206 /*
207 * Convert an index returned by ARM_LPAE_PGD_IDX(), which can point into
208 * a concatenated PGD, into the maximum number of entries that can be
209 * mapped in the same table page.
210 */
arm_lpae_max_entries(int i,struct arm_lpae_io_pgtable * data)211 static inline int arm_lpae_max_entries(int i, struct arm_lpae_io_pgtable *data)
212 {
213 int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
214
215 return ptes_per_table - (i & (ptes_per_table - 1));
216 }
217
218 /*
219 * Check if concatenated PGDs are mandatory according to Arm DDI0487 (K.a)
220 * 1) R_DXBSH: For 16KB, and 48-bit input size, use level 1 instead of 0.
221 * 2) R_SRKBC: After de-ciphering the table for PA size and valid initial lookup
222 * a) 40 bits PA size with 4K: use level 1 instead of level 0 (2 tables for ias = oas)
223 * b) 40 bits PA size with 16K: use level 2 instead of level 1 (16 tables for ias = oas)
224 * c) 42 bits PA size with 4K: use level 1 instead of level 0 (8 tables for ias = oas)
225 * d) 48 bits PA size with 16K: use level 1 instead of level 0 (2 tables for ias = oas)
226 */
arm_lpae_concat_mandatory(struct io_pgtable_cfg * cfg,struct arm_lpae_io_pgtable * data)227 static inline bool arm_lpae_concat_mandatory(struct io_pgtable_cfg *cfg,
228 struct arm_lpae_io_pgtable *data)
229 {
230 unsigned int ias = cfg->ias;
231 unsigned int oas = cfg->oas;
232
233 /* Covers 1 and 2.d */
234 if ((ARM_LPAE_GRANULE(data) == SZ_16K) && (data->start_level == 0))
235 return (oas == 48) || (ias == 48);
236
237 /* Covers 2.a and 2.c */
238 if ((ARM_LPAE_GRANULE(data) == SZ_4K) && (data->start_level == 0))
239 return (oas == 40) || (oas == 42);
240
241 /* Case 2.b */
242 return (ARM_LPAE_GRANULE(data) == SZ_16K) &&
243 (data->start_level == 1) && (oas == 40);
244 }
245
__arm_lpae_dma_addr(void * pages)246 static dma_addr_t __arm_lpae_dma_addr(void *pages)
247 {
248 return (dma_addr_t)virt_to_phys(pages);
249 }
250
__arm_lpae_alloc_pages(size_t size,gfp_t gfp,struct io_pgtable_cfg * cfg,void * cookie)251 static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
252 struct io_pgtable_cfg *cfg,
253 void *cookie)
254 {
255 struct device *dev = cfg->iommu_dev;
256 size_t alloc_size;
257 dma_addr_t dma;
258 void *pages;
259
260 /*
261 * For very small starting-level translation tables the HW requires a
262 * minimum alignment of at least 64 to cover all cases.
263 */
264 alloc_size = max(size, 64);
265 if (cfg->alloc)
266 pages = cfg->alloc(cookie, alloc_size, gfp);
267 else
268 pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp,
269 alloc_size);
270
271 if (!pages)
272 return NULL;
273
274 if (!cfg->coherent_walk) {
275 dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
276 if (dma_mapping_error(dev, dma))
277 goto out_free;
278 /*
279 * We depend on the IOMMU being able to work with any physical
280 * address directly, so if the DMA layer suggests otherwise by
281 * translating or truncating them, that bodes very badly...
282 */
283 if (dma != virt_to_phys(pages))
284 goto out_unmap;
285 }
286
287 return pages;
288
289 out_unmap:
290 dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
291 dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
292
293 out_free:
294 if (cfg->free)
295 cfg->free(cookie, pages, size);
296 else
297 iommu_free_pages(pages);
298
299 return NULL;
300 }
301
__arm_lpae_free_pages(void * pages,size_t size,struct io_pgtable_cfg * cfg,void * cookie)302 static void __arm_lpae_free_pages(void *pages, size_t size,
303 struct io_pgtable_cfg *cfg,
304 void *cookie)
305 {
306 if (!cfg->coherent_walk)
307 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
308 size, DMA_TO_DEVICE);
309
310 if (cfg->free)
311 cfg->free(cookie, pages, size);
312 else
313 iommu_free_pages(pages);
314 }
315
__arm_lpae_sync_pte(arm_lpae_iopte * ptep,int num_entries,struct io_pgtable_cfg * cfg)316 static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
317 struct io_pgtable_cfg *cfg)
318 {
319 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),
320 sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
321 }
322
__arm_lpae_clear_pte(arm_lpae_iopte * ptep,struct io_pgtable_cfg * cfg,int num_entries)323 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries)
324 {
325 for (int i = 0; i < num_entries; i++)
326 ptep[i] = 0;
327
328 if (!cfg->coherent_walk && num_entries)
329 __arm_lpae_sync_pte(ptep, num_entries, cfg);
330 }
331
332 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
333 struct iommu_iotlb_gather *gather,
334 unsigned long iova, size_t size, size_t pgcount,
335 int lvl, arm_lpae_iopte *ptep);
336
__arm_lpae_init_pte(struct arm_lpae_io_pgtable * data,phys_addr_t paddr,arm_lpae_iopte prot,int lvl,int num_entries,arm_lpae_iopte * ptep)337 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
338 phys_addr_t paddr, arm_lpae_iopte prot,
339 int lvl, int num_entries, arm_lpae_iopte *ptep)
340 {
341 arm_lpae_iopte pte = prot;
342 struct io_pgtable_cfg *cfg = &data->iop.cfg;
343 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
344 int i;
345
346 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
347 pte |= ARM_LPAE_PTE_TYPE_PAGE;
348 else
349 pte |= ARM_LPAE_PTE_TYPE_BLOCK;
350
351 for (i = 0; i < num_entries; i++)
352 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data);
353
354 if (!cfg->coherent_walk)
355 __arm_lpae_sync_pte(ptep, num_entries, cfg);
356 }
357
arm_lpae_init_pte(struct arm_lpae_io_pgtable * data,unsigned long iova,phys_addr_t paddr,arm_lpae_iopte prot,int lvl,int num_entries,arm_lpae_iopte * ptep)358 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
359 unsigned long iova, phys_addr_t paddr,
360 arm_lpae_iopte prot, int lvl, int num_entries,
361 arm_lpae_iopte *ptep)
362 {
363 int i;
364
365 for (i = 0; i < num_entries; i++)
366 if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {
367 /* We require an unmap first */
368 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));
369 return -EEXIST;
370 } else if (iopte_type(ptep[i]) == ARM_LPAE_PTE_TYPE_TABLE) {
371 /*
372 * We need to unmap and free the old table before
373 * overwriting it with a block entry.
374 */
375 arm_lpae_iopte *tblp;
376 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
377
378 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
379 if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1,
380 lvl, tblp) != sz) {
381 WARN_ON(1);
382 return -EINVAL;
383 }
384 }
385
386 __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep);
387 return 0;
388 }
389
arm_lpae_install_table(arm_lpae_iopte * table,arm_lpae_iopte * ptep,arm_lpae_iopte curr,struct arm_lpae_io_pgtable * data)390 static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
391 arm_lpae_iopte *ptep,
392 arm_lpae_iopte curr,
393 struct arm_lpae_io_pgtable *data)
394 {
395 arm_lpae_iopte old, new;
396 struct io_pgtable_cfg *cfg = &data->iop.cfg;
397
398 new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
399 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
400 new |= ARM_LPAE_PTE_NSTABLE;
401
402 /*
403 * Ensure the table itself is visible before its PTE can be.
404 * Whilst we could get away with cmpxchg64_release below, this
405 * doesn't have any ordering semantics when !CONFIG_SMP.
406 */
407 dma_wmb();
408
409 old = cmpxchg64_relaxed(ptep, curr, new);
410
411 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC))
412 return old;
413
414 /* Even if it's not ours, there's no point waiting; just kick it */
415 __arm_lpae_sync_pte(ptep, 1, cfg);
416 if (old == curr)
417 WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC);
418
419 return old;
420 }
421
__arm_lpae_map(struct arm_lpae_io_pgtable * data,unsigned long iova,phys_addr_t paddr,size_t size,size_t pgcount,arm_lpae_iopte prot,int lvl,arm_lpae_iopte * ptep,gfp_t gfp,size_t * mapped)422 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
423 phys_addr_t paddr, size_t size, size_t pgcount,
424 arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,
425 gfp_t gfp, size_t *mapped)
426 {
427 arm_lpae_iopte *cptep, pte;
428 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
429 size_t tblsz = ARM_LPAE_GRANULE(data);
430 struct io_pgtable_cfg *cfg = &data->iop.cfg;
431 int ret = 0, num_entries, max_entries, map_idx_start;
432
433 /* Find our entry at the current level */
434 map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
435 ptep += map_idx_start;
436
437 /* If we can install a leaf entry at this level, then do so */
438 if (size == block_size) {
439 max_entries = arm_lpae_max_entries(map_idx_start, data);
440 num_entries = min_t(int, pgcount, max_entries);
441 ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
442 if (!ret)
443 *mapped += num_entries * size;
444
445 return ret;
446 }
447
448 /* We can't allocate tables at the final level */
449 if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))
450 return -EINVAL;
451
452 /* Grab a pointer to the next level */
453 pte = READ_ONCE(*ptep);
454 if (!pte) {
455 cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie);
456 if (!cptep)
457 return -ENOMEM;
458
459 pte = arm_lpae_install_table(cptep, ptep, 0, data);
460 if (pte)
461 __arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie);
462 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {
463 __arm_lpae_sync_pte(ptep, 1, cfg);
464 }
465
466 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
467 cptep = iopte_deref(pte, data);
468 } else if (pte) {
469 /* We require an unmap first */
470 WARN_ON(!(cfg->quirks & IO_PGTABLE_QUIRK_NO_WARN));
471 return -EEXIST;
472 }
473
474 /* Rinse, repeat */
475 return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1,
476 cptep, gfp, mapped);
477 }
478
arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable * data,int prot)479 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
480 int prot)
481 {
482 arm_lpae_iopte pte;
483
484 if (data->iop.fmt == ARM_64_LPAE_S1 ||
485 data->iop.fmt == ARM_32_LPAE_S1) {
486 pte = ARM_LPAE_PTE_nG;
487 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
488 pte |= ARM_LPAE_PTE_AP_RDONLY;
489 else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD)
490 pte |= ARM_LPAE_PTE_DBM;
491 if (!(prot & IOMMU_PRIV))
492 pte |= ARM_LPAE_PTE_AP_UNPRIV;
493 } else {
494 pte = ARM_LPAE_PTE_HAP_FAULT;
495 if (prot & IOMMU_READ)
496 pte |= ARM_LPAE_PTE_HAP_READ;
497 if (prot & IOMMU_WRITE)
498 pte |= ARM_LPAE_PTE_HAP_WRITE;
499 }
500
501 /*
502 * Note that this logic is structured to accommodate Mali LPAE
503 * having stage-1-like attributes but stage-2-like permissions.
504 */
505 if (data->iop.fmt == ARM_64_LPAE_S2 ||
506 data->iop.fmt == ARM_32_LPAE_S2) {
507 if (prot & IOMMU_MMIO) {
508 pte |= ARM_LPAE_PTE_MEMATTR_DEV;
509 } else if (prot & IOMMU_CACHE) {
510 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
511 pte |= ARM_LPAE_PTE_MEMATTR_FWB_WB;
512 else
513 pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
514 } else {
515 pte |= ARM_LPAE_PTE_MEMATTR_NC;
516 }
517 } else {
518 if (prot & IOMMU_MMIO)
519 pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
520 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
521 else if (prot & IOMMU_CACHE)
522 pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
523 << ARM_LPAE_PTE_ATTRINDX_SHIFT);
524 }
525
526 /*
527 * Also Mali has its own notions of shareability wherein its Inner
528 * domain covers the cores within the GPU, and its Outer domain is
529 * "outside the GPU" (i.e. either the Inner or System domain in CPU
530 * terms, depending on coherency).
531 */
532 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)
533 pte |= ARM_LPAE_PTE_SH_IS;
534 else
535 pte |= ARM_LPAE_PTE_SH_OS;
536
537 if (prot & IOMMU_NOEXEC)
538 pte |= ARM_LPAE_PTE_XN;
539
540 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
541 pte |= ARM_LPAE_PTE_NS;
542
543 if (data->iop.fmt != ARM_MALI_LPAE)
544 pte |= ARM_LPAE_PTE_AF;
545
546 return pte;
547 }
548
arm_lpae_map_pages(struct io_pgtable_ops * ops,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int iommu_prot,gfp_t gfp,size_t * mapped)549 static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
550 phys_addr_t paddr, size_t pgsize, size_t pgcount,
551 int iommu_prot, gfp_t gfp, size_t *mapped)
552 {
553 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
554 struct io_pgtable_cfg *cfg = &data->iop.cfg;
555 arm_lpae_iopte *ptep = data->pgd;
556 int ret, lvl = data->start_level;
557 arm_lpae_iopte prot;
558 long iaext = (s64)iova >> cfg->ias;
559
560 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize))
561 return -EINVAL;
562
563 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
564 iaext = ~iaext;
565 if (WARN_ON(iaext || paddr >> cfg->oas))
566 return -ERANGE;
567
568 if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
569 return -EINVAL;
570
571 prot = arm_lpae_prot_to_pte(data, iommu_prot);
572 ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl,
573 ptep, gfp, mapped);
574 /*
575 * Synchronise all PTE updates for the new mapping before there's
576 * a chance for anything to kick off a table walk for the new iova.
577 */
578 wmb();
579
580 return ret;
581 }
582
__arm_lpae_free_pgtable(struct arm_lpae_io_pgtable * data,int lvl,arm_lpae_iopte * ptep)583 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
584 arm_lpae_iopte *ptep)
585 {
586 arm_lpae_iopte *start, *end;
587 unsigned long table_size;
588
589 if (lvl == data->start_level)
590 table_size = ARM_LPAE_PGD_SIZE(data);
591 else
592 table_size = ARM_LPAE_GRANULE(data);
593
594 start = ptep;
595
596 /* Only leaf entries at the last level */
597 if (lvl == ARM_LPAE_MAX_LEVELS - 1)
598 end = ptep;
599 else
600 end = (void *)ptep + table_size;
601
602 while (ptep != end) {
603 arm_lpae_iopte pte = *ptep++;
604
605 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
606 continue;
607
608 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
609 }
610
611 __arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie);
612 }
613
arm_lpae_free_pgtable(struct io_pgtable * iop)614 static void arm_lpae_free_pgtable(struct io_pgtable *iop)
615 {
616 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
617
618 __arm_lpae_free_pgtable(data, data->start_level, data->pgd);
619 kfree(data);
620 }
621
__arm_lpae_unmap(struct arm_lpae_io_pgtable * data,struct iommu_iotlb_gather * gather,unsigned long iova,size_t size,size_t pgcount,int lvl,arm_lpae_iopte * ptep)622 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
623 struct iommu_iotlb_gather *gather,
624 unsigned long iova, size_t size, size_t pgcount,
625 int lvl, arm_lpae_iopte *ptep)
626 {
627 arm_lpae_iopte pte;
628 struct io_pgtable *iop = &data->iop;
629 int i = 0, num_entries, max_entries, unmap_idx_start;
630
631 /* Something went horribly wrong and we ran out of page table */
632 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
633 return 0;
634
635 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
636 ptep += unmap_idx_start;
637 pte = READ_ONCE(*ptep);
638 if (!pte) {
639 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));
640 return -ENOENT;
641 }
642
643 /* If the size matches this level, we're in the right place */
644 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
645 max_entries = arm_lpae_max_entries(unmap_idx_start, data);
646 num_entries = min_t(int, pgcount, max_entries);
647
648 /* Find and handle non-leaf entries */
649 for (i = 0; i < num_entries; i++) {
650 pte = READ_ONCE(ptep[i]);
651 if (!pte) {
652 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));
653 break;
654 }
655
656 if (!iopte_leaf(pte, lvl, iop->fmt)) {
657 __arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1);
658
659 /* Also flush any partial walks */
660 io_pgtable_tlb_flush_walk(iop, iova + i * size, size,
661 ARM_LPAE_GRANULE(data));
662 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
663 }
664 }
665
666 /* Clear the remaining entries */
667 __arm_lpae_clear_pte(ptep, &iop->cfg, i);
668
669 if (gather && !iommu_iotlb_gather_queued(gather))
670 for (int j = 0; j < i; j++)
671 io_pgtable_tlb_add_page(iop, gather, iova + j * size, size);
672
673 return i * size;
674 } else if (iopte_leaf(pte, lvl, iop->fmt)) {
675 WARN_ONCE(true, "Unmap of a partial large IOPTE is not allowed");
676 return 0;
677 }
678
679 /* Keep on walkin' */
680 ptep = iopte_deref(pte, data);
681 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);
682 }
683
arm_lpae_unmap_pages(struct io_pgtable_ops * ops,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)684 static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova,
685 size_t pgsize, size_t pgcount,
686 struct iommu_iotlb_gather *gather)
687 {
688 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
689 struct io_pgtable_cfg *cfg = &data->iop.cfg;
690 arm_lpae_iopte *ptep = data->pgd;
691 long iaext = (s64)iova >> cfg->ias;
692
693 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount))
694 return 0;
695
696 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
697 iaext = ~iaext;
698 if (WARN_ON(iaext))
699 return 0;
700
701 return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount,
702 data->start_level, ptep);
703 }
704
705 struct io_pgtable_walk_data {
706 struct io_pgtable *iop;
707 void *data;
708 int (*visit)(struct io_pgtable_walk_data *walk_data, int lvl,
709 arm_lpae_iopte *ptep, size_t size);
710 unsigned long flags;
711 u64 addr;
712 const u64 end;
713 };
714
715 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data,
716 struct io_pgtable_walk_data *walk_data,
717 arm_lpae_iopte *ptep,
718 int lvl);
719
720 struct iova_to_phys_data {
721 arm_lpae_iopte pte;
722 int lvl;
723 };
724
visit_iova_to_phys(struct io_pgtable_walk_data * walk_data,int lvl,arm_lpae_iopte * ptep,size_t size)725 static int visit_iova_to_phys(struct io_pgtable_walk_data *walk_data, int lvl,
726 arm_lpae_iopte *ptep, size_t size)
727 {
728 struct iova_to_phys_data *data = walk_data->data;
729 data->pte = *ptep;
730 data->lvl = lvl;
731 return 0;
732 }
733
arm_lpae_iova_to_phys(struct io_pgtable_ops * ops,unsigned long iova)734 static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
735 unsigned long iova)
736 {
737 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
738 struct iova_to_phys_data d;
739 struct io_pgtable_walk_data walk_data = {
740 .data = &d,
741 .visit = visit_iova_to_phys,
742 .addr = iova,
743 .end = iova + 1,
744 };
745 int ret;
746
747 ret = __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level);
748 if (ret)
749 return 0;
750
751 iova &= (ARM_LPAE_BLOCK_SIZE(d.lvl, data) - 1);
752 return iopte_to_paddr(d.pte, data) | iova;
753 }
754
visit_pgtable_walk(struct io_pgtable_walk_data * walk_data,int lvl,arm_lpae_iopte * ptep,size_t size)755 static int visit_pgtable_walk(struct io_pgtable_walk_data *walk_data, int lvl,
756 arm_lpae_iopte *ptep, size_t size)
757 {
758 struct arm_lpae_io_pgtable_walk_data *data = walk_data->data;
759 data->ptes[lvl] = *ptep;
760 return 0;
761 }
762
arm_lpae_pgtable_walk(struct io_pgtable_ops * ops,unsigned long iova,void * wd)763 static int arm_lpae_pgtable_walk(struct io_pgtable_ops *ops, unsigned long iova,
764 void *wd)
765 {
766 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
767 struct io_pgtable_walk_data walk_data = {
768 .data = wd,
769 .visit = visit_pgtable_walk,
770 .addr = iova,
771 .end = iova + 1,
772 };
773
774 return __arm_lpae_iopte_walk(data, &walk_data, data->pgd, data->start_level);
775 }
776
io_pgtable_visit(struct arm_lpae_io_pgtable * data,struct io_pgtable_walk_data * walk_data,arm_lpae_iopte * ptep,int lvl)777 static int io_pgtable_visit(struct arm_lpae_io_pgtable *data,
778 struct io_pgtable_walk_data *walk_data,
779 arm_lpae_iopte *ptep, int lvl)
780 {
781 struct io_pgtable *iop = &data->iop;
782 arm_lpae_iopte pte = READ_ONCE(*ptep);
783
784 size_t size = ARM_LPAE_BLOCK_SIZE(lvl, data);
785 int ret = walk_data->visit(walk_data, lvl, ptep, size);
786 if (ret)
787 return ret;
788
789 if (iopte_leaf(pte, lvl, iop->fmt)) {
790 walk_data->addr += size;
791 return 0;
792 }
793
794 if (!iopte_table(pte, lvl)) {
795 return -EINVAL;
796 }
797
798 ptep = iopte_deref(pte, data);
799 return __arm_lpae_iopte_walk(data, walk_data, ptep, lvl + 1);
800 }
801
__arm_lpae_iopte_walk(struct arm_lpae_io_pgtable * data,struct io_pgtable_walk_data * walk_data,arm_lpae_iopte * ptep,int lvl)802 static int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data,
803 struct io_pgtable_walk_data *walk_data,
804 arm_lpae_iopte *ptep,
805 int lvl)
806 {
807 u32 idx;
808 int max_entries, ret;
809
810 if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
811 return -EINVAL;
812
813 if (lvl == data->start_level)
814 max_entries = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte);
815 else
816 max_entries = ARM_LPAE_PTES_PER_TABLE(data);
817
818 for (idx = ARM_LPAE_LVL_IDX(walk_data->addr, lvl, data);
819 (idx < max_entries) && (walk_data->addr < walk_data->end); ++idx) {
820 ret = io_pgtable_visit(data, walk_data, ptep + idx, lvl);
821 if (ret)
822 return ret;
823 }
824
825 return 0;
826 }
827
visit_dirty(struct io_pgtable_walk_data * walk_data,int lvl,arm_lpae_iopte * ptep,size_t size)828 static int visit_dirty(struct io_pgtable_walk_data *walk_data, int lvl,
829 arm_lpae_iopte *ptep, size_t size)
830 {
831 struct iommu_dirty_bitmap *dirty = walk_data->data;
832
833 if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt))
834 return 0;
835
836 if (iopte_writeable_dirty(*ptep)) {
837 iommu_dirty_bitmap_record(dirty, walk_data->addr, size);
838 if (!(walk_data->flags & IOMMU_DIRTY_NO_CLEAR))
839 iopte_set_writeable_clean(ptep);
840 }
841
842 return 0;
843 }
844
arm_lpae_read_and_clear_dirty(struct io_pgtable_ops * ops,unsigned long iova,size_t size,unsigned long flags,struct iommu_dirty_bitmap * dirty)845 static int arm_lpae_read_and_clear_dirty(struct io_pgtable_ops *ops,
846 unsigned long iova, size_t size,
847 unsigned long flags,
848 struct iommu_dirty_bitmap *dirty)
849 {
850 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
851 struct io_pgtable_cfg *cfg = &data->iop.cfg;
852 struct io_pgtable_walk_data walk_data = {
853 .iop = &data->iop,
854 .data = dirty,
855 .visit = visit_dirty,
856 .flags = flags,
857 .addr = iova,
858 .end = iova + size,
859 };
860 arm_lpae_iopte *ptep = data->pgd;
861 int lvl = data->start_level;
862
863 if (WARN_ON(!size))
864 return -EINVAL;
865 if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1)))
866 return -EINVAL;
867 if (data->iop.fmt != ARM_64_LPAE_S1)
868 return -EINVAL;
869
870 return __arm_lpae_iopte_walk(data, &walk_data, ptep, lvl);
871 }
872
arm_lpae_restrict_pgsizes(struct io_pgtable_cfg * cfg)873 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
874 {
875 unsigned long granule, page_sizes;
876 unsigned int max_addr_bits = 48;
877
878 /*
879 * We need to restrict the supported page sizes to match the
880 * translation regime for a particular granule. Aim to match
881 * the CPU page size if possible, otherwise prefer smaller sizes.
882 * While we're at it, restrict the block sizes to match the
883 * chosen granule.
884 */
885 if (cfg->pgsize_bitmap & PAGE_SIZE)
886 granule = PAGE_SIZE;
887 else if (cfg->pgsize_bitmap & ~PAGE_MASK)
888 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);
889 else if (cfg->pgsize_bitmap & PAGE_MASK)
890 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);
891 else
892 granule = 0;
893
894 switch (granule) {
895 case SZ_4K:
896 page_sizes = (SZ_4K | SZ_2M | SZ_1G);
897 break;
898 case SZ_16K:
899 page_sizes = (SZ_16K | SZ_32M);
900 break;
901 case SZ_64K:
902 max_addr_bits = 52;
903 page_sizes = (SZ_64K | SZ_512M);
904 if (cfg->oas > 48)
905 page_sizes |= 1ULL << 42; /* 4TB */
906 break;
907 default:
908 page_sizes = 0;
909 }
910
911 cfg->pgsize_bitmap &= page_sizes;
912 cfg->ias = min(cfg->ias, max_addr_bits);
913 cfg->oas = min(cfg->oas, max_addr_bits);
914 }
915
916 static struct arm_lpae_io_pgtable *
arm_lpae_alloc_pgtable(struct io_pgtable_cfg * cfg)917 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
918 {
919 struct arm_lpae_io_pgtable *data;
920 int levels, va_bits, pg_shift;
921
922 arm_lpae_restrict_pgsizes(cfg);
923
924 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))
925 return NULL;
926
927 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)
928 return NULL;
929
930 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)
931 return NULL;
932
933 data = kmalloc(sizeof(*data), GFP_KERNEL);
934 if (!data)
935 return NULL;
936
937 pg_shift = __ffs(cfg->pgsize_bitmap);
938 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));
939
940 va_bits = cfg->ias - pg_shift;
941 levels = DIV_ROUND_UP(va_bits, data->bits_per_level);
942 data->start_level = ARM_LPAE_MAX_LEVELS - levels;
943
944 /* Calculate the actual size of our pgd (without concatenation) */
945 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
946
947 data->iop.ops = (struct io_pgtable_ops) {
948 .map_pages = arm_lpae_map_pages,
949 .unmap_pages = arm_lpae_unmap_pages,
950 .iova_to_phys = arm_lpae_iova_to_phys,
951 .read_and_clear_dirty = arm_lpae_read_and_clear_dirty,
952 .pgtable_walk = arm_lpae_pgtable_walk,
953 };
954
955 return data;
956 }
957
958 static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg * cfg,void * cookie)959 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
960 {
961 u64 reg;
962 struct arm_lpae_io_pgtable *data;
963 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;
964 bool tg1;
965
966 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |
967 IO_PGTABLE_QUIRK_ARM_TTBR1 |
968 IO_PGTABLE_QUIRK_ARM_OUTER_WBWA |
969 IO_PGTABLE_QUIRK_ARM_HD |
970 IO_PGTABLE_QUIRK_NO_WARN))
971 return NULL;
972
973 data = arm_lpae_alloc_pgtable(cfg);
974 if (!data)
975 return NULL;
976
977 /* TCR */
978 if (cfg->coherent_walk) {
979 tcr->sh = ARM_LPAE_TCR_SH_IS;
980 tcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
981 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
982 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)
983 goto out_free_data;
984 } else {
985 tcr->sh = ARM_LPAE_TCR_SH_OS;
986 tcr->irgn = ARM_LPAE_TCR_RGN_NC;
987 if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA))
988 tcr->orgn = ARM_LPAE_TCR_RGN_NC;
989 else
990 tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
991 }
992
993 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;
994 switch (ARM_LPAE_GRANULE(data)) {
995 case SZ_4K:
996 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K;
997 break;
998 case SZ_16K:
999 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K;
1000 break;
1001 case SZ_64K:
1002 tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K;
1003 break;
1004 }
1005
1006 switch (cfg->oas) {
1007 case 32:
1008 tcr->ips = ARM_LPAE_TCR_PS_32_BIT;
1009 break;
1010 case 36:
1011 tcr->ips = ARM_LPAE_TCR_PS_36_BIT;
1012 break;
1013 case 40:
1014 tcr->ips = ARM_LPAE_TCR_PS_40_BIT;
1015 break;
1016 case 42:
1017 tcr->ips = ARM_LPAE_TCR_PS_42_BIT;
1018 break;
1019 case 44:
1020 tcr->ips = ARM_LPAE_TCR_PS_44_BIT;
1021 break;
1022 case 48:
1023 tcr->ips = ARM_LPAE_TCR_PS_48_BIT;
1024 break;
1025 case 52:
1026 tcr->ips = ARM_LPAE_TCR_PS_52_BIT;
1027 break;
1028 default:
1029 goto out_free_data;
1030 }
1031
1032 tcr->tsz = 64ULL - cfg->ias;
1033
1034 /* MAIRs */
1035 reg = (ARM_LPAE_MAIR_ATTR_NC
1036 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
1037 (ARM_LPAE_MAIR_ATTR_WBRWA
1038 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
1039 (ARM_LPAE_MAIR_ATTR_DEVICE
1040 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) |
1041 (ARM_LPAE_MAIR_ATTR_INC_OWBRWA
1042 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE));
1043
1044 cfg->arm_lpae_s1_cfg.mair = reg;
1045
1046 /* Looking good; allocate a pgd */
1047 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
1048 GFP_KERNEL, cfg, cookie);
1049 if (!data->pgd)
1050 goto out_free_data;
1051
1052 /* Ensure the empty pgd is visible before any actual TTBR write */
1053 wmb();
1054
1055 /* TTBR */
1056 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
1057 return &data->iop;
1058
1059 out_free_data:
1060 kfree(data);
1061 return NULL;
1062 }
1063
1064 static struct io_pgtable *
arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg * cfg,void * cookie)1065 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
1066 {
1067 u64 sl;
1068 struct arm_lpae_io_pgtable *data;
1069 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;
1070
1071 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB |
1072 IO_PGTABLE_QUIRK_NO_WARN))
1073 return NULL;
1074
1075 data = arm_lpae_alloc_pgtable(cfg);
1076 if (!data)
1077 return NULL;
1078
1079 if (arm_lpae_concat_mandatory(cfg, data)) {
1080 if (WARN_ON((ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte)) >
1081 ARM_LPAE_S2_MAX_CONCAT_PAGES))
1082 return NULL;
1083 data->pgd_bits += data->bits_per_level;
1084 data->start_level++;
1085 }
1086
1087 /* VTCR */
1088 if (cfg->coherent_walk) {
1089 vtcr->sh = ARM_LPAE_TCR_SH_IS;
1090 vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
1091 vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
1092 } else {
1093 vtcr->sh = ARM_LPAE_TCR_SH_OS;
1094 vtcr->irgn = ARM_LPAE_TCR_RGN_NC;
1095 vtcr->orgn = ARM_LPAE_TCR_RGN_NC;
1096 }
1097
1098 sl = data->start_level;
1099
1100 switch (ARM_LPAE_GRANULE(data)) {
1101 case SZ_4K:
1102 vtcr->tg = ARM_LPAE_TCR_TG0_4K;
1103 sl++; /* SL0 format is different for 4K granule size */
1104 break;
1105 case SZ_16K:
1106 vtcr->tg = ARM_LPAE_TCR_TG0_16K;
1107 break;
1108 case SZ_64K:
1109 vtcr->tg = ARM_LPAE_TCR_TG0_64K;
1110 break;
1111 }
1112
1113 switch (cfg->oas) {
1114 case 32:
1115 vtcr->ps = ARM_LPAE_TCR_PS_32_BIT;
1116 break;
1117 case 36:
1118 vtcr->ps = ARM_LPAE_TCR_PS_36_BIT;
1119 break;
1120 case 40:
1121 vtcr->ps = ARM_LPAE_TCR_PS_40_BIT;
1122 break;
1123 case 42:
1124 vtcr->ps = ARM_LPAE_TCR_PS_42_BIT;
1125 break;
1126 case 44:
1127 vtcr->ps = ARM_LPAE_TCR_PS_44_BIT;
1128 break;
1129 case 48:
1130 vtcr->ps = ARM_LPAE_TCR_PS_48_BIT;
1131 break;
1132 case 52:
1133 vtcr->ps = ARM_LPAE_TCR_PS_52_BIT;
1134 break;
1135 default:
1136 goto out_free_data;
1137 }
1138
1139 vtcr->tsz = 64ULL - cfg->ias;
1140 vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK;
1141
1142 /* Allocate pgd pages */
1143 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
1144 GFP_KERNEL, cfg, cookie);
1145 if (!data->pgd)
1146 goto out_free_data;
1147
1148 /* Ensure the empty pgd is visible before any actual TTBR write */
1149 wmb();
1150
1151 /* VTTBR */
1152 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
1153 return &data->iop;
1154
1155 out_free_data:
1156 kfree(data);
1157 return NULL;
1158 }
1159
1160 static struct io_pgtable *
arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg * cfg,void * cookie)1161 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
1162 {
1163 if (cfg->ias > 32 || cfg->oas > 40)
1164 return NULL;
1165
1166 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1167 return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
1168 }
1169
1170 static struct io_pgtable *
arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg * cfg,void * cookie)1171 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
1172 {
1173 if (cfg->ias > 40 || cfg->oas > 40)
1174 return NULL;
1175
1176 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1177 return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
1178 }
1179
1180 static struct io_pgtable *
arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg * cfg,void * cookie)1181 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
1182 {
1183 struct arm_lpae_io_pgtable *data;
1184
1185 /* No quirks for Mali (hopefully) */
1186 if (cfg->quirks)
1187 return NULL;
1188
1189 if (cfg->ias > 48 || cfg->oas > 40)
1190 return NULL;
1191
1192 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
1193
1194 data = arm_lpae_alloc_pgtable(cfg);
1195 if (!data)
1196 return NULL;
1197
1198 /* Mali seems to need a full 4-level table regardless of IAS */
1199 if (data->start_level > 0) {
1200 data->start_level = 0;
1201 data->pgd_bits = 0;
1202 }
1203 /*
1204 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the
1205 * best we can do is mimic the out-of-tree driver and hope that the
1206 * "implementation-defined caching policy" is good enough. Similarly,
1207 * we'll use it for the sake of a valid attribute for our 'device'
1208 * index, although callers should never request that in practice.
1209 */
1210 cfg->arm_mali_lpae_cfg.memattr =
1211 (ARM_MALI_LPAE_MEMATTR_IMP_DEF
1212 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
1213 (ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC
1214 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
1215 (ARM_MALI_LPAE_MEMATTR_IMP_DEF
1216 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV));
1217
1218 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
1219 cfg, cookie);
1220 if (!data->pgd)
1221 goto out_free_data;
1222
1223 /* Ensure the empty pgd is visible before TRANSTAB can be written */
1224 wmb();
1225
1226 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |
1227 ARM_MALI_LPAE_TTBR_READ_INNER |
1228 ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
1229 if (cfg->coherent_walk)
1230 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;
1231
1232 return &data->iop;
1233
1234 out_free_data:
1235 kfree(data);
1236 return NULL;
1237 }
1238
1239 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
1240 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1241 .alloc = arm_64_lpae_alloc_pgtable_s1,
1242 .free = arm_lpae_free_pgtable,
1243 };
1244
1245 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = {
1246 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1247 .alloc = arm_64_lpae_alloc_pgtable_s2,
1248 .free = arm_lpae_free_pgtable,
1249 };
1250
1251 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = {
1252 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1253 .alloc = arm_32_lpae_alloc_pgtable_s1,
1254 .free = arm_lpae_free_pgtable,
1255 };
1256
1257 struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
1258 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1259 .alloc = arm_32_lpae_alloc_pgtable_s2,
1260 .free = arm_lpae_free_pgtable,
1261 };
1262
1263 struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
1264 .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR,
1265 .alloc = arm_mali_lpae_alloc_pgtable,
1266 .free = arm_lpae_free_pgtable,
1267 };
1268