Lines Matching full:cfg
229 static inline bool arm_lpae_concat_mandatory(struct io_pgtable_cfg *cfg, in arm_lpae_concat_mandatory() argument
232 unsigned int ias = cfg->ias; in arm_lpae_concat_mandatory()
233 unsigned int oas = cfg->oas; in arm_lpae_concat_mandatory()
254 struct io_pgtable_cfg *cfg, in __arm_lpae_alloc_pages() argument
257 struct device *dev = cfg->iommu_dev; in __arm_lpae_alloc_pages()
267 if (cfg->alloc) in __arm_lpae_alloc_pages()
268 pages = cfg->alloc(cookie, alloc_size, gfp); in __arm_lpae_alloc_pages()
276 if (!cfg->coherent_walk) { in __arm_lpae_alloc_pages()
296 if (cfg->free) in __arm_lpae_alloc_pages()
297 cfg->free(cookie, pages, size); in __arm_lpae_alloc_pages()
305 struct io_pgtable_cfg *cfg, in __arm_lpae_free_pages() argument
308 if (!cfg->coherent_walk) in __arm_lpae_free_pages()
309 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), in __arm_lpae_free_pages()
312 if (cfg->free) in __arm_lpae_free_pages()
313 cfg->free(cookie, pages, size); in __arm_lpae_free_pages()
319 struct io_pgtable_cfg *cfg) in __arm_lpae_sync_pte() argument
321 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), in __arm_lpae_sync_pte()
325 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries) in __arm_lpae_clear_pte() argument
330 if (!cfg->coherent_walk && num_entries) in __arm_lpae_clear_pte()
331 __arm_lpae_sync_pte(ptep, num_entries, cfg); in __arm_lpae_clear_pte()
344 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_lpae_init_pte() local
356 if (!cfg->coherent_walk) in __arm_lpae_init_pte()
357 __arm_lpae_sync_pte(ptep, num_entries, cfg); in __arm_lpae_init_pte()
370 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); in arm_lpae_init_pte()
398 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_install_table() local
401 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_install_table()
413 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) in arm_lpae_install_table()
417 __arm_lpae_sync_pte(ptep, 1, cfg); in arm_lpae_install_table()
432 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_lpae_map() local
457 cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie); in __arm_lpae_map()
463 __arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie); in __arm_lpae_map()
464 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { in __arm_lpae_map()
465 __arm_lpae_sync_pte(ptep, 1, cfg); in __arm_lpae_map()
472 WARN_ON(!(cfg->quirks & IO_PGTABLE_QUIRK_NO_WARN)); in __arm_lpae_map()
491 else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD) in arm_lpae_prot_to_pte()
512 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB) in arm_lpae_prot_to_pte()
542 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_prot_to_pte()
556 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_map_pages() local
560 long iaext = (s64)iova >> cfg->ias; in arm_lpae_map_pages()
562 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) in arm_lpae_map_pages()
565 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_map_pages()
567 if (WARN_ON(iaext || paddr >> cfg->oas)) in arm_lpae_map_pages()
613 __arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie); in __arm_lpae_free_pgtable()
641 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); in __arm_lpae_unmap()
654 WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN)); in __arm_lpae_unmap()
659 __arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1); in __arm_lpae_unmap()
669 __arm_lpae_clear_pte(ptep, &iop->cfg, i); in __arm_lpae_unmap()
691 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_unmap_pages() local
693 long iaext = (s64)iova >> cfg->ias; in arm_lpae_unmap_pages()
695 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) in arm_lpae_unmap_pages()
698 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_unmap_pages()
853 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_read_and_clear_dirty() local
867 if (WARN_ON((iova + size - 1) & ~(BIT(cfg->ias) - 1))) in arm_lpae_read_and_clear_dirty()
875 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) in arm_lpae_restrict_pgsizes() argument
887 if (cfg->pgsize_bitmap & PAGE_SIZE) in arm_lpae_restrict_pgsizes()
889 else if (cfg->pgsize_bitmap & ~PAGE_MASK) in arm_lpae_restrict_pgsizes()
890 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); in arm_lpae_restrict_pgsizes()
891 else if (cfg->pgsize_bitmap & PAGE_MASK) in arm_lpae_restrict_pgsizes()
892 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); in arm_lpae_restrict_pgsizes()
906 if (cfg->oas > 48) in arm_lpae_restrict_pgsizes()
913 cfg->pgsize_bitmap &= page_sizes; in arm_lpae_restrict_pgsizes()
914 cfg->ias = min(cfg->ias, max_addr_bits); in arm_lpae_restrict_pgsizes()
915 cfg->oas = min(cfg->oas, max_addr_bits); in arm_lpae_restrict_pgsizes()
919 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) in arm_lpae_alloc_pgtable() argument
924 arm_lpae_restrict_pgsizes(cfg); in arm_lpae_alloc_pgtable()
926 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) in arm_lpae_alloc_pgtable()
929 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
932 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
939 pg_shift = __ffs(cfg->pgsize_bitmap); in arm_lpae_alloc_pgtable()
942 va_bits = cfg->ias - pg_shift; in arm_lpae_alloc_pgtable()
961 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) in arm_64_lpae_alloc_pgtable_s1() argument
965 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; in arm_64_lpae_alloc_pgtable_s1()
968 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | in arm_64_lpae_alloc_pgtable_s1()
975 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s1()
980 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s1()
984 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA) in arm_64_lpae_alloc_pgtable_s1()
989 if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) in arm_64_lpae_alloc_pgtable_s1()
995 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; in arm_64_lpae_alloc_pgtable_s1()
1008 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s1()
1034 tcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s1()
1046 cfg->arm_lpae_s1_cfg.mair = reg; in arm_64_lpae_alloc_pgtable_s1()
1050 GFP_KERNEL, cfg, cookie); in arm_64_lpae_alloc_pgtable_s1()
1058 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s1()
1067 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) in arm_64_lpae_alloc_pgtable_s2() argument
1071 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; in arm_64_lpae_alloc_pgtable_s2()
1073 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_S2FWB | in arm_64_lpae_alloc_pgtable_s2()
1077 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s2()
1081 if (arm_lpae_concat_mandatory(cfg, data)) { in arm_64_lpae_alloc_pgtable_s2()
1090 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s2()
1115 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s2()
1141 vtcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s2()
1146 GFP_KERNEL, cfg, cookie); in arm_64_lpae_alloc_pgtable_s2()
1154 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s2()
1163 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) in arm_32_lpae_alloc_pgtable_s1() argument
1165 if (cfg->ias > 32 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s1()
1168 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s1()
1169 return arm_64_lpae_alloc_pgtable_s1(cfg, cookie); in arm_32_lpae_alloc_pgtable_s1()
1173 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) in arm_32_lpae_alloc_pgtable_s2() argument
1175 if (cfg->ias > 40 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s2()
1178 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s2()
1179 return arm_64_lpae_alloc_pgtable_s2(cfg, cookie); in arm_32_lpae_alloc_pgtable_s2()
1183 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) in arm_mali_lpae_alloc_pgtable() argument
1188 if (cfg->quirks) in arm_mali_lpae_alloc_pgtable()
1191 if (cfg->ias > 48 || cfg->oas > 40) in arm_mali_lpae_alloc_pgtable()
1194 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_mali_lpae_alloc_pgtable()
1196 data = arm_lpae_alloc_pgtable(cfg); in arm_mali_lpae_alloc_pgtable()
1212 cfg->arm_mali_lpae_cfg.memattr = in arm_mali_lpae_alloc_pgtable()
1221 cfg, cookie); in arm_mali_lpae_alloc_pgtable()
1228 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | in arm_mali_lpae_alloc_pgtable()
1231 if (cfg->coherent_walk) in arm_mali_lpae_alloc_pgtable()
1232 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER; in arm_mali_lpae_alloc_pgtable()
1303 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_dump_ops() local
1305 pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", in arm_lpae_dump_ops()
1306 cfg->pgsize_bitmap, cfg->ias); in arm_lpae_dump_ops()
1318 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) in arm_lpae_run_tests() argument
1331 cfg_cookie = cfg; in arm_lpae_run_tests()
1332 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); in arm_lpae_run_tests()
1355 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1378 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1403 size = 1UL << __fls(cfg->pgsize_bitmap); in arm_lpae_run_tests()
1404 iova = (1UL << cfg->ias) - size; in arm_lpae_run_tests()
1435 struct io_pgtable_cfg cfg = { in arm_lpae_do_selftests() local
1445 cfg.iommu_dev = &dev->dev; in arm_lpae_do_selftests()
1451 cfg.pgsize_bitmap = pgsize[i]; in arm_lpae_do_selftests()
1452 cfg.ias = address_size[k]; in arm_lpae_do_selftests()
1453 cfg.oas = address_size[j]; in arm_lpae_do_selftests()
1455 pgsize[i], cfg.ias, cfg.oas); in arm_lpae_do_selftests()
1456 if (arm_lpae_run_tests(&cfg)) in arm_lpae_do_selftests()