33 	container_of((x), struct arm_lpae_io_pgtable, iop)
160 	struct io_pgtable	iop;  member
344 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_init_pte()
348 	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)  in __arm_lpae_init_pte()
368 		if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {  in arm_lpae_init_pte()
370 			WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));  in arm_lpae_init_pte()
398 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_install_table()
432 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in __arm_lpae_map()
457 		cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie);  in __arm_lpae_map()
463 			__arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie);  in __arm_lpae_map()
468 	if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {  in __arm_lpae_map()
486 	if (data->iop.fmt == ARM_64_LPAE_S1 ||  in arm_lpae_prot_to_pte()
487 	    data->iop.fmt == ARM_32_LPAE_S1) {  in arm_lpae_prot_to_pte()
491 		else if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_HD)  in arm_lpae_prot_to_pte()
507 	if (data->iop.fmt == ARM_64_LPAE_S2 ||  in arm_lpae_prot_to_pte()
508 	    data->iop.fmt == ARM_32_LPAE_S2) {  in arm_lpae_prot_to_pte()
512 			if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)  in arm_lpae_prot_to_pte()
534 	if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)  in arm_lpae_prot_to_pte()
542 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)  in arm_lpae_prot_to_pte()
545 	if (data->iop.fmt != ARM_MALI_LPAE)  in arm_lpae_prot_to_pte()
556 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_map_pages()
607 		if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))  in __arm_lpae_free_pgtable()
613 	__arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie);  in __arm_lpae_free_pgtable()
616 static void arm_lpae_free_pgtable(struct io_pgtable *iop)  in arm_lpae_free_pgtable()  argument
618 	struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);  in arm_lpae_free_pgtable()
630 	struct io_pgtable *iop = &data->iop;  in __arm_lpae_unmap()  local
641 		WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));  in __arm_lpae_unmap()
654 				WARN_ON(!(data->iop.cfg.quirks & IO_PGTABLE_QUIRK_NO_WARN));  in __arm_lpae_unmap()
658 			if (!iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
659 				__arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1);  in __arm_lpae_unmap()
662 				io_pgtable_tlb_flush_walk(iop, iova + i * size, size,  in __arm_lpae_unmap()
669 		__arm_lpae_clear_pte(ptep, &iop->cfg, i);  in __arm_lpae_unmap()
673 				io_pgtable_tlb_add_page(iop, gather, iova + j * size, size);  in __arm_lpae_unmap()
676 	} else if (iopte_leaf(pte, lvl, iop->fmt)) {  in __arm_lpae_unmap()
691 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_unmap_pages()
708 	struct io_pgtable		*iop;  member
783 	struct io_pgtable *iop = &data->iop;  in io_pgtable_visit()  local
791 	if (iopte_leaf(pte, lvl, iop->fmt)) {  in io_pgtable_visit()
835 	if (!iopte_leaf(*ptep, lvl, walk_data->iop->fmt))  in visit_dirty()
853 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_read_and_clear_dirty()
855 		.iop = &data->iop,  in arm_lpae_read_and_clear_dirty()
869 	if (data->iop.fmt != ARM_64_LPAE_S1)  in arm_lpae_read_and_clear_dirty()
949 	data->iop.ops = (struct io_pgtable_ops) {  in arm_lpae_alloc_pgtable()
1059 	return &data->iop;  in arm_64_lpae_alloc_pgtable_s1()
1155 	return &data->iop;  in arm_64_lpae_alloc_pgtable_s2()
1234 	return &data->iop;  in arm_mali_lpae_alloc_pgtable()
1303 	struct io_pgtable_cfg *cfg = &data->iop.cfg;  in arm_lpae_dump_ops()