46 static size_t calc_pgsize(struct msm_iommu_pagetable *pagetable,  in calc_pgsize()  argument
56 	pgsizes = pagetable->pgsize_bitmap & GENMASK(__fls(size), 0);  in calc_pgsize()
72 	pgsizes = pagetable->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);  in calc_pgsize()
104 	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);  in msm_iommu_pagetable_unmap()  local
105 	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;  in msm_iommu_pagetable_unmap()
112 		pgsize = calc_pgsize(pagetable, iova, iova, size, &count);  in msm_iommu_pagetable_unmap()
129 	iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain);  in msm_iommu_pagetable_unmap()
136 	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);  in msm_iommu_pagetable_map_prr()  local
137 	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;  in msm_iommu_pagetable_map_prr()
138 	struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);  in msm_iommu_pagetable_map_prr()
168 	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);  in msm_iommu_pagetable_map()  local
169 	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;  in msm_iommu_pagetable_map()
198 			pgsize = calc_pgsize(pagetable, addr, phys, size, &count);  in msm_iommu_pagetable_map()
223 	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);  in msm_iommu_pagetable_destroy()  local
224 	struct msm_iommu *iommu = to_msm_iommu(pagetable->parent);  in msm_iommu_pagetable_destroy()
226 		dev_get_drvdata(pagetable->parent->dev);  in msm_iommu_pagetable_destroy()
229 	 * If this is the last attached pagetable for the parent,  in msm_iommu_pagetable_destroy()
244 	free_io_pgtable_ops(pagetable->pgtbl_ops);  in msm_iommu_pagetable_destroy()
245 	kfree(pagetable);  in msm_iommu_pagetable_destroy()
251 	struct msm_iommu_pagetable *pagetable;  in msm_iommu_pagetable_params()  local
256 	pagetable = to_pagetable(mmu);  in msm_iommu_pagetable_params()
259 		*ttbr = pagetable->ttbr;  in msm_iommu_pagetable_params()
262 		*asid = pagetable->asid;  in msm_iommu_pagetable_params()
277 	struct msm_iommu_pagetable *pagetable;  in msm_iommu_pagetable_walk()  local
283 	pagetable = to_pagetable(mmu);  in msm_iommu_pagetable_walk()
285 	if (!pagetable->pgtbl_ops->pgtable_walk)  in msm_iommu_pagetable_walk()
288 	pagetable->pgtbl_ops->pgtable_walk(pagetable->pgtbl_ops, iova, &wd);  in msm_iommu_pagetable_walk()
325 	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);  in get_pt_cache()  local
326 	return to_msm_iommu(pagetable->parent)->pt_cache;  in get_pt_cache()
383 	struct msm_iommu_pagetable *pagetable = cookie;  in msm_iommu_pagetable_alloc_pt()  local
384 	struct msm_mmu_prealloc *p = pagetable->base.prealloc;  in msm_iommu_pagetable_alloc_pt()
388 	if (unlikely(!pagetable->root_page_table)) {  in msm_iommu_pagetable_alloc_pt()
391 		p = alloc_pages_node(dev_to_node(pagetable->iommu_dev),  in msm_iommu_pagetable_alloc_pt()
394 		pagetable->root_page_table = page;  in msm_iommu_pagetable_alloc_pt()
428 	struct msm_iommu_pagetable *pagetable = cookie;  in msm_iommu_pagetable_free_pt()  local
430 	if (unlikely(pagetable->root_page_table == data)) {  in msm_iommu_pagetable_free_pt()
432 		pagetable->root_page_table = NULL;  in msm_iommu_pagetable_free_pt()
436 	kmem_cache_free(get_pt_cache(&pagetable->base), data);  in msm_iommu_pagetable_free_pt()
450 	struct msm_iommu_pagetable *pagetable = cookie;  in msm_iommu_tlb_flush_all()  local
453 	if (!pm_runtime_get_if_in_use(pagetable->iommu_dev))  in msm_iommu_tlb_flush_all()
456 	adreno_smmu = dev_get_drvdata(pagetable->parent->dev);  in msm_iommu_tlb_flush_all()
458 	pagetable->tlb->tlb_flush_all((void *)adreno_smmu->cookie);  in msm_iommu_tlb_flush_all()
460 	pm_runtime_put_autosuspend(pagetable->iommu_dev);  in msm_iommu_tlb_flush_all()
466 	struct msm_iommu_pagetable *pagetable = cookie;  in msm_iommu_tlb_flush_walk()  local
469 	if (!pm_runtime_get_if_in_use(pagetable->iommu_dev))  in msm_iommu_tlb_flush_walk()
472 	adreno_smmu = dev_get_drvdata(pagetable->parent->dev);  in msm_iommu_tlb_flush_walk()
474 	pagetable->tlb->tlb_flush_walk(iova, size, granule, (void *)adreno_smmu->cookie);  in msm_iommu_tlb_flush_walk()
476 	pm_runtime_put_autosuspend(pagetable->iommu_dev);  in msm_iommu_tlb_flush_walk()
508 	struct msm_iommu_pagetable *pagetable;  in msm_iommu_pagetable_create()  local
513 	/* Get the pagetable configuration from the domain */  in msm_iommu_pagetable_create()
524 	pagetable = kzalloc(sizeof(*pagetable), GFP_KERNEL);  in msm_iommu_pagetable_create()
525 	if (!pagetable)  in msm_iommu_pagetable_create()
528 	msm_mmu_init(&pagetable->base, parent->dev, &pagetable_funcs,  in msm_iommu_pagetable_create()
567 	pagetable->iommu_dev = ttbr1_cfg->iommu_dev;  in msm_iommu_pagetable_create()
568 	pagetable->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1,  in msm_iommu_pagetable_create()
569 		&ttbr0_cfg, pagetable);  in msm_iommu_pagetable_create()
571 	if (!pagetable->pgtbl_ops) {  in msm_iommu_pagetable_create()
572 		kfree(pagetable);  in msm_iommu_pagetable_create()
577 	 * If this is the first pagetable that we've allocated, send it back to  in msm_iommu_pagetable_create()
586 			free_io_pgtable_ops(pagetable->pgtbl_ops);  in msm_iommu_pagetable_create()
587 			kfree(pagetable);  in msm_iommu_pagetable_create()
613 	pagetable->parent = parent;  in msm_iommu_pagetable_create()
614 	pagetable->tlb = ttbr1_cfg->tlb;  in msm_iommu_pagetable_create()
615 	pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap;  in msm_iommu_pagetable_create()
616 	pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr;  in msm_iommu_pagetable_create()
624 	pagetable->asid = 0;  in msm_iommu_pagetable_create()
626 	return &pagetable->base;  in msm_iommu_pagetable_create()