93 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_pgtables_free()  argument
100 		pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];  in ivpu_mmu_pgtables_free()
101 		pud_dma = pgtable->pgd_dma_ptr[pgd_idx];  in ivpu_mmu_pgtables_free()
107 			pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
108 			pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
114 				pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
115 				pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
120 			kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_pgtables_free()
124 		kfree(pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_pgtables_free()
125 		kfree(pgtable->pte_ptrs[pgd_idx]);  in ivpu_mmu_pgtables_free()
129 	ivpu_pgtable_free_page(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);  in ivpu_mmu_pgtables_free()
130 	pgtable->pgd_dma_ptr = NULL;  in ivpu_mmu_pgtables_free()
131 	pgtable->pgd_dma = 0;  in ivpu_mmu_pgtables_free()
135 ivpu_mmu_ensure_pgd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_ensure_pgd()  argument
137 	u64 *pgd_dma_ptr = pgtable->pgd_dma_ptr;  in ivpu_mmu_ensure_pgd()
147 	pgtable->pgd_dma_ptr = pgd_dma_ptr;  in ivpu_mmu_ensure_pgd()
148 	pgtable->pgd_dma = pgd_dma;  in ivpu_mmu_ensure_pgd()
154 ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)  in ivpu_mmu_ensure_pud()  argument
156 	u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];  in ivpu_mmu_ensure_pud()
166 	drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
167 	pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pud()
168 	if (!pgtable->pmd_ptrs[pgd_idx])  in ivpu_mmu_ensure_pud()
171 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
172 	pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pud()
173 	if (!pgtable->pte_ptrs[pgd_idx])  in ivpu_mmu_ensure_pud()
176 	pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;  in ivpu_mmu_ensure_pud()
177 	pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pud()
182 	kfree(pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
190 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,  in ivpu_mmu_ensure_pmd()  argument
193 	u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_ensure_pmd()
203 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_ensure_pmd()
204 	pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pmd()
205 	if (!pgtable->pte_ptrs[pgd_idx][pud_idx])  in ivpu_mmu_ensure_pmd()
208 	pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;  in ivpu_mmu_ensure_pmd()
209 	pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pmd()
219 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,  in ivpu_mmu_ensure_pte()  argument
222 	u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_ensure_pte()
232 	pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;  in ivpu_mmu_ensure_pte()
233 	pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pte()
251 	if (!ivpu_mmu_ensure_pgd(vdev, &ctx->pgtable))  in ivpu_mmu_context_map_page()
255 	if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))  in ivpu_mmu_context_map_page()
259 	if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))  in ivpu_mmu_context_map_page()
263 	pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);  in ivpu_mmu_context_map_page()
306 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;  in ivpu_mmu_context_unmap_page()
345 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] |= IVPU_MMU_ENTRY_FLAG_RO;  in ivpu_mmu_context_set_page_ro()
356 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] &= ~IVPU_MMU_ENTRY_FLAG_CONT;  in ivpu_mmu_context_split_page()
472 		ret = ivpu_mmu_cd_set(vdev, ctx->id, &ctx->pgtable);  in ivpu_mmu_context_map_sgt()
590 	ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);  in ivpu_mmu_context_fini()
612 	if (!ivpu_mmu_ensure_pgd(vdev, &vdev->rctx.pgtable)) {  in ivpu_mmu_reserved_context_init()
618 	ret = ivpu_mmu_cd_set(vdev, vdev->rctx.id, &vdev->rctx.pgtable);  in ivpu_mmu_reserved_context_init()