xref: /linux/drivers/accel/ivpu/ivpu_mmu_context.c (revision e005fd94e2e5867f2a4e66e5df85069cda6f0db4) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/highmem.h>
8 #include <linux/set_memory.h>
9 #include <linux/vmalloc.h>
10 
11 #include <drm/drm_cache.h>
12 
13 #include "ivpu_drv.h"
14 #include "ivpu_hw.h"
15 #include "ivpu_mmu.h"
16 #include "ivpu_mmu_context.h"
17 
18 #define IVPU_MMU_VPU_ADDRESS_MASK        GENMASK(47, 12)
19 #define IVPU_MMU_PGD_INDEX_MASK          GENMASK(47, 39)
20 #define IVPU_MMU_PUD_INDEX_MASK          GENMASK(38, 30)
21 #define IVPU_MMU_PMD_INDEX_MASK          GENMASK(29, 21)
22 #define IVPU_MMU_PTE_INDEX_MASK          GENMASK(20, 12)
23 #define IVPU_MMU_ENTRY_FLAGS_MASK        (BIT(52) | GENMASK(11, 0))
24 #define IVPU_MMU_ENTRY_FLAG_CONT         BIT(52)
25 #define IVPU_MMU_ENTRY_FLAG_NG           BIT(11)
26 #define IVPU_MMU_ENTRY_FLAG_AF           BIT(10)
27 #define IVPU_MMU_ENTRY_FLAG_RO           BIT(7)
28 #define IVPU_MMU_ENTRY_FLAG_USER         BIT(6)
29 #define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)
30 #define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE    BIT(1)
31 #define IVPU_MMU_ENTRY_FLAG_VALID        BIT(0)
32 
33 #define IVPU_MMU_PAGE_SIZE       SZ_4K
34 #define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16)
35 #define IVPU_MMU_PTE_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)
36 #define IVPU_MMU_PMD_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)
37 #define IVPU_MMU_PUD_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE)
38 #define IVPU_MMU_PGD_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE)
39 #define IVPU_MMU_PGTABLE_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))
40 
41 #define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000
42 #define IVPU_MMU_ENTRY_VALID   (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID)
43 #define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)
44 #define IVPU_MMU_ENTRY_MAPPED  (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
45 				IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
46 
47 static void *ivpu_pgtable_alloc_page(struct ivpu_device *vdev, dma_addr_t *dma)
48 {
49 	dma_addr_t dma_addr;
50 	struct page *page;
51 	void *cpu;
52 
53 	page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
54 	if (!page)
55 		return NULL;
56 
57 	set_pages_array_wc(&page, 1);
58 
59 	dma_addr = dma_map_page(vdev->drm.dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
60 	if (dma_mapping_error(vdev->drm.dev, dma_addr))
61 		goto err_free_page;
62 
63 	cpu = vmap(&page, 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL));
64 	if (!cpu)
65 		goto err_dma_unmap_page;
66 
67 
68 	*dma = dma_addr;
69 	return cpu;
70 
71 err_dma_unmap_page:
72 	dma_unmap_page(vdev->drm.dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
73 
74 err_free_page:
75 	put_page(page);
76 	return NULL;
77 }
78 
79 static void ivpu_pgtable_free_page(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
80 {
81 	struct page *page;
82 
83 	if (cpu_addr) {
84 		page = vmalloc_to_page(cpu_addr);
85 		vunmap(cpu_addr);
86 		dma_unmap_page(vdev->drm.dev, dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK, PAGE_SIZE,
87 			       DMA_BIDIRECTIONAL);
88 		set_pages_array_wb(&page, 1);
89 		put_page(page);
90 	}
91 }
92 
93 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
94 {
95 	int pgd_idx, pud_idx, pmd_idx;
96 	dma_addr_t pud_dma, pmd_dma, pte_dma;
97 	u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr;
98 
99 	for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) {
100 		pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
101 		pud_dma = pgtable->pgd_dma_ptr[pgd_idx];
102 
103 		if (!pud_dma_ptr)
104 			continue;
105 
106 		for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) {
107 			pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
108 			pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];
109 
110 			if (!pmd_dma_ptr)
111 				continue;
112 
113 			for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) {
114 				pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
115 				pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];
116 
117 				ivpu_pgtable_free_page(vdev, pte_dma_ptr, pte_dma);
118 			}
119 
120 			kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);
121 			ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);
122 		}
123 
124 		kfree(pgtable->pmd_ptrs[pgd_idx]);
125 		kfree(pgtable->pte_ptrs[pgd_idx]);
126 		ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);
127 	}
128 
129 	ivpu_pgtable_free_page(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
130 	pgtable->pgd_dma_ptr = NULL;
131 	pgtable->pgd_dma = 0;
132 }
133 
134 static u64*
135 ivpu_mmu_ensure_pgd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
136 {
137 	u64 *pgd_dma_ptr = pgtable->pgd_dma_ptr;
138 	dma_addr_t pgd_dma;
139 
140 	if (pgd_dma_ptr)
141 		return pgd_dma_ptr;
142 
143 	pgd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pgd_dma);
144 	if (!pgd_dma_ptr)
145 		return NULL;
146 
147 	pgtable->pgd_dma_ptr = pgd_dma_ptr;
148 	pgtable->pgd_dma = pgd_dma;
149 
150 	return pgd_dma_ptr;
151 }
152 
153 static u64*
154 ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)
155 {
156 	u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
157 	dma_addr_t pud_dma;
158 
159 	if (pud_dma_ptr)
160 		return pud_dma_ptr;
161 
162 	pud_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pud_dma);
163 	if (!pud_dma_ptr)
164 		return NULL;
165 
166 	drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);
167 	pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
168 	if (!pgtable->pmd_ptrs[pgd_idx])
169 		goto err_free_pud_dma_ptr;
170 
171 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);
172 	pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
173 	if (!pgtable->pte_ptrs[pgd_idx])
174 		goto err_free_pmd_ptrs;
175 
176 	pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;
177 	pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;
178 
179 	return pud_dma_ptr;
180 
181 err_free_pmd_ptrs:
182 	kfree(pgtable->pmd_ptrs[pgd_idx]);
183 
184 err_free_pud_dma_ptr:
185 	ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);
186 	return NULL;
187 }
188 
189 static u64*
190 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,
191 		    int pud_idx)
192 {
193 	u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
194 	dma_addr_t pmd_dma;
195 
196 	if (pmd_dma_ptr)
197 		return pmd_dma_ptr;
198 
199 	pmd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pmd_dma);
200 	if (!pmd_dma_ptr)
201 		return NULL;
202 
203 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);
204 	pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
205 	if (!pgtable->pte_ptrs[pgd_idx][pud_idx])
206 		goto err_free_pmd_dma_ptr;
207 
208 	pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;
209 	pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;
210 
211 	return pmd_dma_ptr;
212 
213 err_free_pmd_dma_ptr:
214 	ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);
215 	return NULL;
216 }
217 
218 static u64*
219 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
220 		    int pgd_idx, int pud_idx, int pmd_idx)
221 {
222 	u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
223 	dma_addr_t pte_dma;
224 
225 	if (pte_dma_ptr)
226 		return pte_dma_ptr;
227 
228 	pte_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pte_dma);
229 	if (!pte_dma_ptr)
230 		return NULL;
231 
232 	pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;
233 	pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;
234 
235 	return pte_dma_ptr;
236 }
237 
238 static int
239 ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
240 			  u64 vpu_addr, dma_addr_t dma_addr, u64 prot)
241 {
242 	u64 *pte;
243 	int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
244 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
245 	int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
246 	int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
247 
248 	drm_WARN_ON(&vdev->drm, ctx->id == IVPU_RESERVED_CONTEXT_MMU_SSID);
249 
250 	/* Allocate PGD - first level page table if needed */
251 	if (!ivpu_mmu_ensure_pgd(vdev, &ctx->pgtable))
252 		return -ENOMEM;
253 
254 	/* Allocate PUD - second level page table if needed */
255 	if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))
256 		return -ENOMEM;
257 
258 	/* Allocate PMD - third level page table if needed */
259 	if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))
260 		return -ENOMEM;
261 
262 	/* Allocate PTE - fourth level page table if needed */
263 	pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);
264 	if (!pte)
265 		return -ENOMEM;
266 
267 	/* Update PTE */
268 	pte[pte_idx] = dma_addr | prot;
269 
270 	return 0;
271 }
272 
273 static int
274 ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
275 			      dma_addr_t dma_addr, u64 prot)
276 {
277 	size_t size = IVPU_MMU_CONT_PAGES_SIZE;
278 
279 	drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size));
280 	drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size));
281 
282 	prot |= IVPU_MMU_ENTRY_FLAG_CONT;
283 
284 	while (size) {
285 		int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
286 
287 		if (ret)
288 			return ret;
289 
290 		size -= IVPU_MMU_PAGE_SIZE;
291 		vpu_addr += IVPU_MMU_PAGE_SIZE;
292 		dma_addr += IVPU_MMU_PAGE_SIZE;
293 	}
294 
295 	return 0;
296 }
297 
298 static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)
299 {
300 	int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
301 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
302 	int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
303 	int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
304 
305 	/* Update PTE with dummy physical address and clear flags */
306 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
307 }
308 
309 static int
310 ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
311 			   u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot)
312 {
313 	int map_size;
314 	int ret;
315 
316 	while (size) {
317 		if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE &&
318 		    IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) {
319 			ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot);
320 			map_size = IVPU_MMU_CONT_PAGES_SIZE;
321 		} else {
322 			ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
323 			map_size = IVPU_MMU_PAGE_SIZE;
324 		}
325 
326 		if (ret)
327 			return ret;
328 
329 		vpu_addr += map_size;
330 		dma_addr += map_size;
331 		size -= map_size;
332 	}
333 
334 	return 0;
335 }
336 
337 static void ivpu_mmu_context_set_page_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
338 					 u64 vpu_addr)
339 {
340 	int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
341 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
342 	int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
343 	int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
344 
345 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] |= IVPU_MMU_ENTRY_FLAG_RO;
346 }
347 
348 static void ivpu_mmu_context_split_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
349 					u64 vpu_addr)
350 {
351 	int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
352 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
353 	int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
354 	int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
355 
356 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] &= ~IVPU_MMU_ENTRY_FLAG_CONT;
357 }
358 
359 static void ivpu_mmu_context_split_64k_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
360 					    u64 vpu_addr)
361 {
362 	u64 start = ALIGN_DOWN(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE);
363 	u64 end = ALIGN(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE);
364 	u64 offset = 0;
365 
366 	ivpu_dbg(vdev, MMU_MAP, "Split 64K page ctx: %u vpu_addr: 0x%llx\n", ctx->id, vpu_addr);
367 
368 	while (start + offset < end) {
369 		ivpu_mmu_context_split_page(vdev, ctx, start + offset);
370 		offset += IVPU_MMU_PAGE_SIZE;
371 	}
372 }
373 
374 int
375 ivpu_mmu_context_set_pages_ro(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
376 			      size_t size)
377 {
378 	u64 end = vpu_addr + size;
379 	size_t size_left = size;
380 	int ret;
381 
382 	if (size == 0)
383 		return 0;
384 
385 	if (drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr | size, IVPU_MMU_PAGE_SIZE)))
386 		return -EINVAL;
387 
388 	mutex_lock(&ctx->lock);
389 
390 	ivpu_dbg(vdev, MMU_MAP, "Set read-only pages ctx: %u vpu_addr: 0x%llx size: %lu\n",
391 		 ctx->id, vpu_addr, size);
392 
393 	if (!ivpu_disable_mmu_cont_pages) {
394 		/* Split 64K contiguous page at the beginning if needed */
395 		if (!IS_ALIGNED(vpu_addr, IVPU_MMU_CONT_PAGES_SIZE))
396 			ivpu_mmu_context_split_64k_page(vdev, ctx, vpu_addr);
397 
398 		/* Split 64K contiguous page at the end if needed */
399 		if (!IS_ALIGNED(vpu_addr + size, IVPU_MMU_CONT_PAGES_SIZE))
400 			ivpu_mmu_context_split_64k_page(vdev, ctx, vpu_addr + size);
401 	}
402 
403 	while (size_left) {
404 		if (vpu_addr < end)
405 			ivpu_mmu_context_set_page_ro(vdev, ctx, vpu_addr);
406 
407 		vpu_addr += IVPU_MMU_PAGE_SIZE;
408 		size_left -= IVPU_MMU_PAGE_SIZE;
409 	}
410 
411 	/* Ensure page table modifications are flushed from wc buffers to memory */
412 	wmb();
413 
414 	mutex_unlock(&ctx->lock);
415 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
416 	if (ret)
417 		ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
418 
419 	return 0;
420 }
421 
422 static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
423 {
424 	while (size) {
425 		ivpu_mmu_context_unmap_page(ctx, vpu_addr);
426 		vpu_addr += IVPU_MMU_PAGE_SIZE;
427 		size -= IVPU_MMU_PAGE_SIZE;
428 	}
429 }
430 
431 int
432 ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
433 			 struct sg_table *sgt, size_t bo_size, bool llc_coherent, bool read_only)
434 {
435 	size_t start_vpu_addr = vpu_addr;
436 	struct scatterlist *sg;
437 	size_t sgt_size = 0;
438 	int ret;
439 	u64 prot;
440 	u64 i;
441 
442 	if (drm_WARN_ON(&vdev->drm, !ctx))
443 		return -EINVAL;
444 
445 	if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
446 		return -EINVAL;
447 
448 	if (vpu_addr & ~IVPU_MMU_VPU_ADDRESS_MASK)
449 		return -EINVAL;
450 
451 	prot = IVPU_MMU_ENTRY_MAPPED;
452 	if (llc_coherent)
453 		prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
454 	if (read_only)
455 		prot |= IVPU_MMU_ENTRY_FLAG_RO;
456 
457 	mutex_lock(&ctx->lock);
458 
459 	for_each_sgtable_dma_sg(sgt, sg, i) {
460 		dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;
461 		size_t size = sg_dma_len(sg) + sg->offset;
462 
463 		ivpu_dbg(vdev, MMU_MAP, "Map ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n",
464 			 ctx->id, dma_addr, vpu_addr, size);
465 
466 		if (sgt_size + size > bo_size) {
467 			ivpu_err(vdev, "Scatter-gather table size exceeds buffer object size\n");
468 			ret = -EINVAL;
469 			goto err_unmap_pages;
470 		}
471 
472 		ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
473 		if (ret) {
474 			ivpu_err(vdev, "Failed to map context pages\n");
475 			goto err_unmap_pages;
476 		}
477 		vpu_addr += size;
478 		sgt_size += size;
479 	}
480 
481 	if (sgt_size < bo_size) {
482 		ivpu_err(vdev, "Scatter-gather table size too small to cover buffer object size\n");
483 		ret = -EINVAL;
484 		goto err_unmap_pages;
485 	}
486 
487 	if (!ctx->is_cd_valid) {
488 		ret = ivpu_mmu_cd_set(vdev, ctx->id, &ctx->pgtable);
489 		if (ret) {
490 			ivpu_err(vdev, "Failed to set context descriptor for context %u: %d\n",
491 				 ctx->id, ret);
492 			goto err_unmap_pages;
493 		}
494 		ctx->is_cd_valid = true;
495 	}
496 
497 	/* Ensure page table modifications are flushed from wc buffers to memory */
498 	wmb();
499 
500 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
501 	if (ret) {
502 		ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
503 		goto err_unmap_pages;
504 	}
505 
506 	mutex_unlock(&ctx->lock);
507 	return 0;
508 
509 err_unmap_pages:
510 	ivpu_mmu_context_unmap_pages(ctx, start_vpu_addr, sgt_size);
511 	mutex_unlock(&ctx->lock);
512 	return ret;
513 }
514 
515 void
516 ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
517 			   u64 vpu_addr, struct sg_table *sgt)
518 {
519 	struct scatterlist *sg;
520 	int ret;
521 	u64 i;
522 
523 	if (drm_WARN_ON(&vdev->drm, !ctx))
524 		return;
525 
526 	mutex_lock(&ctx->lock);
527 
528 	for_each_sgtable_dma_sg(sgt, sg, i) {
529 		dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;
530 		size_t size = sg_dma_len(sg) + sg->offset;
531 
532 		ivpu_dbg(vdev, MMU_MAP, "Unmap ctx: %u dma_addr: 0x%llx vpu_addr: 0x%llx size: %lu\n",
533 			 ctx->id, dma_addr, vpu_addr, size);
534 
535 		ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
536 		vpu_addr += size;
537 	}
538 
539 	/* Ensure page table modifications are flushed from wc buffers to memory */
540 	wmb();
541 
542 	mutex_unlock(&ctx->lock);
543 
544 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
545 	if (ret)
546 		ivpu_warn_ratelimited(vdev, "Failed to invalidate TLB for ctx %u: %d\n",
547 				      ctx->id, ret);
548 }
549 
550 int
551 ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range,
552 			     u64 size, struct drm_mm_node *node)
553 {
554 	int ret;
555 
556 	WARN_ON(!range);
557 
558 	mutex_lock(&ctx->lock);
559 	if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) {
560 		ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,
561 						  range->start, range->end, DRM_MM_INSERT_BEST);
562 		if (!ret)
563 			goto unlock;
564 	}
565 
566 	ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,
567 					  range->start, range->end, DRM_MM_INSERT_BEST);
568 unlock:
569 	mutex_unlock(&ctx->lock);
570 	return ret;
571 }
572 
573 void
574 ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
575 {
576 	mutex_lock(&ctx->lock);
577 	drm_mm_remove_node(node);
578 	mutex_unlock(&ctx->lock);
579 }
580 
581 void ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id)
582 {
583 	u64 start, end;
584 
585 	mutex_init(&ctx->lock);
586 
587 	if (!context_id) {
588 		start = vdev->hw->ranges.runtime.start;
589 		end = vdev->hw->ranges.shave.end;
590 	} else {
591 		start = min_t(u64, vdev->hw->ranges.user.start, vdev->hw->ranges.shave.start);
592 		end = max_t(u64, vdev->hw->ranges.user.end, vdev->hw->ranges.dma.end);
593 	}
594 
595 	drm_mm_init(&ctx->mm, start, end - start);
596 	ctx->id = context_id;
597 }
598 
599 void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
600 {
601 	if (ctx->is_cd_valid) {
602 		ivpu_mmu_cd_clear(vdev, ctx->id);
603 		ctx->is_cd_valid = false;
604 	}
605 
606 	mutex_destroy(&ctx->lock);
607 	ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);
608 	drm_mm_takedown(&ctx->mm);
609 }
610 
611 void ivpu_mmu_global_context_init(struct ivpu_device *vdev)
612 {
613 	ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);
614 }
615 
616 void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)
617 {
618 	ivpu_mmu_context_fini(vdev, &vdev->gctx);
619 }
620 
621 int ivpu_mmu_reserved_context_init(struct ivpu_device *vdev)
622 {
623 	int ret;
624 
625 	ivpu_mmu_context_init(vdev, &vdev->rctx, IVPU_RESERVED_CONTEXT_MMU_SSID);
626 
627 	mutex_lock(&vdev->rctx.lock);
628 
629 	if (!ivpu_mmu_ensure_pgd(vdev, &vdev->rctx.pgtable)) {
630 		ivpu_err(vdev, "Failed to allocate root page table for reserved context\n");
631 		ret = -ENOMEM;
632 		goto err_ctx_fini;
633 	}
634 
635 	ret = ivpu_mmu_cd_set(vdev, vdev->rctx.id, &vdev->rctx.pgtable);
636 	if (ret) {
637 		ivpu_err(vdev, "Failed to set context descriptor for reserved context\n");
638 		goto err_ctx_fini;
639 	}
640 
641 	mutex_unlock(&vdev->rctx.lock);
642 	return ret;
643 
644 err_ctx_fini:
645 	mutex_unlock(&vdev->rctx.lock);
646 	ivpu_mmu_context_fini(vdev, &vdev->rctx);
647 	return ret;
648 }
649 
650 void ivpu_mmu_reserved_context_fini(struct ivpu_device *vdev)
651 {
652 	ivpu_mmu_cd_clear(vdev, vdev->rctx.id);
653 	ivpu_mmu_context_fini(vdev, &vdev->rctx);
654 }
655