xref: /linux/drivers/accel/ivpu/ivpu_mmu_context.c (revision fd7d598270724cc787982ea48bbe17ad383a8b7f) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/bitfield.h>
7 #include <linux/highmem.h>
8 
9 #include "ivpu_drv.h"
10 #include "ivpu_hw.h"
11 #include "ivpu_mmu.h"
12 #include "ivpu_mmu_context.h"
13 
14 #define IVPU_MMU_VPU_ADDRESS_MASK        GENMASK(47, 12)
15 #define IVPU_MMU_PGD_INDEX_MASK          GENMASK(47, 39)
16 #define IVPU_MMU_PUD_INDEX_MASK          GENMASK(38, 30)
17 #define IVPU_MMU_PMD_INDEX_MASK          GENMASK(29, 21)
18 #define IVPU_MMU_PTE_INDEX_MASK          GENMASK(20, 12)
19 #define IVPU_MMU_ENTRY_FLAGS_MASK        (BIT(52) | GENMASK(11, 0))
20 #define IVPU_MMU_ENTRY_FLAG_CONT         BIT(52)
21 #define IVPU_MMU_ENTRY_FLAG_NG           BIT(11)
22 #define IVPU_MMU_ENTRY_FLAG_AF           BIT(10)
23 #define IVPU_MMU_ENTRY_FLAG_USER         BIT(6)
24 #define IVPU_MMU_ENTRY_FLAG_LLC_COHERENT BIT(2)
25 #define IVPU_MMU_ENTRY_FLAG_TYPE_PAGE    BIT(1)
26 #define IVPU_MMU_ENTRY_FLAG_VALID        BIT(0)
27 
28 #define IVPU_MMU_PAGE_SIZE       SZ_4K
29 #define IVPU_MMU_CONT_PAGES_SIZE (IVPU_MMU_PAGE_SIZE * 16)
30 #define IVPU_MMU_PTE_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PAGE_SIZE)
31 #define IVPU_MMU_PMD_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PTE_MAP_SIZE)
32 #define IVPU_MMU_PUD_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PMD_MAP_SIZE)
33 #define IVPU_MMU_PGD_MAP_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * IVPU_MMU_PUD_MAP_SIZE)
34 #define IVPU_MMU_PGTABLE_SIZE    (IVPU_MMU_PGTABLE_ENTRIES * sizeof(u64))
35 
36 #define IVPU_MMU_DUMMY_ADDRESS 0xdeadb000
37 #define IVPU_MMU_ENTRY_VALID   (IVPU_MMU_ENTRY_FLAG_TYPE_PAGE | IVPU_MMU_ENTRY_FLAG_VALID)
38 #define IVPU_MMU_ENTRY_INVALID (IVPU_MMU_DUMMY_ADDRESS & ~IVPU_MMU_ENTRY_FLAGS_MASK)
39 #define IVPU_MMU_ENTRY_MAPPED  (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
40 				IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
41 
42 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
43 {
44 	dma_addr_t pgd_dma;
45 
46 	pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma,
47 						  GFP_KERNEL);
48 	if (!pgtable->pgd_dma_ptr)
49 		return -ENOMEM;
50 
51 	pgtable->pgd_dma = pgd_dma;
52 
53 	return 0;
54 }
55 
56 static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
57 {
58 	if (cpu_addr)
59 		dma_free_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, cpu_addr,
60 				  dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK);
61 }
62 
63 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
64 {
65 	int pgd_idx, pud_idx, pmd_idx;
66 	dma_addr_t pud_dma, pmd_dma, pte_dma;
67 	u64 *pud_dma_ptr, *pmd_dma_ptr, *pte_dma_ptr;
68 
69 	for (pgd_idx = 0; pgd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pgd_idx) {
70 		pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
71 		pud_dma = pgtable->pgd_dma_ptr[pgd_idx];
72 
73 		if (!pud_dma_ptr)
74 			continue;
75 
76 		for (pud_idx = 0; pud_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pud_idx) {
77 			pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
78 			pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];
79 
80 			if (!pmd_dma_ptr)
81 				continue;
82 
83 			for (pmd_idx = 0; pmd_idx < IVPU_MMU_PGTABLE_ENTRIES; ++pmd_idx) {
84 				pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
85 				pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];
86 
87 				ivpu_mmu_pgtable_free(vdev, pte_dma_ptr, pte_dma);
88 			}
89 
90 			kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);
91 			ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
92 		}
93 
94 		kfree(pgtable->pmd_ptrs[pgd_idx]);
95 		kfree(pgtable->pte_ptrs[pgd_idx]);
96 		ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
97 	}
98 
99 	ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
100 }
101 
102 static u64*
103 ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)
104 {
105 	u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];
106 	dma_addr_t pud_dma;
107 
108 	if (pud_dma_ptr)
109 		return pud_dma_ptr;
110 
111 	pud_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL);
112 	if (!pud_dma_ptr)
113 		return NULL;
114 
115 	drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);
116 	pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
117 	if (!pgtable->pmd_ptrs[pgd_idx])
118 		goto err_free_pud_dma_ptr;
119 
120 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);
121 	pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
122 	if (!pgtable->pte_ptrs[pgd_idx])
123 		goto err_free_pmd_ptrs;
124 
125 	pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;
126 	pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;
127 
128 	return pud_dma_ptr;
129 
130 err_free_pmd_ptrs:
131 	kfree(pgtable->pmd_ptrs[pgd_idx]);
132 
133 err_free_pud_dma_ptr:
134 	ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
135 	return NULL;
136 }
137 
138 static u64*
139 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,
140 		    int pud_idx)
141 {
142 	u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];
143 	dma_addr_t pmd_dma;
144 
145 	if (pmd_dma_ptr)
146 		return pmd_dma_ptr;
147 
148 	pmd_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
149 	if (!pmd_dma_ptr)
150 		return NULL;
151 
152 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);
153 	pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);
154 	if (!pgtable->pte_ptrs[pgd_idx][pud_idx])
155 		goto err_free_pmd_dma_ptr;
156 
157 	pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;
158 	pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;
159 
160 	return pmd_dma_ptr;
161 
162 err_free_pmd_dma_ptr:
163 	ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
164 	return NULL;
165 }
166 
167 static u64*
168 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
169 		    int pgd_idx, int pud_idx, int pmd_idx)
170 {
171 	u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
172 	dma_addr_t pte_dma;
173 
174 	if (pte_dma_ptr)
175 		return pte_dma_ptr;
176 
177 	pte_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
178 	if (!pte_dma_ptr)
179 		return NULL;
180 
181 	pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;
182 	pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;
183 
184 	return pte_dma_ptr;
185 }
186 
187 static int
188 ivpu_mmu_context_map_page(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
189 			  u64 vpu_addr, dma_addr_t dma_addr, u64 prot)
190 {
191 	u64 *pte;
192 	int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
193 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
194 	int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
195 	int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
196 
197 	/* Allocate PUD - second level page table if needed */
198 	if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))
199 		return -ENOMEM;
200 
201 	/* Allocate PMD - third level page table if needed */
202 	if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))
203 		return -ENOMEM;
204 
205 	/* Allocate PTE - fourth level page table if needed */
206 	pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);
207 	if (!pte)
208 		return -ENOMEM;
209 
210 	/* Update PTE */
211 	pte[pte_idx] = dma_addr | prot;
212 
213 	return 0;
214 }
215 
216 static int
217 ivpu_mmu_context_map_cont_64k(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u64 vpu_addr,
218 			      dma_addr_t dma_addr, u64 prot)
219 {
220 	size_t size = IVPU_MMU_CONT_PAGES_SIZE;
221 
222 	drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vpu_addr, size));
223 	drm_WARN_ON(&vdev->drm, !IS_ALIGNED(dma_addr, size));
224 
225 	prot |= IVPU_MMU_ENTRY_FLAG_CONT;
226 
227 	while (size) {
228 		int ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
229 
230 		if (ret)
231 			return ret;
232 
233 		size -= IVPU_MMU_PAGE_SIZE;
234 		vpu_addr += IVPU_MMU_PAGE_SIZE;
235 		dma_addr += IVPU_MMU_PAGE_SIZE;
236 	}
237 
238 	return 0;
239 }
240 
241 static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_addr)
242 {
243 	int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
244 	int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
245 	int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
246 	int pte_idx = FIELD_GET(IVPU_MMU_PTE_INDEX_MASK, vpu_addr);
247 
248 	/* Update PTE with dummy physical address and clear flags */
249 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
250 }
251 
252 static void
253 ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
254 {
255 	struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable;
256 	u64 end_addr = vpu_addr + size;
257 
258 	/* Align to PMD entry (2 MB) */
259 	vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
260 
261 	while (vpu_addr < end_addr) {
262 		int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
263 		u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE;
264 
265 		while (vpu_addr < end_addr && vpu_addr < pud_end) {
266 			int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
267 			u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
268 
269 			while (vpu_addr < end_addr && vpu_addr < pmd_end) {
270 				int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
271 
272 				clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx],
273 						    IVPU_MMU_PGTABLE_SIZE);
274 				vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
275 			}
276 			clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx],
277 					    IVPU_MMU_PGTABLE_SIZE);
278 		}
279 		clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE);
280 	}
281 	clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE);
282 }
283 
284 static int
285 ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
286 			   u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot)
287 {
288 	int map_size;
289 	int ret;
290 
291 	while (size) {
292 		if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE &&
293 		    IS_ALIGNED(vpu_addr | dma_addr, IVPU_MMU_CONT_PAGES_SIZE)) {
294 			ret = ivpu_mmu_context_map_cont_64k(vdev, ctx, vpu_addr, dma_addr, prot);
295 			map_size = IVPU_MMU_CONT_PAGES_SIZE;
296 		} else {
297 			ret = ivpu_mmu_context_map_page(vdev, ctx, vpu_addr, dma_addr, prot);
298 			map_size = IVPU_MMU_PAGE_SIZE;
299 		}
300 
301 		if (ret)
302 			return ret;
303 
304 		vpu_addr += map_size;
305 		dma_addr += map_size;
306 		size -= map_size;
307 	}
308 
309 	return 0;
310 }
311 
312 static void ivpu_mmu_context_unmap_pages(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
313 {
314 	while (size) {
315 		ivpu_mmu_context_unmap_page(ctx, vpu_addr);
316 		vpu_addr += IVPU_MMU_PAGE_SIZE;
317 		size -= IVPU_MMU_PAGE_SIZE;
318 	}
319 }
320 
321 int
322 ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
323 			 u64 vpu_addr, struct sg_table *sgt,  bool llc_coherent)
324 {
325 	struct scatterlist *sg;
326 	int ret;
327 	u64 prot;
328 	u64 i;
329 
330 	if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
331 		return -EINVAL;
332 
333 	if (vpu_addr & ~IVPU_MMU_VPU_ADDRESS_MASK)
334 		return -EINVAL;
335 
336 	prot = IVPU_MMU_ENTRY_MAPPED;
337 	if (llc_coherent)
338 		prot |= IVPU_MMU_ENTRY_FLAG_LLC_COHERENT;
339 
340 	mutex_lock(&ctx->lock);
341 
342 	for_each_sgtable_dma_sg(sgt, sg, i) {
343 		dma_addr_t dma_addr = sg_dma_address(sg) - sg->offset;
344 		size_t size = sg_dma_len(sg) + sg->offset;
345 
346 		ret = ivpu_mmu_context_map_pages(vdev, ctx, vpu_addr, dma_addr, size, prot);
347 		if (ret) {
348 			ivpu_err(vdev, "Failed to map context pages\n");
349 			mutex_unlock(&ctx->lock);
350 			return ret;
351 		}
352 		ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
353 		vpu_addr += size;
354 	}
355 
356 	mutex_unlock(&ctx->lock);
357 
358 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
359 	if (ret)
360 		ivpu_err(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
361 	return ret;
362 }
363 
364 void
365 ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
366 			   u64 vpu_addr, struct sg_table *sgt)
367 {
368 	struct scatterlist *sg;
369 	int ret;
370 	u64 i;
371 
372 	if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
373 		ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr);
374 
375 	mutex_lock(&ctx->lock);
376 
377 	for_each_sgtable_dma_sg(sgt, sg, i) {
378 		size_t size = sg_dma_len(sg) + sg->offset;
379 
380 		ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
381 		ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
382 		vpu_addr += size;
383 	}
384 
385 	mutex_unlock(&ctx->lock);
386 
387 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
388 	if (ret)
389 		ivpu_warn(vdev, "Failed to invalidate TLB for ctx %u: %d\n", ctx->id, ret);
390 }
391 
392 int
393 ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
394 				    const struct ivpu_addr_range *range,
395 				    u64 size, struct drm_mm_node *node)
396 {
397 	lockdep_assert_held(&ctx->lock);
398 
399 	if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) {
400 		if (!drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,
401 						 range->start, range->end, DRM_MM_INSERT_BEST))
402 			return 0;
403 	}
404 
405 	return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,
406 					   range->start, range->end, DRM_MM_INSERT_BEST);
407 }
408 
409 void
410 ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
411 {
412 	lockdep_assert_held(&ctx->lock);
413 
414 	drm_mm_remove_node(node);
415 }
416 
417 static int
418 ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 context_id)
419 {
420 	u64 start, end;
421 	int ret;
422 
423 	mutex_init(&ctx->lock);
424 	INIT_LIST_HEAD(&ctx->bo_list);
425 
426 	ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);
427 	if (ret) {
428 		ivpu_err(vdev, "Failed to initialize pgtable for ctx %u: %d\n", context_id, ret);
429 		return ret;
430 	}
431 
432 	if (!context_id) {
433 		start = vdev->hw->ranges.global.start;
434 		end = vdev->hw->ranges.shave.end;
435 	} else {
436 		start = vdev->hw->ranges.user.start;
437 		end = vdev->hw->ranges.dma.end;
438 	}
439 
440 	drm_mm_init(&ctx->mm, start, end - start);
441 	ctx->id = context_id;
442 
443 	return 0;
444 }
445 
446 static void ivpu_mmu_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
447 {
448 	if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr))
449 		return;
450 
451 	mutex_destroy(&ctx->lock);
452 	ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);
453 	drm_mm_takedown(&ctx->mm);
454 
455 	ctx->pgtable.pgd_dma_ptr = NULL;
456 	ctx->pgtable.pgd_dma = 0;
457 }
458 
459 int ivpu_mmu_global_context_init(struct ivpu_device *vdev)
460 {
461 	return ivpu_mmu_context_init(vdev, &vdev->gctx, IVPU_GLOBAL_CONTEXT_MMU_SSID);
462 }
463 
464 void ivpu_mmu_global_context_fini(struct ivpu_device *vdev)
465 {
466 	return ivpu_mmu_context_fini(vdev, &vdev->gctx);
467 }
468 
469 int ivpu_mmu_reserved_context_init(struct ivpu_device *vdev)
470 {
471 	return ivpu_mmu_user_context_init(vdev, &vdev->rctx, IVPU_RESERVED_CONTEXT_MMU_SSID);
472 }
473 
474 void ivpu_mmu_reserved_context_fini(struct ivpu_device *vdev)
475 {
476 	return ivpu_mmu_user_context_fini(vdev, &vdev->rctx);
477 }
478 
479 void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid)
480 {
481 	struct ivpu_file_priv *file_priv;
482 
483 	xa_lock(&vdev->context_xa);
484 
485 	file_priv = xa_load(&vdev->context_xa, ssid);
486 	if (file_priv)
487 		file_priv->has_mmu_faults = true;
488 
489 	xa_unlock(&vdev->context_xa);
490 }
491 
492 int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u32 ctx_id)
493 {
494 	int ret;
495 
496 	drm_WARN_ON(&vdev->drm, !ctx_id);
497 
498 	ret = ivpu_mmu_context_init(vdev, ctx, ctx_id);
499 	if (ret) {
500 		ivpu_err(vdev, "Failed to initialize context %u: %d\n", ctx_id, ret);
501 		return ret;
502 	}
503 
504 	ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);
505 	if (ret) {
506 		ivpu_err(vdev, "Failed to set page table for context %u: %d\n", ctx_id, ret);
507 		goto err_context_fini;
508 	}
509 
510 	return 0;
511 
512 err_context_fini:
513 	ivpu_mmu_context_fini(vdev, ctx);
514 	return ret;
515 }
516 
517 void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
518 {
519 	drm_WARN_ON(&vdev->drm, !ctx->id);
520 
521 	ivpu_mmu_clear_pgtable(vdev, ctx->id);
522 	ivpu_mmu_context_fini(vdev, ctx);
523 }
524