xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <drm/drm_drv.h>
25 
26 #include "amdgpu.h"
27 #include "amdgpu_trace.h"
28 #include "amdgpu_vm.h"
29 #include "amdgpu_vm_internal.h"
30 #include "amdgpu_job.h"
31 
32 /*
33  * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
34  */
35 struct amdgpu_vm_pt_cursor {
36 	uint64_t pfn;
37 	struct amdgpu_vm_bo_base *parent;
38 	struct amdgpu_vm_bo_base *entry;
39 	unsigned int level;
40 };
41 
42 /**
43  * amdgpu_vm_pt_level_shift - return the addr shift for each level
44  *
45  * @adev: amdgpu_device pointer
46  * @level: VMPT level
47  *
48  * Returns:
49  * The number of bits the pfn needs to be right shifted for a level.
50  */
51 static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev,
52 					     unsigned int level)
53 {
54 	switch (level) {
55 	case AMDGPU_VM_PDB3:
56 	case AMDGPU_VM_PDB2:
57 	case AMDGPU_VM_PDB1:
58 	case AMDGPU_VM_PDB0:
59 		return 9 * (AMDGPU_VM_PDB0 - level) +
60 			adev->vm_manager.block_size;
61 	case AMDGPU_VM_PTB:
62 		return 0;
63 	default:
64 		return ~0;
65 	}
66 }
67 
68 /**
69  * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT
70  *
71  * @adev: amdgpu_device pointer
72  * @level: VMPT level
73  *
74  * Returns:
75  * The number of entries in a page directory or page table.
76  */
77 static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev,
78 					     unsigned int level)
79 {
80 	unsigned int shift;
81 
82 	shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level);
83 	if (level == adev->vm_manager.root_level)
84 		/* For the root directory */
85 		return round_up(adev->vm_manager.max_pfn, 1ULL << shift)
86 			>> shift;
87 	else if (level != AMDGPU_VM_PTB)
88 		/* Everything in between */
89 		return 512;
90 
91 	/* For the page tables on the leaves */
92 	return AMDGPU_VM_PTE_COUNT(adev);
93 }
94 
95 /**
96  * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT
97  *
98  * @adev: amdgpu_device pointer
99  * @level: VMPT level
100  *
101  * Returns:
102  * The mask to extract the entry number of a PD/PT from an address.
103  */
104 static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev,
105 					  unsigned int level)
106 {
107 	if (level <= adev->vm_manager.root_level)
108 		return 0xffffffff;
109 	else if (level != AMDGPU_VM_PTB)
110 		return 0x1ff;
111 	else
112 		return AMDGPU_VM_PTE_COUNT(adev) - 1;
113 }
114 
115 /**
116  * amdgpu_vm_pt_size - returns the size of the page table in bytes
117  *
118  * @adev: amdgpu_device pointer
119  * @level: VMPT level
120  *
121  * Returns:
122  * The size of the BO for a page directory or page table in bytes.
123  */
124 static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev,
125 				      unsigned int level)
126 {
127 	return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8);
128 }
129 
130 /**
131  * amdgpu_vm_pt_parent - get the parent page directory
132  *
133  * @pt: child page table
134  *
135  * Helper to get the parent entry for the child page table. NULL if we are at
136  * the root page directory.
137  */
138 static struct amdgpu_vm_bo_base *
139 amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt)
140 {
141 	struct amdgpu_bo *parent = pt->bo->parent;
142 
143 	if (!parent)
144 		return NULL;
145 
146 	return parent->vm_bo;
147 }
148 
149 /**
150  * amdgpu_vm_pt_start - start PD/PT walk
151  *
152  * @adev: amdgpu_device pointer
153  * @vm: amdgpu_vm structure
154  * @start: start address of the walk
155  * @cursor: state to initialize
156  *
157  * Initialize a amdgpu_vm_pt_cursor to start a walk.
158  */
159 static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
160 			       struct amdgpu_vm *vm, uint64_t start,
161 			       struct amdgpu_vm_pt_cursor *cursor)
162 {
163 	cursor->pfn = start;
164 	cursor->parent = NULL;
165 	cursor->entry = &vm->root;
166 	cursor->level = adev->vm_manager.root_level;
167 }
168 
169 /**
170  * amdgpu_vm_pt_descendant - go to child node
171  *
172  * @adev: amdgpu_device pointer
173  * @cursor: current state
174  *
175  * Walk to the child node of the current node.
176  * Returns:
177  * True if the walk was possible, false otherwise.
178  */
179 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
180 				    struct amdgpu_vm_pt_cursor *cursor)
181 {
182 	unsigned int mask, shift, idx;
183 
184 	if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry ||
185 	    !cursor->entry->bo)
186 		return false;
187 
188 	mask = amdgpu_vm_pt_entries_mask(adev, cursor->level);
189 	shift = amdgpu_vm_pt_level_shift(adev, cursor->level);
190 
191 	++cursor->level;
192 	idx = (cursor->pfn >> shift) & mask;
193 	cursor->parent = cursor->entry;
194 	cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx];
195 	return true;
196 }
197 
198 /**
199  * amdgpu_vm_pt_sibling - go to sibling node
200  *
201  * @adev: amdgpu_device pointer
202  * @cursor: current state
203  *
204  * Walk to the sibling node of the current node.
205  * Returns:
206  * True if the walk was possible, false otherwise.
207  */
208 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
209 				 struct amdgpu_vm_pt_cursor *cursor)
210 {
211 
212 	unsigned int shift, num_entries;
213 	struct amdgpu_bo_vm *parent;
214 
215 	/* Root doesn't have a sibling */
216 	if (!cursor->parent)
217 		return false;
218 
219 	/* Go to our parents and see if we got a sibling */
220 	shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1);
221 	num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1);
222 	parent = to_amdgpu_bo_vm(cursor->parent->bo);
223 
224 	if (cursor->entry == &parent->entries[num_entries - 1])
225 		return false;
226 
227 	cursor->pfn += 1ULL << shift;
228 	cursor->pfn &= ~((1ULL << shift) - 1);
229 	++cursor->entry;
230 	return true;
231 }
232 
233 /**
234  * amdgpu_vm_pt_ancestor - go to parent node
235  *
236  * @cursor: current state
237  *
238  * Walk to the parent node of the current node.
239  * Returns:
240  * True if the walk was possible, false otherwise.
241  */
242 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
243 {
244 	if (!cursor->parent)
245 		return false;
246 
247 	--cursor->level;
248 	cursor->entry = cursor->parent;
249 	cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
250 	return true;
251 }
252 
253 /**
254  * amdgpu_vm_pt_next - get next PD/PT in hieratchy
255  *
256  * @adev: amdgpu_device pointer
257  * @cursor: current state
258  *
259  * Walk the PD/PT tree to the next node.
260  */
261 static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
262 			      struct amdgpu_vm_pt_cursor *cursor)
263 {
264 	/* First try a newborn child */
265 	if (amdgpu_vm_pt_descendant(adev, cursor))
266 		return;
267 
268 	/* If that didn't worked try to find a sibling */
269 	while (!amdgpu_vm_pt_sibling(adev, cursor)) {
270 		/* No sibling, go to our parents and grandparents */
271 		if (!amdgpu_vm_pt_ancestor(cursor)) {
272 			cursor->pfn = ~0ll;
273 			return;
274 		}
275 	}
276 }
277 
278 /**
279  * amdgpu_vm_pt_first_dfs - start a deep first search
280  *
281  * @adev: amdgpu_device structure
282  * @vm: amdgpu_vm structure
283  * @start: optional cursor to start with
284  * @cursor: state to initialize
285  *
286  * Starts a deep first traversal of the PD/PT tree.
287  */
288 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
289 				   struct amdgpu_vm *vm,
290 				   struct amdgpu_vm_pt_cursor *start,
291 				   struct amdgpu_vm_pt_cursor *cursor)
292 {
293 	if (start)
294 		*cursor = *start;
295 	else
296 		amdgpu_vm_pt_start(adev, vm, 0, cursor);
297 
298 	while (amdgpu_vm_pt_descendant(adev, cursor))
299 		;
300 }
301 
302 /**
303  * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
304  *
305  * @start: starting point for the search
306  * @entry: current entry
307  *
308  * Returns:
309  * True when the search should continue, false otherwise.
310  */
311 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
312 				      struct amdgpu_vm_bo_base *entry)
313 {
314 	return entry && (!start || entry != start->entry);
315 }
316 
317 /**
318  * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
319  *
320  * @adev: amdgpu_device structure
321  * @cursor: current state
322  *
323  * Move the cursor to the next node in a deep first search.
324  */
325 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
326 				  struct amdgpu_vm_pt_cursor *cursor)
327 {
328 	if (!cursor->entry)
329 		return;
330 
331 	if (!cursor->parent)
332 		cursor->entry = NULL;
333 	else if (amdgpu_vm_pt_sibling(adev, cursor))
334 		while (amdgpu_vm_pt_descendant(adev, cursor))
335 			;
336 	else
337 		amdgpu_vm_pt_ancestor(cursor);
338 }
339 
340 /*
341  * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
342  */
343 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)		\
344 	for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)),		\
345 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
346 	     amdgpu_vm_pt_continue_dfs((start), (entry));			\
347 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
348 
349 /**
350  * amdgpu_vm_pt_clear - initially clear the PDs/PTs
351  *
352  * @adev: amdgpu_device pointer
353  * @vm: VM to clear BO from
354  * @vmbo: BO to clear
355  * @immediate: use an immediate update
356  *
357  * Root PD needs to be reserved when calling this.
358  *
359  * Returns:
360  * 0 on success, errno otherwise.
361  */
362 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
363 		       struct amdgpu_bo_vm *vmbo, bool immediate)
364 {
365 	unsigned int level = adev->vm_manager.root_level;
366 	struct ttm_operation_ctx ctx = { true, false };
367 	struct amdgpu_vm_update_params params;
368 	struct amdgpu_bo *ancestor = &vmbo->bo;
369 	unsigned int entries;
370 	struct amdgpu_bo *bo = &vmbo->bo;
371 	uint64_t value = 0, flags = 0;
372 	uint64_t addr;
373 	int r, idx;
374 
375 	/* Figure out our place in the hierarchy */
376 	if (ancestor->parent) {
377 		++level;
378 		while (ancestor->parent->parent) {
379 			++level;
380 			ancestor = ancestor->parent;
381 		}
382 	}
383 
384 	entries = amdgpu_bo_size(bo) / 8;
385 
386 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
387 	if (r)
388 		return r;
389 
390 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
391 		return -ENODEV;
392 
393 	r = vm->update_funcs->map_table(vmbo);
394 	if (r)
395 		goto exit;
396 
397 	memset(&params, 0, sizeof(params));
398 	params.adev = adev;
399 	params.vm = vm;
400 	params.immediate = immediate;
401 
402 	r = vm->update_funcs->prepare(&params, NULL,
403 				      AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR);
404 	if (r)
405 		goto exit;
406 
407 	addr = 0;
408 
409 	if (adev->asic_type >= CHIP_VEGA10) {
410 		if (level != AMDGPU_VM_PTB) {
411 			if (vm->is_npa)
412 				flags = adev->gmc.noretry_flags;
413 			/* Handle leaf PDEs as PTEs */
414 			flags |= AMDGPU_PDE_PTE_FLAG(adev);
415 			amdgpu_gmc_get_vm_pde(adev, level,
416 					      &value, &flags);
417 		} else if (vm->is_npa) {
418 			flags = adev->gmc.noretry_flags;
419 		} else {
420 			/* Workaround for fault priority problem on GMC9 */
421 			flags = AMDGPU_PTE_EXECUTABLE | adev->gmc.init_pte_flags;
422 		}
423 	}
424 
425 	r = vm->update_funcs->update(&params, vmbo, addr, 0, entries,
426 				     value, flags);
427 	if (r)
428 		goto exit;
429 
430 	r = vm->update_funcs->commit(&params, NULL);
431 exit:
432 	drm_dev_exit(idx);
433 	return r;
434 }
435 
436 /**
437  * amdgpu_vm_pt_create - create bo for PD/PT
438  *
439  * @adev: amdgpu_device pointer
440  * @vm: requesting vm
441  * @level: the page table level
442  * @immediate: use a immediate update
443  * @vmbo: pointer to the buffer object pointer
444  * @xcp_id: GPU partition id
445  */
446 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
447 			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
448 			int32_t xcp_id)
449 {
450 	struct amdgpu_bo_param bp;
451 	unsigned int num_entries;
452 	int r;
453 
454 	memset(&bp, 0, sizeof(bp));
455 
456 	bp.size = amdgpu_vm_pt_size(adev, level);
457 	bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
458 
459 	if (!adev->gmc.is_app_apu)
460 		bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
461 	else
462 		bp.domain = AMDGPU_GEM_DOMAIN_GTT;
463 
464 	bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain);
465 	bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
466 		AMDGPU_GEM_CREATE_CPU_GTT_USWC;
467 
468 	if (level < AMDGPU_VM_PTB)
469 		num_entries = amdgpu_vm_pt_num_entries(adev, level);
470 	else
471 		num_entries = 0;
472 
473 	bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries);
474 
475 	if (vm->use_cpu_for_update)
476 		bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
477 
478 	bp.type = ttm_bo_type_kernel;
479 	bp.no_wait_gpu = immediate;
480 	bp.xcp_id_plus1 = xcp_id + 1;
481 
482 	if (vm->root.bo)
483 		bp.resv = vm->root.bo->tbo.base.resv;
484 
485 	r = amdgpu_bo_create_vm(adev, &bp, vmbo);
486 	if (r)
487 		return r;
488 
489 	/* Assumes that reservation is shared with the VM root and that the
490 	 * reservation is locked
491 	 */
492 	if (vm->root.bo && vm->is_npa) {
493 		struct amdgpu_bo *pt_bo = &(*vmbo)->bo;
494 
495 		r = amdgpu_bo_pin(pt_bo, AMDGPU_GEM_DOMAIN_VRAM);
496 		if (r) {
497 			amdgpu_bo_unref(&pt_bo);
498 			return r;
499 		}
500 	}
501 
502 	return 0;
503 }
504 
505 /**
506  * amdgpu_vm_pt_alloc - Allocate a specific page table
507  *
508  * @p: see amdgpu_vm_update_params definition
509  * @cursor: Which page table to allocate
510  *
511  * Make sure a specific page table or directory is allocated.
512  *
513  * Returns:
514  *
515  * 0 on success or a negative error code on failure.
516  */
517 static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p,
518 			      struct amdgpu_vm_pt_cursor *cursor)
519 {
520 	struct amdgpu_vm_bo_base *entry = cursor->entry;
521 	struct amdgpu_bo *pt_bo;
522 	struct amdgpu_bo_vm *pt;
523 	int r, r2;
524 
525 	if (entry->bo)
526 		return 0;
527 
528 	amdgpu_vm_end_critical(p);
529 	r = amdgpu_vm_pt_create(p->adev, p->vm, cursor->level, p->immediate,
530 				&pt, p->vm->root.bo->xcp_id);
531 	r2 = amdgpu_vm_begin_critical(p);
532 	if (r)
533 		return r;
534 	if (r2)
535 		goto error_free_pt;
536 
537 	/* Keep a reference to the root directory to avoid
538 	 * freeing them up in the wrong order.
539 	 */
540 	pt_bo = &pt->bo;
541 	pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo);
542 	amdgpu_vm_bo_base_init(entry, p->vm, pt_bo);
543 	r = amdgpu_vm_pt_clear(p->adev, p->vm, pt, p->immediate);
544 	if (r)
545 		goto error_unpin;
546 
547 	return 0;
548 
549 error_unpin:
550 	if (p->vm->is_npa)
551 		amdgpu_bo_unpin(pt_bo);
552 error_free_pt:
553 	amdgpu_bo_unref(&pt_bo);
554 	return r;
555 }
556 
557 /**
558  * amdgpu_vm_pt_free - free one PD/PT
559  *
560  * @entry: PDE to free
561  */
562 static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
563 {
564 	if (!entry->bo)
565 		return;
566 
567 	if (entry->vm->is_npa)
568 		amdgpu_bo_unpin(entry->bo);
569 
570 	amdgpu_vm_update_stats(entry, entry->bo->tbo.resource, -1);
571 	entry->bo->vm_bo = NULL;
572 	ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
573 
574 	list_del(&entry->vm_status);
575 	amdgpu_bo_unref(&entry->bo);
576 }
577 
578 /**
579  * amdgpu_vm_pt_free_list - free PD/PT levels
580  *
581  * @adev: amdgpu device structure
582  * @params: see amdgpu_vm_update_params definition
583  *
584  * Free the page directory objects saved in the flush list
585  */
586 void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
587 			    struct amdgpu_vm_update_params *params)
588 {
589 	struct amdgpu_vm_bo_base *entry, *next;
590 	bool unlocked = params->unlocked;
591 
592 	if (list_empty(&params->tlb_flush_waitlist))
593 		return;
594 
595 	/*
596 	 * unlocked unmap clear page table leaves, warning to free the page entry.
597 	 */
598 	WARN_ON(unlocked);
599 
600 	list_for_each_entry_safe(entry, next, &params->tlb_flush_waitlist, vm_status)
601 		amdgpu_vm_pt_free(entry);
602 }
603 
604 /**
605  * amdgpu_vm_pt_add_list - add PD/PT level to the flush list
606  *
607  * @params: parameters for the update
608  * @cursor: first PT entry to start DF search from, non NULL
609  *
610  * This list will be freed after TLB flush.
611  */
612 static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
613 				  struct amdgpu_vm_pt_cursor *cursor)
614 {
615 	struct amdgpu_vm_pt_cursor seek;
616 	struct amdgpu_vm_bo_base *entry;
617 
618 	for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
619 		if (entry && entry->bo)
620 			list_move(&entry->vm_status, &params->tlb_flush_waitlist);
621 	}
622 
623 	/* enter start node now */
624 	list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist);
625 }
626 
627 /**
628  * amdgpu_vm_pt_free_root - free root PD
629  * @adev: amdgpu device structure
630  * @vm: amdgpu vm structure
631  *
632  * Free the root page directory and everything below it.
633  */
634 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm)
635 {
636 	struct amdgpu_vm_pt_cursor cursor;
637 	struct amdgpu_vm_bo_base *entry;
638 
639 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) {
640 		if (entry)
641 			amdgpu_vm_pt_free(entry);
642 	}
643 }
644 
645 /**
646  * amdgpu_vm_pde_update - update a single level in the hierarchy
647  *
648  * @params: parameters for the update
649  * @entry: entry to update
650  *
651  * Makes sure the requested entry in parent is up to date.
652  */
653 int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
654 			 struct amdgpu_vm_bo_base *entry)
655 {
656 	struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry);
657 	struct amdgpu_bo *bo, *pbo;
658 	struct amdgpu_vm *vm = params->vm;
659 	uint64_t pde, pt, flags;
660 	unsigned int level;
661 
662 	if (WARN_ON(!parent))
663 		return -EINVAL;
664 
665 	bo = parent->bo;
666 	for (level = 0, pbo = bo->parent; pbo; ++level)
667 		pbo = pbo->parent;
668 
669 	level += params->adev->vm_manager.root_level;
670 	amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags);
671 	pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8;
672 	return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt,
673 					1, 0, flags);
674 }
675 
676 /**
677  * amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags
678  *
679  * @adev: amdgpu_device pointer
680  * @flags: pointer to PTE flags
681  *
682  * Update PTE no-retry flags when TF is enabled.
683  */
684 static void amdgpu_vm_pte_update_noretry_flags(struct amdgpu_device *adev,
685 						uint64_t *flags)
686 {
687 	/*
688 	 * Update no-retry flags with the corresponding TF
689 	 * no-retry combination.
690 	 */
691 	if ((*flags & AMDGPU_VM_NORETRY_FLAGS) == AMDGPU_VM_NORETRY_FLAGS) {
692 		*flags &= ~AMDGPU_VM_NORETRY_FLAGS;
693 		*flags |= adev->gmc.noretry_flags;
694 	}
695 }
696 
697 /*
698  * amdgpu_vm_pte_update_flags - figure out flags for PTE updates
699  *
700  * Make sure to set the right flags for the PTEs at the desired level.
701  */
702 static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
703 				       struct amdgpu_bo_vm *pt,
704 				       unsigned int level,
705 				       uint64_t pe, uint64_t addr,
706 				       unsigned int count, uint32_t incr,
707 				       uint64_t flags)
708 {
709 	struct amdgpu_device *adev = params->adev;
710 
711 	if (level != AMDGPU_VM_PTB) {
712 		flags |= AMDGPU_PDE_PTE_FLAG(params->adev);
713 		amdgpu_gmc_get_vm_pde(adev, level, &addr, &flags);
714 
715 	} else if (adev->asic_type >= CHIP_VEGA10 &&
716 		   !(flags & AMDGPU_PTE_VALID) &&
717 		   !(flags & AMDGPU_PTE_PRT_FLAG(params->adev))) {
718 
719 		/* Workaround for fault priority problem on GMC9 and GFX12,
720 		 * EXECUTABLE for GMC9 fault priority and init_pte_flags
721 		 * (e.g. AMDGPU_PTE_IS_PTE on GFX12)
722 		 */
723 		flags |= AMDGPU_PTE_EXECUTABLE | adev->gmc.init_pte_flags;
724 	}
725 
726 	/*
727 	 * Update no-retry flags to use the no-retry flag combination
728 	 * with TF enabled. The AMDGPU_VM_NORETRY_FLAGS flag combination
729 	 * does not work when TF is enabled. So, replace them with
730 	 * AMDGPU_VM_NORETRY_FLAGS_TF flag combination which works for
731 	 * all cases.
732 	 */
733 	if (level == AMDGPU_VM_PTB)
734 		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
735 
736 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
737 					 flags);
738 }
739 
740 /**
741  * amdgpu_vm_pte_fragment - get fragment for PTEs
742  *
743  * @params: see amdgpu_vm_update_params definition
744  * @start: first PTE to handle
745  * @end: last PTE to handle
746  * @flags: hw mapping flags
747  * @frag: resulting fragment size
748  * @frag_end: end of this fragment
749  *
750  * Returns the first possible fragment for the start and end address.
751  */
752 static void amdgpu_vm_pte_fragment(struct amdgpu_vm_update_params *params,
753 				   uint64_t start, uint64_t end, uint64_t flags,
754 				   unsigned int *frag, uint64_t *frag_end)
755 {
756 	/**
757 	 * The MC L1 TLB supports variable sized pages, based on a fragment
758 	 * field in the PTE. When this field is set to a non-zero value, page
759 	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
760 	 * flags are considered valid for all PTEs within the fragment range
761 	 * and corresponding mappings are assumed to be physically contiguous.
762 	 *
763 	 * The L1 TLB can store a single PTE for the whole fragment,
764 	 * significantly increasing the space available for translation
765 	 * caching. This leads to large improvements in throughput when the
766 	 * TLB is under pressure.
767 	 *
768 	 * The L2 TLB distributes small and large fragments into two
769 	 * asymmetric partitions. The large fragment cache is significantly
770 	 * larger. Thus, we try to use large fragments wherever possible.
771 	 * Userspace can support this by aligning virtual base address and
772 	 * allocation size to the fragment size.
773 	 *
774 	 * Starting with Vega10 the fragment size only controls the L1. The L2
775 	 * is now directly feed with small/huge/giant pages from the walker.
776 	 */
777 	unsigned int max_frag;
778 
779 	if (params->adev->asic_type < CHIP_VEGA10)
780 		max_frag = params->adev->vm_manager.fragment_size;
781 	else
782 		max_frag = 31;
783 
784 	/* system pages are non continuously */
785 	if (params->pages_addr) {
786 		*frag = 0;
787 		*frag_end = end;
788 		return;
789 	}
790 
791 	/* This intentionally wraps around if no bit is set */
792 	*frag = min_t(unsigned int, ffs(start) - 1, fls64(end - start) - 1);
793 	if (*frag >= max_frag) {
794 		*frag = max_frag;
795 		*frag_end = end & ~((1ULL << max_frag) - 1);
796 	} else {
797 		*frag_end = start + (1 << *frag);
798 	}
799 }
800 
801 /**
802  * amdgpu_vm_ptes_update - make sure that page tables are valid
803  *
804  * @params: see amdgpu_vm_update_params definition
805  * @start: start of GPU address range
806  * @end: end of GPU address range
807  * @dst: destination address to map to, the next dst inside the function
808  * @flags: mapping flags
809  *
810  * Update the page tables in the range @start - @end.
811  *
812  * Returns:
813  * 0 for success, -EINVAL for failure.
814  */
815 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
816 			  uint64_t start, uint64_t end,
817 			  uint64_t dst, uint64_t flags)
818 {
819 	struct amdgpu_device *adev = params->adev;
820 	struct amdgpu_vm_pt_cursor cursor;
821 	uint64_t frag_start = start, frag_end;
822 	unsigned int frag;
823 	int r;
824 
825 	/* figure out the initial fragment */
826 	amdgpu_vm_pte_fragment(params, frag_start, end, flags, &frag,
827 			       &frag_end);
828 
829 	/* walk over the address space and update the PTs */
830 	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
831 	while (cursor.pfn < end) {
832 		unsigned int shift, parent_shift, mask;
833 		uint64_t incr, entry_end, pe_start;
834 		struct amdgpu_bo *pt;
835 
836 		if (!params->unlocked) {
837 			/* make sure that the page tables covering the
838 			 * address range are actually allocated
839 			 */
840 			r = amdgpu_vm_pt_alloc(params, &cursor);
841 			if (r)
842 				return r;
843 		}
844 
845 		shift = amdgpu_vm_pt_level_shift(adev, cursor.level);
846 		parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1);
847 		if (params->unlocked) {
848 			/* Unlocked updates are only allowed on the leaves */
849 			if (amdgpu_vm_pt_descendant(adev, &cursor))
850 				continue;
851 		} else if (adev->asic_type < CHIP_VEGA10 &&
852 			   (flags & AMDGPU_PTE_VALID)) {
853 			/* No huge page support before GMC v9 */
854 			if (cursor.level != AMDGPU_VM_PTB) {
855 				if (!amdgpu_vm_pt_descendant(adev, &cursor))
856 					return -ENOENT;
857 				continue;
858 			}
859 		} else if (frag < shift) {
860 			/* We can't use this level when the fragment size is
861 			 * smaller than the address shift. Go to the next
862 			 * child entry and try again.
863 			 */
864 			if (amdgpu_vm_pt_descendant(adev, &cursor))
865 				continue;
866 		} else if (frag >= parent_shift) {
867 			/* If the fragment size is even larger than the parent
868 			 * shift we should go up one level and check it again.
869 			 */
870 			if (!amdgpu_vm_pt_ancestor(&cursor))
871 				return -EINVAL;
872 			continue;
873 		}
874 
875 		pt = cursor.entry->bo;
876 		if (!pt) {
877 			/* We need all PDs and PTs for mapping something, */
878 			if (flags & AMDGPU_PTE_VALID)
879 				return -ENOENT;
880 
881 			/* but unmapping something can happen at a higher
882 			 * level.
883 			 */
884 			if (!amdgpu_vm_pt_ancestor(&cursor))
885 				return -EINVAL;
886 
887 			pt = cursor.entry->bo;
888 			shift = parent_shift;
889 			frag_end = max(frag_end, ALIGN(frag_start + 1,
890 				   1ULL << shift));
891 		}
892 
893 		/* Looks good so far, calculate parameters for the update */
894 		incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift;
895 		mask = amdgpu_vm_pt_entries_mask(adev, cursor.level);
896 		pe_start = ((cursor.pfn >> shift) & mask) * 8;
897 
898 		if (cursor.level < AMDGPU_VM_PTB && params->unlocked)
899 			/*
900 			 * MMU notifier callback unlocked unmap huge page, leave is PDE entry,
901 			 * only clear one entry. Next entry search again for PDE or PTE leave.
902 			 */
903 			entry_end = 1ULL << shift;
904 		else
905 			entry_end = ((uint64_t)mask + 1) << shift;
906 		entry_end += cursor.pfn & ~(entry_end - 1);
907 		entry_end = min(entry_end, end);
908 
909 		do {
910 			uint64_t upd_end = min(entry_end, frag_end);
911 			unsigned int nptes = (upd_end - frag_start) >> shift;
912 			uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag);
913 
914 			/* This can happen when we set higher level PDs to
915 			 * silent to stop fault floods.
916 			 */
917 			nptes = max(nptes, 1u);
918 
919 			trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
920 						    min(nptes, 32u), dst, incr,
921 						    upd_flags);
922 			amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
923 						   cursor.level, pe_start, dst,
924 						   nptes, incr, upd_flags);
925 
926 			pe_start += nptes * 8;
927 			dst += nptes * incr;
928 
929 			frag_start = upd_end;
930 			if (frag_start >= frag_end) {
931 				/* figure out the next fragment */
932 				amdgpu_vm_pte_fragment(params, frag_start, end,
933 						       flags, &frag, &frag_end);
934 				if (frag < shift)
935 					break;
936 			}
937 		} while (frag_start < entry_end);
938 
939 		if (amdgpu_vm_pt_descendant(adev, &cursor)) {
940 			/* Free all child entries.
941 			 * Update the tables with the flags and addresses and free up subsequent
942 			 * tables in the case of huge pages or freed up areas.
943 			 * This is the maximum you can free, because all other page tables are not
944 			 * completely covered by the range and so potentially still in use.
945 			 */
946 			while (cursor.pfn < frag_start) {
947 				/* Make sure previous mapping is freed */
948 				if (cursor.entry->bo) {
949 					params->needs_flush = true;
950 					amdgpu_vm_pt_add_list(params, &cursor);
951 				}
952 				amdgpu_vm_pt_next(adev, &cursor);
953 			}
954 
955 		} else if (frag >= shift) {
956 			/* or just move on to the next on the same level. */
957 			amdgpu_vm_pt_next(adev, &cursor);
958 		}
959 	}
960 
961 	return 0;
962 }
963 
964 /**
965  * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible
966  * @adev: amdgpu device structure
967  * @vm: amdgpu vm structure
968  *
969  * make root page directory and everything below it cpu accessible.
970  */
971 int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm)
972 {
973 	struct amdgpu_vm_pt_cursor cursor;
974 	struct amdgpu_vm_bo_base *entry;
975 
976 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) {
977 
978 		struct amdgpu_bo_vm *bo;
979 		int r;
980 
981 		if (entry->bo) {
982 			bo = to_amdgpu_bo_vm(entry->bo);
983 			r = vm->update_funcs->map_table(bo);
984 			if (r)
985 				return r;
986 		}
987 	}
988 
989 	return 0;
990 }
991