88 static void gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)  in gen8_ppgtt_notify_vgt()  argument
90 	struct drm_i915_private *i915 = ppgtt->vm.i915;  in gen8_ppgtt_notify_vgt()
91 	struct intel_uncore *uncore = ppgtt->vm.gt->uncore;  in gen8_ppgtt_notify_vgt()
96 		atomic_inc(px_used(ppgtt->pd)); /* never remove */  in gen8_ppgtt_notify_vgt()
98 		atomic_dec(px_used(ppgtt->pd));  in gen8_ppgtt_notify_vgt()
102 	if (i915_vm_is_4lvl(&ppgtt->vm)) {  in gen8_ppgtt_notify_vgt()
103 		const u64 daddr = px_dma(ppgtt->pd);  in gen8_ppgtt_notify_vgt()
115 			const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);  in gen8_ppgtt_notify_vgt()
190 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);  in gen8_pdp_for_page_index()  local
193 		return ppgtt->pd;  in gen8_pdp_for_page_index()
195 		return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top));  in gen8_pdp_for_page_index()
224 	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_cleanup()  local
230 		gen8_ppgtt_notify_vgt(ppgtt, false);  in gen8_ppgtt_cleanup()
232 	if (ppgtt->pd)  in gen8_ppgtt_cleanup()
233 		__gen8_ppgtt_cleanup(vm, ppgtt->pd,  in gen8_ppgtt_cleanup()
455 gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt,  in gen8_ppgtt_insert_pte()  argument
463 	const gen8_pte_t pte_encode = ppgtt->vm.pte_encode(0, pat_index, flags);  in gen8_ppgtt_insert_pte()
740 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_insert()  local
755 			idx = gen8_ppgtt_insert_pte(ppgtt, pdp, &iter, idx,  in gen8_ppgtt_insert()
892 static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)  in gen8_preallocate_top_level_pdp()  argument
894 	struct i915_address_space *vm = &ppgtt->vm;  in gen8_preallocate_top_level_pdp()
895 	struct i915_page_directory *pd = ppgtt->pd;  in gen8_preallocate_top_level_pdp()
1006 	struct i915_ppgtt *ppgtt;  in gen8_ppgtt_create()  local
1009 	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);  in gen8_ppgtt_create()
1010 	if (!ppgtt)  in gen8_ppgtt_create()
1013 	ppgtt_init(ppgtt, gt, lmem_pt_obj_flags);  in gen8_ppgtt_create()
1014 	ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2;  in gen8_ppgtt_create()
1015 	ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t));  in gen8_ppgtt_create()
1025 	ppgtt->vm.has_read_only = !IS_GRAPHICS_VER(gt->i915, 11, 12);  in gen8_ppgtt_create()
1028 		ppgtt->vm.alloc_pt_dma = alloc_pt_lmem;  in gen8_ppgtt_create()
1030 		ppgtt->vm.alloc_pt_dma = alloc_pt_dma;  in gen8_ppgtt_create()
1038 	ppgtt->vm.alloc_scratch_dma = alloc_pt_dma;  in gen8_ppgtt_create()
1041 		ppgtt->vm.pte_encode = gen12_pte_encode;  in gen8_ppgtt_create()
1043 		ppgtt->vm.pte_encode = gen8_pte_encode;  in gen8_ppgtt_create()
1045 	ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND;  in gen8_ppgtt_create()
1046 	ppgtt->vm.insert_entries = gen8_ppgtt_insert;  in gen8_ppgtt_create()
1048 		ppgtt->vm.insert_page = xehp_ppgtt_insert_entry;  in gen8_ppgtt_create()
1050 		ppgtt->vm.insert_page = gen8_ppgtt_insert_entry;  in gen8_ppgtt_create()
1051 	ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;  in gen8_ppgtt_create()
1052 	ppgtt->vm.clear_range = gen8_ppgtt_clear;  in gen8_ppgtt_create()
1053 	ppgtt->vm.foreach = gen8_ppgtt_foreach;  in gen8_ppgtt_create()
1054 	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;  in gen8_ppgtt_create()
1056 	err = gen8_init_scratch(&ppgtt->vm);  in gen8_ppgtt_create()
1060 	pd = gen8_alloc_top_pd(&ppgtt->vm);  in gen8_ppgtt_create()
1065 	ppgtt->pd = pd;  in gen8_ppgtt_create()
1067 	if (!i915_vm_is_4lvl(&ppgtt->vm)) {  in gen8_ppgtt_create()
1068 		err = gen8_preallocate_top_level_pdp(ppgtt);  in gen8_ppgtt_create()
1074 		gen8_ppgtt_notify_vgt(ppgtt, true);  in gen8_ppgtt_create()
1076 	err = gen8_init_rsvd(&ppgtt->vm);  in gen8_ppgtt_create()
1080 	return ppgtt;  in gen8_ppgtt_create()
1083 	i915_vm_put(&ppgtt->vm);  in gen8_ppgtt_create()