96 		atomic_inc(px_used(ppgtt->pd)); /* never remove */  in gen8_ppgtt_notify_vgt()
98 		atomic_dec(px_used(ppgtt->pd));  in gen8_ppgtt_notify_vgt()
103 		const u64 daddr = px_dma(ppgtt->pd);  in gen8_ppgtt_notify_vgt()
193 		return ppgtt->pd;  in gen8_pdp_for_page_index()
195 		return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top));  in gen8_pdp_for_page_index()
205 				 struct i915_page_directory *pd,  in __gen8_ppgtt_cleanup()  argument
209 		void **pde = pd->entry;  in __gen8_ppgtt_cleanup()
219 	free_px(vm, &pd->pt, lvl);  in __gen8_ppgtt_cleanup()
232 	if (ppgtt->pd)  in gen8_ppgtt_cleanup()
233 		__gen8_ppgtt_cleanup(vm, ppgtt->pd,  in gen8_ppgtt_cleanup()
240 			      struct i915_page_directory * const pd,  in __gen8_ppgtt_clear()  argument
251 		  idx, len, atomic_read(px_used(pd)));  in __gen8_ppgtt_clear()
252 	GEM_BUG_ON(!len || len >= atomic_read(px_used(pd)));  in __gen8_ppgtt_clear()
255 		struct i915_page_table *pt = pd->entry[idx];  in __gen8_ppgtt_clear()
259 			GTT_TRACE("%s(%p):{ lvl:%d, idx:%d, start:%llx, end:%llx } removing pd\n",  in __gen8_ppgtt_clear()
261 			clear_pd_entry(pd, idx, scratch);  in __gen8_ppgtt_clear()
300 		if (release_pd_entry(pd, idx, pt, scratch))  in __gen8_ppgtt_clear()
318 	__gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_clear()
324 			       struct i915_page_directory * const pd,  in __gen8_ppgtt_alloc()  argument
334 		  idx, len, atomic_read(px_used(pd)));  in __gen8_ppgtt_alloc()
337 	spin_lock(&pd->lock);  in __gen8_ppgtt_alloc()
338 	GEM_BUG_ON(!atomic_read(px_used(pd))); /* Must be pinned! */  in __gen8_ppgtt_alloc()
340 		struct i915_page_table *pt = pd->entry[idx];  in __gen8_ppgtt_alloc()
343 			spin_unlock(&pd->lock);  in __gen8_ppgtt_alloc()
353 			spin_lock(&pd->lock);  in __gen8_ppgtt_alloc()
354 			if (likely(!pd->entry[idx])) {  in __gen8_ppgtt_alloc()
357 				set_pd_entry(pd, idx, pt);  in __gen8_ppgtt_alloc()
359 				pt = pd->entry[idx];  in __gen8_ppgtt_alloc()
365 			spin_unlock(&pd->lock);  in __gen8_ppgtt_alloc()
370 			spin_lock(&pd->lock);  in __gen8_ppgtt_alloc()
387 	spin_unlock(&pd->lock);  in __gen8_ppgtt_alloc()
402 	__gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_alloc()
407 				 struct i915_page_directory *pd,  in __gen8_ppgtt_foreach()  argument
418 	spin_lock(&pd->lock);  in __gen8_ppgtt_foreach()
420 		struct i915_page_table *pt = pd->entry[idx];  in __gen8_ppgtt_foreach()
423 		spin_unlock(&pd->lock);  in __gen8_ppgtt_foreach()
433 		spin_lock(&pd->lock);  in __gen8_ppgtt_foreach()
436 	spin_unlock(&pd->lock);  in __gen8_ppgtt_foreach()
449 	__gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_foreach()
462 	struct i915_page_directory *pd;  in gen8_ppgtt_insert_pte()  local
466 	pd = i915_pd_entry(pdp, gen8_pd_index(idx, 2));  in gen8_ppgtt_insert_pte()
467 	vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1)));  in gen8_ppgtt_insert_pte()
490 				pd = pdp->entry[gen8_pd_index(idx, 2)];  in gen8_ppgtt_insert_pte()
494 			vaddr = px_vaddr(i915_pt_entry(pd, gen8_pd_index(idx, 1)));  in gen8_ppgtt_insert_pte()
519 		struct i915_page_directory * const pd =  in xehp_ppgtt_insert_huge()  local
522 			i915_pt_entry(pd, __gen8_pte_index(start, 1));  in xehp_ppgtt_insert_huge()
539 			vaddr = px_vaddr(pd);  in xehp_ppgtt_insert_huge()
561 					vaddr = px_vaddr(pd);  in xehp_ppgtt_insert_huge()
626 		struct i915_page_directory * const pd =  in gen8_ppgtt_insert_huge()  local
642 			vaddr = px_vaddr(pd);  in gen8_ppgtt_insert_huge()
645 				i915_pt_entry(pd, __gen8_pte_index(start, 1));  in gen8_ppgtt_insert_huge()
704 			vaddr = px_vaddr(pd);  in gen8_ppgtt_insert_huge()
722 				vaddr = px_vaddr(i915_pt_entry(pd, maybe_64K));  in gen8_ppgtt_insert_huge()
772 	struct i915_page_directory *pd =  in gen8_ppgtt_insert_entry()  local
774 	struct i915_page_table *pt = i915_pt_entry(pd, gen8_pd_index(idx, 1));  in gen8_ppgtt_insert_entry()
793 	struct i915_page_directory *pd =  in xehp_ppgtt_insert_entry_lm()  local
795 	struct i915_page_table *pt = i915_pt_entry(pd, gen8_pd_index(idx, 1));  in xehp_ppgtt_insert_entry_lm()
804 		vaddr = px_vaddr(pd);  in xehp_ppgtt_insert_entry_lm()
895 	struct i915_page_directory *pd = ppgtt->pd;  in gen8_preallocate_top_level_pdp()  local
916 		set_pd_entry(pd, idx, pde);  in gen8_preallocate_top_level_pdp()
928 	struct i915_page_directory *pd;  in gen8_alloc_top_pd()  local
933 	pd = __alloc_pd(count);  in gen8_alloc_top_pd()
934 	if (unlikely(!pd))  in gen8_alloc_top_pd()
937 	pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);  in gen8_alloc_top_pd()
938 	if (IS_ERR(pd->pt.base)) {  in gen8_alloc_top_pd()
939 		err = PTR_ERR(pd->pt.base);  in gen8_alloc_top_pd()
940 		pd->pt.base = NULL;  in gen8_alloc_top_pd()
944 	err = map_pt_dma(vm, pd->pt.base);  in gen8_alloc_top_pd()
948 	fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count);  in gen8_alloc_top_pd()
949 	atomic_inc(px_used(pd)); /* mark as pinned */  in gen8_alloc_top_pd()
950 	return pd;  in gen8_alloc_top_pd()
953 	free_pd(vm, pd);  in gen8_alloc_top_pd()
1005 	struct i915_page_directory *pd;  in gen8_ppgtt_create()  local
1060 	pd = gen8_alloc_top_pd(&ppgtt->vm);  in gen8_ppgtt_create()
1061 	if (IS_ERR(pd)) {  in gen8_ppgtt_create()
1062 		err = PTR_ERR(pd);  in gen8_ppgtt_create()
1065 	ppgtt->pd = pd;  in gen8_ppgtt_create()