59 static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,  in __xe_pt_empty_pte()  argument
62 	struct xe_device *xe = tile_to_xe(tile);  in __xe_pt_empty_pte()
64 	u8 id = tile->id;  in __xe_pt_empty_pte()
100 struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,  in xe_pt_create()  argument
118 	bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,  in xe_pt_create()
120 				  XE_BO_FLAG_VRAM_IF_DGFX(tile) |  in xe_pt_create()
135 	xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);  in xe_pt_create()
155 void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,  in xe_pt_populate_empty()  argument
169 		empty = __xe_pt_empty_pte(tile, vm, pt->level);  in xe_pt_populate_empty()
265 	struct xe_tile *tile;  member
551 		xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1);  in xe_pt_stage_bind_entry()
559 			xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);  in xe_pt_stage_bind_entry()
570 		if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&  in xe_pt_stage_bind_entry()
610 xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_stage_bind()  argument
613 	struct xe_device *xe = tile_to_xe(tile);  in xe_pt_stage_bind()
626 		.tile = tile,  in xe_pt_stage_bind()
633 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];  in xe_pt_stage_bind()
761 	struct xe_tile *tile;  member
789 		xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,  in xe_pt_zap_ptes_entry()
818 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)  in xe_pt_zap_ptes()  argument
826 		.tile = tile,  in xe_pt_zap_ptes()
828 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];  in xe_pt_zap_ptes()
831 	if (!(pt_mask & BIT(tile->id)))  in xe_pt_zap_ptes()
841 xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,  in xe_vm_populate_pgtable()  argument
852 			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *  in xe_vm_populate_pgtable()
1004 xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_prepare_bind()  argument
1010 	err = xe_pt_stage_bind(tile, vma, entries, num_entries);  in xe_pt_prepare_bind()
1012 		xe_tile_assert(tile, *num_entries);  in xe_pt_prepare_bind()
1390 	struct xe_tile *tile;  member
1510 static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_stage_unbind()  argument
1520 		.tile = tile,  in xe_pt_stage_unbind()
1525 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];  in xe_pt_stage_unbind()
1535 				  struct xe_tile *tile, struct iosys_map *map,  in xe_migrate_clear_pgtable_callback()  argument
1540 	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);  in xe_migrate_clear_pgtable_callback()
1545 			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *  in xe_migrate_clear_pgtable_callback()
1643 static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,  in bind_op_prepare()  argument
1659 	pt_op->rebind = BIT(tile->id) & vma->tile_present;  in bind_op_prepare()
1661 	err = vma_reserve_fences(tile_to_xe(tile), vma);  in bind_op_prepare()
1665 	err = xe_pt_prepare_bind(tile, vma, pt_op->entries,  in bind_op_prepare()
1668 		xe_tile_assert(tile, pt_op->num_entries <=  in bind_op_prepare()
1670 		xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in bind_op_prepare()
1696 		vma->tile_staged |= BIT(tile->id);  in bind_op_prepare()
1707 static int unbind_op_prepare(struct xe_tile *tile,  in unbind_op_prepare()  argument
1715 	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))  in unbind_op_prepare()
1735 	err = vma_reserve_fences(tile_to_xe(tile), vma);  in unbind_op_prepare()
1739 	pt_op->num_entries = xe_pt_stage_unbind(tile, vma, pt_op->entries);  in unbind_op_prepare()
1741 	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in unbind_op_prepare()
1754 		      struct xe_tile *tile,  in op_prepare()  argument
1767 		err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma);  in op_prepare()
1771 		err = unbind_op_prepare(tile, pt_update_ops,  in op_prepare()
1775 			err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
1780 			err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
1786 		err = unbind_op_prepare(tile, pt_update_ops,  in op_prepare()
1790 		err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
1820 int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_prepare()  argument
1823 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_prepare()
1825 	int shift = tile->media_gt ? 1 : 0;  in xe_pt_update_ops_prepare()
1834 				      tile_to_xe(tile)->info.tile_count << shift);  in xe_pt_update_ops_prepare()
1839 		err = op_prepare(vops->vm, tile, pt_update_ops, op);  in xe_pt_update_ops_prepare()
1845 	xe_tile_assert(tile, pt_update_ops->current_op <=  in xe_pt_update_ops_prepare()
1858 static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,  in bind_op_commit()  argument
1874 	vma->tile_present |= BIT(tile->id);  in bind_op_commit()
1875 	vma->tile_staged &= ~BIT(tile->id);  in bind_op_commit()
1891 static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,  in unbind_op_commit()  argument
1907 	vma->tile_present &= ~BIT(tile->id);  in unbind_op_commit()
1921 		      struct xe_tile *tile,  in op_commit()  argument
1933 		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,  in op_commit()
1937 		unbind_op_commit(vm, tile, pt_update_ops,  in op_commit()
1942 			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,  in op_commit()
1945 			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,  in op_commit()
1949 		unbind_op_commit(vm, tile, pt_update_ops,  in op_commit()
1953 		bind_op_commit(vm, tile, pt_update_ops,  in op_commit()
1985 xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_run()  argument
1989 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_run()
2002 		.tile_id = tile->id,  in xe_pt_update_ops_run()
2009 		xe_tile_assert(tile, xe_vm_in_fault_mode(vm));  in xe_pt_update_ops_run()
2026 		if (tile->media_gt) {  in xe_pt_update_ops_run()
2051 	fence = xe_migrate_update_pgtables(tile->migrate, &update);  in xe_pt_update_ops_run()
2066 	if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,  in xe_pt_update_ops_run()
2076 		invalidation_fence_init(tile->primary_gt, ifence, fence,  in xe_pt_update_ops_run()
2080 			invalidation_fence_init(tile->media_gt, mfence, fence,  in xe_pt_update_ops_run()
2102 			op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);  in xe_pt_update_ops_run()
2115 			op_commit(vops->vm, tile, pt_update_ops, op,  in xe_pt_update_ops_run()
2132 	if (err != -EAGAIN && tile->id)  in xe_pt_update_ops_run()
2146 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_fini()  argument
2149 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_fini()
2160 	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);  in xe_pt_update_ops_fini()
2170 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_abort()  argument
2173 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_abort()
2195 	xe_pt_update_ops_fini(tile, vops);  in xe_pt_update_ops_abort()