61 static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,  in __xe_pt_empty_pte()  argument
64 	struct xe_device *xe = tile_to_xe(tile);  in __xe_pt_empty_pte()
66 	u8 id = tile->id;  in __xe_pt_empty_pte()
90  * @tile: The tile to create for.
103 struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,  in xe_pt_create()  argument
121 	bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) |  in xe_pt_create()
130 	bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,  in xe_pt_create()
143 	xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);  in xe_pt_create()
156  * @tile: The tile the scratch pagetable of which to use.
160  * Populate the page-table bo of @pt with entries pointing into the tile's
163 void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,  in xe_pt_populate_empty()  argument
177 		empty = __xe_pt_empty_pte(tile, vm, pt->level);  in xe_pt_populate_empty()
289 	/** @tile: The tile we're building for. */
290 	struct xe_tile *tile;  member
596 		xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1,  in xe_pt_stage_bind_entry()
605 			xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);  in xe_pt_stage_bind_entry()
616 		if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&  in xe_pt_stage_bind_entry()
681  * @tile: The tile we're building for.
699 xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_stage_bind()  argument
704 	struct xe_device *xe = tile_to_xe(tile);  in xe_pt_stage_bind()
716 		.tile = tile,  in xe_pt_stage_bind()
724 	struct xe_pt *pt = vm->pt_root[tile->id];  in xe_pt_stage_bind()
847 	/** @tile: The tile we're building for */
848 	struct xe_tile *tile;  member
876 		xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,  in xe_pt_zap_ptes_entry()
891  * @tile: The tile we're zapping for.
905 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)  in xe_pt_zap_ptes()  argument
913 		.tile = tile,  in xe_pt_zap_ptes()
915 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];  in xe_pt_zap_ptes()
923 	if (!(pt_mask & BIT(tile->id)))  in xe_pt_zap_ptes()
934  * @tile: The tile we're zapping for.
947 bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,  in xe_pt_zap_ptes_range()  argument
956 		.tile = tile,  in xe_pt_zap_ptes_range()
958 	struct xe_pt *pt = vm->pt_root[tile->id];  in xe_pt_zap_ptes_range()
975 	if (!(pt_mask & BIT(tile->id)))  in xe_pt_zap_ptes_range()
985 xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,  in xe_vm_populate_pgtable()  argument
996 			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *  in xe_vm_populate_pgtable()
1159 xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,  in xe_pt_prepare_bind()  argument
1167 	err = xe_pt_stage_bind(tile, vma, range, entries, num_entries,  in xe_pt_prepare_bind()
1170 		xe_tile_assert(tile, *num_entries);  in xe_pt_prepare_bind()
1535 	/** @tile: The tile we're unbinding from. */
1536 	struct xe_tile *tile;  member
1645  * @tile: The tile we're unbinding for.
1658 static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,  in xe_pt_stage_unbind()  argument
1673 		.tile = tile,  in xe_pt_stage_unbind()
1678 	struct xe_pt *pt = vm->pt_root[tile->id];  in xe_pt_stage_unbind()
1688 				  struct xe_tile *tile, struct iosys_map *map,  in xe_migrate_clear_pgtable_callback()  argument
1693 	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);  in xe_migrate_clear_pgtable_callback()
1698 			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *  in xe_migrate_clear_pgtable_callback()
1796 static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,  in bind_op_prepare()  argument
1804 	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));  in bind_op_prepare()
1813 	pt_op->rebind = BIT(tile->id) & vma->tile_present;  in bind_op_prepare()
1815 	err = vma_reserve_fences(tile_to_xe(tile), vma);  in bind_op_prepare()
1819 	err = xe_pt_prepare_bind(tile, vma, NULL, pt_op->entries,  in bind_op_prepare()
1822 		xe_tile_assert(tile, pt_op->num_entries <=  in bind_op_prepare()
1824 		xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in bind_op_prepare()
1852 		vma->tile_staged |= BIT(tile->id);  in bind_op_prepare()
1863 static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile,  in bind_range_prepare()  argument
1871 	xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma));  in bind_range_prepare()
1879 	pt_op->rebind = BIT(tile->id) & range->tile_present;  in bind_range_prepare()
1881 	err = xe_pt_prepare_bind(tile, vma, range, pt_op->entries,  in bind_range_prepare()
1884 		xe_tile_assert(tile, pt_op->num_entries <=  in bind_range_prepare()
1886 		xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in bind_range_prepare()
1905 static int unbind_op_prepare(struct xe_tile *tile,  in unbind_op_prepare()  argument
1913 	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))  in unbind_op_prepare()
1916 	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));  in unbind_op_prepare()
1927 	err = vma_reserve_fences(tile_to_xe(tile), vma);  in unbind_op_prepare()
1931 	pt_op->num_entries = xe_pt_stage_unbind(tile, xe_vma_vm(vma),  in unbind_op_prepare()
1934 	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in unbind_op_prepare()
1974 				struct xe_tile *tile,  in unbind_range_prepare()  argument
1981 	if (!(range->tile_present & BIT(tile->id)))  in unbind_range_prepare()
1992 	pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,  in unbind_range_prepare()
1995 	xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,  in unbind_range_prepare()
2002 		xe_vm_has_valid_gpu_mapping(tile, range->tile_present,  in unbind_range_prepare()
2013 		      struct xe_tile *tile,  in op_prepare()  argument
2028 		err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma,  in op_prepare()
2039 		err = unbind_op_prepare(tile, pt_update_ops, old);  in op_prepare()
2042 			err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
2047 			err = bind_op_prepare(vm, tile, pt_update_ops,  in op_prepare()
2060 		err = unbind_op_prepare(tile, pt_update_ops, vma);  in op_prepare()
2072 				err = bind_range_prepare(vm, tile, pt_update_ops,  in op_prepare()
2078 			err = bind_op_prepare(vm, tile, pt_update_ops, vma, false);  in op_prepare()
2087 			err = bind_range_prepare(vm, tile, pt_update_ops,  in op_prepare()
2091 			err = unbind_range_prepare(vm, tile, pt_update_ops,  in op_prepare()
2112  * @tile: Tile of PT update operations
2121 int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_prepare()  argument
2124 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_prepare()
2126 	int shift = tile->media_gt ? 1 : 0;  in xe_pt_update_ops_prepare()
2135 				      tile_to_xe(tile)->info.tile_count << shift);  in xe_pt_update_ops_prepare()
2140 		err = op_prepare(vops->vm, tile, pt_update_ops, op);  in xe_pt_update_ops_prepare()
2146 	xe_tile_assert(tile, pt_update_ops->current_op <=  in xe_pt_update_ops_prepare()
2159 static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,  in bind_op_commit()  argument
2164 	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));  in bind_op_commit()
2178 	WRITE_ONCE(vma->tile_present, vma->tile_present | BIT(tile->id));  in bind_op_commit()
2181 			   vma->tile_invalidated | BIT(tile->id));  in bind_op_commit()
2184 			   vma->tile_invalidated & ~BIT(tile->id));  in bind_op_commit()
2185 	vma->tile_staged &= ~BIT(tile->id);  in bind_op_commit()
2201 static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,  in unbind_op_commit()  argument
2206 	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));  in unbind_op_commit()
2219 	vma->tile_present &= ~BIT(tile->id);  in unbind_op_commit()
2245 		      struct xe_tile *tile,  in op_commit()  argument
2258 		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,  in op_commit()
2268 		unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2);  in op_commit()
2271 			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,  in op_commit()
2274 			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,  in op_commit()
2283 			unbind_op_commit(vm, tile, pt_update_ops, vma, fence,  in op_commit()
2296 				range_present_and_invalidated_tile(vm, range, tile->id);  in op_commit()
2298 			bind_op_commit(vm, tile, pt_update_ops, vma, fence,  in op_commit()
2307 			range_present_and_invalidated_tile(vm, op->map_range.range, tile->id);  in op_commit()
2311 				   ~BIT(tile->id));  in op_commit()
2347  * @tile: Tile of PT update operations
2357 xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_run()  argument
2361 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_run()
2374 		.tile_id = tile->id,  in xe_pt_update_ops_run()
2381 		xe_tile_assert(tile, xe_vm_in_fault_mode(vm));  in xe_pt_update_ops_run()
2395 			to_dep_scheduler(q, tile->primary_gt);  in xe_pt_update_ops_run()
2397 		ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval,  in xe_pt_update_ops_run()
2408 		if (tile->media_gt) {  in xe_pt_update_ops_run()
2409 			dep_scheduler = to_dep_scheduler(q, tile->media_gt);  in xe_pt_update_ops_run()
2412 						       &tile->media_gt->tlb_inval,  in xe_pt_update_ops_run()
2442 	fence = xe_migrate_update_pgtables(tile->migrate, &update);  in xe_pt_update_ops_run()
2457 	if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,  in xe_pt_update_ops_run()
2467 		ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence);  in xe_pt_update_ops_run()
2472 			mfence = xe_tlb_inval_job_push(mjob, tile->migrate,  in xe_pt_update_ops_run()
2494 			op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);  in xe_pt_update_ops_run()
2507 			op_commit(vops->vm, tile, pt_update_ops, op, ifence,  in xe_pt_update_ops_run()
2527 	if (err != -EAGAIN && err != -ENODATA && tile->id)  in xe_pt_update_ops_run()
2536  * @tile: Tile of PT update operations
2541 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_fini()  argument
2544 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_fini()
2555 	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);  in xe_pt_update_ops_fini()
2560  * @tile: Tile of PT update operations
2565 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)  in xe_pt_update_ops_abort()  argument
2568 		&vops->pt_update_ops[tile->id];  in xe_pt_update_ops_abort()
2590 	xe_pt_update_ops_fini(tile, vops);  in xe_pt_update_ops_abort()