Lines Matching refs:xe
115 bool vf_migration = IS_SRIOV_VF(vm->xe) && in wait_for_existing_preempt_fences()
116 xe_sriov_vf_migration_supported(vm->xe); in wait_for_existing_preempt_fences()
128 xe_assert(vm->xe, vf_migration); in wait_for_existing_preempt_fences()
166 xe_assert(vm->xe, link != list); in arm_preempt_fences()
229 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); in xe_vm_add_compute_exec_queue()
232 err = xe_validation_exec_lock(&ctx, &vm_exec, &vm->xe->val); in xe_vm_add_compute_exec_queue()
346 if (!try_wait_for_completion(&vm->xe->pm_block)) in xe_gpuvm_validate()
437 struct xe_device *xe = vm->xe; in vm_suspend_rebind_worker() local
440 mutex_lock(&xe->rebind_resume_lock); in vm_suspend_rebind_worker()
441 if (!try_wait_for_completion(&vm->xe->pm_block)) { in vm_suspend_rebind_worker()
443 list_move_tail(&vm->preempt.pm_activate_link, &xe->rebind_resume_list); in vm_suspend_rebind_worker()
445 mutex_unlock(&xe->rebind_resume_lock); in vm_suspend_rebind_worker()
459 queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work); in xe_vm_resume_rebind_worker()
473 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); in preempt_rebind_work_func()
485 if (!try_wait_for_completion(&vm->xe->pm_block) && vm_suspend_rebind_worker(vm)) { in preempt_rebind_work_func()
498 err = xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, in preempt_rebind_work_func()
548 spin_lock(&vm->xe->ttm.lru_lock); in preempt_rebind_work_func()
550 spin_unlock(&vm->xe->ttm.lru_lock); in preempt_rebind_work_func()
568 if (IS_SRIOV_VF(vm->xe) && in preempt_rebind_work_func()
569 xe_sriov_vf_migration_supported(vm->xe)) { in preempt_rebind_work_func()
579 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err); in preempt_rebind_work_func()
612 drm_warn(&vm->xe->drm, in xe_vm_add_fault_entry_pf()
783 xe_assert(vm->xe, vma->tile_present); in xe_vm_rebind()
830 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vma_rebind()
834 for_each_tile(tile, vm->xe, id) { in xe_vma_rebind()
920 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vm_range_rebind()
921 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); in xe_vm_range_rebind()
925 for_each_tile(tile, vm->xe, id) { in xe_vm_range_rebind()
1002 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vm_range_unbind()
1008 for_each_tile(tile, vm->xe, id) { in xe_vm_range_unbind()
1080 xe_assert(vm->xe, start < end); in xe_vma_create()
1081 xe_assert(vm->xe, end < vm->size); in xe_vma_create()
1111 for_each_tile(tile, vm->xe, id) in xe_vma_create()
1114 if (vm->xe->info.has_atomic_enable_pte_bit) in xe_vma_create()
1225 xe_assert(vm->xe, list_empty(&vma->combined_links.destroy)); in xe_vma_destroy()
1228 xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED); in xe_vma_destroy()
1281 struct xe_device *xe = xe_vma_vm(vma)->xe; in xe_vma_destroy_unlocked() local
1286 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { in xe_vma_destroy_unlocked()
1293 xe_assert(xe, !err); in xe_vma_destroy_unlocked()
1306 xe_assert(vm->xe, start + range <= vm->size); in xe_vm_find_overlapping_vma()
1317 xe_assert(vm->xe, xe_vma_vm(vma) == vm); in xe_vm_insert_vma()
1330 xe_assert(vm->xe, xe_vma_vm(vma) == vm); in xe_vm_remove_vma()
1413 struct xe_device *xe = xe_bo_device(bo); in pde_pat_index() local
1428 pat_index = xe->pat.idx[XE_CACHE_WB]; in pde_pat_index()
1430 pat_index = xe->pat.idx[XE_CACHE_NONE]; in pde_pat_index()
1432 xe_assert(xe, pat_index <= 3); in pde_pat_index()
1490 static u64 xelp_pte_encode_addr(struct xe_device *xe, u64 addr, in xelp_pte_encode_addr() argument
1497 xe_assert(xe, !(flags & ~XE_PTE_PS64)); in xelp_pte_encode_addr()
1536 static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile, in xe_vm_create_scratch() argument
1565 for_each_tile(tile, vm->xe, id) { in xe_vm_free_scratch()
1584 for_each_tile(tile, vm->xe, id) { in xe_vm_pt_destroy()
1592 static void xe_vm_init_prove_locking(struct xe_device *xe, struct xe_vm *vm) in xe_vm_init_prove_locking() argument
1602 might_lock(&xe_root_mmio_gt(xe)->uc.guc.ct.lock); in xe_vm_init_prove_locking()
1606 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) in xe_vm_create() argument
1620 xe_assert(xe, !((flags & XE_VM_FLAG_GSC) && (flags & XE_VM_FLAG_FAULT_MODE))); in xe_vm_create()
1626 vm->xe = xe; in xe_vm_create()
1628 vm->size = 1ull << xe->info.va_bits; in xe_vm_create()
1667 vm->preempt.min_run_period_ms = xe->min_run_period_pf_ms; in xe_vm_create()
1669 vm->preempt.min_run_period_ms = xe->min_run_period_lr_ms; in xe_vm_create()
1672 xe_vm_init_prove_locking(xe, vm); in xe_vm_create()
1674 for_each_tile(tile, xe, id) in xe_vm_create()
1687 xe_pm_runtime_get_noresume(xe); in xe_vm_create()
1695 vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm); in xe_vm_create()
1701 drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm, in xe_vm_create()
1707 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, in xe_vm_create()
1712 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) in xe_vm_create()
1715 for_each_tile(tile, xe, id) { in xe_vm_create()
1720 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level, in xe_vm_create()
1735 for_each_tile(tile, xe, id) { in xe_vm_create()
1739 err = xe_vm_create_scratch(xe, tile, vm, &exec); in xe_vm_create()
1759 for_each_tile(tile, xe, id) { in xe_vm_create()
1771 for_each_tile(tile, xe, id) { in xe_vm_create()
1781 q = xe_exec_queue_create_bind(xe, tile, vm, create_flags, 0); in xe_vm_create()
1790 if (xef && xe->info.has_asid) { in xe_vm_create()
1793 down_write(&xe->usm.lock); in xe_vm_create()
1794 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, in xe_vm_create()
1796 &xe->usm.next_asid, GFP_NOWAIT); in xe_vm_create()
1797 up_write(&xe->usm.lock); in xe_vm_create()
1819 for_each_tile(tile, xe, id) in xe_vm_create()
1821 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); in xe_vm_create()
1826 xe_pm_runtime_put(xe); in xe_vm_create()
1832 struct xe_device *xe = vm->xe; in xe_vm_close() local
1836 bound = drm_dev_enter(&xe->drm, &idx); in xe_vm_close()
1855 for_each_tile(tile, xe, id) in xe_vm_close()
1857 xe_pt_clear(xe, vm->pt_root[id]); in xe_vm_close()
1859 for_each_gt(gt, xe, id) in xe_vm_close()
1875 struct xe_device *xe = vm->xe; in xe_vm_close_and_put() local
1881 xe_assert(xe, !vm->preempt.num_exec_queues); in xe_vm_close_and_put()
1885 mutex_lock(&xe->rebind_resume_lock); in xe_vm_close_and_put()
1887 mutex_unlock(&xe->rebind_resume_lock); in xe_vm_close_and_put()
1894 for_each_tile(tile, xe, id) { in xe_vm_close_and_put()
1905 for_each_tile(tile, xe, id) { in xe_vm_close_and_put()
1963 down_write(&xe->usm.lock); in xe_vm_close_and_put()
1967 xe_assert(xe, xe->info.has_asid); in xe_vm_close_and_put()
1968 xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION)); in xe_vm_close_and_put()
1970 lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); in xe_vm_close_and_put()
1971 xe_assert(xe, lookup == vm); in xe_vm_close_and_put()
1973 up_write(&xe->usm.lock); in xe_vm_close_and_put()
1977 for_each_tile(tile, xe, id) in xe_vm_close_and_put()
1987 struct xe_device *xe = vm->xe; in vm_destroy_work_func() local
1992 xe_assert(xe, !vm->size); in vm_destroy_work_func()
2000 xe_pm_runtime_put(xe); in vm_destroy_work_func()
2002 for_each_tile(tile, xe, id) in vm_destroy_work_func()
2007 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); in vm_destroy_work_func()
2070 struct xe_device *xe = to_xe_device(dev); in xe_vm_create_ioctl() local
2073 struct xe_gt *wa_gt = xe_root_mmio_gt(xe); in xe_vm_create_ioctl()
2079 if (XE_IOCTL_DBG(xe, args->extensions)) in xe_vm_create_ioctl()
2085 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && in xe_vm_create_ioctl()
2086 !xe->info.has_usm)) in xe_vm_create_ioctl()
2089 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_vm_create_ioctl()
2092 if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) in xe_vm_create_ioctl()
2095 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && in xe_vm_create_ioctl()
2097 !xe->info.needs_scratch)) in xe_vm_create_ioctl()
2100 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && in xe_vm_create_ioctl()
2104 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) && in xe_vm_create_ioctl()
2117 vm = xe_vm_create(xe, flags, xef); in xe_vm_create_ioctl()
2144 struct xe_device *xe = to_xe_device(dev); in xe_vm_destroy_ioctl() local
2150 if (XE_IOCTL_DBG(xe, args->pad) || in xe_vm_destroy_ioctl()
2151 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_vm_destroy_ioctl()
2156 if (XE_IOCTL_DBG(xe, !vm)) in xe_vm_destroy_ioctl()
2158 else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) in xe_vm_destroy_ioctl()
2213 struct xe_device *xe = to_xe_device(dev); in xe_vm_query_vmas_attrs_ioctl() local
2221 if (XE_IOCTL_DBG(xe, in xe_vm_query_vmas_attrs_ioctl()
2231 if (XE_IOCTL_DBG(xe, !vm)) in xe_vm_query_vmas_attrs_ioctl()
2323 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) in print_op() argument
2329 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx", in print_op()
2334 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", in print_op()
2338 vm_dbg(&xe->drm, in print_op()
2343 vm_dbg(&xe->drm, in print_op()
2350 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", in print_op()
2356 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx", in print_op()
2360 drm_warn(&xe->drm, "NOT POSSIBLE\n"); in print_op()
2364 static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) in print_op() argument
2415 vm_dbg(&vm->xe->drm, in vm_bind_ioctl_ops_create()
2446 xe_assert(vm->xe, bo); in vm_bind_ioctl_ops_create()
2463 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); in vm_bind_ioctl_ops_create()
2504 ctx.devmem_possible = IS_DGFX(vm->xe) && in vm_bind_ioctl_ops_create()
2507 for_each_tile(tile, vm->xe, id) in vm_bind_ioctl_ops_create()
2515 xe_device_get_root_tile(vm->xe)); in vm_bind_ioctl_ops_create()
2517 tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] - in vm_bind_ioctl_ops_create()
2564 print_op(vm->xe, __op); in vm_bind_ioctl_ops_create()
2590 xe_validation_guard(&ctx, &vm->xe->val, &exec, in new_vma()
2737 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); in xe_vma_op_commit()
2769 struct xe_device *xe = vm->xe; in vm_bind_ioctl_ops_parse() local
2777 for_each_tile(tile, vm->xe, id) in vm_bind_ioctl_ops_parse()
2868 vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx", in vm_bind_ioctl_ops_parse()
2897 vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx", in vm_bind_ioctl_ops_parse()
2939 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); in vm_bind_ioctl_ops_parse()
3012 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); in xe_vma_op_unwind()
3110 bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); in prefetch_ranges()
3133 drm_dbg(&vm->xe->drm, in prefetch_ranges()
3142 …drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe… in prefetch_ranges()
3151 drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n", in prefetch_ranges()
3240 xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC || in op_lock_and_prep()
3267 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); in op_lock_and_prep()
3311 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK) in vm_bind_ioctl_ops_lock_and_prep()
3359 for_each_tile(tile, vm->xe, id) { in vm_ops_setup_tile_args()
3392 for_each_tile(tile, vm->xe, id) { in ops_execute()
3412 for_each_tile(tile, vm->xe, id) { in ops_execute()
3425 for_each_tile(tile, vm->xe, id) { in ops_execute()
3448 xe_assert(vm->xe, current_fence == n_fence); in ops_execute()
3453 for_each_tile(tile, vm->xe, id) { in ops_execute()
3463 for_each_tile(tile, vm->xe, id) { in ops_execute()
3506 drm_warn(&vm->xe->drm, "NOT POSSIBLE\n"); in op_add_ufence()
3546 xe_validation_guard(&ctx, &vm->xe->val, &exec, in vm_bind_ioctl_ops_execute()
3592 static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, in vm_bind_ioctl_check_args() argument
3599 if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || in vm_bind_ioctl_check_args()
3600 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in vm_bind_ioctl_check_args()
3603 if (XE_IOCTL_DBG(xe, args->extensions)) in vm_bind_ioctl_check_args()
3606 if (XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS)) in vm_bind_ioctl_check_args()
3622 if (XE_IOCTL_DBG(xe, err)) { in vm_bind_ioctl_check_args()
3646 if (XE_IOCTL_DBG(xe, is_cpu_addr_mirror && in vm_bind_ioctl_check_args()
3653 if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { in vm_bind_ioctl_check_args()
3658 pat_index = array_index_nospec(pat_index, xe->pat.n_entries); in vm_bind_ioctl_check_args()
3660 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); in vm_bind_ioctl_check_args()
3661 comp_en = xe_pat_index_get_comp_en(xe, pat_index); in vm_bind_ioctl_check_args()
3662 if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */ in vm_bind_ioctl_check_args()
3672 if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || in vm_bind_ioctl_check_args()
3673 XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || in vm_bind_ioctl_check_args()
3674 XE_IOCTL_DBG(xe, obj && (is_null || is_cpu_addr_mirror)) || in vm_bind_ioctl_check_args()
3675 XE_IOCTL_DBG(xe, obj_offset && (is_null || in vm_bind_ioctl_check_args()
3677 XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP && in vm_bind_ioctl_check_args()
3679 XE_IOCTL_DBG(xe, is_decompress && in vm_bind_ioctl_check_args()
3680 xe_pat_index_get_comp_en(xe, pat_index)) || in vm_bind_ioctl_check_args()
3681 XE_IOCTL_DBG(xe, !obj && in vm_bind_ioctl_check_args()
3684 XE_IOCTL_DBG(xe, !obj && in vm_bind_ioctl_check_args()
3686 XE_IOCTL_DBG(xe, addr && in vm_bind_ioctl_check_args()
3688 XE_IOCTL_DBG(xe, range && in vm_bind_ioctl_check_args()
3690 XE_IOCTL_DBG(xe, obj && in vm_bind_ioctl_check_args()
3692 XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && in vm_bind_ioctl_check_args()
3694 XE_IOCTL_DBG(xe, !IS_DGFX(xe) && coh_mode == XE_COH_NONE && in vm_bind_ioctl_check_args()
3696 XE_IOCTL_DBG(xe, xe_device_is_l2_flush_optimized(xe) && in vm_bind_ioctl_check_args()
3700 XE_IOCTL_DBG(xe, comp_en && in vm_bind_ioctl_check_args()
3702 XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR && in vm_bind_ioctl_check_args()
3704 XE_IOCTL_DBG(xe, obj && in vm_bind_ioctl_check_args()
3706 XE_IOCTL_DBG(xe, prefetch_region && in vm_bind_ioctl_check_args()
3708 XE_IOCTL_DBG(xe, (prefetch_region != DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC && in vm_bind_ioctl_check_args()
3710 (prefetch_region >= (sizeof(xe->info.mem_region_mask) * 8) || in vm_bind_ioctl_check_args()
3711 !(BIT(prefetch_region) & xe->info.mem_region_mask)))) || in vm_bind_ioctl_check_args()
3712 XE_IOCTL_DBG(xe, obj && in vm_bind_ioctl_check_args()
3714 XE_IOCTL_DBG(xe, (flags & DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET) && in vm_bind_ioctl_check_args()
3720 if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) || in vm_bind_ioctl_check_args()
3721 XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) || in vm_bind_ioctl_check_args()
3722 XE_IOCTL_DBG(xe, range & ~PAGE_MASK) || in vm_bind_ioctl_check_args()
3723 XE_IOCTL_DBG(xe, !range && in vm_bind_ioctl_check_args()
3729 if (is_decompress && (XE_IOCTL_DBG(xe, !xe_device_has_flat_ccs(xe)) || in vm_bind_ioctl_check_args()
3730 XE_IOCTL_DBG(xe, GRAPHICS_VER(xe) < 20) || in vm_bind_ioctl_check_args()
3731 XE_IOCTL_DBG(xe, !IS_DGFX(xe)))) { in vm_bind_ioctl_check_args()
3782 static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo, in xe_vm_bind_ioctl_validate_bo() argument
3789 if (XE_IOCTL_DBG(xe, (bo->flags & XE_BO_FLAG_NO_COMPRESSION) && in xe_vm_bind_ioctl_validate_bo()
3790 xe_pat_index_get_comp_en(xe, pat_index))) in xe_vm_bind_ioctl_validate_bo()
3793 if (XE_IOCTL_DBG(xe, range > xe_bo_size(bo)) || in xe_vm_bind_ioctl_validate_bo()
3794 XE_IOCTL_DBG(xe, obj_offset > in xe_vm_bind_ioctl_validate_bo()
3808 (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)) { in xe_vm_bind_ioctl_validate_bo()
3809 if (XE_IOCTL_DBG(xe, obj_offset & in xe_vm_bind_ioctl_validate_bo()
3811 XE_IOCTL_DBG(xe, addr & XE_64K_PAGE_MASK) || in xe_vm_bind_ioctl_validate_bo()
3812 XE_IOCTL_DBG(xe, range & XE_64K_PAGE_MASK)) { in xe_vm_bind_ioctl_validate_bo()
3817 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); in xe_vm_bind_ioctl_validate_bo()
3819 if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && in xe_vm_bind_ioctl_validate_bo()
3823 } else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) { in xe_vm_bind_ioctl_validate_bo()
3837 comp_en = xe_pat_index_get_comp_en(xe, pat_index); in xe_vm_bind_ioctl_validate_bo()
3838 if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && comp_en)) in xe_vm_bind_ioctl_validate_bo()
3841 if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && xe_device_is_l2_flush_optimized(xe) && in xe_vm_bind_ioctl_validate_bo()
3848 if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0)) in xe_vm_bind_ioctl_validate_bo()
3856 struct xe_device *xe = to_xe_device(dev); in xe_vm_bind_ioctl() local
3873 if (XE_IOCTL_DBG(xe, !vm)) in xe_vm_bind_ioctl()
3876 err = vm_bind_ioctl_check_args(xe, vm, args, &bind_ops); in xe_vm_bind_ioctl()
3882 if (XE_IOCTL_DBG(xe, !q)) { in xe_vm_bind_ioctl()
3887 if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { in xe_vm_bind_ioctl()
3893 if (XE_IOCTL_DBG(xe, q && vm != q->user_vm)) { in xe_vm_bind_ioctl()
3905 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { in xe_vm_bind_ioctl()
3914 if (XE_IOCTL_DBG(xe, range > vm->size) || in xe_vm_bind_ioctl()
3915 XE_IOCTL_DBG(xe, addr > vm->size - range)) { in xe_vm_bind_ioctl()
3951 if (XE_IOCTL_DBG(xe, !gem_obj)) { in xe_vm_bind_ioctl()
3957 err = xe_vm_bind_ioctl_validate_bo(xe, bos[i], addr, range, in xe_vm_bind_ioctl()
3976 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], in xe_vm_bind_ioctl()
3991 if (XE_IOCTL_DBG(xe, num_ufence > 1)) { in xe_vm_bind_ioctl()
4029 vm->xe->vm_inject_error_position = in xe_vm_bind_ioctl()
4030 (vm->xe->vm_inject_error_position + 1) % in xe_vm_bind_ioctl()
4132 fault_entry.address = xe_device_canonicalize_addr(vm->xe, entry->address); in fill_faults()
4187 struct xe_device *xe = to_xe_device(drm); in xe_vm_get_property_ioctl() local
4193 if (XE_IOCTL_DBG(xe, (args->reserved[0] || args->reserved[1] || in xe_vm_get_property_ioctl()
4199 if (XE_IOCTL_DBG(xe, !vm)) in xe_vm_get_property_ioctl()
4242 vm->xe->pat.idx[cache_lvl]); in xe_vm_bind_kernel_bo()
4252 xe_assert(vm->xe, !list_empty(&vops.list)); in xe_vm_bind_kernel_bo()
4331 struct xe_device *xe = xe_vma_vm(vma)->xe; in xe_vm_invalidate_vma_submit() local
4338 xe_assert(xe, !xe_vma_is_null(vma)); in xe_vm_invalidate_vma_submit()
4339 xe_assert(xe, !xe_vma_is_cpu_addr_mirror(vma)); in xe_vm_invalidate_vma_submit()
4342 vm_dbg(&vm->xe->drm, in xe_vm_invalidate_vma_submit()
4367 for_each_tile(tile, xe, id) in xe_vm_invalidate_vma_submit()
4371 xe_device_wmb(xe); in xe_vm_invalidate_vma_submit()
4373 ret = xe_tlb_inval_range_tilemask_submit(xe, xe_vma_vm(vma)->usm.asid, in xe_vm_invalidate_vma_submit()
4424 err = xe_pxp_bo_key_check(vm->xe->pxp, bo); in xe_vm_validate_protected()
4670 int xe_vma_need_vram_for_atomic(struct xe_device *xe, struct xe_vma *vma, bool is_atomic) in xe_vma_need_vram_for_atomic() argument
4675 if (!IS_DGFX(xe) || !is_atomic) in xe_vma_need_vram_for_atomic()
4685 return !xe->info.has_device_atomics_on_smem; in xe_vma_need_vram_for_atomic()
4750 xe_assert(vm->xe, !remap_op); in xe_vm_alloc_vma()
4751 xe_assert(vm->xe, xe_vma_has_no_bo(vma)); in xe_vm_alloc_vma()
4757 xe_assert(vm->xe, remap_op); in xe_vm_alloc_vma()
4768 print_op(vm->xe, __op); in xe_vm_alloc_vma()
4846 vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); in xe_vm_alloc_madvise_vma()
4910 vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx", in xe_vm_alloc_cpu_addr_mirror_vma()
4926 struct xe_device *xe = vm->xe; in xe_vm_add_exec_queue() local
4929 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_KERNEL)); in xe_vm_add_exec_queue()
4930 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); in xe_vm_add_exec_queue()
4931 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); in xe_vm_add_exec_queue()
4932 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_MIGRATE)); in xe_vm_add_exec_queue()
4933 xe_assert(xe, vm->xef); in xe_vm_add_exec_queue()
4934 xe_assert(xe, vm == q->vm); in xe_vm_add_exec_queue()
4936 if (!xe->info.has_ctx_tlb_inval) in xe_vm_add_exec_queue()
4955 if (!vm->xe->info.has_ctx_tlb_inval) in xe_vm_remove_exec_queue()