Lines Matching +full:p +full:- +full:tile
1 // SPDX-License-Identifier: MIT
8 #include <linux/dma-fence-array.h>
47 return vm->gpuvm.r_obj; in xe_vm_obj()
51 * xe_vm_drm_exec_lock() - Lock the vm's resv with a drm_exec transaction
68 lockdep_assert_held(&vm->lock); in preempt_fences_waiting()
71 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in preempt_fences_waiting()
72 if (!q->lr.pfence || in preempt_fences_waiting()
74 &q->lr.pfence->flags)) { in preempt_fences_waiting()
93 lockdep_assert_held(&vm->lock); in alloc_preempt_fences()
96 if (*count >= vm->preempt.num_exec_queues) in alloc_preempt_fences()
99 for (; *count < vm->preempt.num_exec_queues; ++(*count)) { in alloc_preempt_fences()
114 bool vf_migration = IS_SRIOV_VF(vm->xe) && in wait_for_existing_preempt_fences()
115 xe_sriov_vf_migration_supported(vm->xe); in wait_for_existing_preempt_fences()
120 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in wait_for_existing_preempt_fences()
121 if (q->lr.pfence) { in wait_for_existing_preempt_fences()
124 timeout = dma_fence_wait_timeout(q->lr.pfence, false, in wait_for_existing_preempt_fences()
127 xe_assert(vm->xe, vf_migration); in wait_for_existing_preempt_fences()
128 return -EAGAIN; in wait_for_existing_preempt_fences()
131 /* Only -ETIME on fence indicates VM needs to be killed */ in wait_for_existing_preempt_fences()
132 if (timeout < 0 || q->lr.pfence->error == -ETIME) in wait_for_existing_preempt_fences()
133 return -ETIME; in wait_for_existing_preempt_fences()
135 dma_fence_put(q->lr.pfence); in wait_for_existing_preempt_fences()
136 q->lr.pfence = NULL; in wait_for_existing_preempt_fences()
148 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in xe_vm_is_idle()
161 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in arm_preempt_fences()
164 link = list->next; in arm_preempt_fences()
165 xe_assert(vm->xe, link != list); in arm_preempt_fences()
168 q, q->lr.context, in arm_preempt_fences()
169 ++q->lr.seqno); in arm_preempt_fences()
170 dma_fence_put(q->lr.pfence); in arm_preempt_fences()
171 q->lr.pfence = fence; in arm_preempt_fences()
182 if (!vm->preempt.num_exec_queues) in add_preempt_fences()
185 err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues); in add_preempt_fences()
189 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) in add_preempt_fences()
190 if (q->lr.pfence) { in add_preempt_fences()
191 dma_resv_add_fence(bo->ttm.base.resv, in add_preempt_fences()
192 q->lr.pfence, in add_preempt_fences()
204 lockdep_assert_held(&vm->lock); in resume_and_reinstall_preempt_fences()
207 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in resume_and_reinstall_preempt_fences()
208 q->ops->resume(q); in resume_and_reinstall_preempt_fences()
210 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->lr.pfence, in resume_and_reinstall_preempt_fences()
218 .vm = &vm->gpuvm, in xe_vm_add_compute_exec_queue()
228 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); in xe_vm_add_compute_exec_queue()
230 down_write(&vm->lock); in xe_vm_add_compute_exec_queue()
231 err = xe_validation_exec_lock(&ctx, &vm_exec, &vm->xe->val); in xe_vm_add_compute_exec_queue()
235 pfence = xe_preempt_fence_create(q, q->lr.context, in xe_vm_add_compute_exec_queue()
236 ++q->lr.seqno); in xe_vm_add_compute_exec_queue()
242 list_add(&q->lr.link, &vm->preempt.exec_queues); in xe_vm_add_compute_exec_queue()
243 ++vm->preempt.num_exec_queues; in xe_vm_add_compute_exec_queue()
244 q->lr.pfence = pfence; in xe_vm_add_compute_exec_queue()
248 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence, in xe_vm_add_compute_exec_queue()
265 up_write(&vm->lock); in xe_vm_add_compute_exec_queue()
272 * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM
283 down_write(&vm->lock); in xe_vm_remove_compute_exec_queue()
284 if (!list_empty(&q->lr.link)) { in xe_vm_remove_compute_exec_queue()
285 list_del_init(&q->lr.link); in xe_vm_remove_compute_exec_queue()
286 --vm->preempt.num_exec_queues; in xe_vm_remove_compute_exec_queue()
288 if (q->lr.pfence) { in xe_vm_remove_compute_exec_queue()
289 dma_fence_enable_sw_signaling(q->lr.pfence); in xe_vm_remove_compute_exec_queue()
290 dma_fence_put(q->lr.pfence); in xe_vm_remove_compute_exec_queue()
291 q->lr.pfence = NULL; in xe_vm_remove_compute_exec_queue()
293 up_write(&vm->lock); in xe_vm_remove_compute_exec_queue()
299 * xe_vm_kill() - VM Kill
301 * @unlocked: Flag indicates the VM's dma-resv is not held
310 lockdep_assert_held(&vm->lock); in xe_vm_kill()
315 vm->flags |= XE_VM_FLAG_BANNED; in xe_vm_kill()
318 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) in xe_vm_kill()
319 q->ops->kill(q); in xe_vm_kill()
329 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); in xe_gpuvm_validate()
333 lockdep_assert_held(&vm->lock); in xe_gpuvm_validate()
335 list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind, in xe_gpuvm_validate()
336 &vm->rebind_list); in xe_gpuvm_validate()
338 if (!try_wait_for_completion(&vm->xe->pm_block)) in xe_gpuvm_validate()
339 return -EAGAIN; in xe_gpuvm_validate()
341 ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false, exec); in xe_gpuvm_validate()
345 vm_bo->evicted = false; in xe_gpuvm_validate()
350 * xe_vm_validate_rebind() - Validate buffer objects and rebind vmas
357 * rebindings may cause evictions and hence the validation-rebind
361 * may return -EINTR or -ERESTARTSYS if interrupted, and -EDEADLK if
372 ret = drm_gpuvm_validate(&vm->gpuvm, exec); in xe_vm_validate_rebind()
379 } while (!list_empty(&vm->gpuvm.evict.list)); in xe_vm_validate_rebind()
382 ret = dma_resv_reserve_fences(obj->resv, num_fences); in xe_vm_validate_rebind()
395 err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, 0); in xe_preempt_work_begin()
400 vm->preempt.rebind_deactivated = true; in xe_preempt_work_begin()
410 err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, 0); in xe_preempt_work_begin()
424 return xe_vm_validate_rebind(vm, exec, vm->preempt.num_exec_queues); in xe_preempt_work_begin()
429 struct xe_device *xe = vm->xe; in vm_suspend_rebind_worker()
432 mutex_lock(&xe->rebind_resume_lock); in vm_suspend_rebind_worker()
433 if (!try_wait_for_completion(&vm->xe->pm_block)) { in vm_suspend_rebind_worker()
435 list_move_tail(&vm->preempt.pm_activate_link, &xe->rebind_resume_list); in vm_suspend_rebind_worker()
437 mutex_unlock(&xe->rebind_resume_lock); in vm_suspend_rebind_worker()
443 * xe_vm_resume_rebind_worker() - Resume the rebind worker.
451 queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work); in xe_vm_resume_rebind_worker()
465 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); in preempt_rebind_work_func()
468 down_write(&vm->lock); in preempt_rebind_work_func()
471 up_write(&vm->lock); in preempt_rebind_work_func()
477 if (!try_wait_for_completion(&vm->xe->pm_block) && vm_suspend_rebind_worker(vm)) { in preempt_rebind_work_func()
478 up_write(&vm->lock); in preempt_rebind_work_func()
490 err = xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, in preempt_rebind_work_func()
522 err = -ETIME; in preempt_rebind_work_func()
534 err = -EAGAIN; in preempt_rebind_work_func()
540 spin_lock(&vm->xe->ttm.lru_lock); in preempt_rebind_work_func()
541 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); in preempt_rebind_work_func()
542 spin_unlock(&vm->xe->ttm.lru_lock); in preempt_rebind_work_func()
552 if (err == -EAGAIN) { in preempt_rebind_work_func()
557 * given this can block the VF post-migration workers from in preempt_rebind_work_func()
560 if (IS_SRIOV_VF(vm->xe) && in preempt_rebind_work_func()
561 xe_sriov_vf_migration_supported(vm->xe)) { in preempt_rebind_work_func()
562 up_write(&vm->lock); in preempt_rebind_work_func()
571 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err); in preempt_rebind_work_func()
574 up_write(&vm->lock); in preempt_rebind_work_func()
586 if (!vops->pt_update_ops[i].num_ops) in xe_vma_ops_alloc()
589 vops->pt_update_ops[i].ops = in xe_vma_ops_alloc()
590 kmalloc_array(vops->pt_update_ops[i].num_ops, in xe_vma_ops_alloc()
591 sizeof(*vops->pt_update_ops[i].ops), in xe_vma_ops_alloc()
593 if (!vops->pt_update_ops[i].ops) in xe_vma_ops_alloc()
594 return array_of_binds ? -ENOBUFS : -ENOMEM; in xe_vma_ops_alloc()
605 vma = gpuva_to_vma(op->base.prefetch.va); in xe_vma_svm_prefetch_op_fini()
607 if (op->base.op == DRM_GPUVA_OP_PREFETCH && xe_vma_is_cpu_addr_mirror(vma)) in xe_vma_svm_prefetch_op_fini()
608 xa_destroy(&op->prefetch_range.range); in xe_vma_svm_prefetch_op_fini()
615 if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) in xe_vma_svm_prefetch_ops_fini()
618 list_for_each_entry(op, &vops->list, link) in xe_vma_svm_prefetch_ops_fini()
629 kfree(vops->pt_update_ops[i].ops); in xe_vma_ops_fini()
641 vops->pt_update_ops[i].num_ops += inc_val; in xe_vma_ops_incr_pt_update_ops()
654 INIT_LIST_HEAD(&op->link); in xe_vm_populate_rebind()
655 op->tile_mask = tile_mask; in xe_vm_populate_rebind()
656 op->base.op = DRM_GPUVA_OP_MAP; in xe_vm_populate_rebind()
657 op->base.map.va.addr = vma->gpuva.va.addr; in xe_vm_populate_rebind()
658 op->base.map.va.range = vma->gpuva.va.range; in xe_vm_populate_rebind()
659 op->base.map.gem.obj = vma->gpuva.gem.obj; in xe_vm_populate_rebind()
660 op->base.map.gem.offset = vma->gpuva.gem.offset; in xe_vm_populate_rebind()
661 op->map.vma = vma; in xe_vm_populate_rebind()
662 op->map.immediate = true; in xe_vm_populate_rebind()
663 op->map.vma_flags = vma->gpuva.flags & XE_VMA_CREATE_MASK; in xe_vm_populate_rebind()
673 return -ENOMEM; in xe_vm_ops_add_rebind()
676 list_add_tail(&op->link, &vops->list); in xe_vm_ops_add_rebind()
696 lockdep_assert_held(&vm->lock); in xe_vm_rebind()
698 list_empty(&vm->rebind_list)) in xe_vm_rebind()
706 list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) { in xe_vm_rebind()
707 xe_assert(vm->xe, vma->tile_present); in xe_vm_rebind()
715 vma->tile_present); in xe_vm_rebind()
729 list_for_each_entry_safe(vma, next, &vm->rebind_list, in xe_vm_rebind()
731 list_del_init(&vma->combined_links.rebind); in xe_vm_rebind()
735 list_del(&op->link); in xe_vm_rebind()
748 struct xe_tile *tile; in xe_vma_rebind() local
752 lockdep_assert_held(&vm->lock); in xe_vma_rebind()
754 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vma_rebind()
758 for_each_tile(tile, vm->xe, id) { in xe_vma_rebind()
760 vops.pt_update_ops[tile->id].q = in xe_vma_rebind()
761 xe_migrate_exec_queue(tile->migrate); in xe_vma_rebind()
778 list_del(&op->link); in xe_vma_rebind()
791 INIT_LIST_HEAD(&op->link); in xe_vm_populate_range_rebind()
792 op->tile_mask = tile_mask; in xe_vm_populate_range_rebind()
793 op->base.op = DRM_GPUVA_OP_DRIVER; in xe_vm_populate_range_rebind()
794 op->subop = XE_VMA_SUBOP_MAP_RANGE; in xe_vm_populate_range_rebind()
795 op->map_range.vma = vma; in xe_vm_populate_range_rebind()
796 op->map_range.range = range; in xe_vm_populate_range_rebind()
809 return -ENOMEM; in xe_vm_ops_add_range_rebind()
812 list_add_tail(&op->link, &vops->list); in xe_vm_ops_add_range_rebind()
819 * xe_vm_range_rebind() - VM range (re)bind
823 * @tile_mask: Tile mask to bind the range to.
838 struct xe_tile *tile; in xe_vm_range_rebind() local
842 lockdep_assert_held(&vm->lock); in xe_vm_range_rebind()
844 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vm_range_rebind()
845 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); in xe_vm_range_rebind()
849 for_each_tile(tile, vm->xe, id) { in xe_vm_range_rebind()
851 vops.pt_update_ops[tile->id].q = in xe_vm_range_rebind()
852 xe_migrate_exec_queue(tile->migrate); in xe_vm_range_rebind()
869 list_del(&op->link); in xe_vm_range_rebind()
880 INIT_LIST_HEAD(&op->link); in xe_vm_populate_range_unbind()
881 op->tile_mask = range->tile_present; in xe_vm_populate_range_unbind()
882 op->base.op = DRM_GPUVA_OP_DRIVER; in xe_vm_populate_range_unbind()
883 op->subop = XE_VMA_SUBOP_UNMAP_RANGE; in xe_vm_populate_range_unbind()
884 op->unmap_range.range = range; in xe_vm_populate_range_unbind()
895 return -ENOMEM; in xe_vm_ops_add_range_unbind()
898 list_add_tail(&op->link, &vops->list); in xe_vm_ops_add_range_unbind()
899 xe_vma_ops_incr_pt_update_ops(vops, range->tile_present, 1); in xe_vm_ops_add_range_unbind()
905 * xe_vm_range_unbind() - VM range unbind
920 struct xe_tile *tile; in xe_vm_range_unbind() local
924 lockdep_assert_held(&vm->lock); in xe_vm_range_unbind()
926 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vm_range_unbind()
928 if (!range->tile_present) in xe_vm_range_unbind()
932 for_each_tile(tile, vm->xe, id) { in xe_vm_range_unbind()
934 vops.pt_update_ops[tile->id].q = in xe_vm_range_unbind()
935 xe_migrate_exec_queue(tile->migrate); in xe_vm_range_unbind()
952 list_del(&op->link); in xe_vm_range_unbind()
976 struct xe_tile *tile; in xe_vma_create() local
981 xe_assert(vm->xe, start < end); in xe_vma_create()
982 xe_assert(vm->xe, end < vm->size); in xe_vma_create()
992 return ERR_PTR(-ENOMEM); in xe_vma_create()
994 vma = &uvma->vma; in xe_vma_create()
998 return ERR_PTR(-ENOMEM); in xe_vma_create()
1001 vma->gpuva.gem.obj = &bo->ttm.base; in xe_vma_create()
1004 INIT_LIST_HEAD(&vma->combined_links.rebind); in xe_vma_create()
1006 INIT_LIST_HEAD(&vma->gpuva.gem.entry); in xe_vma_create()
1007 vma->gpuva.vm = &vm->gpuvm; in xe_vma_create()
1008 vma->gpuva.va.addr = start; in xe_vma_create()
1009 vma->gpuva.va.range = end - start + 1; in xe_vma_create()
1010 vma->gpuva.flags = flags; in xe_vma_create()
1012 for_each_tile(tile, vm->xe, id) in xe_vma_create()
1013 vma->tile_mask |= 0x1 << id; in xe_vma_create()
1015 if (vm->xe->info.has_atomic_enable_pte_bit) in xe_vma_create()
1016 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; in xe_vma_create()
1018 vma->attr = *attr; in xe_vma_create()
1025 vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base); in xe_vma_create()
1032 drm_gem_object_get(&bo->ttm.base); in xe_vma_create()
1033 vma->gpuva.gem.offset = bo_offset_or_userptr; in xe_vma_create()
1034 drm_gpuva_link(&vma->gpuva, vm_bo); in xe_vma_create()
1039 u64 size = end - start + 1; in xe_vma_create()
1042 vma->gpuva.gem.offset = bo_offset_or_userptr; in xe_vma_create()
1061 if (vma->ufence) { in xe_vma_destroy_late()
1062 xe_sync_ufence_put(vma->ufence); in xe_vma_destroy_late()
1063 vma->ufence = NULL; in xe_vma_destroy_late()
1093 INIT_WORK(&vma->destroy_work, vma_destroy_work_func); in vma_destroy_cb()
1094 queue_work(system_unbound_wq, &vma->destroy_work); in vma_destroy_cb()
1101 lockdep_assert_held_write(&vm->lock); in xe_vma_destroy()
1102 xe_assert(vm->xe, list_empty(&vma->combined_links.destroy)); in xe_vma_destroy()
1105 xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED); in xe_vma_destroy()
1110 drm_gpuva_unlink(&vma->gpuva); in xe_vma_destroy()
1115 int ret = dma_fence_add_callback(fence, &vma->destroy_cb, in xe_vma_destroy()
1119 XE_WARN_ON(ret != -ENOENT); in xe_vma_destroy()
1128 * xe_vm_lock_vma() - drm_exec utility to lock a vma
1134 * may return -EDEADLK on WW transaction contention and -EINTR if
1146 if (!err && bo && !bo->vm) in xe_vm_lock_vma()
1147 err = drm_exec_lock_obj(exec, &bo->ttm.base); in xe_vm_lock_vma()
1154 struct xe_device *xe = xe_vma_vm(vma)->xe; in xe_vma_destroy_unlocked()
1159 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { in xe_vma_destroy_unlocked()
1174 lockdep_assert_held(&vm->lock); in xe_vm_find_overlapping_vma()
1179 xe_assert(vm->xe, start + range <= vm->size); in xe_vm_find_overlapping_vma()
1181 gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range); in xe_vm_find_overlapping_vma()
1190 xe_assert(vm->xe, xe_vma_vm(vma) == vm); in xe_vm_insert_vma()
1191 lockdep_assert_held(&vm->lock); in xe_vm_insert_vma()
1193 mutex_lock(&vm->snap_mutex); in xe_vm_insert_vma()
1194 err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva); in xe_vm_insert_vma()
1195 mutex_unlock(&vm->snap_mutex); in xe_vm_insert_vma()
1203 xe_assert(vm->xe, xe_vma_vm(vma) == vm); in xe_vm_remove_vma()
1204 lockdep_assert_held(&vm->lock); in xe_vm_remove_vma()
1206 mutex_lock(&vm->snap_mutex); in xe_vm_remove_vma()
1207 drm_gpuva_remove(&vma->gpuva); in xe_vm_remove_vma()
1208 mutex_unlock(&vm->snap_mutex); in xe_vm_remove_vma()
1209 if (vm->usm.last_fault_vma == vma) in xe_vm_remove_vma()
1210 vm->usm.last_fault_vma = NULL; in xe_vm_remove_vma()
1222 return &op->base; in xe_vm_op_alloc()
1290 * We only have two bits to encode the PAT index in non-leaf nodes, but in pde_pat_index()
1294 * non-leaf nodes are instead under driver control so the chosen index in pde_pat_index()
1300 if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached) in pde_pat_index()
1301 pat_index = xe->pat.idx[XE_CACHE_WB]; in pde_pat_index()
1303 pat_index = xe->pat.idx[XE_CACHE_NONE]; in pde_pat_index()
1386 * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the
1387 * given tile and vm.
1389 * @tile: tile to set up for.
1393 * Sets up a pagetable tree with one page-table per level and a single
1394 * leaf PTE. All pagetable entries point to the single page-table or,
1400 static int xe_vm_create_scratch(struct xe_device *xe, struct xe_tile *tile, in xe_vm_create_scratch() argument
1403 u8 id = tile->id; in xe_vm_create_scratch()
1406 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) { in xe_vm_create_scratch()
1407 vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i, exec); in xe_vm_create_scratch()
1408 if (IS_ERR(vm->scratch_pt[id][i])) { in xe_vm_create_scratch()
1409 int err = PTR_ERR(vm->scratch_pt[id][i]); in xe_vm_create_scratch()
1411 vm->scratch_pt[id][i] = NULL; in xe_vm_create_scratch()
1414 xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); in xe_vm_create_scratch()
1423 struct xe_tile *tile; in xe_vm_free_scratch() local
1429 for_each_tile(tile, vm->xe, id) { in xe_vm_free_scratch()
1432 if (!vm->pt_root[id]) in xe_vm_free_scratch()
1435 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i) in xe_vm_free_scratch()
1436 if (vm->scratch_pt[id][i]) in xe_vm_free_scratch()
1437 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL); in xe_vm_free_scratch()
1443 struct xe_tile *tile; in xe_vm_pt_destroy() local
1448 for_each_tile(tile, vm->xe, id) { in xe_vm_pt_destroy()
1449 if (vm->pt_root[id]) { in xe_vm_pt_destroy()
1450 xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); in xe_vm_pt_destroy()
1451 vm->pt_root[id] = NULL; in xe_vm_pt_destroy()
1463 struct xe_tile *tile; in xe_vm_create() local
1467 * Since the GSCCS is not user-accessible, we don't expect a GSC VM to in xe_vm_create()
1474 return ERR_PTR(-ENOMEM); in xe_vm_create()
1476 vm->xe = xe; in xe_vm_create()
1478 vm->size = 1ull << xe->info.va_bits; in xe_vm_create()
1479 vm->flags = flags; in xe_vm_create()
1482 vm->xef = xe_file_get(xef); in xe_vm_create()
1484 * GSC VMs are kernel-owned, only used for PXP ops and can sometimes be in xe_vm_create()
1486 * under a user-VM lock when the PXP session is started at exec_queue in xe_vm_create()
1494 __init_rwsem(&vm->lock, "gsc_vm", &gsc_vm_key); in xe_vm_create()
1496 init_rwsem(&vm->lock); in xe_vm_create()
1498 mutex_init(&vm->snap_mutex); in xe_vm_create()
1500 INIT_LIST_HEAD(&vm->rebind_list); in xe_vm_create()
1502 INIT_LIST_HEAD(&vm->userptr.repin_list); in xe_vm_create()
1503 INIT_LIST_HEAD(&vm->userptr.invalidated); in xe_vm_create()
1504 spin_lock_init(&vm->userptr.invalidated_lock); in xe_vm_create()
1506 ttm_lru_bulk_move_init(&vm->lru_bulk_move); in xe_vm_create()
1508 INIT_WORK(&vm->destroy_work, vm_destroy_work_func); in xe_vm_create()
1510 INIT_LIST_HEAD(&vm->preempt.exec_queues); in xe_vm_create()
1511 vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ in xe_vm_create()
1513 for_each_tile(tile, xe, id) in xe_vm_create()
1514 xe_range_fence_tree_init(&vm->rftree[id]); in xe_vm_create()
1516 vm->pt_ops = &xelp_pt_ops; in xe_vm_create()
1519 * Long-running workloads are not protected by the scheduler references. in xe_vm_create()
1520 * By design, run_job for long-running workloads returns NULL and the in xe_vm_create()
1525 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); in xe_vm_create()
1527 INIT_LIST_HEAD(&vm->preempt.pm_activate_link); in xe_vm_create()
1534 vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm); in xe_vm_create()
1536 err = -ENOMEM; in xe_vm_create()
1540 drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm, in xe_vm_create()
1541 vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops); in xe_vm_create()
1546 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, in xe_vm_create()
1551 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) in xe_vm_create()
1552 vm->flags |= XE_VM_FLAG_64K; in xe_vm_create()
1554 for_each_tile(tile, xe, id) { in xe_vm_create()
1556 tile->id != XE_VM_FLAG_TILE_ID(flags)) in xe_vm_create()
1559 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level, in xe_vm_create()
1561 if (IS_ERR(vm->pt_root[id])) { in xe_vm_create()
1562 err = PTR_ERR(vm->pt_root[id]); in xe_vm_create()
1563 vm->pt_root[id] = NULL; in xe_vm_create()
1574 for_each_tile(tile, xe, id) { in xe_vm_create()
1575 if (!vm->pt_root[id]) in xe_vm_create()
1578 err = xe_vm_create_scratch(xe, tile, vm, &exec); in xe_vm_create()
1589 vm->batch_invalidate_tlb = true; in xe_vm_create()
1592 if (vm->flags & XE_VM_FLAG_LR_MODE) { in xe_vm_create()
1593 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); in xe_vm_create()
1594 vm->batch_invalidate_tlb = false; in xe_vm_create()
1598 for_each_tile(tile, xe, id) { in xe_vm_create()
1599 if (!vm->pt_root[id]) in xe_vm_create()
1602 xe_pt_populate_empty(tile, vm, vm->pt_root[id]); in xe_vm_create()
1610 for_each_tile(tile, xe, id) { in xe_vm_create()
1614 if (!vm->pt_root[id]) in xe_vm_create()
1617 q = xe_exec_queue_create_bind(xe, tile, create_flags, 0); in xe_vm_create()
1622 vm->q[id] = q; in xe_vm_create()
1626 if (xef && xe->info.has_asid) { in xe_vm_create()
1629 down_write(&xe->usm.lock); in xe_vm_create()
1630 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, in xe_vm_create()
1631 XA_LIMIT(1, XE_MAX_ASID - 1), in xe_vm_create()
1632 &xe->usm.next_asid, GFP_KERNEL); in xe_vm_create()
1633 up_write(&xe->usm.lock); in xe_vm_create()
1637 vm->usm.asid = asid; in xe_vm_create()
1650 vm->size = 0; /* close the vm */ in xe_vm_create()
1654 mutex_destroy(&vm->snap_mutex); in xe_vm_create()
1655 for_each_tile(tile, xe, id) in xe_vm_create()
1656 xe_range_fence_tree_fini(&vm->rftree[id]); in xe_vm_create()
1657 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); in xe_vm_create()
1658 if (vm->xef) in xe_vm_create()
1659 xe_file_put(vm->xef); in xe_vm_create()
1668 struct xe_device *xe = vm->xe; in xe_vm_close()
1672 bound = drm_dev_enter(&xe->drm, &idx); in xe_vm_close()
1674 down_write(&vm->lock); in xe_vm_close()
1678 vm->size = 0; in xe_vm_close()
1680 if (!((vm->flags & XE_VM_FLAG_MIGRATION))) { in xe_vm_close()
1681 struct xe_tile *tile; in xe_vm_close() local
1691 for_each_tile(tile, xe, id) in xe_vm_close()
1692 if (vm->pt_root[id]) in xe_vm_close()
1693 xe_pt_clear(xe, vm->pt_root[id]); in xe_vm_close()
1696 xe_tlb_inval_vm(>->tlb_inval, vm); in xe_vm_close()
1702 up_write(&vm->lock); in xe_vm_close()
1711 struct xe_device *xe = vm->xe; in xe_vm_close_and_put()
1712 struct xe_tile *tile; in xe_vm_close_and_put() local
1717 xe_assert(xe, !vm->preempt.num_exec_queues); in xe_vm_close_and_put()
1721 mutex_lock(&xe->rebind_resume_lock); in xe_vm_close_and_put()
1722 list_del_init(&vm->preempt.pm_activate_link); in xe_vm_close_and_put()
1723 mutex_unlock(&xe->rebind_resume_lock); in xe_vm_close_and_put()
1724 flush_work(&vm->preempt.rebind_work); in xe_vm_close_and_put()
1729 down_write(&vm->lock); in xe_vm_close_and_put()
1730 for_each_tile(tile, xe, id) { in xe_vm_close_and_put()
1731 if (vm->q[id]) { in xe_vm_close_and_put()
1734 xe_exec_queue_last_fence_put(vm->q[id], vm); in xe_vm_close_and_put()
1736 xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i); in xe_vm_close_and_put()
1739 up_write(&vm->lock); in xe_vm_close_and_put()
1741 for_each_tile(tile, xe, id) { in xe_vm_close_and_put()
1742 if (vm->q[id]) { in xe_vm_close_and_put()
1743 xe_exec_queue_kill(vm->q[id]); in xe_vm_close_and_put()
1744 xe_exec_queue_put(vm->q[id]); in xe_vm_close_and_put()
1745 vm->q[id] = NULL; in xe_vm_close_and_put()
1749 down_write(&vm->lock); in xe_vm_close_and_put()
1751 drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) { in xe_vm_close_and_put()
1756 vma->gpuva.flags |= XE_VMA_DESTROYED; in xe_vm_close_and_put()
1763 if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) { in xe_vm_close_and_put()
1764 list_del_init(&vma->combined_links.rebind); in xe_vm_close_and_put()
1769 list_move_tail(&vma->combined_links.destroy, &contested); in xe_vm_close_and_put()
1770 vma->gpuva.flags |= XE_VMA_DESTROYED; in xe_vm_close_and_put()
1785 * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL in xe_vm_close_and_put()
1791 list_del_init(&vma->combined_links.destroy); in xe_vm_close_and_put()
1797 up_write(&vm->lock); in xe_vm_close_and_put()
1799 down_write(&xe->usm.lock); in xe_vm_close_and_put()
1800 if (vm->usm.asid) { in xe_vm_close_and_put()
1803 xe_assert(xe, xe->info.has_asid); in xe_vm_close_and_put()
1804 xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION)); in xe_vm_close_and_put()
1806 lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); in xe_vm_close_and_put()
1809 up_write(&xe->usm.lock); in xe_vm_close_and_put()
1811 for_each_tile(tile, xe, id) in xe_vm_close_and_put()
1812 xe_range_fence_tree_fini(&vm->rftree[id]); in xe_vm_close_and_put()
1821 struct xe_device *xe = vm->xe; in vm_destroy_work_func()
1822 struct xe_tile *tile; in vm_destroy_work_func() local
1826 xe_assert(xe, !vm->size); in vm_destroy_work_func()
1829 flush_work(&vm->preempt.rebind_work); in vm_destroy_work_func()
1831 mutex_destroy(&vm->snap_mutex); in vm_destroy_work_func()
1833 if (vm->flags & XE_VM_FLAG_LR_MODE) in vm_destroy_work_func()
1836 for_each_tile(tile, xe, id) in vm_destroy_work_func()
1837 XE_WARN_ON(vm->pt_root[id]); in vm_destroy_work_func()
1841 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); in vm_destroy_work_func()
1843 if (vm->xef) in vm_destroy_work_func()
1844 xe_file_put(vm->xef); in vm_destroy_work_func()
1854 queue_work(system_unbound_wq, &vm->destroy_work); in xe_vm_free()
1861 mutex_lock(&xef->vm.lock); in xe_vm_lookup()
1862 vm = xa_load(&xef->vm.xa, id); in xe_vm_lookup()
1865 mutex_unlock(&xef->vm.lock); in xe_vm_lookup()
1870 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) in xe_vm_pdp4_descriptor() argument
1872 return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0); in xe_vm_pdp4_descriptor()
1878 return q ? q : vm->q[0]; in to_wait_exec_queue()
1912 if (XE_IOCTL_DBG(xe, args->extensions)) in xe_vm_create_ioctl()
1913 return -EINVAL; in xe_vm_create_ioctl()
1916 args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; in xe_vm_create_ioctl()
1918 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && in xe_vm_create_ioctl()
1919 !xe->info.has_usm)) in xe_vm_create_ioctl()
1920 return -EINVAL; in xe_vm_create_ioctl()
1922 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_vm_create_ioctl()
1923 return -EINVAL; in xe_vm_create_ioctl()
1925 if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) in xe_vm_create_ioctl()
1926 return -EINVAL; in xe_vm_create_ioctl()
1928 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && in xe_vm_create_ioctl()
1929 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && in xe_vm_create_ioctl()
1930 !xe->info.needs_scratch)) in xe_vm_create_ioctl()
1931 return -EINVAL; in xe_vm_create_ioctl()
1933 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && in xe_vm_create_ioctl()
1934 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) in xe_vm_create_ioctl()
1935 return -EINVAL; in xe_vm_create_ioctl()
1937 if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) in xe_vm_create_ioctl()
1939 if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) in xe_vm_create_ioctl()
1941 if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) in xe_vm_create_ioctl()
1949 /* Warning: Security issue - never enable by default */ in xe_vm_create_ioctl()
1950 args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); in xe_vm_create_ioctl()
1954 err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); in xe_vm_create_ioctl()
1958 args->vm_id = id; in xe_vm_create_ioctl()
1977 if (XE_IOCTL_DBG(xe, args->pad) || in xe_vm_destroy_ioctl()
1978 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_vm_destroy_ioctl()
1979 return -EINVAL; in xe_vm_destroy_ioctl()
1981 mutex_lock(&xef->vm.lock); in xe_vm_destroy_ioctl()
1982 vm = xa_load(&xef->vm.xa, args->vm_id); in xe_vm_destroy_ioctl()
1984 err = -ENOENT; in xe_vm_destroy_ioctl()
1985 else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) in xe_vm_destroy_ioctl()
1986 err = -EBUSY; in xe_vm_destroy_ioctl()
1988 xa_erase(&xef->vm.xa, args->vm_id); in xe_vm_destroy_ioctl()
1989 mutex_unlock(&xef->vm.lock); in xe_vm_destroy_ioctl()
2002 lockdep_assert_held(&vm->lock); in xe_vm_query_vmas()
2003 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) in xe_vm_query_vmas()
2015 lockdep_assert_held(&vm->lock); in get_mem_attrs()
2017 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { in get_mem_attrs()
2021 return -ENOSPC; in get_mem_attrs()
2025 attrs[i].atomic.val = vma->attr.atomic_access; in get_mem_attrs()
2026 attrs[i].pat_index.val = vma->attr.pat_index; in get_mem_attrs()
2027 attrs[i].preferred_mem_loc.devmem_fd = vma->attr.preferred_loc.devmem_fd; in get_mem_attrs()
2029 vma->attr.preferred_loc.migration_policy; in get_mem_attrs()
2044 u64 __user *attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); in xe_vm_query_vmas_attrs_ioctl()
2049 ((args->num_mem_ranges == 0 && in xe_vm_query_vmas_attrs_ioctl()
2050 (attrs_user || args->sizeof_mem_range_attr != 0)) || in xe_vm_query_vmas_attrs_ioctl()
2051 (args->num_mem_ranges > 0 && in xe_vm_query_vmas_attrs_ioctl()
2053 args->sizeof_mem_range_attr != in xe_vm_query_vmas_attrs_ioctl()
2055 return -EINVAL; in xe_vm_query_vmas_attrs_ioctl()
2057 vm = xe_vm_lookup(xef, args->vm_id); in xe_vm_query_vmas_attrs_ioctl()
2059 return -EINVAL; in xe_vm_query_vmas_attrs_ioctl()
2061 err = down_read_interruptible(&vm->lock); in xe_vm_query_vmas_attrs_ioctl()
2065 attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); in xe_vm_query_vmas_attrs_ioctl()
2067 if (args->num_mem_ranges == 0 && !attrs_user) { in xe_vm_query_vmas_attrs_ioctl()
2068 args->num_mem_ranges = xe_vm_query_vmas(vm, args->start, args->start + args->range); in xe_vm_query_vmas_attrs_ioctl()
2069 args->sizeof_mem_range_attr = sizeof(struct drm_xe_mem_range_attr); in xe_vm_query_vmas_attrs_ioctl()
2073 mem_attrs = kvmalloc_array(args->num_mem_ranges, args->sizeof_mem_range_attr, in xe_vm_query_vmas_attrs_ioctl()
2077 err = args->num_mem_ranges > 1 ? -ENOBUFS : -ENOMEM; in xe_vm_query_vmas_attrs_ioctl()
2081 memset(mem_attrs, 0, args->num_mem_ranges * args->sizeof_mem_range_attr); in xe_vm_query_vmas_attrs_ioctl()
2082 err = get_mem_attrs(vm, &args->num_mem_ranges, args->start, in xe_vm_query_vmas_attrs_ioctl()
2083 args->start + args->range, mem_attrs); in xe_vm_query_vmas_attrs_ioctl()
2088 args->sizeof_mem_range_attr * args->num_mem_ranges); in xe_vm_query_vmas_attrs_ioctl()
2090 err = -EFAULT; in xe_vm_query_vmas_attrs_ioctl()
2095 up_read(&vm->lock); in xe_vm_query_vmas_attrs_ioctl()
2103 if (page_addr > xe_vma_end(vma) - 1 || in vma_matches()
2104 page_addr + SZ_4K - 1 < xe_vma_start(vma)) in vma_matches()
2111 * xe_vm_find_vma_by_addr() - Find a VMA by its address
2120 if (vm->usm.last_fault_vma) { /* Fast lookup */ in xe_vm_find_vma_by_addr()
2121 if (vma_matches(vm->usm.last_fault_vma, page_addr)) in xe_vm_find_vma_by_addr()
2122 vma = vm->usm.last_fault_vma; in xe_vm_find_vma_by_addr()
2140 vma->gpuva.flags |= XE_VMA_DESTROYED; in prep_vma_destroy()
2154 switch (op->op) { in print_op()
2156 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx", in print_op()
2157 (ULL)op->map.va.addr, (ULL)op->map.va.range); in print_op()
2160 vma = gpuva_to_vma(op->remap.unmap->va); in print_op()
2161 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", in print_op()
2163 op->remap.unmap->keep ? 1 : 0); in print_op()
2164 if (op->remap.prev) in print_op()
2165 vm_dbg(&xe->drm, in print_op()
2167 (ULL)op->remap.prev->va.addr, in print_op()
2168 (ULL)op->remap.prev->va.range); in print_op()
2169 if (op->remap.next) in print_op()
2170 vm_dbg(&xe->drm, in print_op()
2172 (ULL)op->remap.next->va.addr, in print_op()
2173 (ULL)op->remap.next->va.range); in print_op()
2176 vma = gpuva_to_vma(op->unmap.va); in print_op()
2177 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", in print_op()
2179 op->unmap.keep ? 1 : 0); in print_op()
2182 vma = gpuva_to_vma(op->prefetch.va); in print_op()
2183 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx", in print_op()
2187 drm_warn(&xe->drm, "NOT POSSIBLE"); in print_op()
2232 struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; in vm_bind_ioctl_ops_create()
2239 lockdep_assert_held_write(&vm->lock); in vm_bind_ioctl_ops_create()
2241 vm_dbg(&vm->xe->drm, in vm_bind_ioctl_ops_create()
2256 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, &map_req); in vm_bind_ioctl_ops_create()
2260 ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range); in vm_bind_ioctl_ops_create()
2263 ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range); in vm_bind_ioctl_ops_create()
2266 xe_assert(vm->xe, bo); in vm_bind_ioctl_ops_create()
2272 vm_bo = drm_gpuvm_bo_obtain(&vm->gpuvm, obj); in vm_bind_ioctl_ops_create()
2283 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in vm_bind_ioctl_ops_create()
2284 ops = ERR_PTR(-EINVAL); in vm_bind_ioctl_ops_create()
2292 if (__op->op == DRM_GPUVA_OP_MAP) { in vm_bind_ioctl_ops_create()
2293 op->map.immediate = in vm_bind_ioctl_ops_create()
2296 op->map.vma_flags |= XE_VMA_READ_ONLY; in vm_bind_ioctl_ops_create()
2298 op->map.vma_flags |= DRM_GPUVA_SPARSE; in vm_bind_ioctl_ops_create()
2300 op->map.vma_flags |= XE_VMA_SYSTEM_ALLOCATOR; in vm_bind_ioctl_ops_create()
2302 op->map.vma_flags |= XE_VMA_DUMPABLE; in vm_bind_ioctl_ops_create()
2304 op->map.vma_flags |= XE_VMA_MADV_AUTORESET; in vm_bind_ioctl_ops_create()
2305 op->map.pat_index = pat_index; in vm_bind_ioctl_ops_create()
2306 op->map.invalidate_on_bind = in vm_bind_ioctl_ops_create()
2308 } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { in vm_bind_ioctl_ops_create()
2309 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); in vm_bind_ioctl_ops_create()
2310 struct xe_tile *tile; in vm_bind_ioctl_ops_create() local
2318 op->prefetch.region = prefetch_region; in vm_bind_ioctl_ops_create()
2323 ctx.devmem_possible = IS_DGFX(vm->xe) && in vm_bind_ioctl_ops_create()
2326 for_each_tile(tile, vm->xe, id) in vm_bind_ioctl_ops_create()
2329 xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC); in vm_bind_ioctl_ops_create()
2330 op->prefetch_range.ranges_count = 0; in vm_bind_ioctl_ops_create()
2331 tile = NULL; in vm_bind_ioctl_ops_create()
2335 xe_device_get_root_tile(vm->xe)); in vm_bind_ioctl_ops_create()
2338 * something to dereference tile from dpagemap. in vm_bind_ioctl_ops_create()
2341 tile = xe_device_get_root_tile(vm->xe); in vm_bind_ioctl_ops_create()
2343 tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] - in vm_bind_ioctl_ops_create()
2347 op->prefetch_range.tile = tile; in vm_bind_ioctl_ops_create()
2351 if (PTR_ERR(svm_range) == -ENOENT) { in vm_bind_ioctl_ops_create()
2366 if (xe_svm_range_validate(vm, svm_range, tile_mask, !!tile)) { in vm_bind_ioctl_ops_create()
2367 xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID"); in vm_bind_ioctl_ops_create()
2371 err = xa_alloc(&op->prefetch_range.range, in vm_bind_ioctl_ops_create()
2378 op->prefetch_range.ranges_count++; in vm_bind_ioctl_ops_create()
2379 vops->flags |= XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH; in vm_bind_ioctl_ops_create()
2380 xe_svm_range_debug(svm_range, "PREFETCH - RANGE CREATED"); in vm_bind_ioctl_ops_create()
2389 print_op(vm->xe, __op); in vm_bind_ioctl_ops_create()
2396 drm_gpuva_ops_free(&vm->gpuvm, ops); in vm_bind_ioctl_ops_create()
2405 struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; in new_vma()
2411 lockdep_assert_held_write(&vm->lock); in new_vma()
2415 xe_validation_guard(&ctx, &vm->xe->val, &exec, in new_vma()
2417 if (!bo->vm) { in new_vma()
2422 err = drm_exec_lock_obj(&exec, &bo->ttm.base); in new_vma()
2428 vma = xe_vma_create(vm, bo, op->gem.offset, in new_vma()
2429 op->va.addr, op->va.addr + in new_vma()
2430 op->va.range - 1, attr, flags); in new_vma()
2434 if (!bo->vm) { in new_vma()
2445 vma = xe_vma_create(vm, NULL, op->gem.offset, in new_vma()
2446 op->va.addr, op->va.addr + in new_vma()
2447 op->va.range - 1, attr, flags); in new_vma()
2465 if (vma->gpuva.flags & XE_VMA_PTE_1G) in xe_vma_max_pte_size()
2467 else if (vma->gpuva.flags & (XE_VMA_PTE_2M | XE_VMA_PTE_COMPACT)) in xe_vma_max_pte_size()
2469 else if (vma->gpuva.flags & XE_VMA_PTE_64K) in xe_vma_max_pte_size()
2471 else if (vma->gpuva.flags & XE_VMA_PTE_4K) in xe_vma_max_pte_size()
2481 vma->gpuva.flags |= XE_VMA_PTE_1G; in xe_vma_set_pte_size()
2484 vma->gpuva.flags |= XE_VMA_PTE_2M; in xe_vma_set_pte_size()
2487 vma->gpuva.flags |= XE_VMA_PTE_64K; in xe_vma_set_pte_size()
2490 vma->gpuva.flags |= XE_VMA_PTE_4K; in xe_vma_set_pte_size()
2499 lockdep_assert_held_write(&vm->lock); in xe_vma_op_commit()
2501 switch (op->base.op) { in xe_vma_op_commit()
2503 err |= xe_vm_insert_vma(vm, op->map.vma); in xe_vma_op_commit()
2505 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2510 gpuva_to_vma(op->base.remap.unmap->va)->tile_present; in xe_vma_op_commit()
2512 prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va), in xe_vma_op_commit()
2514 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2516 if (op->remap.prev) { in xe_vma_op_commit()
2517 err |= xe_vm_insert_vma(vm, op->remap.prev); in xe_vma_op_commit()
2519 op->flags |= XE_VMA_OP_PREV_COMMITTED; in xe_vma_op_commit()
2520 if (!err && op->remap.skip_prev) { in xe_vma_op_commit()
2521 op->remap.prev->tile_present = in xe_vma_op_commit()
2523 op->remap.prev = NULL; in xe_vma_op_commit()
2526 if (op->remap.next) { in xe_vma_op_commit()
2527 err |= xe_vm_insert_vma(vm, op->remap.next); in xe_vma_op_commit()
2529 op->flags |= XE_VMA_OP_NEXT_COMMITTED; in xe_vma_op_commit()
2530 if (!err && op->remap.skip_next) { in xe_vma_op_commit()
2531 op->remap.next->tile_present = in xe_vma_op_commit()
2533 op->remap.next = NULL; in xe_vma_op_commit()
2539 op->base.remap.unmap->va->va.addr = op->remap.start; in xe_vma_op_commit()
2540 op->base.remap.unmap->va->va.range = op->remap.range; in xe_vma_op_commit()
2545 prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true); in xe_vma_op_commit()
2546 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2549 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2552 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in xe_vma_op_commit()
2559 * xe_vma_has_default_mem_attrs - Check if a VMA has default memory attributes
2566 * - `atomic_access` is `DRM_XE_VMA_ATOMIC_UNDEFINED`
2567 * - `pat_index` is equal to `default_pat_index`
2568 * - `preferred_loc.devmem_fd` is `DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE`
2569 * - `preferred_loc.migration_policy` is `DRM_XE_MIGRATE_ALL_PAGES`
2575 return (vma->attr.atomic_access == DRM_XE_ATOMIC_UNDEFINED && in xe_vma_has_default_mem_attrs()
2576 vma->attr.pat_index == vma->attr.default_pat_index && in xe_vma_has_default_mem_attrs()
2577 vma->attr.preferred_loc.devmem_fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && in xe_vma_has_default_mem_attrs()
2578 vma->attr.preferred_loc.migration_policy == DRM_XE_MIGRATE_ALL_PAGES); in xe_vma_has_default_mem_attrs()
2584 struct xe_device *xe = vm->xe; in vm_bind_ioctl_ops_parse()
2586 struct xe_tile *tile; in vm_bind_ioctl_ops_parse() local
2590 lockdep_assert_held_write(&vm->lock); in vm_bind_ioctl_ops_parse()
2592 for_each_tile(tile, vm->xe, id) in vm_bind_ioctl_ops_parse()
2600 INIT_LIST_HEAD(&op->link); in vm_bind_ioctl_ops_parse()
2601 list_add_tail(&op->link, &vops->list); in vm_bind_ioctl_ops_parse()
2602 op->tile_mask = tile_mask; in vm_bind_ioctl_ops_parse()
2604 switch (op->base.op) { in vm_bind_ioctl_ops_parse()
2613 .default_pat_index = op->map.pat_index, in vm_bind_ioctl_ops_parse()
2614 .pat_index = op->map.pat_index, in vm_bind_ioctl_ops_parse()
2617 flags |= op->map.vma_flags & XE_VMA_CREATE_MASK; in vm_bind_ioctl_ops_parse()
2619 vma = new_vma(vm, &op->base.map, &default_attr, in vm_bind_ioctl_ops_parse()
2624 op->map.vma = vma; in vm_bind_ioctl_ops_parse()
2625 if (((op->map.immediate || !xe_vm_in_fault_mode(vm)) && in vm_bind_ioctl_ops_parse()
2626 !(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) || in vm_bind_ioctl_ops_parse()
2627 op->map.invalidate_on_bind) in vm_bind_ioctl_ops_parse()
2629 op->tile_mask, 1); in vm_bind_ioctl_ops_parse()
2635 gpuva_to_vma(op->base.remap.unmap->va); in vm_bind_ioctl_ops_parse()
2640 if (op->base.remap.prev) in vm_bind_ioctl_ops_parse()
2641 start = op->base.remap.prev->va.addr + in vm_bind_ioctl_ops_parse()
2642 op->base.remap.prev->va.range; in vm_bind_ioctl_ops_parse()
2643 if (op->base.remap.next) in vm_bind_ioctl_ops_parse()
2644 end = op->base.remap.next->va.addr; in vm_bind_ioctl_ops_parse()
2648 if (vops->flags & XE_VMA_OPS_FLAG_MADVISE) in vm_bind_ioctl_ops_parse()
2651 return -EBUSY; in vm_bind_ioctl_ops_parse()
2654 op->remap.start = xe_vma_start(old); in vm_bind_ioctl_ops_parse()
2655 op->remap.range = xe_vma_size(old); in vm_bind_ioctl_ops_parse()
2657 flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK; in vm_bind_ioctl_ops_parse()
2658 if (op->base.remap.prev) { in vm_bind_ioctl_ops_parse()
2659 vma = new_vma(vm, op->base.remap.prev, in vm_bind_ioctl_ops_parse()
2660 &old->attr, flags); in vm_bind_ioctl_ops_parse()
2664 op->remap.prev = vma; in vm_bind_ioctl_ops_parse()
2670 op->remap.skip_prev = skip || in vm_bind_ioctl_ops_parse()
2674 if (op->remap.skip_prev) { in vm_bind_ioctl_ops_parse()
2676 op->remap.range -= in vm_bind_ioctl_ops_parse()
2677 xe_vma_end(vma) - in vm_bind_ioctl_ops_parse()
2679 op->remap.start = xe_vma_end(vma); in vm_bind_ioctl_ops_parse()
2680 vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx", in vm_bind_ioctl_ops_parse()
2681 (ULL)op->remap.start, in vm_bind_ioctl_ops_parse()
2682 (ULL)op->remap.range); in vm_bind_ioctl_ops_parse()
2688 if (op->base.remap.next) { in vm_bind_ioctl_ops_parse()
2689 vma = new_vma(vm, op->base.remap.next, in vm_bind_ioctl_ops_parse()
2690 &old->attr, flags); in vm_bind_ioctl_ops_parse()
2694 op->remap.next = vma; in vm_bind_ioctl_ops_parse()
2700 op->remap.skip_next = skip || in vm_bind_ioctl_ops_parse()
2704 if (op->remap.skip_next) { in vm_bind_ioctl_ops_parse()
2706 op->remap.range -= in vm_bind_ioctl_ops_parse()
2707 xe_vma_end(old) - in vm_bind_ioctl_ops_parse()
2709 vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx", in vm_bind_ioctl_ops_parse()
2710 (ULL)op->remap.start, in vm_bind_ioctl_ops_parse()
2711 (ULL)op->remap.range); in vm_bind_ioctl_ops_parse()
2719 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, num_remap_ops); in vm_bind_ioctl_ops_parse()
2723 vma = gpuva_to_vma(op->base.unmap.va); in vm_bind_ioctl_ops_parse()
2728 return -EBUSY; in vm_bind_ioctl_ops_parse()
2731 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); in vm_bind_ioctl_ops_parse()
2734 vma = gpuva_to_vma(op->base.prefetch.va); in vm_bind_ioctl_ops_parse()
2743 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, in vm_bind_ioctl_ops_parse()
2744 op->prefetch_range.ranges_count); in vm_bind_ioctl_ops_parse()
2746 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); in vm_bind_ioctl_ops_parse()
2750 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in vm_bind_ioctl_ops_parse()
2765 lockdep_assert_held_write(&vm->lock); in xe_vma_op_unwind()
2767 switch (op->base.op) { in xe_vma_op_unwind()
2769 if (op->map.vma) { in xe_vma_op_unwind()
2770 prep_vma_destroy(vm, op->map.vma, post_commit); in xe_vma_op_unwind()
2771 xe_vma_destroy_unlocked(op->map.vma); in xe_vma_op_unwind()
2776 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); in xe_vma_op_unwind()
2780 vma->gpuva.flags &= ~XE_VMA_DESTROYED; in xe_vma_op_unwind()
2789 struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vma_op_unwind()
2791 if (op->remap.prev) { in xe_vma_op_unwind()
2792 prep_vma_destroy(vm, op->remap.prev, prev_post_commit); in xe_vma_op_unwind()
2793 xe_vma_destroy_unlocked(op->remap.prev); in xe_vma_op_unwind()
2795 if (op->remap.next) { in xe_vma_op_unwind()
2796 prep_vma_destroy(vm, op->remap.next, next_post_commit); in xe_vma_op_unwind()
2797 xe_vma_destroy_unlocked(op->remap.next); in xe_vma_op_unwind()
2801 vma->gpuva.flags &= ~XE_VMA_DESTROYED; in xe_vma_op_unwind()
2812 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in xe_vma_op_unwind()
2822 for (i = num_ops_list - 1; i >= 0; --i) { in vm_bind_ioctl_ops_unwind()
2833 op->flags & XE_VMA_OP_COMMITTED, in vm_bind_ioctl_ops_unwind()
2834 op->flags & XE_VMA_OP_PREV_COMMITTED, in vm_bind_ioctl_ops_unwind()
2835 op->flags & XE_VMA_OP_NEXT_COMMITTED); in vm_bind_ioctl_ops_unwind()
2848 if (!bo->vm) in vma_lock_and_validate()
2849 err = drm_exec_lock_obj(exec, &bo->ttm.base); in vma_lock_and_validate()
2861 if (vma->ufence) { in check_ufence()
2862 struct xe_user_fence * const f = vma->ufence; in check_ufence()
2865 return -EBUSY; in check_ufence()
2867 vma->ufence = NULL; in check_ufence()
2876 bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); in prefetch_ranges()
2877 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); in prefetch_ranges()
2878 struct xe_tile *tile = op->prefetch_range.tile; in prefetch_ranges() local
2891 ctx.device_private_page_owner = xe_svm_devm_owner(vm->xe); in prefetch_ranges()
2894 xa_for_each(&op->prefetch_range.range, i, svm_range) { in prefetch_ranges()
2895 if (!tile) in prefetch_ranges()
2898 if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, !!tile)) { in prefetch_ranges()
2899 err = xe_svm_alloc_vram(tile, svm_range, &ctx); in prefetch_ranges()
2901 …drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe… in prefetch_ranges()
2902 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); in prefetch_ranges()
2903 return -ENODATA; in prefetch_ranges()
2905 xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM"); in prefetch_ranges()
2910 drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n", in prefetch_ranges()
2911 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); in prefetch_ranges()
2912 if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) in prefetch_ranges()
2913 err = -ENODATA; in prefetch_ranges()
2916 xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE"); in prefetch_ranges()
2933 res_evict = !(vops->flags & XE_VMA_OPS_ARRAY_OF_BINDS); in op_lock_and_prep()
2935 switch (op->base.op) { in op_lock_and_prep()
2937 if (!op->map.invalidate_on_bind) in op_lock_and_prep()
2938 err = vma_lock_and_validate(exec, op->map.vma, in op_lock_and_prep()
2941 op->map.immediate); in op_lock_and_prep()
2944 err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va)); in op_lock_and_prep()
2949 gpuva_to_vma(op->base.remap.unmap->va), in op_lock_and_prep()
2951 if (!err && op->remap.prev) in op_lock_and_prep()
2952 err = vma_lock_and_validate(exec, op->remap.prev, in op_lock_and_prep()
2954 if (!err && op->remap.next) in op_lock_and_prep()
2955 err = vma_lock_and_validate(exec, op->remap.next, in op_lock_and_prep()
2959 err = check_ufence(gpuva_to_vma(op->base.unmap.va)); in op_lock_and_prep()
2964 gpuva_to_vma(op->base.unmap.va), in op_lock_and_prep()
2969 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); in op_lock_and_prep()
2973 region = op->prefetch.region; in op_lock_and_prep()
2974 xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC || in op_lock_and_prep()
2979 gpuva_to_vma(op->base.prefetch.va), in op_lock_and_prep()
2989 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_lock_and_prep()
3000 if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) in vm_bind_ioctl_ops_prefetch_ranges()
3003 list_for_each_entry(op, &vops->list, link) { in vm_bind_ioctl_ops_prefetch_ranges()
3004 if (op->base.op == DRM_GPUVA_OP_PREFETCH) { in vm_bind_ioctl_ops_prefetch_ranges()
3025 list_for_each_entry(op, &vops->list, link) { in vm_bind_ioctl_ops_lock_and_prep()
3032 if (vops->inject_error && in vm_bind_ioctl_ops_lock_and_prep()
3033 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK) in vm_bind_ioctl_ops_lock_and_prep()
3034 return -ENOSPC; in vm_bind_ioctl_ops_lock_and_prep()
3042 switch (op->base.op) { in op_trace()
3044 trace_xe_vma_bind(op->map.vma); in op_trace()
3047 trace_xe_vma_unbind(gpuva_to_vma(op->base.remap.unmap->va)); in op_trace()
3048 if (op->remap.prev) in op_trace()
3049 trace_xe_vma_bind(op->remap.prev); in op_trace()
3050 if (op->remap.next) in op_trace()
3051 trace_xe_vma_bind(op->remap.next); in op_trace()
3054 trace_xe_vma_unbind(gpuva_to_vma(op->base.unmap.va)); in op_trace()
3057 trace_xe_vma_bind(gpuva_to_vma(op->base.prefetch.va)); in op_trace()
3070 list_for_each_entry(op, &vops->list, link) in trace_xe_vm_ops_execute()
3076 struct xe_exec_queue *q = vops->q; in vm_ops_setup_tile_args()
3077 struct xe_tile *tile; in vm_ops_setup_tile_args() local
3081 for_each_tile(tile, vm->xe, id) { in vm_ops_setup_tile_args()
3082 if (vops->pt_update_ops[id].num_ops) in vm_ops_setup_tile_args()
3085 if (vops->pt_update_ops[id].q) in vm_ops_setup_tile_args()
3089 vops->pt_update_ops[id].q = q; in vm_ops_setup_tile_args()
3090 if (vm->pt_root[id] && !list_empty(&q->multi_gt_list)) in vm_ops_setup_tile_args()
3093 vops->pt_update_ops[id].q = vm->q[id]; in vm_ops_setup_tile_args()
3103 struct xe_tile *tile; in ops_execute() local
3112 return ERR_PTR(-ENODATA); in ops_execute()
3114 if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) { in ops_execute()
3115 for_each_tile(tile, vm->xe, id) in ops_execute()
3118 for_each_tile(tile, vm->xe, id) in ops_execute()
3124 fence = ERR_PTR(-ENOMEM); in ops_execute()
3130 fence = ERR_PTR(-ENOMEM); in ops_execute()
3134 for_each_tile(tile, vm->xe, id) { in ops_execute()
3135 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3138 err = xe_pt_update_ops_prepare(tile, vops); in ops_execute()
3147 for_each_tile(tile, vm->xe, id) { in ops_execute()
3148 struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q; in ops_execute()
3152 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3155 fence = xe_pt_update_ops_run(tile, vops); in ops_execute()
3161 if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT) in ops_execute()
3164 xe_migrate_job_lock(tile->migrate, q); in ops_execute()
3168 xe_migrate_job_unlock(tile->migrate, q); in ops_execute()
3171 xe_assert(vm->xe, current_fence == n_fence); in ops_execute()
3174 fence = &cf->base; in ops_execute()
3176 for_each_tile(tile, vm->xe, id) { in ops_execute()
3177 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3180 xe_pt_update_ops_fini(tile, vops); in ops_execute()
3186 for_each_tile(tile, vm->xe, id) { in ops_execute()
3187 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3190 xe_pt_update_ops_abort(tile, vops); in ops_execute()
3193 dma_fence_put(fences[--current_fence]); in ops_execute()
3204 if (vma->ufence) in vma_add_ufence()
3205 xe_sync_ufence_put(vma->ufence); in vma_add_ufence()
3206 vma->ufence = __xe_sync_ufence_get(ufence); in vma_add_ufence()
3212 switch (op->base.op) { in op_add_ufence()
3214 vma_add_ufence(op->map.vma, ufence); in op_add_ufence()
3217 if (op->remap.prev) in op_add_ufence()
3218 vma_add_ufence(op->remap.prev, ufence); in op_add_ufence()
3219 if (op->remap.next) in op_add_ufence()
3220 vma_add_ufence(op->remap.next, ufence); in op_add_ufence()
3225 vma_add_ufence(gpuva_to_vma(op->base.prefetch.va), ufence); in op_add_ufence()
3228 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_add_ufence()
3239 ufence = find_ufence_get(vops->syncs, vops->num_syncs); in vm_bind_ioctl_ops_fini()
3240 list_for_each_entry(op, &vops->list, link) { in vm_bind_ioctl_ops_fini()
3244 if (op->base.op == DRM_GPUVA_OP_UNMAP) in vm_bind_ioctl_ops_fini()
3245 xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence); in vm_bind_ioctl_ops_fini()
3246 else if (op->base.op == DRM_GPUVA_OP_REMAP) in vm_bind_ioctl_ops_fini()
3247 xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), in vm_bind_ioctl_ops_fini()
3253 for (i = 0; i < vops->num_syncs; i++) in vm_bind_ioctl_ops_fini()
3254 xe_sync_entry_signal(vops->syncs + i, fence); in vm_bind_ioctl_ops_fini()
3266 lockdep_assert_held_write(&vm->lock); in vm_bind_ioctl_ops_execute()
3268 xe_validation_guard(&ctx, &vm->xe->val, &exec, in vm_bind_ioctl_ops_execute()
3283 if (PTR_ERR(fence) == -ENODATA) in vm_bind_ioctl_ops_execute()
3320 if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || in vm_bind_ioctl_check_args()
3321 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in vm_bind_ioctl_check_args()
3322 return -EINVAL; in vm_bind_ioctl_check_args()
3324 if (XE_IOCTL_DBG(xe, args->extensions)) in vm_bind_ioctl_check_args()
3325 return -EINVAL; in vm_bind_ioctl_check_args()
3327 if (args->num_binds > 1) { in vm_bind_ioctl_check_args()
3329 u64_to_user_ptr(args->vector_of_binds); in vm_bind_ioctl_check_args()
3331 *bind_ops = kvmalloc_array(args->num_binds, in vm_bind_ioctl_check_args()
3336 return args->num_binds > 1 ? -ENOBUFS : -ENOMEM; in vm_bind_ioctl_check_args()
3340 args->num_binds); in vm_bind_ioctl_check_args()
3342 err = -EFAULT; in vm_bind_ioctl_check_args()
3346 *bind_ops = &args->bind; in vm_bind_ioctl_check_args()
3349 for (i = 0; i < args->num_binds; ++i) { in vm_bind_ioctl_check_args()
3366 err = -EINVAL; in vm_bind_ioctl_check_args()
3370 if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { in vm_bind_ioctl_check_args()
3371 err = -EINVAL; in vm_bind_ioctl_check_args()
3375 pat_index = array_index_nospec(pat_index, xe->pat.n_entries); in vm_bind_ioctl_check_args()
3379 err = -EINVAL; in vm_bind_ioctl_check_args()
3384 err = -EINVAL; in vm_bind_ioctl_check_args()
3416 (prefetch_region >= (sizeof(xe->info.mem_region_mask) * 8) || in vm_bind_ioctl_check_args()
3417 !(BIT(prefetch_region) & xe->info.mem_region_mask)))) || in vm_bind_ioctl_check_args()
3422 err = -EINVAL; in vm_bind_ioctl_check_args()
3431 err = -EINVAL; in vm_bind_ioctl_check_args()
3439 if (args->num_binds > 1) in vm_bind_ioctl_check_args()
3473 INIT_LIST_HEAD(&vops->list); in xe_vma_ops_init()
3474 vops->vm = vm; in xe_vma_ops_init()
3475 vops->q = q; in xe_vma_ops_init()
3476 vops->syncs = syncs; in xe_vma_ops_init()
3477 vops->num_syncs = num_syncs; in xe_vma_ops_init()
3478 vops->flags = 0; in xe_vma_ops_init()
3489 xe_bo_size(bo) - range)) { in xe_vm_bind_ioctl_validate_bo()
3490 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3501 if ((bo->flags & XE_BO_FLAG_INTERNAL_64K) && in xe_vm_bind_ioctl_validate_bo()
3502 (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)) { in xe_vm_bind_ioctl_validate_bo()
3507 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3512 if (bo->cpu_caching) { in xe_vm_bind_ioctl_validate_bo()
3514 bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) { in xe_vm_bind_ioctl_validate_bo()
3515 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3519 * Imported dma-buf from a different device should in xe_vm_bind_ioctl_validate_bo()
3524 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3530 if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0)) in xe_vm_bind_ioctl_validate_bo()
3531 return -ENOEXEC; in xe_vm_bind_ioctl_validate_bo()
3554 vm = xe_vm_lookup(xef, args->vm_id); in xe_vm_bind_ioctl()
3556 return -EINVAL; in xe_vm_bind_ioctl()
3562 if (args->exec_queue_id) { in xe_vm_bind_ioctl()
3563 q = xe_exec_queue_lookup(xef, args->exec_queue_id); in xe_vm_bind_ioctl()
3565 err = -ENOENT; in xe_vm_bind_ioctl()
3569 if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { in xe_vm_bind_ioctl()
3570 err = -EINVAL; in xe_vm_bind_ioctl()
3578 err = down_write_killable(&vm->lock); in xe_vm_bind_ioctl()
3583 err = -ENOENT; in xe_vm_bind_ioctl()
3587 for (i = 0; i < args->num_binds; ++i) { in xe_vm_bind_ioctl()
3591 if (XE_IOCTL_DBG(xe, range > vm->size) || in xe_vm_bind_ioctl()
3592 XE_IOCTL_DBG(xe, addr > vm->size - range)) { in xe_vm_bind_ioctl()
3593 err = -EINVAL; in xe_vm_bind_ioctl()
3598 if (args->num_binds) { in xe_vm_bind_ioctl()
3599 bos = kvcalloc(args->num_binds, sizeof(*bos), in xe_vm_bind_ioctl()
3603 err = -ENOMEM; in xe_vm_bind_ioctl()
3607 ops = kvcalloc(args->num_binds, sizeof(*ops), in xe_vm_bind_ioctl()
3611 err = -ENOMEM; in xe_vm_bind_ioctl()
3616 for (i = 0; i < args->num_binds; ++i) { in xe_vm_bind_ioctl()
3631 err = -ENOENT; in xe_vm_bind_ioctl()
3643 if (args->num_syncs) { in xe_vm_bind_ioctl()
3644 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); in xe_vm_bind_ioctl()
3646 err = -ENOMEM; in xe_vm_bind_ioctl()
3651 syncs_user = u64_to_user_ptr(args->syncs); in xe_vm_bind_ioctl()
3652 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { in xe_vm_bind_ioctl()
3653 struct xe_exec_queue *__q = q ?: vm->q[0]; in xe_vm_bind_ioctl()
3657 __q->ufence_syncobj, in xe_vm_bind_ioctl()
3658 ++__q->ufence_timeline_value, in xe_vm_bind_ioctl()
3661 (!args->num_binds ? in xe_vm_bind_ioctl()
3671 err = -EINVAL; in xe_vm_bind_ioctl()
3675 if (!args->num_binds) { in xe_vm_bind_ioctl()
3676 err = -ENODATA; in xe_vm_bind_ioctl()
3681 if (args->num_binds > 1) in xe_vm_bind_ioctl()
3683 for (i = 0; i < args->num_binds; ++i) { in xe_vm_bind_ioctl()
3708 vm->xe->vm_inject_error_position = in xe_vm_bind_ioctl()
3709 (vm->xe->vm_inject_error_position + 1) % in xe_vm_bind_ioctl()
3717 err = -ENODATA; in xe_vm_bind_ioctl()
3721 err = xe_vma_ops_alloc(&vops, args->num_binds > 1); in xe_vm_bind_ioctl()
3736 if (err && err != -ENODATA) in xe_vm_bind_ioctl()
3737 vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds); in xe_vm_bind_ioctl()
3739 for (i = args->num_binds - 1; i >= 0; --i) in xe_vm_bind_ioctl()
3741 drm_gpuva_ops_free(&vm->gpuvm, ops[i]); in xe_vm_bind_ioctl()
3743 if (err == -ENODATA) in xe_vm_bind_ioctl()
3745 while (num_syncs--) in xe_vm_bind_ioctl()
3750 for (i = 0; i < args->num_binds; ++i) in xe_vm_bind_ioctl()
3757 up_write(&vm->lock); in xe_vm_bind_ioctl()
3762 if (args->num_binds > 1) in xe_vm_bind_ioctl()
3770 * xe_vm_bind_kernel_bo - bind a kernel BO to a VM
3777 * Execute a VM bind map operation on a kernel-owned BO to bind it into a
3778 * kernel-owned VM.
3797 down_write(&vm->lock); in xe_vm_bind_kernel_bo()
3803 vm->xe->pat.idx[cache_lvl]); in xe_vm_bind_kernel_bo()
3813 xe_assert(vm->xe, !list_empty(&vops.list)); in xe_vm_bind_kernel_bo()
3824 if (err && err != -ENODATA) in xe_vm_bind_kernel_bo()
3828 drm_gpuva_ops_free(&vm->gpuvm, ops); in xe_vm_bind_kernel_bo()
3831 up_write(&vm->lock); in xe_vm_bind_kernel_bo()
3845 * xe_vm_lock() - Lock the vm's dma_resv object
3849 * Return: 0 on success, -EINTR if @intr is true and the wait for a
3866 * xe_vm_unlock() - Unlock the vm's dma_resv object
3877 * xe_vm_range_tilemask_tlb_inval - Issue a TLB invalidation on this tilemask for an
3893 struct xe_tile *tile; in xe_vm_range_tilemask_tlb_inval() local
3901 for_each_tile(tile, vm->xe, id) { in xe_vm_range_tilemask_tlb_inval()
3905 xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3908 err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3910 vm->usm.asid); in xe_vm_range_tilemask_tlb_inval()
3915 if (!tile->media_gt) in xe_vm_range_tilemask_tlb_inval()
3918 xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3921 err = xe_tlb_inval_range(&tile->media_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3923 vm->usm.asid); in xe_vm_range_tilemask_tlb_inval()
3937 * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
3948 struct xe_device *xe = xe_vma_vm(vma)->xe; in xe_vm_invalidate_vma()
3950 struct xe_tile *tile; in xe_vm_invalidate_vma() local
3959 vm_dbg(&vm->xe->drm, in xe_vm_invalidate_vma()
3964 * Check that we don't race with page-table updates, tile_invalidated in xe_vm_invalidate_vma()
3969 lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || in xe_vm_invalidate_vma()
3970 (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && in xe_vm_invalidate_vma()
3971 lockdep_is_held(&xe_vm_resv(vm)->lock.base))); in xe_vm_invalidate_vma()
3974 (&to_userptr_vma(vma)->userptr.notifier, in xe_vm_invalidate_vma()
3975 to_userptr_vma(vma)->userptr.pages.notifier_seq)); in xe_vm_invalidate_vma()
3984 for_each_tile(tile, xe, id) in xe_vm_invalidate_vma()
3985 if (xe_pt_zap_ptes(tile, vma)) in xe_vm_invalidate_vma()
3994 WRITE_ONCE(vma->tile_invalidated, vma->tile_mask); in xe_vm_invalidate_vma()
4005 return -ENODEV; in xe_vm_validate_protected()
4007 mutex_lock(&vm->snap_mutex); in xe_vm_validate_protected()
4009 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { in xe_vm_validate_protected()
4011 struct xe_bo *bo = vma->gpuva.gem.obj ? in xe_vm_validate_protected()
4012 gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; in xe_vm_validate_protected()
4018 err = xe_pxp_bo_key_check(vm->xe->pxp, bo); in xe_vm_validate_protected()
4024 mutex_unlock(&vm->snap_mutex); in xe_vm_validate_protected()
4048 mutex_lock(&vm->snap_mutex); in xe_vm_snapshot_capture()
4049 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { in xe_vm_snapshot_capture()
4050 if (gpuva->flags & XE_VMA_DUMPABLE) in xe_vm_snapshot_capture()
4057 snap = num_snaps ? ERR_PTR(-ENOMEM) : ERR_PTR(-ENODEV); in xe_vm_snapshot_capture()
4061 snap->num_snaps = num_snaps; in xe_vm_snapshot_capture()
4063 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { in xe_vm_snapshot_capture()
4065 struct xe_bo *bo = vma->gpuva.gem.obj ? in xe_vm_snapshot_capture()
4066 gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; in xe_vm_snapshot_capture()
4068 if (!(gpuva->flags & XE_VMA_DUMPABLE)) in xe_vm_snapshot_capture()
4071 snap->snap[i].ofs = xe_vma_start(vma); in xe_vm_snapshot_capture()
4072 snap->snap[i].len = xe_vma_size(vma); in xe_vm_snapshot_capture()
4074 snap->snap[i].bo = xe_bo_get(bo); in xe_vm_snapshot_capture()
4075 snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); in xe_vm_snapshot_capture()
4078 to_userptr_vma(vma)->userptr.notifier.mm; in xe_vm_snapshot_capture()
4081 snap->snap[i].mm = mm; in xe_vm_snapshot_capture()
4083 snap->snap[i].data = ERR_PTR(-EFAULT); in xe_vm_snapshot_capture()
4085 snap->snap[i].bo_ofs = xe_vma_userptr(vma); in xe_vm_snapshot_capture()
4087 snap->snap[i].data = ERR_PTR(-ENOENT); in xe_vm_snapshot_capture()
4093 mutex_unlock(&vm->snap_mutex); in xe_vm_snapshot_capture()
4102 for (int i = 0; i < snap->num_snaps; i++) { in xe_vm_snapshot_capture_delayed()
4103 struct xe_bo *bo = snap->snap[i].bo; in xe_vm_snapshot_capture_delayed()
4106 if (IS_ERR(snap->snap[i].data)) in xe_vm_snapshot_capture_delayed()
4109 snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER); in xe_vm_snapshot_capture_delayed()
4110 if (!snap->snap[i].data) { in xe_vm_snapshot_capture_delayed()
4111 snap->snap[i].data = ERR_PTR(-ENOMEM); in xe_vm_snapshot_capture_delayed()
4116 err = xe_bo_read(bo, snap->snap[i].bo_ofs, in xe_vm_snapshot_capture_delayed()
4117 snap->snap[i].data, snap->snap[i].len); in xe_vm_snapshot_capture_delayed()
4119 void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs; in xe_vm_snapshot_capture_delayed()
4121 kthread_use_mm(snap->snap[i].mm); in xe_vm_snapshot_capture_delayed()
4122 if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len)) in xe_vm_snapshot_capture_delayed()
4125 err = -EFAULT; in xe_vm_snapshot_capture_delayed()
4126 kthread_unuse_mm(snap->snap[i].mm); in xe_vm_snapshot_capture_delayed()
4128 mmput(snap->snap[i].mm); in xe_vm_snapshot_capture_delayed()
4129 snap->snap[i].mm = NULL; in xe_vm_snapshot_capture_delayed()
4133 kvfree(snap->snap[i].data); in xe_vm_snapshot_capture_delayed()
4134 snap->snap[i].data = ERR_PTR(err); in xe_vm_snapshot_capture_delayed()
4139 snap->snap[i].bo = NULL; in xe_vm_snapshot_capture_delayed()
4143 void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p) in xe_vm_snapshot_print() argument
4148 drm_printf(p, "[0].error: %li\n", PTR_ERR(snap)); in xe_vm_snapshot_print()
4152 for (i = 0; i < snap->num_snaps; i++) { in xe_vm_snapshot_print()
4153 drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); in xe_vm_snapshot_print()
4155 if (IS_ERR(snap->snap[i].data)) { in xe_vm_snapshot_print()
4156 drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, in xe_vm_snapshot_print()
4157 PTR_ERR(snap->snap[i].data)); in xe_vm_snapshot_print()
4161 drm_printf(p, "[%llx].data: ", snap->snap[i].ofs); in xe_vm_snapshot_print()
4163 for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) { in xe_vm_snapshot_print()
4164 u32 *val = snap->snap[i].data + j; in xe_vm_snapshot_print()
4167 drm_puts(p, ascii85_encode(*val, dumped)); in xe_vm_snapshot_print()
4170 drm_puts(p, "\n"); in xe_vm_snapshot_print()
4172 if (drm_coredump_printer_is_full(p)) in xe_vm_snapshot_print()
4184 for (i = 0; i < snap->num_snaps; i++) { in xe_vm_snapshot_free()
4185 if (!IS_ERR(snap->snap[i].data)) in xe_vm_snapshot_free()
4186 kvfree(snap->snap[i].data); in xe_vm_snapshot_free()
4187 xe_bo_put(snap->snap[i].bo); in xe_vm_snapshot_free()
4188 if (snap->snap[i].mm) in xe_vm_snapshot_free()
4189 mmput(snap->snap[i].mm); in xe_vm_snapshot_free()
4195 * xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations
4204 * 1 - Migration to VRAM is required
4205 * 0 - Migration is not required
4206 * -EACCES - Invalid access for atomic memory attr
4211 u32 atomic_access = xe_vma_bo(vma) ? xe_vma_bo(vma)->attr.atomic_access : in xe_vma_need_vram_for_atomic()
4212 vma->attr.atomic_access; in xe_vma_need_vram_for_atomic()
4218 * NOTE: The checks implemented here are platform-specific. For in xe_vma_need_vram_for_atomic()
4224 return !xe->info.has_device_atomics_on_smem; in xe_vma_need_vram_for_atomic()
4227 return -EACCES; in xe_vma_need_vram_for_atomic()
4249 lockdep_assert_held_write(&vm->lock); in xe_vm_alloc_vma()
4252 ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req); in xe_vm_alloc_vma()
4254 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, map_req); in xe_vm_alloc_vma()
4259 if (list_empty(&ops->list)) { in xe_vm_alloc_vma()
4269 if (__op->op == DRM_GPUVA_OP_UNMAP) { in xe_vm_alloc_vma()
4270 vma = gpuva_to_vma(op->base.unmap.va); in xe_vm_alloc_vma()
4272 default_pat = vma->attr.default_pat_index; in xe_vm_alloc_vma()
4273 vma_flags = vma->gpuva.flags; in xe_vm_alloc_vma()
4276 if (__op->op == DRM_GPUVA_OP_REMAP) { in xe_vm_alloc_vma()
4277 vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vm_alloc_vma()
4278 default_pat = vma->attr.default_pat_index; in xe_vm_alloc_vma()
4279 vma_flags = vma->gpuva.flags; in xe_vm_alloc_vma()
4282 if (__op->op == DRM_GPUVA_OP_MAP) { in xe_vm_alloc_vma()
4283 op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; in xe_vm_alloc_vma()
4284 op->map.pat_index = default_pat; in xe_vm_alloc_vma()
4287 if (__op->op == DRM_GPUVA_OP_REMAP) { in xe_vm_alloc_vma()
4288 vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vm_alloc_vma()
4289 xe_assert(vm->xe, !remap_op); in xe_vm_alloc_vma()
4290 xe_assert(vm->xe, xe_vma_has_no_bo(vma)); in xe_vm_alloc_vma()
4292 vma_flags = vma->gpuva.flags; in xe_vm_alloc_vma()
4295 if (__op->op == DRM_GPUVA_OP_MAP) { in xe_vm_alloc_vma()
4296 xe_assert(vm->xe, remap_op); in xe_vm_alloc_vma()
4304 op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; in xe_vm_alloc_vma()
4307 print_op(vm->xe, __op); in xe_vm_alloc_vma()
4325 if (__op->op == DRM_GPUVA_OP_UNMAP) { in xe_vm_alloc_vma()
4326 vma = gpuva_to_vma(op->base.unmap.va); in xe_vm_alloc_vma()
4332 } else if (__op->op == DRM_GPUVA_OP_REMAP) { in xe_vm_alloc_vma()
4333 vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vm_alloc_vma()
4338 tmp_attr = vma->attr; in xe_vm_alloc_vma()
4340 xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL); in xe_vm_alloc_vma()
4341 } else if (__op->op == DRM_GPUVA_OP_MAP) { in xe_vm_alloc_vma()
4342 vma = op->map.vma; in xe_vm_alloc_vma()
4348 vma->attr = tmp_attr; in xe_vm_alloc_vma()
4353 drm_gpuva_ops_free(&vm->gpuvm, ops); in xe_vm_alloc_vma()
4359 drm_gpuva_ops_free(&vm->gpuvm, ops); in xe_vm_alloc_vma()
4364 * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops
4380 lockdep_assert_held_write(&vm->lock); in xe_vm_alloc_madvise_vma()
4382 vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); in xe_vm_alloc_madvise_vma()
4388 * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma
4404 lockdep_assert_held_write(&vm->lock); in xe_vm_alloc_cpu_addr_mirror_vma()
4406 vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx", in xe_vm_alloc_cpu_addr_mirror_vma()