Lines Matching +full:vm +full:- +full:map
1 // SPDX-License-Identifier: MIT
8 #include <linux/dma-fence-array.h>
45 static struct drm_gem_object *xe_vm_obj(struct xe_vm *vm) in xe_vm_obj() argument
47 return vm->gpuvm.r_obj; in xe_vm_obj()
51 * xe_vm_drm_exec_lock() - Lock the vm's resv with a drm_exec transaction
52 * @vm: The vm whose resv is to be locked.
55 * Helper to lock the vm's resv as part of a drm_exec transaction.
59 int xe_vm_drm_exec_lock(struct xe_vm *vm, struct drm_exec *exec) in xe_vm_drm_exec_lock() argument
61 return drm_exec_lock_obj(exec, xe_vm_obj(vm)); in xe_vm_drm_exec_lock()
64 static bool preempt_fences_waiting(struct xe_vm *vm) in preempt_fences_waiting() argument
68 lockdep_assert_held(&vm->lock); in preempt_fences_waiting()
69 xe_vm_assert_held(vm); in preempt_fences_waiting()
71 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in preempt_fences_waiting()
72 if (!q->lr.pfence || in preempt_fences_waiting()
74 &q->lr.pfence->flags)) { in preempt_fences_waiting()
90 static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, in alloc_preempt_fences() argument
93 lockdep_assert_held(&vm->lock); in alloc_preempt_fences()
94 xe_vm_assert_held(vm); in alloc_preempt_fences()
96 if (*count >= vm->preempt.num_exec_queues) in alloc_preempt_fences()
99 for (; *count < vm->preempt.num_exec_queues; ++(*count)) { in alloc_preempt_fences()
111 static int wait_for_existing_preempt_fences(struct xe_vm *vm) in wait_for_existing_preempt_fences() argument
115 xe_vm_assert_held(vm); in wait_for_existing_preempt_fences()
117 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in wait_for_existing_preempt_fences()
118 if (q->lr.pfence) { in wait_for_existing_preempt_fences()
119 long timeout = dma_fence_wait(q->lr.pfence, false); in wait_for_existing_preempt_fences()
121 /* Only -ETIME on fence indicates VM needs to be killed */ in wait_for_existing_preempt_fences()
122 if (timeout < 0 || q->lr.pfence->error == -ETIME) in wait_for_existing_preempt_fences()
123 return -ETIME; in wait_for_existing_preempt_fences()
125 dma_fence_put(q->lr.pfence); in wait_for_existing_preempt_fences()
126 q->lr.pfence = NULL; in wait_for_existing_preempt_fences()
133 static bool xe_vm_is_idle(struct xe_vm *vm) in xe_vm_is_idle() argument
137 xe_vm_assert_held(vm); in xe_vm_is_idle()
138 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in xe_vm_is_idle()
146 static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) in arm_preempt_fences() argument
151 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in arm_preempt_fences()
154 link = list->next; in arm_preempt_fences()
155 xe_assert(vm->xe, link != list); in arm_preempt_fences()
158 q, q->lr.context, in arm_preempt_fences()
159 ++q->lr.seqno); in arm_preempt_fences()
160 dma_fence_put(q->lr.pfence); in arm_preempt_fences()
161 q->lr.pfence = fence; in arm_preempt_fences()
165 static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) in add_preempt_fences() argument
172 if (!vm->preempt.num_exec_queues) in add_preempt_fences()
175 err = dma_resv_reserve_fences(bo->ttm.base.resv, vm->preempt.num_exec_queues); in add_preempt_fences()
179 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) in add_preempt_fences()
180 if (q->lr.pfence) { in add_preempt_fences()
181 dma_resv_add_fence(bo->ttm.base.resv, in add_preempt_fences()
182 q->lr.pfence, in add_preempt_fences()
189 static void resume_and_reinstall_preempt_fences(struct xe_vm *vm, in resume_and_reinstall_preempt_fences() argument
194 lockdep_assert_held(&vm->lock); in resume_and_reinstall_preempt_fences()
195 xe_vm_assert_held(vm); in resume_and_reinstall_preempt_fences()
197 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) { in resume_and_reinstall_preempt_fences()
198 q->ops->resume(q); in resume_and_reinstall_preempt_fences()
200 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->lr.pfence, in resume_and_reinstall_preempt_fences()
205 int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) in xe_vm_add_compute_exec_queue() argument
208 .vm = &vm->gpuvm, in xe_vm_add_compute_exec_queue()
218 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); in xe_vm_add_compute_exec_queue()
220 down_write(&vm->lock); in xe_vm_add_compute_exec_queue()
221 err = xe_validation_exec_lock(&ctx, &vm_exec, &vm->xe->val); in xe_vm_add_compute_exec_queue()
225 pfence = xe_preempt_fence_create(q, q->lr.context, in xe_vm_add_compute_exec_queue()
226 ++q->lr.seqno); in xe_vm_add_compute_exec_queue()
232 list_add(&q->lr.link, &vm->preempt.exec_queues); in xe_vm_add_compute_exec_queue()
233 ++vm->preempt.num_exec_queues; in xe_vm_add_compute_exec_queue()
234 q->lr.pfence = pfence; in xe_vm_add_compute_exec_queue()
236 xe_svm_notifier_lock(vm); in xe_vm_add_compute_exec_queue()
238 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, pfence, in xe_vm_add_compute_exec_queue()
242 * Check to see if a preemption on VM is in flight or userptr in xe_vm_add_compute_exec_queue()
244 * other preempt fences on the VM. in xe_vm_add_compute_exec_queue()
246 wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm); in xe_vm_add_compute_exec_queue()
250 xe_svm_notifier_unlock(vm); in xe_vm_add_compute_exec_queue()
255 up_write(&vm->lock); in xe_vm_add_compute_exec_queue()
262 * xe_vm_remove_compute_exec_queue() - Remove compute exec queue from VM
263 * @vm: The VM.
268 void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) in xe_vm_remove_compute_exec_queue() argument
270 if (!xe_vm_in_preempt_fence_mode(vm)) in xe_vm_remove_compute_exec_queue()
273 down_write(&vm->lock); in xe_vm_remove_compute_exec_queue()
274 if (!list_empty(&q->lr.link)) { in xe_vm_remove_compute_exec_queue()
275 list_del_init(&q->lr.link); in xe_vm_remove_compute_exec_queue()
276 --vm->preempt.num_exec_queues; in xe_vm_remove_compute_exec_queue()
278 if (q->lr.pfence) { in xe_vm_remove_compute_exec_queue()
279 dma_fence_enable_sw_signaling(q->lr.pfence); in xe_vm_remove_compute_exec_queue()
280 dma_fence_put(q->lr.pfence); in xe_vm_remove_compute_exec_queue()
281 q->lr.pfence = NULL; in xe_vm_remove_compute_exec_queue()
283 up_write(&vm->lock); in xe_vm_remove_compute_exec_queue()
289 * xe_vm_kill() - VM Kill
290 * @vm: The VM.
291 * @unlocked: Flag indicates the VM's dma-resv is not held
293 * Kill the VM by setting banned flag indicated VM is no longer available for
294 * use. If in preempt fence mode, also kill all exec queue attached to the VM.
296 void xe_vm_kill(struct xe_vm *vm, bool unlocked) in xe_vm_kill() argument
300 lockdep_assert_held(&vm->lock); in xe_vm_kill()
303 xe_vm_lock(vm, false); in xe_vm_kill()
305 vm->flags |= XE_VM_FLAG_BANNED; in xe_vm_kill()
306 trace_xe_vm_kill(vm); in xe_vm_kill()
308 list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) in xe_vm_kill()
309 q->ops->kill(q); in xe_vm_kill()
312 xe_vm_unlock(vm); in xe_vm_kill()
314 /* TODO: Inform user the VM is banned */ in xe_vm_kill()
319 struct xe_vm *vm = gpuvm_to_vm(vm_bo->vm); in xe_gpuvm_validate() local
323 lockdep_assert_held(&vm->lock); in xe_gpuvm_validate()
325 list_move_tail(&gpuva_to_vma(gpuva)->combined_links.rebind, in xe_gpuvm_validate()
326 &vm->rebind_list); in xe_gpuvm_validate()
328 if (!try_wait_for_completion(&vm->xe->pm_block)) in xe_gpuvm_validate()
329 return -EAGAIN; in xe_gpuvm_validate()
331 ret = xe_bo_validate(gem_to_xe_bo(vm_bo->obj), vm, false, exec); in xe_gpuvm_validate()
335 vm_bo->evicted = false; in xe_gpuvm_validate()
340 * xe_vm_validate_rebind() - Validate buffer objects and rebind vmas
341 * @vm: The vm for which we are rebinding.
347 * rebindings may cause evictions and hence the validation-rebind
351 * may return -EINTR or -ERESTARTSYS if interrupted, and -EDEADLK if
354 int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, in xe_vm_validate_rebind() argument
362 ret = drm_gpuvm_validate(&vm->gpuvm, exec); in xe_vm_validate_rebind()
366 ret = xe_vm_rebind(vm, false); in xe_vm_validate_rebind()
369 } while (!list_empty(&vm->gpuvm.evict.list)); in xe_vm_validate_rebind()
372 ret = dma_resv_reserve_fences(obj->resv, num_fences); in xe_vm_validate_rebind()
380 static int xe_preempt_work_begin(struct drm_exec *exec, struct xe_vm *vm, in xe_preempt_work_begin() argument
385 err = drm_gpuvm_prepare_vm(&vm->gpuvm, exec, 0); in xe_preempt_work_begin()
389 if (xe_vm_is_idle(vm)) { in xe_preempt_work_begin()
390 vm->preempt.rebind_deactivated = true; in xe_preempt_work_begin()
395 if (!preempt_fences_waiting(vm)) { in xe_preempt_work_begin()
400 err = drm_gpuvm_prepare_objects(&vm->gpuvm, exec, 0); in xe_preempt_work_begin()
404 err = wait_for_existing_preempt_fences(vm); in xe_preempt_work_begin()
414 return xe_vm_validate_rebind(vm, exec, vm->preempt.num_exec_queues); in xe_preempt_work_begin()
417 static bool vm_suspend_rebind_worker(struct xe_vm *vm) in vm_suspend_rebind_worker() argument
419 struct xe_device *xe = vm->xe; in vm_suspend_rebind_worker()
422 mutex_lock(&xe->rebind_resume_lock); in vm_suspend_rebind_worker()
423 if (!try_wait_for_completion(&vm->xe->pm_block)) { in vm_suspend_rebind_worker()
425 list_move_tail(&vm->preempt.pm_activate_link, &xe->rebind_resume_list); in vm_suspend_rebind_worker()
427 mutex_unlock(&xe->rebind_resume_lock); in vm_suspend_rebind_worker()
433 * xe_vm_resume_rebind_worker() - Resume the rebind worker.
434 * @vm: The vm whose preempt worker to resume.
439 void xe_vm_resume_rebind_worker(struct xe_vm *vm) in xe_vm_resume_rebind_worker() argument
441 queue_work(vm->xe->ordered_wq, &vm->preempt.rebind_work); in xe_vm_resume_rebind_worker()
446 struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work); in preempt_rebind_work_func() local
455 xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); in preempt_rebind_work_func()
456 trace_xe_vm_rebind_worker_enter(vm); in preempt_rebind_work_func()
458 down_write(&vm->lock); in preempt_rebind_work_func()
460 if (xe_vm_is_closed_or_banned(vm)) { in preempt_rebind_work_func()
461 up_write(&vm->lock); in preempt_rebind_work_func()
462 trace_xe_vm_rebind_worker_exit(vm); in preempt_rebind_work_func()
467 if (!try_wait_for_completion(&vm->xe->pm_block) && vm_suspend_rebind_worker(vm)) { in preempt_rebind_work_func()
468 up_write(&vm->lock); in preempt_rebind_work_func()
472 if (xe_vm_userptr_check_repin(vm)) { in preempt_rebind_work_func()
473 err = xe_vm_userptr_pin(vm); in preempt_rebind_work_func()
478 err = xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, in preempt_rebind_work_func()
486 err = xe_preempt_work_begin(&exec, vm, &done); in preempt_rebind_work_func()
495 err = alloc_preempt_fences(vm, &preempt_fences, &fence_count); in preempt_rebind_work_func()
499 xe_vm_set_validation_exec(vm, &exec); in preempt_rebind_work_func()
500 err = xe_vm_rebind(vm, true); in preempt_rebind_work_func()
501 xe_vm_set_validation_exec(vm, NULL); in preempt_rebind_work_func()
505 /* Wait on rebinds and munmap style VM unbinds */ in preempt_rebind_work_func()
506 wait = dma_resv_wait_timeout(xe_vm_resv(vm), in preempt_rebind_work_func()
510 err = -ETIME; in preempt_rebind_work_func()
519 xe_svm_notifier_lock(vm); in preempt_rebind_work_func()
520 if (retry_required(tries, vm)) { in preempt_rebind_work_func()
521 xe_svm_notifier_unlock(vm); in preempt_rebind_work_func()
522 err = -EAGAIN; in preempt_rebind_work_func()
528 spin_lock(&vm->xe->ttm.lru_lock); in preempt_rebind_work_func()
529 ttm_lru_bulk_move_tail(&vm->lru_bulk_move); in preempt_rebind_work_func()
530 spin_unlock(&vm->xe->ttm.lru_lock); in preempt_rebind_work_func()
533 arm_preempt_fences(vm, &preempt_fences); in preempt_rebind_work_func()
534 resume_and_reinstall_preempt_fences(vm, &exec); in preempt_rebind_work_func()
535 xe_svm_notifier_unlock(vm); in preempt_rebind_work_func()
540 if (err == -EAGAIN) { in preempt_rebind_work_func()
541 trace_xe_vm_rebind_worker_retry(vm); in preempt_rebind_work_func()
546 drm_warn(&vm->xe->drm, "VM worker error: %d\n", err); in preempt_rebind_work_func()
547 xe_vm_kill(vm, true); in preempt_rebind_work_func()
549 up_write(&vm->lock); in preempt_rebind_work_func()
553 trace_xe_vm_rebind_worker_exit(vm); in preempt_rebind_work_func()
561 if (!vops->pt_update_ops[i].num_ops) in xe_vma_ops_alloc()
564 vops->pt_update_ops[i].ops = in xe_vma_ops_alloc()
565 kmalloc_array(vops->pt_update_ops[i].num_ops, in xe_vma_ops_alloc()
566 sizeof(*vops->pt_update_ops[i].ops), in xe_vma_ops_alloc()
568 if (!vops->pt_update_ops[i].ops) in xe_vma_ops_alloc()
569 return array_of_binds ? -ENOBUFS : -ENOMEM; in xe_vma_ops_alloc()
580 vma = gpuva_to_vma(op->base.prefetch.va); in xe_vma_svm_prefetch_op_fini()
582 if (op->base.op == DRM_GPUVA_OP_PREFETCH && xe_vma_is_cpu_addr_mirror(vma)) in xe_vma_svm_prefetch_op_fini()
583 xa_destroy(&op->prefetch_range.range); in xe_vma_svm_prefetch_op_fini()
590 if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) in xe_vma_svm_prefetch_ops_fini()
593 list_for_each_entry(op, &vops->list, link) in xe_vma_svm_prefetch_ops_fini()
604 kfree(vops->pt_update_ops[i].ops); in xe_vma_ops_fini()
616 vops->pt_update_ops[i].num_ops += inc_val; in xe_vma_ops_incr_pt_update_ops()
629 INIT_LIST_HEAD(&op->link); in xe_vm_populate_rebind()
630 op->tile_mask = tile_mask; in xe_vm_populate_rebind()
631 op->base.op = DRM_GPUVA_OP_MAP; in xe_vm_populate_rebind()
632 op->base.map.va.addr = vma->gpuva.va.addr; in xe_vm_populate_rebind()
633 op->base.map.va.range = vma->gpuva.va.range; in xe_vm_populate_rebind()
634 op->base.map.gem.obj = vma->gpuva.gem.obj; in xe_vm_populate_rebind()
635 op->base.map.gem.offset = vma->gpuva.gem.offset; in xe_vm_populate_rebind()
636 op->map.vma = vma; in xe_vm_populate_rebind()
637 op->map.immediate = true; in xe_vm_populate_rebind()
638 op->map.vma_flags = vma->gpuva.flags & XE_VMA_CREATE_MASK; in xe_vm_populate_rebind()
648 return -ENOMEM; in xe_vm_ops_add_rebind()
651 list_add_tail(&op->link, &vops->list); in xe_vm_ops_add_rebind()
657 static struct dma_fence *ops_execute(struct xe_vm *vm,
659 static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
663 int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) in xe_vm_rebind() argument
671 lockdep_assert_held(&vm->lock); in xe_vm_rebind()
672 if ((xe_vm_in_lr_mode(vm) && !rebind_worker) || in xe_vm_rebind()
673 list_empty(&vm->rebind_list)) in xe_vm_rebind()
676 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); in xe_vm_rebind()
680 xe_vm_assert_held(vm); in xe_vm_rebind()
681 list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) { in xe_vm_rebind()
682 xe_assert(vm->xe, vma->tile_present); in xe_vm_rebind()
690 vma->tile_present); in xe_vm_rebind()
699 fence = ops_execute(vm, &vops); in xe_vm_rebind()
704 list_for_each_entry_safe(vma, next, &vm->rebind_list, in xe_vm_rebind()
706 list_del_init(&vma->combined_links.rebind); in xe_vm_rebind()
710 list_del(&op->link); in xe_vm_rebind()
718 struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask) in xe_vma_rebind() argument
727 lockdep_assert_held(&vm->lock); in xe_vma_rebind()
728 xe_vm_assert_held(vm); in xe_vma_rebind()
729 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vma_rebind()
731 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); in xe_vma_rebind()
732 for_each_tile(tile, vm->xe, id) { in xe_vma_rebind()
734 vops.pt_update_ops[tile->id].q = in xe_vma_rebind()
735 xe_migrate_exec_queue(tile->migrate); in xe_vma_rebind()
748 fence = ops_execute(vm, &vops); in xe_vma_rebind()
752 list_del(&op->link); in xe_vma_rebind()
765 INIT_LIST_HEAD(&op->link); in xe_vm_populate_range_rebind()
766 op->tile_mask = tile_mask; in xe_vm_populate_range_rebind()
767 op->base.op = DRM_GPUVA_OP_DRIVER; in xe_vm_populate_range_rebind()
768 op->subop = XE_VMA_SUBOP_MAP_RANGE; in xe_vm_populate_range_rebind()
769 op->map_range.vma = vma; in xe_vm_populate_range_rebind()
770 op->map_range.range = range; in xe_vm_populate_range_rebind()
783 return -ENOMEM; in xe_vm_ops_add_range_rebind()
786 list_add_tail(&op->link, &vops->list); in xe_vm_ops_add_range_rebind()
793 * xe_vm_range_rebind() - VM range (re)bind
794 * @vm: The VM which the range belongs to.
804 struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm, in xe_vm_range_rebind() argument
816 lockdep_assert_held(&vm->lock); in xe_vm_range_rebind()
817 xe_vm_assert_held(vm); in xe_vm_range_rebind()
818 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vm_range_rebind()
819 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); in xe_vm_range_rebind()
821 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); in xe_vm_range_rebind()
822 for_each_tile(tile, vm->xe, id) { in xe_vm_range_rebind()
824 vops.pt_update_ops[tile->id].q = in xe_vm_range_rebind()
825 xe_migrate_exec_queue(tile->migrate); in xe_vm_range_rebind()
838 fence = ops_execute(vm, &vops); in xe_vm_range_rebind()
842 list_del(&op->link); in xe_vm_range_rebind()
853 INIT_LIST_HEAD(&op->link); in xe_vm_populate_range_unbind()
854 op->tile_mask = range->tile_present; in xe_vm_populate_range_unbind()
855 op->base.op = DRM_GPUVA_OP_DRIVER; in xe_vm_populate_range_unbind()
856 op->subop = XE_VMA_SUBOP_UNMAP_RANGE; in xe_vm_populate_range_unbind()
857 op->unmap_range.range = range; in xe_vm_populate_range_unbind()
868 return -ENOMEM; in xe_vm_ops_add_range_unbind()
871 list_add_tail(&op->link, &vops->list); in xe_vm_ops_add_range_unbind()
872 xe_vma_ops_incr_pt_update_ops(vops, range->tile_present, 1); in xe_vm_ops_add_range_unbind()
878 * xe_vm_range_unbind() - VM range unbind
879 * @vm: The VM which the range belongs to.
887 struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm, in xe_vm_range_unbind() argument
897 lockdep_assert_held(&vm->lock); in xe_vm_range_unbind()
898 xe_vm_assert_held(vm); in xe_vm_range_unbind()
899 xe_assert(vm->xe, xe_vm_in_fault_mode(vm)); in xe_vm_range_unbind()
901 if (!range->tile_present) in xe_vm_range_unbind()
904 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); in xe_vm_range_unbind()
905 for_each_tile(tile, vm->xe, id) { in xe_vm_range_unbind()
907 vops.pt_update_ops[tile->id].q = in xe_vm_range_unbind()
908 xe_migrate_exec_queue(tile->migrate); in xe_vm_range_unbind()
921 fence = ops_execute(vm, &vops); in xe_vm_range_unbind()
925 list_del(&op->link); in xe_vm_range_unbind()
941 static struct xe_vma *xe_vma_create(struct xe_vm *vm, in xe_vma_create() argument
954 xe_assert(vm->xe, start < end); in xe_vma_create()
955 xe_assert(vm->xe, end < vm->size); in xe_vma_create()
965 return ERR_PTR(-ENOMEM); in xe_vma_create()
967 vma = &uvma->vma; in xe_vma_create()
971 return ERR_PTR(-ENOMEM); in xe_vma_create()
974 vma->gpuva.gem.obj = &bo->ttm.base; in xe_vma_create()
977 INIT_LIST_HEAD(&vma->combined_links.rebind); in xe_vma_create()
979 INIT_LIST_HEAD(&vma->gpuva.gem.entry); in xe_vma_create()
980 vma->gpuva.vm = &vm->gpuvm; in xe_vma_create()
981 vma->gpuva.va.addr = start; in xe_vma_create()
982 vma->gpuva.va.range = end - start + 1; in xe_vma_create()
983 vma->gpuva.flags = flags; in xe_vma_create()
985 for_each_tile(tile, vm->xe, id) in xe_vma_create()
986 vma->tile_mask |= 0x1 << id; in xe_vma_create()
988 if (vm->xe->info.has_atomic_enable_pte_bit) in xe_vma_create()
989 vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; in xe_vma_create()
991 vma->attr = *attr; in xe_vma_create()
998 vm_bo = drm_gpuvm_bo_obtain(vma->gpuva.vm, &bo->ttm.base); in xe_vma_create()
1005 drm_gem_object_get(&bo->ttm.base); in xe_vma_create()
1006 vma->gpuva.gem.offset = bo_offset_or_userptr; in xe_vma_create()
1007 drm_gpuva_link(&vma->gpuva, vm_bo); in xe_vma_create()
1012 u64 size = end - start + 1; in xe_vma_create()
1015 vma->gpuva.gem.offset = bo_offset_or_userptr; in xe_vma_create()
1024 xe_vm_get(vm); in xe_vma_create()
1032 struct xe_vm *vm = xe_vma_vm(vma); in xe_vma_destroy_late() local
1034 if (vma->ufence) { in xe_vma_destroy_late()
1035 xe_sync_ufence_put(vma->ufence); in xe_vma_destroy_late()
1036 vma->ufence = NULL; in xe_vma_destroy_late()
1043 xe_vm_put(vm); in xe_vma_destroy_late()
1045 xe_vm_put(vm); in xe_vma_destroy_late()
1066 INIT_WORK(&vma->destroy_work, vma_destroy_work_func); in vma_destroy_cb()
1067 queue_work(system_unbound_wq, &vma->destroy_work); in vma_destroy_cb()
1072 struct xe_vm *vm = xe_vma_vm(vma); in xe_vma_destroy() local
1074 lockdep_assert_held_write(&vm->lock); in xe_vma_destroy()
1075 xe_assert(vm->xe, list_empty(&vma->combined_links.destroy)); in xe_vma_destroy()
1078 xe_assert(vm->xe, vma->gpuva.flags & XE_VMA_DESTROYED); in xe_vma_destroy()
1083 drm_gpuva_unlink(&vma->gpuva); in xe_vma_destroy()
1086 xe_vm_assert_held(vm); in xe_vma_destroy()
1088 int ret = dma_fence_add_callback(fence, &vma->destroy_cb, in xe_vma_destroy()
1092 XE_WARN_ON(ret != -ENOENT); in xe_vma_destroy()
1101 * xe_vm_lock_vma() - drm_exec utility to lock a vma
1103 * @vma: The vma for witch we want to lock the vm resv and any attached
1107 * may return -EDEADLK on WW transaction contention and -EINTR if
1112 struct xe_vm *vm = xe_vma_vm(vma); in xe_vm_lock_vma() local
1116 XE_WARN_ON(!vm); in xe_vm_lock_vma()
1118 err = drm_exec_lock_obj(exec, xe_vm_obj(vm)); in xe_vm_lock_vma()
1119 if (!err && bo && !bo->vm) in xe_vm_lock_vma()
1120 err = drm_exec_lock_obj(exec, &bo->ttm.base); in xe_vm_lock_vma()
1127 struct xe_device *xe = xe_vma_vm(vma)->xe; in xe_vma_destroy_unlocked()
1132 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { in xe_vma_destroy_unlocked()
1143 xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range) in xe_vm_find_overlapping_vma() argument
1147 lockdep_assert_held(&vm->lock); in xe_vm_find_overlapping_vma()
1149 if (xe_vm_is_closed_or_banned(vm)) in xe_vm_find_overlapping_vma()
1152 xe_assert(vm->xe, start + range <= vm->size); in xe_vm_find_overlapping_vma()
1154 gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range); in xe_vm_find_overlapping_vma()
1159 static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma) in xe_vm_insert_vma() argument
1163 xe_assert(vm->xe, xe_vma_vm(vma) == vm); in xe_vm_insert_vma()
1164 lockdep_assert_held(&vm->lock); in xe_vm_insert_vma()
1166 mutex_lock(&vm->snap_mutex); in xe_vm_insert_vma()
1167 err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva); in xe_vm_insert_vma()
1168 mutex_unlock(&vm->snap_mutex); in xe_vm_insert_vma()
1174 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma) in xe_vm_remove_vma() argument
1176 xe_assert(vm->xe, xe_vma_vm(vma) == vm); in xe_vm_remove_vma()
1177 lockdep_assert_held(&vm->lock); in xe_vm_remove_vma()
1179 mutex_lock(&vm->snap_mutex); in xe_vm_remove_vma()
1180 drm_gpuva_remove(&vma->gpuva); in xe_vm_remove_vma()
1181 mutex_unlock(&vm->snap_mutex); in xe_vm_remove_vma()
1182 if (vm->usm.last_fault_vma == vma) in xe_vm_remove_vma()
1183 vm->usm.last_fault_vma = NULL; in xe_vm_remove_vma()
1195 return &op->base; in xe_vm_op_alloc()
1263 * We only have two bits to encode the PAT index in non-leaf nodes, but in pde_pat_index()
1267 * non-leaf nodes are instead under driver control so the chosen index in pde_pat_index()
1273 if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached) in pde_pat_index()
1274 pat_index = xe->pat.idx[XE_CACHE_WB]; in pde_pat_index()
1276 pat_index = xe->pat.idx[XE_CACHE_NONE]; in pde_pat_index()
1359 * xe_vm_create_scratch() - Setup a scratch memory pagetable tree for the
1360 * given tile and vm.
1363 * @vm: vm to set up for.
1364 * @exec: The struct drm_exec object used to lock the vm resv.
1366 * Sets up a pagetable tree with one page-table per level and a single
1367 * leaf PTE. All pagetable entries point to the single page-table or,
1374 struct xe_vm *vm, struct drm_exec *exec) in xe_vm_create_scratch() argument
1376 u8 id = tile->id; in xe_vm_create_scratch()
1379 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; i++) { in xe_vm_create_scratch()
1380 vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i, exec); in xe_vm_create_scratch()
1381 if (IS_ERR(vm->scratch_pt[id][i])) { in xe_vm_create_scratch()
1382 int err = PTR_ERR(vm->scratch_pt[id][i]); in xe_vm_create_scratch()
1384 vm->scratch_pt[id][i] = NULL; in xe_vm_create_scratch()
1387 xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); in xe_vm_create_scratch()
1394 static void xe_vm_free_scratch(struct xe_vm *vm) in xe_vm_free_scratch() argument
1399 if (!xe_vm_has_scratch(vm)) in xe_vm_free_scratch()
1402 for_each_tile(tile, vm->xe, id) { in xe_vm_free_scratch()
1405 if (!vm->pt_root[id]) in xe_vm_free_scratch()
1408 for (i = MAX_HUGEPTE_LEVEL; i < vm->pt_root[id]->level; ++i) in xe_vm_free_scratch()
1409 if (vm->scratch_pt[id][i]) in xe_vm_free_scratch()
1410 xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, NULL); in xe_vm_free_scratch()
1414 static void xe_vm_pt_destroy(struct xe_vm *vm) in xe_vm_pt_destroy() argument
1419 xe_vm_assert_held(vm); in xe_vm_pt_destroy()
1421 for_each_tile(tile, vm->xe, id) { in xe_vm_pt_destroy()
1422 if (vm->pt_root[id]) { in xe_vm_pt_destroy()
1423 xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); in xe_vm_pt_destroy()
1424 vm->pt_root[id] = NULL; in xe_vm_pt_destroy()
1434 struct xe_vm *vm; in xe_vm_create() local
1440 * Since the GSCCS is not user-accessible, we don't expect a GSC VM to in xe_vm_create()
1445 vm = kzalloc(sizeof(*vm), GFP_KERNEL); in xe_vm_create()
1446 if (!vm) in xe_vm_create()
1447 return ERR_PTR(-ENOMEM); in xe_vm_create()
1449 vm->xe = xe; in xe_vm_create()
1451 vm->size = 1ull << xe->info.va_bits; in xe_vm_create()
1452 vm->flags = flags; in xe_vm_create()
1455 vm->xef = xe_file_get(xef); in xe_vm_create()
1457 * GSC VMs are kernel-owned, only used for PXP ops and can sometimes be in xe_vm_create()
1459 * under a user-VM lock when the PXP session is started at exec_queue in xe_vm_create()
1467 __init_rwsem(&vm->lock, "gsc_vm", &gsc_vm_key); in xe_vm_create()
1469 init_rwsem(&vm->lock); in xe_vm_create()
1471 mutex_init(&vm->snap_mutex); in xe_vm_create()
1473 INIT_LIST_HEAD(&vm->rebind_list); in xe_vm_create()
1475 INIT_LIST_HEAD(&vm->userptr.repin_list); in xe_vm_create()
1476 INIT_LIST_HEAD(&vm->userptr.invalidated); in xe_vm_create()
1477 spin_lock_init(&vm->userptr.invalidated_lock); in xe_vm_create()
1479 ttm_lru_bulk_move_init(&vm->lru_bulk_move); in xe_vm_create()
1481 INIT_WORK(&vm->destroy_work, vm_destroy_work_func); in xe_vm_create()
1483 INIT_LIST_HEAD(&vm->preempt.exec_queues); in xe_vm_create()
1484 vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ in xe_vm_create()
1487 xe_range_fence_tree_init(&vm->rftree[id]); in xe_vm_create()
1489 vm->pt_ops = &xelp_pt_ops; in xe_vm_create()
1492 * Long-running workloads are not protected by the scheduler references. in xe_vm_create()
1493 * By design, run_job for long-running workloads returns NULL and the in xe_vm_create()
1494 * scheduler drops all the references of it, hence protecting the VM in xe_vm_create()
1498 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); in xe_vm_create()
1500 INIT_LIST_HEAD(&vm->preempt.pm_activate_link); in xe_vm_create()
1503 err = xe_svm_init(vm); in xe_vm_create()
1507 vm_resv_obj = drm_gpuvm_resv_object_alloc(&xe->drm); in xe_vm_create()
1509 err = -ENOMEM; in xe_vm_create()
1513 drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm, in xe_vm_create()
1514 vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops); in xe_vm_create()
1519 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, in xe_vm_create()
1521 err = xe_vm_drm_exec_lock(vm, &exec); in xe_vm_create()
1524 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) in xe_vm_create()
1525 vm->flags |= XE_VM_FLAG_64K; in xe_vm_create()
1529 tile->id != XE_VM_FLAG_TILE_ID(flags)) in xe_vm_create()
1532 vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level, in xe_vm_create()
1534 if (IS_ERR(vm->pt_root[id])) { in xe_vm_create()
1535 err = PTR_ERR(vm->pt_root[id]); in xe_vm_create()
1536 vm->pt_root[id] = NULL; in xe_vm_create()
1537 xe_vm_pt_destroy(vm); in xe_vm_create()
1546 if (xe_vm_has_scratch(vm)) { in xe_vm_create()
1548 if (!vm->pt_root[id]) in xe_vm_create()
1551 err = xe_vm_create_scratch(xe, tile, vm, &exec); in xe_vm_create()
1553 xe_vm_free_scratch(vm); in xe_vm_create()
1554 xe_vm_pt_destroy(vm); in xe_vm_create()
1562 vm->batch_invalidate_tlb = true; in xe_vm_create()
1565 if (vm->flags & XE_VM_FLAG_LR_MODE) { in xe_vm_create()
1566 INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); in xe_vm_create()
1567 vm->batch_invalidate_tlb = false; in xe_vm_create()
1572 if (!vm->pt_root[id]) in xe_vm_create()
1575 xe_pt_populate_empty(tile, vm, vm->pt_root[id]); in xe_vm_create()
1581 /* Kernel migration VM shouldn't have a circular loop.. */ in xe_vm_create()
1587 if (!vm->pt_root[id]) in xe_vm_create()
1595 vm->q[id] = q; in xe_vm_create()
1601 vm->composite_fence_ctx = dma_fence_context_alloc(1); in xe_vm_create()
1603 if (xef && xe->info.has_asid) { in xe_vm_create()
1606 down_write(&xe->usm.lock); in xe_vm_create()
1607 err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, in xe_vm_create()
1608 XA_LIMIT(1, XE_MAX_ASID - 1), in xe_vm_create()
1609 &xe->usm.next_asid, GFP_KERNEL); in xe_vm_create()
1610 up_write(&xe->usm.lock); in xe_vm_create()
1614 vm->usm.asid = asid; in xe_vm_create()
1617 trace_xe_vm_create(vm); in xe_vm_create()
1619 return vm; in xe_vm_create()
1622 xe_vm_close_and_put(vm); in xe_vm_create()
1627 vm->size = 0; /* close the vm */ in xe_vm_create()
1628 xe_svm_fini(vm); in xe_vm_create()
1631 mutex_destroy(&vm->snap_mutex); in xe_vm_create()
1633 xe_range_fence_tree_fini(&vm->rftree[id]); in xe_vm_create()
1634 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); in xe_vm_create()
1635 if (vm->xef) in xe_vm_create()
1636 xe_file_put(vm->xef); in xe_vm_create()
1637 kfree(vm); in xe_vm_create()
1643 static void xe_vm_close(struct xe_vm *vm) in xe_vm_close() argument
1645 struct xe_device *xe = vm->xe; in xe_vm_close()
1649 bound = drm_dev_enter(&xe->drm, &idx); in xe_vm_close()
1651 down_write(&vm->lock); in xe_vm_close()
1652 if (xe_vm_in_fault_mode(vm)) in xe_vm_close()
1653 xe_svm_notifier_lock(vm); in xe_vm_close()
1655 vm->size = 0; in xe_vm_close()
1657 if (!((vm->flags & XE_VM_FLAG_MIGRATION))) { in xe_vm_close()
1663 dma_resv_wait_timeout(xe_vm_resv(vm), in xe_vm_close()
1669 if (vm->pt_root[id]) in xe_vm_close()
1670 xe_pt_clear(xe, vm->pt_root[id]); in xe_vm_close()
1673 xe_tlb_inval_vm(>->tlb_inval, vm); in xe_vm_close()
1677 if (xe_vm_in_fault_mode(vm)) in xe_vm_close()
1678 xe_svm_notifier_unlock(vm); in xe_vm_close()
1679 up_write(&vm->lock); in xe_vm_close()
1685 void xe_vm_close_and_put(struct xe_vm *vm) in xe_vm_close_and_put() argument
1688 struct xe_device *xe = vm->xe; in xe_vm_close_and_put()
1694 xe_assert(xe, !vm->preempt.num_exec_queues); in xe_vm_close_and_put()
1696 xe_vm_close(vm); in xe_vm_close_and_put()
1697 if (xe_vm_in_preempt_fence_mode(vm)) { in xe_vm_close_and_put()
1698 mutex_lock(&xe->rebind_resume_lock); in xe_vm_close_and_put()
1699 list_del_init(&vm->preempt.pm_activate_link); in xe_vm_close_and_put()
1700 mutex_unlock(&xe->rebind_resume_lock); in xe_vm_close_and_put()
1701 flush_work(&vm->preempt.rebind_work); in xe_vm_close_and_put()
1703 if (xe_vm_in_fault_mode(vm)) in xe_vm_close_and_put()
1704 xe_svm_close(vm); in xe_vm_close_and_put()
1706 down_write(&vm->lock); in xe_vm_close_and_put()
1708 if (vm->q[id]) in xe_vm_close_and_put()
1709 xe_exec_queue_last_fence_put(vm->q[id], vm); in xe_vm_close_and_put()
1711 up_write(&vm->lock); in xe_vm_close_and_put()
1714 if (vm->q[id]) { in xe_vm_close_and_put()
1715 xe_exec_queue_kill(vm->q[id]); in xe_vm_close_and_put()
1716 xe_exec_queue_put(vm->q[id]); in xe_vm_close_and_put()
1717 vm->q[id] = NULL; in xe_vm_close_and_put()
1721 down_write(&vm->lock); in xe_vm_close_and_put()
1722 xe_vm_lock(vm, false); in xe_vm_close_and_put()
1723 drm_gpuvm_for_each_va_safe(gpuva, next, &vm->gpuvm) { in xe_vm_close_and_put()
1727 xe_svm_notifier_lock(vm); in xe_vm_close_and_put()
1728 vma->gpuva.flags |= XE_VMA_DESTROYED; in xe_vm_close_and_put()
1729 xe_svm_notifier_unlock(vm); in xe_vm_close_and_put()
1732 xe_vm_remove_vma(vm, vma); in xe_vm_close_and_put()
1735 if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm) { in xe_vm_close_and_put()
1736 list_del_init(&vma->combined_links.rebind); in xe_vm_close_and_put()
1741 list_move_tail(&vma->combined_links.destroy, &contested); in xe_vm_close_and_put()
1742 vma->gpuva.flags |= XE_VMA_DESTROYED; in xe_vm_close_and_put()
1746 * All vm operations will add shared fences to resv. in xe_vm_close_and_put()
1752 xe_vm_free_scratch(vm); in xe_vm_close_and_put()
1753 xe_vm_pt_destroy(vm); in xe_vm_close_and_put()
1754 xe_vm_unlock(vm); in xe_vm_close_and_put()
1757 * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL in xe_vm_close_and_put()
1763 list_del_init(&vma->combined_links.destroy); in xe_vm_close_and_put()
1767 xe_svm_fini(vm); in xe_vm_close_and_put()
1769 up_write(&vm->lock); in xe_vm_close_and_put()
1771 down_write(&xe->usm.lock); in xe_vm_close_and_put()
1772 if (vm->usm.asid) { in xe_vm_close_and_put()
1775 xe_assert(xe, xe->info.has_asid); in xe_vm_close_and_put()
1776 xe_assert(xe, !(vm->flags & XE_VM_FLAG_MIGRATION)); in xe_vm_close_and_put()
1778 lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); in xe_vm_close_and_put()
1779 xe_assert(xe, lookup == vm); in xe_vm_close_and_put()
1781 up_write(&xe->usm.lock); in xe_vm_close_and_put()
1784 xe_range_fence_tree_fini(&vm->rftree[id]); in xe_vm_close_and_put()
1786 xe_vm_put(vm); in xe_vm_close_and_put()
1791 struct xe_vm *vm = in vm_destroy_work_func() local
1793 struct xe_device *xe = vm->xe; in vm_destroy_work_func()
1798 xe_assert(xe, !vm->size); in vm_destroy_work_func()
1800 if (xe_vm_in_preempt_fence_mode(vm)) in vm_destroy_work_func()
1801 flush_work(&vm->preempt.rebind_work); in vm_destroy_work_func()
1803 mutex_destroy(&vm->snap_mutex); in vm_destroy_work_func()
1805 if (vm->flags & XE_VM_FLAG_LR_MODE) in vm_destroy_work_func()
1809 XE_WARN_ON(vm->pt_root[id]); in vm_destroy_work_func()
1811 trace_xe_vm_free(vm); in vm_destroy_work_func()
1813 ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move); in vm_destroy_work_func()
1815 if (vm->xef) in vm_destroy_work_func()
1816 xe_file_put(vm->xef); in vm_destroy_work_func()
1818 kfree(vm); in vm_destroy_work_func()
1823 struct xe_vm *vm = container_of(gpuvm, struct xe_vm, gpuvm); in xe_vm_free() local
1825 /* To destroy the VM we need to be able to sleep */ in xe_vm_free()
1826 queue_work(system_unbound_wq, &vm->destroy_work); in xe_vm_free()
1831 struct xe_vm *vm; in xe_vm_lookup() local
1833 mutex_lock(&xef->vm.lock); in xe_vm_lookup()
1834 vm = xa_load(&xef->vm.xa, id); in xe_vm_lookup()
1835 if (vm) in xe_vm_lookup()
1836 xe_vm_get(vm); in xe_vm_lookup()
1837 mutex_unlock(&xef->vm.lock); in xe_vm_lookup()
1839 return vm; in xe_vm_lookup()
1842 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) in xe_vm_pdp4_descriptor() argument
1844 return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0); in xe_vm_pdp4_descriptor()
1848 to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) in to_wait_exec_queue() argument
1850 return q ? q : vm->q[0]; in to_wait_exec_queue()
1878 struct xe_vm *vm; in xe_vm_create_ioctl() local
1883 if (XE_IOCTL_DBG(xe, args->extensions)) in xe_vm_create_ioctl()
1884 return -EINVAL; in xe_vm_create_ioctl()
1887 args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; in xe_vm_create_ioctl()
1889 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && in xe_vm_create_ioctl()
1890 !xe->info.has_usm)) in xe_vm_create_ioctl()
1891 return -EINVAL; in xe_vm_create_ioctl()
1893 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_vm_create_ioctl()
1894 return -EINVAL; in xe_vm_create_ioctl()
1896 if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) in xe_vm_create_ioctl()
1897 return -EINVAL; in xe_vm_create_ioctl()
1899 if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && in xe_vm_create_ioctl()
1900 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && in xe_vm_create_ioctl()
1901 !xe->info.needs_scratch)) in xe_vm_create_ioctl()
1902 return -EINVAL; in xe_vm_create_ioctl()
1904 if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && in xe_vm_create_ioctl()
1905 args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) in xe_vm_create_ioctl()
1906 return -EINVAL; in xe_vm_create_ioctl()
1908 if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) in xe_vm_create_ioctl()
1910 if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) in xe_vm_create_ioctl()
1912 if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) in xe_vm_create_ioctl()
1915 vm = xe_vm_create(xe, flags, xef); in xe_vm_create_ioctl()
1916 if (IS_ERR(vm)) in xe_vm_create_ioctl()
1917 return PTR_ERR(vm); in xe_vm_create_ioctl()
1920 /* Warning: Security issue - never enable by default */ in xe_vm_create_ioctl()
1921 args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, XE_PAGE_SIZE); in xe_vm_create_ioctl()
1925 err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); in xe_vm_create_ioctl()
1929 args->vm_id = id; in xe_vm_create_ioctl()
1934 xe_vm_close_and_put(vm); in xe_vm_create_ioctl()
1945 struct xe_vm *vm; in xe_vm_destroy_ioctl() local
1948 if (XE_IOCTL_DBG(xe, args->pad) || in xe_vm_destroy_ioctl()
1949 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_vm_destroy_ioctl()
1950 return -EINVAL; in xe_vm_destroy_ioctl()
1952 mutex_lock(&xef->vm.lock); in xe_vm_destroy_ioctl()
1953 vm = xa_load(&xef->vm.xa, args->vm_id); in xe_vm_destroy_ioctl()
1954 if (XE_IOCTL_DBG(xe, !vm)) in xe_vm_destroy_ioctl()
1955 err = -ENOENT; in xe_vm_destroy_ioctl()
1956 else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) in xe_vm_destroy_ioctl()
1957 err = -EBUSY; in xe_vm_destroy_ioctl()
1959 xa_erase(&xef->vm.xa, args->vm_id); in xe_vm_destroy_ioctl()
1960 mutex_unlock(&xef->vm.lock); in xe_vm_destroy_ioctl()
1963 xe_vm_close_and_put(vm); in xe_vm_destroy_ioctl()
1968 static int xe_vm_query_vmas(struct xe_vm *vm, u64 start, u64 end) in xe_vm_query_vmas() argument
1973 lockdep_assert_held(&vm->lock); in xe_vm_query_vmas()
1974 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) in xe_vm_query_vmas()
1980 static int get_mem_attrs(struct xe_vm *vm, u32 *num_vmas, u64 start, in get_mem_attrs() argument
1986 lockdep_assert_held(&vm->lock); in get_mem_attrs()
1988 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { in get_mem_attrs()
1992 return -ENOSPC; in get_mem_attrs()
1996 attrs[i].atomic.val = vma->attr.atomic_access; in get_mem_attrs()
1997 attrs[i].pat_index.val = vma->attr.pat_index; in get_mem_attrs()
1998 attrs[i].preferred_mem_loc.devmem_fd = vma->attr.preferred_loc.devmem_fd; in get_mem_attrs()
2000 vma->attr.preferred_loc.migration_policy; in get_mem_attrs()
2015 u64 __user *attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); in xe_vm_query_vmas_attrs_ioctl()
2016 struct xe_vm *vm; in xe_vm_query_vmas_attrs_ioctl() local
2020 ((args->num_mem_ranges == 0 && in xe_vm_query_vmas_attrs_ioctl()
2021 (attrs_user || args->sizeof_mem_range_attr != 0)) || in xe_vm_query_vmas_attrs_ioctl()
2022 (args->num_mem_ranges > 0 && in xe_vm_query_vmas_attrs_ioctl()
2024 args->sizeof_mem_range_attr != in xe_vm_query_vmas_attrs_ioctl()
2026 return -EINVAL; in xe_vm_query_vmas_attrs_ioctl()
2028 vm = xe_vm_lookup(xef, args->vm_id); in xe_vm_query_vmas_attrs_ioctl()
2029 if (XE_IOCTL_DBG(xe, !vm)) in xe_vm_query_vmas_attrs_ioctl()
2030 return -EINVAL; in xe_vm_query_vmas_attrs_ioctl()
2032 err = down_read_interruptible(&vm->lock); in xe_vm_query_vmas_attrs_ioctl()
2036 attrs_user = u64_to_user_ptr(args->vector_of_mem_attr); in xe_vm_query_vmas_attrs_ioctl()
2038 if (args->num_mem_ranges == 0 && !attrs_user) { in xe_vm_query_vmas_attrs_ioctl()
2039 args->num_mem_ranges = xe_vm_query_vmas(vm, args->start, args->start + args->range); in xe_vm_query_vmas_attrs_ioctl()
2040 args->sizeof_mem_range_attr = sizeof(struct drm_xe_mem_range_attr); in xe_vm_query_vmas_attrs_ioctl()
2044 mem_attrs = kvmalloc_array(args->num_mem_ranges, args->sizeof_mem_range_attr, in xe_vm_query_vmas_attrs_ioctl()
2048 err = args->num_mem_ranges > 1 ? -ENOBUFS : -ENOMEM; in xe_vm_query_vmas_attrs_ioctl()
2052 memset(mem_attrs, 0, args->num_mem_ranges * args->sizeof_mem_range_attr); in xe_vm_query_vmas_attrs_ioctl()
2053 err = get_mem_attrs(vm, &args->num_mem_ranges, args->start, in xe_vm_query_vmas_attrs_ioctl()
2054 args->start + args->range, mem_attrs); in xe_vm_query_vmas_attrs_ioctl()
2059 args->sizeof_mem_range_attr * args->num_mem_ranges); in xe_vm_query_vmas_attrs_ioctl()
2061 err = -EFAULT; in xe_vm_query_vmas_attrs_ioctl()
2066 up_read(&vm->lock); in xe_vm_query_vmas_attrs_ioctl()
2068 xe_vm_put(vm); in xe_vm_query_vmas_attrs_ioctl()
2074 if (page_addr > xe_vma_end(vma) - 1 || in vma_matches()
2075 page_addr + SZ_4K - 1 < xe_vma_start(vma)) in vma_matches()
2082 * xe_vm_find_vma_by_addr() - Find a VMA by its address
2084 * @vm: the xe_vm the vma belongs to
2087 struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr) in xe_vm_find_vma_by_addr() argument
2091 if (vm->usm.last_fault_vma) { /* Fast lookup */ in xe_vm_find_vma_by_addr()
2092 if (vma_matches(vm->usm.last_fault_vma, page_addr)) in xe_vm_find_vma_by_addr()
2093 vma = vm->usm.last_fault_vma; in xe_vm_find_vma_by_addr()
2096 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K); in xe_vm_find_vma_by_addr()
2107 static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma, in prep_vma_destroy() argument
2110 xe_svm_notifier_lock(vm); in prep_vma_destroy()
2111 vma->gpuva.flags |= XE_VMA_DESTROYED; in prep_vma_destroy()
2112 xe_svm_notifier_unlock(vm); in prep_vma_destroy()
2114 xe_vm_remove_vma(vm, vma); in prep_vma_destroy()
2125 switch (op->op) { in print_op()
2127 vm_dbg(&xe->drm, "MAP: addr=0x%016llx, range=0x%016llx", in print_op()
2128 (ULL)op->map.va.addr, (ULL)op->map.va.range); in print_op()
2131 vma = gpuva_to_vma(op->remap.unmap->va); in print_op()
2132 vm_dbg(&xe->drm, "REMAP:UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", in print_op()
2134 op->remap.unmap->keep ? 1 : 0); in print_op()
2135 if (op->remap.prev) in print_op()
2136 vm_dbg(&xe->drm, in print_op()
2138 (ULL)op->remap.prev->va.addr, in print_op()
2139 (ULL)op->remap.prev->va.range); in print_op()
2140 if (op->remap.next) in print_op()
2141 vm_dbg(&xe->drm, in print_op()
2143 (ULL)op->remap.next->va.addr, in print_op()
2144 (ULL)op->remap.next->va.range); in print_op()
2147 vma = gpuva_to_vma(op->unmap.va); in print_op()
2148 vm_dbg(&xe->drm, "UNMAP: addr=0x%016llx, range=0x%016llx, keep=%d", in print_op()
2150 op->unmap.keep ? 1 : 0); in print_op()
2153 vma = gpuva_to_vma(op->prefetch.va); in print_op()
2154 vm_dbg(&xe->drm, "PREFETCH: addr=0x%016llx, range=0x%016llx", in print_op()
2158 drm_warn(&xe->drm, "NOT POSSIBLE"); in print_op()
2167 static bool __xe_vm_needs_clear_scratch_pages(struct xe_vm *vm, u32 bind_flags) in __xe_vm_needs_clear_scratch_pages() argument
2169 if (!xe_vm_in_fault_mode(vm)) in __xe_vm_needs_clear_scratch_pages()
2172 if (!xe_vm_has_scratch(vm)) in __xe_vm_needs_clear_scratch_pages()
2197 vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops, in vm_bind_ioctl_ops_create() argument
2203 struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; in vm_bind_ioctl_ops_create()
2210 lockdep_assert_held_write(&vm->lock); in vm_bind_ioctl_ops_create()
2212 vm_dbg(&vm->xe->drm, in vm_bind_ioctl_ops_create()
2221 .map.va.addr = addr, in vm_bind_ioctl_ops_create()
2222 .map.va.range = range, in vm_bind_ioctl_ops_create()
2223 .map.gem.obj = obj, in vm_bind_ioctl_ops_create()
2224 .map.gem.offset = bo_offset_or_userptr, in vm_bind_ioctl_ops_create()
2227 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, &map_req); in vm_bind_ioctl_ops_create()
2231 ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range); in vm_bind_ioctl_ops_create()
2234 ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range); in vm_bind_ioctl_ops_create()
2237 xe_assert(vm->xe, bo); in vm_bind_ioctl_ops_create()
2243 vm_bo = drm_gpuvm_bo_obtain(&vm->gpuvm, obj); in vm_bind_ioctl_ops_create()
2254 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in vm_bind_ioctl_ops_create()
2255 ops = ERR_PTR(-EINVAL); in vm_bind_ioctl_ops_create()
2263 if (__op->op == DRM_GPUVA_OP_MAP) { in vm_bind_ioctl_ops_create()
2264 op->map.immediate = in vm_bind_ioctl_ops_create()
2267 op->map.vma_flags |= XE_VMA_READ_ONLY; in vm_bind_ioctl_ops_create()
2269 op->map.vma_flags |= DRM_GPUVA_SPARSE; in vm_bind_ioctl_ops_create()
2271 op->map.vma_flags |= XE_VMA_SYSTEM_ALLOCATOR; in vm_bind_ioctl_ops_create()
2273 op->map.vma_flags |= XE_VMA_DUMPABLE; in vm_bind_ioctl_ops_create()
2275 op->map.vma_flags |= XE_VMA_MADV_AUTORESET; in vm_bind_ioctl_ops_create()
2276 op->map.pat_index = pat_index; in vm_bind_ioctl_ops_create()
2277 op->map.invalidate_on_bind = in vm_bind_ioctl_ops_create()
2278 __xe_vm_needs_clear_scratch_pages(vm, flags); in vm_bind_ioctl_ops_create()
2279 } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { in vm_bind_ioctl_ops_create()
2280 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); in vm_bind_ioctl_ops_create()
2289 op->prefetch.region = prefetch_region; in vm_bind_ioctl_ops_create()
2294 ctx.devmem_possible = IS_DGFX(vm->xe) && in vm_bind_ioctl_ops_create()
2297 for_each_tile(tile, vm->xe, id) in vm_bind_ioctl_ops_create()
2300 xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC); in vm_bind_ioctl_ops_create()
2301 op->prefetch_range.ranges_count = 0; in vm_bind_ioctl_ops_create()
2306 xe_device_get_root_tile(vm->xe)); in vm_bind_ioctl_ops_create()
2312 tile = xe_device_get_root_tile(vm->xe); in vm_bind_ioctl_ops_create()
2314 tile = &vm->xe->tiles[region_to_mem_type[prefetch_region] - in vm_bind_ioctl_ops_create()
2318 op->prefetch_range.tile = tile; in vm_bind_ioctl_ops_create()
2320 svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx); in vm_bind_ioctl_ops_create()
2322 if (PTR_ERR(svm_range) == -ENOENT) { in vm_bind_ioctl_ops_create()
2323 u64 ret = xe_svm_find_vma_start(vm, addr, range_end, vma); in vm_bind_ioctl_ops_create()
2337 if (xe_svm_range_validate(vm, svm_range, tile_mask, !!tile)) { in vm_bind_ioctl_ops_create()
2338 xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID"); in vm_bind_ioctl_ops_create()
2342 err = xa_alloc(&op->prefetch_range.range, in vm_bind_ioctl_ops_create()
2349 op->prefetch_range.ranges_count++; in vm_bind_ioctl_ops_create()
2350 vops->flags |= XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH; in vm_bind_ioctl_ops_create()
2351 xe_svm_range_debug(svm_range, "PREFETCH - RANGE CREATED"); in vm_bind_ioctl_ops_create()
2360 print_op(vm->xe, __op); in vm_bind_ioctl_ops_create()
2367 drm_gpuva_ops_free(&vm->gpuvm, ops); in vm_bind_ioctl_ops_create()
2373 static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, in new_vma() argument
2376 struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; in new_vma()
2382 lockdep_assert_held_write(&vm->lock); in new_vma()
2386 xe_validation_guard(&ctx, &vm->xe->val, &exec, in new_vma()
2388 if (!bo->vm) { in new_vma()
2389 err = drm_exec_lock_obj(&exec, xe_vm_obj(vm)); in new_vma()
2393 err = drm_exec_lock_obj(&exec, &bo->ttm.base); in new_vma()
2399 vma = xe_vma_create(vm, bo, op->gem.offset, in new_vma()
2400 op->va.addr, op->va.addr + in new_vma()
2401 op->va.range - 1, attr, flags); in new_vma()
2405 if (!bo->vm) { in new_vma()
2406 err = add_preempt_fences(vm, bo); in new_vma()
2408 prep_vma_destroy(vm, vma, false); in new_vma()
2416 vma = xe_vma_create(vm, NULL, op->gem.offset, in new_vma()
2417 op->va.addr, op->va.addr + in new_vma()
2418 op->va.range - 1, attr, flags); in new_vma()
2426 prep_vma_destroy(vm, vma, false); in new_vma()
2436 if (vma->gpuva.flags & XE_VMA_PTE_1G) in xe_vma_max_pte_size()
2438 else if (vma->gpuva.flags & (XE_VMA_PTE_2M | XE_VMA_PTE_COMPACT)) in xe_vma_max_pte_size()
2440 else if (vma->gpuva.flags & XE_VMA_PTE_64K) in xe_vma_max_pte_size()
2442 else if (vma->gpuva.flags & XE_VMA_PTE_4K) in xe_vma_max_pte_size()
2452 vma->gpuva.flags |= XE_VMA_PTE_1G; in xe_vma_set_pte_size()
2455 vma->gpuva.flags |= XE_VMA_PTE_2M; in xe_vma_set_pte_size()
2458 vma->gpuva.flags |= XE_VMA_PTE_64K; in xe_vma_set_pte_size()
2461 vma->gpuva.flags |= XE_VMA_PTE_4K; in xe_vma_set_pte_size()
2466 static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) in xe_vma_op_commit() argument
2470 lockdep_assert_held_write(&vm->lock); in xe_vma_op_commit()
2472 switch (op->base.op) { in xe_vma_op_commit()
2474 err |= xe_vm_insert_vma(vm, op->map.vma); in xe_vma_op_commit()
2476 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2481 gpuva_to_vma(op->base.remap.unmap->va)->tile_present; in xe_vma_op_commit()
2483 prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va), in xe_vma_op_commit()
2485 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2487 if (op->remap.prev) { in xe_vma_op_commit()
2488 err |= xe_vm_insert_vma(vm, op->remap.prev); in xe_vma_op_commit()
2490 op->flags |= XE_VMA_OP_PREV_COMMITTED; in xe_vma_op_commit()
2491 if (!err && op->remap.skip_prev) { in xe_vma_op_commit()
2492 op->remap.prev->tile_present = in xe_vma_op_commit()
2494 op->remap.prev = NULL; in xe_vma_op_commit()
2497 if (op->remap.next) { in xe_vma_op_commit()
2498 err |= xe_vm_insert_vma(vm, op->remap.next); in xe_vma_op_commit()
2500 op->flags |= XE_VMA_OP_NEXT_COMMITTED; in xe_vma_op_commit()
2501 if (!err && op->remap.skip_next) { in xe_vma_op_commit()
2502 op->remap.next->tile_present = in xe_vma_op_commit()
2504 op->remap.next = NULL; in xe_vma_op_commit()
2508 /* Adjust for partial unbind after removing VMA from VM */ in xe_vma_op_commit()
2510 op->base.remap.unmap->va->va.addr = op->remap.start; in xe_vma_op_commit()
2511 op->base.remap.unmap->va->va.range = op->remap.range; in xe_vma_op_commit()
2516 prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true); in xe_vma_op_commit()
2517 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2520 op->flags |= XE_VMA_OP_COMMITTED; in xe_vma_op_commit()
2523 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in xe_vma_op_commit()
2530 * xe_vma_has_default_mem_attrs - Check if a VMA has default memory attributes
2537 * - `atomic_access` is `DRM_XE_VMA_ATOMIC_UNDEFINED`
2538 * - `pat_index` is equal to `default_pat_index`
2539 * - `preferred_loc.devmem_fd` is `DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE`
2540 * - `preferred_loc.migration_policy` is `DRM_XE_MIGRATE_ALL_PAGES`
2546 return (vma->attr.atomic_access == DRM_XE_ATOMIC_UNDEFINED && in xe_vma_has_default_mem_attrs()
2547 vma->attr.pat_index == vma->attr.default_pat_index && in xe_vma_has_default_mem_attrs()
2548 vma->attr.preferred_loc.devmem_fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && in xe_vma_has_default_mem_attrs()
2549 vma->attr.preferred_loc.migration_policy == DRM_XE_MIGRATE_ALL_PAGES); in xe_vma_has_default_mem_attrs()
2552 static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, in vm_bind_ioctl_ops_parse() argument
2555 struct xe_device *xe = vm->xe; in vm_bind_ioctl_ops_parse()
2561 lockdep_assert_held_write(&vm->lock); in vm_bind_ioctl_ops_parse()
2563 for_each_tile(tile, vm->xe, id) in vm_bind_ioctl_ops_parse()
2571 INIT_LIST_HEAD(&op->link); in vm_bind_ioctl_ops_parse()
2572 list_add_tail(&op->link, &vops->list); in vm_bind_ioctl_ops_parse()
2573 op->tile_mask = tile_mask; in vm_bind_ioctl_ops_parse()
2575 switch (op->base.op) { in vm_bind_ioctl_ops_parse()
2584 .default_pat_index = op->map.pat_index, in vm_bind_ioctl_ops_parse()
2585 .pat_index = op->map.pat_index, in vm_bind_ioctl_ops_parse()
2588 flags |= op->map.vma_flags & XE_VMA_CREATE_MASK; in vm_bind_ioctl_ops_parse()
2590 vma = new_vma(vm, &op->base.map, &default_attr, in vm_bind_ioctl_ops_parse()
2595 op->map.vma = vma; in vm_bind_ioctl_ops_parse()
2596 if (((op->map.immediate || !xe_vm_in_fault_mode(vm)) && in vm_bind_ioctl_ops_parse()
2597 !(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR)) || in vm_bind_ioctl_ops_parse()
2598 op->map.invalidate_on_bind) in vm_bind_ioctl_ops_parse()
2600 op->tile_mask, 1); in vm_bind_ioctl_ops_parse()
2606 gpuva_to_vma(op->base.remap.unmap->va); in vm_bind_ioctl_ops_parse()
2611 if (op->base.remap.prev) in vm_bind_ioctl_ops_parse()
2612 start = op->base.remap.prev->va.addr + in vm_bind_ioctl_ops_parse()
2613 op->base.remap.prev->va.range; in vm_bind_ioctl_ops_parse()
2614 if (op->base.remap.next) in vm_bind_ioctl_ops_parse()
2615 end = op->base.remap.next->va.addr; in vm_bind_ioctl_ops_parse()
2618 xe_svm_has_mapping(vm, start, end)) { in vm_bind_ioctl_ops_parse()
2619 if (vops->flags & XE_VMA_OPS_FLAG_MADVISE) in vm_bind_ioctl_ops_parse()
2620 xe_svm_unmap_address_range(vm, start, end); in vm_bind_ioctl_ops_parse()
2622 return -EBUSY; in vm_bind_ioctl_ops_parse()
2625 op->remap.start = xe_vma_start(old); in vm_bind_ioctl_ops_parse()
2626 op->remap.range = xe_vma_size(old); in vm_bind_ioctl_ops_parse()
2628 flags |= op->base.remap.unmap->va->flags & XE_VMA_CREATE_MASK; in vm_bind_ioctl_ops_parse()
2629 if (op->base.remap.prev) { in vm_bind_ioctl_ops_parse()
2630 vma = new_vma(vm, op->base.remap.prev, in vm_bind_ioctl_ops_parse()
2631 &old->attr, flags); in vm_bind_ioctl_ops_parse()
2635 op->remap.prev = vma; in vm_bind_ioctl_ops_parse()
2641 op->remap.skip_prev = skip || in vm_bind_ioctl_ops_parse()
2645 if (op->remap.skip_prev) { in vm_bind_ioctl_ops_parse()
2647 op->remap.range -= in vm_bind_ioctl_ops_parse()
2648 xe_vma_end(vma) - in vm_bind_ioctl_ops_parse()
2650 op->remap.start = xe_vma_end(vma); in vm_bind_ioctl_ops_parse()
2651 vm_dbg(&xe->drm, "REMAP:SKIP_PREV: addr=0x%016llx, range=0x%016llx", in vm_bind_ioctl_ops_parse()
2652 (ULL)op->remap.start, in vm_bind_ioctl_ops_parse()
2653 (ULL)op->remap.range); in vm_bind_ioctl_ops_parse()
2659 if (op->base.remap.next) { in vm_bind_ioctl_ops_parse()
2660 vma = new_vma(vm, op->base.remap.next, in vm_bind_ioctl_ops_parse()
2661 &old->attr, flags); in vm_bind_ioctl_ops_parse()
2665 op->remap.next = vma; in vm_bind_ioctl_ops_parse()
2671 op->remap.skip_next = skip || in vm_bind_ioctl_ops_parse()
2675 if (op->remap.skip_next) { in vm_bind_ioctl_ops_parse()
2677 op->remap.range -= in vm_bind_ioctl_ops_parse()
2678 xe_vma_end(old) - in vm_bind_ioctl_ops_parse()
2680 vm_dbg(&xe->drm, "REMAP:SKIP_NEXT: addr=0x%016llx, range=0x%016llx", in vm_bind_ioctl_ops_parse()
2681 (ULL)op->remap.start, in vm_bind_ioctl_ops_parse()
2682 (ULL)op->remap.range); in vm_bind_ioctl_ops_parse()
2690 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, num_remap_ops); in vm_bind_ioctl_ops_parse()
2694 vma = gpuva_to_vma(op->base.unmap.va); in vm_bind_ioctl_ops_parse()
2697 xe_svm_has_mapping(vm, xe_vma_start(vma), in vm_bind_ioctl_ops_parse()
2699 return -EBUSY; in vm_bind_ioctl_ops_parse()
2702 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); in vm_bind_ioctl_ops_parse()
2705 vma = gpuva_to_vma(op->base.prefetch.va); in vm_bind_ioctl_ops_parse()
2714 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, in vm_bind_ioctl_ops_parse()
2715 op->prefetch_range.ranges_count); in vm_bind_ioctl_ops_parse()
2717 xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask, 1); in vm_bind_ioctl_ops_parse()
2721 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in vm_bind_ioctl_ops_parse()
2724 err = xe_vma_op_commit(vm, op); in vm_bind_ioctl_ops_parse()
2732 static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, in xe_vma_op_unwind() argument
2736 lockdep_assert_held_write(&vm->lock); in xe_vma_op_unwind()
2738 switch (op->base.op) { in xe_vma_op_unwind()
2740 if (op->map.vma) { in xe_vma_op_unwind()
2741 prep_vma_destroy(vm, op->map.vma, post_commit); in xe_vma_op_unwind()
2742 xe_vma_destroy_unlocked(op->map.vma); in xe_vma_op_unwind()
2747 struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); in xe_vma_op_unwind()
2750 xe_svm_notifier_lock(vm); in xe_vma_op_unwind()
2751 vma->gpuva.flags &= ~XE_VMA_DESTROYED; in xe_vma_op_unwind()
2752 xe_svm_notifier_unlock(vm); in xe_vma_op_unwind()
2754 xe_vm_insert_vma(vm, vma); in xe_vma_op_unwind()
2760 struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vma_op_unwind()
2762 if (op->remap.prev) { in xe_vma_op_unwind()
2763 prep_vma_destroy(vm, op->remap.prev, prev_post_commit); in xe_vma_op_unwind()
2764 xe_vma_destroy_unlocked(op->remap.prev); in xe_vma_op_unwind()
2766 if (op->remap.next) { in xe_vma_op_unwind()
2767 prep_vma_destroy(vm, op->remap.next, next_post_commit); in xe_vma_op_unwind()
2768 xe_vma_destroy_unlocked(op->remap.next); in xe_vma_op_unwind()
2771 xe_svm_notifier_lock(vm); in xe_vma_op_unwind()
2772 vma->gpuva.flags &= ~XE_VMA_DESTROYED; in xe_vma_op_unwind()
2773 xe_svm_notifier_unlock(vm); in xe_vma_op_unwind()
2775 xe_vm_insert_vma(vm, vma); in xe_vma_op_unwind()
2783 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in xe_vma_op_unwind()
2787 static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, in vm_bind_ioctl_ops_unwind() argument
2793 for (i = num_ops_list - 1; i >= 0; --i) { in vm_bind_ioctl_ops_unwind()
2803 xe_vma_op_unwind(vm, op, in vm_bind_ioctl_ops_unwind()
2804 op->flags & XE_VMA_OP_COMMITTED, in vm_bind_ioctl_ops_unwind()
2805 op->flags & XE_VMA_OP_PREV_COMMITTED, in vm_bind_ioctl_ops_unwind()
2806 op->flags & XE_VMA_OP_NEXT_COMMITTED); in vm_bind_ioctl_ops_unwind()
2815 struct xe_vm *vm = xe_vma_vm(vma); in vma_lock_and_validate() local
2819 if (!bo->vm) in vma_lock_and_validate()
2820 err = drm_exec_lock_obj(exec, &bo->ttm.base); in vma_lock_and_validate()
2822 err = xe_bo_validate(bo, vm, in vma_lock_and_validate()
2823 !xe_vm_in_preempt_fence_mode(vm) && in vma_lock_and_validate()
2832 if (vma->ufence) { in check_ufence()
2833 struct xe_user_fence * const f = vma->ufence; in check_ufence()
2836 return -EBUSY; in check_ufence()
2838 vma->ufence = NULL; in check_ufence()
2845 static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_op *op) in prefetch_ranges() argument
2847 bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); in prefetch_ranges()
2848 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); in prefetch_ranges()
2849 struct xe_tile *tile = op->prefetch_range.tile; in prefetch_ranges()
2862 ctx.device_private_page_owner = xe_svm_devm_owner(vm->xe); in prefetch_ranges()
2865 xa_for_each(&op->prefetch_range.range, i, svm_range) { in prefetch_ranges()
2867 xe_svm_range_migrate_to_smem(vm, svm_range); in prefetch_ranges()
2872 …drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe… in prefetch_ranges()
2873 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); in prefetch_ranges()
2874 return -ENODATA; in prefetch_ranges()
2876 xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM"); in prefetch_ranges()
2879 err = xe_svm_range_get_pages(vm, svm_range, &ctx); in prefetch_ranges()
2881 drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n", in prefetch_ranges()
2882 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); in prefetch_ranges()
2883 if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) in prefetch_ranges()
2884 err = -ENODATA; in prefetch_ranges()
2887 xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE"); in prefetch_ranges()
2893 static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, in op_lock_and_prep() argument
2900 * We only allow evicting a BO within the VM if it is not part of an in op_lock_and_prep()
2904 res_evict = !(vops->flags & XE_VMA_OPS_ARRAY_OF_BINDS); in op_lock_and_prep()
2906 switch (op->base.op) { in op_lock_and_prep()
2908 if (!op->map.invalidate_on_bind) in op_lock_and_prep()
2909 err = vma_lock_and_validate(exec, op->map.vma, in op_lock_and_prep()
2911 !xe_vm_in_fault_mode(vm) || in op_lock_and_prep()
2912 op->map.immediate); in op_lock_and_prep()
2915 err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va)); in op_lock_and_prep()
2920 gpuva_to_vma(op->base.remap.unmap->va), in op_lock_and_prep()
2922 if (!err && op->remap.prev) in op_lock_and_prep()
2923 err = vma_lock_and_validate(exec, op->remap.prev, in op_lock_and_prep()
2925 if (!err && op->remap.next) in op_lock_and_prep()
2926 err = vma_lock_and_validate(exec, op->remap.next, in op_lock_and_prep()
2930 err = check_ufence(gpuva_to_vma(op->base.unmap.va)); in op_lock_and_prep()
2935 gpuva_to_vma(op->base.unmap.va), in op_lock_and_prep()
2940 struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); in op_lock_and_prep()
2944 region = op->prefetch.region; in op_lock_and_prep()
2945 xe_assert(vm->xe, region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC || in op_lock_and_prep()
2950 gpuva_to_vma(op->base.prefetch.va), in op_lock_and_prep()
2960 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_lock_and_prep()
2966 static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops) in vm_bind_ioctl_ops_prefetch_ranges() argument
2971 if (!(vops->flags & XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH)) in vm_bind_ioctl_ops_prefetch_ranges()
2974 list_for_each_entry(op, &vops->list, link) { in vm_bind_ioctl_ops_prefetch_ranges()
2975 if (op->base.op == DRM_GPUVA_OP_PREFETCH) { in vm_bind_ioctl_ops_prefetch_ranges()
2976 err = prefetch_ranges(vm, op); in vm_bind_ioctl_ops_prefetch_ranges()
2986 struct xe_vm *vm, in vm_bind_ioctl_ops_lock_and_prep() argument
2992 err = drm_exec_lock_obj(exec, xe_vm_obj(vm)); in vm_bind_ioctl_ops_lock_and_prep()
2996 list_for_each_entry(op, &vops->list, link) { in vm_bind_ioctl_ops_lock_and_prep()
2997 err = op_lock_and_prep(exec, vm, vops, op); in vm_bind_ioctl_ops_lock_and_prep()
3003 if (vops->inject_error && in vm_bind_ioctl_ops_lock_and_prep()
3004 vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK) in vm_bind_ioctl_ops_lock_and_prep()
3005 return -ENOSPC; in vm_bind_ioctl_ops_lock_and_prep()
3013 switch (op->base.op) { in op_trace()
3015 trace_xe_vma_bind(op->map.vma); in op_trace()
3018 trace_xe_vma_unbind(gpuva_to_vma(op->base.remap.unmap->va)); in op_trace()
3019 if (op->remap.prev) in op_trace()
3020 trace_xe_vma_bind(op->remap.prev); in op_trace()
3021 if (op->remap.next) in op_trace()
3022 trace_xe_vma_bind(op->remap.next); in op_trace()
3025 trace_xe_vma_unbind(gpuva_to_vma(op->base.unmap.va)); in op_trace()
3028 trace_xe_vma_bind(gpuva_to_vma(op->base.prefetch.va)); in op_trace()
3041 list_for_each_entry(op, &vops->list, link) in trace_xe_vm_ops_execute()
3045 static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops) in vm_ops_setup_tile_args() argument
3047 struct xe_exec_queue *q = vops->q; in vm_ops_setup_tile_args()
3052 for_each_tile(tile, vm->xe, id) { in vm_ops_setup_tile_args()
3053 if (vops->pt_update_ops[id].num_ops) in vm_ops_setup_tile_args()
3056 if (vops->pt_update_ops[id].q) in vm_ops_setup_tile_args()
3060 vops->pt_update_ops[id].q = q; in vm_ops_setup_tile_args()
3061 if (vm->pt_root[id] && !list_empty(&q->multi_gt_list)) in vm_ops_setup_tile_args()
3064 vops->pt_update_ops[id].q = vm->q[id]; in vm_ops_setup_tile_args()
3071 static struct dma_fence *ops_execute(struct xe_vm *vm, in ops_execute() argument
3081 number_tiles = vm_ops_setup_tile_args(vm, vops); in ops_execute()
3083 return ERR_PTR(-ENODATA); in ops_execute()
3089 fence = ERR_PTR(-ENOMEM); in ops_execute()
3094 for_each_tile(tile, vm->xe, id) { in ops_execute()
3095 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3107 for_each_tile(tile, vm->xe, id) { in ops_execute()
3108 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3121 vm->composite_fence_ctx, in ops_execute()
3122 vm->composite_fence_seqno++, in ops_execute()
3125 --vm->composite_fence_seqno; in ops_execute()
3126 fence = ERR_PTR(-ENOMEM); in ops_execute()
3129 fence = &cf->base; in ops_execute()
3132 for_each_tile(tile, vm->xe, id) { in ops_execute()
3133 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3142 for_each_tile(tile, vm->xe, id) { in ops_execute()
3143 if (!vops->pt_update_ops[id].num_ops) in ops_execute()
3149 dma_fence_put(fences[--current_fence]); in ops_execute()
3154 trace_xe_vm_ops_fail(vm); in ops_execute()
3160 if (vma->ufence) in vma_add_ufence()
3161 xe_sync_ufence_put(vma->ufence); in vma_add_ufence()
3162 vma->ufence = __xe_sync_ufence_get(ufence); in vma_add_ufence()
3165 static void op_add_ufence(struct xe_vm *vm, struct xe_vma_op *op, in op_add_ufence() argument
3168 switch (op->base.op) { in op_add_ufence()
3170 vma_add_ufence(op->map.vma, ufence); in op_add_ufence()
3173 if (op->remap.prev) in op_add_ufence()
3174 vma_add_ufence(op->remap.prev, ufence); in op_add_ufence()
3175 if (op->remap.next) in op_add_ufence()
3176 vma_add_ufence(op->remap.next, ufence); in op_add_ufence()
3181 vma_add_ufence(gpuva_to_vma(op->base.prefetch.va), ufence); in op_add_ufence()
3184 drm_warn(&vm->xe->drm, "NOT POSSIBLE"); in op_add_ufence()
3188 static void vm_bind_ioctl_ops_fini(struct xe_vm *vm, struct xe_vma_ops *vops, in vm_bind_ioctl_ops_fini() argument
3191 struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, vops->q); in vm_bind_ioctl_ops_fini()
3196 ufence = find_ufence_get(vops->syncs, vops->num_syncs); in vm_bind_ioctl_ops_fini()
3197 list_for_each_entry(op, &vops->list, link) { in vm_bind_ioctl_ops_fini()
3199 op_add_ufence(vm, op, ufence); in vm_bind_ioctl_ops_fini()
3201 if (op->base.op == DRM_GPUVA_OP_UNMAP) in vm_bind_ioctl_ops_fini()
3202 xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence); in vm_bind_ioctl_ops_fini()
3203 else if (op->base.op == DRM_GPUVA_OP_REMAP) in vm_bind_ioctl_ops_fini()
3204 xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), in vm_bind_ioctl_ops_fini()
3210 for (i = 0; i < vops->num_syncs; i++) in vm_bind_ioctl_ops_fini()
3211 xe_sync_entry_signal(vops->syncs + i, fence); in vm_bind_ioctl_ops_fini()
3212 xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); in vm_bind_ioctl_ops_fini()
3216 static struct dma_fence *vm_bind_ioctl_ops_execute(struct xe_vm *vm, in vm_bind_ioctl_ops_execute() argument
3224 lockdep_assert_held_write(&vm->lock); in vm_bind_ioctl_ops_execute()
3226 xe_validation_guard(&ctx, &vm->xe->val, &exec, in vm_bind_ioctl_ops_execute()
3231 err = vm_bind_ioctl_ops_lock_and_prep(&exec, vm, vops); in vm_bind_ioctl_ops_execute()
3237 xe_vm_set_validation_exec(vm, &exec); in vm_bind_ioctl_ops_execute()
3238 fence = ops_execute(vm, vops); in vm_bind_ioctl_ops_execute()
3239 xe_vm_set_validation_exec(vm, NULL); in vm_bind_ioctl_ops_execute()
3241 if (PTR_ERR(fence) == -ENODATA) in vm_bind_ioctl_ops_execute()
3242 vm_bind_ioctl_ops_fini(vm, vops, NULL); in vm_bind_ioctl_ops_execute()
3246 vm_bind_ioctl_ops_fini(vm, vops, fence); in vm_bind_ioctl_ops_execute()
3271 static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, in vm_bind_ioctl_check_args() argument
3278 if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || in vm_bind_ioctl_check_args()
3279 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in vm_bind_ioctl_check_args()
3280 return -EINVAL; in vm_bind_ioctl_check_args()
3282 if (XE_IOCTL_DBG(xe, args->extensions)) in vm_bind_ioctl_check_args()
3283 return -EINVAL; in vm_bind_ioctl_check_args()
3285 if (args->num_binds > 1) { in vm_bind_ioctl_check_args()
3287 u64_to_user_ptr(args->vector_of_binds); in vm_bind_ioctl_check_args()
3289 *bind_ops = kvmalloc_array(args->num_binds, in vm_bind_ioctl_check_args()
3294 return args->num_binds > 1 ? -ENOBUFS : -ENOMEM; in vm_bind_ioctl_check_args()
3298 args->num_binds); in vm_bind_ioctl_check_args()
3300 err = -EFAULT; in vm_bind_ioctl_check_args()
3304 *bind_ops = &args->bind; in vm_bind_ioctl_check_args()
3307 for (i = 0; i < args->num_binds; ++i) { in vm_bind_ioctl_check_args()
3322 (!xe_vm_in_fault_mode(vm) || in vm_bind_ioctl_check_args()
3324 err = -EINVAL; in vm_bind_ioctl_check_args()
3328 if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { in vm_bind_ioctl_check_args()
3329 err = -EINVAL; in vm_bind_ioctl_check_args()
3333 pat_index = array_index_nospec(pat_index, xe->pat.n_entries); in vm_bind_ioctl_check_args()
3337 err = -EINVAL; in vm_bind_ioctl_check_args()
3342 err = -EINVAL; in vm_bind_ioctl_check_args()
3373 !(BIT(prefetch_region) & xe->info.mem_region_mask))) || in vm_bind_ioctl_check_args()
3378 err = -EINVAL; in vm_bind_ioctl_check_args()
3387 err = -EINVAL; in vm_bind_ioctl_check_args()
3395 if (args->num_binds > 1) in vm_bind_ioctl_check_args()
3401 static int vm_bind_ioctl_signal_fences(struct xe_vm *vm, in vm_bind_ioctl_signal_fences() argument
3410 to_wait_exec_queue(vm, q), vm); in vm_bind_ioctl_signal_fences()
3417 xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm, in vm_bind_ioctl_signal_fences()
3424 static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm, in xe_vma_ops_init() argument
3429 INIT_LIST_HEAD(&vops->list); in xe_vma_ops_init()
3430 vops->vm = vm; in xe_vma_ops_init()
3431 vops->q = q; in xe_vma_ops_init()
3432 vops->syncs = syncs; in xe_vma_ops_init()
3433 vops->num_syncs = num_syncs; in xe_vma_ops_init()
3434 vops->flags = 0; in xe_vma_ops_init()
3445 xe_bo_size(bo) - range)) { in xe_vm_bind_ioctl_validate_bo()
3446 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3457 if ((bo->flags & XE_BO_FLAG_INTERNAL_64K) && in xe_vm_bind_ioctl_validate_bo()
3458 (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)) { in xe_vm_bind_ioctl_validate_bo()
3463 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3468 if (bo->cpu_caching) { in xe_vm_bind_ioctl_validate_bo()
3470 bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) { in xe_vm_bind_ioctl_validate_bo()
3471 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3475 * Imported dma-buf from a different device should in xe_vm_bind_ioctl_validate_bo()
3480 return -EINVAL; in xe_vm_bind_ioctl_validate_bo()
3486 if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0)) in xe_vm_bind_ioctl_validate_bo()
3487 return -ENOEXEC; in xe_vm_bind_ioctl_validate_bo()
3500 struct xe_vm *vm; in xe_vm_bind_ioctl() local
3510 vm = xe_vm_lookup(xef, args->vm_id); in xe_vm_bind_ioctl()
3511 if (XE_IOCTL_DBG(xe, !vm)) in xe_vm_bind_ioctl()
3512 return -EINVAL; in xe_vm_bind_ioctl()
3514 err = vm_bind_ioctl_check_args(xe, vm, args, &bind_ops); in xe_vm_bind_ioctl()
3518 if (args->exec_queue_id) { in xe_vm_bind_ioctl()
3519 q = xe_exec_queue_lookup(xef, args->exec_queue_id); in xe_vm_bind_ioctl()
3521 err = -ENOENT; in xe_vm_bind_ioctl()
3525 if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { in xe_vm_bind_ioctl()
3526 err = -EINVAL; in xe_vm_bind_ioctl()
3532 xe_svm_flush(vm); in xe_vm_bind_ioctl()
3534 err = down_write_killable(&vm->lock); in xe_vm_bind_ioctl()
3538 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { in xe_vm_bind_ioctl()
3539 err = -ENOENT; in xe_vm_bind_ioctl()
3543 for (i = 0; i < args->num_binds; ++i) { in xe_vm_bind_ioctl()
3547 if (XE_IOCTL_DBG(xe, range > vm->size) || in xe_vm_bind_ioctl()
3548 XE_IOCTL_DBG(xe, addr > vm->size - range)) { in xe_vm_bind_ioctl()
3549 err = -EINVAL; in xe_vm_bind_ioctl()
3554 if (args->num_binds) { in xe_vm_bind_ioctl()
3555 bos = kvcalloc(args->num_binds, sizeof(*bos), in xe_vm_bind_ioctl()
3559 err = -ENOMEM; in xe_vm_bind_ioctl()
3563 ops = kvcalloc(args->num_binds, sizeof(*ops), in xe_vm_bind_ioctl()
3567 err = -ENOMEM; in xe_vm_bind_ioctl()
3572 for (i = 0; i < args->num_binds; ++i) { in xe_vm_bind_ioctl()
3587 err = -ENOENT; in xe_vm_bind_ioctl()
3599 if (args->num_syncs) { in xe_vm_bind_ioctl()
3600 syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); in xe_vm_bind_ioctl()
3602 err = -ENOMEM; in xe_vm_bind_ioctl()
3607 syncs_user = u64_to_user_ptr(args->syncs); in xe_vm_bind_ioctl()
3608 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { in xe_vm_bind_ioctl()
3611 (xe_vm_in_lr_mode(vm) ? in xe_vm_bind_ioctl()
3613 (!args->num_binds ? in xe_vm_bind_ioctl()
3623 err = -EINVAL; in xe_vm_bind_ioctl()
3627 if (!args->num_binds) { in xe_vm_bind_ioctl()
3628 err = -ENODATA; in xe_vm_bind_ioctl()
3632 xe_vma_ops_init(&vops, vm, q, syncs, num_syncs); in xe_vm_bind_ioctl()
3633 if (args->num_binds > 1) in xe_vm_bind_ioctl()
3635 for (i = 0; i < args->num_binds; ++i) { in xe_vm_bind_ioctl()
3644 ops[i] = vm_bind_ioctl_ops_create(vm, &vops, bos[i], obj_offset, in xe_vm_bind_ioctl()
3653 err = vm_bind_ioctl_ops_parse(vm, ops[i], &vops); in xe_vm_bind_ioctl()
3660 vm->xe->vm_inject_error_position = in xe_vm_bind_ioctl()
3661 (vm->xe->vm_inject_error_position + 1) % in xe_vm_bind_ioctl()
3669 err = -ENODATA; in xe_vm_bind_ioctl()
3673 err = xe_vma_ops_alloc(&vops, args->num_binds > 1); in xe_vm_bind_ioctl()
3677 err = vm_bind_ioctl_ops_prefetch_ranges(vm, &vops); in xe_vm_bind_ioctl()
3681 fence = vm_bind_ioctl_ops_execute(vm, &vops); in xe_vm_bind_ioctl()
3688 if (err && err != -ENODATA) in xe_vm_bind_ioctl()
3689 vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds); in xe_vm_bind_ioctl()
3691 for (i = args->num_binds - 1; i >= 0; --i) in xe_vm_bind_ioctl()
3693 drm_gpuva_ops_free(&vm->gpuvm, ops[i]); in xe_vm_bind_ioctl()
3695 if (err == -ENODATA) in xe_vm_bind_ioctl()
3696 err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs); in xe_vm_bind_ioctl()
3697 while (num_syncs--) in xe_vm_bind_ioctl()
3702 for (i = 0; i < args->num_binds; ++i) in xe_vm_bind_ioctl()
3709 up_write(&vm->lock); in xe_vm_bind_ioctl()
3714 if (args->num_binds > 1) in xe_vm_bind_ioctl()
3717 xe_vm_put(vm); in xe_vm_bind_ioctl()
3722 * xe_vm_bind_kernel_bo - bind a kernel BO to a VM
3723 * @vm: VM to bind the BO to
3729 * Execute a VM bind map operation on a kernel-owned BO to bind it into a
3730 * kernel-owned VM.
3735 struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo, in xe_vm_bind_kernel_bo() argument
3745 xe_vm_get(vm); in xe_vm_bind_kernel_bo()
3749 down_write(&vm->lock); in xe_vm_bind_kernel_bo()
3751 xe_vma_ops_init(&vops, vm, q, NULL, 0); in xe_vm_bind_kernel_bo()
3753 ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo), in xe_vm_bind_kernel_bo()
3755 vm->xe->pat.idx[cache_lvl]); in xe_vm_bind_kernel_bo()
3761 err = vm_bind_ioctl_ops_parse(vm, ops, &vops); in xe_vm_bind_kernel_bo()
3765 xe_assert(vm->xe, !list_empty(&vops.list)); in xe_vm_bind_kernel_bo()
3771 fence = vm_bind_ioctl_ops_execute(vm, &vops); in xe_vm_bind_kernel_bo()
3776 if (err && err != -ENODATA) in xe_vm_bind_kernel_bo()
3777 vm_bind_ioctl_ops_unwind(vm, &ops, 1); in xe_vm_bind_kernel_bo()
3780 drm_gpuva_ops_free(&vm->gpuvm, ops); in xe_vm_bind_kernel_bo()
3783 up_write(&vm->lock); in xe_vm_bind_kernel_bo()
3787 xe_vm_put(vm); in xe_vm_bind_kernel_bo()
3797 * xe_vm_lock() - Lock the vm's dma_resv object
3798 * @vm: The struct xe_vm whose lock is to be locked
3801 * Return: 0 on success, -EINTR if @intr is true and the wait for a
3805 int xe_vm_lock(struct xe_vm *vm, bool intr) in xe_vm_lock() argument
3810 ret = dma_resv_lock_interruptible(xe_vm_resv(vm), NULL); in xe_vm_lock()
3812 ret = dma_resv_lock(xe_vm_resv(vm), NULL); in xe_vm_lock()
3818 * xe_vm_unlock() - Unlock the vm's dma_resv object
3819 * @vm: The struct xe_vm whose lock is to be released.
3823 void xe_vm_unlock(struct xe_vm *vm) in xe_vm_unlock() argument
3825 dma_resv_unlock(xe_vm_resv(vm)); in xe_vm_unlock()
3829 * xe_vm_range_tilemask_tlb_inval - Issue a TLB invalidation on this tilemask for an
3831 * @vm: The VM
3840 int xe_vm_range_tilemask_tlb_inval(struct xe_vm *vm, u64 start, in xe_vm_range_tilemask_tlb_inval() argument
3853 for_each_tile(tile, vm->xe, id) { in xe_vm_range_tilemask_tlb_inval()
3857 xe_tlb_inval_fence_init(&tile->primary_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3860 err = xe_tlb_inval_range(&tile->primary_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3862 vm->usm.asid); in xe_vm_range_tilemask_tlb_inval()
3867 if (!tile->media_gt) in xe_vm_range_tilemask_tlb_inval()
3870 xe_tlb_inval_fence_init(&tile->media_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3873 err = xe_tlb_inval_range(&tile->media_gt->tlb_inval, in xe_vm_range_tilemask_tlb_inval()
3875 vm->usm.asid); in xe_vm_range_tilemask_tlb_inval()
3889 * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
3900 struct xe_device *xe = xe_vma_vm(vma)->xe; in xe_vm_invalidate_vma()
3901 struct xe_vm *vm = xe_vma_vm(vma); in xe_vm_invalidate_vma() local
3911 vm_dbg(&vm->xe->drm, in xe_vm_invalidate_vma()
3916 * Check that we don't race with page-table updates, tile_invalidated in xe_vm_invalidate_vma()
3921 lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) || in xe_vm_invalidate_vma()
3922 (lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && in xe_vm_invalidate_vma()
3923 lockdep_is_held(&xe_vm_resv(vm)->lock.base))); in xe_vm_invalidate_vma()
3926 (&to_userptr_vma(vma)->userptr.notifier, in xe_vm_invalidate_vma()
3927 to_userptr_vma(vma)->userptr.pages.notifier_seq)); in xe_vm_invalidate_vma()
3928 WARN_ON_ONCE(!dma_resv_test_signaled(xe_vm_resv(vm), in xe_vm_invalidate_vma()
3946 WRITE_ONCE(vma->tile_invalidated, vma->tile_mask); in xe_vm_invalidate_vma()
3951 int xe_vm_validate_protected(struct xe_vm *vm) in xe_vm_validate_protected() argument
3956 if (!vm) in xe_vm_validate_protected()
3957 return -ENODEV; in xe_vm_validate_protected()
3959 mutex_lock(&vm->snap_mutex); in xe_vm_validate_protected()
3961 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { in xe_vm_validate_protected()
3963 struct xe_bo *bo = vma->gpuva.gem.obj ? in xe_vm_validate_protected()
3964 gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; in xe_vm_validate_protected()
3970 err = xe_pxp_bo_key_check(vm->xe->pxp, bo); in xe_vm_validate_protected()
3976 mutex_unlock(&vm->snap_mutex); in xe_vm_validate_protected()
3991 struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm) in xe_vm_snapshot_capture() argument
3997 if (!vm) in xe_vm_snapshot_capture()
4000 mutex_lock(&vm->snap_mutex); in xe_vm_snapshot_capture()
4001 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { in xe_vm_snapshot_capture()
4002 if (gpuva->flags & XE_VMA_DUMPABLE) in xe_vm_snapshot_capture()
4009 snap = num_snaps ? ERR_PTR(-ENOMEM) : ERR_PTR(-ENODEV); in xe_vm_snapshot_capture()
4013 snap->num_snaps = num_snaps; in xe_vm_snapshot_capture()
4015 drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { in xe_vm_snapshot_capture()
4017 struct xe_bo *bo = vma->gpuva.gem.obj ? in xe_vm_snapshot_capture()
4018 gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; in xe_vm_snapshot_capture()
4020 if (!(gpuva->flags & XE_VMA_DUMPABLE)) in xe_vm_snapshot_capture()
4023 snap->snap[i].ofs = xe_vma_start(vma); in xe_vm_snapshot_capture()
4024 snap->snap[i].len = xe_vma_size(vma); in xe_vm_snapshot_capture()
4026 snap->snap[i].bo = xe_bo_get(bo); in xe_vm_snapshot_capture()
4027 snap->snap[i].bo_ofs = xe_vma_bo_offset(vma); in xe_vm_snapshot_capture()
4030 to_userptr_vma(vma)->userptr.notifier.mm; in xe_vm_snapshot_capture()
4033 snap->snap[i].mm = mm; in xe_vm_snapshot_capture()
4035 snap->snap[i].data = ERR_PTR(-EFAULT); in xe_vm_snapshot_capture()
4037 snap->snap[i].bo_ofs = xe_vma_userptr(vma); in xe_vm_snapshot_capture()
4039 snap->snap[i].data = ERR_PTR(-ENOENT); in xe_vm_snapshot_capture()
4045 mutex_unlock(&vm->snap_mutex); in xe_vm_snapshot_capture()
4054 for (int i = 0; i < snap->num_snaps; i++) { in xe_vm_snapshot_capture_delayed()
4055 struct xe_bo *bo = snap->snap[i].bo; in xe_vm_snapshot_capture_delayed()
4058 if (IS_ERR(snap->snap[i].data)) in xe_vm_snapshot_capture_delayed()
4061 snap->snap[i].data = kvmalloc(snap->snap[i].len, GFP_USER); in xe_vm_snapshot_capture_delayed()
4062 if (!snap->snap[i].data) { in xe_vm_snapshot_capture_delayed()
4063 snap->snap[i].data = ERR_PTR(-ENOMEM); in xe_vm_snapshot_capture_delayed()
4068 err = xe_bo_read(bo, snap->snap[i].bo_ofs, in xe_vm_snapshot_capture_delayed()
4069 snap->snap[i].data, snap->snap[i].len); in xe_vm_snapshot_capture_delayed()
4071 void __user *userptr = (void __user *)(size_t)snap->snap[i].bo_ofs; in xe_vm_snapshot_capture_delayed()
4073 kthread_use_mm(snap->snap[i].mm); in xe_vm_snapshot_capture_delayed()
4074 if (!copy_from_user(snap->snap[i].data, userptr, snap->snap[i].len)) in xe_vm_snapshot_capture_delayed()
4077 err = -EFAULT; in xe_vm_snapshot_capture_delayed()
4078 kthread_unuse_mm(snap->snap[i].mm); in xe_vm_snapshot_capture_delayed()
4080 mmput(snap->snap[i].mm); in xe_vm_snapshot_capture_delayed()
4081 snap->snap[i].mm = NULL; in xe_vm_snapshot_capture_delayed()
4085 kvfree(snap->snap[i].data); in xe_vm_snapshot_capture_delayed()
4086 snap->snap[i].data = ERR_PTR(err); in xe_vm_snapshot_capture_delayed()
4091 snap->snap[i].bo = NULL; in xe_vm_snapshot_capture_delayed()
4104 for (i = 0; i < snap->num_snaps; i++) { in xe_vm_snapshot_print()
4105 drm_printf(p, "[%llx].length: 0x%lx\n", snap->snap[i].ofs, snap->snap[i].len); in xe_vm_snapshot_print()
4107 if (IS_ERR(snap->snap[i].data)) { in xe_vm_snapshot_print()
4108 drm_printf(p, "[%llx].error: %li\n", snap->snap[i].ofs, in xe_vm_snapshot_print()
4109 PTR_ERR(snap->snap[i].data)); in xe_vm_snapshot_print()
4113 drm_printf(p, "[%llx].data: ", snap->snap[i].ofs); in xe_vm_snapshot_print()
4115 for (j = 0; j < snap->snap[i].len; j += sizeof(u32)) { in xe_vm_snapshot_print()
4116 u32 *val = snap->snap[i].data + j; in xe_vm_snapshot_print()
4136 for (i = 0; i < snap->num_snaps; i++) { in xe_vm_snapshot_free()
4137 if (!IS_ERR(snap->snap[i].data)) in xe_vm_snapshot_free()
4138 kvfree(snap->snap[i].data); in xe_vm_snapshot_free()
4139 xe_bo_put(snap->snap[i].bo); in xe_vm_snapshot_free()
4140 if (snap->snap[i].mm) in xe_vm_snapshot_free()
4141 mmput(snap->snap[i].mm); in xe_vm_snapshot_free()
4147 * xe_vma_need_vram_for_atomic - Check if VMA needs VRAM migration for atomic operations
4156 * 1 - Migration to VRAM is required
4157 * 0 - Migration is not required
4158 * -EACCES - Invalid access for atomic memory attr
4163 u32 atomic_access = xe_vma_bo(vma) ? xe_vma_bo(vma)->attr.atomic_access : in xe_vma_need_vram_for_atomic()
4164 vma->attr.atomic_access; in xe_vma_need_vram_for_atomic()
4170 * NOTE: The checks implemented here are platform-specific. For in xe_vma_need_vram_for_atomic()
4176 return !xe->info.has_device_atomics_on_smem; in xe_vma_need_vram_for_atomic()
4179 return -EACCES; in xe_vma_need_vram_for_atomic()
4188 static int xe_vm_alloc_vma(struct xe_vm *vm, in xe_vm_alloc_vma() argument
4201 lockdep_assert_held_write(&vm->lock); in xe_vm_alloc_vma()
4204 ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req); in xe_vm_alloc_vma()
4206 ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, map_req); in xe_vm_alloc_vma()
4211 if (list_empty(&ops->list)) { in xe_vm_alloc_vma()
4221 if (__op->op == DRM_GPUVA_OP_UNMAP) { in xe_vm_alloc_vma()
4222 vma = gpuva_to_vma(op->base.unmap.va); in xe_vm_alloc_vma()
4224 default_pat = vma->attr.default_pat_index; in xe_vm_alloc_vma()
4225 vma_flags = vma->gpuva.flags; in xe_vm_alloc_vma()
4228 if (__op->op == DRM_GPUVA_OP_REMAP) { in xe_vm_alloc_vma()
4229 vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vm_alloc_vma()
4230 default_pat = vma->attr.default_pat_index; in xe_vm_alloc_vma()
4231 vma_flags = vma->gpuva.flags; in xe_vm_alloc_vma()
4234 if (__op->op == DRM_GPUVA_OP_MAP) { in xe_vm_alloc_vma()
4235 op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; in xe_vm_alloc_vma()
4236 op->map.pat_index = default_pat; in xe_vm_alloc_vma()
4239 if (__op->op == DRM_GPUVA_OP_REMAP) { in xe_vm_alloc_vma()
4240 vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vm_alloc_vma()
4241 xe_assert(vm->xe, !remap_op); in xe_vm_alloc_vma()
4242 xe_assert(vm->xe, xe_vma_has_no_bo(vma)); in xe_vm_alloc_vma()
4244 vma_flags = vma->gpuva.flags; in xe_vm_alloc_vma()
4247 if (__op->op == DRM_GPUVA_OP_MAP) { in xe_vm_alloc_vma()
4248 xe_assert(vm->xe, remap_op); in xe_vm_alloc_vma()
4256 op->map.vma_flags |= vma_flags & XE_VMA_CREATE_MASK; in xe_vm_alloc_vma()
4259 print_op(vm->xe, __op); in xe_vm_alloc_vma()
4262 xe_vma_ops_init(&vops, vm, NULL, NULL, 0); in xe_vm_alloc_vma()
4267 err = vm_bind_ioctl_ops_parse(vm, ops, &vops); in xe_vm_alloc_vma()
4271 xe_vm_lock(vm, false); in xe_vm_alloc_vma()
4277 if (__op->op == DRM_GPUVA_OP_UNMAP) { in xe_vm_alloc_vma()
4278 vma = gpuva_to_vma(op->base.unmap.va); in xe_vm_alloc_vma()
4284 } else if (__op->op == DRM_GPUVA_OP_REMAP) { in xe_vm_alloc_vma()
4285 vma = gpuva_to_vma(op->base.remap.unmap->va); in xe_vm_alloc_vma()
4287 * VMA, so they can be assigned to newly MAP created vma. in xe_vm_alloc_vma()
4290 tmp_attr = vma->attr; in xe_vm_alloc_vma()
4292 xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va), NULL); in xe_vm_alloc_vma()
4293 } else if (__op->op == DRM_GPUVA_OP_MAP) { in xe_vm_alloc_vma()
4294 vma = op->map.vma; in xe_vm_alloc_vma()
4295 /* In case of madvise call, MAP will always be follwed by REMAP. in xe_vm_alloc_vma()
4300 vma->attr = tmp_attr; in xe_vm_alloc_vma()
4304 xe_vm_unlock(vm); in xe_vm_alloc_vma()
4305 drm_gpuva_ops_free(&vm->gpuvm, ops); in xe_vm_alloc_vma()
4309 vm_bind_ioctl_ops_unwind(vm, &ops, 1); in xe_vm_alloc_vma()
4311 drm_gpuva_ops_free(&vm->gpuvm, ops); in xe_vm_alloc_vma()
4316 * xe_vm_alloc_madvise_vma - Allocate VMA's with madvise ops
4317 * @vm: Pointer to the xe_vm structure
4325 int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range) in xe_vm_alloc_madvise_vma() argument
4328 .map.va.addr = start, in xe_vm_alloc_madvise_vma()
4329 .map.va.range = range, in xe_vm_alloc_madvise_vma()
4332 lockdep_assert_held_write(&vm->lock); in xe_vm_alloc_madvise_vma()
4334 vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range); in xe_vm_alloc_madvise_vma()
4336 return xe_vm_alloc_vma(vm, &map_req, true); in xe_vm_alloc_madvise_vma()
4340 * xe_vm_alloc_cpu_addr_mirror_vma - Allocate CPU addr mirror vma
4341 * @vm: Pointer to the xe_vm structure
4349 int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t range) in xe_vm_alloc_cpu_addr_mirror_vma() argument
4352 .map.va.addr = start, in xe_vm_alloc_cpu_addr_mirror_vma()
4353 .map.va.range = range, in xe_vm_alloc_cpu_addr_mirror_vma()
4356 lockdep_assert_held_write(&vm->lock); in xe_vm_alloc_cpu_addr_mirror_vma()
4358 vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx", in xe_vm_alloc_cpu_addr_mirror_vma()
4361 return xe_vm_alloc_vma(vm, &map_req, false); in xe_vm_alloc_cpu_addr_mirror_vma()