Lines Matching full:vm
49 * be guaranteed by a vma reference under the vm lock should keep a reference.
53 static int xe_svm_get_pagemaps(struct xe_vm *vm);
55 void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem) in xe_svm_private_page_owner() argument
57 return force_smem ? NULL : vm->svm.peer.owner; in xe_svm_private_page_owner()
130 xe_svm_garbage_collector_add_range(struct xe_vm *vm, struct xe_svm_range *range, in xe_svm_garbage_collector_add_range() argument
133 struct xe_device *xe = vm->xe; in xe_svm_garbage_collector_add_range()
139 spin_lock(&vm->svm.garbage_collector.lock); in xe_svm_garbage_collector_add_range()
142 &vm->svm.garbage_collector.range_list); in xe_svm_garbage_collector_add_range()
143 spin_unlock(&vm->svm.garbage_collector.lock); in xe_svm_garbage_collector_add_range()
145 queue_work(xe->usm.pf_wq, &vm->svm.garbage_collector.work); in xe_svm_garbage_collector_add_range()
154 xe_svm_range_notifier_event_begin(struct xe_vm *vm, struct drm_gpusvm_range *r, in xe_svm_range_notifier_event_begin() argument
159 struct xe_device *xe = vm->xe; in xe_svm_range_notifier_event_begin()
164 xe_svm_assert_in_notifier(vm); in xe_svm_range_notifier_event_begin()
184 if (xe_pt_zap_ptes_range(tile, vm, range)) { in xe_svm_range_notifier_event_begin()
204 xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r, in xe_svm_range_notifier_event_end() argument
209 xe_svm_assert_in_notifier(vm); in xe_svm_range_notifier_event_end()
211 drm_gpusvm_range_unmap_pages(&vm->svm.gpusvm, r, &ctx); in xe_svm_range_notifier_event_end()
212 if (!xe_vm_is_closed(vm) && mmu_range->event == MMU_NOTIFY_UNMAP) in xe_svm_range_notifier_event_end()
213 xe_svm_garbage_collector_add_range(vm, to_xe_range(r), in xe_svm_range_notifier_event_end()
228 struct xe_vm *vm = gpusvm_to_vm(gpusvm); in xe_svm_invalidate() local
230 struct xe_device *xe = vm->xe; in xe_svm_invalidate()
238 xe_svm_assert_in_notifier(vm); in xe_svm_invalidate()
242 vm->usm.asid, gpusvm, notifier->notifier.invalidate_seq, in xe_svm_invalidate()
258 if (xe_vm_is_closed(vm)) in xe_svm_invalidate()
262 * XXX: Less than ideal to always wait on VM's resv slots if an in xe_svm_invalidate()
266 err = dma_resv_wait_timeout(xe_vm_resv(vm), in xe_svm_invalidate()
273 tile_mask |= xe_svm_range_notifier_event_begin(vm, r, mmu_range, in xe_svm_invalidate()
281 err = xe_tlb_inval_range_tilemask_submit(xe, vm->usm.asid, adj_start, adj_end, in xe_svm_invalidate()
289 xe_svm_range_notifier_event_end(vm, r, mmu_range); in xe_svm_invalidate()
299 static int __xe_svm_garbage_collector(struct xe_vm *vm, in __xe_svm_garbage_collector() argument
306 xe_vm_lock(vm, false); in __xe_svm_garbage_collector()
307 fence = xe_vm_range_unbind(vm, range); in __xe_svm_garbage_collector()
308 xe_vm_unlock(vm); in __xe_svm_garbage_collector()
313 drm_gpusvm_range_remove(&vm->svm.gpusvm, &range->base); in __xe_svm_garbage_collector()
331 static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end) in xe_svm_range_set_default_attr() argument
337 vma = xe_vm_find_vma_by_addr(vm, start); in xe_svm_range_set_default_attr()
342 drm_dbg(&vm->xe->drm, "Skipping madvise reset for vma.\n"); in xe_svm_range_set_default_attr()
346 vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx", in xe_svm_range_set_default_attr()
358 xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end); in xe_svm_range_set_default_attr()
363 vm_dbg(&vm->xe->drm, "New VMA start=0x%016llx, vma_end=0x%016llx", start, end); in xe_svm_range_set_default_attr()
365 err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start); in xe_svm_range_set_default_attr()
367 drm_warn(&vm->xe->drm, "New VMA MAP failed: %pe\n", ERR_PTR(err)); in xe_svm_range_set_default_attr()
368 xe_vm_kill(vm, true); in xe_svm_range_set_default_attr()
379 static int xe_svm_garbage_collector(struct xe_vm *vm) in xe_svm_garbage_collector() argument
386 lockdep_assert_held_write(&vm->lock); in xe_svm_garbage_collector()
388 if (xe_vm_is_closed_or_banned(vm)) in xe_svm_garbage_collector()
392 spin_lock(&vm->svm.garbage_collector.lock); in xe_svm_garbage_collector()
393 range = list_first_entry_or_null(&vm->svm.garbage_collector.range_list, in xe_svm_garbage_collector()
403 spin_unlock(&vm->svm.garbage_collector.lock); in xe_svm_garbage_collector()
405 err = __xe_svm_garbage_collector(vm, range); in xe_svm_garbage_collector()
407 drm_warn(&vm->xe->drm, in xe_svm_garbage_collector()
410 xe_vm_kill(vm, true); in xe_svm_garbage_collector()
414 err = xe_svm_range_set_default_attr(vm, range_start, range_end); in xe_svm_garbage_collector()
422 spin_unlock(&vm->svm.garbage_collector.lock); in xe_svm_garbage_collector()
429 struct xe_vm *vm = container_of(w, struct xe_vm, in xe_svm_garbage_collector_work_func() local
432 down_write(&vm->lock); in xe_svm_garbage_collector_work_func()
433 xe_svm_garbage_collector(vm); in xe_svm_garbage_collector_work_func()
434 up_write(&vm->lock); in xe_svm_garbage_collector_work_func()
814 static int xe_svm_get_pagemaps(struct xe_vm *vm) in xe_svm_get_pagemaps() argument
837 static void xe_svm_put_pagemaps(struct xe_vm *vm) in xe_svm_put_pagemaps() argument
839 struct xe_device *xe = vm->xe; in xe_svm_put_pagemaps()
844 struct xe_pagemap *xpagemap = vm->svm.pagemaps[id]; in xe_svm_put_pagemaps()
848 vm->svm.pagemaps[id] = NULL; in xe_svm_put_pagemaps()
876 * @vm: The VM.
878 * Initialize SVM state which is embedded within the VM.
882 int xe_svm_init(struct xe_vm *vm) in xe_svm_init() argument
886 if (vm->flags & XE_VM_FLAG_FAULT_MODE) { in xe_svm_init()
887 spin_lock_init(&vm->svm.garbage_collector.lock); in xe_svm_init()
888 INIT_LIST_HEAD(&vm->svm.garbage_collector.range_list); in xe_svm_init()
889 INIT_WORK(&vm->svm.garbage_collector.work, in xe_svm_init()
892 vm->svm.peer.private = XE_PEER_VM; in xe_svm_init()
893 err = drm_pagemap_acquire_owner(&vm->svm.peer, &xe_owner_list, in xe_svm_init()
898 err = xe_svm_get_pagemaps(vm); in xe_svm_init()
900 drm_pagemap_release_owner(&vm->svm.peer); in xe_svm_init()
904 err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM", &vm->xe->drm, in xe_svm_init()
905 current->mm, 0, vm->size, in xe_svm_init()
909 drm_gpusvm_driver_set_lock(&vm->svm.gpusvm, &vm->lock); in xe_svm_init()
912 xe_svm_put_pagemaps(vm); in xe_svm_init()
913 drm_pagemap_release_owner(&vm->svm.peer); in xe_svm_init()
917 err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", in xe_svm_init()
918 &vm->xe->drm, NULL, 0, 0, 0, NULL, in xe_svm_init()
927 * @vm: The VM.
931 void xe_svm_close(struct xe_vm *vm) in xe_svm_close() argument
933 xe_assert(vm->xe, xe_vm_is_closed(vm)); in xe_svm_close()
934 disable_work_sync(&vm->svm.garbage_collector.work); in xe_svm_close()
935 xe_svm_put_pagemaps(vm); in xe_svm_close()
936 drm_pagemap_release_owner(&vm->svm.peer); in xe_svm_close()
941 * @vm: The VM.
943 * Finalize SVM state which is embedded within the VM.
945 void xe_svm_fini(struct xe_vm *vm) in xe_svm_fini() argument
947 xe_assert(vm->xe, xe_vm_is_closed(vm)); in xe_svm_fini()
949 drm_gpusvm_fini(&vm->svm.gpusvm); in xe_svm_fini()
961 struct xe_vm *vm = range_to_vm(&range->base); in xe_svm_range_has_pagemap() local
964 xe_svm_notifier_lock(vm); in xe_svm_range_has_pagemap()
966 xe_svm_notifier_unlock(vm); in xe_svm_range_has_pagemap()
983 * @vm: xe_vm pointer
989 void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range) in xe_svm_range_migrate_to_smem() argument
992 drm_gpusvm_range_evict(&vm->svm.gpusvm, &range->base); in xe_svm_range_migrate_to_smem()
997 * @vm: xe_vm pointer
1008 bool xe_svm_range_validate(struct xe_vm *vm, in xe_svm_range_validate() argument
1014 xe_svm_notifier_lock(vm); in xe_svm_range_validate()
1022 xe_svm_notifier_unlock(vm); in xe_svm_range_validate()
1029 * @vm: xe_vm pointer
1036 * range [start, end] in the given VM. It adjusts the range based on the
1042 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 start, u64 end, struct xe_vma *vma) in xe_svm_find_vma_start() argument
1044 return drm_gpusvm_find_vma_start(&vm->svm.gpusvm, in xe_svm_find_vma_start()
1143 struct xe_vm *vm = range_to_vm(&range->base); in xe_svm_range_needs_migrate_to_vram() local
1149 xe_assert(vm->xe, IS_DGFX(vm->xe)); in xe_svm_range_needs_migrate_to_vram()
1152 drm_dbg(&vm->xe->drm, "Range is already in VRAM\n"); in xe_svm_range_needs_migrate_to_vram()
1156 if (range_size < SZ_64K && !supports_4K_migration(vm->xe)) { in xe_svm_range_needs_migrate_to_vram()
1157 drm_dbg(&vm->xe->drm, "Platform doesn't support SZ_4K range migration\n"); in xe_svm_range_needs_migrate_to_vram()
1213 static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
1217 int devmem_possible = IS_DGFX(vm->xe) &&
1225 vm->xe->atomic_svm_timeslice_ms : 0,
1237 lockdep_assert_held_write(&vm->lock);
1238 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma));
1244 err = xe_svm_garbage_collector(vm);
1250 ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap);
1251 range = xe_svm_range_find_or_insert(vm, fault_addr, vma, &ctx);
1281 drm_dbg(&vm->xe->drm,
1283 vm->usm.asid, ERR_PTR(err));
1296 drm_err(&vm->xe->drm,
1298 vm->usm.asid, ERR_PTR(err));
1308 err = xe_svm_range_get_pages(vm, range, &ctx);
1313 drm_dbg(&vm->xe->drm,
1315 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
1319 drm_err(&vm->xe->drm,
1321 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
1328 drm_dbg(&vm->xe->drm, "After page collect data location is %sin \"%s\".\n",
1337 xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) {
1338 err = xe_vm_drm_exec_lock(vm, &exec);
1341 xe_vm_set_validation_exec(vm, &exec);
1342 fence = xe_vm_range_rebind(vm, vma, range, BIT(tile->id));
1343 xe_vm_set_validation_exec(vm, NULL);
1375 * @vm: The VM.
1386 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, in xe_svm_handle_pagefault() argument
1392 need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); in xe_svm_handle_pagefault()
1396 ret = __xe_svm_handle_pagefault(vm, vma, gt, fault_addr, in xe_svm_handle_pagefault()
1403 vma = xe_vm_find_vma_by_addr(vm, fault_addr); in xe_svm_handle_pagefault()
1414 * @vm: The VM.
1422 bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end) in xe_svm_has_mapping() argument
1424 return drm_gpusvm_has_mapping(&vm->svm.gpusvm, start, end); in xe_svm_has_mapping()
1429 * @vm: The VM
1435 void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) in xe_svm_unmap_address_range() argument
1439 lockdep_assert_held_write(&vm->lock); in xe_svm_unmap_address_range()
1441 drm_gpusvm_for_each_notifier_safe(notifier, next, &vm->svm.gpusvm, start, end) { in xe_svm_unmap_address_range()
1447 if (IS_DGFX(vm->xe) && xe_svm_range_in_vram(to_xe_range(range))) in xe_svm_unmap_address_range()
1448 drm_gpusvm_range_evict(&vm->svm.gpusvm, range); in xe_svm_unmap_address_range()
1450 __xe_svm_garbage_collector(vm, to_xe_range(range)); in xe_svm_unmap_address_range()
1452 spin_lock(&vm->svm.garbage_collector.lock); in xe_svm_unmap_address_range()
1454 spin_unlock(&vm->svm.garbage_collector.lock); in xe_svm_unmap_address_range()
1478 * @vm: xe_vm pointer
1488 struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, in xe_svm_range_find_or_insert() argument
1493 r = drm_gpusvm_range_find_or_insert(&vm->svm.gpusvm, max(addr, xe_vma_start(vma)), in xe_svm_range_find_or_insert()
1503 * @vm: Pointer to the struct xe_vm
1512 int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, in xe_svm_range_get_pages() argument
1517 err = drm_gpusvm_range_get_pages(&vm->svm.gpusvm, &range->base, ctx); in xe_svm_range_get_pages()
1520 drm_gpusvm_range_evict(&vm->svm.gpusvm, &range->base); in xe_svm_range_get_pages()
1528 * @vm: Pointer to the xe_vm structure
1537 u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) in xe_svm_ranges_zap_ptes_in_range() argument
1546 lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && in xe_svm_ranges_zap_ptes_in_range()
1547 lockdep_is_held_type(&vm->lock, 0)); in xe_svm_ranges_zap_ptes_in_range()
1549 drm_gpusvm_for_each_notifier(notifier, &vm->svm.gpusvm, start, end) { in xe_svm_ranges_zap_ptes_in_range()
1556 for_each_tile(tile, vm->xe, id) { in xe_svm_ranges_zap_ptes_in_range()
1557 if (xe_pt_zap_ptes_range(tile, vm, range)) { in xe_svm_ranges_zap_ptes_in_range()
1627 struct xe_vm *vm = range_to_vm(&range->base); in xe_svm_alloc_vram() local
1629 struct xe_device *xe = vm->xe; in xe_svm_alloc_vram()
1637 xe_svm_private_page_owner(vm, false), in xe_svm_alloc_vram()
1879 static int xe_svm_get_pagemaps(struct xe_vm *vm) in xe_svm_get_pagemaps() argument
1881 struct xe_device *xe = vm->xe; in xe_svm_get_pagemaps()
1896 vm->svm.pagemaps[id] = xpagemap; in xe_svm_get_pagemaps()
1900 xe_svm_put_pagemaps(vm); in xe_svm_get_pagemaps()
2048 * @vm: The VM.
2052 void xe_svm_flush(struct xe_vm *vm) in xe_svm_flush() argument
2054 if (xe_vm_in_fault_mode(vm)) in xe_svm_flush()
2055 flush_work(&vm->svm.garbage_collector.work); in xe_svm_flush()