305 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,  in drm_gpusvm_range_find()  argument
310 	itree = interval_tree_iter_first(&notifier->root, start, end - 1);  in drm_gpusvm_range_find()
336 	struct drm_gpusvm_notifier *notifier =  in drm_gpusvm_notifier_invalidate()  local
337 		container_of(mni, typeof(*notifier), notifier);  in drm_gpusvm_notifier_invalidate()
338 	struct drm_gpusvm *gpusvm = notifier->gpusvm;  in drm_gpusvm_notifier_invalidate()
345 	gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);  in drm_gpusvm_notifier_invalidate()
447 				       struct drm_gpusvm_notifier *notifier)  in drm_gpusvm_notifier_insert()  argument
452 	interval_tree_insert(&notifier->itree, &gpusvm->root);  in drm_gpusvm_notifier_insert()
454 	node = rb_prev(&notifier->itree.rb);  in drm_gpusvm_notifier_insert()
460 	list_add(&notifier->entry, head);  in drm_gpusvm_notifier_insert()
471 				       struct drm_gpusvm_notifier *notifier)  in drm_gpusvm_notifier_remove()  argument
473 	interval_tree_remove(&notifier->itree, &gpusvm->root);  in drm_gpusvm_notifier_remove()
474 	list_del(&notifier->entry);  in drm_gpusvm_notifier_remove()
486 	struct drm_gpusvm_notifier *notifier, *next;  in drm_gpusvm_fini()  local
488 	drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) {  in drm_gpusvm_fini()
494 		mmu_interval_notifier_remove(&notifier->notifier);  in drm_gpusvm_fini()
495 		notifier->flags.removed = true;  in drm_gpusvm_fini()
497 		drm_gpusvm_for_each_range_safe(range, __next, notifier, 0,  in drm_gpusvm_fini()
520 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_notifier_alloc()  local
523 		notifier = gpusvm->ops->notifier_alloc();  in drm_gpusvm_notifier_alloc()
525 		notifier = kzalloc_obj(*notifier);  in drm_gpusvm_notifier_alloc()
527 	if (!notifier)  in drm_gpusvm_notifier_alloc()
530 	notifier->gpusvm = gpusvm;  in drm_gpusvm_notifier_alloc()
531 	notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size);  in drm_gpusvm_notifier_alloc()
532 	notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1;  in drm_gpusvm_notifier_alloc()
533 	INIT_LIST_HEAD(&notifier->entry);  in drm_gpusvm_notifier_alloc()
534 	notifier->root = RB_ROOT_CACHED;  in drm_gpusvm_notifier_alloc()
535 	INIT_LIST_HEAD(&notifier->range_list);  in drm_gpusvm_notifier_alloc()
537 	return notifier;  in drm_gpusvm_notifier_alloc()
548 				     struct drm_gpusvm_notifier *notifier)  in drm_gpusvm_notifier_free()  argument
550 	WARN_ON(!RB_EMPTY_ROOT(&notifier->root.rb_root));  in drm_gpusvm_notifier_free()
553 		gpusvm->ops->notifier_free(notifier);  in drm_gpusvm_notifier_free()
555 		kfree(notifier);  in drm_gpusvm_notifier_free()
576 static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_range_insert()  argument
582 	drm_gpusvm_notifier_lock(notifier->gpusvm);  in drm_gpusvm_range_insert()
583 	interval_tree_insert(&range->itree, &notifier->root);  in drm_gpusvm_range_insert()
589 		head = &notifier->range_list;  in drm_gpusvm_range_insert()
592 	drm_gpusvm_notifier_unlock(notifier->gpusvm);  in drm_gpusvm_range_insert()
602 static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier,  in __drm_gpusvm_range_remove()  argument
605 	interval_tree_remove(&range->itree, &notifier->root);  in __drm_gpusvm_range_remove()
623 		       struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_range_alloc()  argument
639 	range->notifier = notifier;  in drm_gpusvm_range_alloc()
692 				   struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_check_pages()  argument
698 		.notifier = &notifier->notifier,  in drm_gpusvm_check_pages()
715 	hmm_range.notifier_seq = mmu_interval_read_begin(&notifier->notifier);  in drm_gpusvm_check_pages()
725 				mmu_interval_read_begin(&notifier->notifier);  in drm_gpusvm_check_pages()
766 	struct mmu_interval_notifier *notifier = &range->notifier->notifier;  in drm_gpusvm_scan_mm()  local
771 		.notifier = notifier,  in drm_gpusvm_scan_mm()
791 	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);  in drm_gpusvm_scan_mm()
801 				mmu_interval_read_begin(notifier);  in drm_gpusvm_scan_mm()
811 	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {  in drm_gpusvm_scan_mm()
886 			    struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_range_chunk_size()  argument
903 		    start >= drm_gpusvm_notifier_start(notifier) &&  in drm_gpusvm_range_chunk_size()
904 		    end <= drm_gpusvm_notifier_end(notifier) &&  in drm_gpusvm_range_chunk_size()
919 		range = drm_gpusvm_range_find(notifier, start, end);  in drm_gpusvm_range_chunk_size()
939 		    !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) {  in drm_gpusvm_range_chunk_size()
1020 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_range_find_or_insert()  local
1038 	notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1);  in drm_gpusvm_range_find_or_insert()
1039 	if (!notifier) {  in drm_gpusvm_range_find_or_insert()
1040 		notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr);  in drm_gpusvm_range_find_or_insert()
1041 		if (IS_ERR(notifier)) {  in drm_gpusvm_range_find_or_insert()
1042 			err = PTR_ERR(notifier);  in drm_gpusvm_range_find_or_insert()
1046 		err = mmu_interval_notifier_insert(&notifier->notifier,  in drm_gpusvm_range_find_or_insert()
1048 						   drm_gpusvm_notifier_start(notifier),  in drm_gpusvm_range_find_or_insert()
1049 						   drm_gpusvm_notifier_size(notifier),  in drm_gpusvm_range_find_or_insert()
1068 	range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1);  in drm_gpusvm_range_find_or_insert()
1079 	chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas,  in drm_gpusvm_range_find_or_insert()
1089 	range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size,  in drm_gpusvm_range_find_or_insert()
1096 	drm_gpusvm_range_insert(notifier, range);  in drm_gpusvm_range_find_or_insert()
1098 		drm_gpusvm_notifier_insert(gpusvm, notifier);  in drm_gpusvm_range_find_or_insert()
1109 		mmu_interval_notifier_remove(&notifier->notifier);  in drm_gpusvm_range_find_or_insert()
1112 		drm_gpusvm_notifier_free(gpusvm, notifier);  in drm_gpusvm_range_find_or_insert()
1220 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_range_remove()  local
1224 	notifier = drm_gpusvm_notifier_find(gpusvm,  in drm_gpusvm_range_remove()
1227 	if (WARN_ON_ONCE(!notifier))  in drm_gpusvm_range_remove()
1233 	__drm_gpusvm_range_remove(notifier, range);  in drm_gpusvm_range_remove()
1238 	if (RB_EMPTY_ROOT(&notifier->root.rb_root)) {  in drm_gpusvm_range_remove()
1239 		if (!notifier->flags.removed)  in drm_gpusvm_range_remove()
1240 			mmu_interval_notifier_remove(&notifier->notifier);  in drm_gpusvm_range_remove()
1241 		drm_gpusvm_notifier_remove(gpusvm, notifier);  in drm_gpusvm_range_remove()
1242 		drm_gpusvm_notifier_free(gpusvm, notifier);  in drm_gpusvm_range_remove()
1385 			 struct mmu_interval_notifier *notifier,  in drm_gpusvm_get_pages()  argument
1392 		.notifier = notifier,  in drm_gpusvm_get_pages()
1416 	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);  in drm_gpusvm_get_pages()
1440 				mmu_interval_read_begin(notifier);  in drm_gpusvm_get_pages()
1464 	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {  in drm_gpusvm_get_pages()
1601 				    &range->notifier->notifier,  in drm_gpusvm_range_get_pages()
1672 	struct mmu_interval_notifier *notifier = &range->notifier->notifier;  in drm_gpusvm_range_evict()  local
1675 		.notifier = notifier,  in drm_gpusvm_range_evict()
1697 		hmm_range.notifier_seq = mmu_interval_read_begin(notifier);  in drm_gpusvm_range_evict()
1728 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_has_mapping()  local
1730 	drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) {  in drm_gpusvm_has_mapping()
1733 		drm_gpusvm_for_each_range(range, notifier, start, end)  in drm_gpusvm_has_mapping()