37  *	list of ranges that fall within the notifier interval.  Notifiers are
47  *	dynamically allocated on GPU fault and removed on an MMU notifier UNMAP
48  *	event. As mentioned above, ranges are tracked in a notifier's Red-Black
53  *	range allocation, notifier allocation, and invalidations.
79  *	notifier callback.
81  * - Notifier callback:
91  * GPU SVM introduces a global notifier lock, which safeguards the notifier's
98  * global lock to a per-notifier lock if finer-grained locking is deemed
133  * notifier callback.
229  * 3) Notifier callback
234  *				 struct drm_gpusvm_notifier *notifier,
242  *		drm_gpusvm_for_each_range(range, notifier, mmu_range->start,
274  * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM
276  * @start: Start address of the notifier
277  * @end: End address of the notifier
297  * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier
298  * @notifier: Pointer to the GPU SVM notifier structure.
305 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,  in drm_gpusvm_range_find()  argument
310 	itree = interval_tree_iter_first(&notifier->root, start, end - 1);  in drm_gpusvm_range_find()
320  * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier.
325  * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU
326  * notifier sequence number and calls the driver invalidate vfunc under
336 	struct drm_gpusvm_notifier *notifier =  in drm_gpusvm_notifier_invalidate()  local
337 		container_of(mni, typeof(*notifier), notifier);  in drm_gpusvm_notifier_invalidate()
338 	struct drm_gpusvm *gpusvm = notifier->gpusvm;  in drm_gpusvm_notifier_invalidate()
345 	gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);  in drm_gpusvm_notifier_invalidate()
352  * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
440  * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier
442  * @notifier: Pointer to the GPU SVM notifier structure
444  * This function inserts the GPU SVM notifier into the GPU SVM RB tree and list.
447 				       struct drm_gpusvm_notifier *notifier)  in drm_gpusvm_notifier_insert()  argument
452 	interval_tree_insert(&notifier->itree, &gpusvm->root);  in drm_gpusvm_notifier_insert()
454 	node = rb_prev(&notifier->itree.rb);  in drm_gpusvm_notifier_insert()
460 	list_add(&notifier->entry, head);  in drm_gpusvm_notifier_insert()
464  * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier
466  * @notifier: Pointer to the GPU SVM notifier structure
468  * This function removes the GPU SVM notifier from the GPU SVM RB tree and list.
471 				       struct drm_gpusvm_notifier *notifier)  in drm_gpusvm_notifier_remove()  argument
473 	interval_tree_remove(&notifier->itree, &gpusvm->root);  in drm_gpusvm_notifier_remove()
474 	list_del(&notifier->entry);  in drm_gpusvm_notifier_remove()
486 	struct drm_gpusvm_notifier *notifier, *next;  in drm_gpusvm_fini()  local
488 	drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) {  in drm_gpusvm_fini()
492 		 * Remove notifier first to avoid racing with any invalidation  in drm_gpusvm_fini()
494 		mmu_interval_notifier_remove(&notifier->notifier);  in drm_gpusvm_fini()
495 		notifier->flags.removed = true;  in drm_gpusvm_fini()
497 		drm_gpusvm_for_each_range_safe(range, __next, notifier, 0,  in drm_gpusvm_fini()
509  * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier
513  * This function allocates and initializes the GPU SVM notifier structure.
515  * Return: Pointer to the allocated GPU SVM notifier on success, ERR_PTR() on failure.
520 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_notifier_alloc()  local
523 		notifier = gpusvm->ops->notifier_alloc();  in drm_gpusvm_notifier_alloc()
525 		notifier = kzalloc(sizeof(*notifier), GFP_KERNEL);  in drm_gpusvm_notifier_alloc()
527 	if (!notifier)  in drm_gpusvm_notifier_alloc()
530 	notifier->gpusvm = gpusvm;  in drm_gpusvm_notifier_alloc()
531 	notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size);  in drm_gpusvm_notifier_alloc()
532 	notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1;  in drm_gpusvm_notifier_alloc()
533 	INIT_LIST_HEAD(&notifier->entry);  in drm_gpusvm_notifier_alloc()
534 	notifier->root = RB_ROOT_CACHED;  in drm_gpusvm_notifier_alloc()
535 	INIT_LIST_HEAD(&notifier->range_list);  in drm_gpusvm_notifier_alloc()
537 	return notifier;  in drm_gpusvm_notifier_alloc()
541  * drm_gpusvm_notifier_free() - Free GPU SVM notifier
543  * @notifier: Pointer to the GPU SVM notifier structure
545  * This function frees the GPU SVM notifier structure.
548 				     struct drm_gpusvm_notifier *notifier)  in drm_gpusvm_notifier_free()  argument
550 	WARN_ON(!RB_EMPTY_ROOT(&notifier->root.rb_root));  in drm_gpusvm_notifier_free()
553 		gpusvm->ops->notifier_free(notifier);  in drm_gpusvm_notifier_free()
555 		kfree(notifier);  in drm_gpusvm_notifier_free()
571  * @notifier: Pointer to the GPU SVM notifier structure
574  * This function inserts the GPU SVM range into the notifier RB tree and list.
576 static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_range_insert()  argument
582 	drm_gpusvm_notifier_lock(notifier->gpusvm);  in drm_gpusvm_range_insert()
583 	interval_tree_insert(&range->itree, &notifier->root);  in drm_gpusvm_range_insert()
589 		head = &notifier->range_list;  in drm_gpusvm_range_insert()
592 	drm_gpusvm_notifier_unlock(notifier->gpusvm);  in drm_gpusvm_range_insert()
597  * @notifier: Pointer to the GPU SVM notifier structure
600  * This macro removes the GPU SVM range from the notifier RB tree and list.
602 static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier,  in __drm_gpusvm_range_remove()  argument
605 	interval_tree_remove(&range->itree, &notifier->root);  in __drm_gpusvm_range_remove()
612  * @notifier: Pointer to the GPU SVM notifier structure
623 		       struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_range_alloc()  argument
639 	range->notifier = notifier;  in drm_gpusvm_range_alloc()
681  * @notifier: Pointer to the GPU SVM notifier structure
692 				   struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_check_pages()  argument
698 		.notifier = &notifier->notifier,  in drm_gpusvm_check_pages()
715 	hmm_range.notifier_seq = mmu_interval_read_begin(&notifier->notifier);  in drm_gpusvm_check_pages()
725 				mmu_interval_read_begin(&notifier->notifier);  in drm_gpusvm_check_pages()
749  * @notifier: Pointer to the GPU SVM notifier structure
765 			    struct drm_gpusvm_notifier *notifier,  in drm_gpusvm_range_chunk_size()  argument
782 		    start >= drm_gpusvm_notifier_start(notifier) &&  in drm_gpusvm_range_chunk_size()
783 		    end <= drm_gpusvm_notifier_end(notifier) &&  in drm_gpusvm_range_chunk_size()
798 		range = drm_gpusvm_range_find(notifier, start, end);  in drm_gpusvm_range_chunk_size()
818 		    !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) {  in drm_gpusvm_range_chunk_size()
899 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_range_find_or_insert()  local
917 	notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1);  in drm_gpusvm_range_find_or_insert()
918 	if (!notifier) {  in drm_gpusvm_range_find_or_insert()
919 		notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr);  in drm_gpusvm_range_find_or_insert()
920 		if (IS_ERR(notifier)) {  in drm_gpusvm_range_find_or_insert()
921 			err = PTR_ERR(notifier);  in drm_gpusvm_range_find_or_insert()
925 		err = mmu_interval_notifier_insert(&notifier->notifier,  in drm_gpusvm_range_find_or_insert()
927 						   drm_gpusvm_notifier_start(notifier),  in drm_gpusvm_range_find_or_insert()
928 						   drm_gpusvm_notifier_size(notifier),  in drm_gpusvm_range_find_or_insert()
947 	range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1);  in drm_gpusvm_range_find_or_insert()
958 	chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas,  in drm_gpusvm_range_find_or_insert()
968 	range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size,  in drm_gpusvm_range_find_or_insert()
975 	drm_gpusvm_range_insert(notifier, range);  in drm_gpusvm_range_find_or_insert()
977 		drm_gpusvm_notifier_insert(gpusvm, notifier);  in drm_gpusvm_range_find_or_insert()
988 		mmu_interval_notifier_remove(&notifier->notifier);  in drm_gpusvm_range_find_or_insert()
991 		drm_gpusvm_notifier_free(gpusvm, notifier);  in drm_gpusvm_range_find_or_insert()
1090  * GPU SVM notifier if no more ranges remain in the notifier. The caller must
1091  * hold a lock to protect range and notifier removal.
1098 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_range_remove()  local
1102 	notifier = drm_gpusvm_notifier_find(gpusvm,  in drm_gpusvm_range_remove()
1105 	if (WARN_ON_ONCE(!notifier))  in drm_gpusvm_range_remove()
1111 	__drm_gpusvm_range_remove(notifier, range);  in drm_gpusvm_range_remove()
1116 	if (RB_EMPTY_ROOT(&notifier->root.rb_root)) {  in drm_gpusvm_range_remove()
1117 		if (!notifier->flags.removed)  in drm_gpusvm_range_remove()
1118 			mmu_interval_notifier_remove(&notifier->notifier);  in drm_gpusvm_range_remove()
1119 		drm_gpusvm_notifier_remove(gpusvm, notifier);  in drm_gpusvm_range_remove()
1120 		drm_gpusvm_notifier_free(gpusvm, notifier);  in drm_gpusvm_range_remove()
1182  * GPU binding. This is akin to a notifier seqno check in the HMM documentation
1204  * GPU binding. This is akin to a notifier seqno check in the HMM documentation
1250  * @notifier: The corresponding notifier for the given CPU range
1263 			 struct mmu_interval_notifier *notifier,  in drm_gpusvm_get_pages()  argument
1270 		.notifier = notifier,  in drm_gpusvm_get_pages()
1291 	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);  in drm_gpusvm_get_pages()
1315 				mmu_interval_read_begin(notifier);  in drm_gpusvm_get_pages()
1326 	 * Perform all dma mappings under the notifier lock to not  in drm_gpusvm_get_pages()
1327 	 * access freed pages. A notifier will either block on  in drm_gpusvm_get_pages()
1328 	 * the notifier lock or unmap dma.  in drm_gpusvm_get_pages()
1339 	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {  in drm_gpusvm_get_pages()
1472 				    &range->notifier->notifier,  in drm_gpusvm_range_get_pages()
1488  * Must be called in the invalidate() callback of the corresponding notifier for
1517  * each GPU SVM range attached to notifier in gpusvm->ops->invalidate for IOMMU
1543 	struct mmu_interval_notifier *notifier = &range->notifier->notifier;  in drm_gpusvm_range_evict()  local
1546 		.notifier = notifier,  in drm_gpusvm_range_evict()
1568 		hmm_range.notifier_seq = mmu_interval_read_begin(notifier);  in drm_gpusvm_range_evict()
1599 	struct drm_gpusvm_notifier *notifier;  in drm_gpusvm_has_mapping()  local
1601 	drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) {  in drm_gpusvm_has_mapping()
1604 		drm_gpusvm_for_each_range(range, notifier, start, end)  in drm_gpusvm_has_mapping()
1615  * @mmu_range: Pointer to the MMU notifier range structure.
1618  * if the range partially falls within the provided MMU notifier range.