45 			 struct mmu_rb_handler **handler)  in hfi1_mmu_rb_register()  argument
74 	*handler = h;  in hfi1_mmu_rb_register()
78 void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)  in hfi1_mmu_rb_unregister()  argument
86 	mmgrab(handler->mn.mm);  in hfi1_mmu_rb_unregister()
89 	mmu_notifier_unregister(&handler->mn, handler->mn.mm);  in hfi1_mmu_rb_unregister()
92 	 * Make sure the wq delete handler is finished running.  It will not  in hfi1_mmu_rb_unregister()
95 	flush_work(&handler->del_work);  in hfi1_mmu_rb_unregister()
99 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_unregister()
100 	while ((node = rb_first_cached(&handler->root))) {  in hfi1_mmu_rb_unregister()
102 		rb_erase_cached(node, &handler->root);  in hfi1_mmu_rb_unregister()
106 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_unregister()
115 	mmdrop(handler->mn.mm);  in hfi1_mmu_rb_unregister()
117 	kfree(handler->free_ptr);  in hfi1_mmu_rb_unregister()
120 int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,  in hfi1_mmu_rb_insert()  argument
129 	if (current->mm != handler->mn.mm)  in hfi1_mmu_rb_insert()
132 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_insert()
133 	node = __mmu_rb_search(handler, mnode->addr, mnode->len);  in hfi1_mmu_rb_insert()
138 	__mmu_int_rb_insert(mnode, &handler->root);  in hfi1_mmu_rb_insert()
139 	list_add_tail(&mnode->list, &handler->lru_list);  in hfi1_mmu_rb_insert()
140 	mnode->handler = handler;  in hfi1_mmu_rb_insert()
142 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_insert()
146 /* Caller must hold handler lock */
147 struct mmu_rb_node *hfi1_mmu_rb_get_first(struct mmu_rb_handler *handler,  in hfi1_mmu_rb_get_first()  argument
153 	node = __mmu_int_rb_iter_first(&handler->root, addr, (addr + len) - 1);  in hfi1_mmu_rb_get_first()
155 		list_move_tail(&node->list, &handler->lru_list);  in hfi1_mmu_rb_get_first()
159 /* Caller must hold handler lock */
160 static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,  in __mmu_rb_search()  argument
167 	if (!handler->ops->filter) {  in __mmu_rb_search()
168 		node = __mmu_int_rb_iter_first(&handler->root, addr,  in __mmu_rb_search()
171 		for (node = __mmu_int_rb_iter_first(&handler->root, addr,  in __mmu_rb_search()
176 			if (handler->ops->filter(node, addr, len))  in __mmu_rb_search()
184  * Must NOT call while holding mnode->handler->lock.
185  * mnode->handler->ops->remove() may sleep and mnode->handler->lock is a
193 	mnode->handler->ops->remove(mnode->handler->ops_arg, mnode);  in release_immediate()
196 /* Caller must hold mnode->handler->lock */
201 	list_move(&mnode->list, &mnode->handler->del_list);  in release_nolock()
202 	queue_work(mnode->handler->wq, &mnode->handler->del_work);  in release_nolock()
207  * Adds mmu_rb_node to mmu_rb_node->handler->del_list and queues
208  * handler->del_work on handler->wq.
209  * Does not remove mmu_rb_node from handler->lru_list or handler->rb_root.
210  * Acquires mmu_rb_node->handler->lock; do not call while already holding
211  * handler->lock.
217 	struct mmu_rb_handler *handler = mnode->handler;  in hfi1_mmu_rb_release()  local
220 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_release()
221 	list_move(&mnode->list, &mnode->handler->del_list);  in hfi1_mmu_rb_release()
222 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_release()
223 	queue_work(handler->wq, &handler->del_work);  in hfi1_mmu_rb_release()
226 void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)  in hfi1_mmu_rb_evict()  argument
233 	if (current->mm != handler->mn.mm)  in hfi1_mmu_rb_evict()
238 	spin_lock_irqsave(&handler->lock, flags);  in hfi1_mmu_rb_evict()
239 	list_for_each_entry_safe(rbnode, ptr, &handler->lru_list, list) {  in hfi1_mmu_rb_evict()
244 		if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg,  in hfi1_mmu_rb_evict()
246 			__mmu_int_rb_remove(rbnode, &handler->root);  in hfi1_mmu_rb_evict()
253 	spin_unlock_irqrestore(&handler->lock, flags);  in hfi1_mmu_rb_evict()
264 	struct mmu_rb_handler *handler =  in mmu_notifier_range_start()  local
266 	struct rb_root_cached *root = &handler->root;  in mmu_notifier_range_start()
270 	spin_lock_irqsave(&handler->lock, flags);  in mmu_notifier_range_start()
282 	spin_unlock_irqrestore(&handler->lock, flags);  in mmu_notifier_range_start()
294 	struct mmu_rb_handler *handler = container_of(work,  in handle_remove()  local
302 	spin_lock_irqsave(&handler->lock, flags);  in handle_remove()
303 	list_replace_init(&handler->del_list, &del_list);  in handle_remove()
304 	spin_unlock_irqrestore(&handler->lock, flags);  in handle_remove()
310 		handler->ops->remove(handler->ops_arg, node);  in handle_remove()