226 vdev_queue_class_tree(vdev_queue_t *vq, zio_priority_t p)  in vdev_queue_class_tree()  argument
228 	return (&vq->vq_class[p].vqc_queued_tree);  in vdev_queue_class_tree()
232 vdev_queue_type_tree(vdev_queue_t *vq, zio_type_t t)  in vdev_queue_type_tree()  argument
236 		return (&vq->vq_read_offset_tree);  in vdev_queue_type_tree()
238 		return (&vq->vq_write_offset_tree);  in vdev_queue_type_tree()
240 		return (&vq->vq_trim_offset_tree);  in vdev_queue_type_tree()
260 	vdev_queue_t *vq = &vd->vdev_queue;  in vdev_queue_init()  local
262 	mutex_init(&vq->vq_lock, NULL, MUTEX_DEFAULT, NULL);  in vdev_queue_init()
263 	vq->vq_vdev = vd;  in vdev_queue_init()
265 	avl_create(&vq->vq_active_tree, vdev_queue_offset_compare,  in vdev_queue_init()
267 	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_READ),  in vdev_queue_init()
270 	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE),  in vdev_queue_init()
273 	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_TRIM),  in vdev_queue_init()
293 		avl_create(vdev_queue_class_tree(vq, p), compfn,  in vdev_queue_init()
297 	vq->vq_last_offset = 0;  in vdev_queue_init()
303 	vdev_queue_t *vq = &vd->vdev_queue;  in vdev_queue_fini()  local
306 		avl_destroy(vdev_queue_class_tree(vq, p));  in vdev_queue_fini()
307 	avl_destroy(&vq->vq_active_tree);  in vdev_queue_fini()
308 	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_READ));  in vdev_queue_fini()
309 	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE));  in vdev_queue_fini()
310 	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_TRIM));  in vdev_queue_fini()
312 	mutex_destroy(&vq->vq_lock);  in vdev_queue_fini()
316 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_add()  argument
321 	avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_add()
322 	avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_add()
332 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_remove()  argument
337 	avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_remove()
338 	avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_remove()
349 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_add()  argument
352 	ASSERT(MUTEX_HELD(&vq->vq_lock));  in vdev_queue_pending_add()
354 	vq->vq_class[zio->io_priority].vqc_active++;  in vdev_queue_pending_add()
355 	avl_add(&vq->vq_active_tree, zio);  in vdev_queue_pending_add()
365 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_remove()  argument
368 	ASSERT(MUTEX_HELD(&vq->vq_lock));  in vdev_queue_pending_remove()
370 	vq->vq_class[zio->io_priority].vqc_active--;  in vdev_queue_pending_remove()
371 	avl_remove(&vq->vq_active_tree, zio);  in vdev_queue_pending_remove()
500 vdev_queue_class_to_issue(vdev_queue_t *vq)  in vdev_queue_class_to_issue()  argument
502 	spa_t *spa = vq->vq_vdev->vdev_spa;  in vdev_queue_class_to_issue()
505 	if (avl_numnodes(&vq->vq_active_tree) >= zfs_vdev_max_active)  in vdev_queue_class_to_issue()
510 		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&  in vdev_queue_class_to_issue()
511 		    vq->vq_class[p].vqc_active <  in vdev_queue_class_to_issue()
521 		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&  in vdev_queue_class_to_issue()
522 		    vq->vq_class[p].vqc_active <  in vdev_queue_class_to_issue()
541 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_aggregate()  argument
548 	avl_tree_t *t = vdev_queue_type_tree(vq, zio->io_type);  in vdev_queue_aggregate()
689 		vdev_queue_io_remove(vq, dio);  in vdev_queue_aggregate()
696 	mutex_exit(&vq->vq_lock);  in vdev_queue_aggregate()
701 	mutex_enter(&vq->vq_lock);  in vdev_queue_aggregate()
707 vdev_queue_io_to_issue(vdev_queue_t *vq)  in vdev_queue_io_to_issue()  argument
716 	ASSERT(MUTEX_HELD(&vq->vq_lock));  in vdev_queue_io_to_issue()
718 	p = vdev_queue_class_to_issue(vq);  in vdev_queue_io_to_issue()
731 	tree = vdev_queue_class_tree(vq, p);  in vdev_queue_io_to_issue()
733 	search.io_offset = vq->vq_last_offset - 1;  in vdev_queue_io_to_issue()
740 	aio = vdev_queue_aggregate(vq, zio);  in vdev_queue_io_to_issue()
744 		vdev_queue_io_remove(vq, zio);  in vdev_queue_io_to_issue()
753 		mutex_exit(&vq->vq_lock);  in vdev_queue_io_to_issue()
756 		mutex_enter(&vq->vq_lock);  in vdev_queue_io_to_issue()
760 	vdev_queue_pending_add(vq, zio);  in vdev_queue_io_to_issue()
761 	vq->vq_last_offset = zio->io_offset + zio->io_size;  in vdev_queue_io_to_issue()
769 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io()  local
805 	mutex_enter(&vq->vq_lock);  in vdev_queue_io()
807 	vdev_queue_io_add(vq, zio);  in vdev_queue_io()
808 	nio = vdev_queue_io_to_issue(vq);  in vdev_queue_io()
809 	mutex_exit(&vq->vq_lock);  in vdev_queue_io()
825 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io_done()  local
828 	mutex_enter(&vq->vq_lock);  in vdev_queue_io_done()
830 	vdev_queue_pending_remove(vq, zio);  in vdev_queue_io_done()
833 	vq->vq_io_complete_ts = gethrtime();  in vdev_queue_io_done()
835 	while ((nio = vdev_queue_io_to_issue(vq)) != NULL) {  in vdev_queue_io_done()
836 		mutex_exit(&vq->vq_lock);  in vdev_queue_io_done()
843 		mutex_enter(&vq->vq_lock);  in vdev_queue_io_done()
846 	mutex_exit(&vq->vq_lock);  in vdev_queue_io_done()
852 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_change_io_priority()  local
879 	mutex_enter(&vq->vq_lock);  in vdev_queue_change_io_priority()
888 	tree = vdev_queue_class_tree(vq, zio->io_priority);  in vdev_queue_change_io_priority()
893 		avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
895 		avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
902 	} else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) {  in vdev_queue_change_io_priority()
906 	mutex_exit(&vq->vq_lock);  in vdev_queue_change_io_priority()