266 	    sizeof (zio_t), offsetof(struct zio, io_queue_node));  in vdev_queue_init()
269 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
272 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
275 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
294 		    sizeof (zio_t), offsetof(struct zio, io_queue_node));  in vdev_queue_init()
316 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_add()  argument
318 	spa_t *spa = zio->io_spa;  in vdev_queue_io_add()
320 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_io_add()
321 	avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_add()
322 	avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_add()
325 	spa->spa_queue_stats[zio->io_priority].spa_queued++;  in vdev_queue_io_add()
332 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_remove()  argument
334 	spa_t *spa = zio->io_spa;  in vdev_queue_io_remove()
336 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_io_remove()
337 	avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_remove()
338 	avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_remove()
341 	ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_queued, >, 0);  in vdev_queue_io_remove()
342 	spa->spa_queue_stats[zio->io_priority].spa_queued--;  in vdev_queue_io_remove()
349 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_add()  argument
351 	spa_t *spa = zio->io_spa;  in vdev_queue_pending_add()
353 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_pending_add()
354 	vq->vq_class[zio->io_priority].vqc_active++;  in vdev_queue_pending_add()
355 	avl_add(&vq->vq_active_tree, zio);  in vdev_queue_pending_add()
358 	spa->spa_queue_stats[zio->io_priority].spa_active++;  in vdev_queue_pending_add()
365 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_remove()  argument
367 	spa_t *spa = zio->io_spa;  in vdev_queue_pending_remove()
369 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_pending_remove()
370 	vq->vq_class[zio->io_priority].vqc_active--;  in vdev_queue_pending_remove()
371 	avl_remove(&vq->vq_active_tree, zio);  in vdev_queue_pending_remove()
374 	ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_active, >, 0);  in vdev_queue_pending_remove()
375 	spa->spa_queue_stats[zio->io_priority].spa_active--;  in vdev_queue_pending_remove()
380 		if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_pending_remove()
382 			ksio->nread += zio->io_size;  in vdev_queue_pending_remove()
383 		} else if (zio->io_type == ZIO_TYPE_WRITE) {  in vdev_queue_pending_remove()
385 			ksio->nwritten += zio->io_size;  in vdev_queue_pending_remove()
541 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_aggregate()  argument
548 	avl_tree_t *t = vdev_queue_type_tree(vq, zio->io_type);  in vdev_queue_aggregate()
549 	enum zio_flag flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT;  in vdev_queue_aggregate()
551 	if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE)  in vdev_queue_aggregate()
558 	if (zio->io_type == ZIO_TYPE_TRIM && !zfs_vdev_aggregate_trim)  in vdev_queue_aggregate()
561 	first = last = zio;  in vdev_queue_aggregate()
563 	if (zio->io_type == ZIO_TYPE_READ)  in vdev_queue_aggregate()
589 	    dio->io_type == zio->io_type) {  in vdev_queue_aggregate()
614 	    dio->io_type == zio->io_type) {  in vdev_queue_aggregate()
630 	if (zio->io_type == ZIO_TYPE_WRITE && mandatory != NULL) {  in vdev_queue_aggregate()
669 	    zio->io_priority, flags | ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE,  in vdev_queue_aggregate()
709 	zio_t *zio, *aio;  in vdev_queue_io_to_issue()  local
735 	zio = avl_nearest(tree, idx, AVL_AFTER);  in vdev_queue_io_to_issue()
736 	if (zio == NULL)  in vdev_queue_io_to_issue()
737 		zio = avl_first(tree);  in vdev_queue_io_to_issue()
738 	ASSERT3U(zio->io_priority, ==, p);  in vdev_queue_io_to_issue()
740 	aio = vdev_queue_aggregate(vq, zio);  in vdev_queue_io_to_issue()
742 		zio = aio;  in vdev_queue_io_to_issue()
744 		vdev_queue_io_remove(vq, zio);  in vdev_queue_io_to_issue()
752 	if (zio->io_flags & ZIO_FLAG_NODATA) {  in vdev_queue_io_to_issue()
754 		zio_vdev_io_bypass(zio);  in vdev_queue_io_to_issue()
755 		zio_execute(zio);  in vdev_queue_io_to_issue()
760 	vdev_queue_pending_add(vq, zio);  in vdev_queue_io_to_issue()
761 	vq->vq_last_offset = zio->io_offset + zio->io_size;  in vdev_queue_io_to_issue()
763 	return (zio);  in vdev_queue_io_to_issue()
767 vdev_queue_io(zio_t *zio)  in vdev_queue_io()  argument
769 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io()
772 	if (zio->io_flags & ZIO_FLAG_DONT_QUEUE)  in vdev_queue_io()
773 		return (zio);  in vdev_queue_io()
779 	if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_io()
780 		ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM);  in vdev_queue_io()
782 		if (zio->io_priority != ZIO_PRIORITY_SYNC_READ &&  in vdev_queue_io()
783 		    zio->io_priority != ZIO_PRIORITY_ASYNC_READ &&  in vdev_queue_io()
784 		    zio->io_priority != ZIO_PRIORITY_SCRUB &&  in vdev_queue_io()
785 		    zio->io_priority != ZIO_PRIORITY_REMOVAL &&  in vdev_queue_io()
786 		    zio->io_priority != ZIO_PRIORITY_INITIALIZING) {  in vdev_queue_io()
787 			zio->io_priority = ZIO_PRIORITY_ASYNC_READ;  in vdev_queue_io()
789 	} else if (zio->io_type == ZIO_TYPE_WRITE) {  in vdev_queue_io()
790 		ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM);  in vdev_queue_io()
792 		if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE &&  in vdev_queue_io()
793 		    zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE &&  in vdev_queue_io()
794 		    zio->io_priority != ZIO_PRIORITY_REMOVAL &&  in vdev_queue_io()
795 		    zio->io_priority != ZIO_PRIORITY_INITIALIZING) {  in vdev_queue_io()
796 			zio->io_priority = ZIO_PRIORITY_ASYNC_WRITE;  in vdev_queue_io()
799 		ASSERT(zio->io_type == ZIO_TYPE_TRIM);  in vdev_queue_io()
800 		ASSERT(zio->io_priority == ZIO_PRIORITY_TRIM);  in vdev_queue_io()
803 	zio->io_flags |= ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE;  in vdev_queue_io()
806 	zio->io_timestamp = gethrtime();  in vdev_queue_io()
807 	vdev_queue_io_add(vq, zio);  in vdev_queue_io()
823 vdev_queue_io_done(zio_t *zio)  in vdev_queue_io_done()  argument
825 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io_done()
830 	vdev_queue_pending_remove(vq, zio);  in vdev_queue_io_done()
832 	zio->io_delta = gethrtime() - zio->io_timestamp;  in vdev_queue_io_done()
850 vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority)  in vdev_queue_change_io_priority()  argument
852 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_change_io_priority()
861 	if (zio->io_priority == ZIO_PRIORITY_NOW)  in vdev_queue_change_io_priority()
864 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_change_io_priority()
867 	if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_change_io_priority()
873 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);  in vdev_queue_change_io_priority()
888 	tree = vdev_queue_class_tree(vq, zio->io_priority);  in vdev_queue_change_io_priority()
889 	if (avl_find(tree, zio, NULL) == zio) {  in vdev_queue_change_io_priority()
890 		spa_t *spa = zio->io_spa;  in vdev_queue_change_io_priority()
891 		zio_priority_t oldpri = zio->io_priority;  in vdev_queue_change_io_priority()
893 		avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
894 		zio->io_priority = priority;  in vdev_queue_change_io_priority()
895 		avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
900 		spa->spa_queue_stats[zio->io_priority].spa_queued++;  in vdev_queue_change_io_priority()
902 	} else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) {  in vdev_queue_change_io_priority()
903 		zio->io_priority = priority;  in vdev_queue_change_io_priority()