189 	spa_vdev_removal_t *svr = kmem_zalloc(sizeof (*svr), KM_SLEEP);  in spa_vdev_removal_create()  local
190 	mutex_init(&svr->svr_lock, NULL, MUTEX_DEFAULT, NULL);  in spa_vdev_removal_create()
191 	cv_init(&svr->svr_cv, NULL, CV_DEFAULT, NULL);  in spa_vdev_removal_create()
192 	svr->svr_allocd_segs = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);  in spa_vdev_removal_create()
193 	svr->svr_vdev_id = vd->vdev_id;  in spa_vdev_removal_create()
196 		svr->svr_frees[i] = range_tree_create(NULL, RANGE_SEG64, NULL,  in spa_vdev_removal_create()
198 		list_create(&svr->svr_new_segments[i],  in spa_vdev_removal_create()
203 	return (svr);  in spa_vdev_removal_create()
207 spa_vdev_removal_destroy(spa_vdev_removal_t *svr)  in spa_vdev_removal_destroy()  argument
210 		ASSERT0(svr->svr_bytes_done[i]);  in spa_vdev_removal_destroy()
211 		ASSERT0(svr->svr_max_offset_to_sync[i]);  in spa_vdev_removal_destroy()
212 		range_tree_destroy(svr->svr_frees[i]);  in spa_vdev_removal_destroy()
213 		list_destroy(&svr->svr_new_segments[i]);  in spa_vdev_removal_destroy()
216 	range_tree_destroy(svr->svr_allocd_segs);  in spa_vdev_removal_destroy()
217 	mutex_destroy(&svr->svr_lock);  in spa_vdev_removal_destroy()
218 	cv_destroy(&svr->svr_cv);  in spa_vdev_removal_destroy()
219 	kmem_free(svr, sizeof (*svr));  in spa_vdev_removal_destroy()
241 	spa_vdev_removal_t *svr = NULL;  in vdev_remove_initiate_sync()  local
245 	svr = spa_vdev_removal_create(vd);  in vdev_remove_initiate_sync()
352 	spa->spa_vdev_removal = svr;  in vdev_remove_initiate_sync()
353 	svr->svr_thread = thread_create(NULL, 0,  in vdev_remove_initiate_sync()
405 		spa_vdev_removal_t *svr = spa_vdev_removal_create(vd);  in spa_remove_init()  local
406 		ASSERT3U(svr->svr_vdev_id, ==, vd->vdev_id);  in spa_remove_init()
415 		spa->spa_vdev_removal = svr;  in spa_remove_init()
446 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in spa_restart_removal()  local
448 	if (svr == NULL)  in spa_restart_removal()
460 	if (svr->svr_thread != NULL)  in spa_restart_removal()
466 	zfs_dbgmsg("restarting removal of %llu", svr->svr_vdev_id);  in spa_restart_removal()
467 	svr->svr_thread = thread_create(NULL, 0, spa_vdev_remove_thread, spa,  in spa_restart_removal()
480 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in free_from_removing_vdev()  local
488 	ASSERT3U(vd->vdev_id, ==, svr->svr_vdev_id);  in free_from_removing_vdev()
490 	mutex_enter(&svr->svr_lock);  in free_from_removing_vdev()
550 		if (size > 0 && offset < svr->svr_max_offset_to_sync[txgoff]) {  in free_from_removing_vdev()
556 			    svr->svr_max_offset_to_sync[txgoff] - offset);  in free_from_removing_vdev()
570 			if (svr->svr_max_offset_to_sync[txgoff] != 0) {  in free_from_removing_vdev()
571 				ASSERT3U(svr->svr_max_offset_to_sync[txgoff],  in free_from_removing_vdev()
574 				    svr->svr_max_offset_to_sync[txgoff];  in free_from_removing_vdev()
589 			range_tree_add(svr->svr_frees[txgoff],  in free_from_removing_vdev()
605 			ASSERT3U(svr->svr_bytes_done[txgoff], >=,  in free_from_removing_vdev()
607 			svr->svr_bytes_done[txgoff] -= inflight_size;  in free_from_removing_vdev()
608 			svr->svr_bytes_done[txg & TXG_MASK] += inflight_size;  in free_from_removing_vdev()
611 	ASSERT0(svr->svr_max_offset_to_sync[TXG_CLEAN(txg) & TXG_MASK]);  in free_from_removing_vdev()
624 		if (svr->svr_allocd_segs != NULL)  in free_from_removing_vdev()
625 			range_tree_clear(svr->svr_allocd_segs, offset, size);  in free_from_removing_vdev()
632 		svr->svr_bytes_done[txg & TXG_MASK] += size;  in free_from_removing_vdev()
634 	mutex_exit(&svr->svr_lock);  in free_from_removing_vdev()
661 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in spa_finish_removal()  local
671 		vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);  in spa_finish_removal()
687 	spa_vdev_removal_destroy(svr);  in spa_finish_removal()
712 	spa_vdev_removal_t *svr = arg;  in vdev_mapping_sync()  local
714 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);  in vdev_mapping_sync()
723 	    &svr->svr_new_segments[txg & TXG_MASK], tx);  in vdev_mapping_sync()
731 	mutex_enter(&svr->svr_lock);  in vdev_mapping_sync()
732 	range_tree_vacate(svr->svr_frees[txg & TXG_MASK],  in vdev_mapping_sync()
734 	ASSERT3U(svr->svr_max_offset_to_sync[txg & TXG_MASK], >=,  in vdev_mapping_sync()
736 	svr->svr_max_offset_to_sync[txg & TXG_MASK] = 0;  in vdev_mapping_sync()
737 	mutex_exit(&svr->svr_lock);  in vdev_mapping_sync()
910 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in spa_vdev_copy_segment()  local
1026 	list_insert_tail(&svr->svr_new_segments[txg & TXG_MASK], entry);  in spa_vdev_copy_segment()
1041 	spa_vdev_removal_t *svr = arg;  in vdev_remove_complete_sync()  local
1043 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);  in vdev_remove_complete_sync()
1048 		ASSERT0(svr->svr_bytes_done[i]);  in vdev_remove_complete_sync()
1057 	ASSERT3P(svr->svr_zaplist, !=, NULL);  in vdev_remove_complete_sync()
1058 	for (nvpair_t *pair = nvlist_next_nvpair(svr->svr_zaplist, NULL);  in vdev_remove_complete_sync()
1060 	    pair = nvlist_next_nvpair(svr->svr_zaplist, pair)) {  in vdev_remove_complete_sync()
1063 	fnvlist_free(svr->svr_zaplist);  in vdev_remove_complete_sync()
1095 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in vdev_remove_replace_with_indirect()  local
1102 	svr->svr_zaplist = fnvlist_alloc();  in vdev_remove_replace_with_indirect()
1103 	vdev_remove_enlist_zaps(vd, svr->svr_zaplist);  in vdev_remove_replace_with_indirect()
1116 	dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_remove_complete_sync, svr,  in vdev_remove_replace_with_indirect()
1124 	mutex_enter(&svr->svr_lock);  in vdev_remove_replace_with_indirect()
1125 	svr->svr_thread = NULL;  in vdev_remove_replace_with_indirect()
1126 	cv_broadcast(&svr->svr_cv);  in vdev_remove_replace_with_indirect()
1127 	mutex_exit(&svr->svr_lock);  in vdev_remove_replace_with_indirect()
1210 spa_vdev_copy_impl(vdev_t *vd, spa_vdev_removal_t *svr, vdev_copy_arg_t *vca,  in spa_vdev_copy_impl()  argument
1216 	mutex_enter(&svr->svr_lock);  in spa_vdev_copy_impl()
1227 		range_tree_t *rt = svr->svr_allocd_segs;  in spa_vdev_copy_impl()
1262 		range_tree_remove(svr->svr_allocd_segs,  in spa_vdev_copy_impl()
1267 		mutex_exit(&svr->svr_lock);  in spa_vdev_copy_impl()
1272 	if (svr->svr_max_offset_to_sync[txg & TXG_MASK] == 0) {  in spa_vdev_copy_impl()
1274 		    svr, 0, ZFS_SPACE_CHECK_NONE, tx);  in spa_vdev_copy_impl()
1277 	svr->svr_max_offset_to_sync[txg & TXG_MASK] = range_tree_max(segs);  in spa_vdev_copy_impl()
1283 	svr->svr_bytes_done[txg & TXG_MASK] += range_tree_space(segs);  in spa_vdev_copy_impl()
1285 	mutex_exit(&svr->svr_lock);  in spa_vdev_copy_impl()
1350 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in spa_vdev_remove_thread()  local
1356 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);  in spa_vdev_remove_thread()
1370 	mutex_enter(&svr->svr_lock);  in spa_vdev_remove_thread()
1378 	    msi < vd->vdev_ms_count && !svr->svr_thread_exit; msi++) {  in spa_vdev_remove_thread()
1382 		ASSERT0(range_tree_space(svr->svr_allocd_segs));  in spa_vdev_remove_thread()
1405 			    svr->svr_allocd_segs, SM_ALLOC));  in spa_vdev_remove_thread()
1408 			    range_tree_add, svr->svr_allocd_segs);  in spa_vdev_remove_thread()
1410 			    range_tree_remove, svr->svr_allocd_segs);  in spa_vdev_remove_thread()
1412 			    range_tree_remove, svr->svr_allocd_segs);  in spa_vdev_remove_thread()
1419 			range_tree_clear(svr->svr_allocd_segs, 0, start_offset);  in spa_vdev_remove_thread()
1426 		    zfs_btree_numnodes(&svr->svr_allocd_segs->rt_root),  in spa_vdev_remove_thread()
1429 		while (!svr->svr_thread_exit &&  in spa_vdev_remove_thread()
1430 		    !range_tree_is_empty(svr->svr_allocd_segs)) {  in spa_vdev_remove_thread()
1432 			mutex_exit(&svr->svr_lock);  in spa_vdev_remove_thread()
1453 				bytes_copied += svr->svr_bytes_done[i];  in spa_vdev_remove_thread()
1455 			    !svr->svr_thread_exit)  in spa_vdev_remove_thread()
1477 			vd = vdev_lookup_top(spa, svr->svr_vdev_id);  in spa_vdev_remove_thread()
1483 			spa_vdev_copy_impl(vd, svr, &vca, &max_alloc, tx);  in spa_vdev_remove_thread()
1486 			mutex_enter(&svr->svr_lock);  in spa_vdev_remove_thread()
1490 	mutex_exit(&svr->svr_lock);  in spa_vdev_remove_thread()
1503 	if (svr->svr_thread_exit) {  in spa_vdev_remove_thread()
1504 		mutex_enter(&svr->svr_lock);  in spa_vdev_remove_thread()
1505 		range_tree_vacate(svr->svr_allocd_segs, NULL, NULL);  in spa_vdev_remove_thread()
1506 		svr->svr_thread = NULL;  in spa_vdev_remove_thread()
1507 		cv_broadcast(&svr->svr_cv);  in spa_vdev_remove_thread()
1508 		mutex_exit(&svr->svr_lock);  in spa_vdev_remove_thread()
1510 		ASSERT0(range_tree_space(svr->svr_allocd_segs));  in spa_vdev_remove_thread()
1518 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in spa_vdev_remove_suspend()  local
1520 	if (svr == NULL)  in spa_vdev_remove_suspend()
1523 	mutex_enter(&svr->svr_lock);  in spa_vdev_remove_suspend()
1524 	svr->svr_thread_exit = B_TRUE;  in spa_vdev_remove_suspend()
1525 	while (svr->svr_thread != NULL)  in spa_vdev_remove_suspend()
1526 		cv_wait(&svr->svr_cv, &svr->svr_lock);  in spa_vdev_remove_suspend()
1527 	svr->svr_thread_exit = B_FALSE;  in spa_vdev_remove_suspend()
1528 	mutex_exit(&svr->svr_lock);  in spa_vdev_remove_suspend()
1551 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in spa_vdev_remove_cancel_sync()  local
1552 	vdev_t *vd = vdev_lookup_top(spa, svr->svr_vdev_id);  in spa_vdev_remove_cancel_sync()
1557 	ASSERT3P(svr->svr_thread, ==, NULL);  in spa_vdev_remove_cancel_sync()
1579 		ASSERT(list_is_empty(&svr->svr_new_segments[i]));  in spa_vdev_remove_cancel_sync()
1580 		ASSERT3U(svr->svr_max_offset_to_sync[i], <=,  in spa_vdev_remove_cancel_sync()
1590 		ASSERT0(range_tree_space(svr->svr_allocd_segs));  in spa_vdev_remove_cancel_sync()
1604 			mutex_enter(&svr->svr_lock);  in spa_vdev_remove_cancel_sync()
1606 			    svr->svr_allocd_segs, SM_ALLOC));  in spa_vdev_remove_cancel_sync()
1609 			    range_tree_add, svr->svr_allocd_segs);  in spa_vdev_remove_cancel_sync()
1611 			    range_tree_remove, svr->svr_allocd_segs);  in spa_vdev_remove_cancel_sync()
1613 			    range_tree_remove, svr->svr_allocd_segs);  in spa_vdev_remove_cancel_sync()
1623 				range_tree_clear(svr->svr_allocd_segs,  in spa_vdev_remove_cancel_sync()
1626 			mutex_exit(&svr->svr_lock);  in spa_vdev_remove_cancel_sync()
1630 		mutex_enter(&svr->svr_lock);  in spa_vdev_remove_cancel_sync()
1631 		range_tree_vacate(svr->svr_allocd_segs,  in spa_vdev_remove_cancel_sync()
1633 		mutex_exit(&svr->svr_lock);  in spa_vdev_remove_cancel_sync()
1664 	svr->svr_bytes_done[dmu_tx_get_txg(tx) & TXG_MASK] = 0;  in spa_vdev_remove_cancel_sync()
1704 	spa_vdev_removal_t *svr = spa->spa_vdev_removal;  in svr_sync()  local
1707 	if (svr == NULL)  in svr_sync()
1716 	if (svr->svr_bytes_done[txgoff] == 0)  in svr_sync()
1722 	spa->spa_removing_phys.sr_copied += svr->svr_bytes_done[txgoff];  in svr_sync()
1723 	svr->svr_bytes_done[txgoff] = 0;  in svr_sync()