146 clear_rebuild_bytes(vdev_t *vd)  in clear_rebuild_bytes()  argument
148 	vdev_stat_t *vs = &vd->vdev_stat;  in clear_rebuild_bytes()
150 	for (uint64_t i = 0; i < vd->vdev_children; i++)  in clear_rebuild_bytes()
151 		clear_rebuild_bytes(vd->vdev_child[i]);  in clear_rebuild_bytes()
153 	mutex_enter(&vd->vdev_stat_lock);  in clear_rebuild_bytes()
155 	mutex_exit(&vd->vdev_stat_lock);  in clear_rebuild_bytes()
162 vdev_rebuild_should_stop(vdev_t *vd)  in vdev_rebuild_should_stop()  argument
164 	return (!vdev_writeable(vd) || vd->vdev_removing ||  in vdev_rebuild_should_stop()
165 	    vd->vdev_rebuild_exit_wanted ||  in vdev_rebuild_should_stop()
166 	    vd->vdev_rebuild_cancel_wanted ||  in vdev_rebuild_should_stop()
167 	    vd->vdev_rebuild_reset_wanted);  in vdev_rebuild_should_stop()
175 vdev_rebuild_should_cancel(vdev_t *vd)  in vdev_rebuild_should_cancel()  argument
177 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_should_cancel()
180 	if (!vdev_resilver_needed(vd, &vrp->vrp_min_txg, &vrp->vrp_max_txg))  in vdev_rebuild_should_cancel()
195 	vdev_t *vd = vdev_lookup_top(spa, vdev_id);  in vdev_rebuild_update_sync()  local
196 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_update_sync()
200 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_update_sync()
210 	VERIFY0(zap_update(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,  in vdev_rebuild_update_sync()
214 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_update_sync()
225 	vdev_t *vd = vdev_lookup_top(spa, vdev_id);  in vdev_rebuild_initiate_sync()  local
226 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_initiate_sync()
229 	ASSERT(vd->vdev_rebuilding);  in vdev_rebuild_initiate_sync()
231 	spa_feature_incr(vd->vdev_spa, SPA_FEATURE_DEVICE_REBUILD, tx);  in vdev_rebuild_initiate_sync()
233 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_initiate_sync()
249 	VERIFY(vdev_resilver_needed(vd, &vrp->vrp_min_txg, &vrp->vrp_max_txg));  in vdev_rebuild_initiate_sync()
251 	VERIFY0(zap_update(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,  in vdev_rebuild_initiate_sync()
257 	    (u_longlong_t)vd->vdev_id, (u_longlong_t)vd->vdev_guid);  in vdev_rebuild_initiate_sync()
259 	ASSERT0P(vd->vdev_rebuild_thread);  in vdev_rebuild_initiate_sync()
260 	vd->vdev_rebuild_thread = thread_create(NULL, 0,  in vdev_rebuild_initiate_sync()
261 	    vdev_rebuild_thread, vd, 0, &p0, TS_RUN, maxclsyspri);  in vdev_rebuild_initiate_sync()
263 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_initiate_sync()
267 vdev_rebuild_log_notify(spa_t *spa, vdev_t *vd, const char *name)  in vdev_rebuild_log_notify()  argument
272 	spa_event_notify(spa, vd, aux, name);  in vdev_rebuild_log_notify()
281 vdev_rebuild_initiate(vdev_t *vd)  in vdev_rebuild_initiate()  argument
283 	spa_t *spa = vd->vdev_spa;  in vdev_rebuild_initiate()
285 	ASSERT(vd->vdev_top == vd);  in vdev_rebuild_initiate()
286 	ASSERT(MUTEX_HELD(&vd->vdev_rebuild_lock));  in vdev_rebuild_initiate()
287 	ASSERT(!vd->vdev_rebuilding);  in vdev_rebuild_initiate()
292 	vd->vdev_rebuilding = B_TRUE;  in vdev_rebuild_initiate()
295 	    (void *)(uintptr_t)vd->vdev_id, tx);  in vdev_rebuild_initiate()
298 	vdev_rebuild_log_notify(spa, vd, ESC_ZFS_RESILVER_START);  in vdev_rebuild_initiate()
309 	vdev_t *vd = vdev_lookup_top(spa, vdev_id);  in vdev_rebuild_complete_sync()  local
310 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_complete_sync()
313 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_complete_sync()
319 	if (vd->vdev_rebuild_reset_wanted) {  in vdev_rebuild_complete_sync()
320 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_complete_sync()
328 	VERIFY0(zap_update(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,  in vdev_rebuild_complete_sync()
332 	vdev_dtl_reassess(vd, tx->tx_txg, vrp->vrp_max_txg, B_TRUE, B_TRUE);  in vdev_rebuild_complete_sync()
333 	spa_feature_decr(vd->vdev_spa, SPA_FEATURE_DEVICE_REBUILD, tx);  in vdev_rebuild_complete_sync()
337 	    (u_longlong_t)vd->vdev_id, (u_longlong_t)vd->vdev_guid);  in vdev_rebuild_complete_sync()
338 	vdev_rebuild_log_notify(spa, vd, ESC_ZFS_RESILVER_FINISH);  in vdev_rebuild_complete_sync()
342 	vd->vdev_rebuilding = B_FALSE;  in vdev_rebuild_complete_sync()
343 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_complete_sync()
359 	cv_broadcast(&vd->vdev_rebuild_cv);  in vdev_rebuild_complete_sync()
373 	vdev_t *vd = vdev_lookup_top(spa, vdev_id);  in vdev_rebuild_cancel_sync()  local
374 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_cancel_sync()
377 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_cancel_sync()
381 	VERIFY0(zap_update(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,  in vdev_rebuild_cancel_sync()
385 	spa_feature_decr(vd->vdev_spa, SPA_FEATURE_DEVICE_REBUILD, tx);  in vdev_rebuild_cancel_sync()
389 	    (u_longlong_t)vd->vdev_id, (u_longlong_t)vd->vdev_guid);  in vdev_rebuild_cancel_sync()
390 	vdev_rebuild_log_notify(spa, vd, ESC_ZFS_RESILVER_FINISH);  in vdev_rebuild_cancel_sync()
392 	vd->vdev_rebuild_cancel_wanted = B_FALSE;  in vdev_rebuild_cancel_sync()
393 	vd->vdev_rebuilding = B_FALSE;  in vdev_rebuild_cancel_sync()
394 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_cancel_sync()
397 	cv_broadcast(&vd->vdev_rebuild_cv);  in vdev_rebuild_cancel_sync()
409 	vdev_t *vd = vdev_lookup_top(spa, vdev_id);  in vdev_rebuild_reset_sync()  local
410 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_reset_sync()
413 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_reset_sync()
416 	ASSERT0P(vd->vdev_rebuild_thread);  in vdev_rebuild_reset_sync()
429 	VERIFY(vdev_resilver_needed(vd, &vrp->vrp_min_txg, &vrp->vrp_max_txg));  in vdev_rebuild_reset_sync()
431 	VERIFY0(zap_update(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap,  in vdev_rebuild_reset_sync()
437 	    (u_longlong_t)vd->vdev_id, (u_longlong_t)vd->vdev_guid);  in vdev_rebuild_reset_sync()
439 	vd->vdev_rebuild_reset_wanted = B_FALSE;  in vdev_rebuild_reset_sync()
440 	ASSERT(vd->vdev_rebuilding);  in vdev_rebuild_reset_sync()
442 	vd->vdev_rebuild_thread = thread_create(NULL, 0,  in vdev_rebuild_reset_sync()
443 	    vdev_rebuild_thread, vd, 0, &p0, TS_RUN, maxclsyspri);  in vdev_rebuild_reset_sync()
445 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_reset_sync()
456 	vdev_t *vd = vdev_lookup_top(spa, vdev_id);  in vdev_rebuild_clear_sync()  local
457 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_clear_sync()
461 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_clear_sync()
465 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_clear_sync()
469 	clear_rebuild_bytes(vd);  in vdev_rebuild_clear_sync()
472 	if (vd->vdev_top_zap != 0 && zap_contains(mos, vd->vdev_top_zap,  in vdev_rebuild_clear_sync()
474 		VERIFY0(zap_update(mos, vd->vdev_top_zap,  in vdev_rebuild_clear_sync()
479 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_clear_sync()
491 	vdev_t *vd = vr->vr_top_vdev;  in vdev_rebuild_cb()  local
494 	if (zio->io_error == ENXIO && !vdev_writeable(vd)) {  in vdev_rebuild_cb()
515 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_rebuild_cb()
523 vdev_rebuild_blkptr_init(blkptr_t *bp, vdev_t *vd, uint64_t start,  in vdev_rebuild_blkptr_init()  argument
526 	ASSERT(vd->vdev_ops == &vdev_draid_ops ||  in vdev_rebuild_blkptr_init()
527 	    vd->vdev_ops == &vdev_mirror_ops ||  in vdev_rebuild_blkptr_init()
528 	    vd->vdev_ops == &vdev_replacing_ops ||  in vdev_rebuild_blkptr_init()
529 	    vd->vdev_ops == &vdev_spare_ops);  in vdev_rebuild_blkptr_init()
531 	uint64_t psize = vd->vdev_ops == &vdev_draid_ops ?  in vdev_rebuild_blkptr_init()
532 	    vdev_draid_asize_to_psize(vd, asize, 0) : asize;  in vdev_rebuild_blkptr_init()
536 	DVA_SET_VDEV(&bp->blk_dva[0], vd->vdev_id);  in vdev_rebuild_blkptr_init()
561 	vdev_t *vd = vr->vr_top_vdev;  in vdev_rebuild_range()  local
562 	spa_t *spa = vd->vdev_spa;  in vdev_rebuild_range()
565 	ASSERT3U(ms_id, ==, start >> vd->vdev_ms_shift);  in vdev_rebuild_range()
566 	ASSERT3U(ms_id, ==, (start + size - 1) >> vd->vdev_ms_shift);  in vdev_rebuild_range()
577 	vdev_rebuild_blkptr_init(&blk, vd, start, size);  in vdev_rebuild_range()
580 	if (!vdev_dtl_need_resilver(vd, &blk.blk_dva[0], psize, TXG_UNKNOWN)) {  in vdev_rebuild_range()
598 	spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);  in vdev_rebuild_range()
599 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_range()
606 		    (void *)(uintptr_t)vd->vdev_id, tx);  in vdev_rebuild_range()
610 	if (vdev_rebuild_should_stop(vd)) {  in vdev_rebuild_range()
614 		spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_rebuild_range()
615 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_range()
619 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_range()
640 	vdev_t *vd = vr->vr_top_vdev;  in vdev_rebuild_ranges()  local
655 		    !vdev_rebuild_should_stop(vd)) {  in vdev_rebuild_ranges()
667 			ASSERT3P(vd->vdev_ops, !=, NULL);  in vdev_rebuild_ranges()
668 			chunk_size = vd->vdev_ops->vdev_op_rebuild_asize(vd,  in vdev_rebuild_ranges()
690 vdev_rebuild_update_bytes_est(vdev_t *vd, uint64_t ms_id)  in vdev_rebuild_update_bytes_est()  argument
692 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_update_bytes_est()
696 	if (vrp->vrp_last_offset < vd->vdev_ms[ms_id]->ms_start)  in vdev_rebuild_update_bytes_est()
699 	for (uint64_t i = ms_id; i < vd->vdev_ms_count; i++) {  in vdev_rebuild_update_bytes_est()
700 		metaslab_t *msp = vd->vdev_ms[i];  in vdev_rebuild_update_bytes_est()
714 vdev_rebuild_load(vdev_t *vd)  in vdev_rebuild_load()  argument
716 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_load()
718 	spa_t *spa = vd->vdev_spa;  in vdev_rebuild_load()
721 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_load()
722 	vd->vdev_rebuilding = B_FALSE;  in vdev_rebuild_load()
726 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_load()
730 	ASSERT(vd->vdev_top == vd);  in vdev_rebuild_load()
732 	err = zap_lookup(spa->spa_meta_objset, vd->vdev_top_zap,  in vdev_rebuild_load()
744 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_load()
749 	vr->vr_top_vdev = vd;  in vdev_rebuild_load()
751 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_load()
763 	vdev_t *vd = arg;  in vdev_rebuild_thread()  local
764 	spa_t *spa = vd->vdev_spa;  in vdev_rebuild_thread()
778 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_thread()
780 	ASSERT3P(vd->vdev_top, ==, vd);  in vdev_rebuild_thread()
781 	ASSERT3P(vd->vdev_rebuild_thread, !=, NULL);  in vdev_rebuild_thread()
782 	ASSERT(vd->vdev_rebuilding);  in vdev_rebuild_thread()
784 	ASSERT3B(vd->vdev_rebuild_cancel_wanted, ==, B_FALSE);  in vdev_rebuild_thread()
786 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_thread()
788 	vr->vr_top_vdev = vd;  in vdev_rebuild_thread()
792 	    ZFS_RT_F_DYN_NAME, vdev_rt_name(vd, "vr_scan_tree"));  in vdev_rebuild_thread()
802 	vdev_rebuild_update_bytes_est(vd, 0);  in vdev_rebuild_thread()
806 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_thread()
812 	for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {  in vdev_rebuild_thread()
813 		metaslab_t *msp = vd->vdev_ms[i];  in vdev_rebuild_thread()
825 		    zfs_rebuild_vdev_limit * vd->vdev_children));  in vdev_rebuild_thread()
833 		if (vdev_rebuild_should_cancel(vd)) {  in vdev_rebuild_thread()
834 			vd->vdev_rebuild_cancel_wanted = B_TRUE;  in vdev_rebuild_thread()
903 			vdev_rebuild_update_bytes_est(vd, i);  in vdev_rebuild_thread()
938 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_thread()
949 		    (void *)(uintptr_t)vd->vdev_id, tx);  in vdev_rebuild_thread()
950 	} else if (vd->vdev_rebuild_cancel_wanted) {  in vdev_rebuild_thread()
956 		    (void *)(uintptr_t)vd->vdev_id, tx);  in vdev_rebuild_thread()
957 	} else if (vd->vdev_rebuild_reset_wanted) {  in vdev_rebuild_thread()
964 		    (void *)(uintptr_t)vd->vdev_id, tx);  in vdev_rebuild_thread()
972 		vd->vdev_rebuilding = B_FALSE;  in vdev_rebuild_thread()
977 	vd->vdev_rebuild_thread = NULL;  in vdev_rebuild_thread()
978 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_thread()
981 	cv_broadcast(&vd->vdev_rebuild_cv);  in vdev_rebuild_thread()
990 vdev_rebuild_active(vdev_t *vd)  in vdev_rebuild_active()  argument
992 	spa_t *spa = vd->vdev_spa;  in vdev_rebuild_active()
995 	if (vd == spa->spa_root_vdev) {  in vdev_rebuild_active()
996 		for (uint64_t i = 0; i < vd->vdev_children; i++) {  in vdev_rebuild_active()
997 			ret = vdev_rebuild_active(vd->vdev_child[i]);  in vdev_rebuild_active()
1001 	} else if (vd->vdev_top_zap != 0) {  in vdev_rebuild_active()
1002 		vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_active()
1005 		mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_active()
1007 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_active()
1018 vdev_rebuild(vdev_t *vd)  in vdev_rebuild()  argument
1020 	vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild()
1023 	ASSERT(vd->vdev_top == vd);  in vdev_rebuild()
1024 	ASSERT(vdev_is_concrete(vd));  in vdev_rebuild()
1025 	ASSERT(!vd->vdev_removing);  in vdev_rebuild()
1026 	ASSERT(spa_feature_is_enabled(vd->vdev_spa,  in vdev_rebuild()
1029 	mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild()
1030 	if (vd->vdev_rebuilding) {  in vdev_rebuild()
1039 		if (!vd->vdev_rebuild_reset_wanted)  in vdev_rebuild()
1040 			vd->vdev_rebuild_reset_wanted = B_TRUE;  in vdev_rebuild()
1042 		vdev_rebuild_initiate(vd);  in vdev_rebuild()
1044 	mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild()
1048 vdev_rebuild_restart_impl(vdev_t *vd)  in vdev_rebuild_restart_impl()  argument
1050 	spa_t *spa = vd->vdev_spa;  in vdev_rebuild_restart_impl()
1052 	if (vd == spa->spa_root_vdev) {  in vdev_rebuild_restart_impl()
1053 		for (uint64_t i = 0; i < vd->vdev_children; i++)  in vdev_rebuild_restart_impl()
1054 			vdev_rebuild_restart_impl(vd->vdev_child[i]);  in vdev_rebuild_restart_impl()
1056 	} else if (vd->vdev_top_zap != 0) {  in vdev_rebuild_restart_impl()
1057 		vdev_rebuild_t *vr = &vd->vdev_rebuild_config;  in vdev_rebuild_restart_impl()
1060 		mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_restart_impl()
1062 		    vdev_writeable(vd) && !vd->vdev_rebuilding) {  in vdev_rebuild_restart_impl()
1065 			vd->vdev_rebuilding = B_TRUE;  in vdev_rebuild_restart_impl()
1066 			vd->vdev_rebuild_thread = thread_create(NULL, 0,  in vdev_rebuild_restart_impl()
1067 			    vdev_rebuild_thread, vd, 0, &p0, TS_RUN,  in vdev_rebuild_restart_impl()
1070 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_restart_impl()
1093 vdev_rebuild_stop_wait(vdev_t *vd)  in vdev_rebuild_stop_wait()  argument
1095 	spa_t *spa = vd->vdev_spa;  in vdev_rebuild_stop_wait()
1100 	if (vd == spa->spa_root_vdev) {  in vdev_rebuild_stop_wait()
1101 		for (uint64_t i = 0; i < vd->vdev_children; i++)  in vdev_rebuild_stop_wait()
1102 			vdev_rebuild_stop_wait(vd->vdev_child[i]);  in vdev_rebuild_stop_wait()
1104 	} else if (vd->vdev_top_zap != 0) {  in vdev_rebuild_stop_wait()
1105 		ASSERT(vd == vd->vdev_top);  in vdev_rebuild_stop_wait()
1107 		mutex_enter(&vd->vdev_rebuild_lock);  in vdev_rebuild_stop_wait()
1108 		if (vd->vdev_rebuild_thread != NULL) {  in vdev_rebuild_stop_wait()
1109 			vd->vdev_rebuild_exit_wanted = B_TRUE;  in vdev_rebuild_stop_wait()
1110 			while (vd->vdev_rebuilding) {  in vdev_rebuild_stop_wait()
1111 				cv_wait(&vd->vdev_rebuild_cv,  in vdev_rebuild_stop_wait()
1112 				    &vd->vdev_rebuild_lock);  in vdev_rebuild_stop_wait()
1114 			vd->vdev_rebuild_exit_wanted = B_FALSE;  in vdev_rebuild_stop_wait()
1116 		mutex_exit(&vd->vdev_rebuild_lock);  in vdev_rebuild_stop_wait()