170 vdev_trim_should_stop(vdev_t *vd)  in vdev_trim_should_stop()  argument
172 	return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||  in vdev_trim_should_stop()
173 	    vd->vdev_detached || vd->vdev_top->vdev_removing ||  in vdev_trim_should_stop()
174 	    vd->vdev_top->vdev_rz_expanding);  in vdev_trim_should_stop()
194 vdev_autotrim_wait_kick(vdev_t *vd, int num_of_kick)  in vdev_autotrim_wait_kick()  argument
196 	mutex_enter(&vd->vdev_autotrim_lock);  in vdev_autotrim_wait_kick()
198 		if (vd->vdev_autotrim_exit_wanted)  in vdev_autotrim_wait_kick()
200 		cv_wait_idle(&vd->vdev_autotrim_kick_cv,  in vdev_autotrim_wait_kick()
201 		    &vd->vdev_autotrim_lock);  in vdev_autotrim_wait_kick()
203 	boolean_t exit_wanted = vd->vdev_autotrim_exit_wanted;  in vdev_autotrim_wait_kick()
204 	mutex_exit(&vd->vdev_autotrim_lock);  in vdev_autotrim_wait_kick()
228 	vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);  in vdev_trim_zap_update_sync()  local
229 	if (vd == NULL || vd->vdev_top->vdev_removing ||  in vdev_trim_zap_update_sync()
230 	    !vdev_is_concrete(vd) || vd->vdev_top->vdev_rz_expanding)  in vdev_trim_zap_update_sync()
233 	uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];  in vdev_trim_zap_update_sync()
234 	vd->vdev_trim_offset[txg & TXG_MASK] = 0;  in vdev_trim_zap_update_sync()
236 	VERIFY3U(vd->vdev_leaf_zap, !=, 0);  in vdev_trim_zap_update_sync()
238 	objset_t *mos = vd->vdev_spa->spa_meta_objset;  in vdev_trim_zap_update_sync()
240 	if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {  in vdev_trim_zap_update_sync()
242 		if (vd->vdev_trim_last_offset == UINT64_MAX)  in vdev_trim_zap_update_sync()
245 		vd->vdev_trim_last_offset = last_offset;  in vdev_trim_zap_update_sync()
246 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,  in vdev_trim_zap_update_sync()
251 	if (vd->vdev_trim_action_time > 0) {  in vdev_trim_zap_update_sync()
252 		uint64_t val = (uint64_t)vd->vdev_trim_action_time;  in vdev_trim_zap_update_sync()
253 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,  in vdev_trim_zap_update_sync()
258 	if (vd->vdev_trim_rate > 0) {  in vdev_trim_zap_update_sync()
259 		uint64_t rate = (uint64_t)vd->vdev_trim_rate;  in vdev_trim_zap_update_sync()
264 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,  in vdev_trim_zap_update_sync()
268 	uint64_t partial = vd->vdev_trim_partial;  in vdev_trim_zap_update_sync()
272 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,  in vdev_trim_zap_update_sync()
275 	uint64_t secure = vd->vdev_trim_secure;  in vdev_trim_zap_update_sync()
279 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,  in vdev_trim_zap_update_sync()
283 	uint64_t trim_state = vd->vdev_trim_state;  in vdev_trim_zap_update_sync()
284 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,  in vdev_trim_zap_update_sync()
294 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state,  in vdev_trim_change_state()  argument
297 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim_change_state()
298 	spa_t *spa = vd->vdev_spa;  in vdev_trim_change_state()
300 	if (new_state == vd->vdev_trim_state)  in vdev_trim_change_state()
307 	*guid = vd->vdev_guid;  in vdev_trim_change_state()
312 	if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {  in vdev_trim_change_state()
313 		vd->vdev_trim_action_time = gethrestime_sec();  in vdev_trim_change_state()
322 		if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||  in vdev_trim_change_state()
323 		    vd->vdev_trim_state == VDEV_TRIM_CANCELED) {  in vdev_trim_change_state()
324 			vd->vdev_trim_last_offset = UINT64_MAX;  in vdev_trim_change_state()
325 			vd->vdev_trim_rate = UINT64_MAX;  in vdev_trim_change_state()
326 			vd->vdev_trim_partial = UINT64_MAX;  in vdev_trim_change_state()
327 			vd->vdev_trim_secure = UINT64_MAX;  in vdev_trim_change_state()
331 			vd->vdev_trim_rate = rate;  in vdev_trim_change_state()
334 			vd->vdev_trim_partial = partial;  in vdev_trim_change_state()
337 			vd->vdev_trim_secure = secure;  in vdev_trim_change_state()
340 	vdev_trim_state_t old_state = vd->vdev_trim_state;  in vdev_trim_change_state()
342 	vd->vdev_trim_state = new_state;  in vdev_trim_change_state()
351 		spa_event_notify(spa, vd, NULL,  in vdev_trim_change_state()
354 		    "vdev=%s activated", vd->vdev_path);  in vdev_trim_change_state()
357 		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND);  in vdev_trim_change_state()
359 		    "vdev=%s suspended", vd->vdev_path);  in vdev_trim_change_state()
364 			spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL);  in vdev_trim_change_state()
366 			    "vdev=%s canceled", vd->vdev_path);  in vdev_trim_change_state()
370 		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH);  in vdev_trim_change_state()
372 		    "vdev=%s complete", vd->vdev_path);  in vdev_trim_change_state()
392 	vdev_t *vd = zio->io_vd;  in vdev_trim_cb()  local
394 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_cb()
395 	if (zio->io_error == ENXIO && !vdev_writeable(vd)) {  in vdev_trim_cb()
402 		    &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];  in vdev_trim_cb()
406 			vd->vdev_stat.vs_trim_errors++;  in vdev_trim_cb()
407 			spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,  in vdev_trim_cb()
410 			spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,  in vdev_trim_cb()
414 		vd->vdev_trim_bytes_done += zio->io_orig_size;  in vdev_trim_cb()
417 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);  in vdev_trim_cb()
418 	vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;  in vdev_trim_cb()
419 	cv_broadcast(&vd->vdev_trim_io_cv);  in vdev_trim_cb()
420 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_cb()
422 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_trim_cb()
434 	vdev_t *vd = zio->io_vd;  in vdev_autotrim_cb()  local
436 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_autotrim_cb()
439 		vd->vdev_stat.vs_trim_errors++;  in vdev_autotrim_cb()
440 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,  in vdev_autotrim_cb()
443 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,  in vdev_autotrim_cb()
447 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);  in vdev_autotrim_cb()
448 	vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;  in vdev_autotrim_cb()
449 	cv_broadcast(&vd->vdev_trim_io_cv);  in vdev_autotrim_cb()
450 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_autotrim_cb()
452 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_autotrim_cb()
464 	vdev_t *vd = zio->io_vd;  in vdev_trim_simple_cb()  local
466 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_simple_cb()
469 		vd->vdev_stat.vs_trim_errors++;  in vdev_trim_simple_cb()
470 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,  in vdev_trim_simple_cb()
473 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,  in vdev_trim_simple_cb()
477 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0);  in vdev_trim_simple_cb()
478 	vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--;  in vdev_trim_simple_cb()
479 	cv_broadcast(&vd->vdev_trim_io_cv);  in vdev_trim_simple_cb()
480 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_simple_cb()
482 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_trim_simple_cb()
501 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_range()  local
502 	spa_t *spa = vd->vdev_spa;  in vdev_trim_range()
505 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_range()
512 		while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&  in vdev_trim_range()
513 		    vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {  in vdev_trim_range()
514 			cv_timedwait_idle(&vd->vdev_trim_io_cv,  in vdev_trim_range()
515 			    &vd->vdev_trim_io_lock, ddi_get_lbolt() +  in vdev_trim_range()
522 	while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] +  in vdev_trim_range()
523 	    vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) {  in vdev_trim_range()
524 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_range()
526 	vd->vdev_trim_inflight[ta->trim_type]++;  in vdev_trim_range()
527 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_range()
533 	spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);  in vdev_trim_range()
534 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_range()
537 	    vd->vdev_trim_offset[txg & TXG_MASK] == 0) {  in vdev_trim_range()
539 		*guid = vd->vdev_guid;  in vdev_trim_range()
551 	    vdev_trim_should_stop(vd)) ||  in vdev_trim_range()
553 	    vdev_autotrim_should_stop(vd->vdev_top))) {  in vdev_trim_range()
554 		mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_range()
555 		vd->vdev_trim_inflight[ta->trim_type]--;  in vdev_trim_range()
556 		mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_range()
557 		spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_trim_range()
558 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_range()
562 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_range()
565 		vd->vdev_trim_offset[txg & TXG_MASK] = start + size;  in vdev_trim_range()
575 	zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,  in vdev_trim_range()
594 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_ranges()  local
599 	spa_t *spa = vd->vdev_spa;  in vdev_trim_ranges()
639 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_ranges()
640 	while (vd->vdev_trim_inflight[0] > 0) {  in vdev_trim_ranges()
641 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_ranges()
643 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_ranges()
660 	vdev_t *vd = (vdev_t *)arg;  in vdev_trim_xlate_progress()  local
663 	vd->vdev_trim_bytes_est += size;  in vdev_trim_xlate_progress()
665 	if (vd->vdev_trim_last_offset >= physical_rs->rs_end) {  in vdev_trim_xlate_progress()
666 		vd->vdev_trim_bytes_done += size;  in vdev_trim_xlate_progress()
667 	} else if (vd->vdev_trim_last_offset > physical_rs->rs_start &&  in vdev_trim_xlate_progress()
668 	    vd->vdev_trim_last_offset <= physical_rs->rs_end) {  in vdev_trim_xlate_progress()
669 		vd->vdev_trim_bytes_done +=  in vdev_trim_xlate_progress()
670 		    vd->vdev_trim_last_offset - physical_rs->rs_start;  in vdev_trim_xlate_progress()
678 vdev_trim_calculate_progress(vdev_t *vd)  in vdev_trim_calculate_progress()  argument
680 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||  in vdev_trim_calculate_progress()
681 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));  in vdev_trim_calculate_progress()
682 	ASSERT(vd->vdev_leaf_zap != 0);  in vdev_trim_calculate_progress()
684 	vd->vdev_trim_bytes_est = 0;  in vdev_trim_calculate_progress()
685 	vd->vdev_trim_bytes_done = 0;  in vdev_trim_calculate_progress()
687 	for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {  in vdev_trim_calculate_progress()
688 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];  in vdev_trim_calculate_progress()
693 		    vdev_get_ndisks(vd->vdev_top);  in vdev_trim_calculate_progress()
705 		vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs);  in vdev_trim_calculate_progress()
706 		if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {  in vdev_trim_calculate_progress()
707 			vd->vdev_trim_bytes_est += ms_free;  in vdev_trim_calculate_progress()
715 			vdev_xlate_walk(vd, &remain_rs,  in vdev_trim_calculate_progress()
719 		if (vd->vdev_trim_last_offset > last_rs_end) {  in vdev_trim_calculate_progress()
720 			vd->vdev_trim_bytes_done += ms_free;  in vdev_trim_calculate_progress()
721 			vd->vdev_trim_bytes_est += ms_free;  in vdev_trim_calculate_progress()
741 			vdev_xlate_walk(vd, &logical_rs,  in vdev_trim_calculate_progress()
742 			    vdev_trim_xlate_progress, vd);  in vdev_trim_calculate_progress()
753 vdev_trim_load(vdev_t *vd)  in vdev_trim_load()  argument
756 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||  in vdev_trim_load()
757 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));  in vdev_trim_load()
758 	ASSERT(vd->vdev_leaf_zap != 0);  in vdev_trim_load()
760 	if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||  in vdev_trim_load()
761 	    vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {  in vdev_trim_load()
762 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
763 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,  in vdev_trim_load()
764 		    sizeof (vd->vdev_trim_last_offset), 1,  in vdev_trim_load()
765 		    &vd->vdev_trim_last_offset);  in vdev_trim_load()
767 			vd->vdev_trim_last_offset = 0;  in vdev_trim_load()
772 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
773 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,  in vdev_trim_load()
774 			    sizeof (vd->vdev_trim_rate), 1,  in vdev_trim_load()
775 			    &vd->vdev_trim_rate);  in vdev_trim_load()
777 				vd->vdev_trim_rate = 0;  in vdev_trim_load()
783 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
784 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,  in vdev_trim_load()
785 			    sizeof (vd->vdev_trim_partial), 1,  in vdev_trim_load()
786 			    &vd->vdev_trim_partial);  in vdev_trim_load()
788 				vd->vdev_trim_partial = 0;  in vdev_trim_load()
794 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
795 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,  in vdev_trim_load()
796 			    sizeof (vd->vdev_trim_secure), 1,  in vdev_trim_load()
797 			    &vd->vdev_trim_secure);  in vdev_trim_load()
799 				vd->vdev_trim_secure = 0;  in vdev_trim_load()
805 	vdev_trim_calculate_progress(vd);  in vdev_trim_load()
814 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_xlate_range_add()  local
823 		if (physical_rs->rs_end <= vd->vdev_trim_last_offset)  in vdev_trim_xlate_range_add()
827 		if (vd->vdev_trim_last_offset > physical_rs->rs_start) {  in vdev_trim_xlate_range_add()
829 			    vd->vdev_trim_last_offset);  in vdev_trim_xlate_range_add()
830 			physical_rs->rs_start = vd->vdev_trim_last_offset;  in vdev_trim_xlate_range_add()
848 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_range_add()  local
866 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_range_add()
867 	vdev_xlate_walk(vd, &logical_rs, vdev_trim_xlate_range_add, arg);  in vdev_trim_range_add()
880 	vdev_t *vd = arg;  in vdev_trim_thread()  local
881 	spa_t *spa = vd->vdev_spa;  in vdev_trim_thread()
890 	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);  in vdev_trim_thread()
892 	ASSERT(vdev_is_concrete(vd));  in vdev_trim_thread()
895 	vd->vdev_trim_last_offset = 0;  in vdev_trim_thread()
896 	vd->vdev_trim_rate = 0;  in vdev_trim_thread()
897 	vd->vdev_trim_partial = 0;  in vdev_trim_thread()
898 	vd->vdev_trim_secure = 0;  in vdev_trim_thread()
900 	VERIFY0(vdev_trim_load(vd));  in vdev_trim_thread()
902 	ta.trim_vdev = vd;  in vdev_trim_thread()
914 	if (vd->vdev_trim_secure) {  in vdev_trim_thread()
920 	for (uint64_t i = 0; !vd->vdev_detached &&  in vdev_trim_thread()
921 	    i < vd->vdev_top->vdev_ms_count; i++) {  in vdev_trim_thread()
922 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];  in vdev_trim_thread()
928 		if (vd->vdev_top->vdev_ms_count != ms_count) {  in vdev_trim_thread()
929 			vdev_trim_calculate_progress(vd);  in vdev_trim_thread()
930 			ms_count = vd->vdev_top->vdev_ms_count;  in vdev_trim_thread()
942 		if (msp->ms_sm == NULL && vd->vdev_trim_partial) {  in vdev_trim_thread()
946 			vdev_trim_calculate_progress(vd);  in vdev_trim_thread()
969 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_thread()
970 	if (!vd->vdev_trim_exit_wanted) {  in vdev_trim_thread()
971 		if (vdev_writeable(vd)) {  in vdev_trim_thread()
972 			vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,  in vdev_trim_thread()
973 			    vd->vdev_trim_rate, vd->vdev_trim_partial,  in vdev_trim_thread()
974 			    vd->vdev_trim_secure);  in vdev_trim_thread()
975 		} else if (vd->vdev_faulted) {  in vdev_trim_thread()
976 			vdev_trim_change_state(vd, VDEV_TRIM_CANCELED,  in vdev_trim_thread()
977 			    vd->vdev_trim_rate, vd->vdev_trim_partial,  in vdev_trim_thread()
978 			    vd->vdev_trim_secure);  in vdev_trim_thread()
981 	ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);  in vdev_trim_thread()
990 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_thread()
992 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_thread()
994 	vd->vdev_trim_thread = NULL;  in vdev_trim_thread()
995 	cv_broadcast(&vd->vdev_trim_cv);  in vdev_trim_thread()
996 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_thread()
1006 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)  in vdev_trim()  argument
1008 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim()
1009 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim()
1010 	ASSERT(vdev_is_concrete(vd));  in vdev_trim()
1011 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);  in vdev_trim()
1012 	ASSERT(!vd->vdev_detached);  in vdev_trim()
1013 	ASSERT(!vd->vdev_trim_exit_wanted);  in vdev_trim()
1014 	ASSERT(!vd->vdev_top->vdev_removing);  in vdev_trim()
1015 	ASSERT(!vd->vdev_rz_expanding);  in vdev_trim()
1017 	vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure);  in vdev_trim()
1018 	vd->vdev_trim_thread = thread_create(NULL, 0,  in vdev_trim()
1019 	    vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri);  in vdev_trim()
1026 vdev_trim_stop_wait_impl(vdev_t *vd)  in vdev_trim_stop_wait_impl()  argument
1028 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim_stop_wait_impl()
1030 	while (vd->vdev_trim_thread != NULL)  in vdev_trim_stop_wait_impl()
1031 		cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);  in vdev_trim_stop_wait_impl()
1033 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);  in vdev_trim_stop_wait_impl()
1034 	vd->vdev_trim_exit_wanted = B_FALSE;  in vdev_trim_stop_wait_impl()
1044 	vdev_t *vd;  in vdev_trim_stop_wait()  local
1049 	while ((vd = list_remove_head(vd_list)) != NULL) {  in vdev_trim_stop_wait()
1050 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_stop_wait()
1051 		vdev_trim_stop_wait_impl(vd);  in vdev_trim_stop_wait()
1052 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_stop_wait()
1066 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list)  in vdev_trim_stop()  argument
1068 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));  in vdev_trim_stop()
1069 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim_stop()
1070 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_stop()
1071 	ASSERT(vdev_is_concrete(vd));  in vdev_trim_stop()
1077 	if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)  in vdev_trim_stop()
1080 	vdev_trim_change_state(vd, tgt_state, 0, 0, 0);  in vdev_trim_stop()
1081 	vd->vdev_trim_exit_wanted = B_TRUE;  in vdev_trim_stop()
1084 		vdev_trim_stop_wait_impl(vd);  in vdev_trim_stop()
1087 		    vd->vdev_spa->spa_export_thread == curthread);  in vdev_trim_stop()
1088 		list_insert_tail(vd_list, vd);  in vdev_trim_stop()
1096 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state,  in vdev_trim_stop_all_impl()  argument
1099 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {  in vdev_trim_stop_all_impl()
1100 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_stop_all_impl()
1101 		vdev_trim_stop(vd, tgt_state, vd_list);  in vdev_trim_stop_all_impl()
1102 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_stop_all_impl()
1106 	for (uint64_t i = 0; i < vd->vdev_children; i++) {  in vdev_trim_stop_all_impl()
1107 		vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,  in vdev_trim_stop_all_impl()
1117 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state)  in vdev_trim_stop_all()  argument
1119 	spa_t *spa = vd->vdev_spa;  in vdev_trim_stop_all()
1129 	vdev_trim_stop_all_impl(vd, tgt_state, &vd_list);  in vdev_trim_stop_all()
1143 	if (vd->vdev_spa->spa_sync_on) {  in vdev_trim_stop_all()
1145 		txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);  in vdev_trim_stop_all()
1155 vdev_trim_restart(vdev_t *vd)  in vdev_trim_restart()  argument
1158 	    vd->vdev_spa->spa_load_thread == curthread);  in vdev_trim_restart()
1159 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));  in vdev_trim_restart()
1161 	if (vd->vdev_leaf_zap != 0) {  in vdev_trim_restart()
1162 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_restart()
1164 		int err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_restart()
1165 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,  in vdev_trim_restart()
1168 		vd->vdev_trim_state = trim_state;  in vdev_trim_restart()
1171 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_restart()
1172 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,  in vdev_trim_restart()
1175 		vd->vdev_trim_action_time = timestamp;  in vdev_trim_restart()
1177 		if ((vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||  in vdev_trim_restart()
1178 		    vd->vdev_offline) && !vd->vdev_top->vdev_rz_expanding) {  in vdev_trim_restart()
1180 			VERIFY0(vdev_trim_load(vd));  in vdev_trim_restart()
1181 		} else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&  in vdev_trim_restart()
1182 		    vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&  in vdev_trim_restart()
1183 		    !vd->vdev_top->vdev_rz_expanding &&  in vdev_trim_restart()
1184 		    vd->vdev_trim_thread == NULL) {  in vdev_trim_restart()
1185 			VERIFY0(vdev_trim_load(vd));  in vdev_trim_restart()
1186 			vdev_trim(vd, vd->vdev_trim_rate,  in vdev_trim_restart()
1187 			    vd->vdev_trim_partial, vd->vdev_trim_secure);  in vdev_trim_restart()
1190 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_restart()
1193 	for (uint64_t i = 0; i < vd->vdev_children; i++) {  in vdev_trim_restart()
1194 		vdev_trim_restart(vd->vdev_child[i]);  in vdev_trim_restart()
1223 	vdev_t *vd = arg;  in vdev_autotrim_thread()  local
1224 	spa_t *spa = vd->vdev_spa;  in vdev_autotrim_thread()
1227 	mutex_enter(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1228 	ASSERT3P(vd->vdev_top, ==, vd);  in vdev_autotrim_thread()
1229 	ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);  in vdev_autotrim_thread()
1230 	mutex_exit(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1233 	while (!vdev_autotrim_should_stop(vd)) {  in vdev_autotrim_thread()
1264 		for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;  in vdev_autotrim_thread()
1266 			metaslab_t *msp = vd->vdev_ms[i];  in vdev_autotrim_thread()
1321 			uint64_t children = vd->vdev_children;  in vdev_autotrim_thread()
1326 				tap[0].trim_vdev = vd;  in vdev_autotrim_thread()
1332 					tap[c].trim_vdev = vd->vdev_child[c];  in vdev_autotrim_thread()
1427 				wait_aborted = vdev_autotrim_wait_kick(vd,  in vdev_autotrim_thread()
1447 			if (vdev_autotrim_should_stop(vd))  in vdev_autotrim_thread()
1453 		vdev_autotrim_wait_kick(vd, 1);  in vdev_autotrim_thread()
1459 	for (uint64_t c = 0; c < vd->vdev_children; c++) {  in vdev_autotrim_thread()
1460 		vdev_t *cvd = vd->vdev_child[c];  in vdev_autotrim_thread()
1477 		for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {  in vdev_autotrim_thread()
1478 			metaslab_t *msp = vd->vdev_ms[i];  in vdev_autotrim_thread()
1486 	mutex_enter(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1487 	ASSERT(vd->vdev_autotrim_thread != NULL);  in vdev_autotrim_thread()
1488 	vd->vdev_autotrim_thread = NULL;  in vdev_autotrim_thread()
1489 	cv_broadcast(&vd->vdev_autotrim_cv);  in vdev_autotrim_thread()
1490 	mutex_exit(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1588 	vdev_t		*vd = arg;  in vdev_trim_l2arc_thread()  local
1589 	spa_t		*spa = vd->vdev_spa;  in vdev_trim_l2arc_thread()
1590 	l2arc_dev_t	*dev = l2arc_vdev_get(vd);  in vdev_trim_l2arc_thread()
1594 	ASSERT(vdev_is_concrete(vd));  in vdev_trim_l2arc_thread()
1597 	vd->vdev_trim_last_offset = 0;  in vdev_trim_l2arc_thread()
1598 	vd->vdev_trim_rate = 0;  in vdev_trim_l2arc_thread()
1599 	vd->vdev_trim_partial = 0;  in vdev_trim_l2arc_thread()
1600 	vd->vdev_trim_secure = 0;  in vdev_trim_l2arc_thread()
1602 	ta.trim_vdev = vd;  in vdev_trim_l2arc_thread()
1609 	physical_rs.rs_start = vd->vdev_trim_bytes_done = 0;  in vdev_trim_l2arc_thread()
1610 	physical_rs.rs_end = vd->vdev_trim_bytes_est =  in vdev_trim_l2arc_thread()
1611 	    vdev_get_min_asize(vd);  in vdev_trim_l2arc_thread()
1616 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1617 	vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);  in vdev_trim_l2arc_thread()
1618 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1623 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_l2arc_thread()
1624 	while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) {  in vdev_trim_l2arc_thread()
1625 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_l2arc_thread()
1627 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_l2arc_thread()
1632 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1633 	if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {  in vdev_trim_l2arc_thread()
1634 		vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,  in vdev_trim_l2arc_thread()
1635 		    vd->vdev_trim_rate, vd->vdev_trim_partial,  in vdev_trim_l2arc_thread()
1636 		    vd->vdev_trim_secure);  in vdev_trim_l2arc_thread()
1638 	ASSERT(vd->vdev_trim_thread != NULL ||  in vdev_trim_l2arc_thread()
1639 	    vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0);  in vdev_trim_l2arc_thread()
1649 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1650 	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);  in vdev_trim_l2arc_thread()
1651 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1659 	spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd,  in vdev_trim_l2arc_thread()
1663 	spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd);  in vdev_trim_l2arc_thread()
1665 	vd->vdev_trim_thread = NULL;  in vdev_trim_l2arc_thread()
1666 	if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE)  in vdev_trim_l2arc_thread()
1669 	cv_broadcast(&vd->vdev_trim_cv);  in vdev_trim_l2arc_thread()
1670 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1690 		vdev_t *vd = spa->spa_l2cache.sav_vdevs[i];  in vdev_trim_l2arc()  local
1691 		l2arc_dev_t *dev = l2arc_vdev_get(vd);  in vdev_trim_l2arc()
1704 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc()
1705 		ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_l2arc()
1706 		ASSERT(vdev_is_concrete(vd));  in vdev_trim_l2arc()
1707 		ASSERT3P(vd->vdev_trim_thread, ==, NULL);  in vdev_trim_l2arc()
1708 		ASSERT(!vd->vdev_detached);  in vdev_trim_l2arc()
1709 		ASSERT(!vd->vdev_trim_exit_wanted);  in vdev_trim_l2arc()
1710 		ASSERT(!vd->vdev_top->vdev_removing);  in vdev_trim_l2arc()
1711 		vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);  in vdev_trim_l2arc()
1712 		vd->vdev_trim_thread = thread_create(NULL, 0,  in vdev_trim_l2arc()
1713 		    vdev_trim_l2arc_thread, vd, 0, &p0, TS_RUN, maxclsyspri);  in vdev_trim_l2arc()
1714 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc()
1723 vdev_trim_simple(vdev_t *vd, uint64_t start, uint64_t size)  in vdev_trim_simple()  argument
1731 	ASSERT(vdev_is_concrete(vd));  in vdev_trim_simple()
1732 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_simple()
1733 	ASSERT(!vd->vdev_detached);  in vdev_trim_simple()
1734 	ASSERT(!vd->vdev_top->vdev_removing);  in vdev_trim_simple()
1735 	ASSERT(!vd->vdev_top->vdev_rz_expanding);  in vdev_trim_simple()
1737 	ta.trim_vdev = vd;  in vdev_trim_simple()
1755 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_simple()
1756 	while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) {  in vdev_trim_simple()
1757 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_simple()
1759 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_simple()