1 // SPDX-License-Identifier: CDDL-1.0
10  * or https://opensource.org/licenses/CDDL-1.0.
42  * TRIM is a feature which is used to notify a SSD that some previously
48  * There are two supported TRIM methods; manual and automatic.
50  * Manual TRIM:
52  * A manual TRIM is initiated by running the 'zpool trim' command.  A single
54  * managing that vdev TRIM process.  This involves iterating over all the
56  * required TRIM I/Os.
61  * the TRIM are regularly written to the pool.  This allows the TRIM to be
64  * Automatic TRIM:
66  * An automatic TRIM is enabled by setting the 'autotrim' pool property
68  * top-level (not leaf) vdev in the pool.  These threads perform the same
69  * core TRIM process as a manual TRIM, but with a few key differences.
71  * 1) Automatic TRIM happens continuously in the background and operates
74  * 2) Each thread is associated with a top-level (not leaf) vdev.  This has
77  *    metaslab is disabled at a time.  Unlike manual TRIM, this means each
78  *    'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
81  * 3) There is no automatic TRIM progress information stored on disk, nor
84  * While the automatic TRIM process is highly effective it is more likely
85  * than a manual TRIM to encounter tiny ranges.  Ranges less than or equal to
87  * TRIM and are skipped.  This means small amounts of freed space may not
94  * For this reason it may be beneficial to occasionally manually TRIM a pool
95  * even when automatic TRIM is enabled.
99  * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
104  * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
109  * Skip uninitialized metaslabs during the TRIM process.  This option is
110  * useful for pools constructed from large thinly-provisioned devices where
111  * TRIM operations are slow.  As a pool ages an increasing fraction of
114  * manual TRIM and will persist for the duration of the requested TRIM.
119  * Maximum number of queued TRIM I/Os per leaf vdev.  The number of
120  * concurrent TRIM I/Os issued to the device is controlled by the
127  * metaslab.  This setting represents a trade-off between issuing more
128  * efficient TRIM operations, by allowing them to be aggregated longer,
134  * time.  This can result is larger TRIM operations, and increased memory
136  * has the opposite effect.  The default value of 32 was determined though
144  * trimmed and a range tree containing the extents to TRIM.  All provided
151 	vdev_t		*trim_vdev;		/* Leaf vdev to TRIM */
153 	zfs_range_tree_t	*trim_tree;	/* TRIM ranges (in metaslab) */
154 	trim_type_t	trim_type;		/* Manual or auto TRIM */
155 	uint64_t	trim_extent_bytes_max;	/* Maximum TRIM I/O size */
156 	uint64_t	trim_extent_bytes_min;	/* Minimum TRIM I/O size */
157 	enum trim_flag	trim_flags;		/* TRIM flags (secure) */
172 	return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||  in vdev_trim_should_stop()
173 	    vd->vdev_detached || vd->vdev_top->vdev_removing ||  in vdev_trim_should_stop()
174 	    vd->vdev_top->vdev_rz_expanding);  in vdev_trim_should_stop()
183 	return (tvd->vdev_autotrim_exit_wanted ||  in vdev_autotrim_should_stop()
184 	    !vdev_writeable(tvd) || tvd->vdev_removing ||  in vdev_autotrim_should_stop()
185 	    tvd->vdev_rz_expanding ||  in vdev_autotrim_should_stop()
186 	    spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF);  in vdev_autotrim_should_stop()
196 	mutex_enter(&vd->vdev_autotrim_lock);  in vdev_autotrim_wait_kick()
198 		if (vd->vdev_autotrim_exit_wanted)  in vdev_autotrim_wait_kick()
200 		cv_wait_idle(&vd->vdev_autotrim_kick_cv,  in vdev_autotrim_wait_kick()
201 		    &vd->vdev_autotrim_lock);  in vdev_autotrim_wait_kick()
203 	boolean_t exit_wanted = vd->vdev_autotrim_exit_wanted;  in vdev_autotrim_wait_kick()
204 	mutex_exit(&vd->vdev_autotrim_lock);  in vdev_autotrim_wait_kick()
210  * The sync task for updating the on-disk state of a manual TRIM.  This
228 	vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);  in vdev_trim_zap_update_sync()
229 	if (vd == NULL || vd->vdev_top->vdev_removing ||  in vdev_trim_zap_update_sync()
230 	    !vdev_is_concrete(vd) || vd->vdev_top->vdev_rz_expanding)  in vdev_trim_zap_update_sync()
233 	uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];  in vdev_trim_zap_update_sync()
234 	vd->vdev_trim_offset[txg & TXG_MASK] = 0;  in vdev_trim_zap_update_sync()
236 	VERIFY3U(vd->vdev_leaf_zap, !=, 0);  in vdev_trim_zap_update_sync()
238 	objset_t *mos = vd->vdev_spa->spa_meta_objset;  in vdev_trim_zap_update_sync()
240 	if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {  in vdev_trim_zap_update_sync()
242 		if (vd->vdev_trim_last_offset == UINT64_MAX)  in vdev_trim_zap_update_sync()
245 		vd->vdev_trim_last_offset = last_offset;  in vdev_trim_zap_update_sync()
246 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,  in vdev_trim_zap_update_sync()
251 	if (vd->vdev_trim_action_time > 0) {  in vdev_trim_zap_update_sync()
252 		uint64_t val = (uint64_t)vd->vdev_trim_action_time;  in vdev_trim_zap_update_sync()
253 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,  in vdev_trim_zap_update_sync()
258 	if (vd->vdev_trim_rate > 0) {  in vdev_trim_zap_update_sync()
259 		uint64_t rate = (uint64_t)vd->vdev_trim_rate;  in vdev_trim_zap_update_sync()
264 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,  in vdev_trim_zap_update_sync()
268 	uint64_t partial = vd->vdev_trim_partial;  in vdev_trim_zap_update_sync()
272 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,  in vdev_trim_zap_update_sync()
275 	uint64_t secure = vd->vdev_trim_secure;  in vdev_trim_zap_update_sync()
279 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,  in vdev_trim_zap_update_sync()
283 	uint64_t trim_state = vd->vdev_trim_state;  in vdev_trim_zap_update_sync()
284 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,  in vdev_trim_zap_update_sync()
289  * Update the on-disk state of a manual TRIM.  This is called to request
290  * that a TRIM be started/suspended/canceled, or to change one of the
291  * TRIM options (partial, secure, rate).
297 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim_change_state()
298 	spa_t *spa = vd->vdev_spa;  in vdev_trim_change_state()
300 	if (new_state == vd->vdev_trim_state)  in vdev_trim_change_state()
307 	*guid = vd->vdev_guid;  in vdev_trim_change_state()
312 	if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {  in vdev_trim_change_state()
313 		vd->vdev_trim_action_time = gethrestime_sec();  in vdev_trim_change_state()
317 	 * If we're activating, then preserve the requested rate and trim  in vdev_trim_change_state()
319 	 * as a sentinel to indicate they should be reset to default values.  in vdev_trim_change_state()
322 		if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||  in vdev_trim_change_state()
323 		    vd->vdev_trim_state == VDEV_TRIM_CANCELED) {  in vdev_trim_change_state()
324 			vd->vdev_trim_last_offset = UINT64_MAX;  in vdev_trim_change_state()
325 			vd->vdev_trim_rate = UINT64_MAX;  in vdev_trim_change_state()
326 			vd->vdev_trim_partial = UINT64_MAX;  in vdev_trim_change_state()
327 			vd->vdev_trim_secure = UINT64_MAX;  in vdev_trim_change_state()
331 			vd->vdev_trim_rate = rate;  in vdev_trim_change_state()
334 			vd->vdev_trim_partial = partial;  in vdev_trim_change_state()
337 			vd->vdev_trim_secure = secure;  in vdev_trim_change_state()
340 	vdev_trim_state_t old_state = vd->vdev_trim_state;  in vdev_trim_change_state()
342 	vd->vdev_trim_state = new_state;  in vdev_trim_change_state()
344 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);  in vdev_trim_change_state()
353 		spa_history_log_internal(spa, "trim", tx,  in vdev_trim_change_state()
354 		    "vdev=%s activated", vd->vdev_path);  in vdev_trim_change_state()
358 		spa_history_log_internal(spa, "trim", tx,  in vdev_trim_change_state()
359 		    "vdev=%s suspended", vd->vdev_path);  in vdev_trim_change_state()
365 			spa_history_log_internal(spa, "trim", tx,  in vdev_trim_change_state()
366 			    "vdev=%s canceled", vd->vdev_path);  in vdev_trim_change_state()
371 		spa_history_log_internal(spa, "trim", tx,  in vdev_trim_change_state()
372 		    "vdev=%s complete", vd->vdev_path);  in vdev_trim_change_state()
374 	default:  in vdev_trim_change_state()
385  * The zio_done_func_t done callback for each manual TRIM issued.  It is
386  * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
387  * and limiting the number of in flight TRIM I/Os.
392 	vdev_t *vd = zio->io_vd;  in vdev_trim_cb()
394 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_cb()
395 	if (zio->io_error == ENXIO && !vdev_writeable(vd)) {  in vdev_trim_cb()
402 		    &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];  in vdev_trim_cb()
403 		*offset = MIN(*offset, zio->io_offset);  in vdev_trim_cb()
405 		if (zio->io_error != 0) {  in vdev_trim_cb()
406 			vd->vdev_stat.vs_trim_errors++;  in vdev_trim_cb()
407 			spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,  in vdev_trim_cb()
408 			    0, 0, 0, 0, 1, zio->io_orig_size);  in vdev_trim_cb()
410 			spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,  in vdev_trim_cb()
411 			    1, zio->io_orig_size, 0, 0, 0, 0);  in vdev_trim_cb()
414 		vd->vdev_trim_bytes_done += zio->io_orig_size;  in vdev_trim_cb()
417 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);  in vdev_trim_cb()
418 	vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;  in vdev_trim_cb()
419 	cv_broadcast(&vd->vdev_trim_io_cv);  in vdev_trim_cb()
420 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_cb()
422 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_trim_cb()
426  * The zio_done_func_t done callback for each automatic TRIM issued.  It
427  * is responsible for updating the TRIM stats and limiting the number of
428  * in flight TRIM I/Os.  Automatic TRIM I/Os are best effort and are
434 	vdev_t *vd = zio->io_vd;  in vdev_autotrim_cb()
436 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_autotrim_cb()
438 	if (zio->io_error != 0) {  in vdev_autotrim_cb()
439 		vd->vdev_stat.vs_trim_errors++;  in vdev_autotrim_cb()
440 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,  in vdev_autotrim_cb()
441 		    0, 0, 0, 0, 1, zio->io_orig_size);  in vdev_autotrim_cb()
443 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,  in vdev_autotrim_cb()
444 		    1, zio->io_orig_size, 0, 0, 0, 0);  in vdev_autotrim_cb()
447 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);  in vdev_autotrim_cb()
448 	vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;  in vdev_autotrim_cb()
449 	cv_broadcast(&vd->vdev_trim_io_cv);  in vdev_autotrim_cb()
450 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_autotrim_cb()
452 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_autotrim_cb()
456  * The zio_done_func_t done callback for each TRIM issued via
457  * vdev_trim_simple(). It is responsible for updating the TRIM stats and
458  * limiting the number of in flight TRIM I/Os.  Simple TRIM I/Os are best
464 	vdev_t *vd = zio->io_vd;  in vdev_trim_simple_cb()
466 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_simple_cb()
468 	if (zio->io_error != 0) {  in vdev_trim_simple_cb()
469 		vd->vdev_stat.vs_trim_errors++;  in vdev_trim_simple_cb()
470 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,  in vdev_trim_simple_cb()
471 		    0, 0, 0, 0, 1, zio->io_orig_size);  in vdev_trim_simple_cb()
473 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,  in vdev_trim_simple_cb()
474 		    1, zio->io_orig_size, 0, 0, 0, 0);  in vdev_trim_simple_cb()
477 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0);  in vdev_trim_simple_cb()
478 	vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--;  in vdev_trim_simple_cb()
479 	cv_broadcast(&vd->vdev_trim_io_cv);  in vdev_trim_simple_cb()
480 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_simple_cb()
482 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_trim_simple_cb()
485  * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
490 	return (ta->trim_bytes_done * 1000 /  in vdev_trim_calculate_rate()
491 	    (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1));  in vdev_trim_calculate_rate()
495  * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
496  * and number of concurrent TRIM I/Os.
501 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_range()
502 	spa_t *spa = vd->vdev_spa;  in vdev_trim_range()
505 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_range()
508 	 * Limit manual TRIM I/Os to the requested rate.  This does not  in vdev_trim_range()
509 	 * apply to automatic TRIM since no per vdev rate can be specified.  in vdev_trim_range()
511 	if (ta->trim_type == TRIM_TYPE_MANUAL) {  in vdev_trim_range()
512 		while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&  in vdev_trim_range()
513 		    vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {  in vdev_trim_range()
514 			cv_timedwait_idle(&vd->vdev_trim_io_cv,  in vdev_trim_range()
515 			    &vd->vdev_trim_io_lock, ddi_get_lbolt() +  in vdev_trim_range()
519 	ta->trim_bytes_done += size;  in vdev_trim_range()
522 	while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] +  in vdev_trim_range()
523 	    vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) {  in vdev_trim_range()
524 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_range()
526 	vd->vdev_trim_inflight[ta->trim_type]++;  in vdev_trim_range()
527 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_range()
529 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);  in vdev_trim_range()
534 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_range()
536 	if (ta->trim_type == TRIM_TYPE_MANUAL &&  in vdev_trim_range()
537 	    vd->vdev_trim_offset[txg & TXG_MASK] == 0) {  in vdev_trim_range()
539 		*guid = vd->vdev_guid;  in vdev_trim_range()
550 	if ((ta->trim_type == TRIM_TYPE_MANUAL &&  in vdev_trim_range()
552 	    (ta->trim_type == TRIM_TYPE_AUTO &&  in vdev_trim_range()
553 	    vdev_autotrim_should_stop(vd->vdev_top))) {  in vdev_trim_range()
554 		mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_range()
555 		vd->vdev_trim_inflight[ta->trim_type]--;  in vdev_trim_range()
556 		mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_range()
557 		spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);  in vdev_trim_range()
558 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_range()
562 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_range()
564 	if (ta->trim_type == TRIM_TYPE_MANUAL)  in vdev_trim_range()
565 		vd->vdev_trim_offset[txg & TXG_MASK] = start + size;  in vdev_trim_range()
567 	if (ta->trim_type == TRIM_TYPE_MANUAL) {  in vdev_trim_range()
569 	} else if (ta->trim_type == TRIM_TYPE_AUTO) {  in vdev_trim_range()
575 	zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,  in vdev_trim_range()
577 	    ta->trim_flags));  in vdev_trim_range()
586  * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
587  * Additional parameters describing how the TRIM should be performed must
594 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_ranges()
595 	zfs_btree_t *t = &ta->trim_tree->rt_root;  in vdev_trim_ranges()
597 	uint64_t extent_bytes_max = ta->trim_extent_bytes_max;  in vdev_trim_ranges()
598 	uint64_t extent_bytes_min = ta->trim_extent_bytes_min;  in vdev_trim_ranges()
599 	spa_t *spa = vd->vdev_spa;  in vdev_trim_ranges()
602 	ta->trim_start_time = gethrtime();  in vdev_trim_ranges()
603 	ta->trim_bytes_done = 0;  in vdev_trim_ranges()
607 		uint64_t size = zfs_rs_get_end(rs, ta->trim_tree) -  in vdev_trim_ranges()
608 		    zfs_rs_get_start(rs, ta->trim_tree);  in vdev_trim_ranges()
611 			spa_iostats_trim_add(spa, ta->trim_type,  in vdev_trim_ranges()
616 		/* Split range into legally-sized physical chunks */  in vdev_trim_ranges()
617 		uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1;  in vdev_trim_ranges()
621 			    zfs_rs_get_start(rs, ta->trim_tree) +  in vdev_trim_ranges()
622 			    (w *extent_bytes_max), MIN(size -  in vdev_trim_ranges()
633 	 * returning. TRIM zios have lower priority over regular or syncing  in vdev_trim_ranges()
634 	 * zios, so all TRIM zios for this metaslab must complete before the  in vdev_trim_ranges()
635 	 * metaslab is re-enabled. Otherwise it's possible write zios to  in vdev_trim_ranges()
636 	 * this metaslab could cut ahead of still queued TRIM zios for this  in vdev_trim_ranges()
639 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_ranges()
640 	while (vd->vdev_trim_inflight[0] > 0) {  in vdev_trim_ranges()
641 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_ranges()
643 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_ranges()
653 	if (physical_rs->rs_end > *last_rs_end)  in vdev_trim_xlate_last_rs_end()
654 		*last_rs_end = physical_rs->rs_end;  in vdev_trim_xlate_last_rs_end()
662 	uint64_t size = physical_rs->rs_end - physical_rs->rs_start;  in vdev_trim_xlate_progress()
663 	vd->vdev_trim_bytes_est += size;  in vdev_trim_xlate_progress()
665 	if (vd->vdev_trim_last_offset >= physical_rs->rs_end) {  in vdev_trim_xlate_progress()
666 		vd->vdev_trim_bytes_done += size;  in vdev_trim_xlate_progress()
667 	} else if (vd->vdev_trim_last_offset > physical_rs->rs_start &&  in vdev_trim_xlate_progress()
668 	    vd->vdev_trim_last_offset <= physical_rs->rs_end) {  in vdev_trim_xlate_progress()
669 		vd->vdev_trim_bytes_done +=  in vdev_trim_xlate_progress()
670 		    vd->vdev_trim_last_offset - physical_rs->rs_start;  in vdev_trim_xlate_progress()
675  * Calculates the completion percentage of a manual TRIM.
680 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||  in vdev_trim_calculate_progress()
681 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));  in vdev_trim_calculate_progress()
682 	ASSERT(vd->vdev_leaf_zap != 0);  in vdev_trim_calculate_progress()
684 	vd->vdev_trim_bytes_est = 0;  in vdev_trim_calculate_progress()
685 	vd->vdev_trim_bytes_done = 0;  in vdev_trim_calculate_progress()
687 	for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {  in vdev_trim_calculate_progress()
688 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];  in vdev_trim_calculate_progress()
689 		mutex_enter(&msp->ms_lock);  in vdev_trim_calculate_progress()
691 		uint64_t ms_free = (msp->ms_size -  in vdev_trim_calculate_progress()
693 		    vdev_get_ndisks(vd->vdev_top);  in vdev_trim_calculate_progress()
701 		logical_rs.rs_start = msp->ms_start;  in vdev_trim_calculate_progress()
702 		logical_rs.rs_end = msp->ms_start + msp->ms_size;  in vdev_trim_calculate_progress()
706 		if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {  in vdev_trim_calculate_progress()
707 			vd->vdev_trim_bytes_est += ms_free;  in vdev_trim_calculate_progress()
708 			mutex_exit(&msp->ms_lock);  in vdev_trim_calculate_progress()
719 		if (vd->vdev_trim_last_offset > last_rs_end) {  in vdev_trim_calculate_progress()
720 			vd->vdev_trim_bytes_done += ms_free;  in vdev_trim_calculate_progress()
721 			vd->vdev_trim_bytes_est += ms_free;  in vdev_trim_calculate_progress()
722 			mutex_exit(&msp->ms_lock);  in vdev_trim_calculate_progress()
733 		zfs_range_tree_t *rt = msp->ms_allocatable;  in vdev_trim_calculate_progress()
734 		zfs_btree_t *bt = &rt->rt_root;  in vdev_trim_calculate_progress()
744 		mutex_exit(&msp->ms_lock);  in vdev_trim_calculate_progress()
749  * Load from disk the vdev's manual TRIM information.  This includes the
750  * state, progress, and options provided when initiating the manual TRIM.
756 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||  in vdev_trim_load()
757 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));  in vdev_trim_load()
758 	ASSERT(vd->vdev_leaf_zap != 0);  in vdev_trim_load()
760 	if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||  in vdev_trim_load()
761 	    vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {  in vdev_trim_load()
762 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
763 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,  in vdev_trim_load()
764 		    sizeof (vd->vdev_trim_last_offset), 1,  in vdev_trim_load()
765 		    &vd->vdev_trim_last_offset);  in vdev_trim_load()
767 			vd->vdev_trim_last_offset = 0;  in vdev_trim_load()
772 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
773 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,  in vdev_trim_load()
774 			    sizeof (vd->vdev_trim_rate), 1,  in vdev_trim_load()
775 			    &vd->vdev_trim_rate);  in vdev_trim_load()
777 				vd->vdev_trim_rate = 0;  in vdev_trim_load()
783 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
784 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,  in vdev_trim_load()
785 			    sizeof (vd->vdev_trim_partial), 1,  in vdev_trim_load()
786 			    &vd->vdev_trim_partial);  in vdev_trim_load()
788 				vd->vdev_trim_partial = 0;  in vdev_trim_load()
794 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_load()
795 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,  in vdev_trim_load()
796 			    sizeof (vd->vdev_trim_secure), 1,  in vdev_trim_load()
797 			    &vd->vdev_trim_secure);  in vdev_trim_load()
799 				vd->vdev_trim_secure = 0;  in vdev_trim_load()
814 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_xlate_range_add()
817 	 * Only a manual trim will be traversing the vdev sequentially.  in vdev_trim_xlate_range_add()
818 	 * For an auto trim all valid ranges should be added.  in vdev_trim_xlate_range_add()
820 	if (ta->trim_type == TRIM_TYPE_MANUAL) {  in vdev_trim_xlate_range_add()
823 		if (physical_rs->rs_end <= vd->vdev_trim_last_offset)  in vdev_trim_xlate_range_add()
826 		/* Pick up where we left off mid-range. */  in vdev_trim_xlate_range_add()
827 		if (vd->vdev_trim_last_offset > physical_rs->rs_start) {  in vdev_trim_xlate_range_add()
828 			ASSERT3U(physical_rs->rs_end, >,  in vdev_trim_xlate_range_add()
829 			    vd->vdev_trim_last_offset);  in vdev_trim_xlate_range_add()
830 			physical_rs->rs_start = vd->vdev_trim_last_offset;  in vdev_trim_xlate_range_add()
834 	ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start);  in vdev_trim_xlate_range_add()
836 	zfs_range_tree_add(ta->trim_tree, physical_rs->rs_start,  in vdev_trim_xlate_range_add()
837 	    physical_rs->rs_end - physical_rs->rs_start);  in vdev_trim_xlate_range_add()
848 	vdev_t *vd = ta->trim_vdev;  in vdev_trim_range_add()
859 		metaslab_t *msp = ta->trim_msp;  in vdev_trim_range_add()
861 		VERIFY3B(msp->ms_loaded, ==, B_TRUE);  in vdev_trim_range_add()
862 		VERIFY(zfs_range_tree_contains(msp->ms_allocatable, start,  in vdev_trim_range_add()
866 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_range_add()
871  * Each manual TRIM thread is responsible for trimming the unallocated
873  * over its top-level metaslabs and issuing TRIM I/O for the space described
881 	spa_t *spa = vd->vdev_spa;  in vdev_trim_thread()
890 	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);  in vdev_trim_thread()
895 	vd->vdev_trim_last_offset = 0;  in vdev_trim_thread()
896 	vd->vdev_trim_rate = 0;  in vdev_trim_thread()
897 	vd->vdev_trim_partial = 0;  in vdev_trim_thread()
898 	vd->vdev_trim_secure = 0;  in vdev_trim_thread()
910 	 * When a secure TRIM has been requested infer that the intent  in vdev_trim_thread()
911 	 * is that everything must be trimmed.  Override the default  in vdev_trim_thread()
912 	 * minimum TRIM size to prevent ranges from being skipped.  in vdev_trim_thread()
914 	if (vd->vdev_trim_secure) {  in vdev_trim_thread()
920 	for (uint64_t i = 0; !vd->vdev_detached &&  in vdev_trim_thread()
921 	    i < vd->vdev_top->vdev_ms_count; i++) {  in vdev_trim_thread()
922 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];  in vdev_trim_thread()
925 		 * If we've expanded the top-level vdev or it's our  in vdev_trim_thread()
928 		if (vd->vdev_top->vdev_ms_count != ms_count) {  in vdev_trim_thread()
930 			ms_count = vd->vdev_top->vdev_ms_count;  in vdev_trim_thread()
935 		mutex_enter(&msp->ms_lock);  in vdev_trim_thread()
939 		 * If a partial TRIM was requested skip metaslabs which have  in vdev_trim_thread()
942 		if (msp->ms_sm == NULL && vd->vdev_trim_partial) {  in vdev_trim_thread()
943 			mutex_exit(&msp->ms_lock);  in vdev_trim_thread()
951 		zfs_range_tree_walk(msp->ms_allocatable, vdev_trim_range_add,  in vdev_trim_thread()
953 		zfs_range_tree_vacate(msp->ms_trim, NULL, NULL);  in vdev_trim_thread()
954 		mutex_exit(&msp->ms_lock);  in vdev_trim_thread()
969 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_thread()
970 	if (!vd->vdev_trim_exit_wanted) {  in vdev_trim_thread()
973 			    vd->vdev_trim_rate, vd->vdev_trim_partial,  in vdev_trim_thread()
974 			    vd->vdev_trim_secure);  in vdev_trim_thread()
975 		} else if (vd->vdev_faulted) {  in vdev_trim_thread()
977 			    vd->vdev_trim_rate, vd->vdev_trim_partial,  in vdev_trim_thread()
978 			    vd->vdev_trim_secure);  in vdev_trim_thread()
981 	ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);  in vdev_trim_thread()
986 	 * check to see if it needs to restart a trim. That thread will be  in vdev_trim_thread()
990 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_thread()
992 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_thread()
994 	vd->vdev_trim_thread = NULL;  in vdev_trim_thread()
995 	cv_broadcast(&vd->vdev_trim_cv);  in vdev_trim_thread()
996 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_thread()
1002  * Initiates a manual TRIM for the vdev_t.  Callers must hold vdev_trim_lock,
1008 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim()
1009 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim()
1011 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);  in vdev_trim()
1012 	ASSERT(!vd->vdev_detached);  in vdev_trim()
1013 	ASSERT(!vd->vdev_trim_exit_wanted);  in vdev_trim()
1014 	ASSERT(!vd->vdev_top->vdev_removing);  in vdev_trim()
1015 	ASSERT(!vd->vdev_rz_expanding);  in vdev_trim()
1018 	vd->vdev_trim_thread = thread_create(NULL, 0,  in vdev_trim()
1028 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim_stop_wait_impl()
1030 	while (vd->vdev_trim_thread != NULL)  in vdev_trim_stop_wait_impl()
1031 		cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);  in vdev_trim_stop_wait_impl()
1033 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);  in vdev_trim_stop_wait_impl()
1034 	vd->vdev_trim_exit_wanted = B_FALSE;  in vdev_trim_stop_wait_impl()
1038  * Wait for vdev trim threads which were listed to cleanly exit.
1047 	    spa->spa_export_thread == curthread);  in vdev_trim_stop_wait()
1050 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_stop_wait()
1052 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_stop_wait()
1060  * required to call vdev_trim_stop_wait() to block for all the trim threads
1068 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));  in vdev_trim_stop()
1069 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));  in vdev_trim_stop()
1070 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_stop()
1074 	 * Allow cancel requests to proceed even if the trim thread has  in vdev_trim_stop()
1077 	if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)  in vdev_trim_stop()
1081 	vd->vdev_trim_exit_wanted = B_TRUE;  in vdev_trim_stop()
1087 		    vd->vdev_spa->spa_export_thread == curthread);  in vdev_trim_stop()
1099 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {  in vdev_trim_stop_all_impl()
1100 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_stop_all_impl()
1102 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_stop_all_impl()
1106 	for (uint64_t i = 0; i < vd->vdev_children; i++) {  in vdev_trim_stop_all_impl()
1107 		vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,  in vdev_trim_stop_all_impl()
1113  * Convenience function to stop trimming of a vdev tree and set all trim
1119 	spa_t *spa = vd->vdev_spa;  in vdev_trim_stop_all()
1124 	    spa->spa_export_thread == curthread);  in vdev_trim_stop_all()
1136 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {  in vdev_trim_stop_all()
1137 		vd_l2cache = spa->spa_l2cache.sav_vdevs[i];  in vdev_trim_stop_all()
1143 	if (vd->vdev_spa->spa_sync_on) {  in vdev_trim_stop_all()
1145 		txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);  in vdev_trim_stop_all()
1152  * Conditionally restarts a manual TRIM given its on-disk state.
1158 	    vd->vdev_spa->spa_load_thread == curthread);  in vdev_trim_restart()
1159 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));  in vdev_trim_restart()
1161 	if (vd->vdev_leaf_zap != 0) {  in vdev_trim_restart()
1162 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_restart()
1164 		int err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_restart()
1165 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,  in vdev_trim_restart()
1168 		vd->vdev_trim_state = trim_state;  in vdev_trim_restart()
1171 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,  in vdev_trim_restart()
1172 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,  in vdev_trim_restart()
1175 		vd->vdev_trim_action_time = timestamp;  in vdev_trim_restart()
1177 		if ((vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||  in vdev_trim_restart()
1178 		    vd->vdev_offline) && !vd->vdev_top->vdev_rz_expanding) {  in vdev_trim_restart()
1181 		} else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&  in vdev_trim_restart()
1182 		    vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&  in vdev_trim_restart()
1183 		    !vd->vdev_top->vdev_rz_expanding &&  in vdev_trim_restart()
1184 		    vd->vdev_trim_thread == NULL) {  in vdev_trim_restart()
1186 			vdev_trim(vd, vd->vdev_trim_rate,  in vdev_trim_restart()
1187 			    vd->vdev_trim_partial, vd->vdev_trim_secure);  in vdev_trim_restart()
1190 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_restart()
1193 	for (uint64_t i = 0; i < vd->vdev_children; i++) {  in vdev_trim_restart()
1194 		vdev_trim_restart(vd->vdev_child[i]);  in vdev_trim_restart()
1199  * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1200  * every TRIM range is contained within ms_allocatable.
1206 	metaslab_t *msp = ta->trim_msp;  in vdev_trim_range_verify()
1208 	VERIFY3B(msp->ms_loaded, ==, B_TRUE);  in vdev_trim_range_verify()
1209 	VERIFY3U(msp->ms_disabled, >, 0);  in vdev_trim_range_verify()
1210 	VERIFY(zfs_range_tree_contains(msp->ms_allocatable, start, size));  in vdev_trim_range_verify()
1214  * Each automatic TRIM thread is responsible for managing the trimming of a
1215  * top-level vdev in the pool.  No automatic TRIM state is maintained on-disk.
1217  * N.B. This behavior is different from a manual TRIM where a thread
1218  * is created for each leaf vdev, instead of each top-level vdev.
1224 	spa_t *spa = vd->vdev_spa;  in vdev_autotrim_thread()
1227 	mutex_enter(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1228 	ASSERT3P(vd->vdev_top, ==, vd);  in vdev_autotrim_thread()
1229 	ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);  in vdev_autotrim_thread()
1230 	mutex_exit(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1244 		 * For example, when zfs_trim_txg_batch = 32 (default) then  in vdev_autotrim_thread()
1260 		 * 2) Selecting non-consecutive metaslabs distributes the  in vdev_autotrim_thread()
1261 		 *    TRIM commands for a group evenly over the entire device.  in vdev_autotrim_thread()
1264 		for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;  in vdev_autotrim_thread()
1266 			metaslab_t *msp = vd->vdev_ms[i];  in vdev_autotrim_thread()
1275 			mutex_enter(&msp->ms_lock);  in vdev_autotrim_thread()
1279 			 * or when there are no recent frees to trim.  in vdev_autotrim_thread()
1281 			if (msp->ms_sm == NULL ||  in vdev_autotrim_thread()
1282 			    zfs_range_tree_is_empty(msp->ms_trim)) {  in vdev_autotrim_thread()
1283 				mutex_exit(&msp->ms_lock);  in vdev_autotrim_thread()
1290 			 * This may happen when a manual TRIM or initialize  in vdev_autotrim_thread()
1292 			 * of a manual TRIM, the ms_trim tree will have been  in vdev_autotrim_thread()
1293 			 * vacated.  Only ranges added after the manual TRIM  in vdev_autotrim_thread()
1295 			 * These will be processed when the automatic TRIM  in vdev_autotrim_thread()
1298 			if (msp->ms_disabled > 1) {  in vdev_autotrim_thread()
1299 				mutex_exit(&msp->ms_lock);  in vdev_autotrim_thread()
1310 			zfs_range_tree_swap(&msp->ms_trim, &trim_tree);  in vdev_autotrim_thread()
1311 			ASSERT(zfs_range_tree_is_empty(msp->ms_trim));  in vdev_autotrim_thread()
1314 			 * There are two cases when constructing the per-vdev  in vdev_autotrim_thread()
1315 			 * trim trees for a metaslab.  If the top-level vdev  in vdev_autotrim_thread()
1318 			 * and a trim tree should be constructed for each.  in vdev_autotrim_thread()
1321 			uint64_t children = vd->vdev_children;  in vdev_autotrim_thread()
1332 					tap[c].trim_vdev = vd->vdev_child[c];  in vdev_autotrim_thread()
1338 				vdev_t *cvd = ta->trim_vdev;  in vdev_autotrim_thread()
1340 				ta->trim_msp = msp;  in vdev_autotrim_thread()
1341 				ta->trim_extent_bytes_max = extent_bytes_max;  in vdev_autotrim_thread()
1342 				ta->trim_extent_bytes_min = extent_bytes_min;  in vdev_autotrim_thread()
1343 				ta->trim_type = TRIM_TYPE_AUTO;  in vdev_autotrim_thread()
1344 				ta->trim_flags = 0;  in vdev_autotrim_thread()
1346 				if (cvd->vdev_detached ||  in vdev_autotrim_thread()
1348 				    !cvd->vdev_has_trim ||  in vdev_autotrim_thread()
1349 				    cvd->vdev_trim_thread != NULL) {  in vdev_autotrim_thread()
1360 				if (!cvd->vdev_ops->vdev_op_leaf)  in vdev_autotrim_thread()
1363 				ta->trim_tree = zfs_range_tree_create(NULL,  in vdev_autotrim_thread()
1369 			mutex_exit(&msp->ms_lock);  in vdev_autotrim_thread()
1373 			 * Issue the TRIM I/Os for all ranges covered by the  in vdev_autotrim_thread()
1374 			 * TRIM trees.  These ranges are safe to TRIM because  in vdev_autotrim_thread()
1382 				 * Always yield to a manual TRIM if one has  in vdev_autotrim_thread()
1385 				if (ta->trim_tree == NULL ||  in vdev_autotrim_thread()
1386 				    ta->trim_vdev->vdev_trim_thread != NULL) {  in vdev_autotrim_thread()
1395 				 * of the required TRIM I/Os.  in vdev_autotrim_thread()
1409 				mutex_enter(&msp->ms_lock);  in vdev_autotrim_thread()
1414 				mutex_exit(&msp->ms_lock);  in vdev_autotrim_thread()
1421 			 * Wait for couples of kicks, to ensure the trim io is  in vdev_autotrim_thread()
1437 				if (ta->trim_tree == NULL)  in vdev_autotrim_thread()
1440 				zfs_range_tree_vacate(ta->trim_tree, NULL,  in vdev_autotrim_thread()
1442 				zfs_range_tree_destroy(ta->trim_tree);  in vdev_autotrim_thread()
1459 	for (uint64_t c = 0; c < vd->vdev_children; c++) {  in vdev_autotrim_thread()
1460 		vdev_t *cvd = vd->vdev_child[c];  in vdev_autotrim_thread()
1461 		mutex_enter(&cvd->vdev_trim_io_lock);  in vdev_autotrim_thread()
1463 		while (cvd->vdev_trim_inflight[1] > 0) {  in vdev_autotrim_thread()
1464 			cv_wait(&cvd->vdev_trim_io_cv,  in vdev_autotrim_thread()
1465 			    &cvd->vdev_trim_io_lock);  in vdev_autotrim_thread()
1467 		mutex_exit(&cvd->vdev_trim_io_lock);  in vdev_autotrim_thread()
1477 		for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {  in vdev_autotrim_thread()
1478 			metaslab_t *msp = vd->vdev_ms[i];  in vdev_autotrim_thread()
1480 			mutex_enter(&msp->ms_lock);  in vdev_autotrim_thread()
1481 			zfs_range_tree_vacate(msp->ms_trim, NULL, NULL);  in vdev_autotrim_thread()
1482 			mutex_exit(&msp->ms_lock);  in vdev_autotrim_thread()
1486 	mutex_enter(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1487 	ASSERT(vd->vdev_autotrim_thread != NULL);  in vdev_autotrim_thread()
1488 	vd->vdev_autotrim_thread = NULL;  in vdev_autotrim_thread()
1489 	cv_broadcast(&vd->vdev_autotrim_cv);  in vdev_autotrim_thread()
1490 	mutex_exit(&vd->vdev_autotrim_lock);  in vdev_autotrim_thread()
1496  * Starts an autotrim thread, if needed, for each top-level vdev which can be
1497  * trimmed.  A top-level vdev which has been evacuated will never be trimmed.
1502 	vdev_t *root_vd = spa->spa_root_vdev;  in vdev_autotrim()
1504 	for (uint64_t i = 0; i < root_vd->vdev_children; i++) {  in vdev_autotrim()
1505 		vdev_t *tvd = root_vd->vdev_child[i];  in vdev_autotrim()
1507 		mutex_enter(&tvd->vdev_autotrim_lock);  in vdev_autotrim()
1508 		if (vdev_writeable(tvd) && !tvd->vdev_removing &&  in vdev_autotrim()
1509 		    tvd->vdev_autotrim_thread == NULL &&  in vdev_autotrim()
1510 		    !tvd->vdev_rz_expanding) {  in vdev_autotrim()
1511 			ASSERT3P(tvd->vdev_top, ==, tvd);  in vdev_autotrim()
1513 			tvd->vdev_autotrim_thread = thread_create(NULL, 0,  in vdev_autotrim()
1516 			ASSERT(tvd->vdev_autotrim_thread != NULL);  in vdev_autotrim()
1518 		mutex_exit(&tvd->vdev_autotrim_lock);  in vdev_autotrim()
1523  * Wait for the vdev_autotrim_thread associated with the passed top-level
1529 	mutex_enter(&tvd->vdev_autotrim_lock);  in vdev_autotrim_stop_wait()
1530 	if (tvd->vdev_autotrim_thread != NULL) {  in vdev_autotrim_stop_wait()
1531 		tvd->vdev_autotrim_exit_wanted = B_TRUE;  in vdev_autotrim_stop_wait()
1532 		cv_broadcast(&tvd->vdev_autotrim_kick_cv);  in vdev_autotrim_stop_wait()
1533 		cv_wait(&tvd->vdev_autotrim_cv,  in vdev_autotrim_stop_wait()
1534 		    &tvd->vdev_autotrim_lock);  in vdev_autotrim_stop_wait()
1536 		ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL);  in vdev_autotrim_stop_wait()
1537 		tvd->vdev_autotrim_exit_wanted = B_FALSE;  in vdev_autotrim_stop_wait()
1539 	mutex_exit(&tvd->vdev_autotrim_lock);  in vdev_autotrim_stop_wait()
1547 	vdev_t *root_vd = spa->spa_root_vdev;  in vdev_autotrim_kick()
1550 	for (uint64_t i = 0; i < root_vd->vdev_children; i++) {  in vdev_autotrim_kick()
1551 		tvd = root_vd->vdev_child[i];  in vdev_autotrim_kick()
1553 		mutex_enter(&tvd->vdev_autotrim_lock);  in vdev_autotrim_kick()
1554 		if (tvd->vdev_autotrim_thread != NULL)  in vdev_autotrim_kick()
1555 			cv_broadcast(&tvd->vdev_autotrim_kick_cv);  in vdev_autotrim_kick()
1556 		mutex_exit(&tvd->vdev_autotrim_lock);  in vdev_autotrim_kick()
1567 	vdev_t *root_vd = spa->spa_root_vdev;  in vdev_autotrim_stop_all()
1569 	for (uint64_t i = 0; i < root_vd->vdev_children; i++)  in vdev_autotrim_stop_all()
1570 		vdev_autotrim_stop_wait(root_vd->vdev_child[i]);  in vdev_autotrim_stop_all()
1580 	    spa->spa_load_thread == curthread);  in vdev_autotrim_restart()
1581 	if (spa->spa_autotrim)  in vdev_autotrim_restart()
1589 	spa_t		*spa = vd->vdev_spa;  in vdev_trim_l2arc_thread()
1597 	vd->vdev_trim_last_offset = 0;  in vdev_trim_l2arc_thread()
1598 	vd->vdev_trim_rate = 0;  in vdev_trim_l2arc_thread()
1599 	vd->vdev_trim_partial = 0;  in vdev_trim_l2arc_thread()
1600 	vd->vdev_trim_secure = 0;  in vdev_trim_l2arc_thread()
1609 	physical_rs.rs_start = vd->vdev_trim_bytes_done = 0;  in vdev_trim_l2arc_thread()
1610 	physical_rs.rs_end = vd->vdev_trim_bytes_est =  in vdev_trim_l2arc_thread()
1614 	    physical_rs.rs_end - physical_rs.rs_start);  in vdev_trim_l2arc_thread()
1616 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1618 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1623 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_l2arc_thread()
1624 	while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) {  in vdev_trim_l2arc_thread()
1625 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_l2arc_thread()
1627 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_l2arc_thread()
1632 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1633 	if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {  in vdev_trim_l2arc_thread()
1635 		    vd->vdev_trim_rate, vd->vdev_trim_partial,  in vdev_trim_l2arc_thread()
1636 		    vd->vdev_trim_secure);  in vdev_trim_l2arc_thread()
1638 	ASSERT(vd->vdev_trim_thread != NULL ||  in vdev_trim_l2arc_thread()
1639 	    vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0);  in vdev_trim_l2arc_thread()
1644 	 * must check to see if it needs to restart a trim. That thread  in vdev_trim_l2arc_thread()
1649 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1650 	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);  in vdev_trim_l2arc_thread()
1651 	mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1659 	spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd,  in vdev_trim_l2arc_thread()
1661 	memset(dev->l2ad_dev_hdr, 0, dev->l2ad_dev_hdr_asize);  in vdev_trim_l2arc_thread()
1663 	spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd);  in vdev_trim_l2arc_thread()
1665 	vd->vdev_trim_thread = NULL;  in vdev_trim_l2arc_thread()
1666 	if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE)  in vdev_trim_l2arc_thread()
1667 		dev->l2ad_trim_all = B_FALSE;  in vdev_trim_l2arc_thread()
1669 	cv_broadcast(&vd->vdev_trim_cv);  in vdev_trim_l2arc_thread()
1670 	mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc_thread()
1676  * Punches out TRIM threads for the L2ARC devices in a spa and assigns them
1677  * to vd->vdev_trim_thread variable. This facilitates the management of
1687 	 * Locate the spa's l2arc devices and kick off TRIM threads.  in vdev_trim_l2arc()
1689 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {  in vdev_trim_l2arc()
1690 		vdev_t *vd = spa->spa_l2cache.sav_vdevs[i];  in vdev_trim_l2arc()
1693 		if (dev == NULL || !dev->l2ad_trim_all) {  in vdev_trim_l2arc()
1695 			 * Don't attempt TRIM if the vdev is UNAVAIL or if the  in vdev_trim_l2arc()
1696 			 * cache device was not marked for whole device TRIM  in vdev_trim_l2arc()
1704 		mutex_enter(&vd->vdev_trim_lock);  in vdev_trim_l2arc()
1705 		ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_l2arc()
1707 		ASSERT3P(vd->vdev_trim_thread, ==, NULL);  in vdev_trim_l2arc()
1708 		ASSERT(!vd->vdev_detached);  in vdev_trim_l2arc()
1709 		ASSERT(!vd->vdev_trim_exit_wanted);  in vdev_trim_l2arc()
1710 		ASSERT(!vd->vdev_top->vdev_removing);  in vdev_trim_l2arc()
1712 		vd->vdev_trim_thread = thread_create(NULL, 0,  in vdev_trim_l2arc()
1714 		mutex_exit(&vd->vdev_trim_lock);  in vdev_trim_l2arc()
1732 	ASSERT(vd->vdev_ops->vdev_op_leaf);  in vdev_trim_simple()
1733 	ASSERT(!vd->vdev_detached);  in vdev_trim_simple()
1734 	ASSERT(!vd->vdev_top->vdev_removing);  in vdev_trim_simple()
1735 	ASSERT(!vd->vdev_top->vdev_rz_expanding);  in vdev_trim_simple()
1748 		    physical_rs.rs_end - physical_rs.rs_start);  in vdev_trim_simple()
1755 	mutex_enter(&vd->vdev_trim_io_lock);  in vdev_trim_simple()
1756 	while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) {  in vdev_trim_simple()
1757 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);  in vdev_trim_simple()
1759 	mutex_exit(&vd->vdev_trim_io_lock);  in vdev_trim_simple()
1780 	"Max size of TRIM commands, larger will be split");
1783 	"Min size of TRIM commands, smaller will be skipped");
1789 	"Min number of txgs to aggregate frees before issuing TRIM");