xref: /freebsd/sys/contrib/openzfs/module/zfs/vdev_trim.c (revision 058ac3e8063366dafa634d9107642e12b038bf09)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2016 by Delphix. All rights reserved.
24  * Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
25  * Copyright (c) 2021 Hewlett Packard Enterprise Development LP
26  */
27 
28 #include <sys/spa.h>
29 #include <sys/spa_impl.h>
30 #include <sys/txg.h>
31 #include <sys/vdev_impl.h>
32 #include <sys/vdev_trim.h>
33 #include <sys/metaslab_impl.h>
34 #include <sys/dsl_synctask.h>
35 #include <sys/zap.h>
36 #include <sys/dmu_tx.h>
37 #include <sys/arc_impl.h>
38 
39 /*
40  * TRIM is a feature which is used to notify a SSD that some previously
41  * written space is no longer allocated by the pool.  This is useful because
42  * writes to a SSD must be performed to blocks which have first been erased.
43  * Ensuring the SSD always has a supply of erased blocks for new writes
44  * helps prevent the performance from deteriorating.
45  *
46  * There are two supported TRIM methods; manual and automatic.
47  *
48  * Manual TRIM:
49  *
50  * A manual TRIM is initiated by running the 'zpool trim' command.  A single
51  * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for
52  * managing that vdev TRIM process.  This involves iterating over all the
53  * metaslabs, calculating the unallocated space ranges, and then issuing the
54  * required TRIM I/Os.
55  *
56  * While a metaslab is being actively trimmed it is not eligible to perform
57  * new allocations.  After traversing all of the metaslabs the thread is
58  * terminated.  Finally, both the requested options and current progress of
59  * the TRIM are regularly written to the pool.  This allows the TRIM to be
60  * suspended and resumed as needed.
61  *
62  * Automatic TRIM:
63  *
64  * An automatic TRIM is enabled by setting the 'autotrim' pool property
65  * to 'on'.  When enabled, a `vdev_autotrim' thread is created for each
66  * top-level (not leaf) vdev in the pool.  These threads perform the same
67  * core TRIM process as a manual TRIM, but with a few key differences.
68  *
69  * 1) Automatic TRIM happens continuously in the background and operates
70  *    solely on recently freed blocks (ms_trim not ms_allocatable).
71  *
72  * 2) Each thread is associated with a top-level (not leaf) vdev.  This has
73  *    the benefit of simplifying the threading model, it makes it easier
74  *    to coordinate administrative commands, and it ensures only a single
75  *    metaslab is disabled at a time.  Unlike manual TRIM, this means each
76  *    'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
77  *    children.
78  *
79  * 3) There is no automatic TRIM progress information stored on disk, nor
80  *    is it reported by 'zpool status'.
81  *
82  * While the automatic TRIM process is highly effective it is more likely
83  * than a manual TRIM to encounter tiny ranges.  Ranges less than or equal to
84  * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently
85  * TRIM and are skipped.  This means small amounts of freed space may not
86  * be automatically trimmed.
87  *
88  * Furthermore, devices with attached hot spares and devices being actively
89  * replaced are skipped.  This is done to avoid adding additional stress to
90  * a potentially unhealthy device and to minimize the required rebuild time.
91  *
92  * For this reason it may be beneficial to occasionally manually TRIM a pool
93  * even when automatic TRIM is enabled.
94  */
95 
96 /*
97  * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
98  */
99 static unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024;
100 
101 /*
102  * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
103  */
104 static unsigned int zfs_trim_extent_bytes_min = 32 * 1024;
105 
106 /*
107  * Skip uninitialized metaslabs during the TRIM process.  This option is
108  * useful for pools constructed from large thinly-provisioned devices where
109  * TRIM operations are slow.  As a pool ages an increasing fraction of
110  * the pools metaslabs will be initialized progressively degrading the
111  * usefulness of this option.  This setting is stored when starting a
112  * manual TRIM and will persist for the duration of the requested TRIM.
113  */
114 unsigned int zfs_trim_metaslab_skip = 0;
115 
116 /*
117  * Maximum number of queued TRIM I/Os per leaf vdev.  The number of
118  * concurrent TRIM I/Os issued to the device is controlled by the
119  * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options.
120  */
121 static unsigned int zfs_trim_queue_limit = 10;
122 
123 /*
124  * The minimum number of transaction groups between automatic trims of a
125  * metaslab.  This setting represents a trade-off between issuing more
126  * efficient TRIM operations, by allowing them to be aggregated longer,
127  * and issuing them promptly so the trimmed space is available.  Note
128  * that this value is a minimum; metaslabs can be trimmed less frequently
129  * when there are a large number of ranges which need to be trimmed.
130  *
131  * Increasing this value will allow frees to be aggregated for a longer
132  * time.  This can result is larger TRIM operations, and increased memory
133  * usage in order to track the ranges to be trimmed.  Decreasing this value
134  * has the opposite effect.  The default value of 32 was determined though
135  * testing to be a reasonable compromise.
136  */
137 static unsigned int zfs_trim_txg_batch = 32;
138 
139 /*
140  * The trim_args are a control structure which describe how a leaf vdev
141  * should be trimmed.  The core elements are the vdev, the metaslab being
142  * trimmed and a range tree containing the extents to TRIM.  All provided
143  * ranges must be within the metaslab.
144  */
145 typedef struct trim_args {
146 	/*
147 	 * These fields are set by the caller of vdev_trim_ranges().
148 	 */
149 	vdev_t		*trim_vdev;		/* Leaf vdev to TRIM */
150 	metaslab_t	*trim_msp;		/* Disabled metaslab */
151 	range_tree_t	*trim_tree;		/* TRIM ranges (in metaslab) */
152 	trim_type_t	trim_type;		/* Manual or auto TRIM */
153 	uint64_t	trim_extent_bytes_max;	/* Maximum TRIM I/O size */
154 	uint64_t	trim_extent_bytes_min;	/* Minimum TRIM I/O size */
155 	enum trim_flag	trim_flags;		/* TRIM flags (secure) */
156 
157 	/*
158 	 * These fields are updated by vdev_trim_ranges().
159 	 */
160 	hrtime_t	trim_start_time;	/* Start time */
161 	uint64_t	trim_bytes_done;	/* Bytes trimmed */
162 } trim_args_t;
163 
164 /*
165  * Determines whether a vdev_trim_thread() should be stopped.
166  */
167 static boolean_t
168 vdev_trim_should_stop(vdev_t *vd)
169 {
170 	return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||
171 	    vd->vdev_detached || vd->vdev_top->vdev_removing);
172 }
173 
174 /*
175  * Determines whether a vdev_autotrim_thread() should be stopped.
176  */
177 static boolean_t
178 vdev_autotrim_should_stop(vdev_t *tvd)
179 {
180 	return (tvd->vdev_autotrim_exit_wanted ||
181 	    !vdev_writeable(tvd) || tvd->vdev_removing ||
182 	    spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF);
183 }
184 
185 /*
186  * The sync task for updating the on-disk state of a manual TRIM.  This
187  * is scheduled by vdev_trim_change_state().
188  */
189 static void
190 vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx)
191 {
192 	/*
193 	 * We pass in the guid instead of the vdev_t since the vdev may
194 	 * have been freed prior to the sync task being processed.  This
195 	 * happens when a vdev is detached as we call spa_config_vdev_exit(),
196 	 * stop the trimming thread, schedule the sync task, and free
197 	 * the vdev. Later when the scheduled sync task is invoked, it would
198 	 * find that the vdev has been freed.
199 	 */
200 	uint64_t guid = *(uint64_t *)arg;
201 	uint64_t txg = dmu_tx_get_txg(tx);
202 	kmem_free(arg, sizeof (uint64_t));
203 
204 	vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
205 	if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
206 		return;
207 
208 	uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];
209 	vd->vdev_trim_offset[txg & TXG_MASK] = 0;
210 
211 	VERIFY3U(vd->vdev_leaf_zap, !=, 0);
212 
213 	objset_t *mos = vd->vdev_spa->spa_meta_objset;
214 
215 	if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {
216 
217 		if (vd->vdev_trim_last_offset == UINT64_MAX)
218 			last_offset = 0;
219 
220 		vd->vdev_trim_last_offset = last_offset;
221 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
222 		    VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
223 		    sizeof (last_offset), 1, &last_offset, tx));
224 	}
225 
226 	if (vd->vdev_trim_action_time > 0) {
227 		uint64_t val = (uint64_t)vd->vdev_trim_action_time;
228 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
229 		    VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val),
230 		    1, &val, tx));
231 	}
232 
233 	if (vd->vdev_trim_rate > 0) {
234 		uint64_t rate = (uint64_t)vd->vdev_trim_rate;
235 
236 		if (rate == UINT64_MAX)
237 			rate = 0;
238 
239 		VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
240 		    VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx));
241 	}
242 
243 	uint64_t partial = vd->vdev_trim_partial;
244 	if (partial == UINT64_MAX)
245 		partial = 0;
246 
247 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
248 	    sizeof (partial), 1, &partial, tx));
249 
250 	uint64_t secure = vd->vdev_trim_secure;
251 	if (secure == UINT64_MAX)
252 		secure = 0;
253 
254 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
255 	    sizeof (secure), 1, &secure, tx));
256 
257 
258 	uint64_t trim_state = vd->vdev_trim_state;
259 	VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
260 	    sizeof (trim_state), 1, &trim_state, tx));
261 }
262 
263 /*
264  * Update the on-disk state of a manual TRIM.  This is called to request
265  * that a TRIM be started/suspended/canceled, or to change one of the
266  * TRIM options (partial, secure, rate).
267  */
268 static void
269 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state,
270     uint64_t rate, boolean_t partial, boolean_t secure)
271 {
272 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
273 	spa_t *spa = vd->vdev_spa;
274 
275 	if (new_state == vd->vdev_trim_state)
276 		return;
277 
278 	/*
279 	 * Copy the vd's guid, this will be freed by the sync task.
280 	 */
281 	uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
282 	*guid = vd->vdev_guid;
283 
284 	/*
285 	 * If we're suspending, then preserve the original start time.
286 	 */
287 	if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {
288 		vd->vdev_trim_action_time = gethrestime_sec();
289 	}
290 
291 	/*
292 	 * If we're activating, then preserve the requested rate and trim
293 	 * method.  Setting the last offset and rate to UINT64_MAX is used
294 	 * as a sentinel to indicate they should be reset to default values.
295 	 */
296 	if (new_state == VDEV_TRIM_ACTIVE) {
297 		if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||
298 		    vd->vdev_trim_state == VDEV_TRIM_CANCELED) {
299 			vd->vdev_trim_last_offset = UINT64_MAX;
300 			vd->vdev_trim_rate = UINT64_MAX;
301 			vd->vdev_trim_partial = UINT64_MAX;
302 			vd->vdev_trim_secure = UINT64_MAX;
303 		}
304 
305 		if (rate != 0)
306 			vd->vdev_trim_rate = rate;
307 
308 		if (partial != 0)
309 			vd->vdev_trim_partial = partial;
310 
311 		if (secure != 0)
312 			vd->vdev_trim_secure = secure;
313 	}
314 
315 	vdev_trim_state_t old_state = vd->vdev_trim_state;
316 	boolean_t resumed = (old_state == VDEV_TRIM_SUSPENDED);
317 	vd->vdev_trim_state = new_state;
318 
319 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
320 	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
321 	dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync,
322 	    guid, tx);
323 
324 	switch (new_state) {
325 	case VDEV_TRIM_ACTIVE:
326 		spa_event_notify(spa, vd, NULL,
327 		    resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START);
328 		spa_history_log_internal(spa, "trim", tx,
329 		    "vdev=%s activated", vd->vdev_path);
330 		break;
331 	case VDEV_TRIM_SUSPENDED:
332 		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND);
333 		spa_history_log_internal(spa, "trim", tx,
334 		    "vdev=%s suspended", vd->vdev_path);
335 		break;
336 	case VDEV_TRIM_CANCELED:
337 		if (old_state == VDEV_TRIM_ACTIVE ||
338 		    old_state == VDEV_TRIM_SUSPENDED) {
339 			spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL);
340 			spa_history_log_internal(spa, "trim", tx,
341 			    "vdev=%s canceled", vd->vdev_path);
342 		}
343 		break;
344 	case VDEV_TRIM_COMPLETE:
345 		spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH);
346 		spa_history_log_internal(spa, "trim", tx,
347 		    "vdev=%s complete", vd->vdev_path);
348 		break;
349 	default:
350 		panic("invalid state %llu", (unsigned long long)new_state);
351 	}
352 
353 	dmu_tx_commit(tx);
354 
355 	if (new_state != VDEV_TRIM_ACTIVE)
356 		spa_notify_waiters(spa);
357 }
358 
359 /*
360  * The zio_done_func_t done callback for each manual TRIM issued.  It is
361  * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
362  * and limiting the number of in flight TRIM I/Os.
363  */
364 static void
365 vdev_trim_cb(zio_t *zio)
366 {
367 	vdev_t *vd = zio->io_vd;
368 
369 	mutex_enter(&vd->vdev_trim_io_lock);
370 	if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
371 		/*
372 		 * The I/O failed because the vdev was unavailable; roll the
373 		 * last offset back. (This works because spa_sync waits on
374 		 * spa_txg_zio before it runs sync tasks.)
375 		 */
376 		uint64_t *offset =
377 		    &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];
378 		*offset = MIN(*offset, zio->io_offset);
379 	} else {
380 		if (zio->io_error != 0) {
381 			vd->vdev_stat.vs_trim_errors++;
382 			spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
383 			    0, 0, 0, 0, 1, zio->io_orig_size);
384 		} else {
385 			spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
386 			    1, zio->io_orig_size, 0, 0, 0, 0);
387 		}
388 
389 		vd->vdev_trim_bytes_done += zio->io_orig_size;
390 	}
391 
392 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);
393 	vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;
394 	cv_broadcast(&vd->vdev_trim_io_cv);
395 	mutex_exit(&vd->vdev_trim_io_lock);
396 
397 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
398 }
399 
400 /*
401  * The zio_done_func_t done callback for each automatic TRIM issued.  It
402  * is responsible for updating the TRIM stats and limiting the number of
403  * in flight TRIM I/Os.  Automatic TRIM I/Os are best effort and are
404  * never reissued on failure.
405  */
406 static void
407 vdev_autotrim_cb(zio_t *zio)
408 {
409 	vdev_t *vd = zio->io_vd;
410 
411 	mutex_enter(&vd->vdev_trim_io_lock);
412 
413 	if (zio->io_error != 0) {
414 		vd->vdev_stat.vs_trim_errors++;
415 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
416 		    0, 0, 0, 0, 1, zio->io_orig_size);
417 	} else {
418 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
419 		    1, zio->io_orig_size, 0, 0, 0, 0);
420 	}
421 
422 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);
423 	vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;
424 	cv_broadcast(&vd->vdev_trim_io_cv);
425 	mutex_exit(&vd->vdev_trim_io_lock);
426 
427 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
428 }
429 
430 /*
431  * The zio_done_func_t done callback for each TRIM issued via
432  * vdev_trim_simple(). It is responsible for updating the TRIM stats and
433  * limiting the number of in flight TRIM I/Os.  Simple TRIM I/Os are best
434  * effort and are never reissued on failure.
435  */
436 static void
437 vdev_trim_simple_cb(zio_t *zio)
438 {
439 	vdev_t *vd = zio->io_vd;
440 
441 	mutex_enter(&vd->vdev_trim_io_lock);
442 
443 	if (zio->io_error != 0) {
444 		vd->vdev_stat.vs_trim_errors++;
445 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,
446 		    0, 0, 0, 0, 1, zio->io_orig_size);
447 	} else {
448 		spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,
449 		    1, zio->io_orig_size, 0, 0, 0, 0);
450 	}
451 
452 	ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0);
453 	vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--;
454 	cv_broadcast(&vd->vdev_trim_io_cv);
455 	mutex_exit(&vd->vdev_trim_io_lock);
456 
457 	spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
458 }
459 /*
460  * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
461  */
462 static uint64_t
463 vdev_trim_calculate_rate(trim_args_t *ta)
464 {
465 	return (ta->trim_bytes_done * 1000 /
466 	    (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1));
467 }
468 
469 /*
470  * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
471  * and number of concurrent TRIM I/Os.
472  */
473 static int
474 vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size)
475 {
476 	vdev_t *vd = ta->trim_vdev;
477 	spa_t *spa = vd->vdev_spa;
478 	void *cb;
479 
480 	mutex_enter(&vd->vdev_trim_io_lock);
481 
482 	/*
483 	 * Limit manual TRIM I/Os to the requested rate.  This does not
484 	 * apply to automatic TRIM since no per vdev rate can be specified.
485 	 */
486 	if (ta->trim_type == TRIM_TYPE_MANUAL) {
487 		while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&
488 		    vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {
489 			cv_timedwait_idle(&vd->vdev_trim_io_cv,
490 			    &vd->vdev_trim_io_lock, ddi_get_lbolt() +
491 			    MSEC_TO_TICK(10));
492 		}
493 	}
494 	ta->trim_bytes_done += size;
495 
496 	/* Limit in flight trimming I/Os */
497 	while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] +
498 	    vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) {
499 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
500 	}
501 	vd->vdev_trim_inflight[ta->trim_type]++;
502 	mutex_exit(&vd->vdev_trim_io_lock);
503 
504 	dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
505 	VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
506 	uint64_t txg = dmu_tx_get_txg(tx);
507 
508 	spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
509 	mutex_enter(&vd->vdev_trim_lock);
510 
511 	if (ta->trim_type == TRIM_TYPE_MANUAL &&
512 	    vd->vdev_trim_offset[txg & TXG_MASK] == 0) {
513 		uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
514 		*guid = vd->vdev_guid;
515 
516 		/* This is the first write of this txg. */
517 		dsl_sync_task_nowait(spa_get_dsl(spa),
518 		    vdev_trim_zap_update_sync, guid, tx);
519 	}
520 
521 	/*
522 	 * We know the vdev_t will still be around since all consumers of
523 	 * vdev_free must stop the trimming first.
524 	 */
525 	if ((ta->trim_type == TRIM_TYPE_MANUAL &&
526 	    vdev_trim_should_stop(vd)) ||
527 	    (ta->trim_type == TRIM_TYPE_AUTO &&
528 	    vdev_autotrim_should_stop(vd->vdev_top))) {
529 		mutex_enter(&vd->vdev_trim_io_lock);
530 		vd->vdev_trim_inflight[ta->trim_type]--;
531 		mutex_exit(&vd->vdev_trim_io_lock);
532 		spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
533 		mutex_exit(&vd->vdev_trim_lock);
534 		dmu_tx_commit(tx);
535 		return (SET_ERROR(EINTR));
536 	}
537 	mutex_exit(&vd->vdev_trim_lock);
538 
539 	if (ta->trim_type == TRIM_TYPE_MANUAL)
540 		vd->vdev_trim_offset[txg & TXG_MASK] = start + size;
541 
542 	if (ta->trim_type == TRIM_TYPE_MANUAL) {
543 		cb = vdev_trim_cb;
544 	} else if (ta->trim_type == TRIM_TYPE_AUTO) {
545 		cb = vdev_autotrim_cb;
546 	} else {
547 		cb = vdev_trim_simple_cb;
548 	}
549 
550 	zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,
551 	    start, size, cb, NULL, ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL,
552 	    ta->trim_flags));
553 	/* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */
554 
555 	dmu_tx_commit(tx);
556 
557 	return (0);
558 }
559 
560 /*
561  * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
562  * Additional parameters describing how the TRIM should be performed must
563  * be set in the trim_args structure.  See the trim_args definition for
564  * additional information.
565  */
566 static int
567 vdev_trim_ranges(trim_args_t *ta)
568 {
569 	vdev_t *vd = ta->trim_vdev;
570 	zfs_btree_t *t = &ta->trim_tree->rt_root;
571 	zfs_btree_index_t idx;
572 	uint64_t extent_bytes_max = ta->trim_extent_bytes_max;
573 	uint64_t extent_bytes_min = ta->trim_extent_bytes_min;
574 	spa_t *spa = vd->vdev_spa;
575 
576 	ta->trim_start_time = gethrtime();
577 	ta->trim_bytes_done = 0;
578 
579 	for (range_seg_t *rs = zfs_btree_first(t, &idx); rs != NULL;
580 	    rs = zfs_btree_next(t, &idx, &idx)) {
581 		uint64_t size = rs_get_end(rs, ta->trim_tree) - rs_get_start(rs,
582 		    ta->trim_tree);
583 
584 		if (extent_bytes_min && size < extent_bytes_min) {
585 			spa_iostats_trim_add(spa, ta->trim_type,
586 			    0, 0, 1, size, 0, 0);
587 			continue;
588 		}
589 
590 		/* Split range into legally-sized physical chunks */
591 		uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1;
592 
593 		for (uint64_t w = 0; w < writes_required; w++) {
594 			int error;
595 
596 			error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE +
597 			    rs_get_start(rs, ta->trim_tree) +
598 			    (w *extent_bytes_max), MIN(size -
599 			    (w * extent_bytes_max), extent_bytes_max));
600 			if (error != 0) {
601 				return (error);
602 			}
603 		}
604 	}
605 
606 	return (0);
607 }
608 
609 static void
610 vdev_trim_xlate_last_rs_end(void *arg, range_seg64_t *physical_rs)
611 {
612 	uint64_t *last_rs_end = (uint64_t *)arg;
613 
614 	if (physical_rs->rs_end > *last_rs_end)
615 		*last_rs_end = physical_rs->rs_end;
616 }
617 
618 static void
619 vdev_trim_xlate_progress(void *arg, range_seg64_t *physical_rs)
620 {
621 	vdev_t *vd = (vdev_t *)arg;
622 
623 	uint64_t size = physical_rs->rs_end - physical_rs->rs_start;
624 	vd->vdev_trim_bytes_est += size;
625 
626 	if (vd->vdev_trim_last_offset >= physical_rs->rs_end) {
627 		vd->vdev_trim_bytes_done += size;
628 	} else if (vd->vdev_trim_last_offset > physical_rs->rs_start &&
629 	    vd->vdev_trim_last_offset <= physical_rs->rs_end) {
630 		vd->vdev_trim_bytes_done +=
631 		    vd->vdev_trim_last_offset - physical_rs->rs_start;
632 	}
633 }
634 
635 /*
636  * Calculates the completion percentage of a manual TRIM.
637  */
638 static void
639 vdev_trim_calculate_progress(vdev_t *vd)
640 {
641 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
642 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
643 	ASSERT(vd->vdev_leaf_zap != 0);
644 
645 	vd->vdev_trim_bytes_est = 0;
646 	vd->vdev_trim_bytes_done = 0;
647 
648 	for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
649 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
650 		mutex_enter(&msp->ms_lock);
651 
652 		uint64_t ms_free = (msp->ms_size -
653 		    metaslab_allocated_space(msp)) /
654 		    vdev_get_ndisks(vd->vdev_top);
655 
656 		/*
657 		 * Convert the metaslab range to a physical range
658 		 * on our vdev. We use this to determine if we are
659 		 * in the middle of this metaslab range.
660 		 */
661 		range_seg64_t logical_rs, physical_rs, remain_rs;
662 		logical_rs.rs_start = msp->ms_start;
663 		logical_rs.rs_end = msp->ms_start + msp->ms_size;
664 
665 		/* Metaslab space after this offset has not been trimmed. */
666 		vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs);
667 		if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {
668 			vd->vdev_trim_bytes_est += ms_free;
669 			mutex_exit(&msp->ms_lock);
670 			continue;
671 		}
672 
673 		/* Metaslab space before this offset has been trimmed */
674 		uint64_t last_rs_end = physical_rs.rs_end;
675 		if (!vdev_xlate_is_empty(&remain_rs)) {
676 			vdev_xlate_walk(vd, &remain_rs,
677 			    vdev_trim_xlate_last_rs_end, &last_rs_end);
678 		}
679 
680 		if (vd->vdev_trim_last_offset > last_rs_end) {
681 			vd->vdev_trim_bytes_done += ms_free;
682 			vd->vdev_trim_bytes_est += ms_free;
683 			mutex_exit(&msp->ms_lock);
684 			continue;
685 		}
686 
687 		/*
688 		 * If we get here, we're in the middle of trimming this
689 		 * metaslab.  Load it and walk the free tree for more
690 		 * accurate progress estimation.
691 		 */
692 		VERIFY0(metaslab_load(msp));
693 
694 		range_tree_t *rt = msp->ms_allocatable;
695 		zfs_btree_t *bt = &rt->rt_root;
696 		zfs_btree_index_t idx;
697 		for (range_seg_t *rs = zfs_btree_first(bt, &idx);
698 		    rs != NULL; rs = zfs_btree_next(bt, &idx, &idx)) {
699 			logical_rs.rs_start = rs_get_start(rs, rt);
700 			logical_rs.rs_end = rs_get_end(rs, rt);
701 
702 			vdev_xlate_walk(vd, &logical_rs,
703 			    vdev_trim_xlate_progress, vd);
704 		}
705 		mutex_exit(&msp->ms_lock);
706 	}
707 }
708 
709 /*
710  * Load from disk the vdev's manual TRIM information.  This includes the
711  * state, progress, and options provided when initiating the manual TRIM.
712  */
713 static int
714 vdev_trim_load(vdev_t *vd)
715 {
716 	int err = 0;
717 	ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
718 	    spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
719 	ASSERT(vd->vdev_leaf_zap != 0);
720 
721 	if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||
722 	    vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {
723 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
724 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
725 		    sizeof (vd->vdev_trim_last_offset), 1,
726 		    &vd->vdev_trim_last_offset);
727 		if (err == ENOENT) {
728 			vd->vdev_trim_last_offset = 0;
729 			err = 0;
730 		}
731 
732 		if (err == 0) {
733 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
734 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,
735 			    sizeof (vd->vdev_trim_rate), 1,
736 			    &vd->vdev_trim_rate);
737 			if (err == ENOENT) {
738 				vd->vdev_trim_rate = 0;
739 				err = 0;
740 			}
741 		}
742 
743 		if (err == 0) {
744 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
745 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
746 			    sizeof (vd->vdev_trim_partial), 1,
747 			    &vd->vdev_trim_partial);
748 			if (err == ENOENT) {
749 				vd->vdev_trim_partial = 0;
750 				err = 0;
751 			}
752 		}
753 
754 		if (err == 0) {
755 			err = zap_lookup(vd->vdev_spa->spa_meta_objset,
756 			    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
757 			    sizeof (vd->vdev_trim_secure), 1,
758 			    &vd->vdev_trim_secure);
759 			if (err == ENOENT) {
760 				vd->vdev_trim_secure = 0;
761 				err = 0;
762 			}
763 		}
764 	}
765 
766 	vdev_trim_calculate_progress(vd);
767 
768 	return (err);
769 }
770 
771 static void
772 vdev_trim_xlate_range_add(void *arg, range_seg64_t *physical_rs)
773 {
774 	trim_args_t *ta = arg;
775 	vdev_t *vd = ta->trim_vdev;
776 
777 	/*
778 	 * Only a manual trim will be traversing the vdev sequentially.
779 	 * For an auto trim all valid ranges should be added.
780 	 */
781 	if (ta->trim_type == TRIM_TYPE_MANUAL) {
782 
783 		/* Only add segments that we have not visited yet */
784 		if (physical_rs->rs_end <= vd->vdev_trim_last_offset)
785 			return;
786 
787 		/* Pick up where we left off mid-range. */
788 		if (vd->vdev_trim_last_offset > physical_rs->rs_start) {
789 			ASSERT3U(physical_rs->rs_end, >,
790 			    vd->vdev_trim_last_offset);
791 			physical_rs->rs_start = vd->vdev_trim_last_offset;
792 		}
793 	}
794 
795 	ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start);
796 
797 	range_tree_add(ta->trim_tree, physical_rs->rs_start,
798 	    physical_rs->rs_end - physical_rs->rs_start);
799 }
800 
801 /*
802  * Convert the logical range into physical ranges and add them to the
803  * range tree passed in the trim_args_t.
804  */
805 static void
806 vdev_trim_range_add(void *arg, uint64_t start, uint64_t size)
807 {
808 	trim_args_t *ta = arg;
809 	vdev_t *vd = ta->trim_vdev;
810 	range_seg64_t logical_rs;
811 	logical_rs.rs_start = start;
812 	logical_rs.rs_end = start + size;
813 
814 	/*
815 	 * Every range to be trimmed must be part of ms_allocatable.
816 	 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this
817 	 * is always the case.
818 	 */
819 	if (zfs_flags & ZFS_DEBUG_TRIM) {
820 		metaslab_t *msp = ta->trim_msp;
821 		VERIFY0(metaslab_load(msp));
822 		VERIFY3B(msp->ms_loaded, ==, B_TRUE);
823 		VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
824 	}
825 
826 	ASSERT(vd->vdev_ops->vdev_op_leaf);
827 	vdev_xlate_walk(vd, &logical_rs, vdev_trim_xlate_range_add, arg);
828 }
829 
830 /*
831  * Each manual TRIM thread is responsible for trimming the unallocated
832  * space for each leaf vdev.  This is accomplished by sequentially iterating
833  * over its top-level metaslabs and issuing TRIM I/O for the space described
834  * by its ms_allocatable.  While a metaslab is undergoing trimming it is
835  * not eligible for new allocations.
836  */
837 static __attribute__((noreturn)) void
838 vdev_trim_thread(void *arg)
839 {
840 	vdev_t *vd = arg;
841 	spa_t *spa = vd->vdev_spa;
842 	trim_args_t ta;
843 	int error = 0;
844 
845 	/*
846 	 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by
847 	 * vdev_trim().  Wait for the updated values to be reflected
848 	 * in the zap in order to start with the requested settings.
849 	 */
850 	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
851 
852 	ASSERT(vdev_is_concrete(vd));
853 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
854 
855 	vd->vdev_trim_last_offset = 0;
856 	vd->vdev_trim_rate = 0;
857 	vd->vdev_trim_partial = 0;
858 	vd->vdev_trim_secure = 0;
859 
860 	VERIFY0(vdev_trim_load(vd));
861 
862 	ta.trim_vdev = vd;
863 	ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
864 	ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min;
865 	ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
866 	ta.trim_type = TRIM_TYPE_MANUAL;
867 	ta.trim_flags = 0;
868 
869 	/*
870 	 * When a secure TRIM has been requested infer that the intent
871 	 * is that everything must be trimmed.  Override the default
872 	 * minimum TRIM size to prevent ranges from being skipped.
873 	 */
874 	if (vd->vdev_trim_secure) {
875 		ta.trim_flags |= ZIO_TRIM_SECURE;
876 		ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
877 	}
878 
879 	uint64_t ms_count = 0;
880 	for (uint64_t i = 0; !vd->vdev_detached &&
881 	    i < vd->vdev_top->vdev_ms_count; i++) {
882 		metaslab_t *msp = vd->vdev_top->vdev_ms[i];
883 
884 		/*
885 		 * If we've expanded the top-level vdev or it's our
886 		 * first pass, calculate our progress.
887 		 */
888 		if (vd->vdev_top->vdev_ms_count != ms_count) {
889 			vdev_trim_calculate_progress(vd);
890 			ms_count = vd->vdev_top->vdev_ms_count;
891 		}
892 
893 		spa_config_exit(spa, SCL_CONFIG, FTAG);
894 		metaslab_disable(msp);
895 		mutex_enter(&msp->ms_lock);
896 		VERIFY0(metaslab_load(msp));
897 
898 		/*
899 		 * If a partial TRIM was requested skip metaslabs which have
900 		 * never been initialized and thus have never been written.
901 		 */
902 		if (msp->ms_sm == NULL && vd->vdev_trim_partial) {
903 			mutex_exit(&msp->ms_lock);
904 			metaslab_enable(msp, B_FALSE, B_FALSE);
905 			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
906 			vdev_trim_calculate_progress(vd);
907 			continue;
908 		}
909 
910 		ta.trim_msp = msp;
911 		range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta);
912 		range_tree_vacate(msp->ms_trim, NULL, NULL);
913 		mutex_exit(&msp->ms_lock);
914 
915 		error = vdev_trim_ranges(&ta);
916 		metaslab_enable(msp, B_TRUE, B_FALSE);
917 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
918 
919 		range_tree_vacate(ta.trim_tree, NULL, NULL);
920 		if (error != 0)
921 			break;
922 	}
923 
924 	spa_config_exit(spa, SCL_CONFIG, FTAG);
925 	mutex_enter(&vd->vdev_trim_io_lock);
926 	while (vd->vdev_trim_inflight[0] > 0) {
927 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
928 	}
929 	mutex_exit(&vd->vdev_trim_io_lock);
930 
931 	range_tree_destroy(ta.trim_tree);
932 
933 	mutex_enter(&vd->vdev_trim_lock);
934 	if (!vd->vdev_trim_exit_wanted) {
935 		if (vdev_writeable(vd)) {
936 			vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
937 			    vd->vdev_trim_rate, vd->vdev_trim_partial,
938 			    vd->vdev_trim_secure);
939 		} else if (vd->vdev_faulted) {
940 			vdev_trim_change_state(vd, VDEV_TRIM_CANCELED,
941 			    vd->vdev_trim_rate, vd->vdev_trim_partial,
942 			    vd->vdev_trim_secure);
943 		}
944 	}
945 	ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);
946 
947 	/*
948 	 * Drop the vdev_trim_lock while we sync out the txg since it's
949 	 * possible that a device might be trying to come online and must
950 	 * check to see if it needs to restart a trim. That thread will be
951 	 * holding the spa_config_lock which would prevent the txg_wait_synced
952 	 * from completing.
953 	 */
954 	mutex_exit(&vd->vdev_trim_lock);
955 	txg_wait_synced(spa_get_dsl(spa), 0);
956 	mutex_enter(&vd->vdev_trim_lock);
957 
958 	vd->vdev_trim_thread = NULL;
959 	cv_broadcast(&vd->vdev_trim_cv);
960 	mutex_exit(&vd->vdev_trim_lock);
961 
962 	thread_exit();
963 }
964 
965 /*
966  * Initiates a manual TRIM for the vdev_t.  Callers must hold vdev_trim_lock,
967  * the vdev_t must be a leaf and cannot already be manually trimming.
968  */
969 void
970 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)
971 {
972 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
973 	ASSERT(vd->vdev_ops->vdev_op_leaf);
974 	ASSERT(vdev_is_concrete(vd));
975 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
976 	ASSERT(!vd->vdev_detached);
977 	ASSERT(!vd->vdev_trim_exit_wanted);
978 	ASSERT(!vd->vdev_top->vdev_removing);
979 
980 	vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure);
981 	vd->vdev_trim_thread = thread_create(NULL, 0,
982 	    vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
983 }
984 
985 /*
986  * Wait for the trimming thread to be terminated (canceled or stopped).
987  */
988 static void
989 vdev_trim_stop_wait_impl(vdev_t *vd)
990 {
991 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
992 
993 	while (vd->vdev_trim_thread != NULL)
994 		cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);
995 
996 	ASSERT3P(vd->vdev_trim_thread, ==, NULL);
997 	vd->vdev_trim_exit_wanted = B_FALSE;
998 }
999 
1000 /*
1001  * Wait for vdev trim threads which were listed to cleanly exit.
1002  */
1003 void
1004 vdev_trim_stop_wait(spa_t *spa, list_t *vd_list)
1005 {
1006 	(void) spa;
1007 	vdev_t *vd;
1008 
1009 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1010 
1011 	while ((vd = list_remove_head(vd_list)) != NULL) {
1012 		mutex_enter(&vd->vdev_trim_lock);
1013 		vdev_trim_stop_wait_impl(vd);
1014 		mutex_exit(&vd->vdev_trim_lock);
1015 	}
1016 }
1017 
1018 /*
1019  * Stop trimming a device, with the resultant trimming state being tgt_state.
1020  * For blocking behavior pass NULL for vd_list.  Otherwise, when a list_t is
1021  * provided the stopping vdev is inserted in to the list.  Callers are then
1022  * required to call vdev_trim_stop_wait() to block for all the trim threads
1023  * to exit.  The caller must hold vdev_trim_lock and must not be writing to
1024  * the spa config, as the trimming thread may try to enter the config as a
1025  * reader before exiting.
1026  */
1027 void
1028 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list)
1029 {
1030 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
1031 	ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
1032 	ASSERT(vd->vdev_ops->vdev_op_leaf);
1033 	ASSERT(vdev_is_concrete(vd));
1034 
1035 	/*
1036 	 * Allow cancel requests to proceed even if the trim thread has
1037 	 * stopped.
1038 	 */
1039 	if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)
1040 		return;
1041 
1042 	vdev_trim_change_state(vd, tgt_state, 0, 0, 0);
1043 	vd->vdev_trim_exit_wanted = B_TRUE;
1044 
1045 	if (vd_list == NULL) {
1046 		vdev_trim_stop_wait_impl(vd);
1047 	} else {
1048 		ASSERT(MUTEX_HELD(&spa_namespace_lock));
1049 		list_insert_tail(vd_list, vd);
1050 	}
1051 }
1052 
1053 /*
1054  * Requests that all listed vdevs stop trimming.
1055  */
1056 static void
1057 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state,
1058     list_t *vd_list)
1059 {
1060 	if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
1061 		mutex_enter(&vd->vdev_trim_lock);
1062 		vdev_trim_stop(vd, tgt_state, vd_list);
1063 		mutex_exit(&vd->vdev_trim_lock);
1064 		return;
1065 	}
1066 
1067 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
1068 		vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,
1069 		    vd_list);
1070 	}
1071 }
1072 
1073 /*
1074  * Convenience function to stop trimming of a vdev tree and set all trim
1075  * thread pointers to NULL.
1076  */
1077 void
1078 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state)
1079 {
1080 	spa_t *spa = vd->vdev_spa;
1081 	list_t vd_list;
1082 	vdev_t *vd_l2cache;
1083 
1084 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1085 
1086 	list_create(&vd_list, sizeof (vdev_t),
1087 	    offsetof(vdev_t, vdev_trim_node));
1088 
1089 	vdev_trim_stop_all_impl(vd, tgt_state, &vd_list);
1090 
1091 	/*
1092 	 * Iterate over cache devices and request stop trimming the
1093 	 * whole device in case we export the pool or remove the cache
1094 	 * device prematurely.
1095 	 */
1096 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1097 		vd_l2cache = spa->spa_l2cache.sav_vdevs[i];
1098 		vdev_trim_stop_all_impl(vd_l2cache, tgt_state, &vd_list);
1099 	}
1100 
1101 	vdev_trim_stop_wait(spa, &vd_list);
1102 
1103 	if (vd->vdev_spa->spa_sync_on) {
1104 		/* Make sure that our state has been synced to disk */
1105 		txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1106 	}
1107 
1108 	list_destroy(&vd_list);
1109 }
1110 
1111 /*
1112  * Conditionally restarts a manual TRIM given its on-disk state.
1113  */
1114 void
1115 vdev_trim_restart(vdev_t *vd)
1116 {
1117 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1118 	ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
1119 
1120 	if (vd->vdev_leaf_zap != 0) {
1121 		mutex_enter(&vd->vdev_trim_lock);
1122 		uint64_t trim_state = VDEV_TRIM_NONE;
1123 		int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1124 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
1125 		    sizeof (trim_state), 1, &trim_state);
1126 		ASSERT(err == 0 || err == ENOENT);
1127 		vd->vdev_trim_state = trim_state;
1128 
1129 		uint64_t timestamp = 0;
1130 		err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1131 		    vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,
1132 		    sizeof (timestamp), 1, &timestamp);
1133 		ASSERT(err == 0 || err == ENOENT);
1134 		vd->vdev_trim_action_time = timestamp;
1135 
1136 		if (vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||
1137 		    vd->vdev_offline) {
1138 			/* load progress for reporting, but don't resume */
1139 			VERIFY0(vdev_trim_load(vd));
1140 		} else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&
1141 		    vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&
1142 		    vd->vdev_trim_thread == NULL) {
1143 			VERIFY0(vdev_trim_load(vd));
1144 			vdev_trim(vd, vd->vdev_trim_rate,
1145 			    vd->vdev_trim_partial, vd->vdev_trim_secure);
1146 		}
1147 
1148 		mutex_exit(&vd->vdev_trim_lock);
1149 	}
1150 
1151 	for (uint64_t i = 0; i < vd->vdev_children; i++) {
1152 		vdev_trim_restart(vd->vdev_child[i]);
1153 	}
1154 }
1155 
1156 /*
1157  * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1158  * every TRIM range is contained within ms_allocatable.
1159  */
1160 static void
1161 vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size)
1162 {
1163 	trim_args_t *ta = arg;
1164 	metaslab_t *msp = ta->trim_msp;
1165 
1166 	VERIFY3B(msp->ms_loaded, ==, B_TRUE);
1167 	VERIFY3U(msp->ms_disabled, >, 0);
1168 	VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
1169 }
1170 
1171 /*
1172  * Each automatic TRIM thread is responsible for managing the trimming of a
1173  * top-level vdev in the pool.  No automatic TRIM state is maintained on-disk.
1174  *
1175  * N.B. This behavior is different from a manual TRIM where a thread
1176  * is created for each leaf vdev, instead of each top-level vdev.
1177  */
1178 static __attribute__((noreturn)) void
1179 vdev_autotrim_thread(void *arg)
1180 {
1181 	vdev_t *vd = arg;
1182 	spa_t *spa = vd->vdev_spa;
1183 	int shift = 0;
1184 
1185 	mutex_enter(&vd->vdev_autotrim_lock);
1186 	ASSERT3P(vd->vdev_top, ==, vd);
1187 	ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);
1188 	mutex_exit(&vd->vdev_autotrim_lock);
1189 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1190 
1191 	while (!vdev_autotrim_should_stop(vd)) {
1192 		int txgs_per_trim = MAX(zfs_trim_txg_batch, 1);
1193 		boolean_t issued_trim = B_FALSE;
1194 		uint64_t extent_bytes_max = zfs_trim_extent_bytes_max;
1195 		uint64_t extent_bytes_min = zfs_trim_extent_bytes_min;
1196 
1197 		/*
1198 		 * All of the metaslabs are divided in to groups of size
1199 		 * num_metaslabs / zfs_trim_txg_batch.  Each of these groups
1200 		 * is composed of metaslabs which are spread evenly over the
1201 		 * device.
1202 		 *
1203 		 * For example, when zfs_trim_txg_batch = 32 (default) then
1204 		 * group 0 will contain metaslabs 0, 32, 64, ...;
1205 		 * group 1 will contain metaslabs 1, 33, 65, ...;
1206 		 * group 2 will contain metaslabs 2, 34, 66, ...; and so on.
1207 		 *
1208 		 * On each pass through the while() loop one of these groups
1209 		 * is selected.  This is accomplished by using a shift value
1210 		 * to select the starting metaslab, then striding over the
1211 		 * metaslabs using the zfs_trim_txg_batch size.  This is
1212 		 * done to accomplish two things.
1213 		 *
1214 		 * 1) By dividing the metaslabs in to groups, and making sure
1215 		 *    that each group takes a minimum of one txg to process.
1216 		 *    Then zfs_trim_txg_batch controls the minimum number of
1217 		 *    txgs which must occur before a metaslab is revisited.
1218 		 *
1219 		 * 2) Selecting non-consecutive metaslabs distributes the
1220 		 *    TRIM commands for a group evenly over the entire device.
1221 		 *    This can be advantageous for certain types of devices.
1222 		 */
1223 		for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;
1224 		    i += txgs_per_trim) {
1225 			metaslab_t *msp = vd->vdev_ms[i];
1226 			range_tree_t *trim_tree;
1227 
1228 			spa_config_exit(spa, SCL_CONFIG, FTAG);
1229 			metaslab_disable(msp);
1230 			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1231 
1232 			mutex_enter(&msp->ms_lock);
1233 
1234 			/*
1235 			 * Skip the metaslab when it has never been allocated
1236 			 * or when there are no recent frees to trim.
1237 			 */
1238 			if (msp->ms_sm == NULL ||
1239 			    range_tree_is_empty(msp->ms_trim)) {
1240 				mutex_exit(&msp->ms_lock);
1241 				metaslab_enable(msp, B_FALSE, B_FALSE);
1242 				continue;
1243 			}
1244 
1245 			/*
1246 			 * Skip the metaslab when it has already been disabled.
1247 			 * This may happen when a manual TRIM or initialize
1248 			 * operation is running concurrently.  In the case
1249 			 * of a manual TRIM, the ms_trim tree will have been
1250 			 * vacated.  Only ranges added after the manual TRIM
1251 			 * disabled the metaslab will be included in the tree.
1252 			 * These will be processed when the automatic TRIM
1253 			 * next revisits this metaslab.
1254 			 */
1255 			if (msp->ms_disabled > 1) {
1256 				mutex_exit(&msp->ms_lock);
1257 				metaslab_enable(msp, B_FALSE, B_FALSE);
1258 				continue;
1259 			}
1260 
1261 			/*
1262 			 * Allocate an empty range tree which is swapped in
1263 			 * for the existing ms_trim tree while it is processed.
1264 			 */
1265 			trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL,
1266 			    0, 0);
1267 			range_tree_swap(&msp->ms_trim, &trim_tree);
1268 			ASSERT(range_tree_is_empty(msp->ms_trim));
1269 
1270 			/*
1271 			 * There are two cases when constructing the per-vdev
1272 			 * trim trees for a metaslab.  If the top-level vdev
1273 			 * has no children then it is also a leaf and should
1274 			 * be trimmed.  Otherwise our children are the leaves
1275 			 * and a trim tree should be constructed for each.
1276 			 */
1277 			trim_args_t *tap;
1278 			uint64_t children = vd->vdev_children;
1279 			if (children == 0) {
1280 				children = 1;
1281 				tap = kmem_zalloc(sizeof (trim_args_t) *
1282 				    children, KM_SLEEP);
1283 				tap[0].trim_vdev = vd;
1284 			} else {
1285 				tap = kmem_zalloc(sizeof (trim_args_t) *
1286 				    children, KM_SLEEP);
1287 
1288 				for (uint64_t c = 0; c < children; c++) {
1289 					tap[c].trim_vdev = vd->vdev_child[c];
1290 				}
1291 			}
1292 
1293 			for (uint64_t c = 0; c < children; c++) {
1294 				trim_args_t *ta = &tap[c];
1295 				vdev_t *cvd = ta->trim_vdev;
1296 
1297 				ta->trim_msp = msp;
1298 				ta->trim_extent_bytes_max = extent_bytes_max;
1299 				ta->trim_extent_bytes_min = extent_bytes_min;
1300 				ta->trim_type = TRIM_TYPE_AUTO;
1301 				ta->trim_flags = 0;
1302 
1303 				if (cvd->vdev_detached ||
1304 				    !vdev_writeable(cvd) ||
1305 				    !cvd->vdev_has_trim ||
1306 				    cvd->vdev_trim_thread != NULL) {
1307 					continue;
1308 				}
1309 
1310 				/*
1311 				 * When a device has an attached hot spare, or
1312 				 * is being replaced it will not be trimmed.
1313 				 * This is done to avoid adding additional
1314 				 * stress to a potentially unhealthy device,
1315 				 * and to minimize the required rebuild time.
1316 				 */
1317 				if (!cvd->vdev_ops->vdev_op_leaf)
1318 					continue;
1319 
1320 				ta->trim_tree = range_tree_create(NULL,
1321 				    RANGE_SEG64, NULL, 0, 0);
1322 				range_tree_walk(trim_tree,
1323 				    vdev_trim_range_add, ta);
1324 			}
1325 
1326 			mutex_exit(&msp->ms_lock);
1327 			spa_config_exit(spa, SCL_CONFIG, FTAG);
1328 
1329 			/*
1330 			 * Issue the TRIM I/Os for all ranges covered by the
1331 			 * TRIM trees.  These ranges are safe to TRIM because
1332 			 * no new allocations will be performed until the call
1333 			 * to metaslab_enabled() below.
1334 			 */
1335 			for (uint64_t c = 0; c < children; c++) {
1336 				trim_args_t *ta = &tap[c];
1337 
1338 				/*
1339 				 * Always yield to a manual TRIM if one has
1340 				 * been started for the child vdev.
1341 				 */
1342 				if (ta->trim_tree == NULL ||
1343 				    ta->trim_vdev->vdev_trim_thread != NULL) {
1344 					continue;
1345 				}
1346 
1347 				/*
1348 				 * After this point metaslab_enable() must be
1349 				 * called with the sync flag set.  This is done
1350 				 * here because vdev_trim_ranges() is allowed
1351 				 * to be interrupted (EINTR) before issuing all
1352 				 * of the required TRIM I/Os.
1353 				 */
1354 				issued_trim = B_TRUE;
1355 
1356 				int error = vdev_trim_ranges(ta);
1357 				if (error)
1358 					break;
1359 			}
1360 
1361 			/*
1362 			 * Verify every range which was trimmed is still
1363 			 * contained within the ms_allocatable tree.
1364 			 */
1365 			if (zfs_flags & ZFS_DEBUG_TRIM) {
1366 				mutex_enter(&msp->ms_lock);
1367 				VERIFY0(metaslab_load(msp));
1368 				VERIFY3P(tap[0].trim_msp, ==, msp);
1369 				range_tree_walk(trim_tree,
1370 				    vdev_trim_range_verify, &tap[0]);
1371 				mutex_exit(&msp->ms_lock);
1372 			}
1373 
1374 			range_tree_vacate(trim_tree, NULL, NULL);
1375 			range_tree_destroy(trim_tree);
1376 
1377 			metaslab_enable(msp, issued_trim, B_FALSE);
1378 			spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1379 
1380 			for (uint64_t c = 0; c < children; c++) {
1381 				trim_args_t *ta = &tap[c];
1382 
1383 				if (ta->trim_tree == NULL)
1384 					continue;
1385 
1386 				range_tree_vacate(ta->trim_tree, NULL, NULL);
1387 				range_tree_destroy(ta->trim_tree);
1388 			}
1389 
1390 			kmem_free(tap, sizeof (trim_args_t) * children);
1391 		}
1392 
1393 		spa_config_exit(spa, SCL_CONFIG, FTAG);
1394 
1395 		/*
1396 		 * After completing the group of metaslabs wait for the next
1397 		 * open txg.  This is done to make sure that a minimum of
1398 		 * zfs_trim_txg_batch txgs will occur before these metaslabs
1399 		 * are trimmed again.
1400 		 */
1401 		txg_wait_open(spa_get_dsl(spa), 0, issued_trim);
1402 
1403 		shift++;
1404 		spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1405 	}
1406 
1407 	for (uint64_t c = 0; c < vd->vdev_children; c++) {
1408 		vdev_t *cvd = vd->vdev_child[c];
1409 		mutex_enter(&cvd->vdev_trim_io_lock);
1410 
1411 		while (cvd->vdev_trim_inflight[1] > 0) {
1412 			cv_wait(&cvd->vdev_trim_io_cv,
1413 			    &cvd->vdev_trim_io_lock);
1414 		}
1415 		mutex_exit(&cvd->vdev_trim_io_lock);
1416 	}
1417 
1418 	spa_config_exit(spa, SCL_CONFIG, FTAG);
1419 
1420 	/*
1421 	 * When exiting because the autotrim property was set to off, then
1422 	 * abandon any unprocessed ms_trim ranges to reclaim the memory.
1423 	 */
1424 	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) {
1425 		for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {
1426 			metaslab_t *msp = vd->vdev_ms[i];
1427 
1428 			mutex_enter(&msp->ms_lock);
1429 			range_tree_vacate(msp->ms_trim, NULL, NULL);
1430 			mutex_exit(&msp->ms_lock);
1431 		}
1432 	}
1433 
1434 	mutex_enter(&vd->vdev_autotrim_lock);
1435 	ASSERT(vd->vdev_autotrim_thread != NULL);
1436 	vd->vdev_autotrim_thread = NULL;
1437 	cv_broadcast(&vd->vdev_autotrim_cv);
1438 	mutex_exit(&vd->vdev_autotrim_lock);
1439 
1440 	thread_exit();
1441 }
1442 
1443 /*
1444  * Starts an autotrim thread, if needed, for each top-level vdev which can be
1445  * trimmed.  A top-level vdev which has been evacuated will never be trimmed.
1446  */
1447 void
1448 vdev_autotrim(spa_t *spa)
1449 {
1450 	vdev_t *root_vd = spa->spa_root_vdev;
1451 
1452 	for (uint64_t i = 0; i < root_vd->vdev_children; i++) {
1453 		vdev_t *tvd = root_vd->vdev_child[i];
1454 
1455 		mutex_enter(&tvd->vdev_autotrim_lock);
1456 		if (vdev_writeable(tvd) && !tvd->vdev_removing &&
1457 		    tvd->vdev_autotrim_thread == NULL) {
1458 			ASSERT3P(tvd->vdev_top, ==, tvd);
1459 
1460 			tvd->vdev_autotrim_thread = thread_create(NULL, 0,
1461 			    vdev_autotrim_thread, tvd, 0, &p0, TS_RUN,
1462 			    maxclsyspri);
1463 			ASSERT(tvd->vdev_autotrim_thread != NULL);
1464 		}
1465 		mutex_exit(&tvd->vdev_autotrim_lock);
1466 	}
1467 }
1468 
1469 /*
1470  * Wait for the vdev_autotrim_thread associated with the passed top-level
1471  * vdev to be terminated (canceled or stopped).
1472  */
1473 void
1474 vdev_autotrim_stop_wait(vdev_t *tvd)
1475 {
1476 	mutex_enter(&tvd->vdev_autotrim_lock);
1477 	if (tvd->vdev_autotrim_thread != NULL) {
1478 		tvd->vdev_autotrim_exit_wanted = B_TRUE;
1479 
1480 		while (tvd->vdev_autotrim_thread != NULL) {
1481 			cv_wait(&tvd->vdev_autotrim_cv,
1482 			    &tvd->vdev_autotrim_lock);
1483 		}
1484 
1485 		ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL);
1486 		tvd->vdev_autotrim_exit_wanted = B_FALSE;
1487 	}
1488 	mutex_exit(&tvd->vdev_autotrim_lock);
1489 }
1490 
1491 /*
1492  * Wait for all of the vdev_autotrim_thread associated with the pool to
1493  * be terminated (canceled or stopped).
1494  */
1495 void
1496 vdev_autotrim_stop_all(spa_t *spa)
1497 {
1498 	vdev_t *root_vd = spa->spa_root_vdev;
1499 
1500 	for (uint64_t i = 0; i < root_vd->vdev_children; i++)
1501 		vdev_autotrim_stop_wait(root_vd->vdev_child[i]);
1502 }
1503 
1504 /*
1505  * Conditionally restart all of the vdev_autotrim_thread's for the pool.
1506  */
1507 void
1508 vdev_autotrim_restart(spa_t *spa)
1509 {
1510 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1511 
1512 	if (spa->spa_autotrim)
1513 		vdev_autotrim(spa);
1514 }
1515 
1516 static __attribute__((noreturn)) void
1517 vdev_trim_l2arc_thread(void *arg)
1518 {
1519 	vdev_t		*vd = arg;
1520 	spa_t		*spa = vd->vdev_spa;
1521 	l2arc_dev_t	*dev = l2arc_vdev_get(vd);
1522 	trim_args_t	ta = {0};
1523 	range_seg64_t 	physical_rs;
1524 
1525 	ASSERT(vdev_is_concrete(vd));
1526 	spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1527 
1528 	vd->vdev_trim_last_offset = 0;
1529 	vd->vdev_trim_rate = 0;
1530 	vd->vdev_trim_partial = 0;
1531 	vd->vdev_trim_secure = 0;
1532 
1533 	ta.trim_vdev = vd;
1534 	ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
1535 	ta.trim_type = TRIM_TYPE_MANUAL;
1536 	ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
1537 	ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
1538 	ta.trim_flags = 0;
1539 
1540 	physical_rs.rs_start = vd->vdev_trim_bytes_done = 0;
1541 	physical_rs.rs_end = vd->vdev_trim_bytes_est =
1542 	    vdev_get_min_asize(vd);
1543 
1544 	range_tree_add(ta.trim_tree, physical_rs.rs_start,
1545 	    physical_rs.rs_end - physical_rs.rs_start);
1546 
1547 	mutex_enter(&vd->vdev_trim_lock);
1548 	vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);
1549 	mutex_exit(&vd->vdev_trim_lock);
1550 
1551 	(void) vdev_trim_ranges(&ta);
1552 
1553 	spa_config_exit(spa, SCL_CONFIG, FTAG);
1554 	mutex_enter(&vd->vdev_trim_io_lock);
1555 	while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) {
1556 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
1557 	}
1558 	mutex_exit(&vd->vdev_trim_io_lock);
1559 
1560 	range_tree_vacate(ta.trim_tree, NULL, NULL);
1561 	range_tree_destroy(ta.trim_tree);
1562 
1563 	mutex_enter(&vd->vdev_trim_lock);
1564 	if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {
1565 		vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
1566 		    vd->vdev_trim_rate, vd->vdev_trim_partial,
1567 		    vd->vdev_trim_secure);
1568 	}
1569 	ASSERT(vd->vdev_trim_thread != NULL ||
1570 	    vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0);
1571 
1572 	/*
1573 	 * Drop the vdev_trim_lock while we sync out the txg since it's
1574 	 * possible that a device might be trying to come online and
1575 	 * must check to see if it needs to restart a trim. That thread
1576 	 * will be holding the spa_config_lock which would prevent the
1577 	 * txg_wait_synced from completing. Same strategy as in
1578 	 * vdev_trim_thread().
1579 	 */
1580 	mutex_exit(&vd->vdev_trim_lock);
1581 	txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1582 	mutex_enter(&vd->vdev_trim_lock);
1583 
1584 	/*
1585 	 * Update the header of the cache device here, before
1586 	 * broadcasting vdev_trim_cv which may lead to the removal
1587 	 * of the device. The same applies for setting l2ad_trim_all to
1588 	 * false.
1589 	 */
1590 	spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd,
1591 	    RW_READER);
1592 	memset(dev->l2ad_dev_hdr, 0, dev->l2ad_dev_hdr_asize);
1593 	l2arc_dev_hdr_update(dev);
1594 	spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd);
1595 
1596 	vd->vdev_trim_thread = NULL;
1597 	if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE)
1598 		dev->l2ad_trim_all = B_FALSE;
1599 
1600 	cv_broadcast(&vd->vdev_trim_cv);
1601 	mutex_exit(&vd->vdev_trim_lock);
1602 
1603 	thread_exit();
1604 }
1605 
1606 /*
1607  * Punches out TRIM threads for the L2ARC devices in a spa and assigns them
1608  * to vd->vdev_trim_thread variable. This facilitates the management of
1609  * trimming the whole cache device using TRIM_TYPE_MANUAL upon addition
1610  * to a pool or pool creation or when the header of the device is invalid.
1611  */
1612 void
1613 vdev_trim_l2arc(spa_t *spa)
1614 {
1615 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
1616 
1617 	/*
1618 	 * Locate the spa's l2arc devices and kick off TRIM threads.
1619 	 */
1620 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1621 		vdev_t *vd = spa->spa_l2cache.sav_vdevs[i];
1622 		l2arc_dev_t *dev = l2arc_vdev_get(vd);
1623 
1624 		if (dev == NULL || !dev->l2ad_trim_all) {
1625 			/*
1626 			 * Don't attempt TRIM if the vdev is UNAVAIL or if the
1627 			 * cache device was not marked for whole device TRIM
1628 			 * (ie l2arc_trim_ahead = 0, or the L2ARC device header
1629 			 * is valid with trim_state = VDEV_TRIM_COMPLETE and
1630 			 * l2ad_log_entries > 0).
1631 			 */
1632 			continue;
1633 		}
1634 
1635 		mutex_enter(&vd->vdev_trim_lock);
1636 		ASSERT(vd->vdev_ops->vdev_op_leaf);
1637 		ASSERT(vdev_is_concrete(vd));
1638 		ASSERT3P(vd->vdev_trim_thread, ==, NULL);
1639 		ASSERT(!vd->vdev_detached);
1640 		ASSERT(!vd->vdev_trim_exit_wanted);
1641 		ASSERT(!vd->vdev_top->vdev_removing);
1642 		vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);
1643 		vd->vdev_trim_thread = thread_create(NULL, 0,
1644 		    vdev_trim_l2arc_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
1645 		mutex_exit(&vd->vdev_trim_lock);
1646 	}
1647 }
1648 
1649 /*
1650  * A wrapper which calls vdev_trim_ranges(). It is intended to be called
1651  * on leaf vdevs.
1652  */
1653 int
1654 vdev_trim_simple(vdev_t *vd, uint64_t start, uint64_t size)
1655 {
1656 	trim_args_t ta = {0};
1657 	range_seg64_t physical_rs;
1658 	int error;
1659 	physical_rs.rs_start = start;
1660 	physical_rs.rs_end = start + size;
1661 
1662 	ASSERT(vdev_is_concrete(vd));
1663 	ASSERT(vd->vdev_ops->vdev_op_leaf);
1664 	ASSERT(!vd->vdev_detached);
1665 	ASSERT(!vd->vdev_top->vdev_removing);
1666 
1667 	ta.trim_vdev = vd;
1668 	ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
1669 	ta.trim_type = TRIM_TYPE_SIMPLE;
1670 	ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
1671 	ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
1672 	ta.trim_flags = 0;
1673 
1674 	ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
1675 
1676 	if (physical_rs.rs_end > physical_rs.rs_start) {
1677 		range_tree_add(ta.trim_tree, physical_rs.rs_start,
1678 		    physical_rs.rs_end - physical_rs.rs_start);
1679 	} else {
1680 		ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
1681 	}
1682 
1683 	error = vdev_trim_ranges(&ta);
1684 
1685 	mutex_enter(&vd->vdev_trim_io_lock);
1686 	while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) {
1687 		cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
1688 	}
1689 	mutex_exit(&vd->vdev_trim_io_lock);
1690 
1691 	range_tree_vacate(ta.trim_tree, NULL, NULL);
1692 	range_tree_destroy(ta.trim_tree);
1693 
1694 	return (error);
1695 }
1696 
1697 EXPORT_SYMBOL(vdev_trim);
1698 EXPORT_SYMBOL(vdev_trim_stop);
1699 EXPORT_SYMBOL(vdev_trim_stop_all);
1700 EXPORT_SYMBOL(vdev_trim_stop_wait);
1701 EXPORT_SYMBOL(vdev_trim_restart);
1702 EXPORT_SYMBOL(vdev_autotrim);
1703 EXPORT_SYMBOL(vdev_autotrim_stop_all);
1704 EXPORT_SYMBOL(vdev_autotrim_stop_wait);
1705 EXPORT_SYMBOL(vdev_autotrim_restart);
1706 EXPORT_SYMBOL(vdev_trim_l2arc);
1707 EXPORT_SYMBOL(vdev_trim_simple);
1708 
1709 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_max, UINT, ZMOD_RW,
1710 	"Max size of TRIM commands, larger will be split");
1711 
1712 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_min, UINT, ZMOD_RW,
1713 	"Min size of TRIM commands, smaller will be skipped");
1714 
1715 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, metaslab_skip, UINT, ZMOD_RW,
1716 	"Skip metaslabs which have never been initialized");
1717 
1718 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, txg_batch, UINT, ZMOD_RW,
1719 	"Min number of txgs to aggregate frees before issuing TRIM");
1720 
1721 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, queue_limit, UINT, ZMOD_RW,
1722 	"Max queued TRIMs outstanding per leaf vdev");
1723