1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2016, 2024 by Delphix. All rights reserved.
25 * Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
26 * Copyright (c) 2021 Hewlett Packard Enterprise Development LP
27 * Copyright 2023 RackTop Systems, Inc.
28 */
29
30 #include <sys/spa.h>
31 #include <sys/spa_impl.h>
32 #include <sys/txg.h>
33 #include <sys/vdev_impl.h>
34 #include <sys/vdev_trim.h>
35 #include <sys/metaslab_impl.h>
36 #include <sys/dsl_synctask.h>
37 #include <sys/zap.h>
38 #include <sys/dmu_tx.h>
39 #include <sys/arc_impl.h>
40
41 /*
42 * TRIM is a feature which is used to notify a SSD that some previously
43 * written space is no longer allocated by the pool. This is useful because
44 * writes to a SSD must be performed to blocks which have first been erased.
45 * Ensuring the SSD always has a supply of erased blocks for new writes
46 * helps prevent the performance from deteriorating.
47 *
48 * There are two supported TRIM methods; manual and automatic.
49 *
50 * Manual TRIM:
51 *
52 * A manual TRIM is initiated by running the 'zpool trim' command. A single
53 * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for
54 * managing that vdev TRIM process. This involves iterating over all the
55 * metaslabs, calculating the unallocated space ranges, and then issuing the
56 * required TRIM I/Os.
57 *
58 * While a metaslab is being actively trimmed it is not eligible to perform
59 * new allocations. After traversing all of the metaslabs the thread is
60 * terminated. Finally, both the requested options and current progress of
61 * the TRIM are regularly written to the pool. This allows the TRIM to be
62 * suspended and resumed as needed.
63 *
64 * Automatic TRIM:
65 *
66 * An automatic TRIM is enabled by setting the 'autotrim' pool property
67 * to 'on'. When enabled, a `vdev_autotrim' thread is created for each
68 * top-level (not leaf) vdev in the pool. These threads perform the same
69 * core TRIM process as a manual TRIM, but with a few key differences.
70 *
71 * 1) Automatic TRIM happens continuously in the background and operates
72 * solely on recently freed blocks (ms_trim not ms_allocatable).
73 *
74 * 2) Each thread is associated with a top-level (not leaf) vdev. This has
75 * the benefit of simplifying the threading model, it makes it easier
76 * to coordinate administrative commands, and it ensures only a single
77 * metaslab is disabled at a time. Unlike manual TRIM, this means each
78 * 'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
79 * children.
80 *
81 * 3) There is no automatic TRIM progress information stored on disk, nor
82 * is it reported by 'zpool status'.
83 *
84 * While the automatic TRIM process is highly effective it is more likely
85 * than a manual TRIM to encounter tiny ranges. Ranges less than or equal to
86 * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently
87 * TRIM and are skipped. This means small amounts of freed space may not
88 * be automatically trimmed.
89 *
90 * Furthermore, devices with attached hot spares and devices being actively
91 * replaced are skipped. This is done to avoid adding additional stress to
92 * a potentially unhealthy device and to minimize the required rebuild time.
93 *
94 * For this reason it may be beneficial to occasionally manually TRIM a pool
95 * even when automatic TRIM is enabled.
96 */
97
98 /*
99 * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
100 */
101 static unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024;
102
103 /*
104 * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
105 */
106 static unsigned int zfs_trim_extent_bytes_min = 32 * 1024;
107
108 /*
109 * Skip uninitialized metaslabs during the TRIM process. This option is
110 * useful for pools constructed from large thinly-provisioned devices where
111 * TRIM operations are slow. As a pool ages an increasing fraction of
112 * the pools metaslabs will be initialized progressively degrading the
113 * usefulness of this option. This setting is stored when starting a
114 * manual TRIM and will persist for the duration of the requested TRIM.
115 */
116 unsigned int zfs_trim_metaslab_skip = 0;
117
118 /*
119 * Maximum number of queued TRIM I/Os per leaf vdev. The number of
120 * concurrent TRIM I/Os issued to the device is controlled by the
121 * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options.
122 */
123 static unsigned int zfs_trim_queue_limit = 10;
124
125 /*
126 * The minimum number of transaction groups between automatic trims of a
127 * metaslab. This setting represents a trade-off between issuing more
128 * efficient TRIM operations, by allowing them to be aggregated longer,
129 * and issuing them promptly so the trimmed space is available. Note
130 * that this value is a minimum; metaslabs can be trimmed less frequently
131 * when there are a large number of ranges which need to be trimmed.
132 *
133 * Increasing this value will allow frees to be aggregated for a longer
134 * time. This can result is larger TRIM operations, and increased memory
135 * usage in order to track the ranges to be trimmed. Decreasing this value
136 * has the opposite effect. The default value of 32 was determined though
137 * testing to be a reasonable compromise.
138 */
139 static unsigned int zfs_trim_txg_batch = 32;
140
141 /*
142 * The trim_args are a control structure which describe how a leaf vdev
143 * should be trimmed. The core elements are the vdev, the metaslab being
144 * trimmed and a range tree containing the extents to TRIM. All provided
145 * ranges must be within the metaslab.
146 */
147 typedef struct trim_args {
148 /*
149 * These fields are set by the caller of vdev_trim_ranges().
150 */
151 vdev_t *trim_vdev; /* Leaf vdev to TRIM */
152 metaslab_t *trim_msp; /* Disabled metaslab */
153 zfs_range_tree_t *trim_tree; /* TRIM ranges (in metaslab) */
154 trim_type_t trim_type; /* Manual or auto TRIM */
155 uint64_t trim_extent_bytes_max; /* Maximum TRIM I/O size */
156 uint64_t trim_extent_bytes_min; /* Minimum TRIM I/O size */
157 enum trim_flag trim_flags; /* TRIM flags (secure) */
158
159 /*
160 * These fields are updated by vdev_trim_ranges().
161 */
162 hrtime_t trim_start_time; /* Start time */
163 uint64_t trim_bytes_done; /* Bytes trimmed */
164 } trim_args_t;
165
166 /*
167 * Determines whether a vdev_trim_thread() should be stopped.
168 */
169 static boolean_t
vdev_trim_should_stop(vdev_t * vd)170 vdev_trim_should_stop(vdev_t *vd)
171 {
172 return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||
173 vd->vdev_detached || vd->vdev_top->vdev_removing ||
174 vd->vdev_top->vdev_rz_expanding);
175 }
176
177 /*
178 * Determines whether a vdev_autotrim_thread() should be stopped.
179 */
180 static boolean_t
vdev_autotrim_should_stop(vdev_t * tvd)181 vdev_autotrim_should_stop(vdev_t *tvd)
182 {
183 return (tvd->vdev_autotrim_exit_wanted ||
184 !vdev_writeable(tvd) || tvd->vdev_removing ||
185 tvd->vdev_rz_expanding ||
186 spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF);
187 }
188
189 /*
190 * Wait for given number of kicks, return true if the wait is aborted due to
191 * vdev_autotrim_exit_wanted.
192 */
193 static boolean_t
vdev_autotrim_wait_kick(vdev_t * vd,int num_of_kick)194 vdev_autotrim_wait_kick(vdev_t *vd, int num_of_kick)
195 {
196 mutex_enter(&vd->vdev_autotrim_lock);
197 for (int i = 0; i < num_of_kick; i++) {
198 if (vd->vdev_autotrim_exit_wanted)
199 break;
200 cv_wait_idle(&vd->vdev_autotrim_kick_cv,
201 &vd->vdev_autotrim_lock);
202 }
203 boolean_t exit_wanted = vd->vdev_autotrim_exit_wanted;
204 mutex_exit(&vd->vdev_autotrim_lock);
205
206 return (exit_wanted);
207 }
208
209 /*
210 * The sync task for updating the on-disk state of a manual TRIM. This
211 * is scheduled by vdev_trim_change_state().
212 */
213 static void
vdev_trim_zap_update_sync(void * arg,dmu_tx_t * tx)214 vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx)
215 {
216 /*
217 * We pass in the guid instead of the vdev_t since the vdev may
218 * have been freed prior to the sync task being processed. This
219 * happens when a vdev is detached as we call spa_config_vdev_exit(),
220 * stop the trimming thread, schedule the sync task, and free
221 * the vdev. Later when the scheduled sync task is invoked, it would
222 * find that the vdev has been freed.
223 */
224 uint64_t guid = *(uint64_t *)arg;
225 uint64_t txg = dmu_tx_get_txg(tx);
226 kmem_free(arg, sizeof (uint64_t));
227
228 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
229 if (vd == NULL || vd->vdev_top->vdev_removing ||
230 !vdev_is_concrete(vd) || vd->vdev_top->vdev_rz_expanding)
231 return;
232
233 uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];
234 vd->vdev_trim_offset[txg & TXG_MASK] = 0;
235
236 VERIFY3U(vd->vdev_leaf_zap, !=, 0);
237
238 objset_t *mos = vd->vdev_spa->spa_meta_objset;
239
240 if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {
241
242 if (vd->vdev_trim_last_offset == UINT64_MAX)
243 last_offset = 0;
244
245 vd->vdev_trim_last_offset = last_offset;
246 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
247 VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
248 sizeof (last_offset), 1, &last_offset, tx));
249 }
250
251 if (vd->vdev_trim_action_time > 0) {
252 uint64_t val = (uint64_t)vd->vdev_trim_action_time;
253 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
254 VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val),
255 1, &val, tx));
256 }
257
258 if (vd->vdev_trim_rate > 0) {
259 uint64_t rate = (uint64_t)vd->vdev_trim_rate;
260
261 if (rate == UINT64_MAX)
262 rate = 0;
263
264 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
265 VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx));
266 }
267
268 uint64_t partial = vd->vdev_trim_partial;
269 if (partial == UINT64_MAX)
270 partial = 0;
271
272 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
273 sizeof (partial), 1, &partial, tx));
274
275 uint64_t secure = vd->vdev_trim_secure;
276 if (secure == UINT64_MAX)
277 secure = 0;
278
279 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
280 sizeof (secure), 1, &secure, tx));
281
282
283 uint64_t trim_state = vd->vdev_trim_state;
284 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
285 sizeof (trim_state), 1, &trim_state, tx));
286 }
287
288 /*
289 * Update the on-disk state of a manual TRIM. This is called to request
290 * that a TRIM be started/suspended/canceled, or to change one of the
291 * TRIM options (partial, secure, rate).
292 */
293 static void
vdev_trim_change_state(vdev_t * vd,vdev_trim_state_t new_state,uint64_t rate,boolean_t partial,boolean_t secure)294 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state,
295 uint64_t rate, boolean_t partial, boolean_t secure)
296 {
297 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
298 spa_t *spa = vd->vdev_spa;
299
300 if (new_state == vd->vdev_trim_state)
301 return;
302
303 /*
304 * Copy the vd's guid, this will be freed by the sync task.
305 */
306 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
307 *guid = vd->vdev_guid;
308
309 /*
310 * If we're suspending, then preserve the original start time.
311 */
312 if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {
313 vd->vdev_trim_action_time = gethrestime_sec();
314 }
315
316 /*
317 * If we're activating, then preserve the requested rate and trim
318 * method. Setting the last offset and rate to UINT64_MAX is used
319 * as a sentinel to indicate they should be reset to default values.
320 */
321 if (new_state == VDEV_TRIM_ACTIVE) {
322 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||
323 vd->vdev_trim_state == VDEV_TRIM_CANCELED) {
324 vd->vdev_trim_last_offset = UINT64_MAX;
325 vd->vdev_trim_rate = UINT64_MAX;
326 vd->vdev_trim_partial = UINT64_MAX;
327 vd->vdev_trim_secure = UINT64_MAX;
328 }
329
330 if (rate != 0)
331 vd->vdev_trim_rate = rate;
332
333 if (partial != 0)
334 vd->vdev_trim_partial = partial;
335
336 if (secure != 0)
337 vd->vdev_trim_secure = secure;
338 }
339
340 vdev_trim_state_t old_state = vd->vdev_trim_state;
341 boolean_t resumed = (old_state == VDEV_TRIM_SUSPENDED);
342 vd->vdev_trim_state = new_state;
343
344 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
345 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND));
346 dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync,
347 guid, tx);
348
349 switch (new_state) {
350 case VDEV_TRIM_ACTIVE:
351 spa_event_notify(spa, vd, NULL,
352 resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START);
353 spa_history_log_internal(spa, "trim", tx,
354 "vdev=%s activated", vd->vdev_path);
355 break;
356 case VDEV_TRIM_SUSPENDED:
357 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND);
358 spa_history_log_internal(spa, "trim", tx,
359 "vdev=%s suspended", vd->vdev_path);
360 break;
361 case VDEV_TRIM_CANCELED:
362 if (old_state == VDEV_TRIM_ACTIVE ||
363 old_state == VDEV_TRIM_SUSPENDED) {
364 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL);
365 spa_history_log_internal(spa, "trim", tx,
366 "vdev=%s canceled", vd->vdev_path);
367 }
368 break;
369 case VDEV_TRIM_COMPLETE:
370 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH);
371 spa_history_log_internal(spa, "trim", tx,
372 "vdev=%s complete", vd->vdev_path);
373 break;
374 default:
375 panic("invalid state %llu", (unsigned long long)new_state);
376 }
377
378 dmu_tx_commit(tx);
379
380 if (new_state != VDEV_TRIM_ACTIVE)
381 spa_notify_waiters(spa);
382 }
383
384 /*
385 * The zio_done_func_t done callback for each manual TRIM issued. It is
386 * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
387 * and limiting the number of in flight TRIM I/Os.
388 */
389 static void
vdev_trim_cb(zio_t * zio)390 vdev_trim_cb(zio_t *zio)
391 {
392 vdev_t *vd = zio->io_vd;
393
394 mutex_enter(&vd->vdev_trim_io_lock);
395 if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
396 /*
397 * The I/O failed because the vdev was unavailable; roll the
398 * last offset back. (This works because spa_sync waits on
399 * spa_txg_zio before it runs sync tasks.)
400 */
401 uint64_t *offset =
402 &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];
403 *offset = MIN(*offset, zio->io_offset);
404 } else {
405 if (zio->io_error != 0) {
406 vd->vdev_stat.vs_trim_errors++;
407 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
408 0, 0, 0, 0, 1, zio->io_orig_size);
409 } else {
410 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
411 1, zio->io_orig_size, 0, 0, 0, 0);
412 }
413
414 vd->vdev_trim_bytes_done += zio->io_orig_size;
415 }
416
417 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);
418 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;
419 cv_broadcast(&vd->vdev_trim_io_cv);
420 mutex_exit(&vd->vdev_trim_io_lock);
421
422 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
423 }
424
425 /*
426 * The zio_done_func_t done callback for each automatic TRIM issued. It
427 * is responsible for updating the TRIM stats and limiting the number of
428 * in flight TRIM I/Os. Automatic TRIM I/Os are best effort and are
429 * never reissued on failure.
430 */
431 static void
vdev_autotrim_cb(zio_t * zio)432 vdev_autotrim_cb(zio_t *zio)
433 {
434 vdev_t *vd = zio->io_vd;
435
436 mutex_enter(&vd->vdev_trim_io_lock);
437
438 if (zio->io_error != 0) {
439 vd->vdev_stat.vs_trim_errors++;
440 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
441 0, 0, 0, 0, 1, zio->io_orig_size);
442 } else {
443 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
444 1, zio->io_orig_size, 0, 0, 0, 0);
445 }
446
447 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);
448 vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;
449 cv_broadcast(&vd->vdev_trim_io_cv);
450 mutex_exit(&vd->vdev_trim_io_lock);
451
452 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
453 }
454
455 /*
456 * The zio_done_func_t done callback for each TRIM issued via
457 * vdev_trim_simple(). It is responsible for updating the TRIM stats and
458 * limiting the number of in flight TRIM I/Os. Simple TRIM I/Os are best
459 * effort and are never reissued on failure.
460 */
461 static void
vdev_trim_simple_cb(zio_t * zio)462 vdev_trim_simple_cb(zio_t *zio)
463 {
464 vdev_t *vd = zio->io_vd;
465
466 mutex_enter(&vd->vdev_trim_io_lock);
467
468 if (zio->io_error != 0) {
469 vd->vdev_stat.vs_trim_errors++;
470 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,
471 0, 0, 0, 0, 1, zio->io_orig_size);
472 } else {
473 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,
474 1, zio->io_orig_size, 0, 0, 0, 0);
475 }
476
477 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0);
478 vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--;
479 cv_broadcast(&vd->vdev_trim_io_cv);
480 mutex_exit(&vd->vdev_trim_io_lock);
481
482 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
483 }
484 /*
485 * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
486 */
487 static uint64_t
vdev_trim_calculate_rate(trim_args_t * ta)488 vdev_trim_calculate_rate(trim_args_t *ta)
489 {
490 return (ta->trim_bytes_done * 1000 /
491 (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1));
492 }
493
494 /*
495 * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
496 * and number of concurrent TRIM I/Os.
497 */
498 static int
vdev_trim_range(trim_args_t * ta,uint64_t start,uint64_t size)499 vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size)
500 {
501 vdev_t *vd = ta->trim_vdev;
502 spa_t *spa = vd->vdev_spa;
503 void *cb;
504
505 mutex_enter(&vd->vdev_trim_io_lock);
506
507 /*
508 * Limit manual TRIM I/Os to the requested rate. This does not
509 * apply to automatic TRIM since no per vdev rate can be specified.
510 */
511 if (ta->trim_type == TRIM_TYPE_MANUAL) {
512 while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&
513 vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {
514 cv_timedwait_idle(&vd->vdev_trim_io_cv,
515 &vd->vdev_trim_io_lock, ddi_get_lbolt() +
516 MSEC_TO_TICK(10));
517 }
518 }
519 ta->trim_bytes_done += size;
520
521 /* Limit in flight trimming I/Os */
522 while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] +
523 vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) {
524 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
525 }
526 vd->vdev_trim_inflight[ta->trim_type]++;
527 mutex_exit(&vd->vdev_trim_io_lock);
528
529 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
530 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND));
531 uint64_t txg = dmu_tx_get_txg(tx);
532
533 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
534 mutex_enter(&vd->vdev_trim_lock);
535
536 if (ta->trim_type == TRIM_TYPE_MANUAL &&
537 vd->vdev_trim_offset[txg & TXG_MASK] == 0) {
538 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
539 *guid = vd->vdev_guid;
540
541 /* This is the first write of this txg. */
542 dsl_sync_task_nowait(spa_get_dsl(spa),
543 vdev_trim_zap_update_sync, guid, tx);
544 }
545
546 /*
547 * We know the vdev_t will still be around since all consumers of
548 * vdev_free must stop the trimming first.
549 */
550 if ((ta->trim_type == TRIM_TYPE_MANUAL &&
551 vdev_trim_should_stop(vd)) ||
552 (ta->trim_type == TRIM_TYPE_AUTO &&
553 vdev_autotrim_should_stop(vd->vdev_top))) {
554 mutex_enter(&vd->vdev_trim_io_lock);
555 vd->vdev_trim_inflight[ta->trim_type]--;
556 mutex_exit(&vd->vdev_trim_io_lock);
557 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
558 mutex_exit(&vd->vdev_trim_lock);
559 dmu_tx_commit(tx);
560 return (SET_ERROR(EINTR));
561 }
562 mutex_exit(&vd->vdev_trim_lock);
563
564 if (ta->trim_type == TRIM_TYPE_MANUAL)
565 vd->vdev_trim_offset[txg & TXG_MASK] = start + size;
566
567 if (ta->trim_type == TRIM_TYPE_MANUAL) {
568 cb = vdev_trim_cb;
569 } else if (ta->trim_type == TRIM_TYPE_AUTO) {
570 cb = vdev_autotrim_cb;
571 } else {
572 cb = vdev_trim_simple_cb;
573 }
574
575 zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,
576 start, size, cb, NULL, ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL,
577 ta->trim_flags));
578 /* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */
579
580 dmu_tx_commit(tx);
581
582 return (0);
583 }
584
585 /*
586 * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
587 * Additional parameters describing how the TRIM should be performed must
588 * be set in the trim_args structure. See the trim_args definition for
589 * additional information.
590 */
591 static int
vdev_trim_ranges(trim_args_t * ta)592 vdev_trim_ranges(trim_args_t *ta)
593 {
594 vdev_t *vd = ta->trim_vdev;
595 zfs_btree_t *t = &ta->trim_tree->rt_root;
596 zfs_btree_index_t idx;
597 uint64_t extent_bytes_max = ta->trim_extent_bytes_max;
598 uint64_t extent_bytes_min = ta->trim_extent_bytes_min;
599 spa_t *spa = vd->vdev_spa;
600 int error = 0;
601
602 ta->trim_start_time = gethrtime();
603 ta->trim_bytes_done = 0;
604
605 for (zfs_range_seg_t *rs = zfs_btree_first(t, &idx); rs != NULL;
606 rs = zfs_btree_next(t, &idx, &idx)) {
607 uint64_t size = zfs_rs_get_end(rs, ta->trim_tree) -
608 zfs_rs_get_start(rs, ta->trim_tree);
609
610 if (extent_bytes_min && size < extent_bytes_min) {
611 spa_iostats_trim_add(spa, ta->trim_type,
612 0, 0, 1, size, 0, 0);
613 continue;
614 }
615
616 /* Split range into legally-sized physical chunks */
617 uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1;
618
619 for (uint64_t w = 0; w < writes_required; w++) {
620 error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE +
621 zfs_rs_get_start(rs, ta->trim_tree) +
622 (w *extent_bytes_max), MIN(size -
623 (w * extent_bytes_max), extent_bytes_max));
624 if (error != 0) {
625 goto done;
626 }
627 }
628 }
629
630 done:
631 /*
632 * Make sure all TRIMs for this metaslab have completed before
633 * returning. TRIM zios have lower priority over regular or syncing
634 * zios, so all TRIM zios for this metaslab must complete before the
635 * metaslab is re-enabled. Otherwise it's possible write zios to
636 * this metaslab could cut ahead of still queued TRIM zios for this
637 * metaslab causing corruption if the ranges overlap.
638 */
639 mutex_enter(&vd->vdev_trim_io_lock);
640 while (vd->vdev_trim_inflight[0] > 0) {
641 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
642 }
643 mutex_exit(&vd->vdev_trim_io_lock);
644
645 return (error);
646 }
647
648 static void
vdev_trim_xlate_last_rs_end(void * arg,zfs_range_seg64_t * physical_rs)649 vdev_trim_xlate_last_rs_end(void *arg, zfs_range_seg64_t *physical_rs)
650 {
651 uint64_t *last_rs_end = (uint64_t *)arg;
652
653 if (physical_rs->rs_end > *last_rs_end)
654 *last_rs_end = physical_rs->rs_end;
655 }
656
657 static void
vdev_trim_xlate_progress(void * arg,zfs_range_seg64_t * physical_rs)658 vdev_trim_xlate_progress(void *arg, zfs_range_seg64_t *physical_rs)
659 {
660 vdev_t *vd = (vdev_t *)arg;
661
662 uint64_t size = physical_rs->rs_end - physical_rs->rs_start;
663 vd->vdev_trim_bytes_est += size;
664
665 if (vd->vdev_trim_last_offset >= physical_rs->rs_end) {
666 vd->vdev_trim_bytes_done += size;
667 } else if (vd->vdev_trim_last_offset > physical_rs->rs_start &&
668 vd->vdev_trim_last_offset <= physical_rs->rs_end) {
669 vd->vdev_trim_bytes_done +=
670 vd->vdev_trim_last_offset - physical_rs->rs_start;
671 }
672 }
673
674 /*
675 * Calculates the completion percentage of a manual TRIM.
676 */
677 static void
vdev_trim_calculate_progress(vdev_t * vd)678 vdev_trim_calculate_progress(vdev_t *vd)
679 {
680 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
681 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
682 ASSERT(vd->vdev_leaf_zap != 0);
683
684 vd->vdev_trim_bytes_est = 0;
685 vd->vdev_trim_bytes_done = 0;
686
687 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
688 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
689 mutex_enter(&msp->ms_lock);
690
691 uint64_t ms_free = (msp->ms_size -
692 metaslab_allocated_space(msp)) /
693 vdev_get_ndisks(vd->vdev_top);
694
695 /*
696 * Convert the metaslab range to a physical range
697 * on our vdev. We use this to determine if we are
698 * in the middle of this metaslab range.
699 */
700 zfs_range_seg64_t logical_rs, physical_rs, remain_rs;
701 logical_rs.rs_start = msp->ms_start;
702 logical_rs.rs_end = msp->ms_start + msp->ms_size;
703
704 /* Metaslab space after this offset has not been trimmed. */
705 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs);
706 if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {
707 vd->vdev_trim_bytes_est += ms_free;
708 mutex_exit(&msp->ms_lock);
709 continue;
710 }
711
712 /* Metaslab space before this offset has been trimmed */
713 uint64_t last_rs_end = physical_rs.rs_end;
714 if (!vdev_xlate_is_empty(&remain_rs)) {
715 vdev_xlate_walk(vd, &remain_rs,
716 vdev_trim_xlate_last_rs_end, &last_rs_end);
717 }
718
719 if (vd->vdev_trim_last_offset > last_rs_end) {
720 vd->vdev_trim_bytes_done += ms_free;
721 vd->vdev_trim_bytes_est += ms_free;
722 mutex_exit(&msp->ms_lock);
723 continue;
724 }
725
726 /*
727 * If we get here, we're in the middle of trimming this
728 * metaslab. Load it and walk the free tree for more
729 * accurate progress estimation.
730 */
731 VERIFY0(metaslab_load(msp));
732
733 zfs_range_tree_t *rt = msp->ms_allocatable;
734 zfs_btree_t *bt = &rt->rt_root;
735 zfs_btree_index_t idx;
736 for (zfs_range_seg_t *rs = zfs_btree_first(bt, &idx);
737 rs != NULL; rs = zfs_btree_next(bt, &idx, &idx)) {
738 logical_rs.rs_start = zfs_rs_get_start(rs, rt);
739 logical_rs.rs_end = zfs_rs_get_end(rs, rt);
740
741 vdev_xlate_walk(vd, &logical_rs,
742 vdev_trim_xlate_progress, vd);
743 }
744 mutex_exit(&msp->ms_lock);
745 }
746 }
747
748 /*
749 * Load from disk the vdev's manual TRIM information. This includes the
750 * state, progress, and options provided when initiating the manual TRIM.
751 */
752 static int
vdev_trim_load(vdev_t * vd)753 vdev_trim_load(vdev_t *vd)
754 {
755 int err = 0;
756 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
757 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
758 ASSERT(vd->vdev_leaf_zap != 0);
759
760 if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||
761 vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {
762 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
763 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
764 sizeof (vd->vdev_trim_last_offset), 1,
765 &vd->vdev_trim_last_offset);
766 if (err == ENOENT) {
767 vd->vdev_trim_last_offset = 0;
768 err = 0;
769 }
770
771 if (err == 0) {
772 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
773 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,
774 sizeof (vd->vdev_trim_rate), 1,
775 &vd->vdev_trim_rate);
776 if (err == ENOENT) {
777 vd->vdev_trim_rate = 0;
778 err = 0;
779 }
780 }
781
782 if (err == 0) {
783 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
784 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
785 sizeof (vd->vdev_trim_partial), 1,
786 &vd->vdev_trim_partial);
787 if (err == ENOENT) {
788 vd->vdev_trim_partial = 0;
789 err = 0;
790 }
791 }
792
793 if (err == 0) {
794 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
795 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
796 sizeof (vd->vdev_trim_secure), 1,
797 &vd->vdev_trim_secure);
798 if (err == ENOENT) {
799 vd->vdev_trim_secure = 0;
800 err = 0;
801 }
802 }
803 }
804
805 vdev_trim_calculate_progress(vd);
806
807 return (err);
808 }
809
810 static void
vdev_trim_xlate_range_add(void * arg,zfs_range_seg64_t * physical_rs)811 vdev_trim_xlate_range_add(void *arg, zfs_range_seg64_t *physical_rs)
812 {
813 trim_args_t *ta = arg;
814 vdev_t *vd = ta->trim_vdev;
815
816 /*
817 * Only a manual trim will be traversing the vdev sequentially.
818 * For an auto trim all valid ranges should be added.
819 */
820 if (ta->trim_type == TRIM_TYPE_MANUAL) {
821
822 /* Only add segments that we have not visited yet */
823 if (physical_rs->rs_end <= vd->vdev_trim_last_offset)
824 return;
825
826 /* Pick up where we left off mid-range. */
827 if (vd->vdev_trim_last_offset > physical_rs->rs_start) {
828 ASSERT3U(physical_rs->rs_end, >,
829 vd->vdev_trim_last_offset);
830 physical_rs->rs_start = vd->vdev_trim_last_offset;
831 }
832 }
833
834 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start);
835
836 zfs_range_tree_add(ta->trim_tree, physical_rs->rs_start,
837 physical_rs->rs_end - physical_rs->rs_start);
838 }
839
840 /*
841 * Convert the logical range into physical ranges and add them to the
842 * range tree passed in the trim_args_t.
843 */
844 static void
vdev_trim_range_add(void * arg,uint64_t start,uint64_t size)845 vdev_trim_range_add(void *arg, uint64_t start, uint64_t size)
846 {
847 trim_args_t *ta = arg;
848 vdev_t *vd = ta->trim_vdev;
849 zfs_range_seg64_t logical_rs;
850 logical_rs.rs_start = start;
851 logical_rs.rs_end = start + size;
852
853 /*
854 * Every range to be trimmed must be part of ms_allocatable.
855 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this
856 * is always the case.
857 */
858 if (zfs_flags & ZFS_DEBUG_TRIM) {
859 metaslab_t *msp = ta->trim_msp;
860 VERIFY0(metaslab_load(msp));
861 VERIFY3B(msp->ms_loaded, ==, B_TRUE);
862 VERIFY(zfs_range_tree_contains(msp->ms_allocatable, start,
863 size));
864 }
865
866 ASSERT(vd->vdev_ops->vdev_op_leaf);
867 vdev_xlate_walk(vd, &logical_rs, vdev_trim_xlate_range_add, arg);
868 }
869
870 /*
871 * Each manual TRIM thread is responsible for trimming the unallocated
872 * space for each leaf vdev. This is accomplished by sequentially iterating
873 * over its top-level metaslabs and issuing TRIM I/O for the space described
874 * by its ms_allocatable. While a metaslab is undergoing trimming it is
875 * not eligible for new allocations.
876 */
877 static __attribute__((noreturn)) void
vdev_trim_thread(void * arg)878 vdev_trim_thread(void *arg)
879 {
880 vdev_t *vd = arg;
881 spa_t *spa = vd->vdev_spa;
882 trim_args_t ta;
883 int error = 0;
884
885 /*
886 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by
887 * vdev_trim(). Wait for the updated values to be reflected
888 * in the zap in order to start with the requested settings.
889 */
890 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
891
892 ASSERT(vdev_is_concrete(vd));
893 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
894
895 vd->vdev_trim_last_offset = 0;
896 vd->vdev_trim_rate = 0;
897 vd->vdev_trim_partial = 0;
898 vd->vdev_trim_secure = 0;
899
900 VERIFY0(vdev_trim_load(vd));
901
902 ta.trim_vdev = vd;
903 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
904 ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min;
905 ta.trim_tree = zfs_range_tree_create_flags(
906 NULL, ZFS_RANGE_SEG64, NULL, 0, 0,
907 ZFS_RT_F_DYN_NAME, vdev_rt_name(vd, "trim_tree"));
908 ta.trim_type = TRIM_TYPE_MANUAL;
909 ta.trim_flags = 0;
910
911 /*
912 * When a secure TRIM has been requested infer that the intent
913 * is that everything must be trimmed. Override the default
914 * minimum TRIM size to prevent ranges from being skipped.
915 */
916 if (vd->vdev_trim_secure) {
917 ta.trim_flags |= ZIO_TRIM_SECURE;
918 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
919 }
920
921 uint64_t ms_count = 0;
922 for (uint64_t i = 0; !vd->vdev_detached &&
923 i < vd->vdev_top->vdev_ms_count; i++) {
924 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
925
926 /*
927 * If we've expanded the top-level vdev or it's our
928 * first pass, calculate our progress.
929 */
930 if (vd->vdev_top->vdev_ms_count != ms_count) {
931 vdev_trim_calculate_progress(vd);
932 ms_count = vd->vdev_top->vdev_ms_count;
933 }
934
935 spa_config_exit(spa, SCL_CONFIG, FTAG);
936 metaslab_disable(msp);
937 mutex_enter(&msp->ms_lock);
938 VERIFY0(metaslab_load(msp));
939
940 /*
941 * If a partial TRIM was requested skip metaslabs which have
942 * never been initialized and thus have never been written.
943 */
944 if (msp->ms_sm == NULL && vd->vdev_trim_partial) {
945 mutex_exit(&msp->ms_lock);
946 metaslab_enable(msp, B_FALSE, B_FALSE);
947 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
948 vdev_trim_calculate_progress(vd);
949 continue;
950 }
951
952 ta.trim_msp = msp;
953 zfs_range_tree_walk(msp->ms_allocatable, vdev_trim_range_add,
954 &ta);
955 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL);
956 mutex_exit(&msp->ms_lock);
957
958 error = vdev_trim_ranges(&ta);
959 metaslab_enable(msp, B_TRUE, B_FALSE);
960 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
961
962 zfs_range_tree_vacate(ta.trim_tree, NULL, NULL);
963 if (error != 0)
964 break;
965 }
966
967 spa_config_exit(spa, SCL_CONFIG, FTAG);
968
969 zfs_range_tree_destroy(ta.trim_tree);
970
971 mutex_enter(&vd->vdev_trim_lock);
972 if (!vd->vdev_trim_exit_wanted) {
973 if (vdev_writeable(vd)) {
974 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
975 vd->vdev_trim_rate, vd->vdev_trim_partial,
976 vd->vdev_trim_secure);
977 } else if (vd->vdev_faulted) {
978 vdev_trim_change_state(vd, VDEV_TRIM_CANCELED,
979 vd->vdev_trim_rate, vd->vdev_trim_partial,
980 vd->vdev_trim_secure);
981 }
982 }
983 ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);
984
985 /*
986 * Drop the vdev_trim_lock while we sync out the txg since it's
987 * possible that a device might be trying to come online and must
988 * check to see if it needs to restart a trim. That thread will be
989 * holding the spa_config_lock which would prevent the txg_wait_synced
990 * from completing.
991 */
992 mutex_exit(&vd->vdev_trim_lock);
993 txg_wait_synced(spa_get_dsl(spa), 0);
994 mutex_enter(&vd->vdev_trim_lock);
995
996 vd->vdev_trim_thread = NULL;
997 cv_broadcast(&vd->vdev_trim_cv);
998 mutex_exit(&vd->vdev_trim_lock);
999
1000 thread_exit();
1001 }
1002
1003 /*
1004 * Initiates a manual TRIM for the vdev_t. Callers must hold vdev_trim_lock,
1005 * the vdev_t must be a leaf and cannot already be manually trimming.
1006 */
1007 void
vdev_trim(vdev_t * vd,uint64_t rate,boolean_t partial,boolean_t secure)1008 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)
1009 {
1010 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
1011 ASSERT(vd->vdev_ops->vdev_op_leaf);
1012 ASSERT(vdev_is_concrete(vd));
1013 ASSERT0P(vd->vdev_trim_thread);
1014 ASSERT(!vd->vdev_detached);
1015 ASSERT(!vd->vdev_trim_exit_wanted);
1016 ASSERT(!vd->vdev_top->vdev_removing);
1017 ASSERT(!vd->vdev_rz_expanding);
1018
1019 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure);
1020 vd->vdev_trim_thread = thread_create(NULL, 0,
1021 vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
1022 }
1023
1024 /*
1025 * Wait for the trimming thread to be terminated (canceled or stopped).
1026 */
1027 static void
vdev_trim_stop_wait_impl(vdev_t * vd)1028 vdev_trim_stop_wait_impl(vdev_t *vd)
1029 {
1030 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
1031
1032 while (vd->vdev_trim_thread != NULL)
1033 cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);
1034
1035 ASSERT0P(vd->vdev_trim_thread);
1036 vd->vdev_trim_exit_wanted = B_FALSE;
1037 }
1038
1039 /*
1040 * Wait for vdev trim threads which were listed to cleanly exit.
1041 */
1042 void
vdev_trim_stop_wait(spa_t * spa,list_t * vd_list)1043 vdev_trim_stop_wait(spa_t *spa, list_t *vd_list)
1044 {
1045 (void) spa;
1046 vdev_t *vd;
1047
1048 ASSERT(MUTEX_HELD(&spa_namespace_lock) ||
1049 spa->spa_export_thread == curthread);
1050
1051 while ((vd = list_remove_head(vd_list)) != NULL) {
1052 mutex_enter(&vd->vdev_trim_lock);
1053 vdev_trim_stop_wait_impl(vd);
1054 mutex_exit(&vd->vdev_trim_lock);
1055 }
1056 }
1057
1058 /*
1059 * Stop trimming a device, with the resultant trimming state being tgt_state.
1060 * For blocking behavior pass NULL for vd_list. Otherwise, when a list_t is
1061 * provided the stopping vdev is inserted in to the list. Callers are then
1062 * required to call vdev_trim_stop_wait() to block for all the trim threads
1063 * to exit. The caller must hold vdev_trim_lock and must not be writing to
1064 * the spa config, as the trimming thread may try to enter the config as a
1065 * reader before exiting.
1066 */
1067 void
vdev_trim_stop(vdev_t * vd,vdev_trim_state_t tgt_state,list_t * vd_list)1068 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list)
1069 {
1070 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
1071 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
1072 ASSERT(vd->vdev_ops->vdev_op_leaf);
1073 ASSERT(vdev_is_concrete(vd));
1074
1075 /*
1076 * Allow cancel requests to proceed even if the trim thread has
1077 * stopped.
1078 */
1079 if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)
1080 return;
1081
1082 vdev_trim_change_state(vd, tgt_state, 0, 0, 0);
1083 vd->vdev_trim_exit_wanted = B_TRUE;
1084
1085 if (vd_list == NULL) {
1086 vdev_trim_stop_wait_impl(vd);
1087 } else {
1088 ASSERT(MUTEX_HELD(&spa_namespace_lock) ||
1089 vd->vdev_spa->spa_export_thread == curthread);
1090 list_insert_tail(vd_list, vd);
1091 }
1092 }
1093
1094 /*
1095 * Requests that all listed vdevs stop trimming.
1096 */
1097 static void
vdev_trim_stop_all_impl(vdev_t * vd,vdev_trim_state_t tgt_state,list_t * vd_list)1098 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state,
1099 list_t *vd_list)
1100 {
1101 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
1102 mutex_enter(&vd->vdev_trim_lock);
1103 vdev_trim_stop(vd, tgt_state, vd_list);
1104 mutex_exit(&vd->vdev_trim_lock);
1105 return;
1106 }
1107
1108 for (uint64_t i = 0; i < vd->vdev_children; i++) {
1109 vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,
1110 vd_list);
1111 }
1112 }
1113
1114 /*
1115 * Convenience function to stop trimming of a vdev tree and set all trim
1116 * thread pointers to NULL.
1117 */
1118 void
vdev_trim_stop_all(vdev_t * vd,vdev_trim_state_t tgt_state)1119 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state)
1120 {
1121 spa_t *spa = vd->vdev_spa;
1122 list_t vd_list;
1123 vdev_t *vd_l2cache;
1124
1125 ASSERT(MUTEX_HELD(&spa_namespace_lock) ||
1126 spa->spa_export_thread == curthread);
1127
1128 list_create(&vd_list, sizeof (vdev_t),
1129 offsetof(vdev_t, vdev_trim_node));
1130
1131 vdev_trim_stop_all_impl(vd, tgt_state, &vd_list);
1132
1133 /*
1134 * Iterate over cache devices and request stop trimming the
1135 * whole device in case we export the pool or remove the cache
1136 * device prematurely.
1137 */
1138 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1139 vd_l2cache = spa->spa_l2cache.sav_vdevs[i];
1140 vdev_trim_stop_all_impl(vd_l2cache, tgt_state, &vd_list);
1141 }
1142
1143 vdev_trim_stop_wait(spa, &vd_list);
1144
1145 if (vd->vdev_spa->spa_sync_on) {
1146 /* Make sure that our state has been synced to disk */
1147 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1148 }
1149
1150 list_destroy(&vd_list);
1151 }
1152
1153 /*
1154 * Conditionally restarts a manual TRIM given its on-disk state.
1155 */
1156 void
vdev_trim_restart(vdev_t * vd)1157 vdev_trim_restart(vdev_t *vd)
1158 {
1159 ASSERT(MUTEX_HELD(&spa_namespace_lock) ||
1160 vd->vdev_spa->spa_load_thread == curthread);
1161 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
1162
1163 if (vd->vdev_leaf_zap != 0) {
1164 mutex_enter(&vd->vdev_trim_lock);
1165 uint64_t trim_state = VDEV_TRIM_NONE;
1166 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1167 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
1168 sizeof (trim_state), 1, &trim_state);
1169 ASSERT(err == 0 || err == ENOENT);
1170 vd->vdev_trim_state = trim_state;
1171
1172 uint64_t timestamp = 0;
1173 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1174 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,
1175 sizeof (timestamp), 1, ×tamp);
1176 ASSERT(err == 0 || err == ENOENT);
1177 vd->vdev_trim_action_time = timestamp;
1178
1179 if ((vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||
1180 vd->vdev_offline) && !vd->vdev_top->vdev_rz_expanding) {
1181 /* load progress for reporting, but don't resume */
1182 VERIFY0(vdev_trim_load(vd));
1183 } else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&
1184 vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&
1185 !vd->vdev_top->vdev_rz_expanding &&
1186 vd->vdev_trim_thread == NULL) {
1187 VERIFY0(vdev_trim_load(vd));
1188 vdev_trim(vd, vd->vdev_trim_rate,
1189 vd->vdev_trim_partial, vd->vdev_trim_secure);
1190 }
1191
1192 mutex_exit(&vd->vdev_trim_lock);
1193 }
1194
1195 for (uint64_t i = 0; i < vd->vdev_children; i++) {
1196 vdev_trim_restart(vd->vdev_child[i]);
1197 }
1198 }
1199
1200 /*
1201 * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1202 * every TRIM range is contained within ms_allocatable.
1203 */
1204 static void
vdev_trim_range_verify(void * arg,uint64_t start,uint64_t size)1205 vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size)
1206 {
1207 trim_args_t *ta = arg;
1208 metaslab_t *msp = ta->trim_msp;
1209
1210 VERIFY3B(msp->ms_loaded, ==, B_TRUE);
1211 VERIFY3U(msp->ms_disabled, >, 0);
1212 VERIFY(zfs_range_tree_contains(msp->ms_allocatable, start, size));
1213 }
1214
1215 /*
1216 * Each automatic TRIM thread is responsible for managing the trimming of a
1217 * top-level vdev in the pool. No automatic TRIM state is maintained on-disk.
1218 *
1219 * N.B. This behavior is different from a manual TRIM where a thread
1220 * is created for each leaf vdev, instead of each top-level vdev.
1221 */
1222 static __attribute__((noreturn)) void
vdev_autotrim_thread(void * arg)1223 vdev_autotrim_thread(void *arg)
1224 {
1225 vdev_t *vd = arg;
1226 spa_t *spa = vd->vdev_spa;
1227 int shift = 0;
1228
1229 mutex_enter(&vd->vdev_autotrim_lock);
1230 ASSERT3P(vd->vdev_top, ==, vd);
1231 ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);
1232 mutex_exit(&vd->vdev_autotrim_lock);
1233 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1234
1235 while (!vdev_autotrim_should_stop(vd)) {
1236 int txgs_per_trim = MAX(zfs_trim_txg_batch, 1);
1237 uint64_t extent_bytes_max = zfs_trim_extent_bytes_max;
1238 uint64_t extent_bytes_min = zfs_trim_extent_bytes_min;
1239
1240 /*
1241 * All of the metaslabs are divided in to groups of size
1242 * num_metaslabs / zfs_trim_txg_batch. Each of these groups
1243 * is composed of metaslabs which are spread evenly over the
1244 * device.
1245 *
1246 * For example, when zfs_trim_txg_batch = 32 (default) then
1247 * group 0 will contain metaslabs 0, 32, 64, ...;
1248 * group 1 will contain metaslabs 1, 33, 65, ...;
1249 * group 2 will contain metaslabs 2, 34, 66, ...; and so on.
1250 *
1251 * On each pass through the while() loop one of these groups
1252 * is selected. This is accomplished by using a shift value
1253 * to select the starting metaslab, then striding over the
1254 * metaslabs using the zfs_trim_txg_batch size. This is
1255 * done to accomplish two things.
1256 *
1257 * 1) By dividing the metaslabs in to groups, and making sure
1258 * that each group takes a minimum of one txg to process.
1259 * Then zfs_trim_txg_batch controls the minimum number of
1260 * txgs which must occur before a metaslab is revisited.
1261 *
1262 * 2) Selecting non-consecutive metaslabs distributes the
1263 * TRIM commands for a group evenly over the entire device.
1264 * This can be advantageous for certain types of devices.
1265 */
1266 for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;
1267 i += txgs_per_trim) {
1268 metaslab_t *msp = vd->vdev_ms[i];
1269 zfs_range_tree_t *trim_tree;
1270 boolean_t issued_trim = B_FALSE;
1271 boolean_t wait_aborted = B_FALSE;
1272
1273 spa_config_exit(spa, SCL_CONFIG, FTAG);
1274 metaslab_disable(msp);
1275 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1276
1277 mutex_enter(&msp->ms_lock);
1278
1279 /*
1280 * Skip the metaslab when it has never been allocated
1281 * or when there are no recent frees to trim.
1282 */
1283 if (msp->ms_sm == NULL ||
1284 zfs_range_tree_is_empty(msp->ms_trim)) {
1285 mutex_exit(&msp->ms_lock);
1286 metaslab_enable(msp, B_FALSE, B_FALSE);
1287 continue;
1288 }
1289
1290 /*
1291 * Skip the metaslab when it has already been disabled.
1292 * This may happen when a manual TRIM or initialize
1293 * operation is running concurrently. In the case
1294 * of a manual TRIM, the ms_trim tree will have been
1295 * vacated. Only ranges added after the manual TRIM
1296 * disabled the metaslab will be included in the tree.
1297 * These will be processed when the automatic TRIM
1298 * next revisits this metaslab.
1299 */
1300 if (msp->ms_disabled > 1) {
1301 mutex_exit(&msp->ms_lock);
1302 metaslab_enable(msp, B_FALSE, B_FALSE);
1303 continue;
1304 }
1305
1306 /*
1307 * Allocate an empty range tree which is swapped in
1308 * for the existing ms_trim tree while it is processed.
1309 */
1310 trim_tree = zfs_range_tree_create_flags(
1311 NULL, ZFS_RANGE_SEG64, NULL, 0, 0,
1312 ZFS_RT_F_DYN_NAME,
1313 vdev_rt_name(vd, "autotrim_tree"));
1314 zfs_range_tree_swap(&msp->ms_trim, &trim_tree);
1315 ASSERT(zfs_range_tree_is_empty(msp->ms_trim));
1316
1317 /*
1318 * There are two cases when constructing the per-vdev
1319 * trim trees for a metaslab. If the top-level vdev
1320 * has no children then it is also a leaf and should
1321 * be trimmed. Otherwise our children are the leaves
1322 * and a trim tree should be constructed for each.
1323 */
1324 trim_args_t *tap;
1325 uint64_t children = vd->vdev_children;
1326 if (children == 0) {
1327 children = 1;
1328 tap = kmem_zalloc(sizeof (trim_args_t) *
1329 children, KM_SLEEP);
1330 tap[0].trim_vdev = vd;
1331 } else {
1332 tap = kmem_zalloc(sizeof (trim_args_t) *
1333 children, KM_SLEEP);
1334
1335 for (uint64_t c = 0; c < children; c++) {
1336 tap[c].trim_vdev = vd->vdev_child[c];
1337 }
1338 }
1339
1340 for (uint64_t c = 0; c < children; c++) {
1341 trim_args_t *ta = &tap[c];
1342 vdev_t *cvd = ta->trim_vdev;
1343
1344 ta->trim_msp = msp;
1345 ta->trim_extent_bytes_max = extent_bytes_max;
1346 ta->trim_extent_bytes_min = extent_bytes_min;
1347 ta->trim_type = TRIM_TYPE_AUTO;
1348 ta->trim_flags = 0;
1349
1350 if (cvd->vdev_detached ||
1351 !vdev_writeable(cvd) ||
1352 !cvd->vdev_has_trim ||
1353 cvd->vdev_trim_thread != NULL) {
1354 continue;
1355 }
1356
1357 /*
1358 * When a device has an attached hot spare, or
1359 * is being replaced it will not be trimmed.
1360 * This is done to avoid adding additional
1361 * stress to a potentially unhealthy device,
1362 * and to minimize the required rebuild time.
1363 */
1364 if (!cvd->vdev_ops->vdev_op_leaf)
1365 continue;
1366
1367 ta->trim_tree = zfs_range_tree_create_flags(
1368 NULL, ZFS_RANGE_SEG64, NULL, 0, 0,
1369 ZFS_RT_F_DYN_NAME,
1370 vdev_rt_name(vd, "autotrim_tree"));
1371 zfs_range_tree_walk(trim_tree,
1372 vdev_trim_range_add, ta);
1373 }
1374
1375 mutex_exit(&msp->ms_lock);
1376 spa_config_exit(spa, SCL_CONFIG, FTAG);
1377
1378 /*
1379 * Issue the TRIM I/Os for all ranges covered by the
1380 * TRIM trees. These ranges are safe to TRIM because
1381 * no new allocations will be performed until the call
1382 * to metaslab_enabled() below.
1383 */
1384 for (uint64_t c = 0; c < children; c++) {
1385 trim_args_t *ta = &tap[c];
1386
1387 /*
1388 * Always yield to a manual TRIM if one has
1389 * been started for the child vdev.
1390 */
1391 if (ta->trim_tree == NULL ||
1392 ta->trim_vdev->vdev_trim_thread != NULL) {
1393 continue;
1394 }
1395
1396 /*
1397 * After this point metaslab_enable() must be
1398 * called with the sync flag set. This is done
1399 * here because vdev_trim_ranges() is allowed
1400 * to be interrupted (EINTR) before issuing all
1401 * of the required TRIM I/Os.
1402 */
1403 issued_trim = B_TRUE;
1404
1405 int error = vdev_trim_ranges(ta);
1406 if (error)
1407 break;
1408 }
1409
1410 /*
1411 * Verify every range which was trimmed is still
1412 * contained within the ms_allocatable tree.
1413 */
1414 if (zfs_flags & ZFS_DEBUG_TRIM) {
1415 mutex_enter(&msp->ms_lock);
1416 VERIFY0(metaslab_load(msp));
1417 VERIFY3P(tap[0].trim_msp, ==, msp);
1418 zfs_range_tree_walk(trim_tree,
1419 vdev_trim_range_verify, &tap[0]);
1420 mutex_exit(&msp->ms_lock);
1421 }
1422
1423 zfs_range_tree_vacate(trim_tree, NULL, NULL);
1424 zfs_range_tree_destroy(trim_tree);
1425
1426 /*
1427 * Wait for couples of kicks, to ensure the trim io is
1428 * synced. If the wait is aborted due to
1429 * vdev_autotrim_exit_wanted, we need to signal
1430 * metaslab_enable() to wait for sync.
1431 */
1432 if (issued_trim) {
1433 wait_aborted = vdev_autotrim_wait_kick(vd,
1434 TXG_CONCURRENT_STATES + TXG_DEFER_SIZE);
1435 }
1436
1437 metaslab_enable(msp, wait_aborted, B_FALSE);
1438 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1439
1440 for (uint64_t c = 0; c < children; c++) {
1441 trim_args_t *ta = &tap[c];
1442
1443 if (ta->trim_tree == NULL)
1444 continue;
1445
1446 zfs_range_tree_vacate(ta->trim_tree, NULL,
1447 NULL);
1448 zfs_range_tree_destroy(ta->trim_tree);
1449 }
1450
1451 kmem_free(tap, sizeof (trim_args_t) * children);
1452
1453 if (vdev_autotrim_should_stop(vd))
1454 break;
1455 }
1456
1457 spa_config_exit(spa, SCL_CONFIG, FTAG);
1458
1459 vdev_autotrim_wait_kick(vd, 1);
1460
1461 shift++;
1462 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1463 }
1464
1465 for (uint64_t c = 0; c < vd->vdev_children; c++) {
1466 vdev_t *cvd = vd->vdev_child[c];
1467 mutex_enter(&cvd->vdev_trim_io_lock);
1468
1469 while (cvd->vdev_trim_inflight[1] > 0) {
1470 cv_wait(&cvd->vdev_trim_io_cv,
1471 &cvd->vdev_trim_io_lock);
1472 }
1473 mutex_exit(&cvd->vdev_trim_io_lock);
1474 }
1475
1476 spa_config_exit(spa, SCL_CONFIG, FTAG);
1477
1478 /*
1479 * When exiting because the autotrim property was set to off, then
1480 * abandon any unprocessed ms_trim ranges to reclaim the memory.
1481 */
1482 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) {
1483 for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {
1484 metaslab_t *msp = vd->vdev_ms[i];
1485
1486 mutex_enter(&msp->ms_lock);
1487 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL);
1488 mutex_exit(&msp->ms_lock);
1489 }
1490 }
1491
1492 mutex_enter(&vd->vdev_autotrim_lock);
1493 ASSERT(vd->vdev_autotrim_thread != NULL);
1494 vd->vdev_autotrim_thread = NULL;
1495 cv_broadcast(&vd->vdev_autotrim_cv);
1496 mutex_exit(&vd->vdev_autotrim_lock);
1497
1498 thread_exit();
1499 }
1500
1501 /*
1502 * Starts an autotrim thread, if needed, for each top-level vdev which can be
1503 * trimmed. A top-level vdev which has been evacuated will never be trimmed.
1504 */
1505 void
vdev_autotrim(spa_t * spa)1506 vdev_autotrim(spa_t *spa)
1507 {
1508 vdev_t *root_vd = spa->spa_root_vdev;
1509
1510 for (uint64_t i = 0; i < root_vd->vdev_children; i++) {
1511 vdev_t *tvd = root_vd->vdev_child[i];
1512
1513 mutex_enter(&tvd->vdev_autotrim_lock);
1514 if (vdev_writeable(tvd) && !tvd->vdev_removing &&
1515 tvd->vdev_autotrim_thread == NULL &&
1516 !tvd->vdev_rz_expanding) {
1517 ASSERT3P(tvd->vdev_top, ==, tvd);
1518
1519 tvd->vdev_autotrim_thread = thread_create(NULL, 0,
1520 vdev_autotrim_thread, tvd, 0, &p0, TS_RUN,
1521 maxclsyspri);
1522 ASSERT(tvd->vdev_autotrim_thread != NULL);
1523 }
1524 mutex_exit(&tvd->vdev_autotrim_lock);
1525 }
1526 }
1527
1528 /*
1529 * Wait for the vdev_autotrim_thread associated with the passed top-level
1530 * vdev to be terminated (canceled or stopped).
1531 */
1532 void
vdev_autotrim_stop_wait(vdev_t * tvd)1533 vdev_autotrim_stop_wait(vdev_t *tvd)
1534 {
1535 mutex_enter(&tvd->vdev_autotrim_lock);
1536 if (tvd->vdev_autotrim_thread != NULL) {
1537 tvd->vdev_autotrim_exit_wanted = B_TRUE;
1538 cv_broadcast(&tvd->vdev_autotrim_kick_cv);
1539 cv_wait(&tvd->vdev_autotrim_cv,
1540 &tvd->vdev_autotrim_lock);
1541
1542 ASSERT0P(tvd->vdev_autotrim_thread);
1543 tvd->vdev_autotrim_exit_wanted = B_FALSE;
1544 }
1545 mutex_exit(&tvd->vdev_autotrim_lock);
1546 }
1547
1548 void
vdev_autotrim_kick(spa_t * spa)1549 vdev_autotrim_kick(spa_t *spa)
1550 {
1551 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER));
1552
1553 vdev_t *root_vd = spa->spa_root_vdev;
1554 vdev_t *tvd;
1555
1556 for (uint64_t i = 0; i < root_vd->vdev_children; i++) {
1557 tvd = root_vd->vdev_child[i];
1558
1559 mutex_enter(&tvd->vdev_autotrim_lock);
1560 if (tvd->vdev_autotrim_thread != NULL)
1561 cv_broadcast(&tvd->vdev_autotrim_kick_cv);
1562 mutex_exit(&tvd->vdev_autotrim_lock);
1563 }
1564 }
1565
1566 /*
1567 * Wait for all of the vdev_autotrim_thread associated with the pool to
1568 * be terminated (canceled or stopped).
1569 */
1570 void
vdev_autotrim_stop_all(spa_t * spa)1571 vdev_autotrim_stop_all(spa_t *spa)
1572 {
1573 vdev_t *root_vd = spa->spa_root_vdev;
1574
1575 for (uint64_t i = 0; i < root_vd->vdev_children; i++)
1576 vdev_autotrim_stop_wait(root_vd->vdev_child[i]);
1577 }
1578
1579 /*
1580 * Conditionally restart all of the vdev_autotrim_thread's for the pool.
1581 */
1582 void
vdev_autotrim_restart(spa_t * spa)1583 vdev_autotrim_restart(spa_t *spa)
1584 {
1585 ASSERT(MUTEX_HELD(&spa_namespace_lock) ||
1586 spa->spa_load_thread == curthread);
1587 if (spa->spa_autotrim)
1588 vdev_autotrim(spa);
1589 }
1590
1591 static __attribute__((noreturn)) void
vdev_trim_l2arc_thread(void * arg)1592 vdev_trim_l2arc_thread(void *arg)
1593 {
1594 vdev_t *vd = arg;
1595 spa_t *spa = vd->vdev_spa;
1596 l2arc_dev_t *dev = l2arc_vdev_get(vd);
1597 trim_args_t ta = {0};
1598 zfs_range_seg64_t physical_rs;
1599
1600 ASSERT(vdev_is_concrete(vd));
1601 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1602
1603 vd->vdev_trim_last_offset = 0;
1604 vd->vdev_trim_rate = 0;
1605 vd->vdev_trim_partial = 0;
1606 vd->vdev_trim_secure = 0;
1607
1608 ta.trim_vdev = vd;
1609 ta.trim_tree = zfs_range_tree_create_flags(
1610 NULL, ZFS_RANGE_SEG64, NULL, 0, 0,
1611 ZFS_RT_F_DYN_NAME, vdev_rt_name(vd, "trim_tree"));
1612 ta.trim_type = TRIM_TYPE_MANUAL;
1613 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
1614 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
1615 ta.trim_flags = 0;
1616
1617 physical_rs.rs_start = vd->vdev_trim_bytes_done = 0;
1618 physical_rs.rs_end = vd->vdev_trim_bytes_est =
1619 vdev_get_min_asize(vd);
1620
1621 zfs_range_tree_add(ta.trim_tree, physical_rs.rs_start,
1622 physical_rs.rs_end - physical_rs.rs_start);
1623
1624 mutex_enter(&vd->vdev_trim_lock);
1625 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);
1626 mutex_exit(&vd->vdev_trim_lock);
1627
1628 (void) vdev_trim_ranges(&ta);
1629
1630 spa_config_exit(spa, SCL_CONFIG, FTAG);
1631 mutex_enter(&vd->vdev_trim_io_lock);
1632 while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) {
1633 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
1634 }
1635 mutex_exit(&vd->vdev_trim_io_lock);
1636
1637 zfs_range_tree_vacate(ta.trim_tree, NULL, NULL);
1638 zfs_range_tree_destroy(ta.trim_tree);
1639
1640 mutex_enter(&vd->vdev_trim_lock);
1641 if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {
1642 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
1643 vd->vdev_trim_rate, vd->vdev_trim_partial,
1644 vd->vdev_trim_secure);
1645 }
1646 ASSERT(vd->vdev_trim_thread != NULL ||
1647 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0);
1648
1649 /*
1650 * Drop the vdev_trim_lock while we sync out the txg since it's
1651 * possible that a device might be trying to come online and
1652 * must check to see if it needs to restart a trim. That thread
1653 * will be holding the spa_config_lock which would prevent the
1654 * txg_wait_synced from completing. Same strategy as in
1655 * vdev_trim_thread().
1656 */
1657 mutex_exit(&vd->vdev_trim_lock);
1658 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1659 mutex_enter(&vd->vdev_trim_lock);
1660
1661 /*
1662 * Update the header of the cache device here, before
1663 * broadcasting vdev_trim_cv which may lead to the removal
1664 * of the device. The same applies for setting l2ad_trim_all to
1665 * false.
1666 */
1667 spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd,
1668 RW_READER);
1669 memset(dev->l2ad_dev_hdr, 0, dev->l2ad_dev_hdr_asize);
1670 l2arc_dev_hdr_update(dev);
1671 spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd);
1672
1673 vd->vdev_trim_thread = NULL;
1674 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE)
1675 dev->l2ad_trim_all = B_FALSE;
1676
1677 cv_broadcast(&vd->vdev_trim_cv);
1678 mutex_exit(&vd->vdev_trim_lock);
1679
1680 thread_exit();
1681 }
1682
1683 /*
1684 * Punches out TRIM threads for the L2ARC devices in a spa and assigns them
1685 * to vd->vdev_trim_thread variable. This facilitates the management of
1686 * trimming the whole cache device using TRIM_TYPE_MANUAL upon addition
1687 * to a pool or pool creation or when the header of the device is invalid.
1688 */
1689 void
vdev_trim_l2arc(spa_t * spa)1690 vdev_trim_l2arc(spa_t *spa)
1691 {
1692 ASSERT(MUTEX_HELD(&spa_namespace_lock));
1693
1694 /*
1695 * Locate the spa's l2arc devices and kick off TRIM threads.
1696 */
1697 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1698 vdev_t *vd = spa->spa_l2cache.sav_vdevs[i];
1699 l2arc_dev_t *dev = l2arc_vdev_get(vd);
1700
1701 if (dev == NULL || !dev->l2ad_trim_all) {
1702 /*
1703 * Don't attempt TRIM if the vdev is UNAVAIL or if the
1704 * cache device was not marked for whole device TRIM
1705 * (ie l2arc_trim_ahead = 0, or the L2ARC device header
1706 * is valid with trim_state = VDEV_TRIM_COMPLETE and
1707 * l2ad_log_entries > 0).
1708 */
1709 continue;
1710 }
1711
1712 mutex_enter(&vd->vdev_trim_lock);
1713 ASSERT(vd->vdev_ops->vdev_op_leaf);
1714 ASSERT(vdev_is_concrete(vd));
1715 ASSERT0P(vd->vdev_trim_thread);
1716 ASSERT(!vd->vdev_detached);
1717 ASSERT(!vd->vdev_trim_exit_wanted);
1718 ASSERT(!vd->vdev_top->vdev_removing);
1719 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);
1720 vd->vdev_trim_thread = thread_create(NULL, 0,
1721 vdev_trim_l2arc_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
1722 mutex_exit(&vd->vdev_trim_lock);
1723 }
1724 }
1725
1726 /*
1727 * A wrapper which calls vdev_trim_ranges(). It is intended to be called
1728 * on leaf vdevs.
1729 */
1730 int
vdev_trim_simple(vdev_t * vd,uint64_t start,uint64_t size)1731 vdev_trim_simple(vdev_t *vd, uint64_t start, uint64_t size)
1732 {
1733 trim_args_t ta = {0};
1734 zfs_range_seg64_t physical_rs;
1735 int error;
1736 physical_rs.rs_start = start;
1737 physical_rs.rs_end = start + size;
1738
1739 ASSERT(vdev_is_concrete(vd));
1740 ASSERT(vd->vdev_ops->vdev_op_leaf);
1741 ASSERT(!vd->vdev_detached);
1742 ASSERT(!vd->vdev_top->vdev_removing);
1743 ASSERT(!vd->vdev_top->vdev_rz_expanding);
1744
1745 ta.trim_vdev = vd;
1746 ta.trim_tree = zfs_range_tree_create_flags(
1747 NULL, ZFS_RANGE_SEG64, NULL, 0, 0,
1748 ZFS_RT_F_DYN_NAME, vdev_rt_name(vd, "trim_tree"));
1749 ta.trim_type = TRIM_TYPE_SIMPLE;
1750 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
1751 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
1752 ta.trim_flags = 0;
1753
1754 ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
1755
1756 if (physical_rs.rs_end > physical_rs.rs_start) {
1757 zfs_range_tree_add(ta.trim_tree, physical_rs.rs_start,
1758 physical_rs.rs_end - physical_rs.rs_start);
1759 } else {
1760 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
1761 }
1762
1763 error = vdev_trim_ranges(&ta);
1764
1765 mutex_enter(&vd->vdev_trim_io_lock);
1766 while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) {
1767 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
1768 }
1769 mutex_exit(&vd->vdev_trim_io_lock);
1770
1771 zfs_range_tree_vacate(ta.trim_tree, NULL, NULL);
1772 zfs_range_tree_destroy(ta.trim_tree);
1773
1774 return (error);
1775 }
1776
1777 EXPORT_SYMBOL(vdev_trim);
1778 EXPORT_SYMBOL(vdev_trim_stop);
1779 EXPORT_SYMBOL(vdev_trim_stop_all);
1780 EXPORT_SYMBOL(vdev_trim_stop_wait);
1781 EXPORT_SYMBOL(vdev_trim_restart);
1782 EXPORT_SYMBOL(vdev_autotrim);
1783 EXPORT_SYMBOL(vdev_autotrim_stop_all);
1784 EXPORT_SYMBOL(vdev_autotrim_stop_wait);
1785 EXPORT_SYMBOL(vdev_autotrim_restart);
1786 EXPORT_SYMBOL(vdev_trim_l2arc);
1787 EXPORT_SYMBOL(vdev_trim_simple);
1788
1789 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_max, UINT, ZMOD_RW,
1790 "Max size of TRIM commands, larger will be split");
1791
1792 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_min, UINT, ZMOD_RW,
1793 "Min size of TRIM commands, smaller will be skipped");
1794
1795 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, metaslab_skip, UINT, ZMOD_RW,
1796 "Skip metaslabs which have never been initialized");
1797
1798 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, txg_batch, UINT, ZMOD_RW,
1799 "Min number of txgs to aggregate frees before issuing TRIM");
1800
1801 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, queue_limit, UINT, ZMOD_RW,
1802 "Max queued TRIMs outstanding per leaf vdev");
1803