1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2016 by Delphix. All rights reserved. 24 * Copyright (c) 2019 by Lawrence Livermore National Security, LLC. 25 * Copyright (c) 2021 Hewlett Packard Enterprise Development LP 26 */ 27 28 #include <sys/spa.h> 29 #include <sys/spa_impl.h> 30 #include <sys/txg.h> 31 #include <sys/vdev_impl.h> 32 #include <sys/vdev_trim.h> 33 #include <sys/metaslab_impl.h> 34 #include <sys/dsl_synctask.h> 35 #include <sys/zap.h> 36 #include <sys/dmu_tx.h> 37 #include <sys/arc_impl.h> 38 39 /* 40 * TRIM is a feature which is used to notify a SSD that some previously 41 * written space is no longer allocated by the pool. This is useful because 42 * writes to a SSD must be performed to blocks which have first been erased. 43 * Ensuring the SSD always has a supply of erased blocks for new writes 44 * helps prevent the performance from deteriorating. 45 * 46 * There are two supported TRIM methods; manual and automatic. 47 * 48 * Manual TRIM: 49 * 50 * A manual TRIM is initiated by running the 'zpool trim' command. A single 51 * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for 52 * managing that vdev TRIM process. This involves iterating over all the 53 * metaslabs, calculating the unallocated space ranges, and then issuing the 54 * required TRIM I/Os. 55 * 56 * While a metaslab is being actively trimmed it is not eligible to perform 57 * new allocations. After traversing all of the metaslabs the thread is 58 * terminated. Finally, both the requested options and current progress of 59 * the TRIM are regularly written to the pool. This allows the TRIM to be 60 * suspended and resumed as needed. 61 * 62 * Automatic TRIM: 63 * 64 * An automatic TRIM is enabled by setting the 'autotrim' pool property 65 * to 'on'. When enabled, a `vdev_autotrim' thread is created for each 66 * top-level (not leaf) vdev in the pool. These threads perform the same 67 * core TRIM process as a manual TRIM, but with a few key differences. 68 * 69 * 1) Automatic TRIM happens continuously in the background and operates 70 * solely on recently freed blocks (ms_trim not ms_allocatable). 71 * 72 * 2) Each thread is associated with a top-level (not leaf) vdev. This has 73 * the benefit of simplifying the threading model, it makes it easier 74 * to coordinate administrative commands, and it ensures only a single 75 * metaslab is disabled at a time. Unlike manual TRIM, this means each 76 * 'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its 77 * children. 78 * 79 * 3) There is no automatic TRIM progress information stored on disk, nor 80 * is it reported by 'zpool status'. 81 * 82 * While the automatic TRIM process is highly effective it is more likely 83 * than a manual TRIM to encounter tiny ranges. Ranges less than or equal to 84 * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently 85 * TRIM and are skipped. This means small amounts of freed space may not 86 * be automatically trimmed. 87 * 88 * Furthermore, devices with attached hot spares and devices being actively 89 * replaced are skipped. This is done to avoid adding additional stress to 90 * a potentially unhealthy device and to minimize the required rebuild time. 91 * 92 * For this reason it may be beneficial to occasionally manually TRIM a pool 93 * even when automatic TRIM is enabled. 94 */ 95 96 /* 97 * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths. 98 */ 99 static unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024; 100 101 /* 102 * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped. 103 */ 104 static unsigned int zfs_trim_extent_bytes_min = 32 * 1024; 105 106 /* 107 * Skip uninitialized metaslabs during the TRIM process. This option is 108 * useful for pools constructed from large thinly-provisioned devices where 109 * TRIM operations are slow. As a pool ages an increasing fraction of 110 * the pools metaslabs will be initialized progressively degrading the 111 * usefulness of this option. This setting is stored when starting a 112 * manual TRIM and will persist for the duration of the requested TRIM. 113 */ 114 unsigned int zfs_trim_metaslab_skip = 0; 115 116 /* 117 * Maximum number of queued TRIM I/Os per leaf vdev. The number of 118 * concurrent TRIM I/Os issued to the device is controlled by the 119 * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options. 120 */ 121 static unsigned int zfs_trim_queue_limit = 10; 122 123 /* 124 * The minimum number of transaction groups between automatic trims of a 125 * metaslab. This setting represents a trade-off between issuing more 126 * efficient TRIM operations, by allowing them to be aggregated longer, 127 * and issuing them promptly so the trimmed space is available. Note 128 * that this value is a minimum; metaslabs can be trimmed less frequently 129 * when there are a large number of ranges which need to be trimmed. 130 * 131 * Increasing this value will allow frees to be aggregated for a longer 132 * time. This can result is larger TRIM operations, and increased memory 133 * usage in order to track the ranges to be trimmed. Decreasing this value 134 * has the opposite effect. The default value of 32 was determined though 135 * testing to be a reasonable compromise. 136 */ 137 static unsigned int zfs_trim_txg_batch = 32; 138 139 /* 140 * The trim_args are a control structure which describe how a leaf vdev 141 * should be trimmed. The core elements are the vdev, the metaslab being 142 * trimmed and a range tree containing the extents to TRIM. All provided 143 * ranges must be within the metaslab. 144 */ 145 typedef struct trim_args { 146 /* 147 * These fields are set by the caller of vdev_trim_ranges(). 148 */ 149 vdev_t *trim_vdev; /* Leaf vdev to TRIM */ 150 metaslab_t *trim_msp; /* Disabled metaslab */ 151 range_tree_t *trim_tree; /* TRIM ranges (in metaslab) */ 152 trim_type_t trim_type; /* Manual or auto TRIM */ 153 uint64_t trim_extent_bytes_max; /* Maximum TRIM I/O size */ 154 uint64_t trim_extent_bytes_min; /* Minimum TRIM I/O size */ 155 enum trim_flag trim_flags; /* TRIM flags (secure) */ 156 157 /* 158 * These fields are updated by vdev_trim_ranges(). 159 */ 160 hrtime_t trim_start_time; /* Start time */ 161 uint64_t trim_bytes_done; /* Bytes trimmed */ 162 } trim_args_t; 163 164 /* 165 * Determines whether a vdev_trim_thread() should be stopped. 166 */ 167 static boolean_t 168 vdev_trim_should_stop(vdev_t *vd) 169 { 170 return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) || 171 vd->vdev_detached || vd->vdev_top->vdev_removing); 172 } 173 174 /* 175 * Determines whether a vdev_autotrim_thread() should be stopped. 176 */ 177 static boolean_t 178 vdev_autotrim_should_stop(vdev_t *tvd) 179 { 180 return (tvd->vdev_autotrim_exit_wanted || 181 !vdev_writeable(tvd) || tvd->vdev_removing || 182 spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF); 183 } 184 185 /* 186 * Wait for given number of kicks, return true if the wait is aborted due to 187 * vdev_autotrim_exit_wanted. 188 */ 189 static boolean_t 190 vdev_autotrim_wait_kick(vdev_t *vd, int num_of_kick) 191 { 192 mutex_enter(&vd->vdev_autotrim_lock); 193 for (int i = 0; i < num_of_kick; i++) { 194 if (vd->vdev_autotrim_exit_wanted) 195 break; 196 cv_wait(&vd->vdev_autotrim_kick_cv, &vd->vdev_autotrim_lock); 197 } 198 boolean_t exit_wanted = vd->vdev_autotrim_exit_wanted; 199 mutex_exit(&vd->vdev_autotrim_lock); 200 201 return (exit_wanted); 202 } 203 204 /* 205 * The sync task for updating the on-disk state of a manual TRIM. This 206 * is scheduled by vdev_trim_change_state(). 207 */ 208 static void 209 vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx) 210 { 211 /* 212 * We pass in the guid instead of the vdev_t since the vdev may 213 * have been freed prior to the sync task being processed. This 214 * happens when a vdev is detached as we call spa_config_vdev_exit(), 215 * stop the trimming thread, schedule the sync task, and free 216 * the vdev. Later when the scheduled sync task is invoked, it would 217 * find that the vdev has been freed. 218 */ 219 uint64_t guid = *(uint64_t *)arg; 220 uint64_t txg = dmu_tx_get_txg(tx); 221 kmem_free(arg, sizeof (uint64_t)); 222 223 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE); 224 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd)) 225 return; 226 227 uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK]; 228 vd->vdev_trim_offset[txg & TXG_MASK] = 0; 229 230 VERIFY3U(vd->vdev_leaf_zap, !=, 0); 231 232 objset_t *mos = vd->vdev_spa->spa_meta_objset; 233 234 if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) { 235 236 if (vd->vdev_trim_last_offset == UINT64_MAX) 237 last_offset = 0; 238 239 vd->vdev_trim_last_offset = last_offset; 240 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 241 VDEV_LEAF_ZAP_TRIM_LAST_OFFSET, 242 sizeof (last_offset), 1, &last_offset, tx)); 243 } 244 245 if (vd->vdev_trim_action_time > 0) { 246 uint64_t val = (uint64_t)vd->vdev_trim_action_time; 247 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 248 VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val), 249 1, &val, tx)); 250 } 251 252 if (vd->vdev_trim_rate > 0) { 253 uint64_t rate = (uint64_t)vd->vdev_trim_rate; 254 255 if (rate == UINT64_MAX) 256 rate = 0; 257 258 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 259 VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx)); 260 } 261 262 uint64_t partial = vd->vdev_trim_partial; 263 if (partial == UINT64_MAX) 264 partial = 0; 265 266 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL, 267 sizeof (partial), 1, &partial, tx)); 268 269 uint64_t secure = vd->vdev_trim_secure; 270 if (secure == UINT64_MAX) 271 secure = 0; 272 273 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE, 274 sizeof (secure), 1, &secure, tx)); 275 276 277 uint64_t trim_state = vd->vdev_trim_state; 278 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE, 279 sizeof (trim_state), 1, &trim_state, tx)); 280 } 281 282 /* 283 * Update the on-disk state of a manual TRIM. This is called to request 284 * that a TRIM be started/suspended/canceled, or to change one of the 285 * TRIM options (partial, secure, rate). 286 */ 287 static void 288 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state, 289 uint64_t rate, boolean_t partial, boolean_t secure) 290 { 291 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 292 spa_t *spa = vd->vdev_spa; 293 294 if (new_state == vd->vdev_trim_state) 295 return; 296 297 /* 298 * Copy the vd's guid, this will be freed by the sync task. 299 */ 300 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 301 *guid = vd->vdev_guid; 302 303 /* 304 * If we're suspending, then preserve the original start time. 305 */ 306 if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) { 307 vd->vdev_trim_action_time = gethrestime_sec(); 308 } 309 310 /* 311 * If we're activating, then preserve the requested rate and trim 312 * method. Setting the last offset and rate to UINT64_MAX is used 313 * as a sentinel to indicate they should be reset to default values. 314 */ 315 if (new_state == VDEV_TRIM_ACTIVE) { 316 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE || 317 vd->vdev_trim_state == VDEV_TRIM_CANCELED) { 318 vd->vdev_trim_last_offset = UINT64_MAX; 319 vd->vdev_trim_rate = UINT64_MAX; 320 vd->vdev_trim_partial = UINT64_MAX; 321 vd->vdev_trim_secure = UINT64_MAX; 322 } 323 324 if (rate != 0) 325 vd->vdev_trim_rate = rate; 326 327 if (partial != 0) 328 vd->vdev_trim_partial = partial; 329 330 if (secure != 0) 331 vd->vdev_trim_secure = secure; 332 } 333 334 vdev_trim_state_t old_state = vd->vdev_trim_state; 335 boolean_t resumed = (old_state == VDEV_TRIM_SUSPENDED); 336 vd->vdev_trim_state = new_state; 337 338 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 339 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 340 dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync, 341 guid, tx); 342 343 switch (new_state) { 344 case VDEV_TRIM_ACTIVE: 345 spa_event_notify(spa, vd, NULL, 346 resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START); 347 spa_history_log_internal(spa, "trim", tx, 348 "vdev=%s activated", vd->vdev_path); 349 break; 350 case VDEV_TRIM_SUSPENDED: 351 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND); 352 spa_history_log_internal(spa, "trim", tx, 353 "vdev=%s suspended", vd->vdev_path); 354 break; 355 case VDEV_TRIM_CANCELED: 356 if (old_state == VDEV_TRIM_ACTIVE || 357 old_state == VDEV_TRIM_SUSPENDED) { 358 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL); 359 spa_history_log_internal(spa, "trim", tx, 360 "vdev=%s canceled", vd->vdev_path); 361 } 362 break; 363 case VDEV_TRIM_COMPLETE: 364 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH); 365 spa_history_log_internal(spa, "trim", tx, 366 "vdev=%s complete", vd->vdev_path); 367 break; 368 default: 369 panic("invalid state %llu", (unsigned long long)new_state); 370 } 371 372 dmu_tx_commit(tx); 373 374 if (new_state != VDEV_TRIM_ACTIVE) 375 spa_notify_waiters(spa); 376 } 377 378 /* 379 * The zio_done_func_t done callback for each manual TRIM issued. It is 380 * responsible for updating the TRIM stats, reissuing failed TRIM I/Os, 381 * and limiting the number of in flight TRIM I/Os. 382 */ 383 static void 384 vdev_trim_cb(zio_t *zio) 385 { 386 vdev_t *vd = zio->io_vd; 387 388 mutex_enter(&vd->vdev_trim_io_lock); 389 if (zio->io_error == ENXIO && !vdev_writeable(vd)) { 390 /* 391 * The I/O failed because the vdev was unavailable; roll the 392 * last offset back. (This works because spa_sync waits on 393 * spa_txg_zio before it runs sync tasks.) 394 */ 395 uint64_t *offset = 396 &vd->vdev_trim_offset[zio->io_txg & TXG_MASK]; 397 *offset = MIN(*offset, zio->io_offset); 398 } else { 399 if (zio->io_error != 0) { 400 vd->vdev_stat.vs_trim_errors++; 401 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL, 402 0, 0, 0, 0, 1, zio->io_orig_size); 403 } else { 404 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL, 405 1, zio->io_orig_size, 0, 0, 0, 0); 406 } 407 408 vd->vdev_trim_bytes_done += zio->io_orig_size; 409 } 410 411 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0); 412 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--; 413 cv_broadcast(&vd->vdev_trim_io_cv); 414 mutex_exit(&vd->vdev_trim_io_lock); 415 416 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 417 } 418 419 /* 420 * The zio_done_func_t done callback for each automatic TRIM issued. It 421 * is responsible for updating the TRIM stats and limiting the number of 422 * in flight TRIM I/Os. Automatic TRIM I/Os are best effort and are 423 * never reissued on failure. 424 */ 425 static void 426 vdev_autotrim_cb(zio_t *zio) 427 { 428 vdev_t *vd = zio->io_vd; 429 430 mutex_enter(&vd->vdev_trim_io_lock); 431 432 if (zio->io_error != 0) { 433 vd->vdev_stat.vs_trim_errors++; 434 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO, 435 0, 0, 0, 0, 1, zio->io_orig_size); 436 } else { 437 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO, 438 1, zio->io_orig_size, 0, 0, 0, 0); 439 } 440 441 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0); 442 vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--; 443 cv_broadcast(&vd->vdev_trim_io_cv); 444 mutex_exit(&vd->vdev_trim_io_lock); 445 446 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 447 } 448 449 /* 450 * The zio_done_func_t done callback for each TRIM issued via 451 * vdev_trim_simple(). It is responsible for updating the TRIM stats and 452 * limiting the number of in flight TRIM I/Os. Simple TRIM I/Os are best 453 * effort and are never reissued on failure. 454 */ 455 static void 456 vdev_trim_simple_cb(zio_t *zio) 457 { 458 vdev_t *vd = zio->io_vd; 459 460 mutex_enter(&vd->vdev_trim_io_lock); 461 462 if (zio->io_error != 0) { 463 vd->vdev_stat.vs_trim_errors++; 464 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE, 465 0, 0, 0, 0, 1, zio->io_orig_size); 466 } else { 467 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE, 468 1, zio->io_orig_size, 0, 0, 0, 0); 469 } 470 471 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0); 472 vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--; 473 cv_broadcast(&vd->vdev_trim_io_cv); 474 mutex_exit(&vd->vdev_trim_io_lock); 475 476 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 477 } 478 /* 479 * Returns the average trim rate in bytes/sec for the ta->trim_vdev. 480 */ 481 static uint64_t 482 vdev_trim_calculate_rate(trim_args_t *ta) 483 { 484 return (ta->trim_bytes_done * 1000 / 485 (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1)); 486 } 487 488 /* 489 * Issues a physical TRIM and takes care of rate limiting (bytes/sec) 490 * and number of concurrent TRIM I/Os. 491 */ 492 static int 493 vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size) 494 { 495 vdev_t *vd = ta->trim_vdev; 496 spa_t *spa = vd->vdev_spa; 497 void *cb; 498 499 mutex_enter(&vd->vdev_trim_io_lock); 500 501 /* 502 * Limit manual TRIM I/Os to the requested rate. This does not 503 * apply to automatic TRIM since no per vdev rate can be specified. 504 */ 505 if (ta->trim_type == TRIM_TYPE_MANUAL) { 506 while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) && 507 vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) { 508 cv_timedwait_idle(&vd->vdev_trim_io_cv, 509 &vd->vdev_trim_io_lock, ddi_get_lbolt() + 510 MSEC_TO_TICK(10)); 511 } 512 } 513 ta->trim_bytes_done += size; 514 515 /* Limit in flight trimming I/Os */ 516 while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] + 517 vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) { 518 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 519 } 520 vd->vdev_trim_inflight[ta->trim_type]++; 521 mutex_exit(&vd->vdev_trim_io_lock); 522 523 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 524 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 525 uint64_t txg = dmu_tx_get_txg(tx); 526 527 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER); 528 mutex_enter(&vd->vdev_trim_lock); 529 530 if (ta->trim_type == TRIM_TYPE_MANUAL && 531 vd->vdev_trim_offset[txg & TXG_MASK] == 0) { 532 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 533 *guid = vd->vdev_guid; 534 535 /* This is the first write of this txg. */ 536 dsl_sync_task_nowait(spa_get_dsl(spa), 537 vdev_trim_zap_update_sync, guid, tx); 538 } 539 540 /* 541 * We know the vdev_t will still be around since all consumers of 542 * vdev_free must stop the trimming first. 543 */ 544 if ((ta->trim_type == TRIM_TYPE_MANUAL && 545 vdev_trim_should_stop(vd)) || 546 (ta->trim_type == TRIM_TYPE_AUTO && 547 vdev_autotrim_should_stop(vd->vdev_top))) { 548 mutex_enter(&vd->vdev_trim_io_lock); 549 vd->vdev_trim_inflight[ta->trim_type]--; 550 mutex_exit(&vd->vdev_trim_io_lock); 551 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 552 mutex_exit(&vd->vdev_trim_lock); 553 dmu_tx_commit(tx); 554 return (SET_ERROR(EINTR)); 555 } 556 mutex_exit(&vd->vdev_trim_lock); 557 558 if (ta->trim_type == TRIM_TYPE_MANUAL) 559 vd->vdev_trim_offset[txg & TXG_MASK] = start + size; 560 561 if (ta->trim_type == TRIM_TYPE_MANUAL) { 562 cb = vdev_trim_cb; 563 } else if (ta->trim_type == TRIM_TYPE_AUTO) { 564 cb = vdev_autotrim_cb; 565 } else { 566 cb = vdev_trim_simple_cb; 567 } 568 569 zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd, 570 start, size, cb, NULL, ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL, 571 ta->trim_flags)); 572 /* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */ 573 574 dmu_tx_commit(tx); 575 576 return (0); 577 } 578 579 /* 580 * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree. 581 * Additional parameters describing how the TRIM should be performed must 582 * be set in the trim_args structure. See the trim_args definition for 583 * additional information. 584 */ 585 static int 586 vdev_trim_ranges(trim_args_t *ta) 587 { 588 vdev_t *vd = ta->trim_vdev; 589 zfs_btree_t *t = &ta->trim_tree->rt_root; 590 zfs_btree_index_t idx; 591 uint64_t extent_bytes_max = ta->trim_extent_bytes_max; 592 uint64_t extent_bytes_min = ta->trim_extent_bytes_min; 593 spa_t *spa = vd->vdev_spa; 594 595 ta->trim_start_time = gethrtime(); 596 ta->trim_bytes_done = 0; 597 598 for (range_seg_t *rs = zfs_btree_first(t, &idx); rs != NULL; 599 rs = zfs_btree_next(t, &idx, &idx)) { 600 uint64_t size = rs_get_end(rs, ta->trim_tree) - rs_get_start(rs, 601 ta->trim_tree); 602 603 if (extent_bytes_min && size < extent_bytes_min) { 604 spa_iostats_trim_add(spa, ta->trim_type, 605 0, 0, 1, size, 0, 0); 606 continue; 607 } 608 609 /* Split range into legally-sized physical chunks */ 610 uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1; 611 612 for (uint64_t w = 0; w < writes_required; w++) { 613 int error; 614 615 error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE + 616 rs_get_start(rs, ta->trim_tree) + 617 (w *extent_bytes_max), MIN(size - 618 (w * extent_bytes_max), extent_bytes_max)); 619 if (error != 0) { 620 return (error); 621 } 622 } 623 } 624 625 return (0); 626 } 627 628 static void 629 vdev_trim_xlate_last_rs_end(void *arg, range_seg64_t *physical_rs) 630 { 631 uint64_t *last_rs_end = (uint64_t *)arg; 632 633 if (physical_rs->rs_end > *last_rs_end) 634 *last_rs_end = physical_rs->rs_end; 635 } 636 637 static void 638 vdev_trim_xlate_progress(void *arg, range_seg64_t *physical_rs) 639 { 640 vdev_t *vd = (vdev_t *)arg; 641 642 uint64_t size = physical_rs->rs_end - physical_rs->rs_start; 643 vd->vdev_trim_bytes_est += size; 644 645 if (vd->vdev_trim_last_offset >= physical_rs->rs_end) { 646 vd->vdev_trim_bytes_done += size; 647 } else if (vd->vdev_trim_last_offset > physical_rs->rs_start && 648 vd->vdev_trim_last_offset <= physical_rs->rs_end) { 649 vd->vdev_trim_bytes_done += 650 vd->vdev_trim_last_offset - physical_rs->rs_start; 651 } 652 } 653 654 /* 655 * Calculates the completion percentage of a manual TRIM. 656 */ 657 static void 658 vdev_trim_calculate_progress(vdev_t *vd) 659 { 660 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 661 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 662 ASSERT(vd->vdev_leaf_zap != 0); 663 664 vd->vdev_trim_bytes_est = 0; 665 vd->vdev_trim_bytes_done = 0; 666 667 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) { 668 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 669 mutex_enter(&msp->ms_lock); 670 671 uint64_t ms_free = (msp->ms_size - 672 metaslab_allocated_space(msp)) / 673 vdev_get_ndisks(vd->vdev_top); 674 675 /* 676 * Convert the metaslab range to a physical range 677 * on our vdev. We use this to determine if we are 678 * in the middle of this metaslab range. 679 */ 680 range_seg64_t logical_rs, physical_rs, remain_rs; 681 logical_rs.rs_start = msp->ms_start; 682 logical_rs.rs_end = msp->ms_start + msp->ms_size; 683 684 /* Metaslab space after this offset has not been trimmed. */ 685 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs); 686 if (vd->vdev_trim_last_offset <= physical_rs.rs_start) { 687 vd->vdev_trim_bytes_est += ms_free; 688 mutex_exit(&msp->ms_lock); 689 continue; 690 } 691 692 /* Metaslab space before this offset has been trimmed */ 693 uint64_t last_rs_end = physical_rs.rs_end; 694 if (!vdev_xlate_is_empty(&remain_rs)) { 695 vdev_xlate_walk(vd, &remain_rs, 696 vdev_trim_xlate_last_rs_end, &last_rs_end); 697 } 698 699 if (vd->vdev_trim_last_offset > last_rs_end) { 700 vd->vdev_trim_bytes_done += ms_free; 701 vd->vdev_trim_bytes_est += ms_free; 702 mutex_exit(&msp->ms_lock); 703 continue; 704 } 705 706 /* 707 * If we get here, we're in the middle of trimming this 708 * metaslab. Load it and walk the free tree for more 709 * accurate progress estimation. 710 */ 711 VERIFY0(metaslab_load(msp)); 712 713 range_tree_t *rt = msp->ms_allocatable; 714 zfs_btree_t *bt = &rt->rt_root; 715 zfs_btree_index_t idx; 716 for (range_seg_t *rs = zfs_btree_first(bt, &idx); 717 rs != NULL; rs = zfs_btree_next(bt, &idx, &idx)) { 718 logical_rs.rs_start = rs_get_start(rs, rt); 719 logical_rs.rs_end = rs_get_end(rs, rt); 720 721 vdev_xlate_walk(vd, &logical_rs, 722 vdev_trim_xlate_progress, vd); 723 } 724 mutex_exit(&msp->ms_lock); 725 } 726 } 727 728 /* 729 * Load from disk the vdev's manual TRIM information. This includes the 730 * state, progress, and options provided when initiating the manual TRIM. 731 */ 732 static int 733 vdev_trim_load(vdev_t *vd) 734 { 735 int err = 0; 736 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 737 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 738 ASSERT(vd->vdev_leaf_zap != 0); 739 740 if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE || 741 vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) { 742 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 743 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET, 744 sizeof (vd->vdev_trim_last_offset), 1, 745 &vd->vdev_trim_last_offset); 746 if (err == ENOENT) { 747 vd->vdev_trim_last_offset = 0; 748 err = 0; 749 } 750 751 if (err == 0) { 752 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 753 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE, 754 sizeof (vd->vdev_trim_rate), 1, 755 &vd->vdev_trim_rate); 756 if (err == ENOENT) { 757 vd->vdev_trim_rate = 0; 758 err = 0; 759 } 760 } 761 762 if (err == 0) { 763 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 764 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL, 765 sizeof (vd->vdev_trim_partial), 1, 766 &vd->vdev_trim_partial); 767 if (err == ENOENT) { 768 vd->vdev_trim_partial = 0; 769 err = 0; 770 } 771 } 772 773 if (err == 0) { 774 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 775 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE, 776 sizeof (vd->vdev_trim_secure), 1, 777 &vd->vdev_trim_secure); 778 if (err == ENOENT) { 779 vd->vdev_trim_secure = 0; 780 err = 0; 781 } 782 } 783 } 784 785 vdev_trim_calculate_progress(vd); 786 787 return (err); 788 } 789 790 static void 791 vdev_trim_xlate_range_add(void *arg, range_seg64_t *physical_rs) 792 { 793 trim_args_t *ta = arg; 794 vdev_t *vd = ta->trim_vdev; 795 796 /* 797 * Only a manual trim will be traversing the vdev sequentially. 798 * For an auto trim all valid ranges should be added. 799 */ 800 if (ta->trim_type == TRIM_TYPE_MANUAL) { 801 802 /* Only add segments that we have not visited yet */ 803 if (physical_rs->rs_end <= vd->vdev_trim_last_offset) 804 return; 805 806 /* Pick up where we left off mid-range. */ 807 if (vd->vdev_trim_last_offset > physical_rs->rs_start) { 808 ASSERT3U(physical_rs->rs_end, >, 809 vd->vdev_trim_last_offset); 810 physical_rs->rs_start = vd->vdev_trim_last_offset; 811 } 812 } 813 814 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start); 815 816 range_tree_add(ta->trim_tree, physical_rs->rs_start, 817 physical_rs->rs_end - physical_rs->rs_start); 818 } 819 820 /* 821 * Convert the logical range into physical ranges and add them to the 822 * range tree passed in the trim_args_t. 823 */ 824 static void 825 vdev_trim_range_add(void *arg, uint64_t start, uint64_t size) 826 { 827 trim_args_t *ta = arg; 828 vdev_t *vd = ta->trim_vdev; 829 range_seg64_t logical_rs; 830 logical_rs.rs_start = start; 831 logical_rs.rs_end = start + size; 832 833 /* 834 * Every range to be trimmed must be part of ms_allocatable. 835 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this 836 * is always the case. 837 */ 838 if (zfs_flags & ZFS_DEBUG_TRIM) { 839 metaslab_t *msp = ta->trim_msp; 840 VERIFY0(metaslab_load(msp)); 841 VERIFY3B(msp->ms_loaded, ==, B_TRUE); 842 VERIFY(range_tree_contains(msp->ms_allocatable, start, size)); 843 } 844 845 ASSERT(vd->vdev_ops->vdev_op_leaf); 846 vdev_xlate_walk(vd, &logical_rs, vdev_trim_xlate_range_add, arg); 847 } 848 849 /* 850 * Each manual TRIM thread is responsible for trimming the unallocated 851 * space for each leaf vdev. This is accomplished by sequentially iterating 852 * over its top-level metaslabs and issuing TRIM I/O for the space described 853 * by its ms_allocatable. While a metaslab is undergoing trimming it is 854 * not eligible for new allocations. 855 */ 856 static __attribute__((noreturn)) void 857 vdev_trim_thread(void *arg) 858 { 859 vdev_t *vd = arg; 860 spa_t *spa = vd->vdev_spa; 861 trim_args_t ta; 862 int error = 0; 863 864 /* 865 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by 866 * vdev_trim(). Wait for the updated values to be reflected 867 * in the zap in order to start with the requested settings. 868 */ 869 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 870 871 ASSERT(vdev_is_concrete(vd)); 872 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 873 874 vd->vdev_trim_last_offset = 0; 875 vd->vdev_trim_rate = 0; 876 vd->vdev_trim_partial = 0; 877 vd->vdev_trim_secure = 0; 878 879 VERIFY0(vdev_trim_load(vd)); 880 881 ta.trim_vdev = vd; 882 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max; 883 ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min; 884 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0); 885 ta.trim_type = TRIM_TYPE_MANUAL; 886 ta.trim_flags = 0; 887 888 /* 889 * When a secure TRIM has been requested infer that the intent 890 * is that everything must be trimmed. Override the default 891 * minimum TRIM size to prevent ranges from being skipped. 892 */ 893 if (vd->vdev_trim_secure) { 894 ta.trim_flags |= ZIO_TRIM_SECURE; 895 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE; 896 } 897 898 uint64_t ms_count = 0; 899 for (uint64_t i = 0; !vd->vdev_detached && 900 i < vd->vdev_top->vdev_ms_count; i++) { 901 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 902 903 /* 904 * If we've expanded the top-level vdev or it's our 905 * first pass, calculate our progress. 906 */ 907 if (vd->vdev_top->vdev_ms_count != ms_count) { 908 vdev_trim_calculate_progress(vd); 909 ms_count = vd->vdev_top->vdev_ms_count; 910 } 911 912 spa_config_exit(spa, SCL_CONFIG, FTAG); 913 metaslab_disable(msp); 914 mutex_enter(&msp->ms_lock); 915 VERIFY0(metaslab_load(msp)); 916 917 /* 918 * If a partial TRIM was requested skip metaslabs which have 919 * never been initialized and thus have never been written. 920 */ 921 if (msp->ms_sm == NULL && vd->vdev_trim_partial) { 922 mutex_exit(&msp->ms_lock); 923 metaslab_enable(msp, B_FALSE, B_FALSE); 924 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 925 vdev_trim_calculate_progress(vd); 926 continue; 927 } 928 929 ta.trim_msp = msp; 930 range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta); 931 range_tree_vacate(msp->ms_trim, NULL, NULL); 932 mutex_exit(&msp->ms_lock); 933 934 error = vdev_trim_ranges(&ta); 935 metaslab_enable(msp, B_TRUE, B_FALSE); 936 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 937 938 range_tree_vacate(ta.trim_tree, NULL, NULL); 939 if (error != 0) 940 break; 941 } 942 943 spa_config_exit(spa, SCL_CONFIG, FTAG); 944 mutex_enter(&vd->vdev_trim_io_lock); 945 while (vd->vdev_trim_inflight[0] > 0) { 946 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 947 } 948 mutex_exit(&vd->vdev_trim_io_lock); 949 950 range_tree_destroy(ta.trim_tree); 951 952 mutex_enter(&vd->vdev_trim_lock); 953 if (!vd->vdev_trim_exit_wanted) { 954 if (vdev_writeable(vd)) { 955 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, 956 vd->vdev_trim_rate, vd->vdev_trim_partial, 957 vd->vdev_trim_secure); 958 } else if (vd->vdev_faulted) { 959 vdev_trim_change_state(vd, VDEV_TRIM_CANCELED, 960 vd->vdev_trim_rate, vd->vdev_trim_partial, 961 vd->vdev_trim_secure); 962 } 963 } 964 ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0); 965 966 /* 967 * Drop the vdev_trim_lock while we sync out the txg since it's 968 * possible that a device might be trying to come online and must 969 * check to see if it needs to restart a trim. That thread will be 970 * holding the spa_config_lock which would prevent the txg_wait_synced 971 * from completing. 972 */ 973 mutex_exit(&vd->vdev_trim_lock); 974 txg_wait_synced(spa_get_dsl(spa), 0); 975 mutex_enter(&vd->vdev_trim_lock); 976 977 vd->vdev_trim_thread = NULL; 978 cv_broadcast(&vd->vdev_trim_cv); 979 mutex_exit(&vd->vdev_trim_lock); 980 981 thread_exit(); 982 } 983 984 /* 985 * Initiates a manual TRIM for the vdev_t. Callers must hold vdev_trim_lock, 986 * the vdev_t must be a leaf and cannot already be manually trimming. 987 */ 988 void 989 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure) 990 { 991 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 992 ASSERT(vd->vdev_ops->vdev_op_leaf); 993 ASSERT(vdev_is_concrete(vd)); 994 ASSERT3P(vd->vdev_trim_thread, ==, NULL); 995 ASSERT(!vd->vdev_detached); 996 ASSERT(!vd->vdev_trim_exit_wanted); 997 ASSERT(!vd->vdev_top->vdev_removing); 998 999 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure); 1000 vd->vdev_trim_thread = thread_create(NULL, 0, 1001 vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri); 1002 } 1003 1004 /* 1005 * Wait for the trimming thread to be terminated (canceled or stopped). 1006 */ 1007 static void 1008 vdev_trim_stop_wait_impl(vdev_t *vd) 1009 { 1010 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 1011 1012 while (vd->vdev_trim_thread != NULL) 1013 cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock); 1014 1015 ASSERT3P(vd->vdev_trim_thread, ==, NULL); 1016 vd->vdev_trim_exit_wanted = B_FALSE; 1017 } 1018 1019 /* 1020 * Wait for vdev trim threads which were listed to cleanly exit. 1021 */ 1022 void 1023 vdev_trim_stop_wait(spa_t *spa, list_t *vd_list) 1024 { 1025 (void) spa; 1026 vdev_t *vd; 1027 1028 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1029 1030 while ((vd = list_remove_head(vd_list)) != NULL) { 1031 mutex_enter(&vd->vdev_trim_lock); 1032 vdev_trim_stop_wait_impl(vd); 1033 mutex_exit(&vd->vdev_trim_lock); 1034 } 1035 } 1036 1037 /* 1038 * Stop trimming a device, with the resultant trimming state being tgt_state. 1039 * For blocking behavior pass NULL for vd_list. Otherwise, when a list_t is 1040 * provided the stopping vdev is inserted in to the list. Callers are then 1041 * required to call vdev_trim_stop_wait() to block for all the trim threads 1042 * to exit. The caller must hold vdev_trim_lock and must not be writing to 1043 * the spa config, as the trimming thread may try to enter the config as a 1044 * reader before exiting. 1045 */ 1046 void 1047 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list) 1048 { 1049 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER)); 1050 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); 1051 ASSERT(vd->vdev_ops->vdev_op_leaf); 1052 ASSERT(vdev_is_concrete(vd)); 1053 1054 /* 1055 * Allow cancel requests to proceed even if the trim thread has 1056 * stopped. 1057 */ 1058 if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED) 1059 return; 1060 1061 vdev_trim_change_state(vd, tgt_state, 0, 0, 0); 1062 vd->vdev_trim_exit_wanted = B_TRUE; 1063 1064 if (vd_list == NULL) { 1065 vdev_trim_stop_wait_impl(vd); 1066 } else { 1067 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1068 list_insert_tail(vd_list, vd); 1069 } 1070 } 1071 1072 /* 1073 * Requests that all listed vdevs stop trimming. 1074 */ 1075 static void 1076 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state, 1077 list_t *vd_list) 1078 { 1079 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) { 1080 mutex_enter(&vd->vdev_trim_lock); 1081 vdev_trim_stop(vd, tgt_state, vd_list); 1082 mutex_exit(&vd->vdev_trim_lock); 1083 return; 1084 } 1085 1086 for (uint64_t i = 0; i < vd->vdev_children; i++) { 1087 vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state, 1088 vd_list); 1089 } 1090 } 1091 1092 /* 1093 * Convenience function to stop trimming of a vdev tree and set all trim 1094 * thread pointers to NULL. 1095 */ 1096 void 1097 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state) 1098 { 1099 spa_t *spa = vd->vdev_spa; 1100 list_t vd_list; 1101 vdev_t *vd_l2cache; 1102 1103 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1104 1105 list_create(&vd_list, sizeof (vdev_t), 1106 offsetof(vdev_t, vdev_trim_node)); 1107 1108 vdev_trim_stop_all_impl(vd, tgt_state, &vd_list); 1109 1110 /* 1111 * Iterate over cache devices and request stop trimming the 1112 * whole device in case we export the pool or remove the cache 1113 * device prematurely. 1114 */ 1115 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1116 vd_l2cache = spa->spa_l2cache.sav_vdevs[i]; 1117 vdev_trim_stop_all_impl(vd_l2cache, tgt_state, &vd_list); 1118 } 1119 1120 vdev_trim_stop_wait(spa, &vd_list); 1121 1122 if (vd->vdev_spa->spa_sync_on) { 1123 /* Make sure that our state has been synced to disk */ 1124 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 1125 } 1126 1127 list_destroy(&vd_list); 1128 } 1129 1130 /* 1131 * Conditionally restarts a manual TRIM given its on-disk state. 1132 */ 1133 void 1134 vdev_trim_restart(vdev_t *vd) 1135 { 1136 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1137 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER)); 1138 1139 if (vd->vdev_leaf_zap != 0) { 1140 mutex_enter(&vd->vdev_trim_lock); 1141 uint64_t trim_state = VDEV_TRIM_NONE; 1142 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, 1143 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE, 1144 sizeof (trim_state), 1, &trim_state); 1145 ASSERT(err == 0 || err == ENOENT); 1146 vd->vdev_trim_state = trim_state; 1147 1148 uint64_t timestamp = 0; 1149 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 1150 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME, 1151 sizeof (timestamp), 1, ×tamp); 1152 ASSERT(err == 0 || err == ENOENT); 1153 vd->vdev_trim_action_time = timestamp; 1154 1155 if (vd->vdev_trim_state == VDEV_TRIM_SUSPENDED || 1156 vd->vdev_offline) { 1157 /* load progress for reporting, but don't resume */ 1158 VERIFY0(vdev_trim_load(vd)); 1159 } else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE && 1160 vdev_writeable(vd) && !vd->vdev_top->vdev_removing && 1161 vd->vdev_trim_thread == NULL) { 1162 VERIFY0(vdev_trim_load(vd)); 1163 vdev_trim(vd, vd->vdev_trim_rate, 1164 vd->vdev_trim_partial, vd->vdev_trim_secure); 1165 } 1166 1167 mutex_exit(&vd->vdev_trim_lock); 1168 } 1169 1170 for (uint64_t i = 0; i < vd->vdev_children; i++) { 1171 vdev_trim_restart(vd->vdev_child[i]); 1172 } 1173 } 1174 1175 /* 1176 * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that 1177 * every TRIM range is contained within ms_allocatable. 1178 */ 1179 static void 1180 vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size) 1181 { 1182 trim_args_t *ta = arg; 1183 metaslab_t *msp = ta->trim_msp; 1184 1185 VERIFY3B(msp->ms_loaded, ==, B_TRUE); 1186 VERIFY3U(msp->ms_disabled, >, 0); 1187 VERIFY(range_tree_contains(msp->ms_allocatable, start, size)); 1188 } 1189 1190 /* 1191 * Each automatic TRIM thread is responsible for managing the trimming of a 1192 * top-level vdev in the pool. No automatic TRIM state is maintained on-disk. 1193 * 1194 * N.B. This behavior is different from a manual TRIM where a thread 1195 * is created for each leaf vdev, instead of each top-level vdev. 1196 */ 1197 static __attribute__((noreturn)) void 1198 vdev_autotrim_thread(void *arg) 1199 { 1200 vdev_t *vd = arg; 1201 spa_t *spa = vd->vdev_spa; 1202 int shift = 0; 1203 1204 mutex_enter(&vd->vdev_autotrim_lock); 1205 ASSERT3P(vd->vdev_top, ==, vd); 1206 ASSERT3P(vd->vdev_autotrim_thread, !=, NULL); 1207 mutex_exit(&vd->vdev_autotrim_lock); 1208 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1209 1210 while (!vdev_autotrim_should_stop(vd)) { 1211 int txgs_per_trim = MAX(zfs_trim_txg_batch, 1); 1212 uint64_t extent_bytes_max = zfs_trim_extent_bytes_max; 1213 uint64_t extent_bytes_min = zfs_trim_extent_bytes_min; 1214 1215 /* 1216 * All of the metaslabs are divided in to groups of size 1217 * num_metaslabs / zfs_trim_txg_batch. Each of these groups 1218 * is composed of metaslabs which are spread evenly over the 1219 * device. 1220 * 1221 * For example, when zfs_trim_txg_batch = 32 (default) then 1222 * group 0 will contain metaslabs 0, 32, 64, ...; 1223 * group 1 will contain metaslabs 1, 33, 65, ...; 1224 * group 2 will contain metaslabs 2, 34, 66, ...; and so on. 1225 * 1226 * On each pass through the while() loop one of these groups 1227 * is selected. This is accomplished by using a shift value 1228 * to select the starting metaslab, then striding over the 1229 * metaslabs using the zfs_trim_txg_batch size. This is 1230 * done to accomplish two things. 1231 * 1232 * 1) By dividing the metaslabs in to groups, and making sure 1233 * that each group takes a minimum of one txg to process. 1234 * Then zfs_trim_txg_batch controls the minimum number of 1235 * txgs which must occur before a metaslab is revisited. 1236 * 1237 * 2) Selecting non-consecutive metaslabs distributes the 1238 * TRIM commands for a group evenly over the entire device. 1239 * This can be advantageous for certain types of devices. 1240 */ 1241 for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count; 1242 i += txgs_per_trim) { 1243 metaslab_t *msp = vd->vdev_ms[i]; 1244 range_tree_t *trim_tree; 1245 boolean_t issued_trim = B_FALSE; 1246 boolean_t wait_aborted = B_FALSE; 1247 1248 spa_config_exit(spa, SCL_CONFIG, FTAG); 1249 metaslab_disable(msp); 1250 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1251 1252 mutex_enter(&msp->ms_lock); 1253 1254 /* 1255 * Skip the metaslab when it has never been allocated 1256 * or when there are no recent frees to trim. 1257 */ 1258 if (msp->ms_sm == NULL || 1259 range_tree_is_empty(msp->ms_trim)) { 1260 mutex_exit(&msp->ms_lock); 1261 metaslab_enable(msp, B_FALSE, B_FALSE); 1262 continue; 1263 } 1264 1265 /* 1266 * Skip the metaslab when it has already been disabled. 1267 * This may happen when a manual TRIM or initialize 1268 * operation is running concurrently. In the case 1269 * of a manual TRIM, the ms_trim tree will have been 1270 * vacated. Only ranges added after the manual TRIM 1271 * disabled the metaslab will be included in the tree. 1272 * These will be processed when the automatic TRIM 1273 * next revisits this metaslab. 1274 */ 1275 if (msp->ms_disabled > 1) { 1276 mutex_exit(&msp->ms_lock); 1277 metaslab_enable(msp, B_FALSE, B_FALSE); 1278 continue; 1279 } 1280 1281 /* 1282 * Allocate an empty range tree which is swapped in 1283 * for the existing ms_trim tree while it is processed. 1284 */ 1285 trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 1286 0, 0); 1287 range_tree_swap(&msp->ms_trim, &trim_tree); 1288 ASSERT(range_tree_is_empty(msp->ms_trim)); 1289 1290 /* 1291 * There are two cases when constructing the per-vdev 1292 * trim trees for a metaslab. If the top-level vdev 1293 * has no children then it is also a leaf and should 1294 * be trimmed. Otherwise our children are the leaves 1295 * and a trim tree should be constructed for each. 1296 */ 1297 trim_args_t *tap; 1298 uint64_t children = vd->vdev_children; 1299 if (children == 0) { 1300 children = 1; 1301 tap = kmem_zalloc(sizeof (trim_args_t) * 1302 children, KM_SLEEP); 1303 tap[0].trim_vdev = vd; 1304 } else { 1305 tap = kmem_zalloc(sizeof (trim_args_t) * 1306 children, KM_SLEEP); 1307 1308 for (uint64_t c = 0; c < children; c++) { 1309 tap[c].trim_vdev = vd->vdev_child[c]; 1310 } 1311 } 1312 1313 for (uint64_t c = 0; c < children; c++) { 1314 trim_args_t *ta = &tap[c]; 1315 vdev_t *cvd = ta->trim_vdev; 1316 1317 ta->trim_msp = msp; 1318 ta->trim_extent_bytes_max = extent_bytes_max; 1319 ta->trim_extent_bytes_min = extent_bytes_min; 1320 ta->trim_type = TRIM_TYPE_AUTO; 1321 ta->trim_flags = 0; 1322 1323 if (cvd->vdev_detached || 1324 !vdev_writeable(cvd) || 1325 !cvd->vdev_has_trim || 1326 cvd->vdev_trim_thread != NULL) { 1327 continue; 1328 } 1329 1330 /* 1331 * When a device has an attached hot spare, or 1332 * is being replaced it will not be trimmed. 1333 * This is done to avoid adding additional 1334 * stress to a potentially unhealthy device, 1335 * and to minimize the required rebuild time. 1336 */ 1337 if (!cvd->vdev_ops->vdev_op_leaf) 1338 continue; 1339 1340 ta->trim_tree = range_tree_create(NULL, 1341 RANGE_SEG64, NULL, 0, 0); 1342 range_tree_walk(trim_tree, 1343 vdev_trim_range_add, ta); 1344 } 1345 1346 mutex_exit(&msp->ms_lock); 1347 spa_config_exit(spa, SCL_CONFIG, FTAG); 1348 1349 /* 1350 * Issue the TRIM I/Os for all ranges covered by the 1351 * TRIM trees. These ranges are safe to TRIM because 1352 * no new allocations will be performed until the call 1353 * to metaslab_enabled() below. 1354 */ 1355 for (uint64_t c = 0; c < children; c++) { 1356 trim_args_t *ta = &tap[c]; 1357 1358 /* 1359 * Always yield to a manual TRIM if one has 1360 * been started for the child vdev. 1361 */ 1362 if (ta->trim_tree == NULL || 1363 ta->trim_vdev->vdev_trim_thread != NULL) { 1364 continue; 1365 } 1366 1367 /* 1368 * After this point metaslab_enable() must be 1369 * called with the sync flag set. This is done 1370 * here because vdev_trim_ranges() is allowed 1371 * to be interrupted (EINTR) before issuing all 1372 * of the required TRIM I/Os. 1373 */ 1374 issued_trim = B_TRUE; 1375 1376 int error = vdev_trim_ranges(ta); 1377 if (error) 1378 break; 1379 } 1380 1381 /* 1382 * Verify every range which was trimmed is still 1383 * contained within the ms_allocatable tree. 1384 */ 1385 if (zfs_flags & ZFS_DEBUG_TRIM) { 1386 mutex_enter(&msp->ms_lock); 1387 VERIFY0(metaslab_load(msp)); 1388 VERIFY3P(tap[0].trim_msp, ==, msp); 1389 range_tree_walk(trim_tree, 1390 vdev_trim_range_verify, &tap[0]); 1391 mutex_exit(&msp->ms_lock); 1392 } 1393 1394 range_tree_vacate(trim_tree, NULL, NULL); 1395 range_tree_destroy(trim_tree); 1396 1397 /* 1398 * Wait for couples of kicks, to ensure the trim io is 1399 * synced. If the wait is aborted due to 1400 * vdev_autotrim_exit_wanted, we need to signal 1401 * metaslab_enable() to wait for sync. 1402 */ 1403 if (issued_trim) { 1404 wait_aborted = vdev_autotrim_wait_kick(vd, 1405 TXG_CONCURRENT_STATES + TXG_DEFER_SIZE); 1406 } 1407 1408 metaslab_enable(msp, wait_aborted, B_FALSE); 1409 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1410 1411 for (uint64_t c = 0; c < children; c++) { 1412 trim_args_t *ta = &tap[c]; 1413 1414 if (ta->trim_tree == NULL) 1415 continue; 1416 1417 range_tree_vacate(ta->trim_tree, NULL, NULL); 1418 range_tree_destroy(ta->trim_tree); 1419 } 1420 1421 kmem_free(tap, sizeof (trim_args_t) * children); 1422 1423 if (vdev_autotrim_should_stop(vd)) 1424 break; 1425 } 1426 1427 spa_config_exit(spa, SCL_CONFIG, FTAG); 1428 1429 vdev_autotrim_wait_kick(vd, 1); 1430 1431 shift++; 1432 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1433 } 1434 1435 for (uint64_t c = 0; c < vd->vdev_children; c++) { 1436 vdev_t *cvd = vd->vdev_child[c]; 1437 mutex_enter(&cvd->vdev_trim_io_lock); 1438 1439 while (cvd->vdev_trim_inflight[1] > 0) { 1440 cv_wait(&cvd->vdev_trim_io_cv, 1441 &cvd->vdev_trim_io_lock); 1442 } 1443 mutex_exit(&cvd->vdev_trim_io_lock); 1444 } 1445 1446 spa_config_exit(spa, SCL_CONFIG, FTAG); 1447 1448 /* 1449 * When exiting because the autotrim property was set to off, then 1450 * abandon any unprocessed ms_trim ranges to reclaim the memory. 1451 */ 1452 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) { 1453 for (uint64_t i = 0; i < vd->vdev_ms_count; i++) { 1454 metaslab_t *msp = vd->vdev_ms[i]; 1455 1456 mutex_enter(&msp->ms_lock); 1457 range_tree_vacate(msp->ms_trim, NULL, NULL); 1458 mutex_exit(&msp->ms_lock); 1459 } 1460 } 1461 1462 mutex_enter(&vd->vdev_autotrim_lock); 1463 ASSERT(vd->vdev_autotrim_thread != NULL); 1464 vd->vdev_autotrim_thread = NULL; 1465 cv_broadcast(&vd->vdev_autotrim_cv); 1466 mutex_exit(&vd->vdev_autotrim_lock); 1467 1468 thread_exit(); 1469 } 1470 1471 /* 1472 * Starts an autotrim thread, if needed, for each top-level vdev which can be 1473 * trimmed. A top-level vdev which has been evacuated will never be trimmed. 1474 */ 1475 void 1476 vdev_autotrim(spa_t *spa) 1477 { 1478 vdev_t *root_vd = spa->spa_root_vdev; 1479 1480 for (uint64_t i = 0; i < root_vd->vdev_children; i++) { 1481 vdev_t *tvd = root_vd->vdev_child[i]; 1482 1483 mutex_enter(&tvd->vdev_autotrim_lock); 1484 if (vdev_writeable(tvd) && !tvd->vdev_removing && 1485 tvd->vdev_autotrim_thread == NULL) { 1486 ASSERT3P(tvd->vdev_top, ==, tvd); 1487 1488 tvd->vdev_autotrim_thread = thread_create(NULL, 0, 1489 vdev_autotrim_thread, tvd, 0, &p0, TS_RUN, 1490 maxclsyspri); 1491 ASSERT(tvd->vdev_autotrim_thread != NULL); 1492 } 1493 mutex_exit(&tvd->vdev_autotrim_lock); 1494 } 1495 } 1496 1497 /* 1498 * Wait for the vdev_autotrim_thread associated with the passed top-level 1499 * vdev to be terminated (canceled or stopped). 1500 */ 1501 void 1502 vdev_autotrim_stop_wait(vdev_t *tvd) 1503 { 1504 mutex_enter(&tvd->vdev_autotrim_lock); 1505 if (tvd->vdev_autotrim_thread != NULL) { 1506 tvd->vdev_autotrim_exit_wanted = B_TRUE; 1507 cv_broadcast(&tvd->vdev_autotrim_kick_cv); 1508 cv_wait(&tvd->vdev_autotrim_cv, 1509 &tvd->vdev_autotrim_lock); 1510 1511 ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL); 1512 tvd->vdev_autotrim_exit_wanted = B_FALSE; 1513 } 1514 mutex_exit(&tvd->vdev_autotrim_lock); 1515 } 1516 1517 void 1518 vdev_autotrim_kick(spa_t *spa) 1519 { 1520 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 1521 1522 vdev_t *root_vd = spa->spa_root_vdev; 1523 vdev_t *tvd; 1524 1525 for (uint64_t i = 0; i < root_vd->vdev_children; i++) { 1526 tvd = root_vd->vdev_child[i]; 1527 1528 mutex_enter(&tvd->vdev_autotrim_lock); 1529 if (tvd->vdev_autotrim_thread != NULL) 1530 cv_broadcast(&tvd->vdev_autotrim_kick_cv); 1531 mutex_exit(&tvd->vdev_autotrim_lock); 1532 } 1533 } 1534 1535 /* 1536 * Wait for all of the vdev_autotrim_thread associated with the pool to 1537 * be terminated (canceled or stopped). 1538 */ 1539 void 1540 vdev_autotrim_stop_all(spa_t *spa) 1541 { 1542 vdev_t *root_vd = spa->spa_root_vdev; 1543 1544 for (uint64_t i = 0; i < root_vd->vdev_children; i++) 1545 vdev_autotrim_stop_wait(root_vd->vdev_child[i]); 1546 } 1547 1548 /* 1549 * Conditionally restart all of the vdev_autotrim_thread's for the pool. 1550 */ 1551 void 1552 vdev_autotrim_restart(spa_t *spa) 1553 { 1554 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1555 1556 if (spa->spa_autotrim) 1557 vdev_autotrim(spa); 1558 } 1559 1560 static __attribute__((noreturn)) void 1561 vdev_trim_l2arc_thread(void *arg) 1562 { 1563 vdev_t *vd = arg; 1564 spa_t *spa = vd->vdev_spa; 1565 l2arc_dev_t *dev = l2arc_vdev_get(vd); 1566 trim_args_t ta = {0}; 1567 range_seg64_t physical_rs; 1568 1569 ASSERT(vdev_is_concrete(vd)); 1570 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 1571 1572 vd->vdev_trim_last_offset = 0; 1573 vd->vdev_trim_rate = 0; 1574 vd->vdev_trim_partial = 0; 1575 vd->vdev_trim_secure = 0; 1576 1577 ta.trim_vdev = vd; 1578 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0); 1579 ta.trim_type = TRIM_TYPE_MANUAL; 1580 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max; 1581 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE; 1582 ta.trim_flags = 0; 1583 1584 physical_rs.rs_start = vd->vdev_trim_bytes_done = 0; 1585 physical_rs.rs_end = vd->vdev_trim_bytes_est = 1586 vdev_get_min_asize(vd); 1587 1588 range_tree_add(ta.trim_tree, physical_rs.rs_start, 1589 physical_rs.rs_end - physical_rs.rs_start); 1590 1591 mutex_enter(&vd->vdev_trim_lock); 1592 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0); 1593 mutex_exit(&vd->vdev_trim_lock); 1594 1595 (void) vdev_trim_ranges(&ta); 1596 1597 spa_config_exit(spa, SCL_CONFIG, FTAG); 1598 mutex_enter(&vd->vdev_trim_io_lock); 1599 while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) { 1600 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 1601 } 1602 mutex_exit(&vd->vdev_trim_io_lock); 1603 1604 range_tree_vacate(ta.trim_tree, NULL, NULL); 1605 range_tree_destroy(ta.trim_tree); 1606 1607 mutex_enter(&vd->vdev_trim_lock); 1608 if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) { 1609 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, 1610 vd->vdev_trim_rate, vd->vdev_trim_partial, 1611 vd->vdev_trim_secure); 1612 } 1613 ASSERT(vd->vdev_trim_thread != NULL || 1614 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0); 1615 1616 /* 1617 * Drop the vdev_trim_lock while we sync out the txg since it's 1618 * possible that a device might be trying to come online and 1619 * must check to see if it needs to restart a trim. That thread 1620 * will be holding the spa_config_lock which would prevent the 1621 * txg_wait_synced from completing. Same strategy as in 1622 * vdev_trim_thread(). 1623 */ 1624 mutex_exit(&vd->vdev_trim_lock); 1625 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 1626 mutex_enter(&vd->vdev_trim_lock); 1627 1628 /* 1629 * Update the header of the cache device here, before 1630 * broadcasting vdev_trim_cv which may lead to the removal 1631 * of the device. The same applies for setting l2ad_trim_all to 1632 * false. 1633 */ 1634 spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd, 1635 RW_READER); 1636 memset(dev->l2ad_dev_hdr, 0, dev->l2ad_dev_hdr_asize); 1637 l2arc_dev_hdr_update(dev); 1638 spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd); 1639 1640 vd->vdev_trim_thread = NULL; 1641 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE) 1642 dev->l2ad_trim_all = B_FALSE; 1643 1644 cv_broadcast(&vd->vdev_trim_cv); 1645 mutex_exit(&vd->vdev_trim_lock); 1646 1647 thread_exit(); 1648 } 1649 1650 /* 1651 * Punches out TRIM threads for the L2ARC devices in a spa and assigns them 1652 * to vd->vdev_trim_thread variable. This facilitates the management of 1653 * trimming the whole cache device using TRIM_TYPE_MANUAL upon addition 1654 * to a pool or pool creation or when the header of the device is invalid. 1655 */ 1656 void 1657 vdev_trim_l2arc(spa_t *spa) 1658 { 1659 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1660 1661 /* 1662 * Locate the spa's l2arc devices and kick off TRIM threads. 1663 */ 1664 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1665 vdev_t *vd = spa->spa_l2cache.sav_vdevs[i]; 1666 l2arc_dev_t *dev = l2arc_vdev_get(vd); 1667 1668 if (dev == NULL || !dev->l2ad_trim_all) { 1669 /* 1670 * Don't attempt TRIM if the vdev is UNAVAIL or if the 1671 * cache device was not marked for whole device TRIM 1672 * (ie l2arc_trim_ahead = 0, or the L2ARC device header 1673 * is valid with trim_state = VDEV_TRIM_COMPLETE and 1674 * l2ad_log_entries > 0). 1675 */ 1676 continue; 1677 } 1678 1679 mutex_enter(&vd->vdev_trim_lock); 1680 ASSERT(vd->vdev_ops->vdev_op_leaf); 1681 ASSERT(vdev_is_concrete(vd)); 1682 ASSERT3P(vd->vdev_trim_thread, ==, NULL); 1683 ASSERT(!vd->vdev_detached); 1684 ASSERT(!vd->vdev_trim_exit_wanted); 1685 ASSERT(!vd->vdev_top->vdev_removing); 1686 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0); 1687 vd->vdev_trim_thread = thread_create(NULL, 0, 1688 vdev_trim_l2arc_thread, vd, 0, &p0, TS_RUN, maxclsyspri); 1689 mutex_exit(&vd->vdev_trim_lock); 1690 } 1691 } 1692 1693 /* 1694 * A wrapper which calls vdev_trim_ranges(). It is intended to be called 1695 * on leaf vdevs. 1696 */ 1697 int 1698 vdev_trim_simple(vdev_t *vd, uint64_t start, uint64_t size) 1699 { 1700 trim_args_t ta = {0}; 1701 range_seg64_t physical_rs; 1702 int error; 1703 physical_rs.rs_start = start; 1704 physical_rs.rs_end = start + size; 1705 1706 ASSERT(vdev_is_concrete(vd)); 1707 ASSERT(vd->vdev_ops->vdev_op_leaf); 1708 ASSERT(!vd->vdev_detached); 1709 ASSERT(!vd->vdev_top->vdev_removing); 1710 1711 ta.trim_vdev = vd; 1712 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0); 1713 ta.trim_type = TRIM_TYPE_SIMPLE; 1714 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max; 1715 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE; 1716 ta.trim_flags = 0; 1717 1718 ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start); 1719 1720 if (physical_rs.rs_end > physical_rs.rs_start) { 1721 range_tree_add(ta.trim_tree, physical_rs.rs_start, 1722 physical_rs.rs_end - physical_rs.rs_start); 1723 } else { 1724 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start); 1725 } 1726 1727 error = vdev_trim_ranges(&ta); 1728 1729 mutex_enter(&vd->vdev_trim_io_lock); 1730 while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) { 1731 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); 1732 } 1733 mutex_exit(&vd->vdev_trim_io_lock); 1734 1735 range_tree_vacate(ta.trim_tree, NULL, NULL); 1736 range_tree_destroy(ta.trim_tree); 1737 1738 return (error); 1739 } 1740 1741 EXPORT_SYMBOL(vdev_trim); 1742 EXPORT_SYMBOL(vdev_trim_stop); 1743 EXPORT_SYMBOL(vdev_trim_stop_all); 1744 EXPORT_SYMBOL(vdev_trim_stop_wait); 1745 EXPORT_SYMBOL(vdev_trim_restart); 1746 EXPORT_SYMBOL(vdev_autotrim); 1747 EXPORT_SYMBOL(vdev_autotrim_stop_all); 1748 EXPORT_SYMBOL(vdev_autotrim_stop_wait); 1749 EXPORT_SYMBOL(vdev_autotrim_restart); 1750 EXPORT_SYMBOL(vdev_trim_l2arc); 1751 EXPORT_SYMBOL(vdev_trim_simple); 1752 1753 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_max, UINT, ZMOD_RW, 1754 "Max size of TRIM commands, larger will be split"); 1755 1756 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_min, UINT, ZMOD_RW, 1757 "Min size of TRIM commands, smaller will be skipped"); 1758 1759 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, metaslab_skip, UINT, ZMOD_RW, 1760 "Skip metaslabs which have never been initialized"); 1761 1762 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, txg_batch, UINT, ZMOD_RW, 1763 "Min number of txgs to aggregate frees before issuing TRIM"); 1764 1765 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, queue_limit, UINT, ZMOD_RW, 1766 "Max queued TRIMs outstanding per leaf vdev"); 1767