Lines Matching +full:vd +full:- +full:supply
9 * or https://opensource.org/licenses/CDDL-1.0.
44 * Ensuring the SSD always has a supply of erased blocks for new writes
67 * top-level (not leaf) vdev in the pool. These threads perform the same
73 * 2) Each thread is associated with a top-level (not leaf) vdev. This has
109 * useful for pools constructed from large thinly-provisioned devices where
126 * metaslab. This setting represents a trade-off between issuing more
169 vdev_trim_should_stop(vdev_t *vd) in vdev_trim_should_stop() argument
171 return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) || in vdev_trim_should_stop()
172 vd->vdev_detached || vd->vdev_top->vdev_removing || in vdev_trim_should_stop()
173 vd->vdev_top->vdev_rz_expanding); in vdev_trim_should_stop()
182 return (tvd->vdev_autotrim_exit_wanted || in vdev_autotrim_should_stop()
183 !vdev_writeable(tvd) || tvd->vdev_removing || in vdev_autotrim_should_stop()
184 tvd->vdev_rz_expanding || in vdev_autotrim_should_stop()
185 spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF); in vdev_autotrim_should_stop()
193 vdev_autotrim_wait_kick(vdev_t *vd, int num_of_kick) in vdev_autotrim_wait_kick() argument
195 mutex_enter(&vd->vdev_autotrim_lock); in vdev_autotrim_wait_kick()
197 if (vd->vdev_autotrim_exit_wanted) in vdev_autotrim_wait_kick()
199 cv_wait_idle(&vd->vdev_autotrim_kick_cv, in vdev_autotrim_wait_kick()
200 &vd->vdev_autotrim_lock); in vdev_autotrim_wait_kick()
202 boolean_t exit_wanted = vd->vdev_autotrim_exit_wanted; in vdev_autotrim_wait_kick()
203 mutex_exit(&vd->vdev_autotrim_lock); in vdev_autotrim_wait_kick()
209 * The sync task for updating the on-disk state of a manual TRIM. This
227 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE); in vdev_trim_zap_update_sync() local
228 if (vd == NULL || vd->vdev_top->vdev_removing || in vdev_trim_zap_update_sync()
229 !vdev_is_concrete(vd) || vd->vdev_top->vdev_rz_expanding) in vdev_trim_zap_update_sync()
232 uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK]; in vdev_trim_zap_update_sync()
233 vd->vdev_trim_offset[txg & TXG_MASK] = 0; in vdev_trim_zap_update_sync()
235 VERIFY3U(vd->vdev_leaf_zap, !=, 0); in vdev_trim_zap_update_sync()
237 objset_t *mos = vd->vdev_spa->spa_meta_objset; in vdev_trim_zap_update_sync()
239 if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) { in vdev_trim_zap_update_sync()
241 if (vd->vdev_trim_last_offset == UINT64_MAX) in vdev_trim_zap_update_sync()
244 vd->vdev_trim_last_offset = last_offset; in vdev_trim_zap_update_sync()
245 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, in vdev_trim_zap_update_sync()
250 if (vd->vdev_trim_action_time > 0) { in vdev_trim_zap_update_sync()
251 uint64_t val = (uint64_t)vd->vdev_trim_action_time; in vdev_trim_zap_update_sync()
252 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, in vdev_trim_zap_update_sync()
257 if (vd->vdev_trim_rate > 0) { in vdev_trim_zap_update_sync()
258 uint64_t rate = (uint64_t)vd->vdev_trim_rate; in vdev_trim_zap_update_sync()
263 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, in vdev_trim_zap_update_sync()
267 uint64_t partial = vd->vdev_trim_partial; in vdev_trim_zap_update_sync()
271 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL, in vdev_trim_zap_update_sync()
274 uint64_t secure = vd->vdev_trim_secure; in vdev_trim_zap_update_sync()
278 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE, in vdev_trim_zap_update_sync()
282 uint64_t trim_state = vd->vdev_trim_state; in vdev_trim_zap_update_sync()
283 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE, in vdev_trim_zap_update_sync()
288 * Update the on-disk state of a manual TRIM. This is called to request
293 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state, in vdev_trim_change_state() argument
296 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); in vdev_trim_change_state()
297 spa_t *spa = vd->vdev_spa; in vdev_trim_change_state()
299 if (new_state == vd->vdev_trim_state) in vdev_trim_change_state()
303 * Copy the vd's guid, this will be freed by the sync task. in vdev_trim_change_state()
306 *guid = vd->vdev_guid; in vdev_trim_change_state()
311 if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) { in vdev_trim_change_state()
312 vd->vdev_trim_action_time = gethrestime_sec(); in vdev_trim_change_state()
321 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE || in vdev_trim_change_state()
322 vd->vdev_trim_state == VDEV_TRIM_CANCELED) { in vdev_trim_change_state()
323 vd->vdev_trim_last_offset = UINT64_MAX; in vdev_trim_change_state()
324 vd->vdev_trim_rate = UINT64_MAX; in vdev_trim_change_state()
325 vd->vdev_trim_partial = UINT64_MAX; in vdev_trim_change_state()
326 vd->vdev_trim_secure = UINT64_MAX; in vdev_trim_change_state()
330 vd->vdev_trim_rate = rate; in vdev_trim_change_state()
333 vd->vdev_trim_partial = partial; in vdev_trim_change_state()
336 vd->vdev_trim_secure = secure; in vdev_trim_change_state()
339 vdev_trim_state_t old_state = vd->vdev_trim_state; in vdev_trim_change_state()
341 vd->vdev_trim_state = new_state; in vdev_trim_change_state()
343 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); in vdev_trim_change_state()
350 spa_event_notify(spa, vd, NULL, in vdev_trim_change_state()
353 "vdev=%s activated", vd->vdev_path); in vdev_trim_change_state()
356 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND); in vdev_trim_change_state()
358 "vdev=%s suspended", vd->vdev_path); in vdev_trim_change_state()
363 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL); in vdev_trim_change_state()
365 "vdev=%s canceled", vd->vdev_path); in vdev_trim_change_state()
369 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH); in vdev_trim_change_state()
371 "vdev=%s complete", vd->vdev_path); in vdev_trim_change_state()
391 vdev_t *vd = zio->io_vd; in vdev_trim_cb() local
393 mutex_enter(&vd->vdev_trim_io_lock); in vdev_trim_cb()
394 if (zio->io_error == ENXIO && !vdev_writeable(vd)) { in vdev_trim_cb()
401 &vd->vdev_trim_offset[zio->io_txg & TXG_MASK]; in vdev_trim_cb()
402 *offset = MIN(*offset, zio->io_offset); in vdev_trim_cb()
404 if (zio->io_error != 0) { in vdev_trim_cb()
405 vd->vdev_stat.vs_trim_errors++; in vdev_trim_cb()
406 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL, in vdev_trim_cb()
407 0, 0, 0, 0, 1, zio->io_orig_size); in vdev_trim_cb()
409 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL, in vdev_trim_cb()
410 1, zio->io_orig_size, 0, 0, 0, 0); in vdev_trim_cb()
413 vd->vdev_trim_bytes_done += zio->io_orig_size; in vdev_trim_cb()
416 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0); in vdev_trim_cb()
417 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--; in vdev_trim_cb()
418 cv_broadcast(&vd->vdev_trim_io_cv); in vdev_trim_cb()
419 mutex_exit(&vd->vdev_trim_io_lock); in vdev_trim_cb()
421 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); in vdev_trim_cb()
433 vdev_t *vd = zio->io_vd; in vdev_autotrim_cb() local
435 mutex_enter(&vd->vdev_trim_io_lock); in vdev_autotrim_cb()
437 if (zio->io_error != 0) { in vdev_autotrim_cb()
438 vd->vdev_stat.vs_trim_errors++; in vdev_autotrim_cb()
439 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO, in vdev_autotrim_cb()
440 0, 0, 0, 0, 1, zio->io_orig_size); in vdev_autotrim_cb()
442 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO, in vdev_autotrim_cb()
443 1, zio->io_orig_size, 0, 0, 0, 0); in vdev_autotrim_cb()
446 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0); in vdev_autotrim_cb()
447 vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--; in vdev_autotrim_cb()
448 cv_broadcast(&vd->vdev_trim_io_cv); in vdev_autotrim_cb()
449 mutex_exit(&vd->vdev_trim_io_lock); in vdev_autotrim_cb()
451 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); in vdev_autotrim_cb()
463 vdev_t *vd = zio->io_vd; in vdev_trim_simple_cb() local
465 mutex_enter(&vd->vdev_trim_io_lock); in vdev_trim_simple_cb()
467 if (zio->io_error != 0) { in vdev_trim_simple_cb()
468 vd->vdev_stat.vs_trim_errors++; in vdev_trim_simple_cb()
469 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE, in vdev_trim_simple_cb()
470 0, 0, 0, 0, 1, zio->io_orig_size); in vdev_trim_simple_cb()
472 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE, in vdev_trim_simple_cb()
473 1, zio->io_orig_size, 0, 0, 0, 0); in vdev_trim_simple_cb()
476 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0); in vdev_trim_simple_cb()
477 vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--; in vdev_trim_simple_cb()
478 cv_broadcast(&vd->vdev_trim_io_cv); in vdev_trim_simple_cb()
479 mutex_exit(&vd->vdev_trim_io_lock); in vdev_trim_simple_cb()
481 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); in vdev_trim_simple_cb()
484 * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
489 return (ta->trim_bytes_done * 1000 / in vdev_trim_calculate_rate()
490 (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1)); in vdev_trim_calculate_rate()
500 vdev_t *vd = ta->trim_vdev; in vdev_trim_range() local
501 spa_t *spa = vd->vdev_spa; in vdev_trim_range()
504 mutex_enter(&vd->vdev_trim_io_lock); in vdev_trim_range()
510 if (ta->trim_type == TRIM_TYPE_MANUAL) { in vdev_trim_range()
511 while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) && in vdev_trim_range()
512 vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) { in vdev_trim_range()
513 cv_timedwait_idle(&vd->vdev_trim_io_cv, in vdev_trim_range()
514 &vd->vdev_trim_io_lock, ddi_get_lbolt() + in vdev_trim_range()
518 ta->trim_bytes_done += size; in vdev_trim_range()
521 while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] + in vdev_trim_range()
522 vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) { in vdev_trim_range()
523 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); in vdev_trim_range()
525 vd->vdev_trim_inflight[ta->trim_type]++; in vdev_trim_range()
526 mutex_exit(&vd->vdev_trim_io_lock); in vdev_trim_range()
528 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); in vdev_trim_range()
532 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER); in vdev_trim_range()
533 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_range()
535 if (ta->trim_type == TRIM_TYPE_MANUAL && in vdev_trim_range()
536 vd->vdev_trim_offset[txg & TXG_MASK] == 0) { in vdev_trim_range()
538 *guid = vd->vdev_guid; in vdev_trim_range()
549 if ((ta->trim_type == TRIM_TYPE_MANUAL && in vdev_trim_range()
550 vdev_trim_should_stop(vd)) || in vdev_trim_range()
551 (ta->trim_type == TRIM_TYPE_AUTO && in vdev_trim_range()
552 vdev_autotrim_should_stop(vd->vdev_top))) { in vdev_trim_range()
553 mutex_enter(&vd->vdev_trim_io_lock); in vdev_trim_range()
554 vd->vdev_trim_inflight[ta->trim_type]--; in vdev_trim_range()
555 mutex_exit(&vd->vdev_trim_io_lock); in vdev_trim_range()
556 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); in vdev_trim_range()
557 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_range()
561 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_range()
563 if (ta->trim_type == TRIM_TYPE_MANUAL) in vdev_trim_range()
564 vd->vdev_trim_offset[txg & TXG_MASK] = start + size; in vdev_trim_range()
566 if (ta->trim_type == TRIM_TYPE_MANUAL) { in vdev_trim_range()
568 } else if (ta->trim_type == TRIM_TYPE_AUTO) { in vdev_trim_range()
574 zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd, in vdev_trim_range()
576 ta->trim_flags)); in vdev_trim_range()
585 * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
593 vdev_t *vd = ta->trim_vdev; in vdev_trim_ranges() local
594 zfs_btree_t *t = &ta->trim_tree->rt_root; in vdev_trim_ranges()
596 uint64_t extent_bytes_max = ta->trim_extent_bytes_max; in vdev_trim_ranges()
597 uint64_t extent_bytes_min = ta->trim_extent_bytes_min; in vdev_trim_ranges()
598 spa_t *spa = vd->vdev_spa; in vdev_trim_ranges()
601 ta->trim_start_time = gethrtime(); in vdev_trim_ranges()
602 ta->trim_bytes_done = 0; in vdev_trim_ranges()
606 uint64_t size = rs_get_end(rs, ta->trim_tree) - rs_get_start(rs, in vdev_trim_ranges()
607 ta->trim_tree); in vdev_trim_ranges()
610 spa_iostats_trim_add(spa, ta->trim_type, in vdev_trim_ranges()
615 /* Split range into legally-sized physical chunks */ in vdev_trim_ranges()
616 uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1; in vdev_trim_ranges()
620 rs_get_start(rs, ta->trim_tree) + in vdev_trim_ranges()
621 (w *extent_bytes_max), MIN(size - in vdev_trim_ranges()
634 * metaslab is re-enabled. Otherwise it's possible write zios to in vdev_trim_ranges()
638 mutex_enter(&vd->vdev_trim_io_lock); in vdev_trim_ranges()
639 while (vd->vdev_trim_inflight[0] > 0) { in vdev_trim_ranges()
640 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); in vdev_trim_ranges()
642 mutex_exit(&vd->vdev_trim_io_lock); in vdev_trim_ranges()
652 if (physical_rs->rs_end > *last_rs_end) in vdev_trim_xlate_last_rs_end()
653 *last_rs_end = physical_rs->rs_end; in vdev_trim_xlate_last_rs_end()
659 vdev_t *vd = (vdev_t *)arg; in vdev_trim_xlate_progress() local
661 uint64_t size = physical_rs->rs_end - physical_rs->rs_start; in vdev_trim_xlate_progress()
662 vd->vdev_trim_bytes_est += size; in vdev_trim_xlate_progress()
664 if (vd->vdev_trim_last_offset >= physical_rs->rs_end) { in vdev_trim_xlate_progress()
665 vd->vdev_trim_bytes_done += size; in vdev_trim_xlate_progress()
666 } else if (vd->vdev_trim_last_offset > physical_rs->rs_start && in vdev_trim_xlate_progress()
667 vd->vdev_trim_last_offset <= physical_rs->rs_end) { in vdev_trim_xlate_progress()
668 vd->vdev_trim_bytes_done += in vdev_trim_xlate_progress()
669 vd->vdev_trim_last_offset - physical_rs->rs_start; in vdev_trim_xlate_progress()
677 vdev_trim_calculate_progress(vdev_t *vd) in vdev_trim_calculate_progress() argument
679 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || in vdev_trim_calculate_progress()
680 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); in vdev_trim_calculate_progress()
681 ASSERT(vd->vdev_leaf_zap != 0); in vdev_trim_calculate_progress()
683 vd->vdev_trim_bytes_est = 0; in vdev_trim_calculate_progress()
684 vd->vdev_trim_bytes_done = 0; in vdev_trim_calculate_progress()
686 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) { in vdev_trim_calculate_progress()
687 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; in vdev_trim_calculate_progress()
688 mutex_enter(&msp->ms_lock); in vdev_trim_calculate_progress()
690 uint64_t ms_free = (msp->ms_size - in vdev_trim_calculate_progress()
692 vdev_get_ndisks(vd->vdev_top); in vdev_trim_calculate_progress()
700 logical_rs.rs_start = msp->ms_start; in vdev_trim_calculate_progress()
701 logical_rs.rs_end = msp->ms_start + msp->ms_size; in vdev_trim_calculate_progress()
704 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs); in vdev_trim_calculate_progress()
705 if (vd->vdev_trim_last_offset <= physical_rs.rs_start) { in vdev_trim_calculate_progress()
706 vd->vdev_trim_bytes_est += ms_free; in vdev_trim_calculate_progress()
707 mutex_exit(&msp->ms_lock); in vdev_trim_calculate_progress()
714 vdev_xlate_walk(vd, &remain_rs, in vdev_trim_calculate_progress()
718 if (vd->vdev_trim_last_offset > last_rs_end) { in vdev_trim_calculate_progress()
719 vd->vdev_trim_bytes_done += ms_free; in vdev_trim_calculate_progress()
720 vd->vdev_trim_bytes_est += ms_free; in vdev_trim_calculate_progress()
721 mutex_exit(&msp->ms_lock); in vdev_trim_calculate_progress()
732 range_tree_t *rt = msp->ms_allocatable; in vdev_trim_calculate_progress()
733 zfs_btree_t *bt = &rt->rt_root; in vdev_trim_calculate_progress()
740 vdev_xlate_walk(vd, &logical_rs, in vdev_trim_calculate_progress()
741 vdev_trim_xlate_progress, vd); in vdev_trim_calculate_progress()
743 mutex_exit(&msp->ms_lock); in vdev_trim_calculate_progress()
752 vdev_trim_load(vdev_t *vd) in vdev_trim_load() argument
755 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || in vdev_trim_load()
756 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); in vdev_trim_load()
757 ASSERT(vd->vdev_leaf_zap != 0); in vdev_trim_load()
759 if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE || in vdev_trim_load()
760 vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) { in vdev_trim_load()
761 err = zap_lookup(vd->vdev_spa->spa_meta_objset, in vdev_trim_load()
762 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET, in vdev_trim_load()
763 sizeof (vd->vdev_trim_last_offset), 1, in vdev_trim_load()
764 &vd->vdev_trim_last_offset); in vdev_trim_load()
766 vd->vdev_trim_last_offset = 0; in vdev_trim_load()
771 err = zap_lookup(vd->vdev_spa->spa_meta_objset, in vdev_trim_load()
772 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE, in vdev_trim_load()
773 sizeof (vd->vdev_trim_rate), 1, in vdev_trim_load()
774 &vd->vdev_trim_rate); in vdev_trim_load()
776 vd->vdev_trim_rate = 0; in vdev_trim_load()
782 err = zap_lookup(vd->vdev_spa->spa_meta_objset, in vdev_trim_load()
783 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL, in vdev_trim_load()
784 sizeof (vd->vdev_trim_partial), 1, in vdev_trim_load()
785 &vd->vdev_trim_partial); in vdev_trim_load()
787 vd->vdev_trim_partial = 0; in vdev_trim_load()
793 err = zap_lookup(vd->vdev_spa->spa_meta_objset, in vdev_trim_load()
794 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE, in vdev_trim_load()
795 sizeof (vd->vdev_trim_secure), 1, in vdev_trim_load()
796 &vd->vdev_trim_secure); in vdev_trim_load()
798 vd->vdev_trim_secure = 0; in vdev_trim_load()
804 vdev_trim_calculate_progress(vd); in vdev_trim_load()
813 vdev_t *vd = ta->trim_vdev; in vdev_trim_xlate_range_add() local
819 if (ta->trim_type == TRIM_TYPE_MANUAL) { in vdev_trim_xlate_range_add()
822 if (physical_rs->rs_end <= vd->vdev_trim_last_offset) in vdev_trim_xlate_range_add()
825 /* Pick up where we left off mid-range. */ in vdev_trim_xlate_range_add()
826 if (vd->vdev_trim_last_offset > physical_rs->rs_start) { in vdev_trim_xlate_range_add()
827 ASSERT3U(physical_rs->rs_end, >, in vdev_trim_xlate_range_add()
828 vd->vdev_trim_last_offset); in vdev_trim_xlate_range_add()
829 physical_rs->rs_start = vd->vdev_trim_last_offset; in vdev_trim_xlate_range_add()
833 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start); in vdev_trim_xlate_range_add()
835 range_tree_add(ta->trim_tree, physical_rs->rs_start, in vdev_trim_xlate_range_add()
836 physical_rs->rs_end - physical_rs->rs_start); in vdev_trim_xlate_range_add()
847 vdev_t *vd = ta->trim_vdev; in vdev_trim_range_add() local
858 metaslab_t *msp = ta->trim_msp; in vdev_trim_range_add()
860 VERIFY3B(msp->ms_loaded, ==, B_TRUE); in vdev_trim_range_add()
861 VERIFY(range_tree_contains(msp->ms_allocatable, start, size)); in vdev_trim_range_add()
864 ASSERT(vd->vdev_ops->vdev_op_leaf); in vdev_trim_range_add()
865 vdev_xlate_walk(vd, &logical_rs, vdev_trim_xlate_range_add, arg); in vdev_trim_range_add()
871 * over its top-level metaslabs and issuing TRIM I/O for the space described
878 vdev_t *vd = arg; in vdev_trim_thread() local
879 spa_t *spa = vd->vdev_spa; in vdev_trim_thread()
888 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); in vdev_trim_thread()
890 ASSERT(vdev_is_concrete(vd)); in vdev_trim_thread()
893 vd->vdev_trim_last_offset = 0; in vdev_trim_thread()
894 vd->vdev_trim_rate = 0; in vdev_trim_thread()
895 vd->vdev_trim_partial = 0; in vdev_trim_thread()
896 vd->vdev_trim_secure = 0; in vdev_trim_thread()
898 VERIFY0(vdev_trim_load(vd)); in vdev_trim_thread()
900 ta.trim_vdev = vd; in vdev_trim_thread()
912 if (vd->vdev_trim_secure) { in vdev_trim_thread()
918 for (uint64_t i = 0; !vd->vdev_detached && in vdev_trim_thread()
919 i < vd->vdev_top->vdev_ms_count; i++) { in vdev_trim_thread()
920 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; in vdev_trim_thread()
923 * If we've expanded the top-level vdev or it's our in vdev_trim_thread()
926 if (vd->vdev_top->vdev_ms_count != ms_count) { in vdev_trim_thread()
927 vdev_trim_calculate_progress(vd); in vdev_trim_thread()
928 ms_count = vd->vdev_top->vdev_ms_count; in vdev_trim_thread()
933 mutex_enter(&msp->ms_lock); in vdev_trim_thread()
940 if (msp->ms_sm == NULL && vd->vdev_trim_partial) { in vdev_trim_thread()
941 mutex_exit(&msp->ms_lock); in vdev_trim_thread()
944 vdev_trim_calculate_progress(vd); in vdev_trim_thread()
949 range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta); in vdev_trim_thread()
950 range_tree_vacate(msp->ms_trim, NULL, NULL); in vdev_trim_thread()
951 mutex_exit(&msp->ms_lock); in vdev_trim_thread()
966 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_thread()
967 if (!vd->vdev_trim_exit_wanted) { in vdev_trim_thread()
968 if (vdev_writeable(vd)) { in vdev_trim_thread()
969 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, in vdev_trim_thread()
970 vd->vdev_trim_rate, vd->vdev_trim_partial, in vdev_trim_thread()
971 vd->vdev_trim_secure); in vdev_trim_thread()
972 } else if (vd->vdev_faulted) { in vdev_trim_thread()
973 vdev_trim_change_state(vd, VDEV_TRIM_CANCELED, in vdev_trim_thread()
974 vd->vdev_trim_rate, vd->vdev_trim_partial, in vdev_trim_thread()
975 vd->vdev_trim_secure); in vdev_trim_thread()
978 ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0); in vdev_trim_thread()
987 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_thread()
989 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_thread()
991 vd->vdev_trim_thread = NULL; in vdev_trim_thread()
992 cv_broadcast(&vd->vdev_trim_cv); in vdev_trim_thread()
993 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_thread()
1003 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure) in vdev_trim() argument
1005 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); in vdev_trim()
1006 ASSERT(vd->vdev_ops->vdev_op_leaf); in vdev_trim()
1007 ASSERT(vdev_is_concrete(vd)); in vdev_trim()
1008 ASSERT3P(vd->vdev_trim_thread, ==, NULL); in vdev_trim()
1009 ASSERT(!vd->vdev_detached); in vdev_trim()
1010 ASSERT(!vd->vdev_trim_exit_wanted); in vdev_trim()
1011 ASSERT(!vd->vdev_top->vdev_removing); in vdev_trim()
1012 ASSERT(!vd->vdev_rz_expanding); in vdev_trim()
1014 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure); in vdev_trim()
1015 vd->vdev_trim_thread = thread_create(NULL, 0, in vdev_trim()
1016 vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri); in vdev_trim()
1023 vdev_trim_stop_wait_impl(vdev_t *vd) in vdev_trim_stop_wait_impl() argument
1025 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); in vdev_trim_stop_wait_impl()
1027 while (vd->vdev_trim_thread != NULL) in vdev_trim_stop_wait_impl()
1028 cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock); in vdev_trim_stop_wait_impl()
1030 ASSERT3P(vd->vdev_trim_thread, ==, NULL); in vdev_trim_stop_wait_impl()
1031 vd->vdev_trim_exit_wanted = B_FALSE; in vdev_trim_stop_wait_impl()
1041 vdev_t *vd; in vdev_trim_stop_wait() local
1044 spa->spa_export_thread == curthread); in vdev_trim_stop_wait()
1046 while ((vd = list_remove_head(vd_list)) != NULL) { in vdev_trim_stop_wait()
1047 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_stop_wait()
1048 vdev_trim_stop_wait_impl(vd); in vdev_trim_stop_wait()
1049 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_stop_wait()
1063 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list) in vdev_trim_stop() argument
1065 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER)); in vdev_trim_stop()
1066 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock)); in vdev_trim_stop()
1067 ASSERT(vd->vdev_ops->vdev_op_leaf); in vdev_trim_stop()
1068 ASSERT(vdev_is_concrete(vd)); in vdev_trim_stop()
1074 if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED) in vdev_trim_stop()
1077 vdev_trim_change_state(vd, tgt_state, 0, 0, 0); in vdev_trim_stop()
1078 vd->vdev_trim_exit_wanted = B_TRUE; in vdev_trim_stop()
1081 vdev_trim_stop_wait_impl(vd); in vdev_trim_stop()
1084 vd->vdev_spa->spa_export_thread == curthread); in vdev_trim_stop()
1085 list_insert_tail(vd_list, vd); in vdev_trim_stop()
1093 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state, in vdev_trim_stop_all_impl() argument
1096 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) { in vdev_trim_stop_all_impl()
1097 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_stop_all_impl()
1098 vdev_trim_stop(vd, tgt_state, vd_list); in vdev_trim_stop_all_impl()
1099 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_stop_all_impl()
1103 for (uint64_t i = 0; i < vd->vdev_children; i++) { in vdev_trim_stop_all_impl()
1104 vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state, in vdev_trim_stop_all_impl()
1114 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state) in vdev_trim_stop_all() argument
1116 spa_t *spa = vd->vdev_spa; in vdev_trim_stop_all()
1121 spa->spa_export_thread == curthread); in vdev_trim_stop_all()
1126 vdev_trim_stop_all_impl(vd, tgt_state, &vd_list); in vdev_trim_stop_all()
1133 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { in vdev_trim_stop_all()
1134 vd_l2cache = spa->spa_l2cache.sav_vdevs[i]; in vdev_trim_stop_all()
1140 if (vd->vdev_spa->spa_sync_on) { in vdev_trim_stop_all()
1142 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); in vdev_trim_stop_all()
1149 * Conditionally restarts a manual TRIM given its on-disk state.
1152 vdev_trim_restart(vdev_t *vd) in vdev_trim_restart() argument
1155 vd->vdev_spa->spa_load_thread == curthread); in vdev_trim_restart()
1156 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER)); in vdev_trim_restart()
1158 if (vd->vdev_leaf_zap != 0) { in vdev_trim_restart()
1159 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_restart()
1161 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, in vdev_trim_restart()
1162 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE, in vdev_trim_restart()
1165 vd->vdev_trim_state = trim_state; in vdev_trim_restart()
1168 err = zap_lookup(vd->vdev_spa->spa_meta_objset, in vdev_trim_restart()
1169 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME, in vdev_trim_restart()
1172 vd->vdev_trim_action_time = timestamp; in vdev_trim_restart()
1174 if ((vd->vdev_trim_state == VDEV_TRIM_SUSPENDED || in vdev_trim_restart()
1175 vd->vdev_offline) && !vd->vdev_top->vdev_rz_expanding) { in vdev_trim_restart()
1177 VERIFY0(vdev_trim_load(vd)); in vdev_trim_restart()
1178 } else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE && in vdev_trim_restart()
1179 vdev_writeable(vd) && !vd->vdev_top->vdev_removing && in vdev_trim_restart()
1180 !vd->vdev_top->vdev_rz_expanding && in vdev_trim_restart()
1181 vd->vdev_trim_thread == NULL) { in vdev_trim_restart()
1182 VERIFY0(vdev_trim_load(vd)); in vdev_trim_restart()
1183 vdev_trim(vd, vd->vdev_trim_rate, in vdev_trim_restart()
1184 vd->vdev_trim_partial, vd->vdev_trim_secure); in vdev_trim_restart()
1187 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_restart()
1190 for (uint64_t i = 0; i < vd->vdev_children; i++) { in vdev_trim_restart()
1191 vdev_trim_restart(vd->vdev_child[i]); in vdev_trim_restart()
1203 metaslab_t *msp = ta->trim_msp; in vdev_trim_range_verify()
1205 VERIFY3B(msp->ms_loaded, ==, B_TRUE); in vdev_trim_range_verify()
1206 VERIFY3U(msp->ms_disabled, >, 0); in vdev_trim_range_verify()
1207 VERIFY(range_tree_contains(msp->ms_allocatable, start, size)); in vdev_trim_range_verify()
1212 * top-level vdev in the pool. No automatic TRIM state is maintained on-disk.
1215 * is created for each leaf vdev, instead of each top-level vdev.
1220 vdev_t *vd = arg; in vdev_autotrim_thread() local
1221 spa_t *spa = vd->vdev_spa; in vdev_autotrim_thread()
1224 mutex_enter(&vd->vdev_autotrim_lock); in vdev_autotrim_thread()
1225 ASSERT3P(vd->vdev_top, ==, vd); in vdev_autotrim_thread()
1226 ASSERT3P(vd->vdev_autotrim_thread, !=, NULL); in vdev_autotrim_thread()
1227 mutex_exit(&vd->vdev_autotrim_lock); in vdev_autotrim_thread()
1230 while (!vdev_autotrim_should_stop(vd)) { in vdev_autotrim_thread()
1257 * 2) Selecting non-consecutive metaslabs distributes the in vdev_autotrim_thread()
1261 for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count; in vdev_autotrim_thread()
1263 metaslab_t *msp = vd->vdev_ms[i]; in vdev_autotrim_thread()
1272 mutex_enter(&msp->ms_lock); in vdev_autotrim_thread()
1278 if (msp->ms_sm == NULL || in vdev_autotrim_thread()
1279 range_tree_is_empty(msp->ms_trim)) { in vdev_autotrim_thread()
1280 mutex_exit(&msp->ms_lock); in vdev_autotrim_thread()
1295 if (msp->ms_disabled > 1) { in vdev_autotrim_thread()
1296 mutex_exit(&msp->ms_lock); in vdev_autotrim_thread()
1307 range_tree_swap(&msp->ms_trim, &trim_tree); in vdev_autotrim_thread()
1308 ASSERT(range_tree_is_empty(msp->ms_trim)); in vdev_autotrim_thread()
1311 * There are two cases when constructing the per-vdev in vdev_autotrim_thread()
1312 * trim trees for a metaslab. If the top-level vdev in vdev_autotrim_thread()
1318 uint64_t children = vd->vdev_children; in vdev_autotrim_thread()
1323 tap[0].trim_vdev = vd; in vdev_autotrim_thread()
1329 tap[c].trim_vdev = vd->vdev_child[c]; in vdev_autotrim_thread()
1335 vdev_t *cvd = ta->trim_vdev; in vdev_autotrim_thread()
1337 ta->trim_msp = msp; in vdev_autotrim_thread()
1338 ta->trim_extent_bytes_max = extent_bytes_max; in vdev_autotrim_thread()
1339 ta->trim_extent_bytes_min = extent_bytes_min; in vdev_autotrim_thread()
1340 ta->trim_type = TRIM_TYPE_AUTO; in vdev_autotrim_thread()
1341 ta->trim_flags = 0; in vdev_autotrim_thread()
1343 if (cvd->vdev_detached || in vdev_autotrim_thread()
1345 !cvd->vdev_has_trim || in vdev_autotrim_thread()
1346 cvd->vdev_trim_thread != NULL) { in vdev_autotrim_thread()
1357 if (!cvd->vdev_ops->vdev_op_leaf) in vdev_autotrim_thread()
1360 ta->trim_tree = range_tree_create(NULL, in vdev_autotrim_thread()
1366 mutex_exit(&msp->ms_lock); in vdev_autotrim_thread()
1382 if (ta->trim_tree == NULL || in vdev_autotrim_thread()
1383 ta->trim_vdev->vdev_trim_thread != NULL) { in vdev_autotrim_thread()
1406 mutex_enter(&msp->ms_lock); in vdev_autotrim_thread()
1411 mutex_exit(&msp->ms_lock); in vdev_autotrim_thread()
1424 wait_aborted = vdev_autotrim_wait_kick(vd, in vdev_autotrim_thread()
1434 if (ta->trim_tree == NULL) in vdev_autotrim_thread()
1437 range_tree_vacate(ta->trim_tree, NULL, NULL); in vdev_autotrim_thread()
1438 range_tree_destroy(ta->trim_tree); in vdev_autotrim_thread()
1443 if (vdev_autotrim_should_stop(vd)) in vdev_autotrim_thread()
1449 vdev_autotrim_wait_kick(vd, 1); in vdev_autotrim_thread()
1455 for (uint64_t c = 0; c < vd->vdev_children; c++) { in vdev_autotrim_thread()
1456 vdev_t *cvd = vd->vdev_child[c]; in vdev_autotrim_thread()
1457 mutex_enter(&cvd->vdev_trim_io_lock); in vdev_autotrim_thread()
1459 while (cvd->vdev_trim_inflight[1] > 0) { in vdev_autotrim_thread()
1460 cv_wait(&cvd->vdev_trim_io_cv, in vdev_autotrim_thread()
1461 &cvd->vdev_trim_io_lock); in vdev_autotrim_thread()
1463 mutex_exit(&cvd->vdev_trim_io_lock); in vdev_autotrim_thread()
1473 for (uint64_t i = 0; i < vd->vdev_ms_count; i++) { in vdev_autotrim_thread()
1474 metaslab_t *msp = vd->vdev_ms[i]; in vdev_autotrim_thread()
1476 mutex_enter(&msp->ms_lock); in vdev_autotrim_thread()
1477 range_tree_vacate(msp->ms_trim, NULL, NULL); in vdev_autotrim_thread()
1478 mutex_exit(&msp->ms_lock); in vdev_autotrim_thread()
1482 mutex_enter(&vd->vdev_autotrim_lock); in vdev_autotrim_thread()
1483 ASSERT(vd->vdev_autotrim_thread != NULL); in vdev_autotrim_thread()
1484 vd->vdev_autotrim_thread = NULL; in vdev_autotrim_thread()
1485 cv_broadcast(&vd->vdev_autotrim_cv); in vdev_autotrim_thread()
1486 mutex_exit(&vd->vdev_autotrim_lock); in vdev_autotrim_thread()
1492 * Starts an autotrim thread, if needed, for each top-level vdev which can be
1493 * trimmed. A top-level vdev which has been evacuated will never be trimmed.
1498 vdev_t *root_vd = spa->spa_root_vdev; in vdev_autotrim()
1500 for (uint64_t i = 0; i < root_vd->vdev_children; i++) { in vdev_autotrim()
1501 vdev_t *tvd = root_vd->vdev_child[i]; in vdev_autotrim()
1503 mutex_enter(&tvd->vdev_autotrim_lock); in vdev_autotrim()
1504 if (vdev_writeable(tvd) && !tvd->vdev_removing && in vdev_autotrim()
1505 tvd->vdev_autotrim_thread == NULL && in vdev_autotrim()
1506 !tvd->vdev_rz_expanding) { in vdev_autotrim()
1507 ASSERT3P(tvd->vdev_top, ==, tvd); in vdev_autotrim()
1509 tvd->vdev_autotrim_thread = thread_create(NULL, 0, in vdev_autotrim()
1512 ASSERT(tvd->vdev_autotrim_thread != NULL); in vdev_autotrim()
1514 mutex_exit(&tvd->vdev_autotrim_lock); in vdev_autotrim()
1519 * Wait for the vdev_autotrim_thread associated with the passed top-level
1525 mutex_enter(&tvd->vdev_autotrim_lock); in vdev_autotrim_stop_wait()
1526 if (tvd->vdev_autotrim_thread != NULL) { in vdev_autotrim_stop_wait()
1527 tvd->vdev_autotrim_exit_wanted = B_TRUE; in vdev_autotrim_stop_wait()
1528 cv_broadcast(&tvd->vdev_autotrim_kick_cv); in vdev_autotrim_stop_wait()
1529 cv_wait(&tvd->vdev_autotrim_cv, in vdev_autotrim_stop_wait()
1530 &tvd->vdev_autotrim_lock); in vdev_autotrim_stop_wait()
1532 ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL); in vdev_autotrim_stop_wait()
1533 tvd->vdev_autotrim_exit_wanted = B_FALSE; in vdev_autotrim_stop_wait()
1535 mutex_exit(&tvd->vdev_autotrim_lock); in vdev_autotrim_stop_wait()
1543 vdev_t *root_vd = spa->spa_root_vdev; in vdev_autotrim_kick()
1546 for (uint64_t i = 0; i < root_vd->vdev_children; i++) { in vdev_autotrim_kick()
1547 tvd = root_vd->vdev_child[i]; in vdev_autotrim_kick()
1549 mutex_enter(&tvd->vdev_autotrim_lock); in vdev_autotrim_kick()
1550 if (tvd->vdev_autotrim_thread != NULL) in vdev_autotrim_kick()
1551 cv_broadcast(&tvd->vdev_autotrim_kick_cv); in vdev_autotrim_kick()
1552 mutex_exit(&tvd->vdev_autotrim_lock); in vdev_autotrim_kick()
1563 vdev_t *root_vd = spa->spa_root_vdev; in vdev_autotrim_stop_all()
1565 for (uint64_t i = 0; i < root_vd->vdev_children; i++) in vdev_autotrim_stop_all()
1566 vdev_autotrim_stop_wait(root_vd->vdev_child[i]); in vdev_autotrim_stop_all()
1576 spa->spa_load_thread == curthread); in vdev_autotrim_restart()
1577 if (spa->spa_autotrim) in vdev_autotrim_restart()
1584 vdev_t *vd = arg; in vdev_trim_l2arc_thread() local
1585 spa_t *spa = vd->vdev_spa; in vdev_trim_l2arc_thread()
1586 l2arc_dev_t *dev = l2arc_vdev_get(vd); in vdev_trim_l2arc_thread()
1590 ASSERT(vdev_is_concrete(vd)); in vdev_trim_l2arc_thread()
1593 vd->vdev_trim_last_offset = 0; in vdev_trim_l2arc_thread()
1594 vd->vdev_trim_rate = 0; in vdev_trim_l2arc_thread()
1595 vd->vdev_trim_partial = 0; in vdev_trim_l2arc_thread()
1596 vd->vdev_trim_secure = 0; in vdev_trim_l2arc_thread()
1598 ta.trim_vdev = vd; in vdev_trim_l2arc_thread()
1605 physical_rs.rs_start = vd->vdev_trim_bytes_done = 0; in vdev_trim_l2arc_thread()
1606 physical_rs.rs_end = vd->vdev_trim_bytes_est = in vdev_trim_l2arc_thread()
1607 vdev_get_min_asize(vd); in vdev_trim_l2arc_thread()
1610 physical_rs.rs_end - physical_rs.rs_start); in vdev_trim_l2arc_thread()
1612 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_l2arc_thread()
1613 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0); in vdev_trim_l2arc_thread()
1614 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_l2arc_thread()
1619 mutex_enter(&vd->vdev_trim_io_lock); in vdev_trim_l2arc_thread()
1620 while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) { in vdev_trim_l2arc_thread()
1621 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); in vdev_trim_l2arc_thread()
1623 mutex_exit(&vd->vdev_trim_io_lock); in vdev_trim_l2arc_thread()
1628 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_l2arc_thread()
1629 if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) { in vdev_trim_l2arc_thread()
1630 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE, in vdev_trim_l2arc_thread()
1631 vd->vdev_trim_rate, vd->vdev_trim_partial, in vdev_trim_l2arc_thread()
1632 vd->vdev_trim_secure); in vdev_trim_l2arc_thread()
1634 ASSERT(vd->vdev_trim_thread != NULL || in vdev_trim_l2arc_thread()
1635 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0); in vdev_trim_l2arc_thread()
1645 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_l2arc_thread()
1646 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); in vdev_trim_l2arc_thread()
1647 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_l2arc_thread()
1655 spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd, in vdev_trim_l2arc_thread()
1657 memset(dev->l2ad_dev_hdr, 0, dev->l2ad_dev_hdr_asize); in vdev_trim_l2arc_thread()
1659 spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd); in vdev_trim_l2arc_thread()
1661 vd->vdev_trim_thread = NULL; in vdev_trim_l2arc_thread()
1662 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE) in vdev_trim_l2arc_thread()
1663 dev->l2ad_trim_all = B_FALSE; in vdev_trim_l2arc_thread()
1665 cv_broadcast(&vd->vdev_trim_cv); in vdev_trim_l2arc_thread()
1666 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_l2arc_thread()
1673 * to vd->vdev_trim_thread variable. This facilitates the management of
1685 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { in vdev_trim_l2arc()
1686 vdev_t *vd = spa->spa_l2cache.sav_vdevs[i]; in vdev_trim_l2arc() local
1687 l2arc_dev_t *dev = l2arc_vdev_get(vd); in vdev_trim_l2arc()
1689 if (dev == NULL || !dev->l2ad_trim_all) { in vdev_trim_l2arc()
1700 mutex_enter(&vd->vdev_trim_lock); in vdev_trim_l2arc()
1701 ASSERT(vd->vdev_ops->vdev_op_leaf); in vdev_trim_l2arc()
1702 ASSERT(vdev_is_concrete(vd)); in vdev_trim_l2arc()
1703 ASSERT3P(vd->vdev_trim_thread, ==, NULL); in vdev_trim_l2arc()
1704 ASSERT(!vd->vdev_detached); in vdev_trim_l2arc()
1705 ASSERT(!vd->vdev_trim_exit_wanted); in vdev_trim_l2arc()
1706 ASSERT(!vd->vdev_top->vdev_removing); in vdev_trim_l2arc()
1707 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0); in vdev_trim_l2arc()
1708 vd->vdev_trim_thread = thread_create(NULL, 0, in vdev_trim_l2arc()
1709 vdev_trim_l2arc_thread, vd, 0, &p0, TS_RUN, maxclsyspri); in vdev_trim_l2arc()
1710 mutex_exit(&vd->vdev_trim_lock); in vdev_trim_l2arc()
1719 vdev_trim_simple(vdev_t *vd, uint64_t start, uint64_t size) in vdev_trim_simple() argument
1727 ASSERT(vdev_is_concrete(vd)); in vdev_trim_simple()
1728 ASSERT(vd->vdev_ops->vdev_op_leaf); in vdev_trim_simple()
1729 ASSERT(!vd->vdev_detached); in vdev_trim_simple()
1730 ASSERT(!vd->vdev_top->vdev_removing); in vdev_trim_simple()
1731 ASSERT(!vd->vdev_top->vdev_rz_expanding); in vdev_trim_simple()
1733 ta.trim_vdev = vd; in vdev_trim_simple()
1744 physical_rs.rs_end - physical_rs.rs_start); in vdev_trim_simple()
1751 mutex_enter(&vd->vdev_trim_io_lock); in vdev_trim_simple()
1752 while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) { in vdev_trim_simple()
1753 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); in vdev_trim_simple()
1755 mutex_exit(&vd->vdev_trim_io_lock); in vdev_trim_simple()