Lines Matching +full:llp +full:- +full:based
1 // SPDX-License-Identifier: CDDL-1.0
10 * or https://opensource.org/licenses/CDDL-1.0.
44 * This file contains all the routines used when modifying on-disk SPA state.
115 * moves it to some Illumos-specific place, or removes it entirely.
121 * prevent CPU-intensive kernel threads from affecting latency on interactive
168 * are so high frequency and short-lived that the taskq itself can become a
170 * additional degree of parallelism specified by the number of threads per-
223 * This is used by zdb to analyze non-idle pools.
245 * A non-zero value for zfs_max_missing_tvds means that we allow importing
246 * pools with missing top-level vdevs. This is strictly intended for advanced
248 * missing devices can only be imported read-only for safety reasons, and their
249 * fail-mode will be automatically set to "continue".
270 * might be incomplete or out-dated.
273 * an out-dated cachefile where a device removal was not registered.
276 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available
279 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1;
295 * points for the livelist to be removed - used to test condense/destroy races
319 * a 256-slot ring buffer for minute-level resolution. The number is limited by
323 * high-resolution data.
340 * up-to-date version of the database.
442 vdev_t *rvd = spa->spa_root_vdev; in spa_prop_get_config()
443 dsl_pool_t *pool = spa->spa_dsl_pool; in spa_prop_get_config()
449 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); in spa_prop_get_config()
470 size - alloc, src); in spa_prop_get_config()
472 spa->spa_checkpoint_info.sci_dspace, src); in spa_prop_get_config()
496 rvd->vdev_state, src); in spa_prop_get_config()
517 if (pool->dp_free_dir != NULL) { in spa_prop_get_config()
519 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, in spa_prop_get_config()
526 if (pool->dp_leak_dir != NULL) { in spa_prop_get_config()
528 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, in spa_prop_get_config()
538 if (spa->spa_comment != NULL) { in spa_prop_get_config()
539 spa_prop_add_list(nv, ZPOOL_PROP_COMMENT, spa->spa_comment, in spa_prop_get_config()
543 if (spa->spa_compatibility != NULL) { in spa_prop_get_config()
545 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); in spa_prop_get_config()
548 if (spa->spa_root != NULL) in spa_prop_get_config()
549 spa_prop_add_list(nv, ZPOOL_PROP_ALTROOT, spa->spa_root, in spa_prop_get_config()
568 if ((dp = list_head(&spa->spa_config_list)) != NULL) { in spa_prop_get_config()
569 if (dp->scd_path == NULL) { in spa_prop_get_config()
572 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { in spa_prop_get_config()
574 dp->scd_path, 0, ZPROP_SRC_LOCAL); in spa_prop_get_config()
585 objset_t *mos = spa->spa_meta_objset; in spa_prop_get()
594 mutex_enter(&spa->spa_props_lock); in spa_prop_get()
602 if (mos == NULL || spa->spa_pool_props_object == 0) in spa_prop_get()
608 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); in spa_prop_get()
616 if ((prop = zpool_name_to_prop(za->za_name)) == in spa_prop_get()
617 ZPOOL_PROP_INVAL && !zfs_prop_user(za->za_name)) in spa_prop_get()
620 switch (za->za_integer_length) { in spa_prop_get()
623 if (za->za_first_integer != in spa_prop_get()
631 za->za_first_integer, FTAG, &ds); in spa_prop_get()
641 intval = za->za_first_integer; in spa_prop_get()
653 strval = kmem_alloc(za->za_num_integers, KM_SLEEP); in spa_prop_get()
654 err = zap_lookup(mos, spa->spa_pool_props_object, in spa_prop_get()
655 za->za_name, 1, za->za_num_integers, strval); in spa_prop_get()
657 kmem_free(strval, za->za_num_integers); in spa_prop_get()
664 spa_prop_add_user(nv, za->za_name, strval, in spa_prop_get()
667 kmem_free(strval, za->za_num_integers); in spa_prop_get()
676 mutex_exit(&spa->spa_props_lock); in spa_prop_get()
781 spa->spa_hostid = hostid; in spa_prop_validate()
802 if (!vdev_is_bootable(spa->spa_root_vdev)) { in spa_prop_validate()
842 * the user to change the in-core failmode property in spa_prop_validate()
850 spa->spa_failmode = intval; in spa_prop_validate()
929 dp->scd_path = spa_strdup(spa_config_path); in spa_configfile_set()
931 dp->scd_path = NULL; in spa_configfile_set()
933 dp->scd_path = spa_strdup(cachefile); in spa_configfile_set()
935 list_insert_head(&spa->spa_config_list, dp); in spa_configfile_set()
985 error = dsl_sync_task(spa->spa_name, NULL, in spa_prop_set()
998 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, in spa_prop_set()
1011 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { in spa_prop_clear_bootfs()
1012 VERIFY(zap_remove(spa->spa_meta_objset, in spa_prop_clear_bootfs()
1013 spa->spa_pool_props_object, in spa_prop_clear_bootfs()
1015 spa->spa_bootfs = 0; in spa_prop_clear_bootfs()
1023 spa_t *spa = dmu_tx_pool(tx)->dp_spa; in spa_change_guid_check()
1024 vdev_t *rvd = spa->spa_root_vdev; in spa_change_guid_check()
1034 vdev_state = rvd->vdev_state; in spa_change_guid_check()
1049 spa_t *spa = dmu_tx_pool(tx)->dp_spa; in spa_change_guid_sync()
1051 vdev_t *rvd = spa->spa_root_vdev; in spa_change_guid_sync()
1056 rvd->vdev_guid = *newguid; in spa_change_guid_sync()
1057 rvd->vdev_guid_sum += (*newguid - oldguid); in spa_change_guid_sync()
1067 * re-import a pool built from a clone of our own vdevs. We will modify
1084 mutex_enter(&spa->spa_vdev_top_lock); in spa_change_guid()
1102 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, in spa_change_guid()
1111 vdev_clear_kobj_evt(spa->spa_root_vdev); in spa_change_guid()
1112 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) in spa_change_guid()
1113 vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]); in spa_change_guid()
1121 mutex_exit(&spa->spa_vdev_top_lock); in spa_change_guid()
1139 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, in spa_error_entry_compare()
1147 * re-initializes them in the process.
1152 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); in spa_get_errlists()
1154 memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t)); in spa_get_errlists()
1155 memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t)); in spa_get_errlists()
1157 avl_create(&spa->spa_errlist_scrub, in spa_get_errlists()
1160 avl_create(&spa->spa_errlist_last, in spa_get_errlists()
1169 enum zti_modes mode = ztip->zti_mode; in spa_taskqs_init()
1170 uint_t value = ztip->zti_value; in spa_taskqs_init()
1171 uint_t count = ztip->zti_count; in spa_taskqs_init()
1172 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; in spa_taskqs_init()
1189 count = MIN(count, spa->spa_alloc_count); in spa_taskqs_init()
1190 while (spa->spa_alloc_count % count != 0 && in spa_taskqs_init()
1191 spa->spa_alloc_count < count * 2) in spa_taskqs_init()
1192 count--; in spa_taskqs_init()
1220 * ------- ------- ------- ------- ------- in spa_taskqs_init()
1234 count--; in spa_taskqs_init()
1251 tqs->stqs_count = 0; in spa_taskqs_init()
1252 tqs->stqs_taskq = NULL; in spa_taskqs_init()
1263 tqs->stqs_count = count; in spa_taskqs_init()
1264 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); in spa_taskqs_init()
1278 if (zio_taskq_sysdc && spa->spa_proc != &p0) { in spa_taskqs_init()
1281 spa->spa_proc, zio_taskq_basedc, flags); in spa_taskqs_init()
1293 INT_MAX, spa->spa_proc, flags); in spa_taskqs_init()
1298 tqs->stqs_taskq[i] = tq; in spa_taskqs_init()
1305 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; in spa_taskqs_fini()
1307 if (tqs->stqs_taskq == NULL) { in spa_taskqs_fini()
1308 ASSERT0(tqs->stqs_count); in spa_taskqs_fini()
1312 for (uint_t i = 0; i < tqs->stqs_count; i++) { in spa_taskqs_fini()
1313 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); in spa_taskqs_fini()
1314 taskq_destroy(tqs->stqs_taskq[i]); in spa_taskqs_fini()
1317 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); in spa_taskqs_fini()
1318 tqs->stqs_taskq = NULL; in spa_taskqs_fini()
1335 * 'null' can only be set on the high-priority queues (queue selection for
1336 * high-priority queues will fall back to the regular queue if the high-pri
1390 /* Per-mode setup */ in spa_taskq_param_set()
1482 * Can only null the high-priority queues; the general- in spa_taskq_param_set()
1532 if (zti->zti_mode == ZTI_MODE_FIXED) in spa_taskq_param_get()
1534 modes[zti->zti_mode], zti->zti_count, in spa_taskq_param_get()
1535 zti->zti_value); in spa_taskq_param_get()
1536 else if (zti->zti_mode == ZTI_MODE_SCALE && zti->zti_value > 0) in spa_taskq_param_get()
1538 modes[zti->zti_mode], zti->zti_value); in spa_taskq_param_get()
1541 modes[zti->zti_mode]); in spa_taskq_param_get()
1559 return (-err); in spa_taskq_read_param_set()
1574 return (-err); in spa_taskq_write_param_set()
1589 return (-err); in spa_taskq_free_param_set()
1599 * On FreeBSD load-time parameters can be set up before malloc() is available,
1612 if (err || req->newptr == NULL) in spa_taskq_read_param()
1625 if (err || req->newptr == NULL) in spa_taskq_write_param()
1638 if (err || req->newptr == NULL) in spa_taskq_free_param()
1654 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; in spa_taskq_dispatch()
1657 ASSERT3P(tqs->stqs_taskq, !=, NULL); in spa_taskq_dispatch()
1658 ASSERT3U(tqs->stqs_count, !=, 0); in spa_taskq_dispatch()
1666 ASSERT(taskq_empty_ent(&zio->io_tqent)); in spa_taskq_dispatch()
1668 if (tqs->stqs_count == 1) { in spa_taskq_dispatch()
1669 tq = tqs->stqs_taskq[0]; in spa_taskq_dispatch()
1672 tq = tqs->stqs_taskq[zio->io_allocator % tqs->stqs_count]; in spa_taskq_dispatch()
1674 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; in spa_taskq_dispatch()
1678 &zio->io_tqent); in spa_taskq_dispatch()
1701 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, in spa_thread()
1702 spa->spa_name); in spa_thread()
1705 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), in spa_thread()
1706 "zpool-%s", spa->spa_name); in spa_thread()
1707 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); in spa_thread()
1714 mutex_enter(&curproc->p_lock); in spa_thread()
1718 curthread->t_bind_pset = zio_taskq_psrset_bind; in spa_thread()
1722 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); in spa_thread()
1725 mutex_exit(&curproc->p_lock); in spa_thread()
1737 spa->spa_proc = curproc; in spa_thread()
1738 spa->spa_did = curthread->t_did; in spa_thread()
1742 mutex_enter(&spa->spa_proc_lock); in spa_thread()
1743 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); in spa_thread()
1745 spa->spa_proc_state = SPA_PROC_ACTIVE; in spa_thread()
1746 cv_broadcast(&spa->spa_proc_cv); in spa_thread()
1749 while (spa->spa_proc_state == SPA_PROC_ACTIVE) in spa_thread()
1750 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); in spa_thread()
1751 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); in spa_thread()
1753 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); in spa_thread()
1754 spa->spa_proc_state = SPA_PROC_GONE; in spa_thread()
1755 spa->spa_proc = &p0; in spa_thread()
1756 cv_broadcast(&spa->spa_proc_cv); in spa_thread()
1759 mutex_enter(&curproc->p_lock); in spa_thread()
1773 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); in spa_activate()
1775 spa->spa_state = POOL_STATE_ACTIVE; in spa_activate()
1776 spa->spa_final_txg = UINT64_MAX; in spa_activate()
1777 spa->spa_mode = mode; in spa_activate()
1778 spa->spa_read_spacemaps = spa_mode_readable_spacemaps; in spa_activate()
1780 spa->spa_normal_class = metaslab_class_create(spa, "normal", in spa_activate()
1782 spa->spa_log_class = metaslab_class_create(spa, "log", msp, B_TRUE); in spa_activate()
1783 spa->spa_embedded_log_class = metaslab_class_create(spa, in spa_activate()
1785 spa->spa_special_class = metaslab_class_create(spa, "special", in spa_activate()
1787 spa->spa_special_embedded_log_class = metaslab_class_create(spa, in spa_activate()
1789 spa->spa_dedup_class = metaslab_class_create(spa, "dedup", in spa_activate()
1793 mutex_enter(&spa->spa_proc_lock); in spa_activate()
1794 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); in spa_activate()
1795 ASSERT(spa->spa_proc == &p0); in spa_activate()
1796 spa->spa_did = 0; in spa_activate()
1800 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { in spa_activate()
1803 spa->spa_proc_state = SPA_PROC_CREATED; in spa_activate()
1804 while (spa->spa_proc_state == SPA_PROC_CREATED) { in spa_activate()
1805 cv_wait(&spa->spa_proc_cv, in spa_activate()
1806 &spa->spa_proc_lock); in spa_activate()
1808 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); in spa_activate()
1809 ASSERT(spa->spa_proc != &p0); in spa_activate()
1810 ASSERT(spa->spa_did != 0); in spa_activate()
1815 spa->spa_name); in spa_activate()
1820 mutex_exit(&spa->spa_proc_lock); in spa_activate()
1823 if (spa->spa_proc == &p0) { in spa_activate()
1828 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, in spa_activate()
1832 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), in spa_activate()
1834 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), in spa_activate()
1836 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), in spa_activate()
1839 txg_list_create(&spa->spa_vdev_txg_list, spa, in spa_activate()
1842 avl_create(&spa->spa_errlist_scrub, in spa_activate()
1845 avl_create(&spa->spa_errlist_last, in spa_activate()
1848 avl_create(&spa->spa_errlist_healed, in spa_activate()
1854 spa_keystore_init(&spa->spa_keystore); in spa_activate()
1857 * This taskq is used to perform zvol-minor-related tasks in spa_activate()
1871 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, in spa_activate()
1877 spa->spa_metaslab_taskq = taskq_create("z_metaslab", in spa_activate()
1886 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, in spa_activate()
1893 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, in spa_activate()
1903 ASSERT(spa->spa_sync_on == B_FALSE); in spa_deactivate()
1904 ASSERT0P(spa->spa_dsl_pool); in spa_deactivate()
1905 ASSERT0P(spa->spa_root_vdev); in spa_deactivate()
1906 ASSERT0P(spa->spa_async_zio_root); in spa_deactivate()
1907 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); in spa_deactivate()
1911 if (spa->spa_zvol_taskq) { in spa_deactivate()
1912 taskq_destroy(spa->spa_zvol_taskq); in spa_deactivate()
1913 spa->spa_zvol_taskq = NULL; in spa_deactivate()
1916 if (spa->spa_metaslab_taskq) { in spa_deactivate()
1917 taskq_destroy(spa->spa_metaslab_taskq); in spa_deactivate()
1918 spa->spa_metaslab_taskq = NULL; in spa_deactivate()
1921 if (spa->spa_prefetch_taskq) { in spa_deactivate()
1922 taskq_destroy(spa->spa_prefetch_taskq); in spa_deactivate()
1923 spa->spa_prefetch_taskq = NULL; in spa_deactivate()
1926 if (spa->spa_upgrade_taskq) { in spa_deactivate()
1927 taskq_destroy(spa->spa_upgrade_taskq); in spa_deactivate()
1928 spa->spa_upgrade_taskq = NULL; in spa_deactivate()
1931 txg_list_destroy(&spa->spa_vdev_txg_list); in spa_deactivate()
1933 list_destroy(&spa->spa_config_dirty_list); in spa_deactivate()
1934 list_destroy(&spa->spa_evicting_os_list); in spa_deactivate()
1935 list_destroy(&spa->spa_state_dirty_list); in spa_deactivate()
1937 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); in spa_deactivate()
1946 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); in spa_deactivate()
1947 VERIFY0(zio_wait(spa->spa_txg_zio[i])); in spa_deactivate()
1948 spa->spa_txg_zio[i] = NULL; in spa_deactivate()
1951 metaslab_class_destroy(spa->spa_normal_class); in spa_deactivate()
1952 spa->spa_normal_class = NULL; in spa_deactivate()
1954 metaslab_class_destroy(spa->spa_log_class); in spa_deactivate()
1955 spa->spa_log_class = NULL; in spa_deactivate()
1957 metaslab_class_destroy(spa->spa_embedded_log_class); in spa_deactivate()
1958 spa->spa_embedded_log_class = NULL; in spa_deactivate()
1960 metaslab_class_destroy(spa->spa_special_class); in spa_deactivate()
1961 spa->spa_special_class = NULL; in spa_deactivate()
1963 metaslab_class_destroy(spa->spa_special_embedded_log_class); in spa_deactivate()
1964 spa->spa_special_embedded_log_class = NULL; in spa_deactivate()
1966 metaslab_class_destroy(spa->spa_dedup_class); in spa_deactivate()
1967 spa->spa_dedup_class = NULL; in spa_deactivate()
1974 avl_destroy(&spa->spa_errlist_scrub); in spa_deactivate()
1975 avl_destroy(&spa->spa_errlist_last); in spa_deactivate()
1976 avl_destroy(&spa->spa_errlist_healed); in spa_deactivate()
1978 spa_keystore_fini(&spa->spa_keystore); in spa_deactivate()
1980 spa->spa_state = POOL_STATE_UNINITIALIZED; in spa_deactivate()
1982 mutex_enter(&spa->spa_proc_lock); in spa_deactivate()
1983 if (spa->spa_proc_state != SPA_PROC_NONE) { in spa_deactivate()
1984 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); in spa_deactivate()
1985 spa->spa_proc_state = SPA_PROC_DEACTIVATE; in spa_deactivate()
1986 cv_broadcast(&spa->spa_proc_cv); in spa_deactivate()
1987 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { in spa_deactivate()
1988 ASSERT(spa->spa_proc != &p0); in spa_deactivate()
1989 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); in spa_deactivate()
1991 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); in spa_deactivate()
1992 spa->spa_proc_state = SPA_PROC_NONE; in spa_deactivate()
1994 ASSERT(spa->spa_proc == &p0); in spa_deactivate()
1995 mutex_exit(&spa->spa_proc_lock); in spa_deactivate()
2002 if (spa->spa_did != 0) { in spa_deactivate()
2003 thread_join(spa->spa_did); in spa_deactivate()
2004 spa->spa_did = 0; in spa_deactivate()
2028 if ((*vdp)->vdev_ops->vdev_op_leaf) in spa_config_parse()
2067 if (!spa->spa_sync_on) in spa_should_flush_logs_on_unload()
2086 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); in spa_unload_log_sm_flush_all()
2089 ASSERT0(spa->spa_log_flushall_txg); in spa_unload_log_sm_flush_all()
2090 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); in spa_unload_log_sm_flush_all()
2093 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); in spa_unload_log_sm_flush_all()
2103 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, in spa_unload_log_sm_metadata()
2105 VERIFY0(sls->sls_mscount); in spa_unload_log_sm_metadata()
2109 while ((e = list_remove_head(&spa->spa_log_summary)) != NULL) { in spa_unload_log_sm_metadata()
2110 VERIFY0(e->lse_mscount); in spa_unload_log_sm_metadata()
2114 spa->spa_unflushed_stats.sus_nblocks = 0; in spa_unload_log_sm_metadata()
2115 spa->spa_unflushed_stats.sus_memused = 0; in spa_unload_log_sm_metadata()
2116 spa->spa_unflushed_stats.sus_blocklimit = 0; in spa_unload_log_sm_metadata()
2122 if (spa->spa_condense_zthr != NULL) { in spa_destroy_aux_threads()
2123 zthr_destroy(spa->spa_condense_zthr); in spa_destroy_aux_threads()
2124 spa->spa_condense_zthr = NULL; in spa_destroy_aux_threads()
2126 if (spa->spa_checkpoint_discard_zthr != NULL) { in spa_destroy_aux_threads()
2127 zthr_destroy(spa->spa_checkpoint_discard_zthr); in spa_destroy_aux_threads()
2128 spa->spa_checkpoint_discard_zthr = NULL; in spa_destroy_aux_threads()
2130 if (spa->spa_livelist_delete_zthr != NULL) { in spa_destroy_aux_threads()
2131 zthr_destroy(spa->spa_livelist_delete_zthr); in spa_destroy_aux_threads()
2132 spa->spa_livelist_delete_zthr = NULL; in spa_destroy_aux_threads()
2134 if (spa->spa_livelist_condense_zthr != NULL) { in spa_destroy_aux_threads()
2135 zthr_destroy(spa->spa_livelist_condense_zthr); in spa_destroy_aux_threads()
2136 spa->spa_livelist_condense_zthr = NULL; in spa_destroy_aux_threads()
2138 if (spa->spa_raidz_expand_zthr != NULL) { in spa_destroy_aux_threads()
2139 zthr_destroy(spa->spa_raidz_expand_zthr); in spa_destroy_aux_threads()
2140 spa->spa_raidz_expand_zthr = NULL; in spa_destroy_aux_threads()
2154 if (curtime < spa->spa_last_noted_txg_time + spa_note_txg_time) { in spa_sync_time_logger()
2158 if (txg > spa->spa_last_noted_txg) { in spa_sync_time_logger()
2159 spa->spa_last_noted_txg_time = curtime; in spa_sync_time_logger()
2160 spa->spa_last_noted_txg = txg; in spa_sync_time_logger()
2162 mutex_enter(&spa->spa_txg_log_time_lock); in spa_sync_time_logger()
2163 dbrrd_add(&spa->spa_txg_log_time, curtime, txg); in spa_sync_time_logger()
2164 mutex_exit(&spa->spa_txg_log_time_lock); in spa_sync_time_logger()
2167 if (curtime < spa->spa_last_flush_txg_time + spa_flush_txg_time) { in spa_sync_time_logger()
2170 spa->spa_last_flush_txg_time = curtime; in spa_sync_time_logger()
2176 &spa->spa_txg_log_time.dbr_minutes, tx)); in spa_sync_time_logger()
2179 &spa->spa_txg_log_time.dbr_days, tx)); in spa_sync_time_logger()
2182 &spa->spa_txg_log_time.dbr_months, tx)); in spa_sync_time_logger()
2190 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); in spa_unload_sync_time_logger()
2194 spa->spa_last_noted_txg_time = 0; in spa_unload_sync_time_logger()
2195 spa->spa_last_flush_txg_time = 0; in spa_unload_sync_time_logger()
2206 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, in spa_load_txg_log_time()
2208 &spa->spa_txg_log_time.dbr_minutes); in spa_load_txg_log_time()
2213 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, in spa_load_txg_log_time()
2215 &spa->spa_txg_log_time.dbr_days); in spa_load_txg_log_time()
2220 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, in spa_load_txg_log_time()
2222 &spa->spa_txg_log_time.dbr_months); in spa_load_txg_log_time()
2236 if (!spa->spa_sync_on) in spa_should_sync_time_logger_on_unload()
2242 if (spa->spa_last_noted_txg == 0) in spa_should_sync_time_logger_on_unload()
2256 spa->spa_export_thread == curthread); in spa_unload()
2269 if (spa->spa_final_txg == UINT64_MAX) { in spa_unload()
2287 if (spa->spa_root_vdev) { in spa_unload()
2288 vdev_t *root_vdev = spa->spa_root_vdev; in spa_unload()
2298 spa->spa_final_txg = spa_last_synced_txg(spa) + in spa_unload()
2306 if (spa->spa_sync_on) { in spa_unload()
2307 txg_sync_stop(spa->spa_dsl_pool); in spa_unload()
2308 spa->spa_sync_on = B_FALSE; in spa_unload()
2315 taskq_wait(spa->spa_metaslab_taskq); in spa_unload()
2317 if (spa->spa_mmp.mmp_thread) in spa_unload()
2323 if (spa->spa_async_zio_root != NULL) { in spa_unload()
2325 (void) zio_wait(spa->spa_async_zio_root[i]); in spa_unload()
2326 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); in spa_unload()
2327 spa->spa_async_zio_root = NULL; in spa_unload()
2330 if (spa->spa_vdev_removal != NULL) { in spa_unload()
2331 spa_vdev_removal_destroy(spa->spa_vdev_removal); in spa_unload()
2332 spa->spa_vdev_removal = NULL; in spa_unload()
2339 bpobj_close(&spa->spa_deferred_bpobj); in spa_unload()
2346 if (spa->spa_root_vdev) in spa_unload()
2347 vdev_free(spa->spa_root_vdev); in spa_unload()
2348 ASSERT0P(spa->spa_root_vdev); in spa_unload()
2353 if (spa->spa_dsl_pool) { in spa_unload()
2354 dsl_pool_close(spa->spa_dsl_pool); in spa_unload()
2355 spa->spa_dsl_pool = NULL; in spa_unload()
2356 spa->spa_meta_objset = NULL; in spa_unload()
2368 if (spa->spa_spares.sav_vdevs) { in spa_unload()
2369 for (int i = 0; i < spa->spa_spares.sav_count; i++) in spa_unload()
2370 vdev_free(spa->spa_spares.sav_vdevs[i]); in spa_unload()
2371 kmem_free(spa->spa_spares.sav_vdevs, in spa_unload()
2372 spa->spa_spares.sav_count * sizeof (void *)); in spa_unload()
2373 spa->spa_spares.sav_vdevs = NULL; in spa_unload()
2375 if (spa->spa_spares.sav_config) { in spa_unload()
2376 nvlist_free(spa->spa_spares.sav_config); in spa_unload()
2377 spa->spa_spares.sav_config = NULL; in spa_unload()
2379 spa->spa_spares.sav_count = 0; in spa_unload()
2381 if (spa->spa_l2cache.sav_vdevs) { in spa_unload()
2382 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { in spa_unload()
2383 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); in spa_unload()
2384 vdev_free(spa->spa_l2cache.sav_vdevs[i]); in spa_unload()
2386 kmem_free(spa->spa_l2cache.sav_vdevs, in spa_unload()
2387 spa->spa_l2cache.sav_count * sizeof (void *)); in spa_unload()
2388 spa->spa_l2cache.sav_vdevs = NULL; in spa_unload()
2390 if (spa->spa_l2cache.sav_config) { in spa_unload()
2391 nvlist_free(spa->spa_l2cache.sav_config); in spa_unload()
2392 spa->spa_l2cache.sav_config = NULL; in spa_unload()
2394 spa->spa_l2cache.sav_count = 0; in spa_unload()
2396 spa->spa_async_suspended = 0; in spa_unload()
2398 spa->spa_indirect_vdevs_loaded = B_FALSE; in spa_unload()
2400 if (spa->spa_comment != NULL) { in spa_unload()
2401 spa_strfree(spa->spa_comment); in spa_unload()
2402 spa->spa_comment = NULL; in spa_unload()
2404 if (spa->spa_compatibility != NULL) { in spa_unload()
2405 spa_strfree(spa->spa_compatibility); in spa_unload()
2406 spa->spa_compatibility = NULL; in spa_unload()
2409 spa->spa_raidz_expand = NULL; in spa_unload()
2410 spa->spa_checkpoint_txg = 0; in spa_unload()
2416 * Load (or re-load) the current list of vdevs describing the active spares for
2419 * then re-generate a more complete list including status information.
2446 if (spa->spa_spares.sav_vdevs) { in spa_load_spares()
2447 for (i = 0; i < spa->spa_spares.sav_count; i++) { in spa_load_spares()
2448 vd = spa->spa_spares.sav_vdevs[i]; in spa_load_spares()
2451 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, in spa_load_spares()
2452 B_FALSE)) != NULL && tvd->vdev_isspare) in spa_load_spares()
2458 kmem_free(spa->spa_spares.sav_vdevs, in spa_load_spares()
2459 spa->spa_spares.sav_count * sizeof (void *)); in spa_load_spares()
2462 if (spa->spa_spares.sav_config == NULL) in spa_load_spares()
2465 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, in spa_load_spares()
2468 spa->spa_spares.sav_count = (int)nspares; in spa_load_spares()
2469 spa->spa_spares.sav_vdevs = NULL; in spa_load_spares()
2483 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), in spa_load_spares()
2485 for (i = 0; i < spa->spa_spares.sav_count; i++) { in spa_load_spares()
2490 spa->spa_spares.sav_vdevs[i] = vd; in spa_load_spares()
2492 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, in spa_load_spares()
2494 if (!tvd->vdev_isspare) in spa_load_spares()
2514 vd->vdev_top = vd; in spa_load_spares()
2515 vd->vdev_aux = &spa->spa_spares; in spa_load_spares()
2528 fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES); in spa_load_spares()
2530 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), in spa_load_spares()
2532 for (i = 0; i < spa->spa_spares.sav_count; i++) in spa_load_spares()
2534 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); in spa_load_spares()
2535 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, in spa_load_spares()
2537 spa->spa_spares.sav_count); in spa_load_spares()
2538 for (i = 0; i < spa->spa_spares.sav_count; i++) in spa_load_spares()
2540 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); in spa_load_spares()
2544 * Load (or re-load) the current list of vdevs describing the active l2cache for
2547 * then re-generate a more complete list including status information.
2549 * not re-opened.
2559 spa_aux_vdev_t *sav = &spa->spa_l2cache; in spa_load_l2cache()
2576 oldvdevs = sav->sav_vdevs; in spa_load_l2cache()
2577 oldnvdevs = sav->sav_count; in spa_load_l2cache()
2578 sav->sav_vdevs = NULL; in spa_load_l2cache()
2579 sav->sav_count = 0; in spa_load_l2cache()
2581 if (sav->sav_config == NULL) { in spa_load_l2cache()
2587 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, in spa_load_l2cache()
2600 if (vd != NULL && guid == vd->vdev_guid) { in spa_load_l2cache()
2625 vd->vdev_top = vd; in spa_load_l2cache()
2626 vd->vdev_aux = sav; in spa_load_l2cache()
2649 sav->sav_vdevs = newvdevs; in spa_load_l2cache()
2650 sav->sav_count = (int)nl2cache; in spa_load_l2cache()
2656 fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE); in spa_load_l2cache()
2658 if (sav->sav_count > 0) in spa_load_l2cache()
2659 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), in spa_load_l2cache()
2661 for (i = 0; i < sav->sav_count; i++) in spa_load_l2cache()
2663 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); in spa_load_l2cache()
2664 fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, in spa_load_l2cache()
2665 (const nvlist_t * const *)l2cache, sav->sav_count); in spa_load_l2cache()
2677 ASSERT(vd->vdev_isl2cache); in spa_load_l2cache()
2679 if (spa_l2cache_exists(vd->vdev_guid, &pool) && in spa_load_l2cache()
2690 for (i = 0; i < sav->sav_count; i++) in spa_load_l2cache()
2692 if (sav->sav_count) in spa_load_l2cache()
2693 kmem_free(l2cache, sav->sav_count * sizeof (void *)); in spa_load_l2cache()
2705 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); in load_nvlist()
2709 nvsize = *(uint64_t *)db->db_data; in load_nvlist()
2713 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, in load_nvlist()
2723 * Concrete top-level vdevs that are not missing and are not logs. At every
2729 vdev_t *rvd = spa->spa_root_vdev; in spa_healthy_core_tvds()
2732 for (uint64_t i = 0; i < rvd->vdev_children; i++) { in spa_healthy_core_tvds()
2733 vdev_t *vd = rvd->vdev_child[i]; in spa_healthy_core_tvds()
2734 if (vd->vdev_islog) in spa_healthy_core_tvds()
2750 for (uint64_t c = 0; c < vd->vdev_children; c++) in spa_check_removed()
2751 spa_check_removed(vd->vdev_child[c]); in spa_check_removed()
2753 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && in spa_check_removed()
2755 zfs_post_autoreplace(vd->vdev_spa, vd); in spa_check_removed()
2756 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); in spa_check_removed()
2763 vdev_t *rvd = spa->spa_root_vdev; in spa_check_for_missing_logs()
2770 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { in spa_check_for_missing_logs()
2774 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), in spa_check_for_missing_logs()
2778 for (uint64_t c = 0; c < rvd->vdev_children; c++) { in spa_check_for_missing_logs()
2779 vdev_t *tvd = rvd->vdev_child[c]; in spa_check_for_missing_logs()
2786 if (tvd->vdev_islog && in spa_check_for_missing_logs()
2787 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { in spa_check_for_missing_logs()
2796 fnvlist_add_nvlist(spa->spa_load_info, in spa_check_for_missing_logs()
2803 kmem_free(child, rvd->vdev_children * sizeof (char **)); in spa_check_for_missing_logs()
2811 for (uint64_t c = 0; c < rvd->vdev_children; c++) { in spa_check_for_missing_logs()
2812 vdev_t *tvd = rvd->vdev_child[c]; in spa_check_for_missing_logs()
2814 if (tvd->vdev_islog && in spa_check_for_missing_logs()
2815 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { in spa_check_for_missing_logs()
2837 switch (spa->spa_log_state) { in spa_check_logs()
2843 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, in spa_check_logs()
2858 vdev_t *rvd = spa->spa_root_vdev; in spa_passivate_log()
2863 for (int c = 0; c < rvd->vdev_children; c++) { in spa_passivate_log()
2864 vdev_t *tvd = rvd->vdev_child[c]; in spa_passivate_log()
2866 if (tvd->vdev_islog) { in spa_passivate_log()
2867 ASSERT0P(tvd->vdev_log_mg); in spa_passivate_log()
2868 metaslab_group_passivate(tvd->vdev_mg); in spa_passivate_log()
2882 vdev_t *rvd = spa->spa_root_vdev; in spa_activate_log()
2886 for (int c = 0; c < rvd->vdev_children; c++) { in spa_activate_log()
2887 vdev_t *tvd = rvd->vdev_child[c]; in spa_activate_log()
2889 if (tvd->vdev_islog) { in spa_activate_log()
2890 ASSERT0P(tvd->vdev_log_mg); in spa_activate_log()
2891 metaslab_group_activate(tvd->vdev_mg); in spa_activate_log()
2909 txg_wait_synced(spa->spa_dsl_pool, 0); in spa_reset_logs()
2917 for (int i = 0; i < sav->sav_count; i++) in spa_aux_check_removed()
2918 spa_check_removed(sav->sav_vdevs[i]); in spa_aux_check_removed()
2924 spa_t *spa = zio->io_spa; in spa_claim_notify()
2926 if (zio->io_error) in spa_claim_notify()
2929 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ in spa_claim_notify()
2930 if (spa->spa_claim_max_txg < BP_GET_BIRTH(zio->io_bp)) in spa_claim_notify()
2931 spa->spa_claim_max_txg = BP_GET_BIRTH(zio->io_bp); in spa_claim_notify()
2932 mutex_exit(&spa->spa_props_lock); in spa_claim_notify()
2944 blkptr_t *bp = zio->io_bp; in spa_load_verify_done()
2945 spa_load_error_t *sle = zio->io_private; in spa_load_verify_done()
2947 int error = zio->io_error; in spa_load_verify_done()
2948 spa_t *spa = zio->io_spa; in spa_load_verify_done()
2950 abd_free(zio->io_abd); in spa_load_verify_done()
2954 atomic_inc_64(&sle->sle_meta_count); in spa_load_verify_done()
2956 atomic_inc_64(&sle->sle_data_count); in spa_load_verify_done()
2959 mutex_enter(&spa->spa_scrub_lock); in spa_load_verify_done()
2960 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); in spa_load_verify_done()
2961 cv_broadcast(&spa->spa_scrub_io_cv); in spa_load_verify_done()
2962 mutex_exit(&spa->spa_scrub_lock); in spa_load_verify_done()
2978 spa_load_error_t *sle = rio->io_private; in spa_load_verify_cb()
2997 atomic_inc_64(&sle->sle_meta_count); in spa_load_verify_cb()
3001 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || in spa_load_verify_cb()
3006 (!spa_load_verify_data || !sle->sle_verify_data)) in spa_load_verify_cb()
3013 mutex_enter(&spa->spa_scrub_lock); in spa_load_verify_cb()
3014 while (spa->spa_load_verify_bytes >= maxinflight_bytes) in spa_load_verify_cb()
3015 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); in spa_load_verify_cb()
3016 spa->spa_load_verify_bytes += size; in spa_load_verify_cb()
3017 mutex_exit(&spa->spa_scrub_lock); in spa_load_verify_cb()
3020 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, in spa_load_verify_cb()
3046 zpool_get_load_policy(spa->spa_config, &policy); in spa_load_verify()
3052 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); in spa_load_verify()
3053 error = dmu_objset_find_dp(spa->spa_dsl_pool, in spa_load_verify()
3054 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, in spa_load_verify()
3056 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); in spa_load_verify()
3071 if (spa->spa_extreme_rewind) { in spa_load_verify()
3079 error = traverse_pool(spa, spa->spa_verify_min_txg, in spa_load_verify()
3085 ASSERT0(spa->spa_load_verify_bytes); in spa_load_verify()
3087 spa->spa_load_meta_errors = sle.sle_meta_count; in spa_load_verify()
3088 spa->spa_load_data_errors = sle.sle_data_count; in spa_load_verify()
3102 spa->spa_load_txg = spa->spa_uberblock.ub_txg; in spa_load_verify()
3103 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; in spa_load_verify()
3105 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; in spa_load_verify()
3106 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME, in spa_load_verify()
3107 spa->spa_load_txg_ts); in spa_load_verify()
3108 fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME, in spa_load_verify()
3110 fnvlist_add_uint64(spa->spa_load_info, in spa_load_verify()
3112 fnvlist_add_uint64(spa->spa_load_info, in spa_load_verify()
3115 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; in spa_load_verify()
3136 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, in spa_prop_find()
3146 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, in spa_dir_prop()
3167 return (spa->spa_livelists_to_delete != 0); in spa_livelist_delete_check()
3182 zio_free(spa, tx->tx_txg, bp); in delete_blkptr_cb()
3183 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, in delete_blkptr_cb()
3184 -bp_get_dsize_sync(spa, bp), in delete_blkptr_cb()
3185 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); in delete_blkptr_cb()
3190 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) in dsl_get_next_livelist_obj() argument
3199 *llp = za->za_first_integer; in dsl_get_next_livelist_obj()
3206 * context: freeing block pointers and updating the pool-wide data
3220 spa_t *spa = sda->spa; in sublist_delete_sync()
3221 dsl_deadlist_t *ll = sda->ll; in sublist_delete_sync()
3222 uint64_t key = sda->key; in sublist_delete_sync()
3223 bplist_t *to_free = sda->to_free; in sublist_delete_sync()
3239 spa_t *spa = lda->spa; in livelist_delete_sync()
3240 uint64_t ll_obj = lda->ll_obj; in livelist_delete_sync()
3241 uint64_t zap_obj = lda->zap_obj; in livelist_delete_sync()
3242 objset_t *mos = spa->spa_meta_objset; in livelist_delete_sync()
3255 spa->spa_livelists_to_delete = 0; in livelist_delete_sync()
3264 * the pool-wide livelist data.
3271 objset_t *mos = spa->spa_meta_objset; in spa_livelist_delete_cb()
3272 uint64_t zap_obj = spa->spa_livelists_to_delete; in spa_livelist_delete_cb()
3288 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, in spa_livelist_delete_cb()
3294 .key = dle->dle_mintxg, in spa_livelist_delete_cb()
3299 (u_longlong_t)dle->dle_bpobj.bpo_object, in spa_livelist_delete_cb()
3300 (u_longlong_t)ll_obj, (longlong_t)count - 1); in spa_livelist_delete_cb()
3327 ASSERT0P(spa->spa_livelist_delete_zthr); in spa_start_livelist_destroy_thread()
3328 spa->spa_livelist_delete_zthr = in spa_start_livelist_destroy_thread()
3346 bplist_append(lna->frees, bp); in livelist_track_new_cb()
3348 bplist_append(lna->allocs, bp); in livelist_track_new_cb()
3365 spa_t *spa = lca->spa; in spa_livelist_condense_sync()
3367 dsl_dataset_t *ds = spa->spa_to_condense.ds; in spa_livelist_condense_sync()
3370 if (spa->spa_to_condense.cancelled) { in spa_livelist_condense_sync()
3375 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; in spa_livelist_condense_sync()
3376 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; in spa_livelist_condense_sync()
3377 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; in spa_livelist_condense_sync()
3387 uint64_t first_obj = first->dle_bpobj.bpo_object; in spa_livelist_condense_sync()
3388 uint64_t next_obj = next->dle_bpobj.bpo_object; in spa_livelist_condense_sync()
3389 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; in spa_livelist_condense_sync()
3390 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; in spa_livelist_condense_sync()
3394 .allocs = &lca->to_keep, in spa_livelist_condense_sync()
3398 if (cur_first_size > lca->first_size) { in spa_livelist_condense_sync()
3399 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, in spa_livelist_condense_sync()
3400 livelist_track_new_cb, &new_bps, lca->first_size)); in spa_livelist_condense_sync()
3402 if (cur_next_size > lca->next_size) { in spa_livelist_condense_sync()
3403 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, in spa_livelist_condense_sync()
3404 livelist_track_new_cb, &new_bps, lca->next_size)); in spa_livelist_condense_sync()
3408 ASSERT(bpobj_is_empty(&first->dle_bpobj)); in spa_livelist_condense_sync()
3409 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); in spa_livelist_condense_sync()
3411 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); in spa_livelist_condense_sync()
3418 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " in spa_livelist_condense_sync()
3419 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, in spa_livelist_condense_sync()
3420 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, in spa_livelist_condense_sync()
3423 (u_longlong_t)first->dle_bpobj.bpo_object, in spa_livelist_condense_sync()
3424 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); in spa_livelist_condense_sync()
3426 dmu_buf_rele(ds->ds_dbuf, spa); in spa_livelist_condense_sync()
3427 spa->spa_to_condense.ds = NULL; in spa_livelist_condense_sync()
3428 bplist_clear(&lca->to_keep); in spa_livelist_condense_sync()
3429 bplist_destroy(&lca->to_keep); in spa_livelist_condense_sync()
3431 spa->spa_to_condense.syncing = B_FALSE; in spa_livelist_condense_sync()
3442 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; in spa_livelist_condense_cb()
3443 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; in spa_livelist_condense_cb()
3448 bplist_create(&lca->to_keep); in spa_livelist_condense_cb()
3465 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, in spa_livelist_condense_cb()
3468 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, in spa_livelist_condense_cb()
3476 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); in spa_livelist_condense_cb()
3485 spa->spa_to_condense.syncing = B_TRUE; in spa_livelist_condense_cb()
3486 lca->spa = spa; in spa_livelist_condense_cb()
3487 lca->first_size = first_size; in spa_livelist_condense_cb()
3488 lca->next_size = next_size; in spa_livelist_condense_cb()
3502 bplist_clear(&lca->to_keep); in spa_livelist_condense_cb()
3503 bplist_destroy(&lca->to_keep); in spa_livelist_condense_cb()
3505 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); in spa_livelist_condense_cb()
3506 spa->spa_to_condense.ds = NULL; in spa_livelist_condense_cb()
3520 if ((spa->spa_to_condense.ds != NULL) && in spa_livelist_condense_cb_check()
3521 (spa->spa_to_condense.syncing == B_FALSE) && in spa_livelist_condense_cb_check()
3522 (spa->spa_to_condense.cancelled == B_FALSE)) { in spa_livelist_condense_cb_check()
3531 spa->spa_to_condense.ds = NULL; in spa_start_livelist_condensing_thread()
3532 spa->spa_to_condense.first = NULL; in spa_start_livelist_condensing_thread()
3533 spa->spa_to_condense.next = NULL; in spa_start_livelist_condensing_thread()
3534 spa->spa_to_condense.syncing = B_FALSE; in spa_start_livelist_condensing_thread()
3535 spa->spa_to_condense.cancelled = B_FALSE; in spa_start_livelist_condensing_thread()
3537 ASSERT0P(spa->spa_livelist_condense_zthr); in spa_start_livelist_condensing_thread()
3538 spa->spa_livelist_condense_zthr = in spa_start_livelist_condensing_thread()
3554 ASSERT0P(spa->spa_checkpoint_discard_zthr); in spa_spawn_aux_threads()
3555 spa->spa_checkpoint_discard_zthr = in spa_spawn_aux_threads()
3562 * Fix up config after a partly-completed split. This is done with the
3563 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off
3611 /* attempt to re-online it */ in spa_try_repair()
3612 vd[i]->vdev_offline = B_FALSE; in spa_try_repair()
3617 vdev_reopen(spa->spa_root_vdev); in spa_try_repair()
3622 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) in spa_try_repair()
3637 vdev_reopen(spa->spa_root_vdev); in spa_try_repair()
3649 spa->spa_load_state = state; in spa_load()
3654 gethrestime(&spa->spa_loaded_ts); in spa_load()
3662 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); in spa_load()
3665 spa->spa_loaded_ts.tv_sec = 0; in spa_load()
3666 spa->spa_loaded_ts.tv_nsec = 0; in spa_load()
3673 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; in spa_load()
3674 spa->spa_ena = 0; in spa_load()
3684 * Count the number of per-vdev ZAPs associated with all of the vdevs in the
3686 * spa's per-vdev ZAP list.
3691 spa_t *spa = vd->vdev_spa; in vdev_count_verify_zaps()
3694 if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) && in vdev_count_verify_zaps()
3695 vd->vdev_root_zap != 0) { in vdev_count_verify_zaps()
3697 ASSERT0(zap_lookup_int(spa->spa_meta_objset, in vdev_count_verify_zaps()
3698 spa->spa_all_vdev_zaps, vd->vdev_root_zap)); in vdev_count_verify_zaps()
3700 if (vd->vdev_top_zap != 0) { in vdev_count_verify_zaps()
3702 ASSERT0(zap_lookup_int(spa->spa_meta_objset, in vdev_count_verify_zaps()
3703 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); in vdev_count_verify_zaps()
3705 if (vd->vdev_leaf_zap != 0) { in vdev_count_verify_zaps()
3707 ASSERT0(zap_lookup_int(spa->spa_meta_objset, in vdev_count_verify_zaps()
3708 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); in vdev_count_verify_zaps()
3711 for (uint64_t i = 0; i < vd->vdev_children; i++) { in vdev_count_verify_zaps()
3712 total += vdev_count_verify_zaps(vd->vdev_child[i]); in vdev_count_verify_zaps()
3748 * Disable the MMP activity check - This is used by zdb which in spa_activity_check_required()
3751 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) in spa_activity_check_required()
3757 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) in spa_activity_check_required()
3766 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && in spa_activity_check_required()
3767 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && in spa_activity_check_required()
3793 * Nanoseconds the activity check must watch for changes on-disk.
3834 ub->ub_mmp_delay) * import_intervals); in spa_activity_check_duration()
3840 (u_longlong_t)ub->ub_mmp_delay, in spa_activity_check_duration()
3845 * zfs-0.7 compatibility case in spa_activity_check_duration()
3849 ub->ub_mmp_delay) * import_intervals); in spa_activity_check_duration()
3854 (u_longlong_t)ub->ub_mmp_delay, in spa_activity_check_duration()
3859 zfs_dbgmsg("pool last imported on non-MMP aware " in spa_activity_check_duration()
3873 * 0 - no activity detected
3874 * EREMOTEIO - remote activity detected
3875 * EINTR - user canceled the operation
3881 uint64_t txg = ub->ub_txg; in spa_activity_check()
3882 uint64_t timestamp = ub->ub_timestamp; in spa_activity_check()
3883 uint64_t mmp_config = ub->ub_mmp_config; in spa_activity_check()
3888 vdev_t *rvd = spa->spa_root_vdev; in spa_activity_check()
3919 /* Add a small random factor in case of simultaneous imports (0-25%) */ in spa_activity_check()
3934 (u_longlong_t)NSEC2MSEC(import_expire - now)); in spa_activity_check()
3939 NSEC2SEC(import_expire - gethrtime())); in spa_activity_check()
3944 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || in spa_activity_check()
3950 (u_longlong_t)txg, (u_longlong_t)ub->ub_txg, in spa_activity_check()
3952 (u_longlong_t)ub->ub_timestamp, in spa_activity_check()
3954 (u_longlong_t)ub->ub_mmp_config); in spa_activity_check()
3966 if (error != -1) { in spa_activity_check()
3980 * spa->spa_load_info nvlist. If the remote hostname or hostid are in spa_activity_check()
3984 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) in spa_activity_check()
3985 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool in spa_activity_check()
3986 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool in spa_activity_check()
3993 fnvlist_add_string(spa->spa_load_info, in spa_activity_check()
4000 fnvlist_add_uint64(spa->spa_load_info, in spa_activity_check()
4005 fnvlist_add_uint64(spa->spa_load_info, in spa_activity_check()
4007 fnvlist_add_uint64(spa->spa_load_info, in spa_activity_check()
4034 vdev_uberblock_load(spa->spa_root_vdev, &best_ub, &best_label); in spa_mmp_remote_host_activity()
4056 if (best_ub.ub_txg != spa->spa_uberblock.ub_txg || in spa_mmp_remote_host_activity()
4057 best_ub.ub_timestamp != spa->spa_uberblock.ub_timestamp) { in spa_mmp_remote_host_activity()
4060 (u_longlong_t)spa->spa_uberblock.ub_txg, in spa_mmp_remote_host_activity()
4062 (u_longlong_t)spa->spa_uberblock.ub_timestamp, in spa_mmp_remote_host_activity()
4070 return (spa_activity_check(spa, &spa->spa_uberblock, spa->spa_config, in spa_mmp_remote_host_activity()
4092 "See: https://openzfs.github.io/openzfs-docs/msg/" in spa_verify_host()
4093 "ZFS-8000-EY", in spa_verify_host()
4109 nvlist_t *nvtree, *nvl, *config = spa->spa_config;
4121 &spa->spa_ubsync.ub_version) != 0)
4122 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL;
4140 if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
4141 spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
4144 if ((spa->spa_load_state == SPA_LOAD_IMPORT ||
4145 spa->spa_load_state == SPA_LOAD_TRYIMPORT) &&
4154 spa->spa_config_guid = pool_guid;
4156 nvlist_free(spa->spa_load_info);
4157 spa->spa_load_info = fnvlist_alloc();
4159 ASSERT0P(spa->spa_comment);
4161 spa->spa_comment = spa_strdup(comment);
4163 ASSERT0P(spa->spa_compatibility);
4166 spa->spa_compatibility = spa_strdup(compatibility);
4169 &spa->spa_config_txg);
4172 spa->spa_config_splitting = fnvlist_dup(nvl);
4183 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
4186 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
4208 ASSERT(spa->spa_root_vdev == rvd);
4209 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
4210 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT);
4229 * spa_missing_tvds_allowed defines how many top-level vdevs can be
4232 if (spa->spa_trust_config) {
4233 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds;
4234 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) {
4235 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile;
4236 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) {
4237 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan;
4239 spa->spa_missing_tvds_allowed = 0;
4242 spa->spa_missing_tvds_allowed =
4243 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed);
4246 error = vdev_open(spa->spa_root_vdev);
4249 if (spa->spa_missing_tvds != 0) {
4250 spa_load_note(spa, "vdev tree has %lld missing top-level "
4251 "vdevs.", (u_longlong_t)spa->spa_missing_tvds);
4252 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) {
4262 spa_load_note(spa, "pools with missing top-level "
4263 "vdevs can only be opened in read-only mode.");
4267 "%lld missing top-level vdevs at this stage.",
4268 (u_longlong_t)spa->spa_missing_tvds_allowed);
4275 if (spa->spa_missing_tvds != 0 || error != 0)
4276 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2);
4291 vdev_t *rvd = spa->spa_root_vdev;
4302 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) {
4315 spa->spa_state = POOL_STATE_ACTIVE;
4316 spa->spa_ubsync = spa->spa_uberblock;
4317 spa->spa_verify_min_txg = spa->spa_extreme_rewind ?
4318 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1;
4319 spa->spa_first_txg = spa->spa_last_ubsync_txg ?
4320 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1;
4321 spa->spa_claim_max_txg = spa->spa_first_txg;
4322 spa->spa_prev_software_version = ub->ub_software_version;
4328 vdev_t *rvd = spa->spa_root_vdev;
4330 uberblock_t *ub = &spa->spa_uberblock;
4338 * we are opening the checkpointed state read-only, we have
4350 if (ub->ub_checkpoint_txg != 0 &&
4364 if (ub->ub_txg == 0) {
4370 if (spa->spa_load_max_txg != UINT64_MAX) {
4372 (u_longlong_t)spa->spa_load_max_txg);
4375 (u_longlong_t)ub->ub_txg);
4376 if (ub->ub_raidz_reflow_info != 0) {
4390 spa->spa_config);
4392 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay &&
4395 fnvlist_add_uint64(spa->spa_load_info,
4401 spa_activity_check(spa, ub, spa->spa_config, B_TRUE);
4407 fnvlist_add_uint64(spa->spa_load_info,
4409 fnvlist_add_uint64(spa->spa_load_info,
4410 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg);
4411 fnvlist_add_uint16(spa->spa_load_info,
4419 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) {
4422 (u_longlong_t)ub->ub_version);
4426 if (ub->ub_version >= SPA_VERSION_FEATURES) {
4449 * Update our in-core representation with the definitive values
4452 nvlist_free(spa->spa_label_features);
4453 spa->spa_label_features = fnvlist_dup(features);
4463 if (ub->ub_version >= SPA_VERSION_FEATURES) {
4468 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features,
4470 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) {
4478 fnvlist_add_nvlist(spa->spa_load_info,
4489 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) {
4491 spa_try_repair(spa, spa->spa_config);
4493 nvlist_free(spa->spa_config_splitting);
4494 spa->spa_config_splitting = NULL;
4509 vdev_t *rvd = spa->spa_root_vdev;
4511 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool);
4517 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset;
4526 vdev_t *mrvd, *rvd = spa->spa_root_vdev;
4532 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE)
4545 if (load_nvlist(spa, spa->spa_config_object, &mos_config)
4555 if (spa->spa_load_state == SPA_LOAD_OPEN) {
4603 spa->spa_root_vdev = mrvd;
4608 * If 'zpool import' used a cached config, then the on-disk hostid and
4619 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_HOSTID,
4622 fnvlist_add_string(spa->spa_load_info, ZPOOL_CONFIG_HOSTNAME,
4636 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY,
4640 spa->spa_config_source = SPA_CONFIG_SRC_MOS;
4648 spa->spa_trust_config = B_TRUE;
4666 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT &&
4667 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) {
4676 if (healthy_tvds_mos - healthy_tvds >=
4679 "top-level vdevs compared to MOS (%lld vs %lld). ",
4700 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) {
4703 (u_longlong_t)spa->spa_uberblock.ub_guid_sum,
4704 (u_longlong_t)rvd->vdev_guid_sum);
4716 vdev_t *rvd = spa->spa_root_vdev;
4746 vdev_t *rvd = spa->spa_root_vdev;
4753 &spa->spa_feat_for_read_obj, B_TRUE) != 0) {
4758 &spa->spa_feat_for_write_obj, B_TRUE) != 0) {
4763 &spa->spa_feat_desc_obj, B_TRUE) != 0) {
4775 spa->spa_load_state == SPA_LOAD_TRYIMPORT) {
4782 fnvlist_add_nvlist(spa->spa_load_info,
4786 fnvlist_add_nvlist(spa->spa_load_info,
4794 fnvlist_add_boolean(spa->spa_load_info,
4801 * import in read-write mode and (if it is not) whether the
4802 * pool is available for import in read-only mode. If the pool
4803 * is available for import in read-write mode, it is displayed
4805 * in read-only mode, it is displayed as unavailable in
4806 * userland. If the pool is available for import in read-only
4807 * mode but not read-write mode, it is displayed as unavailable
4809 * available for open in read-only mode.
4813 * the pool can be opened read-only before returning to
4825 * Load refcounts for ZFS features from disk into an in-memory
4834 spa->spa_feat_refcount_cache[i] = refcount;
4836 spa->spa_feat_refcount_cache[i] =
4850 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0)
4861 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
4871 vdev_t *rvd = spa->spa_root_vdev;
4873 spa->spa_is_initializing = B_TRUE;
4874 error = dsl_pool_open(spa->spa_dsl_pool);
4875 spa->spa_is_initializing = B_FALSE;
4889 vdev_t *rvd = spa->spa_root_vdev;
4892 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
4894 sizeof (spa->spa_cksum_salt.zcs_bytes),
4895 spa->spa_cksum_salt.zcs_bytes);
4898 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
4899 sizeof (spa->spa_cksum_salt.zcs_bytes));
4908 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj);
4910 spa_load_failed(spa, "error opening deferred-frees bpobj "
4917 * (raid-z deflation). If we have an older pool, this will not
4920 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE);
4925 &spa->spa_creation_version, B_FALSE);
4936 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last,
4942 &spa->spa_errlog_scrub, B_FALSE);
4948 &spa->spa_scrubbed_last_txg, B_FALSE);
4957 &spa->spa_livelists_to_delete, B_FALSE);
4965 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE);
4970 * Load the per-vdev ZAP map. If we have an older pool, this will not
4978 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) {
4984 &spa->spa_all_vdev_zaps, B_FALSE);
4989 spa->spa_avz_action = AVZ_ACTION_INITIALIZE;
4990 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
4997 * we have orphaned per-vdev ZAPs in the MOS. Defer their
5000 spa->spa_avz_action = AVZ_ACTION_DESTROY;
5005 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev));
5009 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
5011 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object,
5019 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs);
5021 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation);
5022 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode);
5023 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand);
5025 &spa->spa_dedup_table_quota);
5026 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost);
5027 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim);
5028 spa->spa_autoreplace = (autoreplace != 0);
5032 * If we are importing a pool with missing top-level vdevs,
5036 if (spa->spa_missing_tvds > 0 &&
5037 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE &&
5038 spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
5041 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE;
5051 vdev_t *rvd = spa->spa_root_vdev;
5054 * If we're assembling the pool from the split-off vdevs of
5062 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object,
5068 if (load_nvlist(spa, spa->spa_spares.sav_object,
5069 &spa->spa_spares.sav_config) != 0) {
5078 spa->spa_spares.sav_sync = B_TRUE;
5085 &spa->spa_l2cache.sav_object, B_FALSE);
5090 if (load_nvlist(spa, spa->spa_l2cache.sav_object,
5091 &spa->spa_l2cache.sav_config) != 0) {
5100 spa->spa_l2cache.sav_sync = B_TRUE;
5110 vdev_t *rvd = spa->spa_root_vdev;
5118 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) {
5119 fnvlist_add_uint64(spa->spa_load_info,
5131 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
5132 spa_check_removed(spa->spa_root_vdev);
5138 if (spa->spa_load_state != SPA_LOAD_IMPORT) {
5139 spa_aux_check_removed(&spa->spa_spares);
5140 spa_aux_check_removed(&spa->spa_l2cache);
5174 vdev_t *rvd = spa->spa_root_vdev;
5189 vdev_t *rvd = spa->spa_root_vdev;
5203 vdev_t *rvd = spa->spa_root_vdev;
5208 if (spa->spa_missing_tvds != 0) {
5227 vdev_t *rvd = spa->spa_root_vdev;
5233 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) {
5259 spa->spa_claiming = B_TRUE;
5262 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
5266 spa->spa_claiming = B_FALSE;
5275 vdev_t *rvd = spa->spa_root_vdev;
5283 * in-core spa_config and update the disk labels.
5285 if (update_config_cache || config_cache_txg != spa->spa_config_txg ||
5286 spa->spa_load_state == SPA_LOAD_IMPORT ||
5287 spa->spa_load_state == SPA_LOAD_RECOVER ||
5288 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM))
5291 for (int c = 0; c < rvd->vdev_children; c++)
5292 if (rvd->vdev_child[c]->vdev_ms_array == 0)
5306 spa_mode_t mode = spa->spa_mode;
5307 int async_suspended = spa->spa_async_suspended;
5318 spa->spa_async_suspended = async_suspended;
5327 ASSERT0(spa->spa_checkpoint_txg);
5329 spa->spa_load_thread == curthread);
5331 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
5344 spa->spa_checkpoint_txg = checkpoint.ub_txg;
5345 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp;
5356 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
5362 * also effectively puts the spa in read-only mode since
5367 spa->spa_trust_config = B_FALSE;
5382 * based on the success of those operations. After this we'll be ready
5432 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5434 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
5448 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg);
5458 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1;
5464 spa->spa_uberblock = checkpoint;
5474 * pool as read-only. Specifically, verifying the checkpointed
5479 vdev_t *rvd = spa->spa_root_vdev;
5484 int children = rvd->vdev_children;
5488 vdev_t *vd = rvd->vdev_child[(c0 + c) % children];
5494 if (vd->vdev_ms_array == 0 || vd->vdev_islog ||
5502 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg);
5504 spa->spa_last_synced_guid = rvd->vdev_guid;
5574 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5579 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE);
5624 spa->spa_load_thread = curthread;
5637 * from the pool and their contents were re-mapped to other vdevs. Note
5675 * Retrieve the list of auxiliary devices - cache devices and spares -
5712 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT);
5716 * read-only mode but not read-write mode. We now have enough
5719 error = spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT,
5744 * pool. If we are importing the pool in read-write mode, a few
5748 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER ||
5749 spa->spa_load_max_txg == UINT64_MAX)) {
5750 uint64_t config_cache_txg = spa->spa_config_txg;
5752 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT);
5758 if (RRSS_GET_STATE(&spa->spa_uberblock) == RRSS_SCRATCH_VALID)
5768 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg);
5778 * Kick-off the syncing thread.
5780 spa->spa_sync_on = B_TRUE;
5781 txg_sync_start(spa->spa_dsl_pool);
5792 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg);
5809 if (vdev_rebuild_active(spa->spa_root_vdev)) {
5811 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) &&
5812 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
5846 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool);
5850 vdev_initialize_restart(spa->spa_root_vdev);
5852 vdev_trim_restart(spa->spa_root_vdev);
5857 zio_handle_import_delay(spa, gethrtime() - load_start);
5865 spa->spa_load_thread = NULL;
5875 spa_mode_t mode = spa->spa_mode;
5880 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1;
5886 (u_longlong_t)spa->spa_load_max_txg);
5908 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) {
5909 spa->spa_load_max_txg = spa->spa_load_txg;
5912 spa->spa_load_max_txg = max_request;
5914 spa->spa_extreme_rewind = B_TRUE;
5922 * When attempting checkpoint-rewind on a pool with no
5926 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT);
5931 if (spa->spa_root_vdev != NULL)
5932 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
5934 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg;
5935 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp;
5952 loadinfo = spa->spa_load_info;
5953 spa->spa_load_info = fnvlist_alloc();
5956 spa->spa_load_max_txg = spa->spa_last_ubsync_txg;
5957 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE;
5965 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg &&
5966 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) {
5967 if (spa->spa_load_max_txg < safe_rewind_txg)
5968 spa->spa_extreme_rewind = B_TRUE;
5972 spa->spa_extreme_rewind = B_FALSE;
5973 spa->spa_load_max_txg = UINT64_MAX;
5987 spa->spa_load_info);
5990 fnvlist_free(spa->spa_load_info);
5991 spa->spa_load_info = loadinfo;
6039 if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
6044 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config,
6052 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
6053 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
6082 if (config != NULL && spa->spa_config) {
6083 *config = fnvlist_dup(spa->spa_config);
6086 spa->spa_load_info);
6090 spa->spa_last_open_failed = error;
6101 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
6109 spa->spa_load_info);
6113 spa->spa_last_open_failed = 0;
6114 spa->spa_last_ubsync_txg = 0;
6115 spa->spa_load_txg = 0;
6154 spa->spa_inject_ref++;
6164 spa->spa_inject_ref--;
6184 if (spa->spa_spares.sav_count == 0)
6188 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config,
6208 vs->vs_state = VDEV_STATE_CANT_OPEN;
6209 vs->vs_aux = VDEV_AUX_SPARED;
6211 vs->vs_state =
6212 spa->spa_spares.sav_vdevs[i]->vdev_state;
6234 if (spa->spa_l2cache.sav_count == 0)
6238 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config,
6255 for (j = 0; j < spa->spa_l2cache.sav_count; j++) {
6257 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) {
6258 vd = spa->spa_l2cache.sav_vdevs[j];
6279 if (spa->spa_feat_for_read_obj != 0) {
6280 for (zap_cursor_init(&zc, spa->spa_meta_objset,
6281 spa->spa_feat_for_read_obj);
6284 ASSERT(za->za_integer_length == sizeof (uint64_t) &&
6285 za->za_num_integers == 1);
6286 VERIFY0(nvlist_add_uint64(features, za->za_name,
6287 za->za_first_integer));
6292 if (spa->spa_feat_for_write_obj != 0) {
6293 for (zap_cursor_init(&zc, spa->spa_meta_objset,
6294 spa->spa_feat_for_write_obj);
6297 ASSERT(za->za_integer_length == sizeof (uint64_t) &&
6298 za->za_num_integers == 1);
6299 VERIFY0(nvlist_add_uint64(features, za->za_name,
6300 za->za_first_integer));
6341 mutex_enter(&spa->spa_feat_stats_lock);
6342 features = spa->spa_feat_stats;
6348 spa->spa_feat_stats = features;
6355 mutex_exit(&spa->spa_feat_stats_lock);
6372 * self-inconsistent.
6379 loadtimes[0] = spa->spa_loaded_ts.tv_sec;
6380 loadtimes[1] = spa->spa_loaded_ts.tv_nsec;
6391 spa->spa_failmode);
6394 spa->spa_suspended);
6434 * specified, as long as they are well-formed.
6465 * Set the pending device list so we correctly handle device in-use
6468 sav->sav_pending = dev;
6469 sav->sav_npending = ndev;
6476 if (!vd->vdev_ops->vdev_op_leaf) {
6482 vd->vdev_top = vd;
6487 vd->vdev_guid);
6500 sav->sav_pending = NULL;
6501 sav->sav_npending = 0;
6513 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES,
6519 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE,
6529 if (sav->sav_config != NULL) {
6538 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config,
6548 fnvlist_remove(sav->sav_config, config);
6550 fnvlist_add_nvlist_array(sav->sav_config, config,
6559 sav->sav_config = fnvlist_alloc();
6560 fnvlist_add_nvlist_array(sav->sav_config, config,
6573 spa_aux_vdev_t *sav = &spa->spa_l2cache;
6575 for (i = 0; i < sav->sav_count; i++) {
6578 vd = sav->sav_vdevs[i];
6581 if (spa_l2cache_exists(vd->vdev_guid, &pool) &&
6595 if (dcp->cp_crypt != ZIO_CRYPT_OFF &&
6596 dcp->cp_crypt != ZIO_CRYPT_INHERIT &&
6664 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME;
6706 spa->spa_first_txg = txg;
6707 spa->spa_uberblock.ub_txg = txg - 1;
6708 spa->spa_uberblock.ub_version = version;
6709 spa->spa_ubsync = spa->spa_uberblock;
6710 spa->spa_load_state = SPA_LOAD_CREATE;
6711 spa->spa_removing_phys.sr_state = DSS_NONE;
6712 spa->spa_removing_phys.sr_removing_vdev = -1;
6713 spa->spa_removing_phys.sr_prev_indirect_vdev = -1;
6714 spa->spa_indirect_vdevs_loaded = B_TRUE;
6719 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *),
6722 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL,
6735 ASSERT(error != 0 || spa->spa_root_vdev == rvd);
6748 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) {
6749 vdev_t *vd = rvd->vdev_child[c];
6771 spa->spa_spares.sav_config = fnvlist_alloc();
6772 fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
6778 spa->spa_spares.sav_sync = B_TRUE;
6786 VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config,
6788 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
6794 spa->spa_l2cache.sav_sync = B_TRUE;
6797 spa->spa_is_initializing = B_TRUE;
6798 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg);
6799 spa->spa_is_initializing = B_FALSE;
6817 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history)
6826 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset,
6830 if (zap_add(spa->spa_meta_objset,
6832 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) {
6836 if (zap_add(spa->spa_meta_objset,
6844 spa->spa_deflate = TRUE;
6845 if (zap_add(spa->spa_meta_objset,
6847 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) {
6853 * Create the deferred-free bpobj. Turn off compression
6854 * because sync-to-convergence takes longer if the blocksize
6857 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx);
6858 dmu_object_set_compress(spa->spa_meta_objset, obj,
6860 if (zap_add(spa->spa_meta_objset,
6865 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj,
6866 spa->spa_meta_objset, obj));
6871 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes,
6872 sizeof (spa->spa_cksum_salt.zcs_bytes));
6877 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
6878 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
6879 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
6880 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND);
6881 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST);
6882 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM);
6883 spa->spa_dedup_table_quota =
6896 spa->spa_sync_on = B_TRUE;
6910 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount);
6911 spa->spa_load_state = SPA_LOAD_NONE;
6921 * Import a non-root pool into the system.
6956 spa->spa_import_flags = flags;
6959 * Verbatim import - Take a pool and insert it into the namespace
6962 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) {
6984 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT;
6987 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0;
6999 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info);
7006 if (spa->spa_spares.sav_config) {
7007 nvlist_free(spa->spa_spares.sav_config);
7008 spa->spa_spares.sav_config = NULL;
7011 if (spa->spa_l2cache.sav_config) {
7012 nvlist_free(spa->spa_l2cache.sav_config);
7013 spa->spa_l2cache.sav_config = NULL;
7040 if (spa->spa_spares.sav_config)
7041 fnvlist_remove(spa->spa_spares.sav_config,
7044 spa->spa_spares.sav_config = fnvlist_alloc();
7045 fnvlist_add_nvlist_array(spa->spa_spares.sav_config,
7051 spa->spa_spares.sav_sync = B_TRUE;
7052 spa->spa_spares.sav_label_sync = B_TRUE;
7056 if (spa->spa_l2cache.sav_config)
7057 fnvlist_remove(spa->spa_l2cache.sav_config,
7060 spa->spa_l2cache.sav_config = fnvlist_alloc();
7061 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config,
7067 spa->spa_l2cache.sav_sync = B_TRUE;
7068 spa->spa_l2cache.sav_label_sync = B_TRUE;
7074 if (spa->spa_autoreplace) {
7075 spa_aux_check_removed(&spa->spa_spares);
7076 spa_aux_check_removed(&spa->spa_l2cache);
7081 * Update the config cache to include the newly-imported pool.
7125 (void) snprintf(name, MAXPATHLEN, "%s-%llx-%s",
7136 zpool_get_load_policy(spa->spa_config, &policy);
7138 spa->spa_load_max_txg = policy.zlp_txg;
7139 spa->spa_extreme_rewind = B_TRUE;
7149 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE;
7151 spa->spa_config_source = SPA_CONFIG_SRC_SCAN;
7162 spa->spa_import_flags |= ZFS_IMPORT_MISSING_LOG;
7169 if (spa->spa_root_vdev != NULL) {
7170 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE);
7174 spa->spa_uberblock.ub_timestamp);
7176 spa->spa_load_info);
7178 spa->spa_errata);
7185 if ((!error || error == EEXIST) && spa->spa_bootfs) {
7193 spa->spa_bootfs, tmpname) == 0) {
7260 if (spa->spa_is_exporting) {
7265 spa->spa_is_exporting = B_TRUE;
7274 if (spa->spa_zvol_taskq) {
7276 taskq_wait(spa->spa_zvol_taskq);
7279 spa->spa_export_thread = curthread;
7282 if (spa->spa_state == POOL_STATE_UNINITIALIZED) {
7292 if (spa->spa_sync_on) {
7293 txg_wait_synced(spa->spa_dsl_pool, 0);
7302 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) {
7311 * notice the spa->spa_export_thread and wait until we signal
7315 if (spa->spa_sync_on) {
7316 vdev_t *rvd = spa->spa_root_vdev;
7350 spa->spa_state = new_state;
7364 * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs.
7373 spa->spa_final_txg = spa_last_synced_txg(spa) +
7387 if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
7392 if (oldconfig && spa->spa_config)
7393 *oldconfig = fnvlist_dup(spa->spa_config);
7396 zio_handle_export_delay(spa, gethrtime() - export_start);
7412 spa->spa_is_exporting = B_FALSE;
7413 spa->spa_export_thread = NULL;
7424 spa->spa_is_exporting = B_FALSE;
7425 spa->spa_export_thread = NULL;
7480 spa_t *spa = dmu_tx_pool(tx)->dp_spa;
7495 vdev_t *rvd = spa->spa_root_vdev;
7508 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */
7518 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0)
7521 if (vd->vdev_children != 0 &&
7532 rvd->vdev_children)) == 0) {
7553 if (spa->spa_vdev_removal != NULL ||
7554 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
7555 for (int c = 0; c < vd->vdev_children; c++) {
7556 tvd = vd->vdev_child[c];
7557 if (spa->spa_vdev_removal != NULL &&
7558 tvd->vdev_ashift != spa->spa_max_ashift) {
7569 if (tvd->vdev_ops == &vdev_mirror_ops) {
7571 cid < tvd->vdev_children; cid++) {
7572 vdev_t *cvd = tvd->vdev_child[cid];
7573 if (!cvd->vdev_ops->vdev_op_leaf) {
7582 if (check_ashift && spa->spa_max_ashift == spa->spa_min_ashift) {
7583 for (int c = 0; c < vd->vdev_children; c++) {
7584 tvd = vd->vdev_child[c];
7585 if (tvd->vdev_ashift != spa->spa_max_ashift) {
7592 for (int c = 0; c < vd->vdev_children; c++) {
7593 tvd = vd->vdev_child[c];
7595 tvd->vdev_id = rvd->vdev_children;
7601 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares,
7604 spa->spa_spares.sav_sync = B_TRUE;
7608 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache,
7611 spa->spa_l2cache.sav_sync = B_TRUE;
7621 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
7622 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr,
7636 * spa_load() checks for added-but-not-initialized vdevs, so that
7669 * draid2:6d:10c:2s-0 DEGRADED 0 0 0
7670 * scsi-0QEMU_QEMU_HARDDISK_d1 ONLINE 0 0 0
7671 * scsi-0QEMU_QEMU_HARDDISK_d2 ONLINE 0 0 0
7672 * scsi-0QEMU_QEMU_HARDDISK_d3 ONLINE 0 0 0
7673 * scsi-0QEMU_QEMU_HARDDISK_d4 ONLINE 0 0 0
7674 * scsi-0QEMU_QEMU_HARDDISK_d5 ONLINE 0 0 0
7675 * scsi-0QEMU_QEMU_HARDDISK_d6 ONLINE 0 0 0
7676 * scsi-0QEMU_QEMU_HARDDISK_d7 ONLINE 0 0 0
7677 * scsi-0QEMU_QEMU_HARDDISK_d8 ONLINE 0 0 0
7678 * scsi-0QEMU_QEMU_HARDDISK_d9 ONLINE 0 0 0
7679 * spare-9 DEGRADED 0 0 0
7680 * replacing-0 DEGRADED 0 93 0
7681 * scsi-0QEMU_QEMU_HARDDISK_d10-part1/old UNAVAIL 0 0 0
7682 * spare-1 DEGRADED 0 0 0
7683 * scsi-0QEMU_QEMU_HARDDISK_d10 REMOVED 0 0 0
7684 * draid2-0-0 ONLINE 0 0 0
7685 * draid2-0-1 ONLINE 0 0 0
7687 * draid2-0-0 INUSE currently in use
7688 * draid2-0-1 INUSE currently in use
7703 ppvd = pvd->vdev_parent;
7717 if ((ppvd->vdev_ops == &vdev_spare_ops) ||
7718 (ppvd->vdev_ops == &vdev_draid_spare_ops))
7719 if (pvd->vdev_ops == &vdev_replacing_ops)
7720 if (newvd->vdev_isspare)
7728 * a mirror, a raidz, or a leaf device that is also a top-level (e.g. a
7749 vdev_t *rvd = spa->spa_root_vdev;
7784 if (spa->spa_vdev_removal != NULL) {
7792 boolean_t raidz = oldvd->vdev_ops == &vdev_raidz_ops;
7801 if (spa->spa_raidz_expand != NULL) {
7805 } else if (!oldvd->vdev_ops->vdev_op_leaf) {
7812 pvd = oldvd->vdev_parent;
7818 if (newrootvd->vdev_children != 1)
7821 newvd = newrootvd->vdev_child[0];
7823 if (!newvd->vdev_ops->vdev_op_leaf)
7832 if ((oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE ||
7833 oldvd->vdev_top->vdev_islog) && newvd->vdev_isspare) {
7840 if (newvd->vdev_ops == &vdev_draid_spare_ops &&
7841 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) {
7852 if (pvd->vdev_top != NULL)
7853 tvd = pvd->vdev_top;
7855 if (tvd->vdev_ops != &vdev_mirror_ops &&
7856 tvd->vdev_ops != &vdev_root_ops &&
7857 tvd->vdev_ops != &vdev_draid_ops) {
7868 if (pvd->vdev_ops != &vdev_mirror_ops &&
7869 pvd->vdev_ops != &vdev_root_ops &&
7879 if (pvd->vdev_ops == &vdev_spare_ops &&
7880 oldvd->vdev_isspare &&
7881 !spa_has_spare(spa, newvd->vdev_guid))
7889 * the same (spare replaces spare, non-spare replaces
7890 * non-spare).
7892 if (pvd->vdev_ops == &vdev_replacing_ops &&
7895 } else if (pvd->vdev_ops == &vdev_spare_ops &&
7896 newvd->vdev_isspare != oldvd->vdev_isspare) {
7906 if (newvd->vdev_isspare)
7915 vdev_t *min_vdev = raidz ? oldvd->vdev_child[0] : oldvd;
7916 if (newvd->vdev_asize < vdev_get_min_asize(min_vdev))
7921 * than the top-level vdev.
7923 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) {
7929 * RAIDZ-expansion-specific checks.
7938 for (int i = 0; i < oldvd->vdev_children; i++) {
7939 if (vdev_is_dead(oldvd->vdev_child[i]) ||
7940 !oldvd->vdev_child[i]->vdev_ops->vdev_op_leaf) {
7944 /* Also fail if reserved boot area is in-use */
7945 if (vdev_check_boot_reserve(spa, oldvd->vdev_child[i])
7957 * move it to a spa_strdup-ed string.
7959 char *tmp = kmem_asprintf("raidz%u-%u",
7960 (uint_t)vdev_get_nparity(oldvd), (uint_t)oldvd->vdev_id);
7964 oldvdpath = spa_strdup(oldvd->vdev_path);
7966 newvdpath = spa_strdup(newvd->vdev_path);
7969 * If this is an in-place replacement, update oldvd's path and devid
7973 spa_strfree(oldvd->vdev_path);
7974 oldvd->vdev_path = kmem_alloc(strlen(newvdpath) + 5,
7976 (void) sprintf(oldvd->vdev_path, "%s/old",
7978 if (oldvd->vdev_devid != NULL) {
7979 spa_strfree(oldvd->vdev_devid);
7980 oldvd->vdev_devid = NULL;
7983 oldvdpath = spa_strdup(oldvd->vdev_path);
7990 if (!raidz && pvd->vdev_ops != pvops) {
7992 ASSERT(pvd->vdev_ops == pvops);
7993 ASSERT(oldvd->vdev_parent == pvd);
7996 ASSERT(pvd->vdev_top->vdev_parent == rvd);
8002 newvd->vdev_id = pvd->vdev_children;
8003 newvd->vdev_crtxg = oldvd->vdev_crtxg;
8011 tvd = newvd->vdev_top;
8012 ASSERT(pvd->vdev_top == tvd);
8013 ASSERT(tvd->vdev_parent == rvd);
8019 * for any dmu_sync-ed blocks. It will propagate upward when
8038 tvd->vdev_rz_expanding = B_TRUE;
8043 dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool,
8045 dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_raidz_attach_sync,
8050 dtl_max_txg - TXG_INITIAL);
8052 if (newvd->vdev_isspare) {
8057 newvd_isspare = newvd->vdev_isspare;
8066 * We do this to ensure that dmu_sync-ed blocks have been
8070 newvd->vdev_rebuild_txg = txg;
8074 newvd->vdev_resilver_txg = txg;
8081 dsl_scan_restart_resilver(spa->spa_dsl_pool,
8087 if (spa->spa_bootfs)
8120 vdev_t *rvd __maybe_unused = spa->spa_root_vdev;
8156 if (!vd->vdev_ops->vdev_op_leaf)
8159 pvd = vd->vdev_parent;
8163 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing
8169 * ask to detach C, which would leave us with just A -- not what
8171 * parent/child relationship hasn't changed -- in this example,
8174 if (pvd->vdev_guid != pguid && pguid != 0)
8180 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops &&
8181 pvd->vdev_ops != &vdev_spare_ops)
8184 ASSERT(pvd->vdev_ops != &vdev_spare_ops ||
8190 if (pvd->vdev_ops != &vdev_replacing_ops &&
8191 pvd->vdev_ops != &vdev_mirror_ops &&
8192 pvd->vdev_ops != &vdev_spare_ops)
8202 ASSERT(pvd->vdev_children >= 2);
8209 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 &&
8210 vd->vdev_path != NULL) {
8211 size_t len = strlen(vd->vdev_path);
8213 for (int c = 0; c < pvd->vdev_children; c++) {
8214 cvd = pvd->vdev_child[c];
8216 if (cvd == vd || cvd->vdev_path == NULL)
8219 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 &&
8220 strcmp(cvd->vdev_path + len, "/old") == 0) {
8221 spa_strfree(cvd->vdev_path);
8222 cvd->vdev_path = spa_strdup(vd->vdev_path);
8235 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) {
8236 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1];
8238 if (last_cvd->vdev_isspare &&
8239 last_cvd->vdev_ops != &vdev_draid_spare_ops) {
8248 * But if we can't do it, don't treat the error as fatal --
8263 cvd = pvd->vdev_child[pvd->vdev_children - 1];
8275 ASSERT(cvd->vdev_isspare);
8277 unspare_guid = cvd->vdev_guid;
8279 cvd->vdev_unspare = B_TRUE;
8286 if (pvd->vdev_children == 1) {
8287 if (pvd->vdev_ops == &vdev_spare_ops)
8288 cvd->vdev_unspare = B_FALSE;
8294 * may have been the previous top-level vdev.
8296 tvd = cvd->vdev_top;
8297 ASSERT(tvd->vdev_parent == rvd);
8311 if (spa->spa_autoexpand) {
8320 * vd->vdev_detached is set and free vd's DTL object in syncing context.
8324 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none");
8326 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t);
8327 vd->vdev_detached = B_TRUE;
8352 if (altspa->spa_state != POOL_STATE_ACTIVE ||
8386 if (vd == NULL || vd->vdev_detached) {
8389 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
8396 mutex_enter(&vd->vdev_initialize_lock);
8407 (vd->vdev_initialize_thread != NULL ||
8408 vd->vdev_top->vdev_removing || vd->vdev_top->vdev_rz_expanding)) {
8409 mutex_exit(&vd->vdev_initialize_lock);
8412 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE &&
8413 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) {
8414 mutex_exit(&vd->vdev_initialize_lock);
8417 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) {
8418 mutex_exit(&vd->vdev_initialize_lock);
8421 vd->vdev_initialize_thread != NULL) {
8422 mutex_exit(&vd->vdev_initialize_lock);
8442 mutex_exit(&vd->vdev_initialize_lock);
8486 txg_wait_synced(spa->spa_dsl_pool, 0);
8504 if (vd == NULL || vd->vdev_detached) {
8507 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) {
8513 } else if (!vd->vdev_has_trim) {
8516 } else if (secure && !vd->vdev_has_securetrim) {
8520 mutex_enter(&vd->vdev_trim_lock);
8530 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing ||
8531 vd->vdev_top->vdev_rz_expanding)) {
8532 mutex_exit(&vd->vdev_trim_lock);
8535 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE &&
8536 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) {
8537 mutex_exit(&vd->vdev_trim_lock);
8540 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) {
8541 mutex_exit(&vd->vdev_trim_lock);
8558 mutex_exit(&vd->vdev_trim_lock);
8607 txg_wait_synced(spa->spa_dsl_pool, 0);
8668 rvd = spa->spa_root_vdev;
8670 for (c = 0; c < rvd->vdev_children; c++) {
8671 vdev_t *vd = rvd->vdev_child[c];
8674 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops &&
8683 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children))
8702 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole ||
8703 spa->spa_root_vdev->vdev_child[c]->vdev_islog) {
8712 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops ==
8731 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops ||
8732 vml[c]->vdev_islog ||
8734 vml[c]->vdev_isspare ||
8735 vml[c]->vdev_isl2cache ||
8737 vml[c]->vdev_children != 0 ||
8738 vml[c]->vdev_state != VDEV_STATE_HEALTHY ||
8739 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) {
8752 vml[c]->vdev_top->vdev_ms_array);
8754 vml[c]->vdev_top->vdev_ms_shift);
8756 vml[c]->vdev_top->vdev_asize);
8758 vml[c]->vdev_top->vdev_ashift);
8760 /* transfer per-vdev ZAPs */
8761 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0);
8763 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap));
8765 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0);
8768 vml[c]->vdev_parent->vdev_top_zap));
8780 vml[c]->vdev_offline = B_TRUE;
8782 vdev_reopen(spa->spa_root_vdev);
8792 mutex_enter(&spa->spa_props_lock);
8793 fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl);
8794 mutex_exit(&spa->spa_props_lock);
8795 spa->spa_config_splitting = nvl;
8796 vdev_config_dirty(spa->spa_root_vdev);
8803 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg);
8812 newspa->spa_avz_action = AVZ_ACTION_REBUILD;
8813 newspa->spa_config_txg = spa->spa_config_txg;
8839 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
8840 mutex_enter(&vml[c]->vdev_initialize_lock);
8843 mutex_exit(&vml[c]->vdev_initialize_lock);
8845 mutex_enter(&vml[c]->vdev_trim_lock);
8847 mutex_exit(&vml[c]->vdev_trim_lock);
8857 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT;
8858 newspa->spa_is_splitting = B_TRUE;
8866 if (newspa->spa_root_vdev != NULL) {
8867 newspa->spa_config_splitting = fnvlist_alloc();
8868 fnvlist_add_uint64(newspa->spa_config_splitting,
8870 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL,
8884 vdev_config_dirty(newspa->spa_root_vdev);
8894 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
8899 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) {
8900 vdev_t *tvd = vml[c]->vdev_top;
8909 &tvd->vdev_dtl_list, vml[c], t);
8915 "vdev=%s", vml[c]->vdev_path);
8920 spa->spa_avz_action = AVZ_ACTION_REBUILD;
8921 vdev_config_dirty(spa->spa_root_vdev);
8922 spa->spa_config_splitting = NULL;
8935 newspa->spa_is_splitting = B_FALSE;
8952 /* re-online all offlined disks */
8955 vml[c]->vdev_offline = B_FALSE;
8963 vdev_reopen(spa->spa_root_vdev);
8965 nvlist_free(spa->spa_config_splitting);
8966 spa->spa_config_splitting = NULL;
8982 for (int c = 0; c < vd->vdev_children; c++) {
8983 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]);
8996 if (vd->vdev_ops == &vdev_replacing_ops) {
8997 ASSERT(vd->vdev_children > 1);
8999 newvd = vd->vdev_child[vd->vdev_children - 1];
9000 oldvd = vd->vdev_child[0];
9012 if (vd->vdev_ops == &vdev_spare_ops) {
9013 vdev_t *first = vd->vdev_child[0];
9014 vdev_t *last = vd->vdev_child[vd->vdev_children - 1];
9016 if (last->vdev_unspare) {
9019 } else if (first->vdev_unspare) {
9041 if (vd->vdev_children > 2) {
9042 newvd = vd->vdev_child[1];
9044 if (newvd->vdev_isspare && last->vdev_isspare &&
9063 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) {
9064 pvd = vd->vdev_parent;
9065 ppvd = pvd->vdev_parent;
9066 guid = vd->vdev_guid;
9067 pguid = pvd->vdev_guid;
9068 ppguid = ppvd->vdev_guid;
9075 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 &&
9076 ppvd->vdev_children == 2) {
9077 ASSERT(pvd->vdev_ops == &vdev_replacing_ops);
9078 sguid = ppvd->vdev_child[1]->vdev_guid;
9080 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd));
9116 if (!vd->vdev_ops->vdev_op_leaf)
9120 if (strcmp(value, vd->vdev_path) != 0) {
9121 spa_strfree(vd->vdev_path);
9122 vd->vdev_path = spa_strdup(value);
9126 if (vd->vdev_fru == NULL) {
9127 vd->vdev_fru = spa_strdup(value);
9129 } else if (strcmp(value, vd->vdev_fru) != 0) {
9130 spa_strfree(vd->vdev_fru);
9131 vd->vdev_fru = spa_strdup(value);
9161 if (dsl_scan_resilvering(spa->spa_dsl_pool))
9164 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd));
9171 if (dsl_scan_resilvering(spa->spa_dsl_pool))
9174 return (dsl_scan_cancel(spa->spa_dsl_pool));
9204 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) {
9213 return (dsl_scan(spa->spa_dsl_pool, func, txgstart, txgend));
9225 if (vd->vdev_remove_wanted) {
9226 vd->vdev_remove_wanted = B_FALSE;
9227 vd->vdev_delayed_close = B_FALSE;
9236 vd->vdev_stat.vs_read_errors = 0;
9237 vd->vdev_stat.vs_write_errors = 0;
9238 vd->vdev_stat.vs_checksum_errors = 0;
9240 vdev_state_dirty(vd->vdev_top);
9246 for (int c = 0; c < vd->vdev_children; c++)
9247 spa_async_remove(spa, vd->vdev_child[c], by_kernel);
9253 if (vd->vdev_fault_wanted) {
9255 vd->vdev_fault_wanted = B_FALSE;
9261 if (!vd->vdev_top->vdev_islog && vd->vdev_aux == NULL &&
9269 for (int c = 0; c < vd->vdev_children; c++)
9270 spa_async_fault_vdev(vd->vdev_child[c], suspend);
9276 if (!spa->spa_autoexpand)
9279 for (int c = 0; c < vd->vdev_children; c++) {
9280 vdev_t *cvd = vd->vdev_child[c];
9284 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL)
9287 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND);
9294 dsl_pool_t *dp = spa->spa_dsl_pool;
9297 ASSERT(spa->spa_sync_on);
9299 mutex_enter(&spa->spa_async_lock);
9300 tasks = spa->spa_async_tasks;
9301 spa->spa_async_tasks = 0;
9302 mutex_exit(&spa->spa_async_lock);
9338 (u_longlong_t)(new_space - old_space));
9350 spa_async_remove(spa, spa->spa_root_vdev, by_kernel);
9351 for (int i = 0; i < spa->spa_l2cache.sav_count; i++)
9352 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i],
9354 for (int i = 0; i < spa->spa_spares.sav_count; i++)
9355 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i],
9362 spa_async_autoexpand(spa, spa->spa_root_vdev);
9372 spa_async_fault_vdev(spa->spa_root_vdev, &suspend);
9391 !vdev_rebuild_active(spa->spa_root_vdev) &&
9393 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)))
9399 vdev_initialize_restart(spa->spa_root_vdev);
9407 vdev_trim_restart(spa->spa_root_vdev);
9445 mutex_enter(&spa->spa_async_lock);
9446 spa->spa_async_thread = NULL;
9447 cv_broadcast(&spa->spa_async_cv);
9448 mutex_exit(&spa->spa_async_lock);
9455 mutex_enter(&spa->spa_async_lock);
9456 spa->spa_async_suspended++;
9457 while (spa->spa_async_thread != NULL)
9458 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock);
9459 mutex_exit(&spa->spa_async_lock);
9463 zthr_t *condense_thread = spa->spa_condense_zthr;
9467 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr;
9471 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
9475 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
9479 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
9487 mutex_enter(&spa->spa_async_lock);
9488 ASSERT(spa->spa_async_suspended != 0);
9489 spa->spa_async_suspended--;
9490 mutex_exit(&spa->spa_async_lock);
9493 zthr_t *condense_thread = spa->spa_condense_zthr;
9497 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr;
9501 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr;
9505 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr;
9509 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
9521 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE;
9522 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE;
9523 if (spa->spa_ccw_fail_time == 0) {
9527 (gethrtime() - spa->spa_ccw_fail_time) <
9537 mutex_enter(&spa->spa_async_lock);
9539 !spa->spa_async_suspended &&
9540 spa->spa_async_thread == NULL)
9541 spa->spa_async_thread = thread_create(NULL, 0,
9543 mutex_exit(&spa->spa_async_lock);
9549 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task);
9550 mutex_enter(&spa->spa_async_lock);
9551 spa->spa_async_tasks |= task;
9552 mutex_exit(&spa->spa_async_lock);
9558 return (spa->spa_async_tasks);
9594 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp,
9595 pio->io_flags));
9633 * would be considered a no-op as spa_deferred_bpobj should
9637 * immediately) for the edge-case scenario where we just
9642 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj,
9660 * saves us a pre-read to get data we don't actually care about.
9667 memset(packed + nvsize, 0, bufsize - nvsize);
9669 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx,
9674 VERIFY0(dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db));
9676 *(uint64_t *)db->db_data = nvsize;
9688 if (!sav->sav_sync)
9696 if (sav->sav_object == 0) {
9697 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset,
9700 VERIFY(zap_update(spa->spa_meta_objset,
9702 &sav->sav_object, tx) == 0);
9706 if (sav->sav_count == 0) {
9710 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP);
9711 for (i = 0; i < sav->sav_count; i++)
9712 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i],
9715 (const nvlist_t * const *)list, sav->sav_count);
9716 for (i = 0; i < sav->sav_count; i++)
9718 kmem_free(list, sav->sav_count * sizeof (void *));
9721 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx);
9724 sav->sav_sync = B_FALSE;
9728 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t.
9729 * The all-vdev ZAP must be empty.
9734 spa_t *spa = vd->vdev_spa;
9736 if (vd->vdev_root_zap != 0 &&
9738 VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
9739 vd->vdev_root_zap, tx));
9741 if (vd->vdev_top_zap != 0) {
9742 VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
9743 vd->vdev_top_zap, tx));
9745 if (vd->vdev_leaf_zap != 0) {
9746 VERIFY0(zap_add_int(spa->spa_meta_objset, avz,
9747 vd->vdev_leaf_zap, tx));
9749 for (uint64_t i = 0; i < vd->vdev_children; i++) {
9750 spa_avz_build(vd->vdev_child[i], avz, tx);
9760 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS,
9761 * its config may not be dirty but we still need to build per-vdev ZAPs.
9765 if (list_is_empty(&spa->spa_config_dirty_list) &&
9766 spa->spa_avz_action == AVZ_ACTION_NONE)
9771 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE ||
9772 spa->spa_avz_action == AVZ_ACTION_INITIALIZE ||
9773 spa->spa_all_vdev_zaps != 0);
9775 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) {
9777 uint64_t new_avz = zap_create(spa->spa_meta_objset,
9779 spa_avz_build(spa->spa_root_vdev, new_avz, tx);
9785 for (zap_cursor_init(&zc, spa->spa_meta_objset,
9786 spa->spa_all_vdev_zaps);
9789 uint64_t vdzap = za->za_first_integer;
9790 if (zap_lookup_int(spa->spa_meta_objset, new_avz,
9796 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap,
9805 VERIFY0(zap_destroy(spa->spa_meta_objset,
9806 spa->spa_all_vdev_zaps, tx));
9809 VERIFY0(zap_update(spa->spa_meta_objset,
9813 spa->spa_all_vdev_zaps = new_avz;
9814 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) {
9819 for (zap_cursor_init(&zc, spa->spa_meta_objset,
9820 spa->spa_all_vdev_zaps);
9823 uint64_t zap = za->za_first_integer;
9824 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx));
9831 VERIFY0(zap_destroy(spa->spa_meta_objset,
9832 spa->spa_all_vdev_zaps, tx));
9833 VERIFY0(zap_remove(spa->spa_meta_objset,
9835 spa->spa_all_vdev_zaps = 0;
9838 if (spa->spa_all_vdev_zaps == 0) {
9839 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset,
9843 spa->spa_avz_action = AVZ_ACTION_NONE;
9846 vdev_construct_zaps(spa->spa_root_vdev, tx);
9848 config = spa_config_generate(spa, spa->spa_root_vdev,
9855 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version)
9857 spa->spa_uberblock.ub_version);
9861 nvlist_free(spa->spa_config_syncing);
9862 spa->spa_config_syncing = config;
9864 spa_sync_nvlist(spa, spa->spa_config_object, config, tx);
9872 spa_t *spa = dmu_tx_pool(tx)->dp_spa;
9877 ASSERT(tx->tx_txg != TXG_INITIAL);
9882 spa->spa_uberblock.ub_version = version;
9883 vdev_config_dirty(spa->spa_root_vdev);
9895 spa_t *spa = dmu_tx_pool(tx)->dp_spa;
9896 objset_t *mos = spa->spa_meta_objset;
9899 mutex_enter(&spa->spa_props_lock);
9922 * 'altroot' is a non-persistent property. It should
9925 ASSERT(spa->spa_root != NULL);
9931 * 'readonly' and 'cachefile' are also non-persistent
9937 if (spa->spa_comment != NULL)
9938 spa_strfree(spa->spa_comment);
9939 spa->spa_comment = spa_strdup(strval);
9948 if (tx->tx_txg != TXG_INITIAL) {
9949 vdev_config_dirty(spa->spa_root_vdev);
9957 if (spa->spa_compatibility != NULL)
9958 spa_strfree(spa->spa_compatibility);
9959 spa->spa_compatibility = spa_strdup(strval);
9963 if (tx->tx_txg != TXG_INITIAL) {
9964 vdev_config_dirty(spa->spa_root_vdev);
9990 if (spa->spa_pool_props_object == 0) {
9991 spa->spa_pool_props_object =
10012 spa->spa_pool_props_object,
10016 spa->spa_pool_props_object,
10031 spa->spa_pool_props_object, propname,
10039 spa->spa_delegation = intval;
10042 spa->spa_bootfs = intval;
10045 spa->spa_failmode = intval;
10048 spa->spa_autotrim = intval;
10053 spa->spa_autoexpand = intval;
10054 if (tx->tx_txg != TXG_INITIAL)
10059 spa->spa_multihost = intval;
10062 spa->spa_dedup_table_quota = intval;
10074 mutex_exit(&spa->spa_props_lock);
10078 * Perform one-time upgrade on-disk changes. spa_version() does not
10090 dsl_pool_t *dp = spa->spa_dsl_pool;
10091 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
10093 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN &&
10094 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) {
10098 spa->spa_minref += 3;
10101 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES &&
10102 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) {
10106 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES &&
10107 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) {
10111 spa->spa_minref += 3;
10114 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES &&
10115 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
10125 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) {
10140 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
10142 VERIFY0(zap_add(spa->spa_meta_objset,
10144 sizeof (spa->spa_cksum_salt.zcs_bytes),
10145 spa->spa_cksum_salt.zcs_bytes, tx));
10148 rrw_exit(&dp->dp_config_rwlock, FTAG);
10154 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping;
10155 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births;
10157 if (vd->vdev_ops == &vdev_indirect_ops) {
10165 ASSERT(vd->vdev_obsolete_sm != NULL);
10166 ASSERT(vd->vdev_removing ||
10167 vd->vdev_ops == &vdev_indirect_ops);
10171 space_map_object(vd->vdev_obsolete_sm));
10173 space_map_allocated(vd->vdev_obsolete_sm));
10175 ASSERT(vd->vdev_obsolete_segments != NULL);
10182 ASSERT0(zfs_range_tree_space(vd->vdev_obsolete_segments));
10186 * Set the top-level vdev's max queue depth. Evaluate each top-level's
10205 vdev_t *rvd = spa->spa_root_vdev;
10206 for (int c = 0; c < rvd->vdev_children; c++) {
10207 vdev_t *vd = rvd->vdev_child[c];
10220 objset_t *mos = spa->spa_meta_objset;
10221 dsl_pool_t *dp = spa->spa_dsl_pool;
10222 uint64_t txg = tx->tx_txg;
10223 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK];
10226 int pass = ++spa->spa_sync_pass;
10229 spa_sync_aux_dev(spa, &spa->spa_spares, tx,
10231 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx,
10252 &spa->spa_deferred_bpobj, tx);
10265 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg))
10271 * dsl_pool_sync() -> dp_sync_tasks may have dirtied
10273 * be a no-op. So we must sync the config to the MOS
10274 * before checking for no-op.
10300 BP_GET_LOGICAL_BIRTH(&spa->spa_uberblock.ub_rootbp) < txg &&
10304 * TXG is a no-op. Avoid syncing deferred frees, so
10305 * that we can keep this TXG as a no-op.
10307 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
10308 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
10309 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg));
10310 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg));
10323 * top-level vdevs that are known to be visible in the config cache
10330 vdev_t *rvd = spa->spa_root_vdev;
10331 uint64_t txg = tx->tx_txg;
10342 if (list_is_empty(&spa->spa_config_dirty_list)) {
10345 int children = rvd->vdev_children;
10350 rvd->vdev_child[(c0 + c) % children];
10356 if (vd->vdev_ms_array == 0 ||
10357 vd->vdev_islog ||
10367 error = vdev_config_sync(rvd->vdev_child,
10368 rvd->vdev_children, txg);
10372 spa->spa_last_synced_guid = rvd->vdev_guid;
10398 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]);
10399 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL,
10416 spa->spa_syncing_txg = txg;
10417 spa->spa_sync_pass = 0;
10424 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
10426 if (vd->vdev_aux == NULL) {
10440 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) {
10449 dsl_pool_t *dp = spa->spa_dsl_pool;
10452 spa->spa_sync_starttime = gethrtime();
10454 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
10455 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq,
10457 NSEC_TO_TICK(spa->spa_deadman_synctime));
10461 * set spa_deflate if we have no raid-z vdevs.
10463 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE &&
10464 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) {
10465 vdev_t *rvd = spa->spa_root_vdev;
10468 for (i = 0; i < rvd->vdev_children; i++) {
10469 vd = rvd->vdev_child[i];
10470 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE)
10473 if (i == rvd->vdev_children) {
10474 spa->spa_deflate = TRUE;
10475 VERIFY0(zap_add(spa->spa_meta_objset,
10477 sizeof (uint64_t), 1, &spa->spa_deflate, tx));
10488 if (!list_is_empty(&spa->spa_config_dirty_list)) {
10491 * the number of ZAPs in the per-vdev ZAP list. This only gets
10497 ASSERT0(zap_count(spa->spa_meta_objset,
10498 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count));
10499 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==,
10504 if (spa->spa_vdev_removal != NULL) {
10505 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]);
10511 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid);
10512 spa->spa_deadman_tqid = 0;
10517 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL)
10524 if (spa->spa_config_syncing != NULL) {
10525 spa_config_set(spa, spa->spa_config_syncing);
10526 spa->spa_config_txg = txg;
10527 spa->spa_config_syncing = NULL;
10535 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)))
10539 metaslab_class_evict_old(spa->spa_normal_class, txg);
10540 metaslab_class_evict_old(spa->spa_log_class, txg);
10542 metaslab_class_evict_old(spa->spa_special_class, txg);
10543 metaslab_class_evict_old(spa->spa_dedup_class, txg);
10556 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg));
10557 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg));
10558 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg));
10563 spa->spa_sync_pass = 0;
10571 spa->spa_ubsync = spa->spa_uberblock;
10610 ASSERT0P(spa->spa_sync_tq);
10611 ASSERT3S(spa->spa_alloc_count, <=, boot_ncpus);
10614 * - do not allow more allocators than cpus.
10615 * - there may be more cpus than allocators.
10616 * - do not allow more sync taskq threads than allocators or cpus.
10618 int nthreads = spa->spa_alloc_count;
10619 spa->spa_syncthreads = kmem_zalloc(sizeof (spa_syncthread_info_t) *
10622 spa->spa_sync_tq = taskq_create_synced(name, nthreads, minclsyspri,
10624 VERIFY(spa->spa_sync_tq != NULL);
10627 spa_syncthread_info_t *ti = spa->spa_syncthreads;
10629 ti->sti_thread = kthreads[i];
10630 ti->sti_allocator = i;
10634 return (spa->spa_sync_tq);
10640 ASSERT(spa->spa_sync_tq != NULL);
10642 taskq_wait(spa->spa_sync_tq);
10643 taskq_destroy(spa->spa_sync_tq);
10644 kmem_free(spa->spa_syncthreads,
10645 sizeof (spa_syncthread_info_t) * spa->spa_alloc_count);
10646 spa->spa_sync_tq = NULL;
10654 if (spa->spa_alloc_count == 1)
10657 mutex_enter(&spa->spa_allocs_use->sau_lock);
10658 uint_t r = spa->spa_allocs_use->sau_rotor;
10660 if (++r == spa->spa_alloc_count)
10662 } while (spa->spa_allocs_use->sau_inuse[r]);
10663 spa->spa_allocs_use->sau_inuse[r] = B_TRUE;
10664 spa->spa_allocs_use->sau_rotor = r;
10665 mutex_exit(&spa->spa_allocs_use->sau_lock);
10667 spa_syncthread_info_t *ti = spa->spa_syncthreads;
10668 for (i = 0; i < spa->spa_alloc_count; i++, ti++) {
10669 if (ti->sti_thread == curthread) {
10670 ti->sti_allocator = r;
10674 ASSERT3S(i, <, spa->spa_alloc_count);
10681 if (spa->spa_alloc_count > 1)
10682 spa->spa_allocs_use->sau_inuse[allocator] = B_FALSE;
10688 zbookmark_phys_t *bm = &zio->io_bookmark;
10689 spa_t *spa = zio->io_spa;
10691 ASSERT(zio->io_type == ZIO_TYPE_WRITE);
10708 if (spa->spa_sync_tq != NULL) {
10709 spa_syncthread_info_t *ti = spa->spa_syncthreads;
10710 for (int i = 0; i < spa->spa_alloc_count; i++, ti++) {
10711 if (ti->sti_thread == curthread) {
10712 zio->io_allocator = ti->sti_allocator;
10722 * into 2^20 block regions, and then hash based on the objset, object,
10725 uint64_t hv = cityhash4(bm->zb_objset, bm->zb_object, bm->zb_level,
10726 bm->zb_blkid >> 20);
10728 zio->io_allocator = (uint_t)hv % spa->spa_alloc_count;
10762 if (spa->spa_state != POOL_STATE_UNINITIALIZED) {
10777 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL)
10781 for (i = 0; i < spa->spa_l2cache.sav_count; i++) {
10782 vd = spa->spa_l2cache.sav_vdevs[i];
10783 if (vd->vdev_guid == guid)
10787 for (i = 0; i < spa->spa_spares.sav_count; i++) {
10788 vd = spa->spa_spares.sav_vdevs[i];
10789 if (vd->vdev_guid == guid)
10805 * This should only be called for a non-faulted pool, and since a
10809 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version));
10810 ASSERT3U(version, >=, spa->spa_uberblock.ub_version);
10812 spa->spa_uberblock.ub_version = version;
10813 vdev_config_dirty(spa->spa_root_vdev);
10827 for (i = 0; i < sav->sav_count; i++)
10828 if (sav->sav_vdevs[i]->vdev_guid == guid)
10831 for (i = 0; i < sav->sav_npending; i++) {
10832 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID,
10843 return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache));
10849 return (spa_has_aux_vdev(spa, guid, &spa->spa_spares));
10861 spa_aux_vdev_t *sav = &spa->spa_spares;
10863 for (i = 0; i < sav->sav_count; i++) {
10864 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool,
10876 vdev_t *rvd = spa->spa_root_vdev;
10879 for (uint64_t c = 0; c < rvd->vdev_children; c++) {
10880 vdev_t *vd = rvd->vdev_child[c];
10883 m += vd->vdev_ms_count;
10889 * Notify any waiting threads that some activity has switched from being in-
10890 * progress to not-in-progress so that the thread can wake up and determine
10900 mutex_enter(&spa->spa_activities_lock);
10901 cv_broadcast(&spa->spa_activities_cv);
10902 mutex_exit(&spa->spa_activities_lock);
10912 mutex_enter(&spa->spa_activities_lock);
10913 spa->spa_waiters_cancel = B_TRUE;
10914 cv_broadcast(&spa->spa_activities_cv);
10915 while (spa->spa_waiters != 0)
10916 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock);
10917 spa->spa_waiters_cancel = B_FALSE;
10918 mutex_exit(&spa->spa_activities_lock);
10925 spa_t *spa = vd->vdev_spa;
10928 ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
10933 &vd->vdev_initialize_lock : &vd->vdev_trim_lock;
10935 mutex_exit(&spa->spa_activities_lock);
10937 mutex_enter(&spa->spa_activities_lock);
10940 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) :
10941 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE);
10947 for (int i = 0; i < vd->vdev_children; i++) {
10948 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i],
10966 mutex_exit(&spa->spa_activities_lock);
10968 mutex_enter(&spa->spa_activities_lock);
10973 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) {
10978 vd = spa->spa_root_vdev;
10989 * ---------------------------
10993 * in-memory representation of the relevant on-disk state which can be used to
10994 * determine whether or not the activity is in progress. The in-memory state and
11000 * When the state is checked, both the activity-specific lock (if there is one)
11001 * and spa_activities_lock are held. In some cases, the activity-specific lock
11004 * thread releases the activity-specific lock and, if the activity is in
11010 * needs to hold its activity-specific lock when updating the state, and this
11021 * activity-specific lock. The order in which spa_activities_lock and the
11024 * completing thread calls spa_notify_waiters with the activity-specific lock
11025 * held, then the waiting thread must also acquire the activity-specific lock
11035 ASSERT(MUTEX_HELD(&spa->spa_activities_lock));
11047 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) ||
11057 mutex_exit(&spa->spa_activities_lock);
11059 mutex_enter(&spa->spa_activities_lock);
11061 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev);
11065 *in_progress = (spa->spa_removing_phys.sr_state ==
11069 *in_progress = vdev_rebuild_active(spa->spa_root_vdev);
11076 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
11078 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB);
11079 scanning = (scn->scn_phys.scn_state == DSS_SCANNING);
11087 vdev_raidz_expand_t *vre = spa->spa_raidz_expand;
11088 *in_progress = (vre != NULL && vre->vre_state == DSS_SCANNING);
11132 mutex_enter(&spa->spa_activities_lock);
11133 spa->spa_waiters++;
11142 if (error || !in_progress || spa->spa_waiters_cancel)
11147 if (cv_wait_sig(&spa->spa_activities_cv,
11148 &spa->spa_activities_lock) == 0) {
11154 spa->spa_waiters--;
11155 cv_signal(&spa->spa_waiters_cv);
11156 mutex_exit(&spa->spa_activities_lock);
11192 ev->resource = resource;
11205 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb);
11305 "Allow importing pool with up to this number of missing top-level "
11306 "vdevs (in read-only mode)");