Lines Matching refs:msp
352 static void metaslab_passivate(metaslab_t *msp, uint64_t weight);
353 static uint64_t metaslab_weight_from_range_tree(metaslab_t *msp);
861 metaslab_t *msp = multilist_sublist_head(mls); in metaslab_class_evict_old() local
863 while (msp != NULL) { in metaslab_class_evict_old()
864 mutex_enter(&msp->ms_lock); in metaslab_class_evict_old()
872 if (!multilist_link_active(&msp->ms_class_txg_node)) { in metaslab_class_evict_old()
873 mutex_exit(&msp->ms_lock); in metaslab_class_evict_old()
878 metaslab_t *next_msp = multilist_sublist_next(mls, msp); in metaslab_class_evict_old()
881 msp->ms_selected_txg + metaslab_unload_delay && in metaslab_class_evict_old()
882 now > msp->ms_selected_time + delay && in metaslab_class_evict_old()
883 (msp->ms_allocator == -1 || in metaslab_class_evict_old()
885 metaslab_evict(msp, txg); in metaslab_class_evict_old()
892 mutex_exit(&msp->ms_lock); in metaslab_class_evict_old()
895 mutex_exit(&msp->ms_lock); in metaslab_class_evict_old()
896 msp = next_msp; in metaslab_class_evict_old()
1182 metaslab_t *msp = mga->mga_primary; in metaslab_group_passivate() local
1183 if (msp != NULL) { in metaslab_group_passivate()
1184 mutex_enter(&msp->ms_lock); in metaslab_group_passivate()
1185 metaslab_passivate(msp, in metaslab_group_passivate()
1186 metaslab_weight_from_range_tree(msp)); in metaslab_group_passivate()
1187 mutex_exit(&msp->ms_lock); in metaslab_group_passivate()
1189 msp = mga->mga_secondary; in metaslab_group_passivate()
1190 if (msp != NULL) { in metaslab_group_passivate()
1191 mutex_enter(&msp->ms_lock); in metaslab_group_passivate()
1192 metaslab_passivate(msp, in metaslab_group_passivate()
1193 metaslab_weight_from_range_tree(msp)); in metaslab_group_passivate()
1194 mutex_exit(&msp->ms_lock); in metaslab_group_passivate()
1256 for (metaslab_t *msp = avl_first(t); in metaslab_group_histogram_verify() local
1257 msp != NULL; msp = AVL_NEXT(t, msp)) { in metaslab_group_histogram_verify()
1258 VERIFY3P(msp->ms_group, ==, mg); in metaslab_group_histogram_verify()
1260 if (msp->ms_sm == NULL) in metaslab_group_histogram_verify()
1265 msp->ms_sm->sm_phys->smp_histogram[i]; in metaslab_group_histogram_verify()
1278 metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_histogram_add() argument
1283 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_group_histogram_add()
1284 if (msp->ms_sm == NULL) in metaslab_group_histogram_add()
1293 msp->ms_sm->sm_phys->smp_histogram[i]; in metaslab_group_histogram_add()
1295 msp->ms_sm->sm_phys->smp_histogram[i]; in metaslab_group_histogram_add()
1302 metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_histogram_remove() argument
1307 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_group_histogram_remove()
1308 if (msp->ms_sm == NULL) in metaslab_group_histogram_remove()
1315 msp->ms_sm->sm_phys->smp_histogram[i]); in metaslab_group_histogram_remove()
1317 msp->ms_sm->sm_phys->smp_histogram[i]); in metaslab_group_histogram_remove()
1322 msp->ms_sm->sm_phys->smp_histogram[i]; in metaslab_group_histogram_remove()
1324 msp->ms_sm->sm_phys->smp_histogram[i]; in metaslab_group_histogram_remove()
1331 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_add() argument
1333 ASSERT(msp->ms_group == NULL); in metaslab_group_add()
1335 msp->ms_group = mg; in metaslab_group_add()
1336 msp->ms_weight = 0; in metaslab_group_add()
1337 avl_add(&mg->mg_metaslab_tree, msp); in metaslab_group_add()
1340 mutex_enter(&msp->ms_lock); in metaslab_group_add()
1341 metaslab_group_histogram_add(mg, msp); in metaslab_group_add()
1342 mutex_exit(&msp->ms_lock); in metaslab_group_add()
1346 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp) in metaslab_group_remove() argument
1348 mutex_enter(&msp->ms_lock); in metaslab_group_remove()
1349 metaslab_group_histogram_remove(mg, msp); in metaslab_group_remove()
1350 mutex_exit(&msp->ms_lock); in metaslab_group_remove()
1353 ASSERT(msp->ms_group == mg); in metaslab_group_remove()
1354 avl_remove(&mg->mg_metaslab_tree, msp); in metaslab_group_remove()
1356 metaslab_class_t *mc = msp->ms_group->mg_class; in metaslab_group_remove()
1358 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); in metaslab_group_remove()
1359 if (multilist_link_active(&msp->ms_class_txg_node)) in metaslab_group_remove()
1360 multilist_sublist_remove(mls, msp); in metaslab_group_remove()
1363 msp->ms_group = NULL; in metaslab_group_remove()
1368 metaslab_group_sort_impl(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) in metaslab_group_sort_impl() argument
1370 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_group_sort_impl()
1372 ASSERT(msp->ms_group == mg); in metaslab_group_sort_impl()
1374 avl_remove(&mg->mg_metaslab_tree, msp); in metaslab_group_sort_impl()
1375 msp->ms_weight = weight; in metaslab_group_sort_impl()
1376 avl_add(&mg->mg_metaslab_tree, msp); in metaslab_group_sort_impl()
1381 metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) in metaslab_group_sort() argument
1388 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_group_sort()
1391 metaslab_group_sort_impl(mg, msp, weight); in metaslab_group_sort()
1410 metaslab_t *msp = vd->vdev_ms[m]; in metaslab_group_fragmentation() local
1412 if (msp->ms_group != mg) in metaslab_group_fragmentation()
1415 if (msp->ms_fragmentation == ZFS_FRAG_INVALID) in metaslab_group_fragmentation()
1419 free = (msp->ms_size - metaslab_allocated_space(msp)) / in metaslab_group_fragmentation()
1422 fragmentation += msp->ms_fragmentation * free; in metaslab_group_fragmentation()
1617 metaslab_largest_allocatable(metaslab_t *msp) in metaslab_largest_allocatable() argument
1619 zfs_btree_t *t = &msp->ms_allocatable_by_size; in metaslab_largest_allocatable()
1625 metaslab_size_tree_full_load(msp->ms_allocatable); in metaslab_largest_allocatable()
1631 return (zfs_rs_get_end(rs, msp->ms_allocatable) - zfs_rs_get_start(rs, in metaslab_largest_allocatable()
1632 msp->ms_allocatable)); in metaslab_largest_allocatable()
1640 metaslab_largest_unflushed_free(metaslab_t *msp) in metaslab_largest_unflushed_free() argument
1642 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_largest_unflushed_free()
1644 if (msp->ms_unflushed_frees == NULL) in metaslab_largest_unflushed_free()
1647 if (zfs_btree_numnodes(&msp->ms_unflushed_frees_by_size) == 0) in metaslab_largest_unflushed_free()
1648 metaslab_size_tree_full_load(msp->ms_unflushed_frees); in metaslab_largest_unflushed_free()
1649 zfs_range_seg_t *rs = zfs_btree_last(&msp->ms_unflushed_frees_by_size, in metaslab_largest_unflushed_free()
1677 uint64_t rstart = zfs_rs_get_start(rs, msp->ms_unflushed_frees); in metaslab_largest_unflushed_free()
1678 uint64_t rsize = zfs_rs_get_end(rs, msp->ms_unflushed_frees) - rstart; in metaslab_largest_unflushed_free()
1682 boolean_t found = zfs_range_tree_find_in(msp->ms_defer[t], in metaslab_largest_unflushed_free()
1693 boolean_t found = zfs_range_tree_find_in(msp->ms_freed, rstart, in metaslab_largest_unflushed_free()
1771 static uint64_t metaslab_df_alloc(metaslab_t *msp, uint64_t size,
1773 static uint64_t metaslab_cf_alloc(metaslab_t *msp, uint64_t size,
1775 static uint64_t metaslab_ndf_alloc(metaslab_t *msp, uint64_t size,
1862 metaslab_df_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, in metaslab_df_alloc() argument
1873 uint64_t *cursor = &msp->ms_lbas[highbit64(align) - 1]; in metaslab_df_alloc()
1874 zfs_range_tree_t *rt = msp->ms_allocatable; in metaslab_df_alloc()
1875 uint_t free_pct = zfs_range_tree_space(rt) * 100 / msp->ms_size; in metaslab_df_alloc()
1878 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_df_alloc()
1884 if (metaslab_largest_allocatable(msp) < metaslab_df_alloc_threshold || in metaslab_df_alloc()
1887 cursor = &msp->ms_lbas[highbit64(align) - 1]; in metaslab_df_alloc()
1894 cursor = &msp->ms_lbas[highbit64(align) - 1]; in metaslab_df_alloc()
1902 if (zfs_btree_numnodes(&msp->ms_allocatable_by_size) == 0) in metaslab_df_alloc()
1903 metaslab_size_tree_full_load(msp->ms_allocatable); in metaslab_df_alloc()
1907 rs = zfs_btree_last(&msp->ms_allocatable_by_size, NULL); in metaslab_df_alloc()
1911 rs = metaslab_block_find(&msp->ms_allocatable_by_size, in metaslab_df_alloc()
1912 rt, msp->ms_start, size, max_size, &where); in metaslab_df_alloc()
1936 metaslab_cf_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, in metaslab_cf_alloc() argument
1939 zfs_range_tree_t *rt = msp->ms_allocatable; in metaslab_cf_alloc()
1940 zfs_btree_t *t = &msp->ms_allocatable_by_size; in metaslab_cf_alloc()
1941 uint64_t *cursor = &msp->ms_lbas[0]; in metaslab_cf_alloc()
1942 uint64_t *cursor_end = &msp->ms_lbas[1]; in metaslab_cf_alloc()
1945 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_cf_alloc()
1953 metaslab_size_tree_full_load(msp->ms_allocatable); in metaslab_cf_alloc()
1986 metaslab_ndf_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, in metaslab_ndf_alloc() argument
1989 zfs_btree_t *t = &msp->ms_allocatable->rt_root; in metaslab_ndf_alloc()
1990 zfs_range_tree_t *rt = msp->ms_allocatable; in metaslab_ndf_alloc()
1995 uint64_t *cursor = &msp->ms_lbas[hbit - 1]; in metaslab_ndf_alloc()
1996 uint64_t max_possible_size = metaslab_largest_allocatable(msp); in metaslab_ndf_alloc()
1998 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_ndf_alloc()
2010 cursor = &msp->ms_lbas[hbit - 1]; in metaslab_ndf_alloc()
2018 t = &msp->ms_allocatable_by_size; in metaslab_ndf_alloc()
2049 metaslab_load_wait(metaslab_t *msp) in metaslab_load_wait() argument
2051 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_load_wait()
2053 while (msp->ms_loading) { in metaslab_load_wait()
2054 ASSERT(!msp->ms_loaded); in metaslab_load_wait()
2055 cv_wait(&msp->ms_load_cv, &msp->ms_lock); in metaslab_load_wait()
2063 metaslab_flush_wait(metaslab_t *msp) in metaslab_flush_wait() argument
2065 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_flush_wait()
2067 while (msp->ms_flushing) in metaslab_flush_wait()
2068 cv_wait(&msp->ms_flush_cv, &msp->ms_lock); in metaslab_flush_wait()
2074 metaslab_t *msp = arg; in metaslab_idx_func() local
2080 return ((unsigned int)msp->ms_id % multilist_get_num_sublists(ml)); in metaslab_idx_func()
2084 metaslab_allocated_space(metaslab_t *msp) in metaslab_allocated_space() argument
2086 return (msp->ms_allocated_space); in metaslab_allocated_space()
2093 metaslab_verify_space(metaslab_t *msp, uint64_t txg) in metaslab_verify_space() argument
2095 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_verify_space()
2099 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_verify_space()
2100 ASSERT(!msp->ms_condensing); in metaslab_verify_space()
2112 if (txg != spa_syncing_txg(spa) || msp->ms_sm == NULL || in metaslab_verify_space()
2113 !msp->ms_loaded) in metaslab_verify_space()
2121 ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0); in metaslab_verify_space()
2123 ASSERT3U(space_map_allocated(msp->ms_sm), >=, in metaslab_verify_space()
2124 zfs_range_tree_space(msp->ms_unflushed_frees)); in metaslab_verify_space()
2126 ASSERT3U(metaslab_allocated_space(msp), ==, in metaslab_verify_space()
2127 space_map_allocated(msp->ms_sm) + in metaslab_verify_space()
2128 zfs_range_tree_space(msp->ms_unflushed_allocs) - in metaslab_verify_space()
2129 zfs_range_tree_space(msp->ms_unflushed_frees)); in metaslab_verify_space()
2131 sm_free_space = msp->ms_size - metaslab_allocated_space(msp); in metaslab_verify_space()
2139 zfs_range_tree_space(msp->ms_allocating[(txg + t) & in metaslab_verify_space()
2142 ASSERT3U(allocating + msp->ms_allocated_this_txg, ==, in metaslab_verify_space()
2143 msp->ms_allocating_total); in metaslab_verify_space()
2145 ASSERT3U(msp->ms_deferspace, ==, in metaslab_verify_space()
2146 zfs_range_tree_space(msp->ms_defer[0]) + in metaslab_verify_space()
2147 zfs_range_tree_space(msp->ms_defer[1])); in metaslab_verify_space()
2149 msp_free_space = zfs_range_tree_space(msp->ms_allocatable) + in metaslab_verify_space()
2150 allocating + msp->ms_deferspace + in metaslab_verify_space()
2151 zfs_range_tree_space(msp->ms_freed); in metaslab_verify_space()
2157 metaslab_aux_histograms_clear(metaslab_t *msp) in metaslab_aux_histograms_clear() argument
2163 ASSERT(msp->ms_loaded); in metaslab_aux_histograms_clear()
2165 memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist)); in metaslab_aux_histograms_clear()
2167 memset(msp->ms_deferhist[t], 0, sizeof (msp->ms_deferhist[t])); in metaslab_aux_histograms_clear()
2204 metaslab_aux_histograms_update(metaslab_t *msp) in metaslab_aux_histograms_update() argument
2206 space_map_t *sm = msp->ms_sm; in metaslab_aux_histograms_update()
2215 if (msp->ms_loaded) { in metaslab_aux_histograms_update()
2216 metaslab_aux_histograms_clear(msp); in metaslab_aux_histograms_update()
2218 metaslab_aux_histogram_add(msp->ms_synchist, in metaslab_aux_histograms_update()
2219 sm->sm_shift, msp->ms_freed); in metaslab_aux_histograms_update()
2222 metaslab_aux_histogram_add(msp->ms_deferhist[t], in metaslab_aux_histograms_update()
2223 sm->sm_shift, msp->ms_defer[t]); in metaslab_aux_histograms_update()
2227 metaslab_aux_histogram_add(msp->ms_synchist, in metaslab_aux_histograms_update()
2228 sm->sm_shift, msp->ms_freeing); in metaslab_aux_histograms_update()
2237 metaslab_aux_histograms_update_done(metaslab_t *msp, boolean_t defer_allowed) in metaslab_aux_histograms_update_done() argument
2239 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_aux_histograms_update_done()
2240 space_map_t *sm = msp->ms_sm; in metaslab_aux_histograms_update_done()
2257 memcpy(msp->ms_deferhist[hist_index], msp->ms_synchist, in metaslab_aux_histograms_update_done()
2258 sizeof (msp->ms_synchist)); in metaslab_aux_histograms_update_done()
2260 memset(msp->ms_deferhist[hist_index], 0, in metaslab_aux_histograms_update_done()
2261 sizeof (msp->ms_deferhist[hist_index])); in metaslab_aux_histograms_update_done()
2263 memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist)); in metaslab_aux_histograms_update_done()
2272 metaslab_verify_weight_and_frag(metaslab_t *msp) in metaslab_verify_weight_and_frag() argument
2274 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_verify_weight_and_frag()
2288 if (msp->ms_group == NULL) in metaslab_verify_weight_and_frag()
2296 vdev_t *vd = msp->ms_group->mg_vd; in metaslab_verify_weight_and_frag()
2306 if (txg_list_member(&vd->vdev_ms_list, msp, t)) in metaslab_verify_weight_and_frag()
2315 if (!spa_writeable(msp->ms_group->mg_vd->vdev_spa)) in metaslab_verify_weight_and_frag()
2319 if (msp->ms_loaded) { in metaslab_verify_weight_and_frag()
2320 zfs_range_tree_stat_verify(msp->ms_allocatable); in metaslab_verify_weight_and_frag()
2321 VERIFY(space_map_histogram_verify(msp->ms_sm, in metaslab_verify_weight_and_frag()
2322 msp->ms_allocatable)); in metaslab_verify_weight_and_frag()
2325 uint64_t weight = msp->ms_weight; in metaslab_verify_weight_and_frag()
2326 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK; in metaslab_verify_weight_and_frag()
2327 boolean_t space_based = WEIGHT_IS_SPACEBASED(msp->ms_weight); in metaslab_verify_weight_and_frag()
2328 uint64_t frag = msp->ms_fragmentation; in metaslab_verify_weight_and_frag()
2329 uint64_t max_segsize = msp->ms_max_size; in metaslab_verify_weight_and_frag()
2331 msp->ms_weight = 0; in metaslab_verify_weight_and_frag()
2332 msp->ms_fragmentation = 0; in metaslab_verify_weight_and_frag()
2347 msp->ms_weight = metaslab_weight(msp, B_TRUE) | was_active; in metaslab_verify_weight_and_frag()
2349 VERIFY3U(max_segsize, ==, msp->ms_max_size); in metaslab_verify_weight_and_frag()
2355 if ((space_based && !WEIGHT_IS_SPACEBASED(msp->ms_weight)) || in metaslab_verify_weight_and_frag()
2356 (!space_based && WEIGHT_IS_SPACEBASED(msp->ms_weight))) { in metaslab_verify_weight_and_frag()
2357 msp->ms_fragmentation = frag; in metaslab_verify_weight_and_frag()
2358 msp->ms_weight = weight; in metaslab_verify_weight_and_frag()
2362 VERIFY3U(msp->ms_fragmentation, ==, frag); in metaslab_verify_weight_and_frag()
2363 VERIFY3U(msp->ms_weight, ==, weight); in metaslab_verify_weight_and_frag()
2389 metaslab_t *msp = multilist_sublist_head(mls); in metaslab_potentially_evict() local
2391 while (msp != NULL && allmem * zfs_metaslab_mem_limit / 100 < in metaslab_potentially_evict()
2396 metaslab_idx_func(&mc->mc_metaslab_txg_list, msp)); in metaslab_potentially_evict()
2398 if (!multilist_link_active(&msp->ms_class_txg_node)) { in metaslab_potentially_evict()
2402 metaslab_t *next_msp = multilist_sublist_next(mls, msp); in metaslab_potentially_evict()
2415 if (msp->ms_loading) { in metaslab_potentially_evict()
2416 msp = next_msp; in metaslab_potentially_evict()
2433 mutex_enter(&msp->ms_lock); in metaslab_potentially_evict()
2434 if (msp->ms_allocator == -1 && msp->ms_sm != NULL && in metaslab_potentially_evict()
2435 msp->ms_allocating_total == 0) { in metaslab_potentially_evict()
2436 metaslab_unload(msp); in metaslab_potentially_evict()
2438 mutex_exit(&msp->ms_lock); in metaslab_potentially_evict()
2439 msp = next_msp; in metaslab_potentially_evict()
2449 metaslab_load_impl(metaslab_t *msp) in metaslab_load_impl() argument
2453 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_load_impl()
2454 ASSERT(msp->ms_loading); in metaslab_load_impl()
2455 ASSERT(!msp->ms_condensing); in metaslab_load_impl()
2481 uint64_t length = msp->ms_synced_length; in metaslab_load_impl()
2482 mutex_exit(&msp->ms_lock); in metaslab_load_impl()
2486 if (msp->ms_allocatable->rt_arg == NULL) { in metaslab_load_impl()
2489 mrap = msp->ms_allocatable->rt_arg; in metaslab_load_impl()
2490 msp->ms_allocatable->rt_ops = NULL; in metaslab_load_impl()
2491 msp->ms_allocatable->rt_arg = NULL; in metaslab_load_impl()
2493 mrap->mra_bt = &msp->ms_allocatable_by_size; in metaslab_load_impl()
2496 if (msp->ms_sm != NULL) { in metaslab_load_impl()
2497 error = space_map_load_length(msp->ms_sm, msp->ms_allocatable, in metaslab_load_impl()
2501 metaslab_rt_create(msp->ms_allocatable, mrap); in metaslab_load_impl()
2502 msp->ms_allocatable->rt_ops = &metaslab_rt_ops; in metaslab_load_impl()
2503 msp->ms_allocatable->rt_arg = mrap; in metaslab_load_impl()
2506 arg.rt = msp->ms_allocatable; in metaslab_load_impl()
2508 zfs_range_tree_walk(msp->ms_allocatable, in metaslab_load_impl()
2515 metaslab_rt_create(msp->ms_allocatable, mrap); in metaslab_load_impl()
2516 msp->ms_allocatable->rt_ops = &metaslab_rt_ops; in metaslab_load_impl()
2517 msp->ms_allocatable->rt_arg = mrap; in metaslab_load_impl()
2523 zfs_range_tree_add(msp->ms_allocatable, in metaslab_load_impl()
2524 msp->ms_start, msp->ms_size); in metaslab_load_impl()
2526 if (msp->ms_new) { in metaslab_load_impl()
2535 msp->ms_unflushed_allocs)); in metaslab_load_impl()
2537 msp->ms_unflushed_frees)); in metaslab_load_impl()
2548 mutex_enter(&msp->ms_sync_lock); in metaslab_load_impl()
2549 mutex_enter(&msp->ms_lock); in metaslab_load_impl()
2551 ASSERT(!msp->ms_condensing); in metaslab_load_impl()
2552 ASSERT(!msp->ms_flushing); in metaslab_load_impl()
2555 mutex_exit(&msp->ms_sync_lock); in metaslab_load_impl()
2559 ASSERT3P(msp->ms_group, !=, NULL); in metaslab_load_impl()
2560 msp->ms_loaded = B_TRUE; in metaslab_load_impl()
2567 zfs_range_tree_walk(msp->ms_unflushed_allocs, in metaslab_load_impl()
2568 zfs_range_tree_remove, msp->ms_allocatable); in metaslab_load_impl()
2569 zfs_range_tree_walk(msp->ms_unflushed_frees, in metaslab_load_impl()
2570 zfs_range_tree_add, msp->ms_allocatable); in metaslab_load_impl()
2572 ASSERT3P(msp->ms_group, !=, NULL); in metaslab_load_impl()
2573 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_load_impl()
2598 zfs_range_tree_walk(msp->ms_freed, in metaslab_load_impl()
2599 zfs_range_tree_remove, msp->ms_allocatable); in metaslab_load_impl()
2617 zfs_range_tree_walk(msp->ms_defer[t], in metaslab_load_impl()
2618 zfs_range_tree_remove, msp->ms_allocatable); in metaslab_load_impl()
2633 uint64_t weight = msp->ms_weight; in metaslab_load_impl()
2634 uint64_t max_size = msp->ms_max_size; in metaslab_load_impl()
2635 metaslab_recalculate_weight_and_sort(msp); in metaslab_load_impl()
2637 ASSERT3U(weight, <=, msp->ms_weight); in metaslab_load_impl()
2638 msp->ms_max_size = metaslab_largest_allocatable(msp); in metaslab_load_impl()
2639 ASSERT3U(max_size, <=, msp->ms_max_size); in metaslab_load_impl()
2641 msp->ms_load_time = load_end; in metaslab_load_impl()
2650 msp->ms_group->mg_class->mc_name, in metaslab_load_impl()
2651 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, in metaslab_load_impl()
2652 (u_longlong_t)msp->ms_id, in metaslab_load_impl()
2653 (u_longlong_t)space_map_length(msp->ms_sm), in metaslab_load_impl()
2654 (u_longlong_t)zfs_range_tree_space(msp->ms_unflushed_allocs), in metaslab_load_impl()
2655 (u_longlong_t)zfs_range_tree_space(msp->ms_unflushed_frees), in metaslab_load_impl()
2656 (u_longlong_t)zfs_range_tree_space(msp->ms_freed), in metaslab_load_impl()
2657 (u_longlong_t)zfs_range_tree_space(msp->ms_defer[0]), in metaslab_load_impl()
2658 (u_longlong_t)zfs_range_tree_space(msp->ms_defer[1]), in metaslab_load_impl()
2659 (longlong_t)((load_start - msp->ms_unload_time) / 1000000), in metaslab_load_impl()
2661 (u_longlong_t)msp->ms_max_size, in metaslab_load_impl()
2662 (u_longlong_t)msp->ms_max_size - max_size, in metaslab_load_impl()
2663 (u_longlong_t)weight, (u_longlong_t)msp->ms_weight); in metaslab_load_impl()
2665 metaslab_verify_space(msp, spa_syncing_txg(spa)); in metaslab_load_impl()
2666 mutex_exit(&msp->ms_sync_lock); in metaslab_load_impl()
2671 metaslab_load(metaslab_t *msp) in metaslab_load() argument
2673 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_load()
2679 metaslab_load_wait(msp); in metaslab_load()
2680 if (msp->ms_loaded) in metaslab_load()
2682 VERIFY(!msp->ms_loading); in metaslab_load()
2683 ASSERT(!msp->ms_condensing); in metaslab_load()
2691 msp->ms_loading = B_TRUE; in metaslab_load()
2698 if (msp->ms_flushing) in metaslab_load()
2699 metaslab_flush_wait(msp); in metaslab_load()
2706 ASSERT(!msp->ms_loaded); in metaslab_load()
2713 if (spa_normal_class(msp->ms_group->mg_class->mc_spa) == in metaslab_load()
2714 msp->ms_group->mg_class) { in metaslab_load()
2715 metaslab_potentially_evict(msp->ms_group->mg_class); in metaslab_load()
2718 int error = metaslab_load_impl(msp); in metaslab_load()
2720 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_load()
2721 msp->ms_loading = B_FALSE; in metaslab_load()
2722 cv_broadcast(&msp->ms_load_cv); in metaslab_load()
2728 metaslab_unload(metaslab_t *msp) in metaslab_unload() argument
2730 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_unload()
2737 if (!msp->ms_loaded) in metaslab_unload()
2740 zfs_range_tree_vacate(msp->ms_allocatable, NULL, NULL); in metaslab_unload()
2741 msp->ms_loaded = B_FALSE; in metaslab_unload()
2742 msp->ms_unload_time = gethrtime(); in metaslab_unload()
2744 msp->ms_activation_weight = 0; in metaslab_unload()
2745 msp->ms_weight &= ~METASLAB_ACTIVE_MASK; in metaslab_unload()
2747 if (msp->ms_group != NULL) { in metaslab_unload()
2748 metaslab_class_t *mc = msp->ms_group->mg_class; in metaslab_unload()
2750 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); in metaslab_unload()
2751 if (multilist_link_active(&msp->ms_class_txg_node)) in metaslab_unload()
2752 multilist_sublist_remove(mls, msp); in metaslab_unload()
2755 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_unload()
2761 msp->ms_group->mg_class->mc_name, in metaslab_unload()
2762 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, in metaslab_unload()
2763 (u_longlong_t)msp->ms_id, in metaslab_unload()
2764 (u_longlong_t)msp->ms_weight, in metaslab_unload()
2765 (u_longlong_t)msp->ms_selected_txg, in metaslab_unload()
2766 (u_longlong_t)(NSEC2SEC(msp->ms_unload_time) - in metaslab_unload()
2767 msp->ms_selected_time), in metaslab_unload()
2768 (u_longlong_t)msp->ms_alloc_txg, in metaslab_unload()
2769 (u_longlong_t)(msp->ms_unload_time - in metaslab_unload()
2770 msp->ms_load_time) / 1000 / 1000, in metaslab_unload()
2771 (u_longlong_t)msp->ms_max_size); in metaslab_unload()
2786 if (msp->ms_group != NULL) in metaslab_unload()
2787 metaslab_recalculate_weight_and_sort(msp); in metaslab_unload()
2798 metaslab_calculate_range_tree_type(vdev_t *vdev, metaslab_t *msp, in metaslab_calculate_range_tree_type() argument
2804 *start = msp->ms_start; in metaslab_calculate_range_tree_type()
2814 metaslab_set_selected_txg(metaslab_t *msp, uint64_t txg) in metaslab_set_selected_txg() argument
2816 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_set_selected_txg()
2817 metaslab_class_t *mc = msp->ms_group->mg_class; in metaslab_set_selected_txg()
2819 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); in metaslab_set_selected_txg()
2820 if (multilist_link_active(&msp->ms_class_txg_node)) in metaslab_set_selected_txg()
2821 multilist_sublist_remove(mls, msp); in metaslab_set_selected_txg()
2822 msp->ms_selected_txg = txg; in metaslab_set_selected_txg()
2823 msp->ms_selected_time = gethrestime_sec(); in metaslab_set_selected_txg()
2824 multilist_sublist_insert_tail(mls, msp); in metaslab_set_selected_txg()
2843 uint64_t txg, metaslab_t **msp) in metaslab_init() argument
2946 *msp = ms; in metaslab_init()
2952 metaslab_fini_flush_data(metaslab_t *msp) in metaslab_fini_flush_data() argument
2954 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_fini_flush_data()
2956 if (metaslab_unflushed_txg(msp) == 0) { in metaslab_fini_flush_data()
2957 ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), in metaslab_fini_flush_data()
2964 avl_remove(&spa->spa_metaslabs_by_flushed, msp); in metaslab_fini_flush_data()
2967 spa_log_sm_decrement_mscount(spa, metaslab_unflushed_txg(msp)); in metaslab_fini_flush_data()
2968 spa_log_summary_decrement_mscount(spa, metaslab_unflushed_txg(msp), in metaslab_fini_flush_data()
2969 metaslab_unflushed_dirty(msp)); in metaslab_fini_flush_data()
2981 metaslab_fini(metaslab_t *msp) in metaslab_fini() argument
2983 metaslab_group_t *mg = msp->ms_group; in metaslab_fini()
2987 metaslab_fini_flush_data(msp); in metaslab_fini()
2989 metaslab_group_remove(mg, msp); in metaslab_fini()
2991 mutex_enter(&msp->ms_lock); in metaslab_fini()
2992 VERIFY(msp->ms_group == NULL); in metaslab_fini()
2999 if (!msp->ms_new) { in metaslab_fini()
3001 -metaslab_allocated_space(msp), 0, -msp->ms_size); in metaslab_fini()
3004 space_map_close(msp->ms_sm); in metaslab_fini()
3005 msp->ms_sm = NULL; in metaslab_fini()
3007 metaslab_unload(msp); in metaslab_fini()
3009 zfs_range_tree_destroy(msp->ms_allocatable); in metaslab_fini()
3010 zfs_range_tree_destroy(msp->ms_freeing); in metaslab_fini()
3011 zfs_range_tree_destroy(msp->ms_freed); in metaslab_fini()
3014 metaslab_unflushed_changes_memused(msp)); in metaslab_fini()
3016 metaslab_unflushed_changes_memused(msp); in metaslab_fini()
3017 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); in metaslab_fini()
3018 zfs_range_tree_destroy(msp->ms_unflushed_allocs); in metaslab_fini()
3019 zfs_range_tree_destroy(msp->ms_checkpointing); in metaslab_fini()
3020 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); in metaslab_fini()
3021 zfs_range_tree_destroy(msp->ms_unflushed_frees); in metaslab_fini()
3024 zfs_range_tree_destroy(msp->ms_allocating[t]); in metaslab_fini()
3027 zfs_range_tree_destroy(msp->ms_defer[t]); in metaslab_fini()
3029 ASSERT0(msp->ms_deferspace); in metaslab_fini()
3032 ASSERT(!txg_list_member(&vd->vdev_ms_list, msp, t)); in metaslab_fini()
3034 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); in metaslab_fini()
3035 zfs_range_tree_destroy(msp->ms_trim); in metaslab_fini()
3037 mutex_exit(&msp->ms_lock); in metaslab_fini()
3038 cv_destroy(&msp->ms_load_cv); in metaslab_fini()
3039 cv_destroy(&msp->ms_flush_cv); in metaslab_fini()
3040 mutex_destroy(&msp->ms_lock); in metaslab_fini()
3041 mutex_destroy(&msp->ms_sync_lock); in metaslab_fini()
3042 ASSERT3U(msp->ms_allocator, ==, -1); in metaslab_fini()
3044 kmem_free(msp, sizeof (metaslab_t)); in metaslab_fini()
3099 metaslab_set_fragmentation(metaslab_t *msp, boolean_t nodirty) in metaslab_set_fragmentation() argument
3101 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_set_fragmentation()
3108 msp->ms_fragmentation = ZFS_FRAG_INVALID; in metaslab_set_fragmentation()
3116 if (msp->ms_sm == NULL) { in metaslab_set_fragmentation()
3117 msp->ms_fragmentation = 0; in metaslab_set_fragmentation()
3125 if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) { in metaslab_set_fragmentation()
3127 vdev_t *vd = msp->ms_group->mg_vd; in metaslab_set_fragmentation()
3139 msp->ms_condense_wanted = B_TRUE; in metaslab_set_fragmentation()
3140 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); in metaslab_set_fragmentation()
3143 (u_longlong_t)msp->ms_id, in metaslab_set_fragmentation()
3146 msp->ms_fragmentation = ZFS_FRAG_INVALID; in metaslab_set_fragmentation()
3152 uint8_t shift = msp->ms_sm->sm_shift; in metaslab_set_fragmentation()
3157 if (msp->ms_sm->sm_phys->smp_histogram[i] == 0) in metaslab_set_fragmentation()
3160 space = msp->ms_sm->sm_phys->smp_histogram[i] << (i + shift); in metaslab_set_fragmentation()
3171 msp->ms_fragmentation = fragmentation; in metaslab_set_fragmentation()
3180 metaslab_space_weight(metaslab_t *msp) in metaslab_space_weight() argument
3182 metaslab_group_t *mg = msp->ms_group; in metaslab_space_weight()
3186 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_space_weight()
3191 space = msp->ms_size - metaslab_allocated_space(msp); in metaslab_space_weight()
3194 msp->ms_fragmentation != ZFS_FRAG_INVALID) { in metaslab_space_weight()
3202 space = (space * (100 - (msp->ms_fragmentation - 1))) / 100; in metaslab_space_weight()
3226 weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count; in metaslab_space_weight()
3236 if (msp->ms_loaded && msp->ms_fragmentation != ZFS_FRAG_INVALID && in metaslab_space_weight()
3237 msp->ms_fragmentation <= zfs_metaslab_fragmentation_threshold) { in metaslab_space_weight()
3238 weight |= (msp->ms_weight & METASLAB_ACTIVE_MASK); in metaslab_space_weight()
3252 metaslab_weight_from_range_tree(metaslab_t *msp) in metaslab_weight_from_range_tree() argument
3257 ASSERT(msp->ms_loaded); in metaslab_weight_from_range_tree()
3261 uint8_t shift = msp->ms_group->mg_vd->vdev_ashift; in metaslab_weight_from_range_tree()
3265 segments += msp->ms_allocatable->rt_histogram[i]; in metaslab_weight_from_range_tree()
3293 metaslab_weight_from_spacemap(metaslab_t *msp) in metaslab_weight_from_spacemap() argument
3295 space_map_t *sm = msp->ms_sm; in metaslab_weight_from_spacemap()
3296 ASSERT(!msp->ms_loaded); in metaslab_weight_from_spacemap()
3311 deferspace_histogram[i] += msp->ms_synchist[i]; in metaslab_weight_from_spacemap()
3314 deferspace_histogram[i] += msp->ms_deferhist[t][i]; in metaslab_weight_from_spacemap()
3340 metaslab_segment_weight(metaslab_t *msp) in metaslab_segment_weight() argument
3342 metaslab_group_t *mg = msp->ms_group; in metaslab_segment_weight()
3346 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_segment_weight()
3351 if (metaslab_allocated_space(msp) == 0) { in metaslab_segment_weight()
3352 int idx = highbit64(msp->ms_size) - 1; in metaslab_segment_weight()
3367 ASSERT3U(msp->ms_sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t)); in metaslab_segment_weight()
3372 if (metaslab_allocated_space(msp) == msp->ms_size) in metaslab_segment_weight()
3379 if (msp->ms_loaded) { in metaslab_segment_weight()
3380 weight = metaslab_weight_from_range_tree(msp); in metaslab_segment_weight()
3382 weight = metaslab_weight_from_spacemap(msp); in metaslab_segment_weight()
3390 if (msp->ms_activation_weight != 0 && weight != 0) in metaslab_segment_weight()
3391 WEIGHT_SET_ACTIVE(weight, WEIGHT_GET_ACTIVE(msp->ms_weight)); in metaslab_segment_weight()
3405 metaslab_should_allocate(metaslab_t *msp, uint64_t asize, boolean_t try_hard) in metaslab_should_allocate() argument
3413 if (unlikely(msp->ms_new)) in metaslab_should_allocate()
3423 if (msp->ms_loaded || in metaslab_should_allocate()
3424 (msp->ms_max_size != 0 && !try_hard && gethrtime() < in metaslab_should_allocate()
3425 msp->ms_unload_time + SEC2NSEC(zfs_metaslab_max_size_cache_sec))) in metaslab_should_allocate()
3426 return (msp->ms_max_size >= asize); in metaslab_should_allocate()
3429 if (!WEIGHT_IS_SPACEBASED(msp->ms_weight)) { in metaslab_should_allocate()
3437 1ULL << (WEIGHT_GET_INDEX(msp->ms_weight) + 1)); in metaslab_should_allocate()
3440 (msp->ms_weight & ~METASLAB_WEIGHT_TYPE)); in metaslab_should_allocate()
3447 metaslab_weight(metaslab_t *msp, boolean_t nodirty) in metaslab_weight() argument
3449 vdev_t *vd = msp->ms_group->mg_vd; in metaslab_weight()
3453 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_weight()
3455 metaslab_set_fragmentation(msp, nodirty); in metaslab_weight()
3467 if (msp->ms_loaded) { in metaslab_weight()
3468 msp->ms_max_size = metaslab_largest_allocatable(msp); in metaslab_weight()
3470 msp->ms_max_size = MAX(msp->ms_max_size, in metaslab_weight()
3471 metaslab_largest_unflushed_free(msp)); in metaslab_weight()
3479 (msp->ms_sm == NULL || msp->ms_sm->sm_dbuf->db_size == in metaslab_weight()
3481 weight = metaslab_segment_weight(msp); in metaslab_weight()
3483 weight = metaslab_space_weight(msp); in metaslab_weight()
3489 metaslab_recalculate_weight_and_sort(metaslab_t *msp) in metaslab_recalculate_weight_and_sort() argument
3491 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_recalculate_weight_and_sort()
3494 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK; in metaslab_recalculate_weight_and_sort()
3495 metaslab_group_sort(msp->ms_group, msp, in metaslab_recalculate_weight_and_sort()
3496 metaslab_weight(msp, B_FALSE) | was_active); in metaslab_recalculate_weight_and_sort()
3500 metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp, in metaslab_activate_allocator() argument
3504 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_activate_allocator()
3511 ASSERT0(msp->ms_activation_weight); in metaslab_activate_allocator()
3512 msp->ms_activation_weight = msp->ms_weight; in metaslab_activate_allocator()
3513 metaslab_group_sort(mg, msp, msp->ms_weight | in metaslab_activate_allocator()
3527 *mspp = msp; in metaslab_activate_allocator()
3528 ASSERT3S(msp->ms_allocator, ==, -1); in metaslab_activate_allocator()
3529 msp->ms_allocator = allocator; in metaslab_activate_allocator()
3530 msp->ms_primary = (activation_weight == METASLAB_WEIGHT_PRIMARY); in metaslab_activate_allocator()
3532 ASSERT0(msp->ms_activation_weight); in metaslab_activate_allocator()
3533 msp->ms_activation_weight = msp->ms_weight; in metaslab_activate_allocator()
3534 metaslab_group_sort_impl(mg, msp, in metaslab_activate_allocator()
3535 msp->ms_weight | activation_weight); in metaslab_activate_allocator()
3542 metaslab_activate(metaslab_t *msp, int allocator, uint64_t activation_weight) in metaslab_activate() argument
3544 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_activate()
3557 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) { in metaslab_activate()
3558 ASSERT(msp->ms_loaded); in metaslab_activate()
3562 int error = metaslab_load(msp); in metaslab_activate()
3564 metaslab_group_sort(msp->ms_group, msp, 0); in metaslab_activate()
3583 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) { in metaslab_activate()
3584 if (msp->ms_allocator != allocator) in metaslab_activate()
3587 if ((msp->ms_weight & activation_weight) == 0) in metaslab_activate()
3591 msp->ms_primary); in metaslab_activate()
3602 if (msp->ms_weight == 0) { in metaslab_activate()
3603 ASSERT0(zfs_range_tree_space(msp->ms_allocatable)); in metaslab_activate()
3607 if ((error = metaslab_activate_allocator(msp->ms_group, msp, in metaslab_activate()
3612 ASSERT(msp->ms_loaded); in metaslab_activate()
3613 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); in metaslab_activate()
3619 metaslab_passivate_allocator(metaslab_group_t *mg, metaslab_t *msp, in metaslab_passivate_allocator() argument
3622 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_passivate_allocator()
3623 ASSERT(msp->ms_loaded); in metaslab_passivate_allocator()
3625 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) { in metaslab_passivate_allocator()
3626 metaslab_group_sort(mg, msp, weight); in metaslab_passivate_allocator()
3631 ASSERT3P(msp->ms_group, ==, mg); in metaslab_passivate_allocator()
3632 ASSERT3S(0, <=, msp->ms_allocator); in metaslab_passivate_allocator()
3633 ASSERT3U(msp->ms_allocator, <, mg->mg_class->mc_spa->spa_alloc_count); in metaslab_passivate_allocator()
3635 metaslab_group_allocator_t *mga = &mg->mg_allocator[msp->ms_allocator]; in metaslab_passivate_allocator()
3636 if (msp->ms_primary) { in metaslab_passivate_allocator()
3637 ASSERT3P(mga->mga_primary, ==, msp); in metaslab_passivate_allocator()
3638 ASSERT(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); in metaslab_passivate_allocator()
3641 ASSERT3P(mga->mga_secondary, ==, msp); in metaslab_passivate_allocator()
3642 ASSERT(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); in metaslab_passivate_allocator()
3645 msp->ms_allocator = -1; in metaslab_passivate_allocator()
3646 metaslab_group_sort_impl(mg, msp, weight); in metaslab_passivate_allocator()
3651 metaslab_passivate(metaslab_t *msp, uint64_t weight) in metaslab_passivate() argument
3660 ASSERT(!WEIGHT_IS_SPACEBASED(msp->ms_weight) || in metaslab_passivate()
3662 zfs_range_tree_space(msp->ms_allocatable) == 0); in metaslab_passivate()
3665 ASSERT(msp->ms_activation_weight != 0); in metaslab_passivate()
3666 msp->ms_activation_weight = 0; in metaslab_passivate()
3667 metaslab_passivate_allocator(msp->ms_group, msp, weight); in metaslab_passivate()
3668 ASSERT0(msp->ms_weight & METASLAB_ACTIVE_MASK); in metaslab_passivate()
3683 metaslab_segment_may_passivate(metaslab_t *msp) in metaslab_segment_may_passivate() argument
3685 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_segment_may_passivate()
3687 if (WEIGHT_IS_SPACEBASED(msp->ms_weight) || spa_sync_pass(spa) > 1) in metaslab_segment_may_passivate()
3695 if (metaslab_largest_allocatable(msp) > in metaslab_segment_may_passivate()
3696 zfs_range_tree_space(msp->ms_allocatable) * 15 / 16) in metaslab_segment_may_passivate()
3704 uint64_t weight = metaslab_weight_from_range_tree(msp); in metaslab_segment_may_passivate()
3705 int activation_idx = WEIGHT_GET_INDEX(msp->ms_activation_weight); in metaslab_segment_may_passivate()
3709 metaslab_passivate(msp, weight); in metaslab_segment_may_passivate()
3715 metaslab_t *msp = arg; in metaslab_preload() local
3716 metaslab_class_t *mc = msp->ms_group->mg_class; in metaslab_preload()
3720 ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock)); in metaslab_preload()
3722 mutex_enter(&msp->ms_lock); in metaslab_preload()
3723 (void) metaslab_load(msp); in metaslab_preload()
3724 metaslab_set_selected_txg(msp, spa_syncing_txg(spa)); in metaslab_preload()
3725 mutex_exit(&msp->ms_lock); in metaslab_preload()
3733 metaslab_t *msp; in metaslab_group_preload() local
3745 for (msp = avl_first(t); msp != NULL; msp = AVL_NEXT(t, msp)) { in metaslab_group_preload()
3746 ASSERT3P(msp->ms_group, ==, mg); in metaslab_group_preload()
3754 if (++m > metaslab_preload_limit && !msp->ms_condense_wanted) { in metaslab_group_preload()
3759 msp, TQ_SLEEP | (m <= spa->spa_alloc_count ? TQ_FRONT : 0)) in metaslab_group_preload()
3787 metaslab_should_condense(metaslab_t *msp) in metaslab_should_condense() argument
3789 space_map_t *sm = msp->ms_sm; in metaslab_should_condense()
3790 vdev_t *vd = msp->ms_group->mg_vd; in metaslab_should_condense()
3793 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_should_condense()
3794 ASSERT(msp->ms_loaded); in metaslab_should_condense()
3802 if (zfs_range_tree_numsegs(msp->ms_allocatable) == 0 || in metaslab_should_condense()
3803 msp->ms_condense_wanted) in metaslab_should_condense()
3809 msp->ms_allocatable, SM_NO_VDEVID); in metaslab_should_condense()
3824 metaslab_condense(metaslab_t *msp, dmu_tx_t *tx) in metaslab_condense() argument
3827 space_map_t *sm = msp->ms_sm; in metaslab_condense()
3829 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_condense()
3831 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_condense()
3832 ASSERT(msp->ms_loaded); in metaslab_condense()
3833 ASSERT(msp->ms_sm != NULL); in metaslab_condense()
3878 ASSERT(zfs_range_tree_is_empty(msp->ms_freed)); /* since it is pass 1 */ in metaslab_condense()
3882 (u_longlong_t)txg, (u_longlong_t)msp->ms_id, msp, in metaslab_condense()
3883 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, in metaslab_condense()
3884 spa->spa_name, (u_longlong_t)space_map_length(msp->ms_sm), in metaslab_condense()
3885 (u_longlong_t)zfs_range_tree_numsegs(msp->ms_allocatable), in metaslab_condense()
3886 msp->ms_condense_wanted ? "TRUE" : "FALSE"); in metaslab_condense()
3888 msp->ms_condense_wanted = B_FALSE; in metaslab_condense()
3892 type = metaslab_calculate_range_tree_type(msp->ms_group->mg_vd, msp, in metaslab_condense()
3898 zfs_range_tree_walk(msp->ms_defer[t], in metaslab_condense()
3903 zfs_range_tree_walk(msp->ms_allocating[(txg + t) & TXG_MASK], in metaslab_condense()
3908 metaslab_unflushed_changes_memused(msp)); in metaslab_condense()
3910 metaslab_unflushed_changes_memused(msp); in metaslab_condense()
3911 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); in metaslab_condense()
3912 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); in metaslab_condense()
3922 msp->ms_condensing = B_TRUE; in metaslab_condense()
3924 mutex_exit(&msp->ms_lock); in metaslab_condense()
3925 uint64_t object = space_map_object(msp->ms_sm); in metaslab_condense()
3934 if (space_map_object(msp->ms_sm) != object) { in metaslab_condense()
3935 object = space_map_object(msp->ms_sm); in metaslab_condense()
3937 msp->ms_group->mg_vd->vdev_ms_array, sizeof (uint64_t) * in metaslab_condense()
3938 msp->ms_id, sizeof (uint64_t), &object, tx); in metaslab_condense()
3954 zfs_range_tree_add(tmp_tree, msp->ms_start, msp->ms_size); in metaslab_condense()
3956 space_map_write(sm, msp->ms_allocatable, SM_FREE, SM_NO_VDEVID, tx); in metaslab_condense()
3963 mutex_enter(&msp->ms_lock); in metaslab_condense()
3965 msp->ms_condensing = B_FALSE; in metaslab_condense()
3966 metaslab_flush_update(msp, tx); in metaslab_condense()
3970 metaslab_unflushed_add(metaslab_t *msp, dmu_tx_t *tx) in metaslab_unflushed_add() argument
3972 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_unflushed_add()
3974 ASSERT(msp->ms_sm != NULL); in metaslab_unflushed_add()
3975 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); in metaslab_unflushed_add()
3976 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); in metaslab_unflushed_add()
3979 metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx); in metaslab_unflushed_add()
3980 metaslab_set_unflushed_dirty(msp, B_TRUE); in metaslab_unflushed_add()
3981 avl_add(&spa->spa_metaslabs_by_flushed, msp); in metaslab_unflushed_add()
3989 metaslab_unflushed_bump(metaslab_t *msp, dmu_tx_t *tx, boolean_t dirty) in metaslab_unflushed_bump() argument
3991 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_unflushed_bump()
3993 ASSERT(msp->ms_sm != NULL); in metaslab_unflushed_bump()
3994 ASSERT(metaslab_unflushed_txg(msp) != 0); in metaslab_unflushed_bump()
3995 ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), ==, msp); in metaslab_unflushed_bump()
3996 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); in metaslab_unflushed_bump()
3997 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); in metaslab_unflushed_bump()
4002 uint64_t ms_prev_flushed_txg = metaslab_unflushed_txg(msp); in metaslab_unflushed_bump()
4003 boolean_t ms_prev_flushed_dirty = metaslab_unflushed_dirty(msp); in metaslab_unflushed_bump()
4005 avl_remove(&spa->spa_metaslabs_by_flushed, msp); in metaslab_unflushed_bump()
4006 metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx); in metaslab_unflushed_bump()
4007 metaslab_set_unflushed_dirty(msp, dirty); in metaslab_unflushed_bump()
4008 avl_add(&spa->spa_metaslabs_by_flushed, msp); in metaslab_unflushed_bump()
4031 metaslab_flush_update(metaslab_t *msp, dmu_tx_t *tx) in metaslab_flush_update() argument
4033 metaslab_group_t *mg = msp->ms_group; in metaslab_flush_update()
4036 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_flush_update()
4045 msp->ms_synced_length = space_map_length(msp->ms_sm); in metaslab_flush_update()
4052 metaslab_unflushed_txg(msp) == 0) in metaslab_flush_update()
4055 metaslab_unflushed_bump(msp, tx, B_FALSE); in metaslab_flush_update()
4059 metaslab_flush(metaslab_t *msp, dmu_tx_t *tx) in metaslab_flush() argument
4061 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; in metaslab_flush()
4063 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_flush()
4067 ASSERT(msp->ms_sm != NULL); in metaslab_flush()
4068 ASSERT(metaslab_unflushed_txg(msp) != 0); in metaslab_flush()
4069 ASSERT(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL) != NULL); in metaslab_flush()
4078 ASSERT3U(metaslab_unflushed_txg(msp), <, dmu_tx_get_txg(tx)); in metaslab_flush()
4084 if (msp->ms_loading) in metaslab_flush()
4087 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); in metaslab_flush()
4088 metaslab_verify_weight_and_frag(msp); in metaslab_flush()
4101 if (msp->ms_loaded && metaslab_should_condense(msp)) { in metaslab_flush()
4102 metaslab_group_t *mg = msp->ms_group; in metaslab_flush()
4111 metaslab_group_histogram_remove(mg, msp); in metaslab_flush()
4113 metaslab_condense(msp, tx); in metaslab_flush()
4115 space_map_histogram_clear(msp->ms_sm); in metaslab_flush()
4116 space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx); in metaslab_flush()
4117 ASSERT(zfs_range_tree_is_empty(msp->ms_freed)); in metaslab_flush()
4119 space_map_histogram_add(msp->ms_sm, in metaslab_flush()
4120 msp->ms_defer[t], tx); in metaslab_flush()
4122 metaslab_aux_histograms_update(msp); in metaslab_flush()
4124 metaslab_group_histogram_add(mg, msp); in metaslab_flush()
4128 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); in metaslab_flush()
4137 metaslab_recalculate_weight_and_sort(msp); in metaslab_flush()
4141 msp->ms_flushing = B_TRUE; in metaslab_flush()
4142 uint64_t sm_len_before = space_map_length(msp->ms_sm); in metaslab_flush()
4144 mutex_exit(&msp->ms_lock); in metaslab_flush()
4145 space_map_write(msp->ms_sm, msp->ms_unflushed_allocs, SM_ALLOC, in metaslab_flush()
4147 space_map_write(msp->ms_sm, msp->ms_unflushed_frees, SM_FREE, in metaslab_flush()
4149 mutex_enter(&msp->ms_lock); in metaslab_flush()
4151 uint64_t sm_len_after = space_map_length(msp->ms_sm); in metaslab_flush()
4157 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, in metaslab_flush()
4158 (u_longlong_t)msp->ms_id, in metaslab_flush()
4160 msp->ms_unflushed_allocs), in metaslab_flush()
4162 msp->ms_unflushed_frees), in metaslab_flush()
4167 metaslab_unflushed_changes_memused(msp)); in metaslab_flush()
4169 metaslab_unflushed_changes_memused(msp); in metaslab_flush()
4170 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); in metaslab_flush()
4171 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); in metaslab_flush()
4173 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); in metaslab_flush()
4174 metaslab_verify_weight_and_frag(msp); in metaslab_flush()
4176 metaslab_flush_update(msp, tx); in metaslab_flush()
4178 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); in metaslab_flush()
4179 metaslab_verify_weight_and_frag(msp); in metaslab_flush()
4181 msp->ms_flushing = B_FALSE; in metaslab_flush()
4182 cv_broadcast(&msp->ms_flush_cv); in metaslab_flush()
4190 metaslab_sync(metaslab_t *msp, uint64_t txg) in metaslab_sync() argument
4192 metaslab_group_t *mg = msp->ms_group; in metaslab_sync()
4196 zfs_range_tree_t *alloctree = msp->ms_allocating[txg & TXG_MASK]; in metaslab_sync()
4204 if (msp->ms_new) { in metaslab_sync()
4206 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); in metaslab_sync()
4207 ASSERT0(zfs_range_tree_space(msp->ms_freed)); in metaslab_sync()
4208 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); in metaslab_sync()
4209 ASSERT0(zfs_range_tree_space(msp->ms_trim)); in metaslab_sync()
4225 zfs_range_tree_is_empty(msp->ms_freeing) && in metaslab_sync()
4226 zfs_range_tree_is_empty(msp->ms_checkpointing) && in metaslab_sync()
4227 !(msp->ms_loaded && msp->ms_condense_wanted && in metaslab_sync()
4254 if (msp->ms_sm == NULL) { in metaslab_sync()
4262 msp->ms_id, sizeof (uint64_t), &new_object, tx); in metaslab_sync()
4264 VERIFY0(space_map_open(&msp->ms_sm, mos, new_object, in metaslab_sync()
4265 msp->ms_start, msp->ms_size, vd->vdev_ashift)); in metaslab_sync()
4266 ASSERT(msp->ms_sm != NULL); in metaslab_sync()
4268 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); in metaslab_sync()
4269 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); in metaslab_sync()
4270 ASSERT0(metaslab_allocated_space(msp)); in metaslab_sync()
4273 if (!zfs_range_tree_is_empty(msp->ms_checkpointing) && in metaslab_sync()
4295 mutex_enter(&msp->ms_sync_lock); in metaslab_sync()
4296 mutex_enter(&msp->ms_lock); in metaslab_sync()
4305 metaslab_group_histogram_remove(mg, msp); in metaslab_sync()
4307 if (spa->spa_sync_pass == 1 && msp->ms_loaded && in metaslab_sync()
4308 metaslab_should_condense(msp)) in metaslab_sync()
4309 metaslab_condense(msp, tx); in metaslab_sync()
4316 mutex_exit(&msp->ms_lock); in metaslab_sync()
4320 if (metaslab_unflushed_txg(msp) == 0) in metaslab_sync()
4321 metaslab_unflushed_add(msp, tx); in metaslab_sync()
4322 else if (!metaslab_unflushed_dirty(msp)) in metaslab_sync()
4323 metaslab_unflushed_bump(msp, tx, B_TRUE); in metaslab_sync()
4327 space_map_write(log_sm, msp->ms_freeing, SM_FREE, in metaslab_sync()
4329 mutex_enter(&msp->ms_lock); in metaslab_sync()
4332 metaslab_unflushed_changes_memused(msp)); in metaslab_sync()
4334 metaslab_unflushed_changes_memused(msp); in metaslab_sync()
4336 msp->ms_unflushed_frees, msp->ms_unflushed_allocs); in metaslab_sync()
4337 zfs_range_tree_remove_xor_add(msp->ms_freeing, in metaslab_sync()
4338 msp->ms_unflushed_allocs, msp->ms_unflushed_frees); in metaslab_sync()
4340 metaslab_unflushed_changes_memused(msp); in metaslab_sync()
4344 space_map_write(msp->ms_sm, alloctree, SM_ALLOC, in metaslab_sync()
4346 space_map_write(msp->ms_sm, msp->ms_freeing, SM_FREE, in metaslab_sync()
4348 mutex_enter(&msp->ms_lock); in metaslab_sync()
4351 msp->ms_allocated_space += zfs_range_tree_space(alloctree); in metaslab_sync()
4352 ASSERT3U(msp->ms_allocated_space, >=, in metaslab_sync()
4353 zfs_range_tree_space(msp->ms_freeing)); in metaslab_sync()
4354 msp->ms_allocated_space -= zfs_range_tree_space(msp->ms_freeing); in metaslab_sync()
4356 if (!zfs_range_tree_is_empty(msp->ms_checkpointing)) { in metaslab_sync()
4366 mutex_exit(&msp->ms_lock); in metaslab_sync()
4368 msp->ms_checkpointing, SM_FREE, SM_NO_VDEVID, tx); in metaslab_sync()
4369 mutex_enter(&msp->ms_lock); in metaslab_sync()
4372 zfs_range_tree_space(msp->ms_checkpointing); in metaslab_sync()
4374 zfs_range_tree_space(msp->ms_checkpointing); in metaslab_sync()
4378 zfs_range_tree_vacate(msp->ms_checkpointing, NULL, NULL); in metaslab_sync()
4381 if (msp->ms_loaded) { in metaslab_sync()
4388 space_map_histogram_clear(msp->ms_sm); in metaslab_sync()
4389 space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx); in metaslab_sync()
4398 space_map_histogram_add(msp->ms_sm, msp->ms_freed, tx); in metaslab_sync()
4408 space_map_histogram_add(msp->ms_sm, in metaslab_sync()
4409 msp->ms_defer[t], tx); in metaslab_sync()
4420 space_map_histogram_add(msp->ms_sm, msp->ms_freeing, tx); in metaslab_sync()
4421 metaslab_aux_histograms_update(msp); in metaslab_sync()
4423 metaslab_group_histogram_add(mg, msp); in metaslab_sync()
4438 zfs_range_tree_swap(&msp->ms_freeing, &msp->ms_freed); in metaslab_sync()
4439 ASSERT0(msp->ms_allocated_this_txg); in metaslab_sync()
4441 zfs_range_tree_vacate(msp->ms_freeing, in metaslab_sync()
4442 zfs_range_tree_add, msp->ms_freed); in metaslab_sync()
4444 msp->ms_allocated_this_txg += zfs_range_tree_space(alloctree); in metaslab_sync()
4447 ASSERT0(zfs_range_tree_space(msp->ms_allocating[txg & TXG_MASK])); in metaslab_sync()
4448 ASSERT0(zfs_range_tree_space(msp->ms_allocating[TXG_CLEAN(txg) in metaslab_sync()
4450 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); in metaslab_sync()
4451 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); in metaslab_sync()
4453 mutex_exit(&msp->ms_lock); in metaslab_sync()
4461 msp->ms_id * sizeof (uint64_t), sizeof (uint64_t), &object, 0)); in metaslab_sync()
4462 VERIFY3U(object, ==, space_map_object(msp->ms_sm)); in metaslab_sync()
4464 mutex_exit(&msp->ms_sync_lock); in metaslab_sync()
4469 metaslab_evict(metaslab_t *msp, uint64_t txg) in metaslab_evict() argument
4471 if (!msp->ms_loaded || msp->ms_disabled != 0) in metaslab_evict()
4476 msp->ms_allocating[(txg + t) & TXG_MASK])); in metaslab_evict()
4478 if (msp->ms_allocator != -1) in metaslab_evict()
4479 metaslab_passivate(msp, msp->ms_weight & ~METASLAB_ACTIVE_MASK); in metaslab_evict()
4482 metaslab_unload(msp); in metaslab_evict()
4490 metaslab_sync_done(metaslab_t *msp, uint64_t txg) in metaslab_sync_done() argument
4492 metaslab_group_t *mg = msp->ms_group; in metaslab_sync_done()
4501 mutex_enter(&msp->ms_lock); in metaslab_sync_done()
4503 if (msp->ms_new) { in metaslab_sync_done()
4505 metaslab_space_update(vd, mg->mg_class, 0, 0, msp->ms_size); in metaslab_sync_done()
4508 VERIFY0(msp->ms_allocated_this_txg); in metaslab_sync_done()
4509 VERIFY0(zfs_range_tree_space(msp->ms_freed)); in metaslab_sync_done()
4512 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); in metaslab_sync_done()
4513 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); in metaslab_sync_done()
4515 defer_tree = &msp->ms_defer[txg % TXG_DEFER_SIZE]; in metaslab_sync_done()
4525 alloc_delta = msp->ms_allocated_this_txg - in metaslab_sync_done()
4526 zfs_range_tree_space(msp->ms_freed); in metaslab_sync_done()
4529 defer_delta = zfs_range_tree_space(msp->ms_freed) - in metaslab_sync_done()
4545 metaslab_load_wait(msp); in metaslab_sync_done()
4560 msp->ms_trim); in metaslab_sync_done()
4562 zfs_range_tree_walk(msp->ms_freed, zfs_range_tree_add, in metaslab_sync_done()
4563 msp->ms_trim); in metaslab_sync_done()
4566 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); in metaslab_sync_done()
4576 msp->ms_loaded ? zfs_range_tree_add : NULL, msp->ms_allocatable); in metaslab_sync_done()
4578 zfs_range_tree_swap(&msp->ms_freed, defer_tree); in metaslab_sync_done()
4580 zfs_range_tree_vacate(msp->ms_freed, in metaslab_sync_done()
4581 msp->ms_loaded ? zfs_range_tree_add : NULL, in metaslab_sync_done()
4582 msp->ms_allocatable); in metaslab_sync_done()
4585 msp->ms_synced_length = space_map_length(msp->ms_sm); in metaslab_sync_done()
4587 msp->ms_deferspace += defer_delta; in metaslab_sync_done()
4588 ASSERT3S(msp->ms_deferspace, >=, 0); in metaslab_sync_done()
4589 ASSERT3S(msp->ms_deferspace, <=, msp->ms_size); in metaslab_sync_done()
4590 if (msp->ms_deferspace != 0) { in metaslab_sync_done()
4595 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); in metaslab_sync_done()
4597 metaslab_aux_histograms_update_done(msp, defer_allowed); in metaslab_sync_done()
4599 if (msp->ms_new) { in metaslab_sync_done()
4600 msp->ms_new = B_FALSE; in metaslab_sync_done()
4610 metaslab_recalculate_weight_and_sort(msp); in metaslab_sync_done()
4612 ASSERT0(zfs_range_tree_space(msp->ms_allocating[txg & TXG_MASK])); in metaslab_sync_done()
4613 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); in metaslab_sync_done()
4614 ASSERT0(zfs_range_tree_space(msp->ms_freed)); in metaslab_sync_done()
4615 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); in metaslab_sync_done()
4616 msp->ms_allocating_total -= msp->ms_allocated_this_txg; in metaslab_sync_done()
4617 msp->ms_allocated_this_txg = 0; in metaslab_sync_done()
4618 mutex_exit(&msp->ms_lock); in metaslab_sync_done()
4649 metaslab_is_unique(metaslab_t *msp, dva_t *dva) in metaslab_is_unique() argument
4656 if (msp->ms_group->mg_vd->vdev_id != DVA_GET_VDEV(dva)) in metaslab_is_unique()
4659 dva_ms_id = DVA_GET_OFFSET(dva) >> msp->ms_group->mg_vd->vdev_ms_shift; in metaslab_is_unique()
4661 return (msp->ms_id != dva_ms_id); in metaslab_is_unique()
4675 metaslab_t *msp, uint64_t psize, uint32_t dva_id, uint64_t offset, in metaslab_trace_add() argument
4705 mat->mat_msp = msp; in metaslab_trace_add()
4712 if (msp != NULL) in metaslab_trace_add()
4713 mat->mat_weight = msp->ms_weight; in metaslab_trace_add()
4801 metaslab_block_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, in metaslab_block_alloc() argument
4805 zfs_range_tree_t *rt = msp->ms_allocatable; in metaslab_block_alloc()
4806 metaslab_class_t *mc = msp->ms_group->mg_class; in metaslab_block_alloc()
4808 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_block_alloc()
4809 VERIFY(!msp->ms_condensing); in metaslab_block_alloc()
4810 VERIFY0(msp->ms_disabled); in metaslab_block_alloc()
4811 VERIFY0(msp->ms_new); in metaslab_block_alloc()
4813 start = mc->mc_ops->msop_alloc(msp, size, max_size, actual_size); in metaslab_block_alloc()
4816 metaslab_group_t *mg = msp->ms_group; in metaslab_block_alloc()
4821 VERIFY3U(zfs_range_tree_space(rt) - size, <=, msp->ms_size); in metaslab_block_alloc()
4823 zfs_range_tree_clear(msp->ms_trim, start, size); in metaslab_block_alloc()
4825 if (zfs_range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK])) in metaslab_block_alloc()
4826 vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg); in metaslab_block_alloc()
4828 zfs_range_tree_add(msp->ms_allocating[txg & TXG_MASK], start, in metaslab_block_alloc()
4830 msp->ms_allocating_total += size; in metaslab_block_alloc()
4833 msp->ms_alloc_txg = txg; in metaslab_block_alloc()
4834 metaslab_verify_space(msp, txg); in metaslab_block_alloc()
4841 msp->ms_max_size = metaslab_largest_allocatable(msp); in metaslab_block_alloc()
4864 metaslab_t *msp = avl_find(t, search, &idx); in find_valid_metaslab() local
4865 if (msp == NULL) in find_valid_metaslab()
4866 msp = avl_nearest(t, idx, AVL_AFTER); in find_valid_metaslab()
4869 for (; msp != NULL; msp = AVL_NEXT(t, msp)) { in find_valid_metaslab()
4878 if (!metaslab_should_allocate(msp, asize, try_hard)) { in find_valid_metaslab()
4879 metaslab_trace_add(zal, mg, msp, asize, d, in find_valid_metaslab()
4888 if (msp->ms_condensing || msp->ms_disabled > 0 || msp->ms_new) in find_valid_metaslab()
4891 *was_active = msp->ms_allocator != -1; in find_valid_metaslab()
4904 if (!metaslab_is_unique(msp, &dva[i])) in find_valid_metaslab()
4912 if (msp != NULL) { in find_valid_metaslab()
4913 search->ms_weight = msp->ms_weight; in find_valid_metaslab()
4914 search->ms_start = msp->ms_start + 1; in find_valid_metaslab()
4915 search->ms_allocator = msp->ms_allocator; in find_valid_metaslab()
4916 search->ms_primary = msp->ms_primary; in find_valid_metaslab()
4918 return (msp); in find_valid_metaslab()
4922 metaslab_active_mask_verify(metaslab_t *msp) in metaslab_active_mask_verify() argument
4924 ASSERT(MUTEX_HELD(&msp->ms_lock)); in metaslab_active_mask_verify()
4929 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0) in metaslab_active_mask_verify()
4932 if (msp->ms_weight & METASLAB_WEIGHT_PRIMARY) { in metaslab_active_mask_verify()
4933 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); in metaslab_active_mask_verify()
4934 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM); in metaslab_active_mask_verify()
4935 VERIFY3S(msp->ms_allocator, !=, -1); in metaslab_active_mask_verify()
4936 VERIFY(msp->ms_primary); in metaslab_active_mask_verify()
4940 if (msp->ms_weight & METASLAB_WEIGHT_SECONDARY) { in metaslab_active_mask_verify()
4941 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); in metaslab_active_mask_verify()
4942 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM); in metaslab_active_mask_verify()
4943 VERIFY3S(msp->ms_allocator, !=, -1); in metaslab_active_mask_verify()
4944 VERIFY(!msp->ms_primary); in metaslab_active_mask_verify()
4948 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) { in metaslab_active_mask_verify()
4949 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); in metaslab_active_mask_verify()
4950 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); in metaslab_active_mask_verify()
4951 VERIFY3S(msp->ms_allocator, ==, -1); in metaslab_active_mask_verify()
4962 metaslab_t *msp = NULL; in metaslab_group_alloc() local
5005 msp = mga->mga_primary; in metaslab_group_alloc()
5013 ASSERT(msp->ms_primary); in metaslab_group_alloc()
5014 ASSERT3S(msp->ms_allocator, ==, allocator); in metaslab_group_alloc()
5015 ASSERT(msp->ms_loaded); in metaslab_group_alloc()
5018 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); in metaslab_group_alloc()
5021 msp = mga->mga_secondary; in metaslab_group_alloc()
5027 ASSERT(!msp->ms_primary); in metaslab_group_alloc()
5028 ASSERT3S(msp->ms_allocator, ==, allocator); in metaslab_group_alloc()
5029 ASSERT(msp->ms_loaded); in metaslab_group_alloc()
5032 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); in metaslab_group_alloc()
5034 msp = find_valid_metaslab(mg, activation_weight, dva, d, in metaslab_group_alloc()
5040 if (msp == NULL) in metaslab_group_alloc()
5042 mutex_enter(&msp->ms_lock); in metaslab_group_alloc()
5044 metaslab_active_mask_verify(msp); in metaslab_group_alloc()
5052 metaslab_t *, msp, uint64_t, activation_weight, in metaslab_group_alloc()
5064 if (was_active && !(msp->ms_weight & METASLAB_ACTIVE_MASK)) { in metaslab_group_alloc()
5065 ASSERT3S(msp->ms_allocator, ==, -1); in metaslab_group_alloc()
5066 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5076 if (!was_active && (msp->ms_weight & METASLAB_ACTIVE_MASK) && in metaslab_group_alloc()
5077 (msp->ms_allocator != -1) && in metaslab_group_alloc()
5078 (msp->ms_allocator != allocator || ((activation_weight == in metaslab_group_alloc()
5079 METASLAB_WEIGHT_PRIMARY) != msp->ms_primary))) { in metaslab_group_alloc()
5080 ASSERT(msp->ms_loaded); in metaslab_group_alloc()
5081 ASSERT((msp->ms_weight & METASLAB_WEIGHT_CLAIM) || in metaslab_group_alloc()
5082 msp->ms_allocator != -1); in metaslab_group_alloc()
5083 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5094 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM && in metaslab_group_alloc()
5096 ASSERT(msp->ms_loaded); in metaslab_group_alloc()
5097 ASSERT3S(msp->ms_allocator, ==, -1); in metaslab_group_alloc()
5098 metaslab_passivate(msp, msp->ms_weight & in metaslab_group_alloc()
5100 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5104 metaslab_set_selected_txg(msp, txg); in metaslab_group_alloc()
5107 metaslab_activate(msp, allocator, activation_weight); in metaslab_group_alloc()
5108 metaslab_active_mask_verify(msp); in metaslab_group_alloc()
5129 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5132 ASSERT(msp->ms_loaded); in metaslab_group_alloc()
5141 if (!metaslab_should_allocate(msp, asize, try_hard)) { in metaslab_group_alloc()
5143 metaslab_trace_add(zal, mg, msp, asize, d, in metaslab_group_alloc()
5155 if (msp->ms_condensing) { in metaslab_group_alloc()
5156 metaslab_trace_add(zal, mg, msp, asize, d, in metaslab_group_alloc()
5159 metaslab_passivate(msp, msp->ms_weight & in metaslab_group_alloc()
5162 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5164 } else if (msp->ms_disabled > 0) { in metaslab_group_alloc()
5165 metaslab_trace_add(zal, mg, msp, asize, d, in metaslab_group_alloc()
5168 metaslab_passivate(msp, msp->ms_weight & in metaslab_group_alloc()
5171 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5175 offset = metaslab_block_alloc(msp, asize, max_asize, txg, in metaslab_group_alloc()
5179 metaslab_trace_add(zal, mg, msp, *actual_asize, d, in metaslab_group_alloc()
5183 metaslab_segment_may_passivate(msp); in metaslab_group_alloc()
5184 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5187 metaslab_trace_add(zal, mg, msp, asize, d, offset, allocator); in metaslab_group_alloc()
5189 ASSERT(msp->ms_loaded); in metaslab_group_alloc()
5196 DTRACE_PROBE2(ms__alloc__failure, metaslab_t *, msp, in metaslab_group_alloc()
5222 if (WEIGHT_IS_SPACEBASED(msp->ms_weight)) { in metaslab_group_alloc()
5223 weight = metaslab_largest_allocatable(msp); in metaslab_group_alloc()
5226 weight = metaslab_weight_from_range_tree(msp); in metaslab_group_alloc()
5230 metaslab_passivate(msp, weight); in metaslab_group_alloc()
5244 weight |= msp->ms_weight & METASLAB_ACTIVE_MASK; in metaslab_group_alloc()
5245 metaslab_group_sort(mg, msp, weight); in metaslab_group_alloc()
5247 metaslab_active_mask_verify(msp); in metaslab_group_alloc()
5255 ASSERT(!metaslab_should_allocate(msp, asize, try_hard)); in metaslab_group_alloc()
5257 mutex_exit(&msp->ms_lock); in metaslab_group_alloc()
5464 metaslab_t *msp; in metaslab_free_concrete() local
5472 msp = vd->vdev_ms[m]; in metaslab_free_concrete()
5474 VERIFY(!msp->ms_condensing); in metaslab_free_concrete()
5475 VERIFY3U(offset, >=, msp->ms_start); in metaslab_free_concrete()
5476 VERIFY3U(offset + asize, <=, msp->ms_start + msp->ms_size); in metaslab_free_concrete()
5482 mutex_enter(&msp->ms_lock); in metaslab_free_concrete()
5483 if (zfs_range_tree_is_empty(msp->ms_freeing) && in metaslab_free_concrete()
5484 zfs_range_tree_is_empty(msp->ms_checkpointing)) { in metaslab_free_concrete()
5485 vdev_dirty(vd, VDD_METASLAB, msp, spa_syncing_txg(spa)); in metaslab_free_concrete()
5490 zfs_range_tree_add(msp->ms_checkpointing, offset, asize); in metaslab_free_concrete()
5492 zfs_range_tree_add(msp->ms_freeing, offset, asize); in metaslab_free_concrete()
5494 mutex_exit(&msp->ms_lock); in metaslab_free_concrete()
5692 metaslab_t *msp; in metaslab_unalloc_dva() local
5720 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; in metaslab_unalloc_dva()
5722 mutex_enter(&msp->ms_lock); in metaslab_unalloc_dva()
5723 zfs_range_tree_remove(msp->ms_allocating[txg & TXG_MASK], in metaslab_unalloc_dva()
5725 msp->ms_allocating_total -= size; in metaslab_unalloc_dva()
5727 VERIFY(!msp->ms_condensing); in metaslab_unalloc_dva()
5728 VERIFY3U(offset, >=, msp->ms_start); in metaslab_unalloc_dva()
5729 VERIFY3U(offset + size, <=, msp->ms_start + msp->ms_size); in metaslab_unalloc_dva()
5730 VERIFY3U(zfs_range_tree_space(msp->ms_allocatable) + size, <=, in metaslab_unalloc_dva()
5731 msp->ms_size); in metaslab_unalloc_dva()
5734 zfs_range_tree_add(msp->ms_allocatable, offset, size); in metaslab_unalloc_dva()
5735 mutex_exit(&msp->ms_lock); in metaslab_unalloc_dva()
5811 metaslab_t *msp; in metaslab_claim_concrete() local
5819 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; in metaslab_claim_concrete()
5821 mutex_enter(&msp->ms_lock); in metaslab_claim_concrete()
5823 if ((txg != 0 && spa_writeable(spa)) || !msp->ms_loaded) { in metaslab_claim_concrete()
5824 error = metaslab_activate(msp, 0, METASLAB_WEIGHT_CLAIM); in metaslab_claim_concrete()
5826 ASSERT(msp->ms_loaded); in metaslab_claim_concrete()
5827 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); in metaslab_claim_concrete()
5833 !zfs_range_tree_contains(msp->ms_allocatable, offset, size)) in metaslab_claim_concrete()
5837 mutex_exit(&msp->ms_lock); in metaslab_claim_concrete()
5841 VERIFY(!msp->ms_condensing); in metaslab_claim_concrete()
5844 VERIFY3U(zfs_range_tree_space(msp->ms_allocatable) - size, <=, in metaslab_claim_concrete()
5845 msp->ms_size); in metaslab_claim_concrete()
5846 zfs_range_tree_remove(msp->ms_allocatable, offset, size); in metaslab_claim_concrete()
5847 zfs_range_tree_clear(msp->ms_trim, offset, size); in metaslab_claim_concrete()
5850 metaslab_class_t *mc = msp->ms_group->mg_class; in metaslab_claim_concrete()
5852 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); in metaslab_claim_concrete()
5853 if (!multilist_link_active(&msp->ms_class_txg_node)) { in metaslab_claim_concrete()
5854 msp->ms_selected_txg = txg; in metaslab_claim_concrete()
5855 multilist_sublist_insert_head(mls, msp); in metaslab_claim_concrete()
5859 if (zfs_range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK])) in metaslab_claim_concrete()
5860 vdev_dirty(vd, VDD_METASLAB, msp, txg); in metaslab_claim_concrete()
5861 zfs_range_tree_add(msp->ms_allocating[txg & TXG_MASK], in metaslab_claim_concrete()
5863 msp->ms_allocating_total += size; in metaslab_claim_concrete()
5866 mutex_exit(&msp->ms_lock); in metaslab_claim_concrete()
6116 metaslab_t *msp; in metaslab_check_free_impl() local
6132 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; in metaslab_check_free_impl()
6134 mutex_enter(&msp->ms_lock); in metaslab_check_free_impl()
6135 if (msp->ms_loaded) { in metaslab_check_free_impl()
6136 zfs_range_tree_verify_not_present(msp->ms_allocatable, in metaslab_check_free_impl()
6151 zfs_range_tree_verify_not_present(msp->ms_freeing, offset, size); in metaslab_check_free_impl()
6152 zfs_range_tree_verify_not_present(msp->ms_checkpointing, offset, size); in metaslab_check_free_impl()
6153 zfs_range_tree_verify_not_present(msp->ms_freed, offset, size); in metaslab_check_free_impl()
6155 zfs_range_tree_verify_not_present(msp->ms_defer[j], offset, in metaslab_check_free_impl()
6157 zfs_range_tree_verify_not_present(msp->ms_trim, offset, size); in metaslab_check_free_impl()
6158 mutex_exit(&msp->ms_lock); in metaslab_check_free_impl()
6213 metaslab_disable(metaslab_t *msp) in metaslab_disable() argument
6215 ASSERT(!MUTEX_HELD(&msp->ms_lock)); in metaslab_disable()
6216 metaslab_group_t *mg = msp->ms_group; in metaslab_disable()
6231 if (msp->ms_disabled == 0) { in metaslab_disable()
6234 mutex_enter(&msp->ms_lock); in metaslab_disable()
6235 msp->ms_disabled++; in metaslab_disable()
6236 mutex_exit(&msp->ms_lock); in metaslab_disable()
6244 metaslab_enable(metaslab_t *msp, boolean_t sync, boolean_t unload) in metaslab_enable() argument
6246 metaslab_group_t *mg = msp->ms_group; in metaslab_enable()
6258 mutex_enter(&msp->ms_lock); in metaslab_enable()
6259 if (--msp->ms_disabled == 0) { in metaslab_enable()
6263 metaslab_unload(msp); in metaslab_enable()
6265 mutex_exit(&msp->ms_lock); in metaslab_enable()