1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2019 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved. 27 * Copyright (c) 2017, Intel Corporation. 28 */ 29 30 #include <sys/zfs_context.h> 31 #include <sys/brt.h> 32 #include <sys/dmu.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/space_map.h> 35 #include <sys/metaslab_impl.h> 36 #include <sys/vdev_impl.h> 37 #include <sys/vdev_draid.h> 38 #include <sys/zio.h> 39 #include <sys/spa_impl.h> 40 #include <sys/zfeature.h> 41 #include <sys/vdev_indirect_mapping.h> 42 #include <sys/zap.h> 43 #include <sys/btree.h> 44 45 #define GANG_ALLOCATION(flags) \ 46 ((flags) & (METASLAB_GANG_CHILD | METASLAB_GANG_HEADER)) 47 48 /* 49 * Metaslab group's per child vdev granularity, in bytes. This is roughly 50 * similar to what would be referred to as the "stripe size" in traditional 51 * RAID arrays. In normal operation, we will try to write this amount of 52 * data to each disk before moving on to the next top-level vdev. 53 */ 54 static uint64_t metaslab_aliquot = 2 * 1024 * 1024; 55 56 /* 57 * For testing, make some blocks above a certain size be gang blocks. 58 */ 59 uint64_t metaslab_force_ganging = SPA_MAXBLOCKSIZE + 1; 60 61 /* 62 * Of blocks of size >= metaslab_force_ganging, actually gang them this often. 63 */ 64 uint_t metaslab_force_ganging_pct = 3; 65 66 /* 67 * In pools where the log space map feature is not enabled we touch 68 * multiple metaslabs (and their respective space maps) with each 69 * transaction group. Thus, we benefit from having a small space map 70 * block size since it allows us to issue more I/O operations scattered 71 * around the disk. So a sane default for the space map block size 72 * is 8~16K. 73 */ 74 int zfs_metaslab_sm_blksz_no_log = (1 << 14); 75 76 /* 77 * When the log space map feature is enabled, we accumulate a lot of 78 * changes per metaslab that are flushed once in a while so we benefit 79 * from a bigger block size like 128K for the metaslab space maps. 80 */ 81 int zfs_metaslab_sm_blksz_with_log = (1 << 17); 82 83 /* 84 * The in-core space map representation is more compact than its on-disk form. 85 * The zfs_condense_pct determines how much more compact the in-core 86 * space map representation must be before we compact it on-disk. 87 * Values should be greater than or equal to 100. 88 */ 89 uint_t zfs_condense_pct = 200; 90 91 /* 92 * Condensing a metaslab is not guaranteed to actually reduce the amount of 93 * space used on disk. In particular, a space map uses data in increments of 94 * MAX(1 << ashift, space_map_blksz), so a metaslab might use the 95 * same number of blocks after condensing. Since the goal of condensing is to 96 * reduce the number of IOPs required to read the space map, we only want to 97 * condense when we can be sure we will reduce the number of blocks used by the 98 * space map. Unfortunately, we cannot precisely compute whether or not this is 99 * the case in metaslab_should_condense since we are holding ms_lock. Instead, 100 * we apply the following heuristic: do not condense a spacemap unless the 101 * uncondensed size consumes greater than zfs_metaslab_condense_block_threshold 102 * blocks. 103 */ 104 static const int zfs_metaslab_condense_block_threshold = 4; 105 106 /* 107 * The zfs_mg_noalloc_threshold defines which metaslab groups should 108 * be eligible for allocation. The value is defined as a percentage of 109 * free space. Metaslab groups that have more free space than 110 * zfs_mg_noalloc_threshold are always eligible for allocations. Once 111 * a metaslab group's free space is less than or equal to the 112 * zfs_mg_noalloc_threshold the allocator will avoid allocating to that 113 * group unless all groups in the pool have reached zfs_mg_noalloc_threshold. 114 * Once all groups in the pool reach zfs_mg_noalloc_threshold then all 115 * groups are allowed to accept allocations. Gang blocks are always 116 * eligible to allocate on any metaslab group. The default value of 0 means 117 * no metaslab group will be excluded based on this criterion. 118 */ 119 static uint_t zfs_mg_noalloc_threshold = 0; 120 121 /* 122 * Metaslab groups are considered eligible for allocations if their 123 * fragmentation metric (measured as a percentage) is less than or 124 * equal to zfs_mg_fragmentation_threshold. If a metaslab group 125 * exceeds this threshold then it will be skipped unless all metaslab 126 * groups within the metaslab class have also crossed this threshold. 127 * 128 * This tunable was introduced to avoid edge cases where we continue 129 * allocating from very fragmented disks in our pool while other, less 130 * fragmented disks, exists. On the other hand, if all disks in the 131 * pool are uniformly approaching the threshold, the threshold can 132 * be a speed bump in performance, where we keep switching the disks 133 * that we allocate from (e.g. we allocate some segments from disk A 134 * making it bypassing the threshold while freeing segments from disk 135 * B getting its fragmentation below the threshold). 136 * 137 * Empirically, we've seen that our vdev selection for allocations is 138 * good enough that fragmentation increases uniformly across all vdevs 139 * the majority of the time. Thus we set the threshold percentage high 140 * enough to avoid hitting the speed bump on pools that are being pushed 141 * to the edge. 142 */ 143 static uint_t zfs_mg_fragmentation_threshold = 95; 144 145 /* 146 * Allow metaslabs to keep their active state as long as their fragmentation 147 * percentage is less than or equal to zfs_metaslab_fragmentation_threshold. An 148 * active metaslab that exceeds this threshold will no longer keep its active 149 * status allowing better metaslabs to be selected. 150 */ 151 static uint_t zfs_metaslab_fragmentation_threshold = 77; 152 153 /* 154 * When set will load all metaslabs when pool is first opened. 155 */ 156 int metaslab_debug_load = B_FALSE; 157 158 /* 159 * When set will prevent metaslabs from being unloaded. 160 */ 161 static int metaslab_debug_unload = B_FALSE; 162 163 /* 164 * Minimum size which forces the dynamic allocator to change 165 * it's allocation strategy. Once the space map cannot satisfy 166 * an allocation of this size then it switches to using more 167 * aggressive strategy (i.e search by size rather than offset). 168 */ 169 uint64_t metaslab_df_alloc_threshold = SPA_OLD_MAXBLOCKSIZE; 170 171 /* 172 * The minimum free space, in percent, which must be available 173 * in a space map to continue allocations in a first-fit fashion. 174 * Once the space map's free space drops below this level we dynamically 175 * switch to using best-fit allocations. 176 */ 177 uint_t metaslab_df_free_pct = 4; 178 179 /* 180 * Maximum distance to search forward from the last offset. Without this 181 * limit, fragmented pools can see >100,000 iterations and 182 * metaslab_block_picker() becomes the performance limiting factor on 183 * high-performance storage. 184 * 185 * With the default setting of 16MB, we typically see less than 500 186 * iterations, even with very fragmented, ashift=9 pools. The maximum number 187 * of iterations possible is: 188 * metaslab_df_max_search / (2 * (1<<ashift)) 189 * With the default setting of 16MB this is 16*1024 (with ashift=9) or 190 * 2048 (with ashift=12). 191 */ 192 static uint_t metaslab_df_max_search = 16 * 1024 * 1024; 193 194 /* 195 * Forces the metaslab_block_picker function to search for at least this many 196 * segments forwards until giving up on finding a segment that the allocation 197 * will fit into. 198 */ 199 static const uint32_t metaslab_min_search_count = 100; 200 201 /* 202 * If we are not searching forward (due to metaslab_df_max_search, 203 * metaslab_df_free_pct, or metaslab_df_alloc_threshold), this tunable 204 * controls what segment is used. If it is set, we will use the largest free 205 * segment. If it is not set, we will use a segment of exactly the requested 206 * size (or larger). 207 */ 208 static int metaslab_df_use_largest_segment = B_FALSE; 209 210 /* 211 * These tunables control how long a metaslab will remain loaded after the 212 * last allocation from it. A metaslab can't be unloaded until at least 213 * metaslab_unload_delay TXG's and metaslab_unload_delay_ms milliseconds 214 * have elapsed. However, zfs_metaslab_mem_limit may cause it to be 215 * unloaded sooner. These settings are intended to be generous -- to keep 216 * metaslabs loaded for a long time, reducing the rate of metaslab loading. 217 */ 218 static uint_t metaslab_unload_delay = 32; 219 static uint_t metaslab_unload_delay_ms = 10 * 60 * 1000; /* ten minutes */ 220 221 /* 222 * Max number of metaslabs per group to preload. 223 */ 224 uint_t metaslab_preload_limit = 10; 225 226 /* 227 * Enable/disable preloading of metaslab. 228 */ 229 static int metaslab_preload_enabled = B_TRUE; 230 231 /* 232 * Enable/disable fragmentation weighting on metaslabs. 233 */ 234 static int metaslab_fragmentation_factor_enabled = B_TRUE; 235 236 /* 237 * Enable/disable lba weighting (i.e. outer tracks are given preference). 238 */ 239 static int metaslab_lba_weighting_enabled = B_TRUE; 240 241 /* 242 * Enable/disable space-based metaslab group biasing. 243 */ 244 static int metaslab_bias_enabled = B_TRUE; 245 246 /* 247 * Control performance-based metaslab group biasing. 248 */ 249 static int metaslab_perf_bias = 1; 250 251 /* 252 * Enable/disable remapping of indirect DVAs to their concrete vdevs. 253 */ 254 static const boolean_t zfs_remap_blkptr_enable = B_TRUE; 255 256 /* 257 * Enable/disable segment-based metaslab selection. 258 */ 259 static int zfs_metaslab_segment_weight_enabled = B_TRUE; 260 261 /* 262 * When using segment-based metaslab selection, we will continue 263 * allocating from the active metaslab until we have exhausted 264 * zfs_metaslab_switch_threshold of its buckets. 265 */ 266 static int zfs_metaslab_switch_threshold = 2; 267 268 /* 269 * Internal switch to enable/disable the metaslab allocation tracing 270 * facility. 271 */ 272 static const boolean_t metaslab_trace_enabled = B_FALSE; 273 274 /* 275 * Maximum entries that the metaslab allocation tracing facility will keep 276 * in a given list when running in non-debug mode. We limit the number 277 * of entries in non-debug mode to prevent us from using up too much memory. 278 * The limit should be sufficiently large that we don't expect any allocation 279 * to every exceed this value. In debug mode, the system will panic if this 280 * limit is ever reached allowing for further investigation. 281 */ 282 static const uint64_t metaslab_trace_max_entries = 5000; 283 284 /* 285 * Maximum number of metaslabs per group that can be disabled 286 * simultaneously. 287 */ 288 static const int max_disabled_ms = 3; 289 290 /* 291 * Time (in seconds) to respect ms_max_size when the metaslab is not loaded. 292 * To avoid 64-bit overflow, don't set above UINT32_MAX. 293 */ 294 static uint64_t zfs_metaslab_max_size_cache_sec = 1 * 60 * 60; /* 1 hour */ 295 296 /* 297 * Maximum percentage of memory to use on storing loaded metaslabs. If loading 298 * a metaslab would take it over this percentage, the oldest selected metaslab 299 * is automatically unloaded. 300 */ 301 static uint_t zfs_metaslab_mem_limit = 25; 302 303 /* 304 * Force the per-metaslab range trees to use 64-bit integers to store 305 * segments. Used for debugging purposes. 306 */ 307 static const boolean_t zfs_metaslab_force_large_segs = B_FALSE; 308 309 /* 310 * By default we only store segments over a certain size in the size-sorted 311 * metaslab trees (ms_allocatable_by_size and 312 * ms_unflushed_frees_by_size). This dramatically reduces memory usage and 313 * improves load and unload times at the cost of causing us to use slightly 314 * larger segments than we would otherwise in some cases. 315 */ 316 static const uint32_t metaslab_by_size_min_shift = 14; 317 318 /* 319 * If not set, we will first try normal allocation. If that fails then 320 * we will do a gang allocation. If that fails then we will do a "try hard" 321 * gang allocation. If that fails then we will have a multi-layer gang 322 * block. 323 * 324 * If set, we will first try normal allocation. If that fails then 325 * we will do a "try hard" allocation. If that fails we will do a gang 326 * allocation. If that fails we will do a "try hard" gang allocation. If 327 * that fails then we will have a multi-layer gang block. 328 */ 329 static int zfs_metaslab_try_hard_before_gang = B_FALSE; 330 331 /* 332 * When not trying hard, we only consider the best zfs_metaslab_find_max_tries 333 * metaslabs. This improves performance, especially when there are many 334 * metaslabs per vdev and the allocation can't actually be satisfied (so we 335 * would otherwise iterate all the metaslabs). If there is a metaslab with a 336 * worse weight but it can actually satisfy the allocation, we won't find it 337 * until trying hard. This may happen if the worse metaslab is not loaded 338 * (and the true weight is better than we have calculated), or due to weight 339 * bucketization. E.g. we are looking for a 60K segment, and the best 340 * metaslabs all have free segments in the 32-63K bucket, but the best 341 * zfs_metaslab_find_max_tries metaslabs have ms_max_size <60KB, and a 342 * subsequent metaslab has ms_max_size >60KB (but fewer segments in this 343 * bucket, and therefore a lower weight). 344 */ 345 static uint_t zfs_metaslab_find_max_tries = 100; 346 347 static uint64_t metaslab_weight(metaslab_t *, boolean_t); 348 static void metaslab_set_fragmentation(metaslab_t *, boolean_t); 349 static void metaslab_free_impl(vdev_t *, uint64_t, uint64_t, boolean_t); 350 static void metaslab_check_free_impl(vdev_t *, uint64_t, uint64_t); 351 352 static void metaslab_passivate(metaslab_t *msp, uint64_t weight); 353 static uint64_t metaslab_weight_from_range_tree(metaslab_t *msp); 354 static void metaslab_flush_update(metaslab_t *, dmu_tx_t *); 355 static unsigned int metaslab_idx_func(multilist_t *, void *); 356 static void metaslab_evict(metaslab_t *, uint64_t); 357 static void metaslab_rt_add(zfs_range_tree_t *rt, zfs_range_seg_t *rs, 358 void *arg); 359 kmem_cache_t *metaslab_alloc_trace_cache; 360 361 typedef struct metaslab_stats { 362 kstat_named_t metaslabstat_trace_over_limit; 363 kstat_named_t metaslabstat_reload_tree; 364 kstat_named_t metaslabstat_too_many_tries; 365 kstat_named_t metaslabstat_try_hard; 366 } metaslab_stats_t; 367 368 static metaslab_stats_t metaslab_stats = { 369 { "trace_over_limit", KSTAT_DATA_UINT64 }, 370 { "reload_tree", KSTAT_DATA_UINT64 }, 371 { "too_many_tries", KSTAT_DATA_UINT64 }, 372 { "try_hard", KSTAT_DATA_UINT64 }, 373 }; 374 375 #define METASLABSTAT_BUMP(stat) \ 376 atomic_inc_64(&metaslab_stats.stat.value.ui64); 377 378 char * 379 metaslab_rt_name(metaslab_group_t *mg, metaslab_t *ms, const char *name) 380 { 381 return (kmem_asprintf("{spa=%s vdev_guid=%llu ms_id=%llu %s}", 382 spa_name(mg->mg_vd->vdev_spa), 383 (u_longlong_t)mg->mg_vd->vdev_guid, 384 (u_longlong_t)ms->ms_id, 385 name)); 386 } 387 388 389 static kstat_t *metaslab_ksp; 390 391 void 392 metaslab_stat_init(void) 393 { 394 ASSERT0P(metaslab_alloc_trace_cache); 395 metaslab_alloc_trace_cache = kmem_cache_create( 396 "metaslab_alloc_trace_cache", sizeof (metaslab_alloc_trace_t), 397 0, NULL, NULL, NULL, NULL, NULL, 0); 398 metaslab_ksp = kstat_create("zfs", 0, "metaslab_stats", 399 "misc", KSTAT_TYPE_NAMED, sizeof (metaslab_stats) / 400 sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 401 if (metaslab_ksp != NULL) { 402 metaslab_ksp->ks_data = &metaslab_stats; 403 kstat_install(metaslab_ksp); 404 } 405 } 406 407 void 408 metaslab_stat_fini(void) 409 { 410 if (metaslab_ksp != NULL) { 411 kstat_delete(metaslab_ksp); 412 metaslab_ksp = NULL; 413 } 414 415 kmem_cache_destroy(metaslab_alloc_trace_cache); 416 metaslab_alloc_trace_cache = NULL; 417 } 418 419 /* 420 * ========================================================================== 421 * Metaslab classes 422 * ========================================================================== 423 */ 424 metaslab_class_t * 425 metaslab_class_create(spa_t *spa, const char *name, 426 const metaslab_ops_t *ops, boolean_t is_log) 427 { 428 metaslab_class_t *mc; 429 430 mc = kmem_zalloc(offsetof(metaslab_class_t, 431 mc_allocator[spa->spa_alloc_count]), KM_SLEEP); 432 433 mc->mc_spa = spa; 434 mc->mc_name = name; 435 mc->mc_ops = ops; 436 mc->mc_is_log = is_log; 437 mc->mc_alloc_io_size = SPA_OLD_MAXBLOCKSIZE; 438 mc->mc_alloc_max = UINT64_MAX; 439 mutex_init(&mc->mc_lock, NULL, MUTEX_DEFAULT, NULL); 440 multilist_create(&mc->mc_metaslab_txg_list, sizeof (metaslab_t), 441 offsetof(metaslab_t, ms_class_txg_node), metaslab_idx_func); 442 for (int i = 0; i < spa->spa_alloc_count; i++) { 443 metaslab_class_allocator_t *mca = &mc->mc_allocator[i]; 444 mutex_init(&mca->mca_lock, NULL, MUTEX_DEFAULT, NULL); 445 avl_create(&mca->mca_tree, zio_bookmark_compare, 446 sizeof (zio_t), offsetof(zio_t, io_queue_node.a)); 447 mca->mca_rotor = NULL; 448 mca->mca_reserved = 0; 449 } 450 451 return (mc); 452 } 453 454 void 455 metaslab_class_destroy(metaslab_class_t *mc) 456 { 457 spa_t *spa = mc->mc_spa; 458 459 ASSERT0(mc->mc_alloc); 460 ASSERT0(mc->mc_deferred); 461 ASSERT0(mc->mc_space); 462 ASSERT0(mc->mc_dspace); 463 464 for (int i = 0; i < spa->spa_alloc_count; i++) { 465 metaslab_class_allocator_t *mca = &mc->mc_allocator[i]; 466 avl_destroy(&mca->mca_tree); 467 mutex_destroy(&mca->mca_lock); 468 ASSERT0P(mca->mca_rotor); 469 ASSERT0(mca->mca_reserved); 470 } 471 mutex_destroy(&mc->mc_lock); 472 multilist_destroy(&mc->mc_metaslab_txg_list); 473 kmem_free(mc, offsetof(metaslab_class_t, 474 mc_allocator[spa->spa_alloc_count])); 475 } 476 477 void 478 metaslab_class_validate(metaslab_class_t *mc) 479 { 480 #ifdef ZFS_DEBUG 481 spa_t *spa = mc->mc_spa; 482 483 /* 484 * Must hold one of the spa_config locks. 485 */ 486 ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) || 487 spa_config_held(spa, SCL_ALL, RW_WRITER)); 488 489 for (int i = 0; i < spa->spa_alloc_count; i++) { 490 metaslab_class_allocator_t *mca = &mc->mc_allocator[i]; 491 metaslab_group_t *mg, *rotor; 492 493 ASSERT0(avl_numnodes(&mca->mca_tree)); 494 ASSERT0(mca->mca_reserved); 495 496 if ((mg = rotor = mca->mca_rotor) == NULL) 497 continue; 498 do { 499 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 500 vdev_t *vd = mg->mg_vd; 501 502 ASSERT3P(vd->vdev_top, ==, vd); 503 ASSERT(vd->vdev_mg == mg || vd->vdev_log_mg == mg); 504 ASSERT3P(mg->mg_class, ==, mc); 505 ASSERT3P(vd->vdev_ops, !=, &vdev_hole_ops); 506 ASSERT0(zfs_refcount_count(&mga->mga_queue_depth)); 507 } while ((mg = mg->mg_next) != rotor); 508 } 509 #endif 510 } 511 512 /* 513 * For each metaslab group in a class pre-calculate allocation quota and 514 * target queue depth to balance their space usage and write performance. 515 * Based on those pre-calculate class allocation throttle threshold for 516 * optimal saturation. onsync is true once per TXG to enable/disable 517 * allocation throttling and update moving average of maximum I/O size. 518 */ 519 void 520 metaslab_class_balance(metaslab_class_t *mc, boolean_t onsync) 521 { 522 metaslab_group_t *mg, *first; 523 524 /* 525 * Must hold one of the spa_config locks. 526 */ 527 ASSERT(spa_config_held(mc->mc_spa, SCL_ALL, RW_READER) || 528 spa_config_held(mc->mc_spa, SCL_ALL, RW_WRITER)); 529 530 if (onsync) 531 metaslab_class_validate(mc); 532 533 if (mc->mc_groups == 0) { 534 if (onsync) 535 mc->mc_alloc_throttle_enabled = B_FALSE; 536 mc->mc_alloc_max = UINT64_MAX; 537 return; 538 } 539 540 if (onsync) { 541 /* 542 * Moving average of maximum allocation size, in absence of 543 * large allocations shrinking to 1/8 of metaslab_aliquot. 544 */ 545 mc->mc_alloc_io_size = (3 * mc->mc_alloc_io_size + 546 metaslab_aliquot / 8) / 4; 547 mc->mc_alloc_throttle_enabled = mc->mc_is_log ? 0 : 548 zio_dva_throttle_enabled; 549 } 550 551 mg = first = mc->mc_allocator[0].mca_rotor; 552 uint64_t children = 0; 553 do { 554 children += vdev_get_ndisks(mg->mg_vd) - 555 vdev_get_nparity(mg->mg_vd); 556 } while ((mg = mg->mg_next) != first); 557 558 uint64_t sum_aliquot = 0; 559 do { 560 vdev_stat_t *vs = &mg->mg_vd->vdev_stat; 561 uint_t ratio; 562 563 /* 564 * Scale allocations per iteration with average number of 565 * children. Wider vdevs need more sequential allocations 566 * to keep decent per-child I/O size. 567 */ 568 uint64_t mg_aliquot = MAX(metaslab_aliquot * children / 569 mc->mc_groups, mc->mc_alloc_io_size * 4); 570 571 /* 572 * Scale allocations per iteration with the vdev capacity, 573 * relative to average. Bigger vdevs should get more to 574 * fill up at the same time as smaller ones. 575 */ 576 if (mc->mc_space > 0 && vs->vs_space > 0) { 577 ratio = vs->vs_space / (mc->mc_space / (mc->mc_groups * 578 256) + 1); 579 mg_aliquot = mg_aliquot * ratio / 256; 580 } 581 582 /* 583 * Scale allocations per iteration with the vdev's free space 584 * fraction, relative to average. Despite the above, vdevs free 585 * space fractions may get imbalanced, for example due to new 586 * vdev addition or different performance. We want free space 587 * fractions to be similar to postpone fragmentation. 588 * 589 * But same time we don't want to throttle vdevs still having 590 * plenty of free space, that appear faster than others, even 591 * if that cause temporary imbalance. Allow them to allocate 592 * more by keeping their allocation queue depth equivalent to 593 * 2.5 full iteration, even if they repeatedly drain it. Later 594 * with the free space reduction gradually reduce the target 595 * queue depth, stronger enforcing the free space balance. 596 */ 597 if (metaslab_bias_enabled && 598 mc->mc_space > 0 && vs->vs_space > 0) { 599 uint64_t vs_free = vs->vs_space > vs->vs_alloc ? 600 vs->vs_space - vs->vs_alloc : 0; 601 uint64_t mc_free = mc->mc_space > mc->mc_alloc ? 602 mc->mc_space - mc->mc_alloc : 0; 603 /* 604 * vs_fr is 16 bit fixed-point free space fraction. 605 * mc_fr is 8 bit fixed-point free space fraction. 606 * ratio as their quotient is 8 bit fixed-point. 607 */ 608 uint_t vs_fr = vs_free / (vs->vs_space / 65536 + 1); 609 uint_t mc_fr = mc_free / (mc->mc_space / 256 + 1); 610 ratio = vs_fr / (mc_fr + 1); 611 mg->mg_aliquot = mg_aliquot * ratio / 256; 612 /* From 2.5x at 25% full to 1x at 75%. */ 613 ratio = MIN(163840, vs_fr * 3 + 16384); 614 mg->mg_queue_target = MAX(mg->mg_aliquot, 615 mg->mg_aliquot * ratio / 65536); 616 } else { 617 mg->mg_aliquot = mg_aliquot; 618 mg->mg_queue_target = mg->mg_aliquot * 2; 619 } 620 sum_aliquot += mg->mg_aliquot; 621 } while ((mg = mg->mg_next) != first); 622 623 /* 624 * Set per-class allocation throttle threshold to 4 iterations through 625 * all the vdevs. This should keep all vdevs busy even if some are 626 * allocating more than we planned for them due to bigger blocks or 627 * better performance. 628 */ 629 mc->mc_alloc_max = sum_aliquot * 4; 630 } 631 632 static void 633 metaslab_class_rotate(metaslab_group_t *mg, int allocator, uint64_t psize, 634 boolean_t success) 635 { 636 metaslab_class_t *mc = mg->mg_class; 637 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 638 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 639 640 /* 641 * Exit fast if there is nothing to rotate, we are not following 642 * the rotor (copies, gangs, etc) or somebody already rotated it. 643 */ 644 if (mc->mc_groups < 2 || mca->mca_rotor != mg) 645 return; 646 647 /* 648 * Always rotate in case of allocation error or a log class. 649 */ 650 if (!success || mc->mc_is_log) 651 goto rotate; 652 653 /* 654 * Allocate from this group if we expect next I/O of the same size to 655 * mostly fit within the allocation quota. Rotate if we expect it to 656 * mostly go over the target queue depth. Meanwhile, to stripe between 657 * groups in configured amounts per child even if we can't reach the 658 * target queue depth, i.e. can't saturate the group write performance, 659 * always rotate after allocating the queue target bytes. 660 */ 661 uint64_t naq = atomic_add_64_nv(&mca->mca_aliquot, psize) + psize / 2; 662 if (naq < mg->mg_aliquot) 663 return; 664 if (naq >= mg->mg_queue_target) 665 goto rotate; 666 if (zfs_refcount_count(&mga->mga_queue_depth) + psize + psize / 2 >= 667 mg->mg_queue_target) 668 goto rotate; 669 670 /* 671 * When the pool is not too busy, prefer restoring the vdev free space 672 * balance instead of getting maximum speed we might not need, so that 673 * we could have more flexibility during more busy times later. 674 */ 675 if (metaslab_perf_bias <= 0) 676 goto rotate; 677 if (metaslab_perf_bias >= 2) 678 return; 679 spa_t *spa = mc->mc_spa; 680 dsl_pool_t *dp = spa_get_dsl(spa); 681 if (dp == NULL) 682 return; 683 uint64_t busy_thresh = zfs_dirty_data_max * 684 (zfs_vdev_async_write_active_min_dirty_percent + 685 zfs_vdev_async_write_active_max_dirty_percent) / 200; 686 if (dp->dp_dirty_total > busy_thresh || spa_has_pending_synctask(spa)) 687 return; 688 689 rotate: 690 mca->mca_rotor = mg->mg_next; 691 mca->mca_aliquot = 0; 692 } 693 694 static void 695 metaslab_class_space_update(metaslab_class_t *mc, int64_t alloc_delta, 696 int64_t defer_delta, int64_t space_delta, int64_t dspace_delta) 697 { 698 atomic_add_64(&mc->mc_alloc, alloc_delta); 699 atomic_add_64(&mc->mc_deferred, defer_delta); 700 atomic_add_64(&mc->mc_space, space_delta); 701 atomic_add_64(&mc->mc_dspace, dspace_delta); 702 } 703 704 const char * 705 metaslab_class_get_name(metaslab_class_t *mc) 706 { 707 return (mc->mc_name); 708 } 709 710 uint64_t 711 metaslab_class_get_alloc(metaslab_class_t *mc) 712 { 713 return (mc->mc_alloc); 714 } 715 716 uint64_t 717 metaslab_class_get_deferred(metaslab_class_t *mc) 718 { 719 return (mc->mc_deferred); 720 } 721 722 uint64_t 723 metaslab_class_get_space(metaslab_class_t *mc) 724 { 725 return (mc->mc_space); 726 } 727 728 uint64_t 729 metaslab_class_get_dspace(metaslab_class_t *mc) 730 { 731 return (spa_deflate(mc->mc_spa) ? mc->mc_dspace : mc->mc_space); 732 } 733 734 void 735 metaslab_class_histogram_verify(metaslab_class_t *mc) 736 { 737 spa_t *spa = mc->mc_spa; 738 vdev_t *rvd = spa->spa_root_vdev; 739 uint64_t *mc_hist; 740 int i; 741 742 if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0) 743 return; 744 745 mc_hist = kmem_zalloc(sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE, 746 KM_SLEEP); 747 748 mutex_enter(&mc->mc_lock); 749 for (int c = 0; c < rvd->vdev_children; c++) { 750 vdev_t *tvd = rvd->vdev_child[c]; 751 metaslab_group_t *mg = vdev_get_mg(tvd, mc); 752 753 /* 754 * Skip any holes, uninitialized top-levels, or 755 * vdevs that are not in this metalab class. 756 */ 757 if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 || 758 mg->mg_class != mc) { 759 continue; 760 } 761 762 IMPLY(mg == mg->mg_vd->vdev_log_mg, 763 mc == spa_embedded_log_class(mg->mg_vd->vdev_spa) || 764 mc == spa_special_embedded_log_class(mg->mg_vd->vdev_spa)); 765 766 for (i = 0; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i++) 767 mc_hist[i] += mg->mg_histogram[i]; 768 } 769 770 for (i = 0; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i++) { 771 VERIFY3U(mc_hist[i], ==, mc->mc_histogram[i]); 772 } 773 774 mutex_exit(&mc->mc_lock); 775 kmem_free(mc_hist, sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE); 776 } 777 778 /* 779 * Calculate the metaslab class's fragmentation metric. The metric 780 * is weighted based on the space contribution of each metaslab group. 781 * The return value will be a number between 0 and 100 (inclusive), or 782 * ZFS_FRAG_INVALID if the metric has not been set. See comment above the 783 * zfs_frag_table for more information about the metric. 784 */ 785 uint64_t 786 metaslab_class_fragmentation(metaslab_class_t *mc) 787 { 788 vdev_t *rvd = mc->mc_spa->spa_root_vdev; 789 uint64_t fragmentation = 0; 790 791 spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER); 792 793 for (int c = 0; c < rvd->vdev_children; c++) { 794 vdev_t *tvd = rvd->vdev_child[c]; 795 metaslab_group_t *mg = tvd->vdev_mg; 796 797 /* 798 * Skip any holes, uninitialized top-levels, 799 * or vdevs that are not in this metalab class. 800 */ 801 if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 || 802 mg->mg_class != mc) { 803 continue; 804 } 805 806 /* 807 * If a metaslab group does not contain a fragmentation 808 * metric then just bail out. 809 */ 810 if (mg->mg_fragmentation == ZFS_FRAG_INVALID) { 811 spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG); 812 return (ZFS_FRAG_INVALID); 813 } 814 815 /* 816 * Determine how much this metaslab_group is contributing 817 * to the overall pool fragmentation metric. 818 */ 819 fragmentation += mg->mg_fragmentation * 820 metaslab_group_get_space(mg); 821 } 822 fragmentation /= metaslab_class_get_space(mc); 823 824 ASSERT3U(fragmentation, <=, 100); 825 spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG); 826 return (fragmentation); 827 } 828 829 /* 830 * Calculate the amount of expandable space that is available in 831 * this metaslab class. If a device is expanded then its expandable 832 * space will be the amount of allocatable space that is currently not 833 * part of this metaslab class. 834 */ 835 uint64_t 836 metaslab_class_expandable_space(metaslab_class_t *mc) 837 { 838 vdev_t *rvd = mc->mc_spa->spa_root_vdev; 839 uint64_t space = 0; 840 841 spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER); 842 for (int c = 0; c < rvd->vdev_children; c++) { 843 vdev_t *tvd = rvd->vdev_child[c]; 844 metaslab_group_t *mg = tvd->vdev_mg; 845 846 if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 || 847 mg->mg_class != mc) { 848 continue; 849 } 850 851 /* 852 * Calculate if we have enough space to add additional 853 * metaslabs. We report the expandable space in terms 854 * of the metaslab size since that's the unit of expansion. 855 */ 856 space += P2ALIGN_TYPED(tvd->vdev_max_asize - tvd->vdev_asize, 857 1ULL << tvd->vdev_ms_shift, uint64_t); 858 } 859 spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG); 860 return (space); 861 } 862 863 void 864 metaslab_class_evict_old(metaslab_class_t *mc, uint64_t txg) 865 { 866 multilist_t *ml = &mc->mc_metaslab_txg_list; 867 uint64_t now = gethrestime_sec(); 868 /* Round delay up to next second. */ 869 uint_t delay = (metaslab_unload_delay_ms + 999) / 1000; 870 for (int i = 0; i < multilist_get_num_sublists(ml); i++) { 871 multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i); 872 metaslab_t *msp = multilist_sublist_head(mls); 873 multilist_sublist_unlock(mls); 874 while (msp != NULL) { 875 mutex_enter(&msp->ms_lock); 876 877 /* 878 * If the metaslab has been removed from the list 879 * (which could happen if we were at the memory limit 880 * and it was evicted during this loop), then we can't 881 * proceed and we should restart the sublist. 882 */ 883 if (!multilist_link_active(&msp->ms_class_txg_node)) { 884 mutex_exit(&msp->ms_lock); 885 i--; 886 break; 887 } 888 mls = multilist_sublist_lock_idx(ml, i); 889 metaslab_t *next_msp = multilist_sublist_next(mls, msp); 890 multilist_sublist_unlock(mls); 891 if (txg > 892 msp->ms_selected_txg + metaslab_unload_delay && 893 now > msp->ms_selected_time + delay && 894 (msp->ms_allocator == -1 || 895 !metaslab_preload_enabled)) { 896 metaslab_evict(msp, txg); 897 } else { 898 /* 899 * Once we've hit a metaslab selected too 900 * recently to evict, we're done evicting for 901 * now. 902 */ 903 mutex_exit(&msp->ms_lock); 904 break; 905 } 906 mutex_exit(&msp->ms_lock); 907 msp = next_msp; 908 } 909 } 910 } 911 912 static int 913 metaslab_compare(const void *x1, const void *x2) 914 { 915 const metaslab_t *m1 = (const metaslab_t *)x1; 916 const metaslab_t *m2 = (const metaslab_t *)x2; 917 918 int sort1 = 0; 919 int sort2 = 0; 920 if (m1->ms_allocator != -1 && m1->ms_primary) 921 sort1 = 1; 922 else if (m1->ms_allocator != -1 && !m1->ms_primary) 923 sort1 = 2; 924 if (m2->ms_allocator != -1 && m2->ms_primary) 925 sort2 = 1; 926 else if (m2->ms_allocator != -1 && !m2->ms_primary) 927 sort2 = 2; 928 929 /* 930 * Sort inactive metaslabs first, then primaries, then secondaries. When 931 * selecting a metaslab to allocate from, an allocator first tries its 932 * primary, then secondary active metaslab. If it doesn't have active 933 * metaslabs, or can't allocate from them, it searches for an inactive 934 * metaslab to activate. If it can't find a suitable one, it will steal 935 * a primary or secondary metaslab from another allocator. 936 */ 937 if (sort1 < sort2) 938 return (-1); 939 if (sort1 > sort2) 940 return (1); 941 942 int cmp = TREE_CMP(m2->ms_weight, m1->ms_weight); 943 if (likely(cmp)) 944 return (cmp); 945 946 IMPLY(TREE_CMP(m1->ms_start, m2->ms_start) == 0, m1 == m2); 947 948 return (TREE_CMP(m1->ms_start, m2->ms_start)); 949 } 950 951 /* 952 * ========================================================================== 953 * Metaslab groups 954 * ========================================================================== 955 */ 956 /* 957 * Update the allocatable flag and the metaslab group's capacity. 958 * The allocatable flag is set to true if the capacity is below 959 * the zfs_mg_noalloc_threshold or has a fragmentation value that is 960 * greater than zfs_mg_fragmentation_threshold. If a metaslab group 961 * transitions from allocatable to non-allocatable or vice versa then the 962 * metaslab group's class is updated to reflect the transition. 963 */ 964 static void 965 metaslab_group_alloc_update(metaslab_group_t *mg) 966 { 967 vdev_t *vd = mg->mg_vd; 968 metaslab_class_t *mc = mg->mg_class; 969 vdev_stat_t *vs = &vd->vdev_stat; 970 boolean_t was_allocatable; 971 boolean_t was_initialized; 972 973 ASSERT(vd == vd->vdev_top); 974 ASSERT3U(spa_config_held(mc->mc_spa, SCL_ALLOC, RW_READER), ==, 975 SCL_ALLOC); 976 977 mutex_enter(&mg->mg_lock); 978 was_allocatable = mg->mg_allocatable; 979 was_initialized = mg->mg_initialized; 980 981 uint64_t free_capacity = ((vs->vs_space - vs->vs_alloc) * 100) / 982 (vs->vs_space + 1); 983 984 mutex_enter(&mc->mc_lock); 985 986 /* 987 * If the metaslab group was just added then it won't 988 * have any space until we finish syncing out this txg. 989 * At that point we will consider it initialized and available 990 * for allocations. We also don't consider non-activated 991 * metaslab groups (e.g. vdevs that are in the middle of being removed) 992 * to be initialized, because they can't be used for allocation. 993 */ 994 mg->mg_initialized = metaslab_group_initialized(mg); 995 if (!was_initialized && mg->mg_initialized) { 996 mc->mc_groups++; 997 } else if (was_initialized && !mg->mg_initialized) { 998 ASSERT3U(mc->mc_groups, >, 0); 999 mc->mc_groups--; 1000 } 1001 if (mg->mg_initialized) 1002 mg->mg_no_free_space = B_FALSE; 1003 1004 /* 1005 * A metaslab group is considered allocatable if it has plenty 1006 * of free space or is not heavily fragmented. We only take 1007 * fragmentation into account if the metaslab group has a valid 1008 * fragmentation metric (i.e. a value between 0 and 100). 1009 */ 1010 mg->mg_allocatable = (mg->mg_activation_count > 0 && 1011 free_capacity > zfs_mg_noalloc_threshold && 1012 (mg->mg_fragmentation == ZFS_FRAG_INVALID || 1013 mg->mg_fragmentation <= zfs_mg_fragmentation_threshold)); 1014 1015 /* 1016 * The mc_alloc_groups maintains a count of the number of 1017 * groups in this metaslab class that are still above the 1018 * zfs_mg_noalloc_threshold. This is used by the allocating 1019 * threads to determine if they should avoid allocations to 1020 * a given group. The allocator will avoid allocations to a group 1021 * if that group has reached or is below the zfs_mg_noalloc_threshold 1022 * and there are still other groups that are above the threshold. 1023 * When a group transitions from allocatable to non-allocatable or 1024 * vice versa we update the metaslab class to reflect that change. 1025 * When the mc_alloc_groups value drops to 0 that means that all 1026 * groups have reached the zfs_mg_noalloc_threshold making all groups 1027 * eligible for allocations. This effectively means that all devices 1028 * are balanced again. 1029 */ 1030 if (was_allocatable && !mg->mg_allocatable) 1031 mc->mc_alloc_groups--; 1032 else if (!was_allocatable && mg->mg_allocatable) 1033 mc->mc_alloc_groups++; 1034 mutex_exit(&mc->mc_lock); 1035 1036 mutex_exit(&mg->mg_lock); 1037 } 1038 1039 int 1040 metaslab_sort_by_flushed(const void *va, const void *vb) 1041 { 1042 const metaslab_t *a = va; 1043 const metaslab_t *b = vb; 1044 1045 int cmp = TREE_CMP(a->ms_unflushed_txg, b->ms_unflushed_txg); 1046 if (likely(cmp)) 1047 return (cmp); 1048 1049 uint64_t a_vdev_id = a->ms_group->mg_vd->vdev_id; 1050 uint64_t b_vdev_id = b->ms_group->mg_vd->vdev_id; 1051 cmp = TREE_CMP(a_vdev_id, b_vdev_id); 1052 if (cmp) 1053 return (cmp); 1054 1055 return (TREE_CMP(a->ms_id, b->ms_id)); 1056 } 1057 1058 metaslab_group_t * 1059 metaslab_group_create(metaslab_class_t *mc, vdev_t *vd) 1060 { 1061 spa_t *spa = mc->mc_spa; 1062 metaslab_group_t *mg; 1063 1064 mg = kmem_zalloc(offsetof(metaslab_group_t, 1065 mg_allocator[spa->spa_alloc_count]), KM_SLEEP); 1066 mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL); 1067 mutex_init(&mg->mg_ms_disabled_lock, NULL, MUTEX_DEFAULT, NULL); 1068 cv_init(&mg->mg_ms_disabled_cv, NULL, CV_DEFAULT, NULL); 1069 avl_create(&mg->mg_metaslab_tree, metaslab_compare, 1070 sizeof (metaslab_t), offsetof(metaslab_t, ms_group_node)); 1071 mg->mg_vd = vd; 1072 mg->mg_class = mc; 1073 mg->mg_activation_count = 0; 1074 mg->mg_initialized = B_FALSE; 1075 mg->mg_no_free_space = B_TRUE; 1076 1077 for (int i = 0; i < spa->spa_alloc_count; i++) { 1078 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 1079 zfs_refcount_create_tracked(&mga->mga_queue_depth); 1080 } 1081 1082 return (mg); 1083 } 1084 1085 void 1086 metaslab_group_destroy(metaslab_group_t *mg) 1087 { 1088 spa_t *spa = mg->mg_class->mc_spa; 1089 1090 ASSERT0P(mg->mg_prev); 1091 ASSERT0P(mg->mg_next); 1092 /* 1093 * We may have gone below zero with the activation count 1094 * either because we never activated in the first place or 1095 * because we're done, and possibly removing the vdev. 1096 */ 1097 ASSERT(mg->mg_activation_count <= 0); 1098 1099 avl_destroy(&mg->mg_metaslab_tree); 1100 mutex_destroy(&mg->mg_lock); 1101 mutex_destroy(&mg->mg_ms_disabled_lock); 1102 cv_destroy(&mg->mg_ms_disabled_cv); 1103 1104 for (int i = 0; i < spa->spa_alloc_count; i++) { 1105 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 1106 zfs_refcount_destroy(&mga->mga_queue_depth); 1107 } 1108 kmem_free(mg, offsetof(metaslab_group_t, 1109 mg_allocator[spa->spa_alloc_count])); 1110 } 1111 1112 void 1113 metaslab_group_activate(metaslab_group_t *mg) 1114 { 1115 metaslab_class_t *mc = mg->mg_class; 1116 spa_t *spa = mc->mc_spa; 1117 metaslab_group_t *mgprev, *mgnext; 1118 1119 ASSERT3U(spa_config_held(spa, SCL_ALLOC, RW_WRITER), !=, 0); 1120 1121 ASSERT0P(mg->mg_prev); 1122 ASSERT0P(mg->mg_next); 1123 ASSERT(mg->mg_activation_count <= 0); 1124 1125 if (++mg->mg_activation_count <= 0) 1126 return; 1127 1128 metaslab_group_alloc_update(mg); 1129 1130 if ((mgprev = mc->mc_allocator[0].mca_rotor) == NULL) { 1131 mg->mg_prev = mg; 1132 mg->mg_next = mg; 1133 } else { 1134 mgnext = mgprev->mg_next; 1135 mg->mg_prev = mgprev; 1136 mg->mg_next = mgnext; 1137 mgprev->mg_next = mg; 1138 mgnext->mg_prev = mg; 1139 } 1140 for (int i = 0; i < spa->spa_alloc_count; i++) { 1141 mc->mc_allocator[i].mca_rotor = mg; 1142 mg = mg->mg_next; 1143 } 1144 metaslab_class_balance(mc, B_FALSE); 1145 } 1146 1147 /* 1148 * Passivate a metaslab group and remove it from the allocation rotor. 1149 * Callers must hold both the SCL_ALLOC and SCL_ZIO lock prior to passivating 1150 * a metaslab group. This function will momentarily drop spa_config_locks 1151 * that are lower than the SCL_ALLOC lock (see comment below). 1152 */ 1153 void 1154 metaslab_group_passivate(metaslab_group_t *mg) 1155 { 1156 metaslab_class_t *mc = mg->mg_class; 1157 spa_t *spa = mc->mc_spa; 1158 metaslab_group_t *mgprev, *mgnext; 1159 int locks = spa_config_held(spa, SCL_ALL, RW_WRITER); 1160 1161 ASSERT3U(spa_config_held(spa, SCL_ALLOC | SCL_ZIO, RW_WRITER), ==, 1162 (SCL_ALLOC | SCL_ZIO)); 1163 1164 if (--mg->mg_activation_count != 0) { 1165 for (int i = 0; i < spa->spa_alloc_count; i++) 1166 ASSERT(mc->mc_allocator[i].mca_rotor != mg); 1167 ASSERT0P(mg->mg_prev); 1168 ASSERT0P(mg->mg_next); 1169 ASSERT(mg->mg_activation_count < 0); 1170 return; 1171 } 1172 1173 /* 1174 * The spa_config_lock is an array of rwlocks, ordered as 1175 * follows (from highest to lowest): 1176 * SCL_CONFIG > SCL_STATE > SCL_L2ARC > SCL_ALLOC > 1177 * SCL_ZIO > SCL_FREE > SCL_VDEV 1178 * (For more information about the spa_config_lock see spa_misc.c) 1179 * The higher the lock, the broader its coverage. When we passivate 1180 * a metaslab group, we must hold both the SCL_ALLOC and the SCL_ZIO 1181 * config locks. However, the metaslab group's taskq might be trying 1182 * to preload metaslabs so we must drop the SCL_ZIO lock and any 1183 * lower locks to allow the I/O to complete. At a minimum, 1184 * we continue to hold the SCL_ALLOC lock, which prevents any future 1185 * allocations from taking place and any changes to the vdev tree. 1186 */ 1187 spa_config_exit(spa, locks & ~(SCL_ZIO - 1), spa); 1188 taskq_wait_outstanding(spa->spa_metaslab_taskq, 0); 1189 spa_config_enter(spa, locks & ~(SCL_ZIO - 1), spa, RW_WRITER); 1190 metaslab_group_alloc_update(mg); 1191 for (int i = 0; i < spa->spa_alloc_count; i++) { 1192 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 1193 metaslab_t *msp = mga->mga_primary; 1194 if (msp != NULL) { 1195 mutex_enter(&msp->ms_lock); 1196 metaslab_passivate(msp, 1197 metaslab_weight(msp, B_TRUE) & 1198 ~METASLAB_ACTIVE_MASK); 1199 mutex_exit(&msp->ms_lock); 1200 } 1201 msp = mga->mga_secondary; 1202 if (msp != NULL) { 1203 mutex_enter(&msp->ms_lock); 1204 metaslab_passivate(msp, 1205 metaslab_weight(msp, B_TRUE) & 1206 ~METASLAB_ACTIVE_MASK); 1207 mutex_exit(&msp->ms_lock); 1208 } 1209 } 1210 1211 mgprev = mg->mg_prev; 1212 mgnext = mg->mg_next; 1213 1214 if (mg == mgnext) { 1215 mgnext = NULL; 1216 } else { 1217 mgprev->mg_next = mgnext; 1218 mgnext->mg_prev = mgprev; 1219 } 1220 for (int i = 0; i < spa->spa_alloc_count; i++) { 1221 if (mc->mc_allocator[i].mca_rotor == mg) 1222 mc->mc_allocator[i].mca_rotor = mgnext; 1223 } 1224 1225 mg->mg_prev = NULL; 1226 mg->mg_next = NULL; 1227 metaslab_class_balance(mc, B_FALSE); 1228 } 1229 1230 boolean_t 1231 metaslab_group_initialized(metaslab_group_t *mg) 1232 { 1233 vdev_t *vd = mg->mg_vd; 1234 vdev_stat_t *vs = &vd->vdev_stat; 1235 1236 return (vs->vs_space != 0 && mg->mg_activation_count > 0); 1237 } 1238 1239 uint64_t 1240 metaslab_group_get_space(metaslab_group_t *mg) 1241 { 1242 /* 1243 * Note that the number of nodes in mg_metaslab_tree may be one less 1244 * than vdev_ms_count, due to the embedded log metaslab. 1245 */ 1246 mutex_enter(&mg->mg_lock); 1247 uint64_t ms_count = avl_numnodes(&mg->mg_metaslab_tree); 1248 mutex_exit(&mg->mg_lock); 1249 return ((1ULL << mg->mg_vd->vdev_ms_shift) * ms_count); 1250 } 1251 1252 void 1253 metaslab_group_histogram_verify(metaslab_group_t *mg) 1254 { 1255 uint64_t *mg_hist; 1256 avl_tree_t *t = &mg->mg_metaslab_tree; 1257 uint64_t ashift = mg->mg_vd->vdev_ashift; 1258 1259 if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0) 1260 return; 1261 1262 mg_hist = kmem_zalloc(sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE, 1263 KM_SLEEP); 1264 1265 ASSERT3U(ZFS_RANGE_TREE_HISTOGRAM_SIZE, >=, 1266 SPACE_MAP_HISTOGRAM_SIZE + ashift); 1267 1268 mutex_enter(&mg->mg_lock); 1269 for (metaslab_t *msp = avl_first(t); 1270 msp != NULL; msp = AVL_NEXT(t, msp)) { 1271 VERIFY3P(msp->ms_group, ==, mg); 1272 /* skip if not active */ 1273 if (msp->ms_sm == NULL) 1274 continue; 1275 1276 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 1277 mg_hist[i + ashift] += 1278 msp->ms_sm->sm_phys->smp_histogram[i]; 1279 } 1280 } 1281 1282 for (int i = 0; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i ++) 1283 VERIFY3U(mg_hist[i], ==, mg->mg_histogram[i]); 1284 1285 mutex_exit(&mg->mg_lock); 1286 1287 kmem_free(mg_hist, sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE); 1288 } 1289 1290 static void 1291 metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp) 1292 { 1293 metaslab_class_t *mc = mg->mg_class; 1294 uint64_t ashift = mg->mg_vd->vdev_ashift; 1295 1296 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1297 if (msp->ms_sm == NULL) 1298 return; 1299 1300 mutex_enter(&mg->mg_lock); 1301 mutex_enter(&mc->mc_lock); 1302 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 1303 IMPLY(mg == mg->mg_vd->vdev_log_mg, 1304 mc == spa_embedded_log_class(mg->mg_vd->vdev_spa) || 1305 mc == spa_special_embedded_log_class(mg->mg_vd->vdev_spa)); 1306 mg->mg_histogram[i + ashift] += 1307 msp->ms_sm->sm_phys->smp_histogram[i]; 1308 mc->mc_histogram[i + ashift] += 1309 msp->ms_sm->sm_phys->smp_histogram[i]; 1310 } 1311 mutex_exit(&mc->mc_lock); 1312 mutex_exit(&mg->mg_lock); 1313 } 1314 1315 void 1316 metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp) 1317 { 1318 metaslab_class_t *mc = mg->mg_class; 1319 uint64_t ashift = mg->mg_vd->vdev_ashift; 1320 1321 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1322 if (msp->ms_sm == NULL) 1323 return; 1324 1325 mutex_enter(&mg->mg_lock); 1326 mutex_enter(&mc->mc_lock); 1327 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 1328 ASSERT3U(mg->mg_histogram[i + ashift], >=, 1329 msp->ms_sm->sm_phys->smp_histogram[i]); 1330 ASSERT3U(mc->mc_histogram[i + ashift], >=, 1331 msp->ms_sm->sm_phys->smp_histogram[i]); 1332 IMPLY(mg == mg->mg_vd->vdev_log_mg, 1333 mc == spa_embedded_log_class(mg->mg_vd->vdev_spa) || 1334 mc == spa_special_embedded_log_class(mg->mg_vd->vdev_spa)); 1335 1336 mg->mg_histogram[i + ashift] -= 1337 msp->ms_sm->sm_phys->smp_histogram[i]; 1338 mc->mc_histogram[i + ashift] -= 1339 msp->ms_sm->sm_phys->smp_histogram[i]; 1340 } 1341 mutex_exit(&mc->mc_lock); 1342 mutex_exit(&mg->mg_lock); 1343 } 1344 1345 static void 1346 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp) 1347 { 1348 ASSERT0P(msp->ms_group); 1349 mutex_enter(&mg->mg_lock); 1350 msp->ms_group = mg; 1351 msp->ms_weight = 0; 1352 avl_add(&mg->mg_metaslab_tree, msp); 1353 mutex_exit(&mg->mg_lock); 1354 1355 mutex_enter(&msp->ms_lock); 1356 metaslab_group_histogram_add(mg, msp); 1357 mutex_exit(&msp->ms_lock); 1358 } 1359 1360 static void 1361 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp) 1362 { 1363 mutex_enter(&msp->ms_lock); 1364 metaslab_group_histogram_remove(mg, msp); 1365 mutex_exit(&msp->ms_lock); 1366 1367 mutex_enter(&mg->mg_lock); 1368 ASSERT(msp->ms_group == mg); 1369 avl_remove(&mg->mg_metaslab_tree, msp); 1370 1371 metaslab_class_t *mc = msp->ms_group->mg_class; 1372 multilist_sublist_t *mls = 1373 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 1374 if (multilist_link_active(&msp->ms_class_txg_node)) 1375 multilist_sublist_remove(mls, msp); 1376 multilist_sublist_unlock(mls); 1377 1378 msp->ms_group = NULL; 1379 mutex_exit(&mg->mg_lock); 1380 } 1381 1382 static void 1383 metaslab_group_sort_impl(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) 1384 { 1385 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1386 ASSERT(MUTEX_HELD(&mg->mg_lock)); 1387 ASSERT(msp->ms_group == mg); 1388 1389 avl_remove(&mg->mg_metaslab_tree, msp); 1390 msp->ms_weight = weight; 1391 avl_add(&mg->mg_metaslab_tree, msp); 1392 1393 } 1394 1395 static void 1396 metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) 1397 { 1398 /* 1399 * Although in principle the weight can be any value, in 1400 * practice we do not use values in the range [1, 511]. 1401 */ 1402 ASSERT(weight >= SPA_MINBLOCKSIZE || weight == 0); 1403 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1404 1405 mutex_enter(&mg->mg_lock); 1406 metaslab_group_sort_impl(mg, msp, weight); 1407 mutex_exit(&mg->mg_lock); 1408 } 1409 1410 /* 1411 * Calculate the fragmentation for a given metaslab group. Weight metaslabs 1412 * on the amount of free space. The return value will be between 0 and 100 1413 * (inclusive), or ZFS_FRAG_INVALID if less than half of the metaslab in this 1414 * group have a fragmentation metric. 1415 */ 1416 uint64_t 1417 metaslab_group_fragmentation(metaslab_group_t *mg) 1418 { 1419 vdev_t *vd = mg->mg_vd; 1420 uint64_t fragmentation = 0; 1421 uint64_t valid_ms = 0, total_ms = 0; 1422 uint64_t free, total_free = 0; 1423 1424 for (int m = 0; m < vd->vdev_ms_count; m++) { 1425 metaslab_t *msp = vd->vdev_ms[m]; 1426 1427 if (msp->ms_group != mg) 1428 continue; 1429 total_ms++; 1430 if (msp->ms_fragmentation == ZFS_FRAG_INVALID) 1431 continue; 1432 1433 valid_ms++; 1434 free = (msp->ms_size - metaslab_allocated_space(msp)) / 1435 SPA_MINBLOCKSIZE; /* To prevent overflows. */ 1436 total_free += free; 1437 fragmentation += msp->ms_fragmentation * free; 1438 } 1439 1440 if (valid_ms < (total_ms + 1) / 2 || total_free == 0) 1441 return (ZFS_FRAG_INVALID); 1442 1443 fragmentation /= total_free; 1444 ASSERT3U(fragmentation, <=, 100); 1445 return (fragmentation); 1446 } 1447 1448 /* 1449 * ========================================================================== 1450 * Range tree callbacks 1451 * ========================================================================== 1452 */ 1453 1454 /* 1455 * Comparison function for the private size-ordered tree using 32-bit 1456 * ranges. Tree is sorted by size, larger sizes at the end of the tree. 1457 */ 1458 __attribute__((always_inline)) inline 1459 static int 1460 metaslab_rangesize32_compare(const void *x1, const void *x2) 1461 { 1462 const zfs_range_seg32_t *r1 = x1; 1463 const zfs_range_seg32_t *r2 = x2; 1464 1465 uint64_t rs_size1 = r1->rs_end - r1->rs_start; 1466 uint64_t rs_size2 = r2->rs_end - r2->rs_start; 1467 1468 int cmp = TREE_CMP(rs_size1, rs_size2); 1469 1470 return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start)); 1471 } 1472 1473 /* 1474 * Comparison function for the private size-ordered tree using 64-bit 1475 * ranges. Tree is sorted by size, larger sizes at the end of the tree. 1476 */ 1477 __attribute__((always_inline)) inline 1478 static int 1479 metaslab_rangesize64_compare(const void *x1, const void *x2) 1480 { 1481 const zfs_range_seg64_t *r1 = x1; 1482 const zfs_range_seg64_t *r2 = x2; 1483 1484 uint64_t rs_size1 = r1->rs_end - r1->rs_start; 1485 uint64_t rs_size2 = r2->rs_end - r2->rs_start; 1486 1487 int cmp = TREE_CMP(rs_size1, rs_size2); 1488 1489 return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start)); 1490 } 1491 1492 typedef struct metaslab_rt_arg { 1493 zfs_btree_t *mra_bt; 1494 uint32_t mra_floor_shift; 1495 } metaslab_rt_arg_t; 1496 1497 struct mssa_arg { 1498 zfs_range_tree_t *rt; 1499 metaslab_rt_arg_t *mra; 1500 }; 1501 1502 static void 1503 metaslab_size_sorted_add(void *arg, uint64_t start, uint64_t size) 1504 { 1505 struct mssa_arg *mssap = arg; 1506 zfs_range_tree_t *rt = mssap->rt; 1507 metaslab_rt_arg_t *mrap = mssap->mra; 1508 zfs_range_seg_max_t seg = {0}; 1509 zfs_rs_set_start(&seg, rt, start); 1510 zfs_rs_set_end(&seg, rt, start + size); 1511 metaslab_rt_add(rt, &seg, mrap); 1512 } 1513 1514 static void 1515 metaslab_size_tree_full_load(zfs_range_tree_t *rt) 1516 { 1517 metaslab_rt_arg_t *mrap = rt->rt_arg; 1518 METASLABSTAT_BUMP(metaslabstat_reload_tree); 1519 ASSERT0(zfs_btree_numnodes(mrap->mra_bt)); 1520 mrap->mra_floor_shift = 0; 1521 struct mssa_arg arg = {0}; 1522 arg.rt = rt; 1523 arg.mra = mrap; 1524 zfs_range_tree_walk(rt, metaslab_size_sorted_add, &arg); 1525 } 1526 1527 1528 ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize32_in_buf, 1529 zfs_range_seg32_t, metaslab_rangesize32_compare) 1530 1531 ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize64_in_buf, 1532 zfs_range_seg64_t, metaslab_rangesize64_compare) 1533 1534 /* 1535 * Create any block allocator specific components. The current allocators 1536 * rely on using both a size-ordered zfs_range_tree_t and an array of 1537 * uint64_t's. 1538 */ 1539 static void 1540 metaslab_rt_create(zfs_range_tree_t *rt, void *arg) 1541 { 1542 metaslab_rt_arg_t *mrap = arg; 1543 zfs_btree_t *size_tree = mrap->mra_bt; 1544 1545 size_t size; 1546 int (*compare) (const void *, const void *); 1547 bt_find_in_buf_f bt_find; 1548 switch (rt->rt_type) { 1549 case ZFS_RANGE_SEG32: 1550 size = sizeof (zfs_range_seg32_t); 1551 compare = metaslab_rangesize32_compare; 1552 bt_find = metaslab_rt_find_rangesize32_in_buf; 1553 break; 1554 case ZFS_RANGE_SEG64: 1555 size = sizeof (zfs_range_seg64_t); 1556 compare = metaslab_rangesize64_compare; 1557 bt_find = metaslab_rt_find_rangesize64_in_buf; 1558 break; 1559 default: 1560 panic("Invalid range seg type %d", rt->rt_type); 1561 } 1562 zfs_btree_create(size_tree, compare, bt_find, size); 1563 mrap->mra_floor_shift = metaslab_by_size_min_shift; 1564 } 1565 1566 static void 1567 metaslab_rt_destroy(zfs_range_tree_t *rt, void *arg) 1568 { 1569 (void) rt; 1570 metaslab_rt_arg_t *mrap = arg; 1571 zfs_btree_t *size_tree = mrap->mra_bt; 1572 1573 zfs_btree_destroy(size_tree); 1574 kmem_free(mrap, sizeof (*mrap)); 1575 } 1576 1577 static void 1578 metaslab_rt_add(zfs_range_tree_t *rt, zfs_range_seg_t *rs, void *arg) 1579 { 1580 metaslab_rt_arg_t *mrap = arg; 1581 zfs_btree_t *size_tree = mrap->mra_bt; 1582 1583 if (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt) < 1584 (1ULL << mrap->mra_floor_shift)) 1585 return; 1586 1587 zfs_btree_add(size_tree, rs); 1588 } 1589 1590 static void 1591 metaslab_rt_remove(zfs_range_tree_t *rt, zfs_range_seg_t *rs, void *arg) 1592 { 1593 metaslab_rt_arg_t *mrap = arg; 1594 zfs_btree_t *size_tree = mrap->mra_bt; 1595 1596 if (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt) < (1ULL << 1597 mrap->mra_floor_shift)) 1598 return; 1599 1600 zfs_btree_remove(size_tree, rs); 1601 } 1602 1603 static void 1604 metaslab_rt_vacate(zfs_range_tree_t *rt, void *arg) 1605 { 1606 metaslab_rt_arg_t *mrap = arg; 1607 zfs_btree_t *size_tree = mrap->mra_bt; 1608 zfs_btree_clear(size_tree); 1609 zfs_btree_destroy(size_tree); 1610 1611 metaslab_rt_create(rt, arg); 1612 } 1613 1614 static const zfs_range_tree_ops_t metaslab_rt_ops = { 1615 .rtop_create = metaslab_rt_create, 1616 .rtop_destroy = metaslab_rt_destroy, 1617 .rtop_add = metaslab_rt_add, 1618 .rtop_remove = metaslab_rt_remove, 1619 .rtop_vacate = metaslab_rt_vacate 1620 }; 1621 1622 /* 1623 * ========================================================================== 1624 * Common allocator routines 1625 * ========================================================================== 1626 */ 1627 1628 /* 1629 * Return the maximum contiguous segment within the metaslab. 1630 */ 1631 uint64_t 1632 metaslab_largest_allocatable(metaslab_t *msp) 1633 { 1634 zfs_btree_t *t = &msp->ms_allocatable_by_size; 1635 zfs_range_seg_t *rs; 1636 1637 if (t == NULL) 1638 return (0); 1639 if (zfs_btree_numnodes(t) == 0) 1640 metaslab_size_tree_full_load(msp->ms_allocatable); 1641 1642 rs = zfs_btree_last(t, NULL); 1643 if (rs == NULL) 1644 return (0); 1645 1646 return (zfs_rs_get_end(rs, msp->ms_allocatable) - zfs_rs_get_start(rs, 1647 msp->ms_allocatable)); 1648 } 1649 1650 /* 1651 * Return the maximum contiguous segment within the unflushed frees of this 1652 * metaslab. 1653 */ 1654 static uint64_t 1655 metaslab_largest_unflushed_free(metaslab_t *msp) 1656 { 1657 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1658 1659 if (msp->ms_unflushed_frees == NULL) 1660 return (0); 1661 1662 if (zfs_btree_numnodes(&msp->ms_unflushed_frees_by_size) == 0) 1663 metaslab_size_tree_full_load(msp->ms_unflushed_frees); 1664 zfs_range_seg_t *rs = zfs_btree_last(&msp->ms_unflushed_frees_by_size, 1665 NULL); 1666 if (rs == NULL) 1667 return (0); 1668 1669 /* 1670 * When a range is freed from the metaslab, that range is added to 1671 * both the unflushed frees and the deferred frees. While the block 1672 * will eventually be usable, if the metaslab were loaded the range 1673 * would not be added to the ms_allocatable tree until TXG_DEFER_SIZE 1674 * txgs had passed. As a result, when attempting to estimate an upper 1675 * bound for the largest currently-usable free segment in the 1676 * metaslab, we need to not consider any ranges currently in the defer 1677 * trees. This algorithm approximates the largest available chunk in 1678 * the largest range in the unflushed_frees tree by taking the first 1679 * chunk. While this may be a poor estimate, it should only remain so 1680 * briefly and should eventually self-correct as frees are no longer 1681 * deferred. Similar logic applies to the ms_freed tree. See 1682 * metaslab_load() for more details. 1683 * 1684 * There are two primary sources of inaccuracy in this estimate. Both 1685 * are tolerated for performance reasons. The first source is that we 1686 * only check the largest segment for overlaps. Smaller segments may 1687 * have more favorable overlaps with the other trees, resulting in 1688 * larger usable chunks. Second, we only look at the first chunk in 1689 * the largest segment; there may be other usable chunks in the 1690 * largest segment, but we ignore them. 1691 */ 1692 uint64_t rstart = zfs_rs_get_start(rs, msp->ms_unflushed_frees); 1693 uint64_t rsize = zfs_rs_get_end(rs, msp->ms_unflushed_frees) - rstart; 1694 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 1695 uint64_t start = 0; 1696 uint64_t size = 0; 1697 boolean_t found = zfs_range_tree_find_in(msp->ms_defer[t], 1698 rstart, rsize, &start, &size); 1699 if (found) { 1700 if (rstart == start) 1701 return (0); 1702 rsize = start - rstart; 1703 } 1704 } 1705 1706 uint64_t start = 0; 1707 uint64_t size = 0; 1708 boolean_t found = zfs_range_tree_find_in(msp->ms_freed, rstart, 1709 rsize, &start, &size); 1710 if (found) 1711 rsize = start - rstart; 1712 1713 return (rsize); 1714 } 1715 1716 static zfs_range_seg_t * 1717 metaslab_block_find(zfs_btree_t *t, zfs_range_tree_t *rt, uint64_t start, 1718 uint64_t size, uint64_t max_size, zfs_btree_index_t *where) 1719 { 1720 zfs_range_seg_t *rs; 1721 zfs_range_seg_max_t rsearch; 1722 1723 zfs_rs_set_start(&rsearch, rt, start); 1724 zfs_rs_set_end(&rsearch, rt, start + max_size); 1725 1726 rs = zfs_btree_find(t, &rsearch, where); 1727 if (rs == NULL) { 1728 if (size == max_size) { 1729 rs = zfs_btree_next(t, where, where); 1730 } else { 1731 /* 1732 * If we're searching for a range, get the largest 1733 * segment in that range, or the smallest one bigger 1734 * than it. 1735 */ 1736 rs = zfs_btree_prev(t, where, where); 1737 if (rs == NULL || zfs_rs_get_end(rs, rt) - 1738 zfs_rs_get_start(rs, rt) < size) { 1739 rs = zfs_btree_next(t, where, where); 1740 } 1741 } 1742 } 1743 1744 return (rs); 1745 } 1746 1747 /* 1748 * This is a helper function that can be used by the allocator to find a 1749 * suitable block to allocate. This will search the specified B-tree looking 1750 * for a block that matches the specified criteria. 1751 */ 1752 static uint64_t 1753 metaslab_block_picker(zfs_range_tree_t *rt, uint64_t *cursor, uint64_t size, 1754 uint64_t max_size, uint64_t max_search, uint64_t *found_size) 1755 { 1756 if (*cursor == 0) 1757 *cursor = rt->rt_start; 1758 zfs_btree_t *bt = &rt->rt_root; 1759 zfs_btree_index_t where; 1760 zfs_range_seg_t *rs = metaslab_block_find(bt, rt, *cursor, size, 1761 max_size, &where); 1762 uint64_t first_found; 1763 int count_searched = 0; 1764 1765 if (rs != NULL) 1766 first_found = zfs_rs_get_start(rs, rt); 1767 1768 while (rs != NULL && (zfs_rs_get_start(rs, rt) - first_found <= 1769 max_search || count_searched < metaslab_min_search_count)) { 1770 uint64_t offset = zfs_rs_get_start(rs, rt); 1771 if (offset + size <= zfs_rs_get_end(rs, rt)) { 1772 *found_size = MIN(zfs_rs_get_end(rs, rt) - offset, 1773 max_size); 1774 *cursor = offset + *found_size; 1775 return (offset); 1776 } 1777 rs = zfs_btree_next(bt, &where, &where); 1778 count_searched++; 1779 } 1780 1781 *cursor = 0; 1782 *found_size = 0; 1783 return (-1ULL); 1784 } 1785 1786 static uint64_t metaslab_df_alloc(metaslab_t *msp, uint64_t size, 1787 uint64_t max_size, uint64_t *found_size); 1788 static uint64_t metaslab_cf_alloc(metaslab_t *msp, uint64_t size, 1789 uint64_t max_size, uint64_t *found_size); 1790 static uint64_t metaslab_ndf_alloc(metaslab_t *msp, uint64_t size, 1791 uint64_t max_size, uint64_t *found_size); 1792 metaslab_ops_t *metaslab_allocator(spa_t *spa); 1793 1794 static metaslab_ops_t metaslab_allocators[] = { 1795 { "dynamic", metaslab_df_alloc }, 1796 { "cursor", metaslab_cf_alloc }, 1797 { "new-dynamic", metaslab_ndf_alloc }, 1798 }; 1799 1800 static int 1801 spa_find_allocator_byname(const char *val) 1802 { 1803 int a = ARRAY_SIZE(metaslab_allocators) - 1; 1804 if (strcmp("new-dynamic", val) == 0) 1805 return (-1); /* remove when ndf is working */ 1806 for (; a >= 0; a--) { 1807 if (strcmp(val, metaslab_allocators[a].msop_name) == 0) 1808 return (a); 1809 } 1810 return (-1); 1811 } 1812 1813 void 1814 spa_set_allocator(spa_t *spa, const char *allocator) 1815 { 1816 int a = spa_find_allocator_byname(allocator); 1817 if (a < 0) a = 0; 1818 spa->spa_active_allocator = a; 1819 zfs_dbgmsg("spa allocator: %s", metaslab_allocators[a].msop_name); 1820 } 1821 1822 int 1823 spa_get_allocator(spa_t *spa) 1824 { 1825 return (spa->spa_active_allocator); 1826 } 1827 1828 #if defined(_KERNEL) 1829 int 1830 param_set_active_allocator_common(const char *val) 1831 { 1832 char *p; 1833 1834 if (val == NULL) 1835 return (SET_ERROR(EINVAL)); 1836 1837 if ((p = strchr(val, '\n')) != NULL) 1838 *p = '\0'; 1839 1840 int a = spa_find_allocator_byname(val); 1841 if (a < 0) 1842 return (SET_ERROR(EINVAL)); 1843 1844 zfs_active_allocator = metaslab_allocators[a].msop_name; 1845 return (0); 1846 } 1847 #endif 1848 1849 metaslab_ops_t * 1850 metaslab_allocator(spa_t *spa) 1851 { 1852 int allocator = spa_get_allocator(spa); 1853 return (&metaslab_allocators[allocator]); 1854 } 1855 1856 /* 1857 * ========================================================================== 1858 * Dynamic Fit (df) block allocator 1859 * 1860 * Search for a free chunk of at least this size, starting from the last 1861 * offset (for this alignment of block) looking for up to 1862 * metaslab_df_max_search bytes (16MB). If a large enough free chunk is not 1863 * found within 16MB, then return a free chunk of exactly the requested size (or 1864 * larger). 1865 * 1866 * If it seems like searching from the last offset will be unproductive, skip 1867 * that and just return a free chunk of exactly the requested size (or larger). 1868 * This is based on metaslab_df_alloc_threshold and metaslab_df_free_pct. This 1869 * mechanism is probably not very useful and may be removed in the future. 1870 * 1871 * The behavior when not searching can be changed to return the largest free 1872 * chunk, instead of a free chunk of exactly the requested size, by setting 1873 * metaslab_df_use_largest_segment. 1874 * ========================================================================== 1875 */ 1876 static uint64_t 1877 metaslab_df_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 1878 uint64_t *found_size) 1879 { 1880 /* 1881 * Find the largest power of 2 block size that evenly divides the 1882 * requested size. This is used to try to allocate blocks with similar 1883 * alignment from the same area of the metaslab (i.e. same cursor 1884 * bucket) but it does not guarantee that other allocations sizes 1885 * may exist in the same region. 1886 */ 1887 uint64_t align = max_size & -max_size; 1888 uint64_t *cursor = &msp->ms_lbas[highbit64(align) - 1]; 1889 zfs_range_tree_t *rt = msp->ms_allocatable; 1890 uint_t free_pct = zfs_range_tree_space(rt) * 100 / msp->ms_size; 1891 uint64_t offset; 1892 1893 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1894 1895 /* 1896 * If we're running low on space, find a segment based on size, 1897 * rather than iterating based on offset. 1898 */ 1899 if (metaslab_largest_allocatable(msp) < metaslab_df_alloc_threshold || 1900 free_pct < metaslab_df_free_pct) { 1901 align = size & -size; 1902 cursor = &msp->ms_lbas[highbit64(align) - 1]; 1903 offset = -1; 1904 } else { 1905 offset = metaslab_block_picker(rt, cursor, size, max_size, 1906 metaslab_df_max_search, found_size); 1907 if (max_size != size && offset == -1) { 1908 align = size & -size; 1909 cursor = &msp->ms_lbas[highbit64(align) - 1]; 1910 offset = metaslab_block_picker(rt, cursor, size, 1911 max_size, metaslab_df_max_search, found_size); 1912 } 1913 } 1914 1915 if (offset == -1) { 1916 zfs_range_seg_t *rs; 1917 if (zfs_btree_numnodes(&msp->ms_allocatable_by_size) == 0) 1918 metaslab_size_tree_full_load(msp->ms_allocatable); 1919 1920 if (metaslab_df_use_largest_segment) { 1921 /* use largest free segment */ 1922 rs = zfs_btree_last(&msp->ms_allocatable_by_size, NULL); 1923 } else { 1924 zfs_btree_index_t where; 1925 /* use segment of this size, or next largest */ 1926 rs = metaslab_block_find(&msp->ms_allocatable_by_size, 1927 rt, msp->ms_start, size, max_size, &where); 1928 } 1929 if (rs != NULL && zfs_rs_get_start(rs, rt) + size <= 1930 zfs_rs_get_end(rs, rt)) { 1931 offset = zfs_rs_get_start(rs, rt); 1932 *found_size = MIN(zfs_rs_get_end(rs, rt) - offset, 1933 max_size); 1934 *cursor = offset + *found_size; 1935 } 1936 } 1937 1938 return (offset); 1939 } 1940 1941 /* 1942 * ========================================================================== 1943 * Cursor fit block allocator - 1944 * Select the largest region in the metaslab, set the cursor to the beginning 1945 * of the range and the cursor_end to the end of the range. As allocations 1946 * are made advance the cursor. Continue allocating from the cursor until 1947 * the range is exhausted and then find a new range. 1948 * ========================================================================== 1949 */ 1950 static uint64_t 1951 metaslab_cf_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 1952 uint64_t *found_size) 1953 { 1954 zfs_range_tree_t *rt = msp->ms_allocatable; 1955 zfs_btree_t *t = &msp->ms_allocatable_by_size; 1956 uint64_t *cursor = &msp->ms_lbas[0]; 1957 uint64_t *cursor_end = &msp->ms_lbas[1]; 1958 uint64_t offset = 0; 1959 1960 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1961 1962 ASSERT3U(*cursor_end, >=, *cursor); 1963 1964 if ((*cursor + size) > *cursor_end) { 1965 zfs_range_seg_t *rs; 1966 1967 if (zfs_btree_numnodes(t) == 0) 1968 metaslab_size_tree_full_load(msp->ms_allocatable); 1969 rs = zfs_btree_last(t, NULL); 1970 if (rs == NULL || (zfs_rs_get_end(rs, rt) - 1971 zfs_rs_get_start(rs, rt)) < size) 1972 return (-1ULL); 1973 1974 *cursor = zfs_rs_get_start(rs, rt); 1975 *cursor_end = zfs_rs_get_end(rs, rt); 1976 } 1977 1978 offset = *cursor; 1979 *found_size = MIN(*cursor_end - offset, max_size); 1980 *cursor = offset + *found_size; 1981 1982 return (offset); 1983 } 1984 1985 /* 1986 * ========================================================================== 1987 * New dynamic fit allocator - 1988 * Select a region that is large enough to allocate 2^metaslab_ndf_clump_shift 1989 * contiguous blocks. If no region is found then just use the largest segment 1990 * that remains. 1991 * ========================================================================== 1992 */ 1993 1994 /* 1995 * Determines desired number of contiguous blocks (2^metaslab_ndf_clump_shift) 1996 * to request from the allocator. 1997 */ 1998 uint64_t metaslab_ndf_clump_shift = 4; 1999 2000 static uint64_t 2001 metaslab_ndf_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 2002 uint64_t *found_size) 2003 { 2004 zfs_btree_t *t = &msp->ms_allocatable->rt_root; 2005 zfs_range_tree_t *rt = msp->ms_allocatable; 2006 zfs_btree_index_t where; 2007 zfs_range_seg_t *rs; 2008 zfs_range_seg_max_t rsearch; 2009 uint64_t hbit = highbit64(max_size); 2010 uint64_t *cursor = &msp->ms_lbas[hbit - 1]; 2011 uint64_t max_possible_size = metaslab_largest_allocatable(msp); 2012 2013 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2014 2015 if (max_possible_size < size) 2016 return (-1ULL); 2017 2018 zfs_rs_set_start(&rsearch, rt, *cursor); 2019 zfs_rs_set_end(&rsearch, rt, *cursor + max_size); 2020 2021 rs = zfs_btree_find(t, &rsearch, &where); 2022 if (rs == NULL || (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt)) < 2023 max_size) { 2024 hbit = highbit64(size); 2025 cursor = &msp->ms_lbas[hbit - 1]; 2026 zfs_rs_set_start(&rsearch, rt, *cursor); 2027 zfs_rs_set_end(&rsearch, rt, *cursor + size); 2028 2029 rs = zfs_btree_find(t, &rsearch, &where); 2030 } 2031 if (rs == NULL || (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt)) < 2032 size) { 2033 t = &msp->ms_allocatable_by_size; 2034 2035 zfs_rs_set_start(&rsearch, rt, 0); 2036 zfs_rs_set_end(&rsearch, rt, MIN(max_possible_size, 2037 1ULL << (hbit + metaslab_ndf_clump_shift))); 2038 2039 rs = zfs_btree_find(t, &rsearch, &where); 2040 if (rs == NULL) 2041 rs = zfs_btree_next(t, &where, &where); 2042 ASSERT(rs != NULL); 2043 } 2044 2045 if ((zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt)) >= size) { 2046 *found_size = MIN(zfs_rs_get_end(rs, rt) - 2047 zfs_rs_get_start(rs, rt), max_size); 2048 *cursor = zfs_rs_get_start(rs, rt) + *found_size; 2049 return (zfs_rs_get_start(rs, rt)); 2050 } 2051 return (-1ULL); 2052 } 2053 2054 /* 2055 * ========================================================================== 2056 * Metaslabs 2057 * ========================================================================== 2058 */ 2059 2060 /* 2061 * Wait for any in-progress metaslab loads to complete. 2062 */ 2063 static void 2064 metaslab_load_wait(metaslab_t *msp) 2065 { 2066 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2067 2068 while (msp->ms_loading) { 2069 ASSERT(!msp->ms_loaded); 2070 cv_wait(&msp->ms_load_cv, &msp->ms_lock); 2071 } 2072 } 2073 2074 /* 2075 * Wait for any in-progress flushing to complete. 2076 */ 2077 static void 2078 metaslab_flush_wait(metaslab_t *msp) 2079 { 2080 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2081 2082 while (msp->ms_flushing) 2083 cv_wait(&msp->ms_flush_cv, &msp->ms_lock); 2084 } 2085 2086 static unsigned int 2087 metaslab_idx_func(multilist_t *ml, void *arg) 2088 { 2089 metaslab_t *msp = arg; 2090 2091 /* 2092 * ms_id values are allocated sequentially, so full 64bit 2093 * division would be a waste of time, so limit it to 32 bits. 2094 */ 2095 return ((unsigned int)msp->ms_id % multilist_get_num_sublists(ml)); 2096 } 2097 2098 uint64_t 2099 metaslab_allocated_space(metaslab_t *msp) 2100 { 2101 return (msp->ms_allocated_space); 2102 } 2103 2104 /* 2105 * Verify that the space accounting on disk matches the in-core range_trees. 2106 */ 2107 static void 2108 metaslab_verify_space(metaslab_t *msp, uint64_t txg) 2109 { 2110 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2111 uint64_t allocating = 0; 2112 uint64_t sm_free_space, msp_free_space; 2113 2114 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2115 ASSERT(!msp->ms_condensing); 2116 2117 if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0) 2118 return; 2119 2120 /* 2121 * We can only verify the metaslab space when we're called 2122 * from syncing context with a loaded metaslab that has an 2123 * allocated space map. Calling this in non-syncing context 2124 * does not provide a consistent view of the metaslab since 2125 * we're performing allocations in the future. 2126 */ 2127 if (txg != spa_syncing_txg(spa) || msp->ms_sm == NULL || 2128 !msp->ms_loaded) 2129 return; 2130 2131 /* 2132 * Even though the smp_alloc field can get negative, 2133 * when it comes to a metaslab's space map, that should 2134 * never be the case. 2135 */ 2136 ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0); 2137 2138 ASSERT3U(space_map_allocated(msp->ms_sm), >=, 2139 zfs_range_tree_space(msp->ms_unflushed_frees)); 2140 2141 ASSERT3U(metaslab_allocated_space(msp), ==, 2142 space_map_allocated(msp->ms_sm) + 2143 zfs_range_tree_space(msp->ms_unflushed_allocs) - 2144 zfs_range_tree_space(msp->ms_unflushed_frees)); 2145 2146 sm_free_space = msp->ms_size - metaslab_allocated_space(msp); 2147 2148 /* 2149 * Account for future allocations since we would have 2150 * already deducted that space from the ms_allocatable. 2151 */ 2152 for (int t = 0; t < TXG_CONCURRENT_STATES; t++) { 2153 allocating += 2154 zfs_range_tree_space(msp->ms_allocating[(txg + t) & 2155 TXG_MASK]); 2156 } 2157 ASSERT3U(allocating + msp->ms_allocated_this_txg, ==, 2158 msp->ms_allocating_total); 2159 2160 ASSERT3U(msp->ms_deferspace, ==, 2161 zfs_range_tree_space(msp->ms_defer[0]) + 2162 zfs_range_tree_space(msp->ms_defer[1])); 2163 2164 msp_free_space = zfs_range_tree_space(msp->ms_allocatable) + 2165 allocating + msp->ms_deferspace + 2166 zfs_range_tree_space(msp->ms_freed); 2167 2168 VERIFY3U(sm_free_space, ==, msp_free_space); 2169 } 2170 2171 static void 2172 metaslab_aux_histograms_clear(metaslab_t *msp) 2173 { 2174 /* 2175 * Auxiliary histograms are only cleared when resetting them, 2176 * which can only happen while the metaslab is loaded. 2177 */ 2178 ASSERT(msp->ms_loaded); 2179 2180 memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist)); 2181 for (int t = 0; t < TXG_DEFER_SIZE; t++) 2182 memset(msp->ms_deferhist[t], 0, sizeof (msp->ms_deferhist[t])); 2183 } 2184 2185 static void 2186 metaslab_aux_histogram_add(uint64_t *histogram, uint64_t shift, 2187 zfs_range_tree_t *rt) 2188 { 2189 /* 2190 * This is modeled after space_map_histogram_add(), so refer to that 2191 * function for implementation details. We want this to work like 2192 * the space map histogram, and not the range tree histogram, as we 2193 * are essentially constructing a delta that will be later subtracted 2194 * from the space map histogram. 2195 */ 2196 int idx = 0; 2197 for (int i = shift; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i++) { 2198 ASSERT3U(i, >=, idx + shift); 2199 histogram[idx] += rt->rt_histogram[i] << (i - idx - shift); 2200 2201 if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) { 2202 ASSERT3U(idx + shift, ==, i); 2203 idx++; 2204 ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE); 2205 } 2206 } 2207 } 2208 2209 /* 2210 * Called at every sync pass that the metaslab gets synced. 2211 * 2212 * The reason is that we want our auxiliary histograms to be updated 2213 * wherever the metaslab's space map histogram is updated. This way 2214 * we stay consistent on which parts of the metaslab space map's 2215 * histogram are currently not available for allocations (e.g because 2216 * they are in the defer, freed, and freeing trees). 2217 */ 2218 static void 2219 metaslab_aux_histograms_update(metaslab_t *msp) 2220 { 2221 space_map_t *sm = msp->ms_sm; 2222 ASSERT(sm != NULL); 2223 2224 /* 2225 * This is similar to the metaslab's space map histogram updates 2226 * that take place in metaslab_sync(). The only difference is that 2227 * we only care about segments that haven't made it into the 2228 * ms_allocatable tree yet. 2229 */ 2230 if (msp->ms_loaded) { 2231 metaslab_aux_histograms_clear(msp); 2232 2233 metaslab_aux_histogram_add(msp->ms_synchist, 2234 sm->sm_shift, msp->ms_freed); 2235 2236 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 2237 metaslab_aux_histogram_add(msp->ms_deferhist[t], 2238 sm->sm_shift, msp->ms_defer[t]); 2239 } 2240 } 2241 2242 metaslab_aux_histogram_add(msp->ms_synchist, 2243 sm->sm_shift, msp->ms_freeing); 2244 } 2245 2246 /* 2247 * Called every time we are done syncing (writing to) the metaslab, 2248 * i.e. at the end of each sync pass. 2249 * [see the comment in metaslab_impl.h for ms_synchist, ms_deferhist] 2250 */ 2251 static void 2252 metaslab_aux_histograms_update_done(metaslab_t *msp, boolean_t defer_allowed) 2253 { 2254 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2255 space_map_t *sm = msp->ms_sm; 2256 2257 if (sm == NULL) { 2258 /* 2259 * We came here from metaslab_init() when creating/opening a 2260 * pool, looking at a metaslab that hasn't had any allocations 2261 * yet. 2262 */ 2263 return; 2264 } 2265 2266 /* 2267 * This is similar to the actions that we take for the ms_freed 2268 * and ms_defer trees in metaslab_sync_done(). 2269 */ 2270 uint64_t hist_index = spa_syncing_txg(spa) % TXG_DEFER_SIZE; 2271 if (defer_allowed) { 2272 memcpy(msp->ms_deferhist[hist_index], msp->ms_synchist, 2273 sizeof (msp->ms_synchist)); 2274 } else { 2275 memset(msp->ms_deferhist[hist_index], 0, 2276 sizeof (msp->ms_deferhist[hist_index])); 2277 } 2278 memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist)); 2279 } 2280 2281 /* 2282 * Ensure that the metaslab's weight and fragmentation are consistent 2283 * with the contents of the histogram (either the range tree's histogram 2284 * or the space map's depending whether the metaslab is loaded). 2285 */ 2286 static void 2287 metaslab_verify_weight_and_frag(metaslab_t *msp) 2288 { 2289 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2290 2291 if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0) 2292 return; 2293 2294 /* 2295 * We can end up here from vdev_remove_complete(), in which case we 2296 * cannot do these assertions because we hold spa config locks and 2297 * thus we are not allowed to read from the DMU. 2298 * 2299 * We check if the metaslab group has been removed and if that's 2300 * the case we return immediately as that would mean that we are 2301 * here from the aforementioned code path. 2302 */ 2303 if (msp->ms_group == NULL) 2304 return; 2305 2306 /* 2307 * Devices being removed always return a weight of 0 and leave 2308 * fragmentation and ms_max_size as is - there is nothing for 2309 * us to verify here. 2310 */ 2311 vdev_t *vd = msp->ms_group->mg_vd; 2312 if (vd->vdev_removing) 2313 return; 2314 2315 /* 2316 * If the metaslab is dirty it probably means that we've done 2317 * some allocations or frees that have changed our histograms 2318 * and thus the weight. 2319 */ 2320 for (int t = 0; t < TXG_SIZE; t++) { 2321 if (txg_list_member(&vd->vdev_ms_list, msp, t)) 2322 return; 2323 } 2324 2325 /* 2326 * This verification checks that our in-memory state is consistent 2327 * with what's on disk. If the pool is read-only then there aren't 2328 * any changes and we just have the initially-loaded state. 2329 */ 2330 if (!spa_writeable(msp->ms_group->mg_vd->vdev_spa)) 2331 return; 2332 2333 /* some extra verification for in-core tree if you can */ 2334 if (msp->ms_loaded) { 2335 zfs_range_tree_stat_verify(msp->ms_allocatable); 2336 VERIFY(space_map_histogram_verify(msp->ms_sm, 2337 msp->ms_allocatable)); 2338 } 2339 2340 uint64_t weight = msp->ms_weight; 2341 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK; 2342 boolean_t space_based = WEIGHT_IS_SPACEBASED(msp->ms_weight); 2343 uint64_t frag = msp->ms_fragmentation; 2344 uint64_t max_segsize = msp->ms_max_size; 2345 2346 msp->ms_weight = 0; 2347 msp->ms_fragmentation = 0; 2348 2349 /* 2350 * This function is used for verification purposes and thus should 2351 * not introduce any side-effects/mutations on the system's state. 2352 * 2353 * Regardless of whether metaslab_weight() thinks this metaslab 2354 * should be active or not, we want to ensure that the actual weight 2355 * (and therefore the value of ms_weight) would be the same if it 2356 * was to be recalculated at this point. 2357 * 2358 * In addition we set the nodirty flag so metaslab_weight() does 2359 * not dirty the metaslab for future TXGs (e.g. when trying to 2360 * force condensing to upgrade the metaslab spacemaps). 2361 */ 2362 msp->ms_weight = metaslab_weight(msp, B_TRUE) | was_active; 2363 2364 VERIFY3U(max_segsize, ==, msp->ms_max_size); 2365 2366 /* 2367 * If the weight type changed then there is no point in doing 2368 * verification. Revert fields to their original values. 2369 */ 2370 if ((space_based && !WEIGHT_IS_SPACEBASED(msp->ms_weight)) || 2371 (!space_based && WEIGHT_IS_SPACEBASED(msp->ms_weight))) { 2372 msp->ms_fragmentation = frag; 2373 msp->ms_weight = weight; 2374 return; 2375 } 2376 2377 VERIFY3U(msp->ms_fragmentation, ==, frag); 2378 VERIFY3U(msp->ms_weight, ==, weight); 2379 } 2380 2381 /* 2382 * If we're over the zfs_metaslab_mem_limit, select the loaded metaslab from 2383 * this class that was used longest ago, and attempt to unload it. We don't 2384 * want to spend too much time in this loop to prevent performance 2385 * degradation, and we expect that most of the time this operation will 2386 * succeed. Between that and the normal unloading processing during txg sync, 2387 * we expect this to keep the metaslab memory usage under control. 2388 */ 2389 static void 2390 metaslab_potentially_evict(metaslab_class_t *mc) 2391 { 2392 #ifdef _KERNEL 2393 uint64_t allmem = arc_all_memory(); 2394 uint64_t inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache); 2395 uint64_t size = spl_kmem_cache_entry_size(zfs_btree_leaf_cache); 2396 uint_t tries = 0; 2397 for (; allmem * zfs_metaslab_mem_limit / 100 < inuse * size && 2398 tries < multilist_get_num_sublists(&mc->mc_metaslab_txg_list) * 2; 2399 tries++) { 2400 unsigned int idx = multilist_get_random_index( 2401 &mc->mc_metaslab_txg_list); 2402 multilist_sublist_t *mls = 2403 multilist_sublist_lock_idx(&mc->mc_metaslab_txg_list, idx); 2404 metaslab_t *msp = multilist_sublist_head(mls); 2405 multilist_sublist_unlock(mls); 2406 while (msp != NULL && allmem * zfs_metaslab_mem_limit / 100 < 2407 inuse * size) { 2408 VERIFY3P(mls, ==, multilist_sublist_lock_idx( 2409 &mc->mc_metaslab_txg_list, idx)); 2410 ASSERT3U(idx, ==, 2411 metaslab_idx_func(&mc->mc_metaslab_txg_list, msp)); 2412 2413 if (!multilist_link_active(&msp->ms_class_txg_node)) { 2414 multilist_sublist_unlock(mls); 2415 break; 2416 } 2417 metaslab_t *next_msp = multilist_sublist_next(mls, msp); 2418 multilist_sublist_unlock(mls); 2419 /* 2420 * If the metaslab is currently loading there are two 2421 * cases. If it's the metaslab we're evicting, we 2422 * can't continue on or we'll panic when we attempt to 2423 * recursively lock the mutex. If it's another 2424 * metaslab that's loading, it can be safely skipped, 2425 * since we know it's very new and therefore not a 2426 * good eviction candidate. We check later once the 2427 * lock is held that the metaslab is fully loaded 2428 * before actually unloading it. 2429 */ 2430 if (msp->ms_loading) { 2431 msp = next_msp; 2432 inuse = 2433 spl_kmem_cache_inuse(zfs_btree_leaf_cache); 2434 continue; 2435 } 2436 /* 2437 * We can't unload metaslabs with no spacemap because 2438 * they're not ready to be unloaded yet. We can't 2439 * unload metaslabs with outstanding allocations 2440 * because doing so could cause the metaslab's weight 2441 * to decrease while it's unloaded, which violates an 2442 * invariant that we use to prevent unnecessary 2443 * loading. We also don't unload metaslabs that are 2444 * currently active because they are high-weight 2445 * metaslabs that are likely to be used in the near 2446 * future. 2447 */ 2448 mutex_enter(&msp->ms_lock); 2449 if (msp->ms_allocator == -1 && msp->ms_sm != NULL && 2450 msp->ms_allocating_total == 0) { 2451 metaslab_unload(msp); 2452 } 2453 mutex_exit(&msp->ms_lock); 2454 msp = next_msp; 2455 inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache); 2456 } 2457 } 2458 #else 2459 (void) mc, (void) zfs_metaslab_mem_limit; 2460 #endif 2461 } 2462 2463 static int 2464 metaslab_load_impl(metaslab_t *msp) 2465 { 2466 int error = 0; 2467 2468 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2469 ASSERT(msp->ms_loading); 2470 ASSERT(!msp->ms_condensing); 2471 2472 /* 2473 * We temporarily drop the lock to unblock other operations while we 2474 * are reading the space map. Therefore, metaslab_sync() and 2475 * metaslab_sync_done() can run at the same time as we do. 2476 * 2477 * If we are using the log space maps, metaslab_sync() can't write to 2478 * the metaslab's space map while we are loading as we only write to 2479 * it when we are flushing the metaslab, and that can't happen while 2480 * we are loading it. 2481 * 2482 * If we are not using log space maps though, metaslab_sync() can 2483 * append to the space map while we are loading. Therefore we load 2484 * only entries that existed when we started the load. Additionally, 2485 * metaslab_sync_done() has to wait for the load to complete because 2486 * there are potential races like metaslab_load() loading parts of the 2487 * space map that are currently being appended by metaslab_sync(). If 2488 * we didn't, the ms_allocatable would have entries that 2489 * metaslab_sync_done() would try to re-add later. 2490 * 2491 * That's why before dropping the lock we remember the synced length 2492 * of the metaslab and read up to that point of the space map, 2493 * ignoring entries appended by metaslab_sync() that happen after we 2494 * drop the lock. 2495 */ 2496 uint64_t length = msp->ms_synced_length; 2497 mutex_exit(&msp->ms_lock); 2498 2499 hrtime_t load_start = gethrtime(); 2500 metaslab_rt_arg_t *mrap; 2501 if (msp->ms_allocatable->rt_arg == NULL) { 2502 mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP); 2503 } else { 2504 mrap = msp->ms_allocatable->rt_arg; 2505 msp->ms_allocatable->rt_ops = NULL; 2506 msp->ms_allocatable->rt_arg = NULL; 2507 } 2508 mrap->mra_bt = &msp->ms_allocatable_by_size; 2509 mrap->mra_floor_shift = metaslab_by_size_min_shift; 2510 2511 if (msp->ms_sm != NULL) { 2512 error = space_map_load_length(msp->ms_sm, msp->ms_allocatable, 2513 SM_FREE, length); 2514 2515 /* Now, populate the size-sorted tree. */ 2516 metaslab_rt_create(msp->ms_allocatable, mrap); 2517 msp->ms_allocatable->rt_ops = &metaslab_rt_ops; 2518 msp->ms_allocatable->rt_arg = mrap; 2519 2520 struct mssa_arg arg = {0}; 2521 arg.rt = msp->ms_allocatable; 2522 arg.mra = mrap; 2523 zfs_range_tree_walk(msp->ms_allocatable, 2524 metaslab_size_sorted_add, &arg); 2525 } else { 2526 /* 2527 * Add the size-sorted tree first, since we don't need to load 2528 * the metaslab from the spacemap. 2529 */ 2530 metaslab_rt_create(msp->ms_allocatable, mrap); 2531 msp->ms_allocatable->rt_ops = &metaslab_rt_ops; 2532 msp->ms_allocatable->rt_arg = mrap; 2533 /* 2534 * The space map has not been allocated yet, so treat 2535 * all the space in the metaslab as free and add it to the 2536 * ms_allocatable tree. 2537 */ 2538 zfs_range_tree_add(msp->ms_allocatable, 2539 msp->ms_start, msp->ms_size); 2540 2541 if (msp->ms_new) { 2542 /* 2543 * If the ms_sm doesn't exist, this means that this 2544 * metaslab hasn't gone through metaslab_sync() and 2545 * thus has never been dirtied. So we shouldn't 2546 * expect any unflushed allocs or frees from previous 2547 * TXGs. 2548 */ 2549 ASSERT(zfs_range_tree_is_empty( 2550 msp->ms_unflushed_allocs)); 2551 ASSERT(zfs_range_tree_is_empty( 2552 msp->ms_unflushed_frees)); 2553 } 2554 } 2555 2556 /* 2557 * We need to grab the ms_sync_lock to prevent metaslab_sync() from 2558 * changing the ms_sm (or log_sm) and the metaslab's range trees 2559 * while we are about to use them and populate the ms_allocatable. 2560 * The ms_lock is insufficient for this because metaslab_sync() doesn't 2561 * hold the ms_lock while writing the ms_checkpointing tree to disk. 2562 */ 2563 mutex_enter(&msp->ms_sync_lock); 2564 mutex_enter(&msp->ms_lock); 2565 2566 ASSERT(!msp->ms_condensing); 2567 ASSERT(!msp->ms_flushing); 2568 2569 if (error != 0) { 2570 mutex_exit(&msp->ms_sync_lock); 2571 return (error); 2572 } 2573 2574 ASSERT3P(msp->ms_group, !=, NULL); 2575 msp->ms_loaded = B_TRUE; 2576 2577 /* 2578 * Apply all the unflushed changes to ms_allocatable right 2579 * away so any manipulations we do below have a clear view 2580 * of what is allocated and what is free. 2581 */ 2582 zfs_range_tree_walk(msp->ms_unflushed_allocs, 2583 zfs_range_tree_remove, msp->ms_allocatable); 2584 zfs_range_tree_walk(msp->ms_unflushed_frees, 2585 zfs_range_tree_add, msp->ms_allocatable); 2586 2587 ASSERT3P(msp->ms_group, !=, NULL); 2588 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2589 if (spa_syncing_log_sm(spa) != NULL) { 2590 ASSERT(spa_feature_is_enabled(spa, 2591 SPA_FEATURE_LOG_SPACEMAP)); 2592 2593 /* 2594 * If we use a log space map we add all the segments 2595 * that are in ms_unflushed_frees so they are available 2596 * for allocation. 2597 * 2598 * ms_allocatable needs to contain all free segments 2599 * that are ready for allocations (thus not segments 2600 * from ms_freeing, ms_freed, and the ms_defer trees). 2601 * But if we grab the lock in this code path at a sync 2602 * pass later that 1, then it also contains the 2603 * segments of ms_freed (they were added to it earlier 2604 * in this path through ms_unflushed_frees). So we 2605 * need to remove all the segments that exist in 2606 * ms_freed from ms_allocatable as they will be added 2607 * later in metaslab_sync_done(). 2608 * 2609 * When there's no log space map, the ms_allocatable 2610 * correctly doesn't contain any segments that exist 2611 * in ms_freed [see ms_synced_length]. 2612 */ 2613 zfs_range_tree_walk(msp->ms_freed, 2614 zfs_range_tree_remove, msp->ms_allocatable); 2615 } 2616 2617 /* 2618 * If we are not using the log space map, ms_allocatable 2619 * contains the segments that exist in the ms_defer trees 2620 * [see ms_synced_length]. Thus we need to remove them 2621 * from ms_allocatable as they will be added again in 2622 * metaslab_sync_done(). 2623 * 2624 * If we are using the log space map, ms_allocatable still 2625 * contains the segments that exist in the ms_defer trees. 2626 * Not because it read them through the ms_sm though. But 2627 * because these segments are part of ms_unflushed_frees 2628 * whose segments we add to ms_allocatable earlier in this 2629 * code path. 2630 */ 2631 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 2632 zfs_range_tree_walk(msp->ms_defer[t], 2633 zfs_range_tree_remove, msp->ms_allocatable); 2634 } 2635 2636 /* 2637 * Call metaslab_recalculate_weight_and_sort() now that the 2638 * metaslab is loaded so we get the metaslab's real weight. 2639 * 2640 * Unless this metaslab was created with older software and 2641 * has not yet been converted to use segment-based weight, we 2642 * expect the new weight to be better or equal to the weight 2643 * that the metaslab had while it was not loaded. This is 2644 * because the old weight does not take into account the 2645 * consolidation of adjacent segments between TXGs. [see 2646 * comment for ms_synchist and ms_deferhist[] for more info] 2647 */ 2648 uint64_t weight = msp->ms_weight; 2649 uint64_t max_size = msp->ms_max_size; 2650 metaslab_recalculate_weight_and_sort(msp); 2651 if (!WEIGHT_IS_SPACEBASED(weight)) 2652 ASSERT3U(weight, <=, msp->ms_weight); 2653 msp->ms_max_size = metaslab_largest_allocatable(msp); 2654 ASSERT3U(max_size, <=, msp->ms_max_size); 2655 hrtime_t load_end = gethrtime(); 2656 msp->ms_load_time = load_end; 2657 zfs_dbgmsg("metaslab_load: txg %llu, spa %s, class %s, vdev_id %llu, " 2658 "ms_id %llu, smp_length %llu, " 2659 "unflushed_allocs %llu, unflushed_frees %llu, " 2660 "freed %llu, defer %llu + %llu, unloaded time %llu ms, " 2661 "loading_time %lld ms, ms_max_size %llu, " 2662 "max size error %lld, " 2663 "old_weight %llx, new_weight %llx", 2664 (u_longlong_t)spa_syncing_txg(spa), spa_name(spa), 2665 msp->ms_group->mg_class->mc_name, 2666 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 2667 (u_longlong_t)msp->ms_id, 2668 (u_longlong_t)space_map_length(msp->ms_sm), 2669 (u_longlong_t)zfs_range_tree_space(msp->ms_unflushed_allocs), 2670 (u_longlong_t)zfs_range_tree_space(msp->ms_unflushed_frees), 2671 (u_longlong_t)zfs_range_tree_space(msp->ms_freed), 2672 (u_longlong_t)zfs_range_tree_space(msp->ms_defer[0]), 2673 (u_longlong_t)zfs_range_tree_space(msp->ms_defer[1]), 2674 (longlong_t)((load_start - msp->ms_unload_time) / 1000000), 2675 (longlong_t)((load_end - load_start) / 1000000), 2676 (u_longlong_t)msp->ms_max_size, 2677 (u_longlong_t)msp->ms_max_size - max_size, 2678 (u_longlong_t)weight, (u_longlong_t)msp->ms_weight); 2679 2680 metaslab_verify_space(msp, spa_syncing_txg(spa)); 2681 mutex_exit(&msp->ms_sync_lock); 2682 return (0); 2683 } 2684 2685 int 2686 metaslab_load(metaslab_t *msp) 2687 { 2688 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2689 2690 /* 2691 * There may be another thread loading the same metaslab, if that's 2692 * the case just wait until the other thread is done and return. 2693 */ 2694 metaslab_load_wait(msp); 2695 if (msp->ms_loaded) 2696 return (0); 2697 VERIFY(!msp->ms_loading); 2698 ASSERT(!msp->ms_condensing); 2699 2700 /* 2701 * We set the loading flag BEFORE potentially dropping the lock to 2702 * wait for an ongoing flush (see ms_flushing below). This way other 2703 * threads know that there is already a thread that is loading this 2704 * metaslab. 2705 */ 2706 msp->ms_loading = B_TRUE; 2707 2708 /* 2709 * Wait for any in-progress flushing to finish as we drop the ms_lock 2710 * both here (during space_map_load()) and in metaslab_flush() (when 2711 * we flush our changes to the ms_sm). 2712 */ 2713 if (msp->ms_flushing) 2714 metaslab_flush_wait(msp); 2715 2716 /* 2717 * In the possibility that we were waiting for the metaslab to be 2718 * flushed (where we temporarily dropped the ms_lock), ensure that 2719 * no one else loaded the metaslab somehow. 2720 */ 2721 ASSERT(!msp->ms_loaded); 2722 2723 /* 2724 * If we're loading a metaslab in the normal class, consider evicting 2725 * another one to keep our memory usage under the limit defined by the 2726 * zfs_metaslab_mem_limit tunable. 2727 */ 2728 if (spa_normal_class(msp->ms_group->mg_class->mc_spa) == 2729 msp->ms_group->mg_class) { 2730 metaslab_potentially_evict(msp->ms_group->mg_class); 2731 } 2732 2733 int error = metaslab_load_impl(msp); 2734 2735 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2736 msp->ms_loading = B_FALSE; 2737 cv_broadcast(&msp->ms_load_cv); 2738 2739 return (error); 2740 } 2741 2742 void 2743 metaslab_unload(metaslab_t *msp) 2744 { 2745 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2746 2747 /* 2748 * This can happen if a metaslab is selected for eviction (in 2749 * metaslab_potentially_evict) and then unloaded during spa_sync (via 2750 * metaslab_class_evict_old). 2751 */ 2752 if (!msp->ms_loaded) 2753 return; 2754 2755 zfs_range_tree_vacate(msp->ms_allocatable, NULL, NULL); 2756 msp->ms_loaded = B_FALSE; 2757 msp->ms_unload_time = gethrtime(); 2758 2759 msp->ms_activation_weight = 0; 2760 msp->ms_weight &= ~METASLAB_ACTIVE_MASK; 2761 2762 if (msp->ms_group != NULL) { 2763 metaslab_class_t *mc = msp->ms_group->mg_class; 2764 multilist_sublist_t *mls = 2765 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 2766 if (multilist_link_active(&msp->ms_class_txg_node)) 2767 multilist_sublist_remove(mls, msp); 2768 multilist_sublist_unlock(mls); 2769 2770 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2771 zfs_dbgmsg("metaslab_unload: txg %llu, spa %s, class %s, " 2772 "vdev_id %llu, ms_id %llu, weight %llx, " 2773 "selected txg %llu (%llu s ago), alloc_txg %llu, " 2774 "loaded %llu ms ago, max_size %llu", 2775 (u_longlong_t)spa_syncing_txg(spa), spa_name(spa), 2776 msp->ms_group->mg_class->mc_name, 2777 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 2778 (u_longlong_t)msp->ms_id, 2779 (u_longlong_t)msp->ms_weight, 2780 (u_longlong_t)msp->ms_selected_txg, 2781 (u_longlong_t)(NSEC2SEC(msp->ms_unload_time) - 2782 msp->ms_selected_time), 2783 (u_longlong_t)msp->ms_alloc_txg, 2784 (u_longlong_t)(msp->ms_unload_time - 2785 msp->ms_load_time) / 1000 / 1000, 2786 (u_longlong_t)msp->ms_max_size); 2787 } 2788 2789 /* 2790 * We explicitly recalculate the metaslab's weight based on its space 2791 * map (as it is now not loaded). We want unload metaslabs to always 2792 * have their weights calculated from the space map histograms, while 2793 * loaded ones have it calculated from their in-core range tree 2794 * [see metaslab_load()]. This way, the weight reflects the information 2795 * available in-core, whether it is loaded or not. 2796 * 2797 * If ms_group == NULL means that we came here from metaslab_fini(), 2798 * at which point it doesn't make sense for us to do the recalculation 2799 * and the sorting. 2800 */ 2801 if (msp->ms_group != NULL) 2802 metaslab_recalculate_weight_and_sort(msp); 2803 } 2804 2805 /* 2806 * We want to optimize the memory use of the per-metaslab range 2807 * trees. To do this, we store the segments in the range trees in 2808 * units of sectors, zero-indexing from the start of the metaslab. If 2809 * the vdev_ms_shift - the vdev_ashift is less than 32, we can store 2810 * the ranges using two uint32_ts, rather than two uint64_ts. 2811 */ 2812 zfs_range_seg_type_t 2813 metaslab_calculate_range_tree_type(vdev_t *vdev, metaslab_t *msp, 2814 uint64_t *start, uint64_t *shift) 2815 { 2816 if (vdev->vdev_ms_shift - vdev->vdev_ashift < 32 && 2817 !zfs_metaslab_force_large_segs) { 2818 *shift = vdev->vdev_ashift; 2819 *start = msp->ms_start; 2820 return (ZFS_RANGE_SEG32); 2821 } else { 2822 *shift = 0; 2823 *start = 0; 2824 return (ZFS_RANGE_SEG64); 2825 } 2826 } 2827 2828 void 2829 metaslab_set_selected_txg(metaslab_t *msp, uint64_t txg) 2830 { 2831 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2832 metaslab_class_t *mc = msp->ms_group->mg_class; 2833 multilist_sublist_t *mls = 2834 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 2835 if (multilist_link_active(&msp->ms_class_txg_node)) 2836 multilist_sublist_remove(mls, msp); 2837 msp->ms_selected_txg = txg; 2838 msp->ms_selected_time = gethrestime_sec(); 2839 multilist_sublist_insert_tail(mls, msp); 2840 multilist_sublist_unlock(mls); 2841 } 2842 2843 void 2844 metaslab_space_update(vdev_t *vd, metaslab_class_t *mc, int64_t alloc_delta, 2845 int64_t defer_delta, int64_t space_delta) 2846 { 2847 vdev_space_update(vd, alloc_delta, defer_delta, space_delta); 2848 2849 ASSERT3P(vd->vdev_spa->spa_root_vdev, ==, vd->vdev_parent); 2850 ASSERT(vd->vdev_ms_count != 0); 2851 2852 metaslab_class_space_update(mc, alloc_delta, defer_delta, space_delta, 2853 vdev_deflated_space(vd, space_delta)); 2854 } 2855 2856 int 2857 metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, 2858 uint64_t txg, metaslab_t **msp) 2859 { 2860 vdev_t *vd = mg->mg_vd; 2861 spa_t *spa = vd->vdev_spa; 2862 objset_t *mos = spa->spa_meta_objset; 2863 metaslab_t *ms; 2864 int error; 2865 2866 ms = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP); 2867 mutex_init(&ms->ms_lock, NULL, MUTEX_DEFAULT, NULL); 2868 mutex_init(&ms->ms_sync_lock, NULL, MUTEX_DEFAULT, NULL); 2869 cv_init(&ms->ms_load_cv, NULL, CV_DEFAULT, NULL); 2870 cv_init(&ms->ms_flush_cv, NULL, CV_DEFAULT, NULL); 2871 multilist_link_init(&ms->ms_class_txg_node); 2872 2873 ms->ms_id = id; 2874 ms->ms_start = id << vd->vdev_ms_shift; 2875 ms->ms_size = 1ULL << vd->vdev_ms_shift; 2876 ms->ms_allocator = -1; 2877 ms->ms_new = B_TRUE; 2878 2879 vdev_ops_t *ops = vd->vdev_ops; 2880 if (ops->vdev_op_metaslab_init != NULL) 2881 ops->vdev_op_metaslab_init(vd, &ms->ms_start, &ms->ms_size); 2882 2883 /* 2884 * We only open space map objects that already exist. All others 2885 * will be opened when we finally allocate an object for it. For 2886 * readonly pools there is no need to open the space map object. 2887 * 2888 * Note: 2889 * When called from vdev_expand(), we can't call into the DMU as 2890 * we are holding the spa_config_lock as a writer and we would 2891 * deadlock [see relevant comment in vdev_metaslab_init()]. in 2892 * that case, the object parameter is zero though, so we won't 2893 * call into the DMU. 2894 */ 2895 if (object != 0 && !(spa->spa_mode == SPA_MODE_READ && 2896 !spa->spa_read_spacemaps)) { 2897 error = space_map_open(&ms->ms_sm, mos, object, ms->ms_start, 2898 ms->ms_size, vd->vdev_ashift); 2899 2900 if (error != 0) { 2901 kmem_free(ms, sizeof (metaslab_t)); 2902 return (error); 2903 } 2904 2905 ASSERT(ms->ms_sm != NULL); 2906 ms->ms_allocated_space = space_map_allocated(ms->ms_sm); 2907 } 2908 2909 uint64_t shift, start; 2910 zfs_range_seg_type_t type = 2911 metaslab_calculate_range_tree_type(vd, ms, &start, &shift); 2912 2913 ms->ms_allocatable = zfs_range_tree_create_flags( 2914 NULL, type, NULL, start, shift, 2915 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_allocatable")); 2916 for (int t = 0; t < TXG_SIZE; t++) { 2917 ms->ms_allocating[t] = zfs_range_tree_create_flags( 2918 NULL, type, NULL, start, shift, 2919 ZFS_RT_F_DYN_NAME, 2920 metaslab_rt_name(mg, ms, "ms_allocating")); 2921 } 2922 ms->ms_freeing = zfs_range_tree_create_flags( 2923 NULL, type, NULL, start, shift, 2924 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_freeing")); 2925 ms->ms_freed = zfs_range_tree_create_flags( 2926 NULL, type, NULL, start, shift, 2927 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_freed")); 2928 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 2929 ms->ms_defer[t] = zfs_range_tree_create_flags( 2930 NULL, type, NULL, start, shift, 2931 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_defer")); 2932 } 2933 ms->ms_checkpointing = zfs_range_tree_create_flags( 2934 NULL, type, NULL, start, shift, 2935 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_checkpointing")); 2936 ms->ms_unflushed_allocs = zfs_range_tree_create_flags( 2937 NULL, type, NULL, start, shift, 2938 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_unflushed_allocs")); 2939 2940 metaslab_rt_arg_t *mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP); 2941 mrap->mra_bt = &ms->ms_unflushed_frees_by_size; 2942 mrap->mra_floor_shift = metaslab_by_size_min_shift; 2943 ms->ms_unflushed_frees = zfs_range_tree_create_flags( 2944 &metaslab_rt_ops, type, mrap, start, shift, 2945 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_unflushed_frees")); 2946 2947 ms->ms_trim = zfs_range_tree_create_flags( 2948 NULL, type, NULL, start, shift, 2949 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_trim")); 2950 2951 metaslab_group_add(mg, ms); 2952 metaslab_set_fragmentation(ms, B_FALSE); 2953 2954 /* 2955 * If we're opening an existing pool (txg == 0) or creating 2956 * a new one (txg == TXG_INITIAL), all space is available now. 2957 * If we're adding space to an existing pool, the new space 2958 * does not become available until after this txg has synced. 2959 * The metaslab's weight will also be initialized when we sync 2960 * out this txg. This ensures that we don't attempt to allocate 2961 * from it before we have initialized it completely. 2962 */ 2963 if (txg <= TXG_INITIAL) { 2964 metaslab_sync_done(ms, 0); 2965 metaslab_space_update(vd, mg->mg_class, 2966 metaslab_allocated_space(ms), 0, 0); 2967 } 2968 2969 if (txg != 0) { 2970 vdev_dirty(vd, 0, NULL, txg); 2971 vdev_dirty(vd, VDD_METASLAB, ms, txg); 2972 } 2973 2974 *msp = ms; 2975 2976 return (0); 2977 } 2978 2979 static void 2980 metaslab_fini_flush_data(metaslab_t *msp) 2981 { 2982 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2983 2984 if (metaslab_unflushed_txg(msp) == 0) { 2985 ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), 2986 ==, NULL); 2987 return; 2988 } 2989 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 2990 2991 mutex_enter(&spa->spa_flushed_ms_lock); 2992 avl_remove(&spa->spa_metaslabs_by_flushed, msp); 2993 mutex_exit(&spa->spa_flushed_ms_lock); 2994 2995 spa_log_sm_decrement_mscount(spa, metaslab_unflushed_txg(msp)); 2996 spa_log_summary_decrement_mscount(spa, metaslab_unflushed_txg(msp), 2997 metaslab_unflushed_dirty(msp)); 2998 } 2999 3000 uint64_t 3001 metaslab_unflushed_changes_memused(metaslab_t *ms) 3002 { 3003 return ((zfs_range_tree_numsegs(ms->ms_unflushed_allocs) + 3004 zfs_range_tree_numsegs(ms->ms_unflushed_frees)) * 3005 ms->ms_unflushed_allocs->rt_root.bt_elem_size); 3006 } 3007 3008 void 3009 metaslab_fini(metaslab_t *msp) 3010 { 3011 metaslab_group_t *mg = msp->ms_group; 3012 vdev_t *vd = mg->mg_vd; 3013 spa_t *spa = vd->vdev_spa; 3014 3015 metaslab_fini_flush_data(msp); 3016 3017 metaslab_group_remove(mg, msp); 3018 3019 mutex_enter(&msp->ms_lock); 3020 VERIFY0P(msp->ms_group); 3021 3022 /* 3023 * If this metaslab hasn't been through metaslab_sync_done() yet its 3024 * space hasn't been accounted for in its vdev and doesn't need to be 3025 * subtracted. 3026 */ 3027 if (!msp->ms_new) { 3028 metaslab_space_update(vd, mg->mg_class, 3029 -metaslab_allocated_space(msp), 0, -msp->ms_size); 3030 3031 } 3032 space_map_close(msp->ms_sm); 3033 msp->ms_sm = NULL; 3034 3035 metaslab_unload(msp); 3036 3037 zfs_range_tree_destroy(msp->ms_allocatable); 3038 zfs_range_tree_destroy(msp->ms_freeing); 3039 zfs_range_tree_destroy(msp->ms_freed); 3040 3041 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 3042 metaslab_unflushed_changes_memused(msp)); 3043 spa->spa_unflushed_stats.sus_memused -= 3044 metaslab_unflushed_changes_memused(msp); 3045 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); 3046 zfs_range_tree_destroy(msp->ms_unflushed_allocs); 3047 zfs_range_tree_destroy(msp->ms_checkpointing); 3048 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); 3049 zfs_range_tree_destroy(msp->ms_unflushed_frees); 3050 3051 for (int t = 0; t < TXG_SIZE; t++) { 3052 zfs_range_tree_destroy(msp->ms_allocating[t]); 3053 } 3054 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 3055 zfs_range_tree_destroy(msp->ms_defer[t]); 3056 } 3057 ASSERT0(msp->ms_deferspace); 3058 3059 for (int t = 0; t < TXG_SIZE; t++) 3060 ASSERT(!txg_list_member(&vd->vdev_ms_list, msp, t)); 3061 3062 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); 3063 zfs_range_tree_destroy(msp->ms_trim); 3064 3065 mutex_exit(&msp->ms_lock); 3066 cv_destroy(&msp->ms_load_cv); 3067 cv_destroy(&msp->ms_flush_cv); 3068 mutex_destroy(&msp->ms_lock); 3069 mutex_destroy(&msp->ms_sync_lock); 3070 ASSERT3U(msp->ms_allocator, ==, -1); 3071 3072 kmem_free(msp, sizeof (metaslab_t)); 3073 } 3074 3075 /* 3076 * This table defines a segment size based fragmentation metric that will 3077 * allow each metaslab to derive its own fragmentation value. This is done 3078 * by calculating the space in each bucket of the spacemap histogram and 3079 * multiplying that by the fragmentation metric in this table. Doing 3080 * this for all buckets and dividing it by the total amount of free 3081 * space in this metaslab (i.e. the total free space in all buckets) gives 3082 * us the fragmentation metric. This means that a high fragmentation metric 3083 * equates to most of the free space being comprised of small segments. 3084 * Conversely, if the metric is low, then most of the free space is in 3085 * large segments. 3086 * 3087 * This table defines 0% fragmented space using 512M segments. Using this value, 3088 * we derive the rest of the table. This table originally went up to 16MB, but 3089 * with larger recordsizes, larger ashifts, and use of raidz3, it is possible 3090 * to have significantly larger allocations than were previously possible. 3091 * Since the fragmentation value is never stored on disk, it is possible to 3092 * change these calculations in the future. 3093 */ 3094 static const int zfs_frag_table[] = { 3095 100, /* 512B */ 3096 99, /* 1K */ 3097 97, /* 2K */ 3098 93, /* 4K */ 3099 88, /* 8K */ 3100 83, /* 16K */ 3101 77, /* 32K */ 3102 71, /* 64K */ 3103 64, /* 128K */ 3104 57, /* 256K */ 3105 50, /* 512K */ 3106 43, /* 1M */ 3107 36, /* 2M */ 3108 29, /* 4M */ 3109 23, /* 8M */ 3110 17, /* 16M */ 3111 12, /* 32M */ 3112 7, /* 64M */ 3113 3, /* 128M */ 3114 1, /* 256M */ 3115 0, /* 512M */ 3116 }; 3117 #define FRAGMENTATION_TABLE_SIZE \ 3118 (sizeof (zfs_frag_table)/(sizeof (zfs_frag_table[0]))) 3119 3120 /* 3121 * Calculate the metaslab's fragmentation metric and set ms_fragmentation. 3122 * Setting this value to ZFS_FRAG_INVALID means that the metaslab has not 3123 * been upgraded and does not support this metric. Otherwise, the return 3124 * value should be in the range [0, 100]. 3125 */ 3126 static void 3127 metaslab_set_fragmentation(metaslab_t *msp, boolean_t nodirty) 3128 { 3129 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 3130 uint64_t fragmentation = 0; 3131 uint64_t total = 0; 3132 boolean_t feature_enabled = spa_feature_is_enabled(spa, 3133 SPA_FEATURE_SPACEMAP_HISTOGRAM); 3134 3135 if (!feature_enabled) { 3136 msp->ms_fragmentation = ZFS_FRAG_INVALID; 3137 return; 3138 } 3139 3140 /* 3141 * A null space map means that the entire metaslab is free 3142 * and thus is not fragmented. 3143 */ 3144 if (msp->ms_sm == NULL) { 3145 msp->ms_fragmentation = 0; 3146 return; 3147 } 3148 3149 /* 3150 * If this metaslab's space map has not been upgraded, flag it 3151 * so that we upgrade next time we encounter it. 3152 */ 3153 if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) { 3154 uint64_t txg = spa_syncing_txg(spa); 3155 vdev_t *vd = msp->ms_group->mg_vd; 3156 3157 /* 3158 * If we've reached the final dirty txg, then we must 3159 * be shutting down the pool. We don't want to dirty 3160 * any data past this point so skip setting the condense 3161 * flag. We can retry this action the next time the pool 3162 * is imported. We also skip marking this metaslab for 3163 * condensing if the caller has explicitly set nodirty. 3164 */ 3165 if (!nodirty && 3166 spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) { 3167 msp->ms_condense_wanted = B_TRUE; 3168 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); 3169 zfs_dbgmsg("txg %llu, requesting force condense: " 3170 "ms_id %llu, vdev_id %llu", (u_longlong_t)txg, 3171 (u_longlong_t)msp->ms_id, 3172 (u_longlong_t)vd->vdev_id); 3173 } 3174 msp->ms_fragmentation = ZFS_FRAG_INVALID; 3175 return; 3176 } 3177 3178 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 3179 uint64_t space = 0; 3180 uint8_t shift = msp->ms_sm->sm_shift; 3181 3182 int idx = MIN(shift - SPA_MINBLOCKSHIFT + i, 3183 FRAGMENTATION_TABLE_SIZE - 1); 3184 3185 if (msp->ms_sm->sm_phys->smp_histogram[i] == 0) 3186 continue; 3187 3188 space = msp->ms_sm->sm_phys->smp_histogram[i] << (i + shift); 3189 total += space; 3190 3191 ASSERT3U(idx, <, FRAGMENTATION_TABLE_SIZE); 3192 fragmentation += space * zfs_frag_table[idx]; 3193 } 3194 3195 if (total > 0) 3196 fragmentation /= total; 3197 ASSERT3U(fragmentation, <=, 100); 3198 3199 msp->ms_fragmentation = fragmentation; 3200 } 3201 3202 /* 3203 * Compute a weight -- a selection preference value -- for the given metaslab. 3204 * This is based on the amount of free space, the level of fragmentation, 3205 * the LBA range, and whether the metaslab is loaded. 3206 */ 3207 static uint64_t 3208 metaslab_space_weight(metaslab_t *msp) 3209 { 3210 metaslab_group_t *mg = msp->ms_group; 3211 vdev_t *vd = mg->mg_vd; 3212 uint64_t weight, space; 3213 3214 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3215 3216 /* 3217 * The baseline weight is the metaslab's free space. 3218 */ 3219 space = msp->ms_size - metaslab_allocated_space(msp); 3220 3221 if (metaslab_fragmentation_factor_enabled && 3222 msp->ms_fragmentation != ZFS_FRAG_INVALID) { 3223 /* 3224 * Use the fragmentation information to inversely scale 3225 * down the baseline weight. We need to ensure that we 3226 * don't exclude this metaslab completely when it's 100% 3227 * fragmented. To avoid this we reduce the fragmented value 3228 * by 1. 3229 */ 3230 space = (space * (100 - (msp->ms_fragmentation - 1))) / 100; 3231 3232 /* 3233 * If space < SPA_MINBLOCKSIZE, then we will not allocate from 3234 * this metaslab again. The fragmentation metric may have 3235 * decreased the space to something smaller than 3236 * SPA_MINBLOCKSIZE, so reset the space to SPA_MINBLOCKSIZE 3237 * so that we can consume any remaining space. 3238 */ 3239 if (space > 0 && space < SPA_MINBLOCKSIZE) 3240 space = SPA_MINBLOCKSIZE; 3241 } 3242 weight = space; 3243 3244 /* 3245 * Modern disks have uniform bit density and constant angular velocity. 3246 * Therefore, the outer recording zones are faster (higher bandwidth) 3247 * than the inner zones by the ratio of outer to inner track diameter, 3248 * which is typically around 2:1. We account for this by assigning 3249 * higher weight to lower metaslabs (multiplier ranging from 2x to 1x). 3250 * In effect, this means that we'll select the metaslab with the most 3251 * free bandwidth rather than simply the one with the most free space. 3252 */ 3253 if (!vd->vdev_nonrot && metaslab_lba_weighting_enabled) { 3254 weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count; 3255 ASSERT(weight >= space && weight <= 2 * space); 3256 } 3257 3258 /* 3259 * If this metaslab is one we're actively using, adjust its 3260 * weight to make it preferable to any inactive metaslab so 3261 * we'll polish it off. If the fragmentation on this metaslab 3262 * has exceed our threshold, then don't mark it active. 3263 */ 3264 if (msp->ms_loaded && msp->ms_fragmentation != ZFS_FRAG_INVALID && 3265 msp->ms_fragmentation <= zfs_metaslab_fragmentation_threshold) { 3266 weight |= (msp->ms_weight & METASLAB_ACTIVE_MASK); 3267 } 3268 3269 WEIGHT_SET_SPACEBASED(weight); 3270 return (weight); 3271 } 3272 3273 /* 3274 * Return the weight of the specified metaslab, according to the segment-based 3275 * weighting algorithm. The metaslab must be loaded. This function can 3276 * be called within a sync pass since it relies only on the metaslab's 3277 * range tree which is always accurate when the metaslab is loaded. 3278 */ 3279 static uint64_t 3280 metaslab_weight_from_range_tree(metaslab_t *msp) 3281 { 3282 uint64_t weight = 0; 3283 uint32_t segments = 0; 3284 3285 ASSERT(msp->ms_loaded); 3286 3287 for (int i = ZFS_RANGE_TREE_HISTOGRAM_SIZE - 1; i >= SPA_MINBLOCKSHIFT; 3288 i--) { 3289 uint8_t shift = msp->ms_group->mg_vd->vdev_ashift; 3290 int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1; 3291 3292 segments <<= 1; 3293 segments += msp->ms_allocatable->rt_histogram[i]; 3294 3295 /* 3296 * The range tree provides more precision than the space map 3297 * and must be downgraded so that all values fit within the 3298 * space map's histogram. This allows us to compare loaded 3299 * vs. unloaded metaslabs to determine which metaslab is 3300 * considered "best". 3301 */ 3302 if (i > max_idx) 3303 continue; 3304 3305 if (segments != 0) { 3306 WEIGHT_SET_COUNT(weight, segments); 3307 WEIGHT_SET_INDEX(weight, i); 3308 WEIGHT_SET_ACTIVE(weight, 0); 3309 break; 3310 } 3311 } 3312 return (weight); 3313 } 3314 3315 /* 3316 * Calculate the weight based on the on-disk histogram. Should be applied 3317 * only to unloaded metaslabs (i.e no incoming allocations) in-order to 3318 * give results consistent with the on-disk state 3319 */ 3320 static uint64_t 3321 metaslab_weight_from_spacemap(metaslab_t *msp) 3322 { 3323 space_map_t *sm = msp->ms_sm; 3324 ASSERT(!msp->ms_loaded); 3325 ASSERT(sm != NULL); 3326 ASSERT3U(space_map_object(sm), !=, 0); 3327 ASSERT3U(sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t)); 3328 3329 /* 3330 * Create a joint histogram from all the segments that have made 3331 * it to the metaslab's space map histogram, that are not yet 3332 * available for allocation because they are still in the freeing 3333 * pipeline (e.g. freeing, freed, and defer trees). Then subtract 3334 * these segments from the space map's histogram to get a more 3335 * accurate weight. 3336 */ 3337 uint64_t deferspace_histogram[SPACE_MAP_HISTOGRAM_SIZE] = {0}; 3338 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) 3339 deferspace_histogram[i] += msp->ms_synchist[i]; 3340 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 3341 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 3342 deferspace_histogram[i] += msp->ms_deferhist[t][i]; 3343 } 3344 } 3345 3346 uint64_t weight = 0; 3347 for (int i = SPACE_MAP_HISTOGRAM_SIZE - 1; i >= 0; i--) { 3348 ASSERT3U(sm->sm_phys->smp_histogram[i], >=, 3349 deferspace_histogram[i]); 3350 uint64_t count = 3351 sm->sm_phys->smp_histogram[i] - deferspace_histogram[i]; 3352 if (count != 0) { 3353 WEIGHT_SET_COUNT(weight, count); 3354 WEIGHT_SET_INDEX(weight, i + sm->sm_shift); 3355 WEIGHT_SET_ACTIVE(weight, 0); 3356 break; 3357 } 3358 } 3359 return (weight); 3360 } 3361 3362 /* 3363 * Compute a segment-based weight for the specified metaslab. The weight 3364 * is determined by highest bucket in the histogram. The information 3365 * for the highest bucket is encoded into the weight value. 3366 */ 3367 static uint64_t 3368 metaslab_segment_weight(metaslab_t *msp) 3369 { 3370 metaslab_group_t *mg = msp->ms_group; 3371 uint64_t weight = 0; 3372 uint8_t shift = mg->mg_vd->vdev_ashift; 3373 3374 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3375 3376 /* 3377 * The metaslab is completely free. 3378 */ 3379 if (metaslab_allocated_space(msp) == 0) { 3380 int idx = highbit64(msp->ms_size) - 1; 3381 int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1; 3382 3383 if (idx < max_idx) { 3384 WEIGHT_SET_COUNT(weight, 1ULL); 3385 WEIGHT_SET_INDEX(weight, idx); 3386 } else { 3387 WEIGHT_SET_COUNT(weight, 1ULL << (idx - max_idx)); 3388 WEIGHT_SET_INDEX(weight, max_idx); 3389 } 3390 WEIGHT_SET_ACTIVE(weight, 0); 3391 ASSERT(!WEIGHT_IS_SPACEBASED(weight)); 3392 return (weight); 3393 } 3394 3395 ASSERT3U(msp->ms_sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t)); 3396 3397 /* 3398 * If the metaslab is fully allocated then just make the weight 0. 3399 */ 3400 if (metaslab_allocated_space(msp) == msp->ms_size) 3401 return (0); 3402 /* 3403 * If the metaslab is already loaded, then use the range tree to 3404 * determine the weight. Otherwise, we rely on the space map information 3405 * to generate the weight. 3406 */ 3407 if (msp->ms_loaded) { 3408 weight = metaslab_weight_from_range_tree(msp); 3409 } else { 3410 weight = metaslab_weight_from_spacemap(msp); 3411 } 3412 3413 /* 3414 * If the metaslab was active the last time we calculated its weight 3415 * then keep it active. We want to consume the entire region that 3416 * is associated with this weight. 3417 */ 3418 if (msp->ms_activation_weight != 0 && weight != 0) 3419 WEIGHT_SET_ACTIVE(weight, WEIGHT_GET_ACTIVE(msp->ms_weight)); 3420 return (weight); 3421 } 3422 3423 /* 3424 * Determine if we should attempt to allocate from this metaslab. If the 3425 * metaslab is loaded, then we can determine if the desired allocation 3426 * can be satisfied by looking at the size of the maximum free segment 3427 * on that metaslab. Otherwise, we make our decision based on the metaslab's 3428 * weight. For segment-based weighting we can determine the maximum 3429 * allocation based on the index encoded in its value. For space-based 3430 * weights we rely on the entire weight (excluding the weight-type bit). 3431 */ 3432 static boolean_t 3433 metaslab_should_allocate(metaslab_t *msp, uint64_t asize, boolean_t try_hard) 3434 { 3435 /* 3436 * This case will usually but not always get caught by the checks below; 3437 * metaslabs can be loaded by various means, including the trim and 3438 * initialize code. Once that happens, without this check they are 3439 * allocatable even before they finish their first txg sync. 3440 */ 3441 if (unlikely(msp->ms_new)) 3442 return (B_FALSE); 3443 3444 /* 3445 * If the metaslab is loaded, ms_max_size is definitive and we can use 3446 * the fast check. If it's not, the ms_max_size is a lower bound (once 3447 * set), and we should use the fast check as long as we're not in 3448 * try_hard and it's been less than zfs_metaslab_max_size_cache_sec 3449 * seconds since the metaslab was unloaded. 3450 */ 3451 if (msp->ms_loaded || 3452 (msp->ms_max_size != 0 && !try_hard && gethrtime() < 3453 msp->ms_unload_time + SEC2NSEC(zfs_metaslab_max_size_cache_sec))) 3454 return (msp->ms_max_size >= asize); 3455 3456 boolean_t should_allocate; 3457 if (!WEIGHT_IS_SPACEBASED(msp->ms_weight)) { 3458 /* 3459 * The metaslab segment weight indicates segments in the 3460 * range [2^i, 2^(i+1)), where i is the index in the weight. 3461 * Since the asize might be in the middle of the range, we 3462 * should attempt the allocation if asize < 2^(i+1). 3463 */ 3464 should_allocate = (asize < 3465 1ULL << (WEIGHT_GET_INDEX(msp->ms_weight) + 1)); 3466 } else { 3467 should_allocate = (asize <= 3468 (msp->ms_weight & ~METASLAB_WEIGHT_TYPE)); 3469 } 3470 3471 return (should_allocate); 3472 } 3473 3474 static uint64_t 3475 metaslab_weight(metaslab_t *msp, boolean_t nodirty) 3476 { 3477 vdev_t *vd = msp->ms_group->mg_vd; 3478 spa_t *spa = vd->vdev_spa; 3479 uint64_t weight; 3480 3481 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3482 3483 metaslab_set_fragmentation(msp, nodirty); 3484 3485 /* 3486 * Update the maximum size. If the metaslab is loaded, this will 3487 * ensure that we get an accurate maximum size if newly freed space 3488 * has been added back into the free tree. If the metaslab is 3489 * unloaded, we check if there's a larger free segment in the 3490 * unflushed frees. This is a lower bound on the largest allocatable 3491 * segment size. Coalescing of adjacent entries may reveal larger 3492 * allocatable segments, but we aren't aware of those until loading 3493 * the space map into a range tree. 3494 */ 3495 if (msp->ms_loaded) { 3496 msp->ms_max_size = metaslab_largest_allocatable(msp); 3497 } else { 3498 msp->ms_max_size = MAX(msp->ms_max_size, 3499 metaslab_largest_unflushed_free(msp)); 3500 } 3501 3502 /* 3503 * Segment-based weighting requires space map histogram support. 3504 */ 3505 if (zfs_metaslab_segment_weight_enabled && 3506 spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) && 3507 (msp->ms_sm == NULL || msp->ms_sm->sm_dbuf->db_size == 3508 sizeof (space_map_phys_t))) { 3509 weight = metaslab_segment_weight(msp); 3510 } else { 3511 weight = metaslab_space_weight(msp); 3512 } 3513 return (weight); 3514 } 3515 3516 void 3517 metaslab_recalculate_weight_and_sort(metaslab_t *msp) 3518 { 3519 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3520 3521 /* note: we preserve the mask (e.g. indication of primary, etc..) */ 3522 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK; 3523 metaslab_group_sort(msp->ms_group, msp, 3524 metaslab_weight(msp, B_FALSE) | was_active); 3525 } 3526 3527 static int 3528 metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp, 3529 int allocator, uint64_t activation_weight) 3530 { 3531 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 3532 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3533 3534 /* 3535 * If we're activating for the claim code, we don't want to actually 3536 * set the metaslab up for a specific allocator. 3537 */ 3538 if (activation_weight == METASLAB_WEIGHT_CLAIM) { 3539 ASSERT0(msp->ms_activation_weight); 3540 msp->ms_activation_weight = msp->ms_weight; 3541 metaslab_group_sort(mg, msp, msp->ms_weight | 3542 activation_weight); 3543 return (0); 3544 } 3545 3546 metaslab_t **mspp = (activation_weight == METASLAB_WEIGHT_PRIMARY ? 3547 &mga->mga_primary : &mga->mga_secondary); 3548 3549 mutex_enter(&mg->mg_lock); 3550 if (*mspp != NULL) { 3551 mutex_exit(&mg->mg_lock); 3552 return (EEXIST); 3553 } 3554 3555 *mspp = msp; 3556 ASSERT3S(msp->ms_allocator, ==, -1); 3557 msp->ms_allocator = allocator; 3558 msp->ms_primary = (activation_weight == METASLAB_WEIGHT_PRIMARY); 3559 3560 ASSERT0(msp->ms_activation_weight); 3561 msp->ms_activation_weight = msp->ms_weight; 3562 metaslab_group_sort_impl(mg, msp, 3563 msp->ms_weight | activation_weight); 3564 mutex_exit(&mg->mg_lock); 3565 3566 return (0); 3567 } 3568 3569 static int 3570 metaslab_activate(metaslab_t *msp, int allocator, uint64_t activation_weight) 3571 { 3572 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3573 3574 /* 3575 * The current metaslab is already activated for us so there 3576 * is nothing to do. Already activated though, doesn't mean 3577 * that this metaslab is activated for our allocator nor our 3578 * requested activation weight. The metaslab could have started 3579 * as an active one for our allocator but changed allocators 3580 * while we were waiting to grab its ms_lock or we stole it 3581 * [see find_valid_metaslab()]. This means that there is a 3582 * possibility of passivating a metaslab of another allocator 3583 * or from a different activation mask, from this thread. 3584 */ 3585 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) { 3586 ASSERT(msp->ms_loaded); 3587 return (0); 3588 } 3589 3590 int error = metaslab_load(msp); 3591 if (error != 0) { 3592 metaslab_group_sort(msp->ms_group, msp, 0); 3593 return (error); 3594 } 3595 3596 /* 3597 * When entering metaslab_load() we may have dropped the 3598 * ms_lock because we were loading this metaslab, or we 3599 * were waiting for another thread to load it for us. In 3600 * that scenario, we recheck the weight of the metaslab 3601 * to see if it was activated by another thread. 3602 * 3603 * If the metaslab was activated for another allocator or 3604 * it was activated with a different activation weight (e.g. 3605 * we wanted to make it a primary but it was activated as 3606 * secondary) we return error (EBUSY). 3607 * 3608 * If the metaslab was activated for the same allocator 3609 * and requested activation mask, skip activating it. 3610 */ 3611 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) { 3612 if (msp->ms_allocator != allocator) 3613 return (EBUSY); 3614 3615 if ((msp->ms_weight & activation_weight) == 0) 3616 return (SET_ERROR(EBUSY)); 3617 3618 EQUIV((activation_weight == METASLAB_WEIGHT_PRIMARY), 3619 msp->ms_primary); 3620 return (0); 3621 } 3622 3623 /* 3624 * If the metaslab has literally 0 space, it will have weight 0. In 3625 * that case, don't bother activating it. This can happen if the 3626 * metaslab had space during find_valid_metaslab, but another thread 3627 * loaded it and used all that space while we were waiting to grab the 3628 * lock. 3629 */ 3630 if (msp->ms_weight == 0) { 3631 ASSERT0(zfs_range_tree_space(msp->ms_allocatable)); 3632 return (SET_ERROR(ENOSPC)); 3633 } 3634 3635 if ((error = metaslab_activate_allocator(msp->ms_group, msp, 3636 allocator, activation_weight)) != 0) { 3637 return (error); 3638 } 3639 3640 ASSERT(msp->ms_loaded); 3641 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 3642 3643 return (0); 3644 } 3645 3646 static void 3647 metaslab_passivate_allocator(metaslab_group_t *mg, metaslab_t *msp, 3648 uint64_t weight) 3649 { 3650 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3651 ASSERT(msp->ms_loaded); 3652 3653 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) { 3654 metaslab_group_sort(mg, msp, weight); 3655 return; 3656 } 3657 3658 mutex_enter(&mg->mg_lock); 3659 ASSERT3P(msp->ms_group, ==, mg); 3660 ASSERT3S(0, <=, msp->ms_allocator); 3661 ASSERT3U(msp->ms_allocator, <, mg->mg_class->mc_spa->spa_alloc_count); 3662 3663 metaslab_group_allocator_t *mga = &mg->mg_allocator[msp->ms_allocator]; 3664 if (msp->ms_primary) { 3665 ASSERT3P(mga->mga_primary, ==, msp); 3666 ASSERT(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); 3667 mga->mga_primary = NULL; 3668 } else { 3669 ASSERT3P(mga->mga_secondary, ==, msp); 3670 ASSERT(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); 3671 mga->mga_secondary = NULL; 3672 } 3673 msp->ms_allocator = -1; 3674 metaslab_group_sort_impl(mg, msp, weight); 3675 mutex_exit(&mg->mg_lock); 3676 } 3677 3678 static void 3679 metaslab_passivate(metaslab_t *msp, uint64_t weight) 3680 { 3681 uint64_t size __maybe_unused = weight & ~METASLAB_WEIGHT_TYPE; 3682 3683 /* 3684 * If size < SPA_MINBLOCKSIZE, then we will not allocate from 3685 * this metaslab again. In that case, it had better be empty, 3686 * or we would be leaving space on the table. 3687 */ 3688 ASSERT(!WEIGHT_IS_SPACEBASED(msp->ms_weight) || 3689 size >= SPA_MINBLOCKSIZE || 3690 zfs_range_tree_space(msp->ms_allocatable) == 0); 3691 ASSERT0(weight & METASLAB_ACTIVE_MASK); 3692 3693 ASSERT(msp->ms_activation_weight != 0); 3694 msp->ms_activation_weight = 0; 3695 metaslab_passivate_allocator(msp->ms_group, msp, weight); 3696 ASSERT0(msp->ms_weight & METASLAB_ACTIVE_MASK); 3697 } 3698 3699 /* 3700 * Segment-based metaslabs are activated once and remain active until 3701 * we either fail an allocation attempt (similar to space-based metaslabs) 3702 * or have exhausted the free space in zfs_metaslab_switch_threshold 3703 * buckets since the metaslab was activated. This function checks to see 3704 * if we've exhausted the zfs_metaslab_switch_threshold buckets in the 3705 * metaslab and passivates it proactively. This will allow us to select a 3706 * metaslab with a larger contiguous region, if any, remaining within this 3707 * metaslab group. If we're in sync pass > 1, then we continue using this 3708 * metaslab so that we don't dirty more block and cause more sync passes. 3709 */ 3710 static void 3711 metaslab_segment_may_passivate(metaslab_t *msp) 3712 { 3713 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 3714 3715 if (WEIGHT_IS_SPACEBASED(msp->ms_weight) || spa_sync_pass(spa) > 1) 3716 return; 3717 3718 /* 3719 * As long as a single largest free segment covers majorioty of free 3720 * space, don't consider the metaslab fragmented. It should allow 3721 * us to fill new unfragmented metaslabs full before switching. 3722 */ 3723 if (metaslab_largest_allocatable(msp) > 3724 zfs_range_tree_space(msp->ms_allocatable) * 15 / 16) 3725 return; 3726 3727 /* 3728 * Since we are in the middle of a sync pass, the most accurate 3729 * information that is accessible to us is the in-core range tree 3730 * histogram; calculate the new weight based on that information. 3731 */ 3732 uint64_t weight = metaslab_weight_from_range_tree(msp); 3733 int activation_idx = WEIGHT_GET_INDEX(msp->ms_activation_weight); 3734 int current_idx = WEIGHT_GET_INDEX(weight); 3735 3736 if (current_idx <= activation_idx - zfs_metaslab_switch_threshold) 3737 metaslab_passivate(msp, weight); 3738 } 3739 3740 static void 3741 metaslab_preload(void *arg) 3742 { 3743 metaslab_t *msp = arg; 3744 metaslab_class_t *mc = msp->ms_group->mg_class; 3745 spa_t *spa = mc->mc_spa; 3746 fstrans_cookie_t cookie = spl_fstrans_mark(); 3747 3748 ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock)); 3749 3750 mutex_enter(&msp->ms_lock); 3751 (void) metaslab_load(msp); 3752 metaslab_set_selected_txg(msp, spa_syncing_txg(spa)); 3753 mutex_exit(&msp->ms_lock); 3754 spl_fstrans_unmark(cookie); 3755 } 3756 3757 static void 3758 metaslab_group_preload(metaslab_group_t *mg) 3759 { 3760 spa_t *spa = mg->mg_vd->vdev_spa; 3761 metaslab_t *msp; 3762 avl_tree_t *t = &mg->mg_metaslab_tree; 3763 int m = 0; 3764 3765 if (spa_shutting_down(spa) || !metaslab_preload_enabled) 3766 return; 3767 3768 mutex_enter(&mg->mg_lock); 3769 3770 /* 3771 * Load the next potential metaslabs 3772 */ 3773 for (msp = avl_first(t); msp != NULL; msp = AVL_NEXT(t, msp)) { 3774 ASSERT3P(msp->ms_group, ==, mg); 3775 3776 /* 3777 * We preload only the maximum number of metaslabs specified 3778 * by metaslab_preload_limit. If a metaslab is being forced 3779 * to condense then we preload it too. This will ensure 3780 * that force condensing happens in the next txg. 3781 */ 3782 if (++m > metaslab_preload_limit && !msp->ms_condense_wanted) { 3783 continue; 3784 } 3785 3786 VERIFY(taskq_dispatch(spa->spa_metaslab_taskq, metaslab_preload, 3787 msp, TQ_SLEEP | (m <= spa->spa_alloc_count ? TQ_FRONT : 0)) 3788 != TASKQID_INVALID); 3789 } 3790 mutex_exit(&mg->mg_lock); 3791 } 3792 3793 /* 3794 * Determine if the space map's on-disk footprint is past our tolerance for 3795 * inefficiency. We would like to use the following criteria to make our 3796 * decision: 3797 * 3798 * 1. Do not condense if the size of the space map object would dramatically 3799 * increase as a result of writing out the free space range tree. 3800 * 3801 * 2. Condense if the on on-disk space map representation is at least 3802 * zfs_condense_pct/100 times the size of the optimal representation 3803 * (i.e. zfs_condense_pct = 110 and in-core = 1MB, optimal = 1.1MB). 3804 * 3805 * 3. Do not condense if the on-disk size of the space map does not actually 3806 * decrease. 3807 * 3808 * Unfortunately, we cannot compute the on-disk size of the space map in this 3809 * context because we cannot accurately compute the effects of compression, etc. 3810 * Instead, we apply the heuristic described in the block comment for 3811 * zfs_metaslab_condense_block_threshold - we only condense if the space used 3812 * is greater than a threshold number of blocks. 3813 */ 3814 static boolean_t 3815 metaslab_should_condense(metaslab_t *msp) 3816 { 3817 space_map_t *sm = msp->ms_sm; 3818 vdev_t *vd = msp->ms_group->mg_vd; 3819 uint64_t vdev_blocksize = 1ULL << vd->vdev_ashift; 3820 3821 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3822 ASSERT(msp->ms_loaded); 3823 ASSERT(sm != NULL); 3824 ASSERT3U(spa_sync_pass(vd->vdev_spa), ==, 1); 3825 3826 /* 3827 * We always condense metaslabs that are empty and metaslabs for 3828 * which a condense request has been made. 3829 */ 3830 if (zfs_range_tree_numsegs(msp->ms_allocatable) == 0 || 3831 msp->ms_condense_wanted) 3832 return (B_TRUE); 3833 3834 uint64_t record_size = MAX(sm->sm_blksz, vdev_blocksize); 3835 uint64_t object_size = space_map_length(sm); 3836 uint64_t optimal_size = space_map_estimate_optimal_size(sm, 3837 msp->ms_allocatable, SM_NO_VDEVID); 3838 3839 return (object_size >= (optimal_size * zfs_condense_pct / 100) && 3840 object_size > zfs_metaslab_condense_block_threshold * record_size); 3841 } 3842 3843 /* 3844 * Condense the on-disk space map representation to its minimized form. 3845 * The minimized form consists of a small number of allocations followed 3846 * by the entries of the free range tree (ms_allocatable). The condensed 3847 * spacemap contains all the entries of previous TXGs (including those in 3848 * the pool-wide log spacemaps; thus this is effectively a superset of 3849 * metaslab_flush()), but this TXG's entries still need to be written. 3850 */ 3851 static void 3852 metaslab_condense(metaslab_t *msp, dmu_tx_t *tx) 3853 { 3854 zfs_range_tree_t *condense_tree; 3855 space_map_t *sm = msp->ms_sm; 3856 uint64_t txg = dmu_tx_get_txg(tx); 3857 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 3858 3859 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3860 ASSERT(msp->ms_loaded); 3861 ASSERT(msp->ms_sm != NULL); 3862 3863 /* 3864 * In order to condense the space map, we need to change it so it 3865 * only describes which segments are currently allocated and free. 3866 * 3867 * All the current free space resides in the ms_allocatable, all 3868 * the ms_defer trees, and all the ms_allocating trees. We ignore 3869 * ms_freed because it is empty because we're in sync pass 1. We 3870 * ignore ms_freeing because these changes are not yet reflected 3871 * in the spacemap (they will be written later this txg). 3872 * 3873 * So to truncate the space map to represent all the entries of 3874 * previous TXGs we do the following: 3875 * 3876 * 1] We create a range tree (condense tree) that is 100% empty. 3877 * 2] We add to it all segments found in the ms_defer trees 3878 * as those segments are marked as free in the original space 3879 * map. We do the same with the ms_allocating trees for the same 3880 * reason. Adding these segments should be a relatively 3881 * inexpensive operation since we expect these trees to have a 3882 * small number of nodes. 3883 * 3] We vacate any unflushed allocs, since they are not frees we 3884 * need to add to the condense tree. Then we vacate any 3885 * unflushed frees as they should already be part of ms_allocatable. 3886 * 4] At this point, we would ideally like to add all segments 3887 * in the ms_allocatable tree from the condense tree. This way 3888 * we would write all the entries of the condense tree as the 3889 * condensed space map, which would only contain freed 3890 * segments with everything else assumed to be allocated. 3891 * 3892 * Doing so can be prohibitively expensive as ms_allocatable can 3893 * be large, and therefore computationally expensive to add to 3894 * the condense_tree. Instead we first sync out an entry marking 3895 * everything as allocated, then the condense_tree and then the 3896 * ms_allocatable, in the condensed space map. While this is not 3897 * optimal, it is typically close to optimal and more importantly 3898 * much cheaper to compute. 3899 * 3900 * 5] Finally, as both of the unflushed trees were written to our 3901 * new and condensed metaslab space map, we basically flushed 3902 * all the unflushed changes to disk, thus we call 3903 * metaslab_flush_update(). 3904 */ 3905 ASSERT3U(spa_sync_pass(spa), ==, 1); 3906 ASSERT(zfs_range_tree_is_empty(msp->ms_freed)); /* since it is pass 1 */ 3907 3908 zfs_dbgmsg("condensing: txg %llu, msp[%llu] %px, vdev id %llu, " 3909 "spa %s, smp size %llu, segments %llu, forcing condense=%s", 3910 (u_longlong_t)txg, (u_longlong_t)msp->ms_id, msp, 3911 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 3912 spa->spa_name, (u_longlong_t)space_map_length(msp->ms_sm), 3913 (u_longlong_t)zfs_range_tree_numsegs(msp->ms_allocatable), 3914 msp->ms_condense_wanted ? "TRUE" : "FALSE"); 3915 3916 msp->ms_condense_wanted = B_FALSE; 3917 3918 zfs_range_seg_type_t type; 3919 uint64_t shift, start; 3920 type = metaslab_calculate_range_tree_type(msp->ms_group->mg_vd, msp, 3921 &start, &shift); 3922 3923 condense_tree = zfs_range_tree_create_flags( 3924 NULL, type, NULL, start, shift, 3925 ZFS_RT_F_DYN_NAME, 3926 metaslab_rt_name(msp->ms_group, msp, "condense_tree")); 3927 3928 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 3929 zfs_range_tree_walk(msp->ms_defer[t], 3930 zfs_range_tree_add, condense_tree); 3931 } 3932 3933 for (int t = 0; t < TXG_CONCURRENT_STATES; t++) { 3934 zfs_range_tree_walk(msp->ms_allocating[(txg + t) & TXG_MASK], 3935 zfs_range_tree_add, condense_tree); 3936 } 3937 3938 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 3939 metaslab_unflushed_changes_memused(msp)); 3940 spa->spa_unflushed_stats.sus_memused -= 3941 metaslab_unflushed_changes_memused(msp); 3942 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); 3943 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); 3944 3945 /* 3946 * We're about to drop the metaslab's lock thus allowing other 3947 * consumers to change it's content. Set the metaslab's ms_condensing 3948 * flag to ensure that allocations on this metaslab do not occur 3949 * while we're in the middle of committing it to disk. This is only 3950 * critical for ms_allocatable as all other range trees use per TXG 3951 * views of their content. 3952 */ 3953 msp->ms_condensing = B_TRUE; 3954 3955 mutex_exit(&msp->ms_lock); 3956 uint64_t object = space_map_object(msp->ms_sm); 3957 space_map_truncate(sm, 3958 spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ? 3959 zfs_metaslab_sm_blksz_with_log : zfs_metaslab_sm_blksz_no_log, tx); 3960 3961 /* 3962 * space_map_truncate() may have reallocated the spacemap object. 3963 * If so, update the vdev_ms_array. 3964 */ 3965 if (space_map_object(msp->ms_sm) != object) { 3966 object = space_map_object(msp->ms_sm); 3967 dmu_write(spa->spa_meta_objset, 3968 msp->ms_group->mg_vd->vdev_ms_array, sizeof (uint64_t) * 3969 msp->ms_id, sizeof (uint64_t), &object, tx, 3970 DMU_READ_NO_PREFETCH); 3971 } 3972 3973 /* 3974 * Note: 3975 * When the log space map feature is enabled, each space map will 3976 * always have ALLOCS followed by FREES for each sync pass. This is 3977 * typically true even when the log space map feature is disabled, 3978 * except from the case where a metaslab goes through metaslab_sync() 3979 * and gets condensed. In that case the metaslab's space map will have 3980 * ALLOCS followed by FREES (due to condensing) followed by ALLOCS 3981 * followed by FREES (due to space_map_write() in metaslab_sync()) for 3982 * sync pass 1. 3983 */ 3984 zfs_range_tree_t *tmp_tree = zfs_range_tree_create_flags( 3985 NULL, type, NULL, start, shift, 3986 ZFS_RT_F_DYN_NAME, 3987 metaslab_rt_name(msp->ms_group, msp, "tmp_tree")); 3988 zfs_range_tree_add(tmp_tree, msp->ms_start, msp->ms_size); 3989 space_map_write(sm, tmp_tree, SM_ALLOC, SM_NO_VDEVID, tx); 3990 space_map_write(sm, msp->ms_allocatable, SM_FREE, SM_NO_VDEVID, tx); 3991 space_map_write(sm, condense_tree, SM_FREE, SM_NO_VDEVID, tx); 3992 3993 zfs_range_tree_vacate(condense_tree, NULL, NULL); 3994 zfs_range_tree_destroy(condense_tree); 3995 zfs_range_tree_vacate(tmp_tree, NULL, NULL); 3996 zfs_range_tree_destroy(tmp_tree); 3997 mutex_enter(&msp->ms_lock); 3998 3999 msp->ms_condensing = B_FALSE; 4000 metaslab_flush_update(msp, tx); 4001 } 4002 4003 static void 4004 metaslab_unflushed_add(metaslab_t *msp, dmu_tx_t *tx) 4005 { 4006 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 4007 ASSERT(spa_syncing_log_sm(spa) != NULL); 4008 ASSERT(msp->ms_sm != NULL); 4009 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); 4010 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); 4011 4012 mutex_enter(&spa->spa_flushed_ms_lock); 4013 metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx); 4014 metaslab_set_unflushed_dirty(msp, B_TRUE); 4015 avl_add(&spa->spa_metaslabs_by_flushed, msp); 4016 mutex_exit(&spa->spa_flushed_ms_lock); 4017 4018 spa_log_sm_increment_current_mscount(spa); 4019 spa_log_summary_add_flushed_metaslab(spa, B_TRUE); 4020 } 4021 4022 void 4023 metaslab_unflushed_bump(metaslab_t *msp, dmu_tx_t *tx, boolean_t dirty) 4024 { 4025 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 4026 ASSERT(spa_syncing_log_sm(spa) != NULL); 4027 ASSERT(msp->ms_sm != NULL); 4028 ASSERT(metaslab_unflushed_txg(msp) != 0); 4029 ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), ==, msp); 4030 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); 4031 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); 4032 4033 VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(spa)); 4034 4035 /* update metaslab's position in our flushing tree */ 4036 uint64_t ms_prev_flushed_txg = metaslab_unflushed_txg(msp); 4037 boolean_t ms_prev_flushed_dirty = metaslab_unflushed_dirty(msp); 4038 mutex_enter(&spa->spa_flushed_ms_lock); 4039 avl_remove(&spa->spa_metaslabs_by_flushed, msp); 4040 metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx); 4041 metaslab_set_unflushed_dirty(msp, dirty); 4042 avl_add(&spa->spa_metaslabs_by_flushed, msp); 4043 mutex_exit(&spa->spa_flushed_ms_lock); 4044 4045 /* update metaslab counts of spa_log_sm_t nodes */ 4046 spa_log_sm_decrement_mscount(spa, ms_prev_flushed_txg); 4047 spa_log_sm_increment_current_mscount(spa); 4048 4049 /* update log space map summary */ 4050 spa_log_summary_decrement_mscount(spa, ms_prev_flushed_txg, 4051 ms_prev_flushed_dirty); 4052 spa_log_summary_add_flushed_metaslab(spa, dirty); 4053 4054 /* cleanup obsolete logs if any */ 4055 spa_cleanup_old_sm_logs(spa, tx); 4056 } 4057 4058 /* 4059 * Called when the metaslab has been flushed (its own spacemap now reflects 4060 * all the contents of the pool-wide spacemap log). Updates the metaslab's 4061 * metadata and any pool-wide related log space map data (e.g. summary, 4062 * obsolete logs, etc..) to reflect that. 4063 */ 4064 static void 4065 metaslab_flush_update(metaslab_t *msp, dmu_tx_t *tx) 4066 { 4067 metaslab_group_t *mg = msp->ms_group; 4068 spa_t *spa = mg->mg_vd->vdev_spa; 4069 4070 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4071 4072 ASSERT3U(spa_sync_pass(spa), ==, 1); 4073 4074 /* 4075 * Just because a metaslab got flushed, that doesn't mean that 4076 * it will pass through metaslab_sync_done(). Thus, make sure to 4077 * update ms_synced_length here in case it doesn't. 4078 */ 4079 msp->ms_synced_length = space_map_length(msp->ms_sm); 4080 4081 /* 4082 * We may end up here from metaslab_condense() without the 4083 * feature being active. In that case this is a no-op. 4084 */ 4085 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP) || 4086 metaslab_unflushed_txg(msp) == 0) 4087 return; 4088 4089 metaslab_unflushed_bump(msp, tx, B_FALSE); 4090 } 4091 4092 boolean_t 4093 metaslab_flush(metaslab_t *msp, dmu_tx_t *tx) 4094 { 4095 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 4096 4097 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4098 ASSERT3U(spa_sync_pass(spa), ==, 1); 4099 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 4100 4101 ASSERT(msp->ms_sm != NULL); 4102 ASSERT(metaslab_unflushed_txg(msp) != 0); 4103 ASSERT(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL) != NULL); 4104 4105 /* 4106 * There is nothing wrong with flushing the same metaslab twice, as 4107 * this codepath should work on that case. However, the current 4108 * flushing scheme makes sure to avoid this situation as we would be 4109 * making all these calls without having anything meaningful to write 4110 * to disk. We assert this behavior here. 4111 */ 4112 ASSERT3U(metaslab_unflushed_txg(msp), <, dmu_tx_get_txg(tx)); 4113 4114 /* 4115 * We can not flush while loading, because then we would 4116 * not load the ms_unflushed_{allocs,frees}. 4117 */ 4118 if (msp->ms_loading) 4119 return (B_FALSE); 4120 4121 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4122 metaslab_verify_weight_and_frag(msp); 4123 4124 /* 4125 * Metaslab condensing is effectively flushing. Therefore if the 4126 * metaslab can be condensed we can just condense it instead of 4127 * flushing it. 4128 * 4129 * Note that metaslab_condense() does call metaslab_flush_update() 4130 * so we can just return immediately after condensing. We also 4131 * don't need to care about setting ms_flushing or broadcasting 4132 * ms_flush_cv, even if we temporarily drop the ms_lock in 4133 * metaslab_condense(), as the metaslab is already loaded. 4134 */ 4135 if (msp->ms_loaded && metaslab_should_condense(msp)) { 4136 metaslab_group_t *mg = msp->ms_group; 4137 4138 /* 4139 * For all histogram operations below refer to the 4140 * comments of metaslab_sync() where we follow a 4141 * similar procedure. 4142 */ 4143 metaslab_group_histogram_verify(mg); 4144 metaslab_class_histogram_verify(mg->mg_class); 4145 metaslab_group_histogram_remove(mg, msp); 4146 4147 metaslab_condense(msp, tx); 4148 4149 space_map_histogram_clear(msp->ms_sm); 4150 space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx); 4151 ASSERT(zfs_range_tree_is_empty(msp->ms_freed)); 4152 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 4153 space_map_histogram_add(msp->ms_sm, 4154 msp->ms_defer[t], tx); 4155 } 4156 metaslab_aux_histograms_update(msp); 4157 4158 metaslab_group_histogram_add(mg, msp); 4159 metaslab_group_histogram_verify(mg); 4160 metaslab_class_histogram_verify(mg->mg_class); 4161 4162 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4163 4164 /* 4165 * Since we recreated the histogram (and potentially 4166 * the ms_sm too while condensing) ensure that the 4167 * weight is updated too because we are not guaranteed 4168 * that this metaslab is dirty and will go through 4169 * metaslab_sync_done(). 4170 */ 4171 metaslab_recalculate_weight_and_sort(msp); 4172 return (B_TRUE); 4173 } 4174 4175 msp->ms_flushing = B_TRUE; 4176 uint64_t sm_len_before = space_map_length(msp->ms_sm); 4177 4178 mutex_exit(&msp->ms_lock); 4179 space_map_write(msp->ms_sm, msp->ms_unflushed_allocs, SM_ALLOC, 4180 SM_NO_VDEVID, tx); 4181 space_map_write(msp->ms_sm, msp->ms_unflushed_frees, SM_FREE, 4182 SM_NO_VDEVID, tx); 4183 mutex_enter(&msp->ms_lock); 4184 4185 uint64_t sm_len_after = space_map_length(msp->ms_sm); 4186 if (zfs_flags & ZFS_DEBUG_LOG_SPACEMAP) { 4187 zfs_dbgmsg("flushing: txg %llu, spa %s, vdev_id %llu, " 4188 "ms_id %llu, unflushed_allocs %llu, unflushed_frees %llu, " 4189 "appended %llu bytes", (u_longlong_t)dmu_tx_get_txg(tx), 4190 spa_name(spa), 4191 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 4192 (u_longlong_t)msp->ms_id, 4193 (u_longlong_t)zfs_range_tree_space( 4194 msp->ms_unflushed_allocs), 4195 (u_longlong_t)zfs_range_tree_space( 4196 msp->ms_unflushed_frees), 4197 (u_longlong_t)(sm_len_after - sm_len_before)); 4198 } 4199 4200 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 4201 metaslab_unflushed_changes_memused(msp)); 4202 spa->spa_unflushed_stats.sus_memused -= 4203 metaslab_unflushed_changes_memused(msp); 4204 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); 4205 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); 4206 4207 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4208 metaslab_verify_weight_and_frag(msp); 4209 4210 metaslab_flush_update(msp, tx); 4211 4212 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4213 metaslab_verify_weight_and_frag(msp); 4214 4215 msp->ms_flushing = B_FALSE; 4216 cv_broadcast(&msp->ms_flush_cv); 4217 return (B_TRUE); 4218 } 4219 4220 /* 4221 * Write a metaslab to disk in the context of the specified transaction group. 4222 */ 4223 void 4224 metaslab_sync(metaslab_t *msp, uint64_t txg) 4225 { 4226 metaslab_group_t *mg = msp->ms_group; 4227 vdev_t *vd = mg->mg_vd; 4228 spa_t *spa = vd->vdev_spa; 4229 objset_t *mos = spa_meta_objset(spa); 4230 zfs_range_tree_t *alloctree = msp->ms_allocating[txg & TXG_MASK]; 4231 dmu_tx_t *tx; 4232 4233 ASSERT(!vd->vdev_ishole); 4234 4235 /* 4236 * This metaslab has just been added so there's no work to do now. 4237 */ 4238 if (msp->ms_new) { 4239 ASSERT0(zfs_range_tree_space(alloctree)); 4240 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4241 ASSERT0(zfs_range_tree_space(msp->ms_freed)); 4242 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4243 ASSERT0(zfs_range_tree_space(msp->ms_trim)); 4244 return; 4245 } 4246 4247 /* 4248 * Normally, we don't want to process a metaslab if there are no 4249 * allocations or frees to perform. However, if the metaslab is being 4250 * forced to condense, it's loaded and we're not beyond the final 4251 * dirty txg, we need to let it through. Not condensing beyond the 4252 * final dirty txg prevents an issue where metaslabs that need to be 4253 * condensed but were loaded for other reasons could cause a panic 4254 * here. By only checking the txg in that branch of the conditional, 4255 * we preserve the utility of the VERIFY statements in all other 4256 * cases. 4257 */ 4258 if (zfs_range_tree_is_empty(alloctree) && 4259 zfs_range_tree_is_empty(msp->ms_freeing) && 4260 zfs_range_tree_is_empty(msp->ms_checkpointing) && 4261 !(msp->ms_loaded && msp->ms_condense_wanted && 4262 txg <= spa_final_dirty_txg(spa))) 4263 return; 4264 4265 4266 VERIFY3U(txg, <=, spa_final_dirty_txg(spa)); 4267 4268 /* 4269 * The only state that can actually be changing concurrently 4270 * with metaslab_sync() is the metaslab's ms_allocatable. No 4271 * other thread can be modifying this txg's alloc, freeing, 4272 * freed, or space_map_phys_t. We drop ms_lock whenever we 4273 * could call into the DMU, because the DMU can call down to 4274 * us (e.g. via zio_free()) at any time. 4275 * 4276 * The spa_vdev_remove_thread() can be reading metaslab state 4277 * concurrently, and it is locked out by the ms_sync_lock. 4278 * Note that the ms_lock is insufficient for this, because it 4279 * is dropped by space_map_write(). 4280 */ 4281 tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg); 4282 4283 /* 4284 * Generate a log space map if one doesn't exist already. 4285 */ 4286 spa_generate_syncing_log_sm(spa, tx); 4287 4288 if (msp->ms_sm == NULL) { 4289 uint64_t new_object = space_map_alloc(mos, 4290 spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ? 4291 zfs_metaslab_sm_blksz_with_log : 4292 zfs_metaslab_sm_blksz_no_log, tx); 4293 VERIFY3U(new_object, !=, 0); 4294 4295 dmu_write(mos, vd->vdev_ms_array, sizeof (uint64_t) * 4296 msp->ms_id, sizeof (uint64_t), &new_object, tx, 4297 DMU_READ_NO_PREFETCH); 4298 4299 VERIFY0(space_map_open(&msp->ms_sm, mos, new_object, 4300 msp->ms_start, msp->ms_size, vd->vdev_ashift)); 4301 ASSERT(msp->ms_sm != NULL); 4302 4303 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); 4304 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); 4305 ASSERT0(metaslab_allocated_space(msp)); 4306 } 4307 4308 if (!zfs_range_tree_is_empty(msp->ms_checkpointing) && 4309 vd->vdev_checkpoint_sm == NULL) { 4310 ASSERT(spa_has_checkpoint(spa)); 4311 4312 uint64_t new_object = space_map_alloc(mos, 4313 zfs_vdev_standard_sm_blksz, tx); 4314 VERIFY3U(new_object, !=, 0); 4315 4316 VERIFY0(space_map_open(&vd->vdev_checkpoint_sm, 4317 mos, new_object, 0, vd->vdev_asize, vd->vdev_ashift)); 4318 ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL); 4319 4320 /* 4321 * We save the space map object as an entry in vdev_top_zap 4322 * so it can be retrieved when the pool is reopened after an 4323 * export or through zdb. 4324 */ 4325 VERIFY0(zap_add(vd->vdev_spa->spa_meta_objset, 4326 vd->vdev_top_zap, VDEV_TOP_ZAP_POOL_CHECKPOINT_SM, 4327 sizeof (new_object), 1, &new_object, tx)); 4328 } 4329 4330 mutex_enter(&msp->ms_sync_lock); 4331 mutex_enter(&msp->ms_lock); 4332 4333 /* 4334 * Note: metaslab_condense() clears the space map's histogram. 4335 * Therefore we must verify and remove this histogram before 4336 * condensing. 4337 */ 4338 metaslab_group_histogram_verify(mg); 4339 metaslab_class_histogram_verify(mg->mg_class); 4340 metaslab_group_histogram_remove(mg, msp); 4341 4342 if (spa->spa_sync_pass == 1 && msp->ms_loaded && 4343 metaslab_should_condense(msp)) 4344 metaslab_condense(msp, tx); 4345 4346 /* 4347 * We'll be going to disk to sync our space accounting, thus we 4348 * drop the ms_lock during that time so allocations coming from 4349 * open-context (ZIL) for future TXGs do not block. 4350 */ 4351 mutex_exit(&msp->ms_lock); 4352 space_map_t *log_sm = spa_syncing_log_sm(spa); 4353 if (log_sm != NULL) { 4354 ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP)); 4355 if (metaslab_unflushed_txg(msp) == 0) 4356 metaslab_unflushed_add(msp, tx); 4357 else if (!metaslab_unflushed_dirty(msp)) 4358 metaslab_unflushed_bump(msp, tx, B_TRUE); 4359 4360 space_map_write(log_sm, alloctree, SM_ALLOC, 4361 vd->vdev_id, tx); 4362 space_map_write(log_sm, msp->ms_freeing, SM_FREE, 4363 vd->vdev_id, tx); 4364 mutex_enter(&msp->ms_lock); 4365 4366 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 4367 metaslab_unflushed_changes_memused(msp)); 4368 spa->spa_unflushed_stats.sus_memused -= 4369 metaslab_unflushed_changes_memused(msp); 4370 zfs_range_tree_remove_xor_add(alloctree, 4371 msp->ms_unflushed_frees, msp->ms_unflushed_allocs); 4372 zfs_range_tree_remove_xor_add(msp->ms_freeing, 4373 msp->ms_unflushed_allocs, msp->ms_unflushed_frees); 4374 spa->spa_unflushed_stats.sus_memused += 4375 metaslab_unflushed_changes_memused(msp); 4376 } else { 4377 ASSERT(!spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP)); 4378 4379 space_map_write(msp->ms_sm, alloctree, SM_ALLOC, 4380 SM_NO_VDEVID, tx); 4381 space_map_write(msp->ms_sm, msp->ms_freeing, SM_FREE, 4382 SM_NO_VDEVID, tx); 4383 mutex_enter(&msp->ms_lock); 4384 } 4385 4386 msp->ms_allocated_space += zfs_range_tree_space(alloctree); 4387 ASSERT3U(msp->ms_allocated_space, >=, 4388 zfs_range_tree_space(msp->ms_freeing)); 4389 msp->ms_allocated_space -= zfs_range_tree_space(msp->ms_freeing); 4390 4391 if (!zfs_range_tree_is_empty(msp->ms_checkpointing)) { 4392 ASSERT(spa_has_checkpoint(spa)); 4393 ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL); 4394 4395 /* 4396 * Since we are doing writes to disk and the ms_checkpointing 4397 * tree won't be changing during that time, we drop the 4398 * ms_lock while writing to the checkpoint space map, for the 4399 * same reason mentioned above. 4400 */ 4401 mutex_exit(&msp->ms_lock); 4402 space_map_write(vd->vdev_checkpoint_sm, 4403 msp->ms_checkpointing, SM_FREE, SM_NO_VDEVID, tx); 4404 mutex_enter(&msp->ms_lock); 4405 4406 spa->spa_checkpoint_info.sci_dspace += 4407 zfs_range_tree_space(msp->ms_checkpointing); 4408 vd->vdev_stat.vs_checkpoint_space += 4409 zfs_range_tree_space(msp->ms_checkpointing); 4410 ASSERT3U(vd->vdev_stat.vs_checkpoint_space, ==, 4411 -space_map_allocated(vd->vdev_checkpoint_sm)); 4412 4413 zfs_range_tree_vacate(msp->ms_checkpointing, NULL, NULL); 4414 } 4415 4416 if (msp->ms_loaded) { 4417 /* 4418 * When the space map is loaded, we have an accurate 4419 * histogram in the range tree. This gives us an opportunity 4420 * to bring the space map's histogram up-to-date so we clear 4421 * it first before updating it. 4422 */ 4423 space_map_histogram_clear(msp->ms_sm); 4424 space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx); 4425 4426 /* 4427 * Since we've cleared the histogram we need to add back 4428 * any free space that has already been processed, plus 4429 * any deferred space. This allows the on-disk histogram 4430 * to accurately reflect all free space even if some space 4431 * is not yet available for allocation (i.e. deferred). 4432 */ 4433 space_map_histogram_add(msp->ms_sm, msp->ms_freed, tx); 4434 4435 /* 4436 * Add back any deferred free space that has not been 4437 * added back into the in-core free tree yet. This will 4438 * ensure that we don't end up with a space map histogram 4439 * that is completely empty unless the metaslab is fully 4440 * allocated. 4441 */ 4442 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 4443 space_map_histogram_add(msp->ms_sm, 4444 msp->ms_defer[t], tx); 4445 } 4446 } 4447 4448 /* 4449 * Always add the free space from this sync pass to the space 4450 * map histogram. We want to make sure that the on-disk histogram 4451 * accounts for all free space. If the space map is not loaded, 4452 * then we will lose some accuracy but will correct it the next 4453 * time we load the space map. 4454 */ 4455 space_map_histogram_add(msp->ms_sm, msp->ms_freeing, tx); 4456 metaslab_aux_histograms_update(msp); 4457 4458 metaslab_group_histogram_add(mg, msp); 4459 metaslab_group_histogram_verify(mg); 4460 metaslab_class_histogram_verify(mg->mg_class); 4461 4462 /* 4463 * For sync pass 1, we avoid traversing this txg's free range tree 4464 * and instead will just swap the pointers for freeing and freed. 4465 * We can safely do this since the freed_tree is guaranteed to be 4466 * empty on the initial pass. 4467 * 4468 * Keep in mind that even if we are currently using a log spacemap 4469 * we want current frees to end up in the ms_allocatable (but not 4470 * get appended to the ms_sm) so their ranges can be reused as usual. 4471 */ 4472 if (spa_sync_pass(spa) == 1) { 4473 zfs_range_tree_swap(&msp->ms_freeing, &msp->ms_freed); 4474 ASSERT0(msp->ms_allocated_this_txg); 4475 } else { 4476 zfs_range_tree_vacate(msp->ms_freeing, 4477 zfs_range_tree_add, msp->ms_freed); 4478 } 4479 msp->ms_allocated_this_txg += zfs_range_tree_space(alloctree); 4480 zfs_range_tree_vacate(alloctree, NULL, NULL); 4481 4482 ASSERT0(zfs_range_tree_space(msp->ms_allocating[txg & TXG_MASK])); 4483 ASSERT0(zfs_range_tree_space(msp->ms_allocating[TXG_CLEAN(txg) 4484 & TXG_MASK])); 4485 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4486 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4487 4488 mutex_exit(&msp->ms_lock); 4489 4490 /* 4491 * Verify that the space map object ID has been recorded in the 4492 * vdev_ms_array. 4493 */ 4494 uint64_t object; 4495 VERIFY0(dmu_read(mos, vd->vdev_ms_array, 4496 msp->ms_id * sizeof (uint64_t), sizeof (uint64_t), &object, 0)); 4497 VERIFY3U(object, ==, space_map_object(msp->ms_sm)); 4498 4499 mutex_exit(&msp->ms_sync_lock); 4500 dmu_tx_commit(tx); 4501 } 4502 4503 static void 4504 metaslab_evict(metaslab_t *msp, uint64_t txg) 4505 { 4506 if (!msp->ms_loaded || msp->ms_disabled != 0) 4507 return; 4508 4509 for (int t = 1; t < TXG_CONCURRENT_STATES; t++) { 4510 VERIFY0(zfs_range_tree_space( 4511 msp->ms_allocating[(txg + t) & TXG_MASK])); 4512 } 4513 if (msp->ms_allocator != -1) 4514 metaslab_passivate(msp, msp->ms_weight & ~METASLAB_ACTIVE_MASK); 4515 4516 if (!metaslab_debug_unload) 4517 metaslab_unload(msp); 4518 } 4519 4520 /* 4521 * Called after a transaction group has completely synced to mark 4522 * all of the metaslab's free space as usable. 4523 */ 4524 void 4525 metaslab_sync_done(metaslab_t *msp, uint64_t txg) 4526 { 4527 metaslab_group_t *mg = msp->ms_group; 4528 vdev_t *vd = mg->mg_vd; 4529 spa_t *spa = vd->vdev_spa; 4530 zfs_range_tree_t **defer_tree; 4531 int64_t alloc_delta, defer_delta; 4532 boolean_t defer_allowed = B_TRUE; 4533 4534 ASSERT(!vd->vdev_ishole); 4535 4536 mutex_enter(&msp->ms_lock); 4537 4538 if (msp->ms_new) { 4539 /* this is a new metaslab, add its capacity to the vdev */ 4540 metaslab_space_update(vd, mg->mg_class, 0, 0, msp->ms_size); 4541 4542 /* there should be no allocations nor frees at this point */ 4543 VERIFY0(msp->ms_allocated_this_txg); 4544 VERIFY0(zfs_range_tree_space(msp->ms_freed)); 4545 } 4546 4547 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4548 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4549 4550 defer_tree = &msp->ms_defer[txg % TXG_DEFER_SIZE]; 4551 4552 uint64_t free_space = metaslab_class_get_space(spa_normal_class(spa)) - 4553 metaslab_class_get_alloc(spa_normal_class(spa)); 4554 if (free_space <= spa_get_slop_space(spa) || vd->vdev_removing || 4555 vd->vdev_rz_expanding) { 4556 defer_allowed = B_FALSE; 4557 } 4558 4559 defer_delta = 0; 4560 alloc_delta = msp->ms_allocated_this_txg - 4561 zfs_range_tree_space(msp->ms_freed); 4562 4563 if (defer_allowed) { 4564 defer_delta = zfs_range_tree_space(msp->ms_freed) - 4565 zfs_range_tree_space(*defer_tree); 4566 } else { 4567 defer_delta -= zfs_range_tree_space(*defer_tree); 4568 } 4569 metaslab_space_update(vd, mg->mg_class, alloc_delta + defer_delta, 4570 defer_delta, 0); 4571 4572 if (spa_syncing_log_sm(spa) == NULL) { 4573 /* 4574 * If there's a metaslab_load() in progress and we don't have 4575 * a log space map, it means that we probably wrote to the 4576 * metaslab's space map. If this is the case, we need to 4577 * make sure that we wait for the load to complete so that we 4578 * have a consistent view at the in-core side of the metaslab. 4579 */ 4580 metaslab_load_wait(msp); 4581 } else { 4582 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 4583 } 4584 4585 /* 4586 * When auto-trimming is enabled, free ranges which are added to 4587 * ms_allocatable are also be added to ms_trim. The ms_trim tree is 4588 * periodically consumed by the vdev_autotrim_thread() which issues 4589 * trims for all ranges and then vacates the tree. The ms_trim tree 4590 * can be discarded at any time with the sole consequence of recent 4591 * frees not being trimmed. 4592 */ 4593 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) { 4594 zfs_range_tree_walk(*defer_tree, zfs_range_tree_add, 4595 msp->ms_trim); 4596 if (!defer_allowed) { 4597 zfs_range_tree_walk(msp->ms_freed, zfs_range_tree_add, 4598 msp->ms_trim); 4599 } 4600 } else { 4601 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); 4602 } 4603 4604 /* 4605 * Move the frees from the defer_tree back to the free 4606 * range tree (if it's loaded). Swap the freed_tree and 4607 * the defer_tree -- this is safe to do because we've 4608 * just emptied out the defer_tree. 4609 */ 4610 zfs_range_tree_vacate(*defer_tree, 4611 msp->ms_loaded ? zfs_range_tree_add : NULL, msp->ms_allocatable); 4612 if (defer_allowed) { 4613 zfs_range_tree_swap(&msp->ms_freed, defer_tree); 4614 } else { 4615 zfs_range_tree_vacate(msp->ms_freed, 4616 msp->ms_loaded ? zfs_range_tree_add : NULL, 4617 msp->ms_allocatable); 4618 } 4619 4620 msp->ms_synced_length = space_map_length(msp->ms_sm); 4621 4622 msp->ms_deferspace += defer_delta; 4623 ASSERT3S(msp->ms_deferspace, >=, 0); 4624 ASSERT3S(msp->ms_deferspace, <=, msp->ms_size); 4625 if (msp->ms_deferspace != 0) { 4626 /* 4627 * Keep syncing this metaslab until all deferred frees 4628 * are back in circulation. 4629 */ 4630 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); 4631 } 4632 metaslab_aux_histograms_update_done(msp, defer_allowed); 4633 4634 if (msp->ms_new) { 4635 msp->ms_new = B_FALSE; 4636 mutex_enter(&mg->mg_lock); 4637 mg->mg_ms_ready++; 4638 mutex_exit(&mg->mg_lock); 4639 } 4640 4641 /* 4642 * Re-sort metaslab within its group now that we've adjusted 4643 * its allocatable space. 4644 */ 4645 metaslab_recalculate_weight_and_sort(msp); 4646 4647 ASSERT0(zfs_range_tree_space(msp->ms_allocating[txg & TXG_MASK])); 4648 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4649 ASSERT0(zfs_range_tree_space(msp->ms_freed)); 4650 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4651 msp->ms_allocating_total -= msp->ms_allocated_this_txg; 4652 msp->ms_allocated_this_txg = 0; 4653 mutex_exit(&msp->ms_lock); 4654 } 4655 4656 void 4657 metaslab_sync_reassess(metaslab_group_t *mg) 4658 { 4659 spa_t *spa = mg->mg_class->mc_spa; 4660 4661 spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER); 4662 mg->mg_fragmentation = metaslab_group_fragmentation(mg); 4663 metaslab_group_alloc_update(mg); 4664 4665 /* 4666 * Preload the next potential metaslabs but only on active 4667 * metaslab groups. We can get into a state where the metaslab 4668 * is no longer active since we dirty metaslabs as we remove a 4669 * a device, thus potentially making the metaslab group eligible 4670 * for preloading. 4671 */ 4672 if (mg->mg_activation_count > 0) { 4673 metaslab_group_preload(mg); 4674 } 4675 spa_config_exit(spa, SCL_ALLOC, FTAG); 4676 } 4677 4678 /* 4679 * When writing a ditto block (i.e. more than one DVA for a given BP) on 4680 * the same vdev as an existing DVA of this BP, then try to allocate it 4681 * on a different metaslab than existing DVAs (i.e. a unique metaslab). 4682 */ 4683 static boolean_t 4684 metaslab_is_unique(metaslab_t *msp, dva_t *dva) 4685 { 4686 uint64_t dva_ms_id; 4687 4688 if (DVA_GET_ASIZE(dva) == 0) 4689 return (B_TRUE); 4690 4691 if (msp->ms_group->mg_vd->vdev_id != DVA_GET_VDEV(dva)) 4692 return (B_TRUE); 4693 4694 dva_ms_id = DVA_GET_OFFSET(dva) >> msp->ms_group->mg_vd->vdev_ms_shift; 4695 4696 return (msp->ms_id != dva_ms_id); 4697 } 4698 4699 /* 4700 * ========================================================================== 4701 * Metaslab allocation tracing facility 4702 * ========================================================================== 4703 */ 4704 4705 /* 4706 * Add an allocation trace element to the allocation tracing list. 4707 */ 4708 static void 4709 metaslab_trace_add(zio_alloc_list_t *zal, metaslab_group_t *mg, 4710 metaslab_t *msp, uint64_t psize, uint32_t dva_id, uint64_t offset, 4711 int allocator) 4712 { 4713 metaslab_alloc_trace_t *mat; 4714 4715 if (!metaslab_trace_enabled) 4716 return; 4717 4718 /* 4719 * When the tracing list reaches its maximum we remove 4720 * the second element in the list before adding a new one. 4721 * By removing the second element we preserve the original 4722 * entry as a clue to what allocations steps have already been 4723 * performed. 4724 */ 4725 if (zal->zal_size == metaslab_trace_max_entries) { 4726 metaslab_alloc_trace_t *mat_next; 4727 #ifdef ZFS_DEBUG 4728 panic("too many entries in allocation list"); 4729 #endif 4730 METASLABSTAT_BUMP(metaslabstat_trace_over_limit); 4731 zal->zal_size--; 4732 mat_next = list_next(&zal->zal_list, list_head(&zal->zal_list)); 4733 list_remove(&zal->zal_list, mat_next); 4734 kmem_cache_free(metaslab_alloc_trace_cache, mat_next); 4735 } 4736 4737 mat = kmem_cache_alloc(metaslab_alloc_trace_cache, KM_SLEEP); 4738 list_link_init(&mat->mat_list_node); 4739 mat->mat_mg = mg; 4740 mat->mat_msp = msp; 4741 mat->mat_size = psize; 4742 mat->mat_dva_id = dva_id; 4743 mat->mat_offset = offset; 4744 mat->mat_weight = 0; 4745 mat->mat_allocator = allocator; 4746 4747 if (msp != NULL) 4748 mat->mat_weight = msp->ms_weight; 4749 4750 /* 4751 * The list is part of the zio so locking is not required. Only 4752 * a single thread will perform allocations for a given zio. 4753 */ 4754 list_insert_tail(&zal->zal_list, mat); 4755 zal->zal_size++; 4756 4757 ASSERT3U(zal->zal_size, <=, metaslab_trace_max_entries); 4758 } 4759 4760 void 4761 metaslab_trace_move(zio_alloc_list_t *old, zio_alloc_list_t *new) 4762 { 4763 ASSERT0(new->zal_size); 4764 list_move_tail(&new->zal_list, &old->zal_list); 4765 new->zal_size = old->zal_size; 4766 list_destroy(&old->zal_list); 4767 } 4768 4769 void 4770 metaslab_trace_init(zio_alloc_list_t *zal) 4771 { 4772 list_create(&zal->zal_list, sizeof (metaslab_alloc_trace_t), 4773 offsetof(metaslab_alloc_trace_t, mat_list_node)); 4774 zal->zal_size = 0; 4775 } 4776 4777 void 4778 metaslab_trace_fini(zio_alloc_list_t *zal) 4779 { 4780 metaslab_alloc_trace_t *mat; 4781 4782 while ((mat = list_remove_head(&zal->zal_list)) != NULL) 4783 kmem_cache_free(metaslab_alloc_trace_cache, mat); 4784 list_destroy(&zal->zal_list); 4785 zal->zal_size = 0; 4786 } 4787 4788 /* 4789 * ========================================================================== 4790 * Metaslab block operations 4791 * ========================================================================== 4792 */ 4793 4794 static void 4795 metaslab_group_alloc_increment(spa_t *spa, uint64_t vdev, int allocator, 4796 int flags, uint64_t psize, const void *tag) 4797 { 4798 if (!(flags & METASLAB_ASYNC_ALLOC) || tag == NULL) 4799 return; 4800 4801 metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; 4802 if (!mg->mg_class->mc_alloc_throttle_enabled) 4803 return; 4804 4805 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 4806 (void) zfs_refcount_add_many(&mga->mga_queue_depth, psize, tag); 4807 } 4808 4809 void 4810 metaslab_group_alloc_increment_all(spa_t *spa, blkptr_t *bp, int allocator, 4811 int flags, uint64_t psize, const void *tag) 4812 { 4813 for (int d = 0; d < BP_GET_NDVAS(bp); d++) { 4814 uint64_t vdev = DVA_GET_VDEV(&bp->blk_dva[d]); 4815 metaslab_group_alloc_increment(spa, vdev, allocator, flags, 4816 psize, tag); 4817 } 4818 } 4819 4820 void 4821 metaslab_group_alloc_decrement(spa_t *spa, uint64_t vdev, int allocator, 4822 int flags, uint64_t psize, const void *tag) 4823 { 4824 if (!(flags & METASLAB_ASYNC_ALLOC) || tag == NULL) 4825 return; 4826 4827 metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; 4828 if (!mg->mg_class->mc_alloc_throttle_enabled) 4829 return; 4830 4831 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 4832 (void) zfs_refcount_remove_many(&mga->mga_queue_depth, psize, tag); 4833 } 4834 4835 static uint64_t 4836 metaslab_block_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 4837 uint64_t txg, uint64_t *actual_size) 4838 { 4839 uint64_t start; 4840 zfs_range_tree_t *rt = msp->ms_allocatable; 4841 metaslab_class_t *mc = msp->ms_group->mg_class; 4842 4843 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4844 VERIFY(!msp->ms_condensing); 4845 VERIFY0(msp->ms_disabled); 4846 VERIFY0(msp->ms_new); 4847 4848 start = mc->mc_ops->msop_alloc(msp, size, max_size, actual_size); 4849 if (start != -1ULL) { 4850 size = *actual_size; 4851 metaslab_group_t *mg = msp->ms_group; 4852 vdev_t *vd = mg->mg_vd; 4853 4854 VERIFY0(P2PHASE(start, 1ULL << vd->vdev_ashift)); 4855 VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 4856 VERIFY3U(zfs_range_tree_space(rt) - size, <=, msp->ms_size); 4857 zfs_range_tree_remove(rt, start, size); 4858 zfs_range_tree_clear(msp->ms_trim, start, size); 4859 4860 if (zfs_range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK])) 4861 vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg); 4862 4863 zfs_range_tree_add(msp->ms_allocating[txg & TXG_MASK], start, 4864 size); 4865 msp->ms_allocating_total += size; 4866 4867 /* Track the last successful allocation */ 4868 msp->ms_alloc_txg = txg; 4869 metaslab_verify_space(msp, txg); 4870 } 4871 4872 /* 4873 * Now that we've attempted the allocation we need to update the 4874 * metaslab's maximum block size since it may have changed. 4875 */ 4876 msp->ms_max_size = metaslab_largest_allocatable(msp); 4877 return (start); 4878 } 4879 4880 /* 4881 * Find the metaslab with the highest weight that is less than what we've 4882 * already tried. In the common case, this means that we will examine each 4883 * metaslab at most once. Note that concurrent callers could reorder metaslabs 4884 * by activation/passivation once we have dropped the mg_lock. If a metaslab is 4885 * activated by another thread, and we fail to allocate from the metaslab we 4886 * have selected, we may not try the newly-activated metaslab, and instead 4887 * activate another metaslab. This is not optimal, but generally does not cause 4888 * any problems (a possible exception being if every metaslab is completely full 4889 * except for the newly-activated metaslab which we fail to examine). 4890 */ 4891 static metaslab_t * 4892 find_valid_metaslab(metaslab_group_t *mg, uint64_t activation_weight, 4893 dva_t *dva, int d, uint64_t asize, int allocator, 4894 boolean_t try_hard, zio_alloc_list_t *zal, metaslab_t *search, 4895 boolean_t *was_active) 4896 { 4897 avl_index_t idx; 4898 avl_tree_t *t = &mg->mg_metaslab_tree; 4899 metaslab_t *msp = avl_find(t, search, &idx); 4900 if (msp == NULL) 4901 msp = avl_nearest(t, idx, AVL_AFTER); 4902 4903 uint_t tries = 0; 4904 for (; msp != NULL; msp = AVL_NEXT(t, msp)) { 4905 int i; 4906 4907 if (!try_hard && tries > zfs_metaslab_find_max_tries) { 4908 METASLABSTAT_BUMP(metaslabstat_too_many_tries); 4909 return (NULL); 4910 } 4911 tries++; 4912 4913 if (!metaslab_should_allocate(msp, asize, try_hard)) { 4914 metaslab_trace_add(zal, mg, msp, asize, d, 4915 TRACE_TOO_SMALL, allocator); 4916 continue; 4917 } 4918 4919 /* 4920 * If the selected metaslab is condensing or disabled, or 4921 * hasn't gone through a metaslab_sync_done(), then skip it. 4922 */ 4923 if (msp->ms_condensing || msp->ms_disabled > 0 || msp->ms_new) 4924 continue; 4925 4926 *was_active = msp->ms_allocator != -1; 4927 /* 4928 * If we're activating as primary, this is our first allocation 4929 * from this disk, so we don't need to check how close we are. 4930 * If the metaslab under consideration was already active, 4931 * we're getting desperate enough to steal another allocator's 4932 * metaslab, so we still don't care about distances. 4933 */ 4934 if (activation_weight == METASLAB_WEIGHT_PRIMARY || *was_active) 4935 break; 4936 4937 if (!try_hard) { 4938 for (i = 0; i < d; i++) { 4939 if (!metaslab_is_unique(msp, &dva[i])) 4940 break; /* try another metaslab */ 4941 } 4942 if (i == d) 4943 break; 4944 } 4945 } 4946 4947 if (msp != NULL) { 4948 search->ms_weight = msp->ms_weight; 4949 search->ms_start = msp->ms_start + 1; 4950 search->ms_allocator = msp->ms_allocator; 4951 search->ms_primary = msp->ms_primary; 4952 } 4953 return (msp); 4954 } 4955 4956 static void 4957 metaslab_active_mask_verify(metaslab_t *msp) 4958 { 4959 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4960 4961 if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0) 4962 return; 4963 4964 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0) 4965 return; 4966 4967 if (msp->ms_weight & METASLAB_WEIGHT_PRIMARY) { 4968 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); 4969 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM); 4970 VERIFY3S(msp->ms_allocator, !=, -1); 4971 VERIFY(msp->ms_primary); 4972 return; 4973 } 4974 4975 if (msp->ms_weight & METASLAB_WEIGHT_SECONDARY) { 4976 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); 4977 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM); 4978 VERIFY3S(msp->ms_allocator, !=, -1); 4979 VERIFY(!msp->ms_primary); 4980 return; 4981 } 4982 4983 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) { 4984 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); 4985 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); 4986 VERIFY3S(msp->ms_allocator, ==, -1); 4987 return; 4988 } 4989 } 4990 4991 static uint64_t 4992 metaslab_group_alloc(metaslab_group_t *mg, zio_alloc_list_t *zal, 4993 uint64_t asize, uint64_t max_asize, uint64_t txg, 4994 dva_t *dva, int d, int allocator, boolean_t try_hard, 4995 uint64_t *actual_asize) 4996 { 4997 metaslab_t *msp = NULL; 4998 uint64_t offset = -1ULL; 4999 5000 uint64_t activation_weight = METASLAB_WEIGHT_PRIMARY; 5001 for (int i = 0; i < d; i++) { 5002 if (activation_weight == METASLAB_WEIGHT_PRIMARY && 5003 DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) { 5004 activation_weight = METASLAB_WEIGHT_SECONDARY; 5005 } else if (activation_weight == METASLAB_WEIGHT_SECONDARY && 5006 DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) { 5007 activation_weight = METASLAB_WEIGHT_CLAIM; 5008 break; 5009 } 5010 } 5011 5012 /* 5013 * If we don't have enough metaslabs active, we just use the 0th slot. 5014 */ 5015 if (allocator >= mg->mg_ms_ready / 3) 5016 allocator = 0; 5017 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 5018 5019 ASSERT3U(mg->mg_vd->vdev_ms_count, >=, 2); 5020 5021 metaslab_t *search = kmem_alloc(sizeof (*search), KM_SLEEP); 5022 search->ms_weight = UINT64_MAX; 5023 search->ms_start = 0; 5024 /* 5025 * At the end of the metaslab tree are the already-active metaslabs, 5026 * first the primaries, then the secondaries. When we resume searching 5027 * through the tree, we need to consider ms_allocator and ms_primary so 5028 * we start in the location right after where we left off, and don't 5029 * accidentally loop forever considering the same metaslabs. 5030 */ 5031 search->ms_allocator = -1; 5032 search->ms_primary = B_TRUE; 5033 for (;;) { 5034 boolean_t was_active = B_FALSE; 5035 5036 mutex_enter(&mg->mg_lock); 5037 5038 if (activation_weight == METASLAB_WEIGHT_PRIMARY && 5039 mga->mga_primary != NULL) { 5040 msp = mga->mga_primary; 5041 5042 /* 5043 * Even though we don't hold the ms_lock for the 5044 * primary metaslab, those fields should not 5045 * change while we hold the mg_lock. Thus it is 5046 * safe to make assertions on them. 5047 */ 5048 ASSERT(msp->ms_primary); 5049 ASSERT3S(msp->ms_allocator, ==, allocator); 5050 ASSERT(msp->ms_loaded); 5051 5052 was_active = B_TRUE; 5053 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 5054 } else if (activation_weight == METASLAB_WEIGHT_SECONDARY && 5055 mga->mga_secondary != NULL) { 5056 msp = mga->mga_secondary; 5057 5058 /* 5059 * See comment above about the similar assertions 5060 * for the primary metaslab. 5061 */ 5062 ASSERT(!msp->ms_primary); 5063 ASSERT3S(msp->ms_allocator, ==, allocator); 5064 ASSERT(msp->ms_loaded); 5065 5066 was_active = B_TRUE; 5067 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 5068 } else { 5069 msp = find_valid_metaslab(mg, activation_weight, dva, d, 5070 asize, allocator, try_hard, zal, search, 5071 &was_active); 5072 } 5073 5074 mutex_exit(&mg->mg_lock); 5075 if (msp == NULL) 5076 break; 5077 mutex_enter(&msp->ms_lock); 5078 5079 metaslab_active_mask_verify(msp); 5080 5081 /* 5082 * This code is disabled out because of issues with 5083 * tracepoints in non-gpl kernel modules. 5084 */ 5085 #if 0 5086 DTRACE_PROBE3(ms__activation__attempt, 5087 metaslab_t *, msp, uint64_t, activation_weight, 5088 boolean_t, was_active); 5089 #endif 5090 5091 /* 5092 * Ensure that the metaslab we have selected is still 5093 * capable of handling our request. It's possible that 5094 * another thread may have changed the weight while we 5095 * were blocked on the metaslab lock. We check the 5096 * active status first to see if we need to set_selected_txg 5097 * a new metaslab. 5098 */ 5099 if (was_active && !(msp->ms_weight & METASLAB_ACTIVE_MASK)) { 5100 ASSERT3S(msp->ms_allocator, ==, -1); 5101 mutex_exit(&msp->ms_lock); 5102 continue; 5103 } 5104 5105 /* 5106 * If the metaslab was activated for another allocator 5107 * while we were waiting in the ms_lock above, or it's 5108 * a primary and we're seeking a secondary (or vice versa), 5109 * we go back and select a new metaslab. 5110 */ 5111 if (!was_active && (msp->ms_weight & METASLAB_ACTIVE_MASK) && 5112 (msp->ms_allocator != -1) && 5113 (msp->ms_allocator != allocator || ((activation_weight == 5114 METASLAB_WEIGHT_PRIMARY) != msp->ms_primary))) { 5115 ASSERT(msp->ms_loaded); 5116 ASSERT((msp->ms_weight & METASLAB_WEIGHT_CLAIM) || 5117 msp->ms_allocator != -1); 5118 mutex_exit(&msp->ms_lock); 5119 continue; 5120 } 5121 5122 /* 5123 * This metaslab was used for claiming regions allocated 5124 * by the ZIL during pool import. Once these regions are 5125 * claimed we don't need to keep the CLAIM bit set 5126 * anymore. Passivate this metaslab to zero its activation 5127 * mask. 5128 */ 5129 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM && 5130 activation_weight != METASLAB_WEIGHT_CLAIM) { 5131 ASSERT(msp->ms_loaded); 5132 ASSERT3S(msp->ms_allocator, ==, -1); 5133 metaslab_passivate(msp, msp->ms_weight & 5134 ~METASLAB_WEIGHT_CLAIM); 5135 mutex_exit(&msp->ms_lock); 5136 continue; 5137 } 5138 5139 metaslab_set_selected_txg(msp, txg); 5140 5141 int activation_error = 5142 metaslab_activate(msp, allocator, activation_weight); 5143 metaslab_active_mask_verify(msp); 5144 5145 /* 5146 * If the metaslab was activated by another thread for 5147 * another allocator or activation_weight (EBUSY), or it 5148 * failed because another metaslab was assigned as primary 5149 * for this allocator (EEXIST) we continue using this 5150 * metaslab for our allocation, rather than going on to a 5151 * worse metaslab (we waited for that metaslab to be loaded 5152 * after all). 5153 * 5154 * If the activation failed due to an I/O error or ENOSPC we 5155 * skip to the next metaslab. 5156 */ 5157 boolean_t activated; 5158 if (activation_error == 0) { 5159 activated = B_TRUE; 5160 } else if (activation_error == EBUSY || 5161 activation_error == EEXIST) { 5162 activated = B_FALSE; 5163 } else { 5164 mutex_exit(&msp->ms_lock); 5165 continue; 5166 } 5167 ASSERT(msp->ms_loaded); 5168 5169 /* 5170 * Now that we have the lock, recheck to see if we should 5171 * continue to use this metaslab for this allocation. The 5172 * the metaslab is now loaded so metaslab_should_allocate() 5173 * can accurately determine if the allocation attempt should 5174 * proceed. 5175 */ 5176 if (!metaslab_should_allocate(msp, asize, try_hard)) { 5177 /* Passivate this metaslab and select a new one. */ 5178 metaslab_trace_add(zal, mg, msp, asize, d, 5179 TRACE_TOO_SMALL, allocator); 5180 goto next; 5181 } 5182 5183 /* 5184 * If this metaslab is currently condensing then pick again 5185 * as we can't manipulate this metaslab until it's committed 5186 * to disk. If this metaslab is being initialized, we shouldn't 5187 * allocate from it since the allocated region might be 5188 * overwritten after allocation. 5189 */ 5190 if (msp->ms_condensing) { 5191 metaslab_trace_add(zal, mg, msp, asize, d, 5192 TRACE_CONDENSING, allocator); 5193 if (activated) { 5194 metaslab_passivate(msp, msp->ms_weight & 5195 ~METASLAB_ACTIVE_MASK); 5196 } 5197 mutex_exit(&msp->ms_lock); 5198 continue; 5199 } else if (msp->ms_disabled > 0) { 5200 metaslab_trace_add(zal, mg, msp, asize, d, 5201 TRACE_DISABLED, allocator); 5202 if (activated) { 5203 metaslab_passivate(msp, msp->ms_weight & 5204 ~METASLAB_ACTIVE_MASK); 5205 } 5206 mutex_exit(&msp->ms_lock); 5207 continue; 5208 } 5209 5210 offset = metaslab_block_alloc(msp, asize, max_asize, txg, 5211 actual_asize); 5212 5213 if (offset != -1ULL) { 5214 metaslab_trace_add(zal, mg, msp, *actual_asize, d, 5215 offset, allocator); 5216 /* Proactively passivate the metaslab, if needed */ 5217 if (activated) 5218 metaslab_segment_may_passivate(msp); 5219 mutex_exit(&msp->ms_lock); 5220 break; 5221 } 5222 metaslab_trace_add(zal, mg, msp, asize, d, offset, allocator); 5223 next: 5224 ASSERT(msp->ms_loaded); 5225 5226 /* 5227 * This code is disabled out because of issues with 5228 * tracepoints in non-gpl kernel modules. 5229 */ 5230 #if 0 5231 DTRACE_PROBE2(ms__alloc__failure, metaslab_t *, msp, 5232 uint64_t, asize); 5233 #endif 5234 5235 /* 5236 * We were unable to allocate from this metaslab so determine 5237 * a new weight for this metaslab. The weight was last 5238 * recalculated either when we loaded it (if this is the first 5239 * TXG it's been loaded in), or the last time a txg was synced 5240 * out. 5241 */ 5242 uint64_t weight; 5243 if (WEIGHT_IS_SPACEBASED(msp->ms_weight)) { 5244 metaslab_set_fragmentation(msp, B_TRUE); 5245 weight = metaslab_space_weight(msp) & 5246 ~METASLAB_ACTIVE_MASK; 5247 } else { 5248 weight = metaslab_weight_from_range_tree(msp); 5249 } 5250 5251 if (activated) { 5252 metaslab_passivate(msp, weight); 5253 } else { 5254 /* 5255 * For the case where we use the metaslab that is 5256 * active for another allocator we want to make 5257 * sure that we retain the activation mask. 5258 */ 5259 weight |= msp->ms_weight & METASLAB_ACTIVE_MASK; 5260 metaslab_group_sort(mg, msp, weight); 5261 } 5262 metaslab_active_mask_verify(msp); 5263 5264 /* 5265 * We have just failed an allocation attempt, check 5266 * that metaslab_should_allocate() agrees. Otherwise, 5267 * we may end up in an infinite loop retrying the same 5268 * metaslab. 5269 */ 5270 ASSERT(!metaslab_should_allocate(msp, asize, try_hard)); 5271 5272 mutex_exit(&msp->ms_lock); 5273 } 5274 kmem_free(search, sizeof (*search)); 5275 5276 if (offset == -1ULL) { 5277 metaslab_trace_add(zal, mg, NULL, asize, d, 5278 TRACE_GROUP_FAILURE, allocator); 5279 if (asize <= vdev_get_min_alloc(mg->mg_vd)) { 5280 /* 5281 * This metaslab group was unable to allocate 5282 * the minimum block size so it must be out of 5283 * space. Notify the allocation throttle to 5284 * skip allocation attempts to this group until 5285 * more space becomes available. 5286 */ 5287 mg->mg_no_free_space = B_TRUE; 5288 } 5289 } 5290 return (offset); 5291 } 5292 5293 static boolean_t 5294 metaslab_group_allocatable(spa_t *spa, metaslab_group_t *mg, uint64_t psize, 5295 int d, int flags, boolean_t try_hard, zio_alloc_list_t *zal, int allocator) 5296 { 5297 metaslab_class_t *mc = mg->mg_class; 5298 vdev_t *vd = mg->mg_vd; 5299 boolean_t allocatable; 5300 5301 /* 5302 * Don't allocate from faulted devices. 5303 */ 5304 if (try_hard) 5305 spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER); 5306 allocatable = vdev_allocatable(vd); 5307 if (try_hard) 5308 spa_config_exit(spa, SCL_ZIO, FTAG); 5309 if (!allocatable) { 5310 metaslab_trace_add(zal, mg, NULL, psize, d, 5311 TRACE_NOT_ALLOCATABLE, allocator); 5312 return (B_FALSE); 5313 } 5314 5315 if (!try_hard) { 5316 /* 5317 * Avoid vdevs with too little space or too fragmented. 5318 */ 5319 if (!GANG_ALLOCATION(flags) && (mg->mg_no_free_space || 5320 (!mg->mg_allocatable && mc->mc_alloc_groups > 0))) { 5321 metaslab_trace_add(zal, mg, NULL, psize, d, 5322 TRACE_NOT_ALLOCATABLE, allocator); 5323 return (B_FALSE); 5324 } 5325 5326 /* 5327 * Avoid writing single-copy data to an unhealthy, 5328 * non-redundant vdev. 5329 */ 5330 if (d == 0 && vd->vdev_state < VDEV_STATE_HEALTHY && 5331 vd->vdev_children == 0) { 5332 metaslab_trace_add(zal, mg, NULL, psize, d, 5333 TRACE_VDEV_ERROR, allocator); 5334 return (B_FALSE); 5335 } 5336 } 5337 5338 return (B_TRUE); 5339 } 5340 5341 static int 5342 metaslab_alloc_dva_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize, 5343 uint64_t max_psize, dva_t *dva, int d, const dva_t *hintdva, uint64_t txg, 5344 int flags, zio_alloc_list_t *zal, int allocator, uint64_t *actual_psize) 5345 { 5346 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 5347 metaslab_group_t *mg = NULL, *rotor; 5348 vdev_t *vd; 5349 boolean_t try_hard = B_FALSE; 5350 5351 ASSERT(!DVA_IS_VALID(&dva[d])); 5352 5353 /* 5354 * For testing, make some blocks above a certain size be gang blocks. 5355 * This will result in more split blocks when using device removal, 5356 * and a large number of split blocks coupled with ztest-induced 5357 * damage can result in extremely long reconstruction times. This 5358 * will also test spilling from special to normal. 5359 */ 5360 if (psize >= metaslab_force_ganging && 5361 metaslab_force_ganging_pct > 0 && 5362 (random_in_range(100) < MIN(metaslab_force_ganging_pct, 100))) { 5363 metaslab_trace_add(zal, NULL, NULL, psize, d, TRACE_FORCE_GANG, 5364 allocator); 5365 return (SET_ERROR(ENOSPC)); 5366 } 5367 if (max_psize > psize && max_psize >= metaslab_force_ganging && 5368 metaslab_force_ganging_pct > 0 && 5369 (random_in_range(100) < MIN(metaslab_force_ganging_pct, 100))) { 5370 max_psize = MAX((psize + max_psize) / 2, 5371 metaslab_force_ganging); 5372 } 5373 ASSERT3U(psize, <=, max_psize); 5374 5375 /* 5376 * Start at the rotor and loop through all mgs until we find something. 5377 * Note that there's no locking on mca_rotor or mca_aliquot because 5378 * nothing actually breaks if we miss a few updates -- we just won't 5379 * allocate quite as evenly. It all balances out over time. 5380 * 5381 * If we are doing ditto or log blocks, try to spread them across 5382 * consecutive vdevs. If we're forced to reuse a vdev before we've 5383 * allocated all of our ditto blocks, then try and spread them out on 5384 * that vdev as much as possible. If it turns out to not be possible, 5385 * gradually lower our standards until anything becomes acceptable. 5386 * Also, allocating on consecutive vdevs (as opposed to random vdevs) 5387 * gives us hope of containing our fault domains to something we're 5388 * able to reason about. Otherwise, any two top-level vdev failures 5389 * will guarantee the loss of data. With consecutive allocation, 5390 * only two adjacent top-level vdev failures will result in data loss. 5391 * 5392 * If we are doing gang blocks (hintdva is non-NULL), try to keep 5393 * ourselves on the same vdev as our gang block header. It makes our 5394 * fault domains something tractable. 5395 */ 5396 if (hintdva && DVA_IS_VALID(&hintdva[d])) { 5397 vd = vdev_lookup_top(spa, DVA_GET_VDEV(&hintdva[d])); 5398 mg = vdev_get_mg(vd, mc); 5399 } 5400 if (mg == NULL && d != 0) { 5401 vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[d - 1])); 5402 mg = vdev_get_mg(vd, mc)->mg_next; 5403 } 5404 if (mg == NULL || mg->mg_class != mc || mg->mg_activation_count <= 0) { 5405 ASSERT(mca->mca_rotor != NULL); 5406 mg = mca->mca_rotor; 5407 } 5408 5409 rotor = mg; 5410 top: 5411 do { 5412 ASSERT(mg->mg_activation_count == 1); 5413 ASSERT(mg->mg_class == mc); 5414 5415 if (!metaslab_group_allocatable(spa, mg, psize, d, flags, 5416 try_hard, zal, allocator)) 5417 goto next; 5418 5419 vd = mg->mg_vd; 5420 uint64_t asize = vdev_psize_to_asize_txg(vd, psize, txg); 5421 ASSERT0(P2PHASE(asize, 1ULL << vd->vdev_ashift)); 5422 uint64_t max_asize = vdev_psize_to_asize_txg(vd, max_psize, 5423 txg); 5424 ASSERT0(P2PHASE(max_asize, 1ULL << vd->vdev_ashift)); 5425 uint64_t offset = metaslab_group_alloc(mg, zal, asize, 5426 max_asize, txg, dva, d, allocator, try_hard, 5427 &asize); 5428 5429 if (offset != -1ULL) { 5430 if (actual_psize) 5431 *actual_psize = vdev_asize_to_psize_txg(vd, 5432 asize, txg); 5433 metaslab_class_rotate(mg, allocator, psize, B_TRUE); 5434 5435 DVA_SET_VDEV(&dva[d], vd->vdev_id); 5436 DVA_SET_OFFSET(&dva[d], offset); 5437 DVA_SET_GANG(&dva[d], 5438 ((flags & METASLAB_GANG_HEADER) ? 1 : 0)); 5439 DVA_SET_ASIZE(&dva[d], asize); 5440 return (0); 5441 } 5442 next: 5443 metaslab_class_rotate(mg, allocator, psize, B_FALSE); 5444 } while ((mg = mg->mg_next) != rotor); 5445 5446 /* 5447 * If we haven't tried hard, perhaps do so now. 5448 */ 5449 if (!try_hard && (zfs_metaslab_try_hard_before_gang || 5450 GANG_ALLOCATION(flags) || (flags & METASLAB_ZIL) != 0 || 5451 psize <= spa->spa_min_alloc)) { 5452 METASLABSTAT_BUMP(metaslabstat_try_hard); 5453 try_hard = B_TRUE; 5454 goto top; 5455 } 5456 5457 memset(&dva[d], 0, sizeof (dva_t)); 5458 5459 metaslab_trace_add(zal, rotor, NULL, psize, d, TRACE_ENOSPC, allocator); 5460 return (SET_ERROR(ENOSPC)); 5461 } 5462 5463 /* 5464 * Allocate a block for the specified i/o. 5465 */ 5466 int 5467 metaslab_alloc_dva(spa_t *spa, metaslab_class_t *mc, uint64_t psize, 5468 dva_t *dva, int d, const dva_t *hintdva, uint64_t txg, int flags, 5469 zio_alloc_list_t *zal, int allocator) 5470 { 5471 return (metaslab_alloc_dva_range(spa, mc, psize, psize, dva, d, hintdva, 5472 txg, flags, zal, allocator, NULL)); 5473 } 5474 5475 void 5476 metaslab_free_concrete(vdev_t *vd, uint64_t offset, uint64_t asize, 5477 boolean_t checkpoint) 5478 { 5479 metaslab_t *msp; 5480 spa_t *spa = vd->vdev_spa; 5481 int m = offset >> vd->vdev_ms_shift; 5482 5483 ASSERT(vdev_is_concrete(vd)); 5484 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5485 VERIFY3U(m, <, vd->vdev_ms_count); 5486 5487 msp = vd->vdev_ms[m]; 5488 5489 VERIFY(!msp->ms_condensing); 5490 VERIFY3U(offset, >=, msp->ms_start); 5491 VERIFY3U(offset + asize, <=, msp->ms_start + msp->ms_size); 5492 VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 5493 VERIFY0(P2PHASE(asize, 1ULL << vd->vdev_ashift)); 5494 5495 metaslab_check_free_impl(vd, offset, asize); 5496 5497 mutex_enter(&msp->ms_lock); 5498 if (zfs_range_tree_is_empty(msp->ms_freeing) && 5499 zfs_range_tree_is_empty(msp->ms_checkpointing)) { 5500 vdev_dirty(vd, VDD_METASLAB, msp, spa_syncing_txg(spa)); 5501 } 5502 5503 if (checkpoint) { 5504 ASSERT(spa_has_checkpoint(spa)); 5505 zfs_range_tree_add(msp->ms_checkpointing, offset, asize); 5506 } else { 5507 zfs_range_tree_add(msp->ms_freeing, offset, asize); 5508 } 5509 mutex_exit(&msp->ms_lock); 5510 } 5511 5512 void 5513 metaslab_free_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, 5514 uint64_t size, void *arg) 5515 { 5516 (void) inner_offset; 5517 boolean_t *checkpoint = arg; 5518 5519 ASSERT3P(checkpoint, !=, NULL); 5520 5521 if (vd->vdev_ops->vdev_op_remap != NULL) 5522 vdev_indirect_mark_obsolete(vd, offset, size); 5523 else 5524 metaslab_free_impl(vd, offset, size, *checkpoint); 5525 } 5526 5527 static void 5528 metaslab_free_impl(vdev_t *vd, uint64_t offset, uint64_t size, 5529 boolean_t checkpoint) 5530 { 5531 spa_t *spa = vd->vdev_spa; 5532 5533 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5534 5535 if (spa_syncing_txg(spa) > spa_freeze_txg(spa)) 5536 return; 5537 5538 if (spa->spa_vdev_removal != NULL && 5539 spa->spa_vdev_removal->svr_vdev_id == vd->vdev_id && 5540 vdev_is_concrete(vd)) { 5541 /* 5542 * Note: we check if the vdev is concrete because when 5543 * we complete the removal, we first change the vdev to be 5544 * an indirect vdev (in open context), and then (in syncing 5545 * context) clear spa_vdev_removal. 5546 */ 5547 free_from_removing_vdev(vd, offset, size); 5548 } else if (vd->vdev_ops->vdev_op_remap != NULL) { 5549 vdev_indirect_mark_obsolete(vd, offset, size); 5550 vd->vdev_ops->vdev_op_remap(vd, offset, size, 5551 metaslab_free_impl_cb, &checkpoint); 5552 } else { 5553 metaslab_free_concrete(vd, offset, size, checkpoint); 5554 } 5555 } 5556 5557 typedef struct remap_blkptr_cb_arg { 5558 blkptr_t *rbca_bp; 5559 spa_remap_cb_t rbca_cb; 5560 vdev_t *rbca_remap_vd; 5561 uint64_t rbca_remap_offset; 5562 void *rbca_cb_arg; 5563 } remap_blkptr_cb_arg_t; 5564 5565 static void 5566 remap_blkptr_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, 5567 uint64_t size, void *arg) 5568 { 5569 remap_blkptr_cb_arg_t *rbca = arg; 5570 blkptr_t *bp = rbca->rbca_bp; 5571 5572 /* We can not remap split blocks. */ 5573 if (size != DVA_GET_ASIZE(&bp->blk_dva[0])) 5574 return; 5575 ASSERT0(inner_offset); 5576 5577 if (rbca->rbca_cb != NULL) { 5578 /* 5579 * At this point we know that we are not handling split 5580 * blocks and we invoke the callback on the previous 5581 * vdev which must be indirect. 5582 */ 5583 ASSERT3P(rbca->rbca_remap_vd->vdev_ops, ==, &vdev_indirect_ops); 5584 5585 rbca->rbca_cb(rbca->rbca_remap_vd->vdev_id, 5586 rbca->rbca_remap_offset, size, rbca->rbca_cb_arg); 5587 5588 /* set up remap_blkptr_cb_arg for the next call */ 5589 rbca->rbca_remap_vd = vd; 5590 rbca->rbca_remap_offset = offset; 5591 } 5592 5593 /* 5594 * The phys birth time is that of dva[0]. This ensures that we know 5595 * when each dva was written, so that resilver can determine which 5596 * blocks need to be scrubbed (i.e. those written during the time 5597 * the vdev was offline). It also ensures that the key used in 5598 * the ARC hash table is unique (i.e. dva[0] + phys_birth). If 5599 * we didn't change the phys_birth, a lookup in the ARC for a 5600 * remapped BP could find the data that was previously stored at 5601 * this vdev + offset. 5602 */ 5603 vdev_t *oldvd = vdev_lookup_top(vd->vdev_spa, 5604 DVA_GET_VDEV(&bp->blk_dva[0])); 5605 vdev_indirect_births_t *vib = oldvd->vdev_indirect_births; 5606 uint64_t physical_birth = vdev_indirect_births_physbirth(vib, 5607 DVA_GET_OFFSET(&bp->blk_dva[0]), DVA_GET_ASIZE(&bp->blk_dva[0])); 5608 5609 /* 5610 * For rewritten blocks, use the old physical birth as the new logical 5611 * birth (representing when the space was allocated) and the removal 5612 * time as the new physical birth (representing when it was actually 5613 * written). 5614 */ 5615 if (BP_GET_REWRITE(bp)) { 5616 uint64_t old_physical_birth = BP_GET_PHYSICAL_BIRTH(bp); 5617 ASSERT3U(old_physical_birth, <, physical_birth); 5618 BP_SET_BIRTH(bp, old_physical_birth, physical_birth); 5619 BP_SET_REWRITE(bp, 0); 5620 } else { 5621 BP_SET_PHYSICAL_BIRTH(bp, physical_birth); 5622 } 5623 5624 DVA_SET_VDEV(&bp->blk_dva[0], vd->vdev_id); 5625 DVA_SET_OFFSET(&bp->blk_dva[0], offset); 5626 } 5627 5628 /* 5629 * If the block pointer contains any indirect DVAs, modify them to refer to 5630 * concrete DVAs. Note that this will sometimes not be possible, leaving 5631 * the indirect DVA in place. This happens if the indirect DVA spans multiple 5632 * segments in the mapping (i.e. it is a "split block"). 5633 * 5634 * If the BP was remapped, calls the callback on the original dva (note the 5635 * callback can be called multiple times if the original indirect DVA refers 5636 * to another indirect DVA, etc). 5637 * 5638 * Returns TRUE if the BP was remapped. 5639 */ 5640 boolean_t 5641 spa_remap_blkptr(spa_t *spa, blkptr_t *bp, spa_remap_cb_t callback, void *arg) 5642 { 5643 remap_blkptr_cb_arg_t rbca; 5644 5645 if (!zfs_remap_blkptr_enable) 5646 return (B_FALSE); 5647 5648 if (!spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)) 5649 return (B_FALSE); 5650 5651 /* 5652 * Dedup BP's can not be remapped, because ddt_phys_select() depends 5653 * on DVA[0] being the same in the BP as in the DDT (dedup table). 5654 */ 5655 if (BP_GET_DEDUP(bp)) 5656 return (B_FALSE); 5657 5658 /* 5659 * Gang blocks can not be remapped, because 5660 * zio_checksum_gang_verifier() depends on the DVA[0] that's in 5661 * the BP used to read the gang block header (GBH) being the same 5662 * as the DVA[0] that we allocated for the GBH. 5663 */ 5664 if (BP_IS_GANG(bp)) 5665 return (B_FALSE); 5666 5667 /* 5668 * Embedded BP's have no DVA to remap. 5669 */ 5670 if (BP_GET_NDVAS(bp) < 1) 5671 return (B_FALSE); 5672 5673 /* 5674 * Cloned blocks can not be remapped since BRT depends on specific 5675 * vdev id and offset in the DVA[0] for its reference counting. 5676 */ 5677 if (!BP_IS_METADATA(bp) && brt_maybe_exists(spa, bp)) 5678 return (B_FALSE); 5679 5680 /* 5681 * Note: we only remap dva[0]. If we remapped other dvas, we 5682 * would no longer know what their phys birth txg is. 5683 */ 5684 dva_t *dva = &bp->blk_dva[0]; 5685 5686 uint64_t offset = DVA_GET_OFFSET(dva); 5687 uint64_t size = DVA_GET_ASIZE(dva); 5688 vdev_t *vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva)); 5689 5690 if (vd->vdev_ops->vdev_op_remap == NULL) 5691 return (B_FALSE); 5692 5693 rbca.rbca_bp = bp; 5694 rbca.rbca_cb = callback; 5695 rbca.rbca_remap_vd = vd; 5696 rbca.rbca_remap_offset = offset; 5697 rbca.rbca_cb_arg = arg; 5698 5699 /* 5700 * remap_blkptr_cb() will be called in order for each level of 5701 * indirection, until a concrete vdev is reached or a split block is 5702 * encountered. old_vd and old_offset are updated within the callback 5703 * as we go from the one indirect vdev to the next one (either concrete 5704 * or indirect again) in that order. 5705 */ 5706 vd->vdev_ops->vdev_op_remap(vd, offset, size, remap_blkptr_cb, &rbca); 5707 5708 /* Check if the DVA wasn't remapped because it is a split block */ 5709 if (DVA_GET_VDEV(&rbca.rbca_bp->blk_dva[0]) == vd->vdev_id) 5710 return (B_FALSE); 5711 5712 return (B_TRUE); 5713 } 5714 5715 /* 5716 * Undo the allocation of a DVA which happened in the given transaction group. 5717 */ 5718 void 5719 metaslab_unalloc_dva(spa_t *spa, const dva_t *dva, uint64_t txg) 5720 { 5721 metaslab_t *msp; 5722 vdev_t *vd; 5723 uint64_t vdev = DVA_GET_VDEV(dva); 5724 uint64_t offset = DVA_GET_OFFSET(dva); 5725 uint64_t size = DVA_GET_ASIZE(dva); 5726 5727 ASSERT(DVA_IS_VALID(dva)); 5728 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5729 5730 if (txg > spa_freeze_txg(spa)) 5731 return; 5732 5733 if ((vd = vdev_lookup_top(spa, vdev)) == NULL || !DVA_IS_VALID(dva) || 5734 (offset >> vd->vdev_ms_shift) >= vd->vdev_ms_count) { 5735 zfs_panic_recover("metaslab_free_dva(): bad DVA %llu:%llu:%llu", 5736 (u_longlong_t)vdev, (u_longlong_t)offset, 5737 (u_longlong_t)size); 5738 return; 5739 } 5740 5741 ASSERT(!vd->vdev_removing); 5742 ASSERT(vdev_is_concrete(vd)); 5743 ASSERT0(vd->vdev_indirect_config.vic_mapping_object); 5744 ASSERT0P(vd->vdev_indirect_mapping); 5745 5746 if (DVA_GET_GANG(dva)) 5747 size = vdev_gang_header_asize(vd); 5748 5749 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; 5750 5751 mutex_enter(&msp->ms_lock); 5752 zfs_range_tree_remove(msp->ms_allocating[txg & TXG_MASK], 5753 offset, size); 5754 msp->ms_allocating_total -= size; 5755 5756 VERIFY(!msp->ms_condensing); 5757 VERIFY3U(offset, >=, msp->ms_start); 5758 VERIFY3U(offset + size, <=, msp->ms_start + msp->ms_size); 5759 VERIFY3U(zfs_range_tree_space(msp->ms_allocatable) + size, <=, 5760 msp->ms_size); 5761 VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 5762 VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 5763 zfs_range_tree_add(msp->ms_allocatable, offset, size); 5764 mutex_exit(&msp->ms_lock); 5765 } 5766 5767 /* 5768 * Free the block represented by the given DVA. 5769 */ 5770 void 5771 metaslab_free_dva(spa_t *spa, const dva_t *dva, boolean_t checkpoint) 5772 { 5773 uint64_t vdev = DVA_GET_VDEV(dva); 5774 uint64_t offset = DVA_GET_OFFSET(dva); 5775 uint64_t size = DVA_GET_ASIZE(dva); 5776 vdev_t *vd = vdev_lookup_top(spa, vdev); 5777 5778 ASSERT(DVA_IS_VALID(dva)); 5779 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5780 5781 if (DVA_GET_GANG(dva)) { 5782 size = vdev_gang_header_asize(vd); 5783 } 5784 5785 metaslab_free_impl(vd, offset, size, checkpoint); 5786 } 5787 5788 /* 5789 * Reserve some space for a future allocation. The reservation system must be 5790 * called before we call into the allocator. If there aren't enough space 5791 * available, the calling I/O will be throttled until another I/O completes and 5792 * its reservation is released. The function returns true if it was successful 5793 * in placing the reservation. 5794 */ 5795 boolean_t 5796 metaslab_class_throttle_reserve(metaslab_class_t *mc, int allocator, 5797 int copies, uint64_t io_size, boolean_t must, boolean_t *more) 5798 { 5799 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 5800 5801 ASSERT(mc->mc_alloc_throttle_enabled); 5802 if (mc->mc_alloc_io_size < io_size) { 5803 mc->mc_alloc_io_size = io_size; 5804 metaslab_class_balance(mc, B_FALSE); 5805 } 5806 if (must || mca->mca_reserved <= mc->mc_alloc_max) { 5807 /* 5808 * The potential race between compare and add is covered by the 5809 * allocator lock in most cases, or irrelevant due to must set. 5810 * But even if we assume some other non-existing scenario, the 5811 * worst that can happen is few more I/Os get to allocation 5812 * earlier, that is not a problem. 5813 */ 5814 int64_t delta = copies * io_size; 5815 *more = (atomic_add_64_nv(&mca->mca_reserved, delta) <= 5816 mc->mc_alloc_max); 5817 return (B_TRUE); 5818 } 5819 *more = B_FALSE; 5820 return (B_FALSE); 5821 } 5822 5823 boolean_t 5824 metaslab_class_throttle_unreserve(metaslab_class_t *mc, int allocator, 5825 int copies, uint64_t io_size) 5826 { 5827 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 5828 5829 ASSERT(mc->mc_alloc_throttle_enabled); 5830 int64_t delta = copies * io_size; 5831 return (atomic_add_64_nv(&mca->mca_reserved, -delta) <= 5832 mc->mc_alloc_max); 5833 } 5834 5835 static int 5836 metaslab_claim_concrete(vdev_t *vd, uint64_t offset, uint64_t size, 5837 uint64_t txg) 5838 { 5839 metaslab_t *msp; 5840 spa_t *spa = vd->vdev_spa; 5841 int error = 0; 5842 5843 if (offset >> vd->vdev_ms_shift >= vd->vdev_ms_count) 5844 return (SET_ERROR(ENXIO)); 5845 5846 ASSERT3P(vd->vdev_ms, !=, NULL); 5847 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; 5848 5849 mutex_enter(&msp->ms_lock); 5850 5851 if ((txg != 0 && spa_writeable(spa)) || !msp->ms_loaded) { 5852 error = metaslab_activate(msp, 0, METASLAB_WEIGHT_CLAIM); 5853 if (error == EBUSY) { 5854 ASSERT(msp->ms_loaded); 5855 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 5856 error = 0; 5857 } 5858 } 5859 5860 if (error == 0 && 5861 !zfs_range_tree_contains(msp->ms_allocatable, offset, size)) 5862 error = SET_ERROR(ENOENT); 5863 5864 if (error || txg == 0) { /* txg == 0 indicates dry run */ 5865 mutex_exit(&msp->ms_lock); 5866 return (error); 5867 } 5868 5869 VERIFY(!msp->ms_condensing); 5870 VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 5871 VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 5872 VERIFY3U(zfs_range_tree_space(msp->ms_allocatable) - size, <=, 5873 msp->ms_size); 5874 zfs_range_tree_remove(msp->ms_allocatable, offset, size); 5875 zfs_range_tree_clear(msp->ms_trim, offset, size); 5876 5877 if (spa_writeable(spa)) { /* don't dirty if we're zdb(8) */ 5878 metaslab_class_t *mc = msp->ms_group->mg_class; 5879 multilist_sublist_t *mls = 5880 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 5881 if (!multilist_link_active(&msp->ms_class_txg_node)) { 5882 msp->ms_selected_txg = txg; 5883 multilist_sublist_insert_head(mls, msp); 5884 } 5885 multilist_sublist_unlock(mls); 5886 5887 if (zfs_range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK])) 5888 vdev_dirty(vd, VDD_METASLAB, msp, txg); 5889 zfs_range_tree_add(msp->ms_allocating[txg & TXG_MASK], 5890 offset, size); 5891 msp->ms_allocating_total += size; 5892 } 5893 5894 mutex_exit(&msp->ms_lock); 5895 5896 return (0); 5897 } 5898 5899 typedef struct metaslab_claim_cb_arg_t { 5900 uint64_t mcca_txg; 5901 int mcca_error; 5902 } metaslab_claim_cb_arg_t; 5903 5904 static void 5905 metaslab_claim_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, 5906 uint64_t size, void *arg) 5907 { 5908 (void) inner_offset; 5909 metaslab_claim_cb_arg_t *mcca_arg = arg; 5910 5911 if (mcca_arg->mcca_error == 0) { 5912 mcca_arg->mcca_error = metaslab_claim_concrete(vd, offset, 5913 size, mcca_arg->mcca_txg); 5914 } 5915 } 5916 5917 int 5918 metaslab_claim_impl(vdev_t *vd, uint64_t offset, uint64_t size, uint64_t txg) 5919 { 5920 if (vd->vdev_ops->vdev_op_remap != NULL) { 5921 metaslab_claim_cb_arg_t arg; 5922 5923 /* 5924 * Only zdb(8) can claim on indirect vdevs. This is used 5925 * to detect leaks of mapped space (that are not accounted 5926 * for in the obsolete counts, spacemap, or bpobj). 5927 */ 5928 ASSERT(!spa_writeable(vd->vdev_spa)); 5929 arg.mcca_error = 0; 5930 arg.mcca_txg = txg; 5931 5932 vd->vdev_ops->vdev_op_remap(vd, offset, size, 5933 metaslab_claim_impl_cb, &arg); 5934 5935 if (arg.mcca_error == 0) { 5936 arg.mcca_error = metaslab_claim_concrete(vd, 5937 offset, size, txg); 5938 } 5939 return (arg.mcca_error); 5940 } else { 5941 return (metaslab_claim_concrete(vd, offset, size, txg)); 5942 } 5943 } 5944 5945 /* 5946 * Intent log support: upon opening the pool after a crash, notify the SPA 5947 * of blocks that the intent log has allocated for immediate write, but 5948 * which are still considered free by the SPA because the last transaction 5949 * group didn't commit yet. 5950 */ 5951 static int 5952 metaslab_claim_dva(spa_t *spa, const dva_t *dva, uint64_t txg) 5953 { 5954 uint64_t vdev = DVA_GET_VDEV(dva); 5955 uint64_t offset = DVA_GET_OFFSET(dva); 5956 uint64_t size = DVA_GET_ASIZE(dva); 5957 vdev_t *vd; 5958 5959 if ((vd = vdev_lookup_top(spa, vdev)) == NULL) { 5960 return (SET_ERROR(ENXIO)); 5961 } 5962 5963 ASSERT(DVA_IS_VALID(dva)); 5964 5965 if (DVA_GET_GANG(dva)) 5966 size = vdev_gang_header_asize(vd); 5967 5968 return (metaslab_claim_impl(vd, offset, size, txg)); 5969 } 5970 5971 int 5972 metaslab_alloc(spa_t *spa, metaslab_class_t *mc, uint64_t psize, blkptr_t *bp, 5973 int ndvas, uint64_t txg, const blkptr_t *hintbp, int flags, 5974 zio_alloc_list_t *zal, int allocator, const void *tag) 5975 { 5976 return (metaslab_alloc_range(spa, mc, psize, psize, bp, ndvas, txg, 5977 hintbp, flags, zal, allocator, tag, NULL)); 5978 } 5979 5980 int 5981 metaslab_alloc_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize, 5982 uint64_t max_psize, blkptr_t *bp, int ndvas, uint64_t txg, 5983 const blkptr_t *hintbp, int flags, zio_alloc_list_t *zal, int allocator, 5984 const void *tag, uint64_t *actual_psize) 5985 { 5986 dva_t *dva = bp->blk_dva; 5987 const dva_t *hintdva = (hintbp != NULL) ? hintbp->blk_dva : NULL; 5988 int error = 0; 5989 5990 ASSERT0(BP_GET_LOGICAL_BIRTH(bp)); 5991 ASSERT0(BP_GET_RAW_PHYSICAL_BIRTH(bp)); 5992 5993 spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER); 5994 5995 if (mc->mc_allocator[allocator].mca_rotor == NULL) { 5996 /* no vdevs in this class */ 5997 spa_config_exit(spa, SCL_ALLOC, FTAG); 5998 return (SET_ERROR(ENOSPC)); 5999 } 6000 6001 ASSERT(ndvas > 0 && ndvas <= spa_max_replication(spa)); 6002 ASSERT0(BP_GET_NDVAS(bp)); 6003 ASSERT(hintbp == NULL || ndvas <= BP_GET_NDVAS(hintbp)); 6004 ASSERT3P(zal, !=, NULL); 6005 6006 uint64_t smallest_psize = UINT64_MAX; 6007 for (int d = 0; d < ndvas; d++) { 6008 uint64_t cur_psize = 0; 6009 error = metaslab_alloc_dva_range(spa, mc, psize, 6010 MIN(smallest_psize, max_psize), dva, d, hintdva, txg, 6011 flags, zal, allocator, actual_psize ? &cur_psize : NULL); 6012 if (error != 0) { 6013 for (d--; d >= 0; d--) { 6014 metaslab_unalloc_dva(spa, &dva[d], txg); 6015 metaslab_group_alloc_decrement(spa, 6016 DVA_GET_VDEV(&dva[d]), allocator, flags, 6017 psize, tag); 6018 memset(&dva[d], 0, sizeof (dva_t)); 6019 } 6020 spa_config_exit(spa, SCL_ALLOC, FTAG); 6021 return (error); 6022 } else { 6023 /* 6024 * Update the metaslab group's queue depth 6025 * based on the newly allocated dva. 6026 */ 6027 metaslab_group_alloc_increment(spa, 6028 DVA_GET_VDEV(&dva[d]), allocator, flags, psize, 6029 tag); 6030 if (actual_psize) 6031 smallest_psize = MIN(cur_psize, smallest_psize); 6032 } 6033 } 6034 ASSERT0(error); 6035 ASSERT(BP_GET_NDVAS(bp) == ndvas); 6036 if (actual_psize) 6037 *actual_psize = smallest_psize; 6038 6039 spa_config_exit(spa, SCL_ALLOC, FTAG); 6040 6041 BP_SET_BIRTH(bp, txg, 0); 6042 6043 return (0); 6044 } 6045 6046 void 6047 metaslab_free(spa_t *spa, const blkptr_t *bp, uint64_t txg, boolean_t now) 6048 { 6049 const dva_t *dva = bp->blk_dva; 6050 int ndvas = BP_GET_NDVAS(bp); 6051 6052 ASSERT(!BP_IS_HOLE(bp)); 6053 ASSERT(!now || BP_GET_BIRTH(bp) >= spa_syncing_txg(spa)); 6054 6055 /* 6056 * If we have a checkpoint for the pool we need to make sure that 6057 * the blocks that we free that are part of the checkpoint won't be 6058 * reused until the checkpoint is discarded or we revert to it. 6059 * 6060 * The checkpoint flag is passed down the metaslab_free code path 6061 * and is set whenever we want to add a block to the checkpoint's 6062 * accounting. That is, we "checkpoint" blocks that existed at the 6063 * time the checkpoint was created and are therefore referenced by 6064 * the checkpointed uberblock. 6065 * 6066 * Note that, we don't checkpoint any blocks if the current 6067 * syncing txg <= spa_checkpoint_txg. We want these frees to sync 6068 * normally as they will be referenced by the checkpointed uberblock. 6069 */ 6070 boolean_t checkpoint = B_FALSE; 6071 if (BP_GET_BIRTH(bp) <= spa->spa_checkpoint_txg && 6072 spa_syncing_txg(spa) > spa->spa_checkpoint_txg) { 6073 /* 6074 * At this point, if the block is part of the checkpoint 6075 * there is no way it was created in the current txg. 6076 */ 6077 ASSERT(!now); 6078 ASSERT3U(spa_syncing_txg(spa), ==, txg); 6079 checkpoint = B_TRUE; 6080 } 6081 6082 spa_config_enter(spa, SCL_FREE, FTAG, RW_READER); 6083 6084 for (int d = 0; d < ndvas; d++) { 6085 if (now) { 6086 metaslab_unalloc_dva(spa, &dva[d], txg); 6087 } else { 6088 ASSERT3U(txg, ==, spa_syncing_txg(spa)); 6089 metaslab_free_dva(spa, &dva[d], checkpoint); 6090 } 6091 } 6092 6093 spa_config_exit(spa, SCL_FREE, FTAG); 6094 } 6095 6096 int 6097 metaslab_claim(spa_t *spa, const blkptr_t *bp, uint64_t txg) 6098 { 6099 const dva_t *dva = bp->blk_dva; 6100 int ndvas = BP_GET_NDVAS(bp); 6101 int error = 0; 6102 6103 ASSERT(!BP_IS_HOLE(bp)); 6104 6105 if (txg != 0) { 6106 /* 6107 * First do a dry run to make sure all DVAs are claimable, 6108 * so we don't have to unwind from partial failures below. 6109 */ 6110 if ((error = metaslab_claim(spa, bp, 0)) != 0) 6111 return (error); 6112 } 6113 6114 spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER); 6115 6116 for (int d = 0; d < ndvas; d++) { 6117 error = metaslab_claim_dva(spa, &dva[d], txg); 6118 if (error != 0) 6119 break; 6120 } 6121 6122 spa_config_exit(spa, SCL_ALLOC, FTAG); 6123 6124 ASSERT(error == 0 || txg == 0); 6125 6126 return (error); 6127 } 6128 6129 static void 6130 metaslab_check_free_impl_cb(uint64_t inner, vdev_t *vd, uint64_t offset, 6131 uint64_t size, void *arg) 6132 { 6133 (void) inner, (void) arg; 6134 6135 if (vd->vdev_ops == &vdev_indirect_ops) 6136 return; 6137 6138 metaslab_check_free_impl(vd, offset, size); 6139 } 6140 6141 static void 6142 metaslab_check_free_impl(vdev_t *vd, uint64_t offset, uint64_t size) 6143 { 6144 metaslab_t *msp; 6145 spa_t *spa __maybe_unused = vd->vdev_spa; 6146 6147 if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0) 6148 return; 6149 6150 if (vd->vdev_ops->vdev_op_remap != NULL) { 6151 vd->vdev_ops->vdev_op_remap(vd, offset, size, 6152 metaslab_check_free_impl_cb, NULL); 6153 return; 6154 } 6155 6156 ASSERT(vdev_is_concrete(vd)); 6157 ASSERT3U(offset >> vd->vdev_ms_shift, <, vd->vdev_ms_count); 6158 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 6159 6160 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; 6161 6162 mutex_enter(&msp->ms_lock); 6163 if (msp->ms_loaded) { 6164 zfs_range_tree_verify_not_present(msp->ms_allocatable, 6165 offset, size); 6166 } 6167 6168 /* 6169 * Check all segments that currently exist in the freeing pipeline. 6170 * 6171 * It would intuitively make sense to also check the current allocating 6172 * tree since metaslab_unalloc_dva() exists for extents that are 6173 * allocated and freed in the same sync pass within the same txg. 6174 * Unfortunately there are places (e.g. the ZIL) where we allocate a 6175 * segment but then we free part of it within the same txg 6176 * [see zil_sync()]. Thus, we don't call zfs_range_tree_verify() in the 6177 * current allocating tree. 6178 */ 6179 zfs_range_tree_verify_not_present(msp->ms_freeing, offset, size); 6180 zfs_range_tree_verify_not_present(msp->ms_checkpointing, offset, size); 6181 zfs_range_tree_verify_not_present(msp->ms_freed, offset, size); 6182 for (int j = 0; j < TXG_DEFER_SIZE; j++) 6183 zfs_range_tree_verify_not_present(msp->ms_defer[j], offset, 6184 size); 6185 zfs_range_tree_verify_not_present(msp->ms_trim, offset, size); 6186 mutex_exit(&msp->ms_lock); 6187 } 6188 6189 void 6190 metaslab_check_free(spa_t *spa, const blkptr_t *bp) 6191 { 6192 if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0) 6193 return; 6194 6195 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER); 6196 for (int i = 0; i < BP_GET_NDVAS(bp); i++) { 6197 uint64_t vdev = DVA_GET_VDEV(&bp->blk_dva[i]); 6198 vdev_t *vd = vdev_lookup_top(spa, vdev); 6199 uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]); 6200 uint64_t size = DVA_GET_ASIZE(&bp->blk_dva[i]); 6201 6202 if (DVA_GET_GANG(&bp->blk_dva[i])) 6203 size = vdev_gang_header_asize(vd); 6204 6205 ASSERT3P(vd, !=, NULL); 6206 6207 metaslab_check_free_impl(vd, offset, size); 6208 } 6209 spa_config_exit(spa, SCL_VDEV, FTAG); 6210 } 6211 6212 static void 6213 metaslab_group_disable_wait(metaslab_group_t *mg) 6214 { 6215 ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock)); 6216 while (mg->mg_disabled_updating) { 6217 cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock); 6218 } 6219 } 6220 6221 static void 6222 metaslab_group_disabled_increment(metaslab_group_t *mg) 6223 { 6224 ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock)); 6225 ASSERT(mg->mg_disabled_updating); 6226 6227 while (mg->mg_ms_disabled >= max_disabled_ms) { 6228 cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock); 6229 } 6230 mg->mg_ms_disabled++; 6231 ASSERT3U(mg->mg_ms_disabled, <=, max_disabled_ms); 6232 } 6233 6234 /* 6235 * Mark the metaslab as disabled to prevent any allocations on this metaslab. 6236 * We must also track how many metaslabs are currently disabled within a 6237 * metaslab group and limit them to prevent allocation failures from 6238 * occurring because all metaslabs are disabled. 6239 */ 6240 void 6241 metaslab_disable(metaslab_t *msp) 6242 { 6243 ASSERT(!MUTEX_HELD(&msp->ms_lock)); 6244 metaslab_group_t *mg = msp->ms_group; 6245 6246 mutex_enter(&mg->mg_ms_disabled_lock); 6247 6248 /* 6249 * To keep an accurate count of how many threads have disabled 6250 * a specific metaslab group, we only allow one thread to mark 6251 * the metaslab group at a time. This ensures that the value of 6252 * ms_disabled will be accurate when we decide to mark a metaslab 6253 * group as disabled. To do this we force all other threads 6254 * to wait till the metaslab's mg_disabled_updating flag is no 6255 * longer set. 6256 */ 6257 metaslab_group_disable_wait(mg); 6258 mg->mg_disabled_updating = B_TRUE; 6259 if (msp->ms_disabled == 0) { 6260 metaslab_group_disabled_increment(mg); 6261 } 6262 mutex_enter(&msp->ms_lock); 6263 msp->ms_disabled++; 6264 mutex_exit(&msp->ms_lock); 6265 6266 mg->mg_disabled_updating = B_FALSE; 6267 cv_broadcast(&mg->mg_ms_disabled_cv); 6268 mutex_exit(&mg->mg_ms_disabled_lock); 6269 } 6270 6271 void 6272 metaslab_enable(metaslab_t *msp, boolean_t sync, boolean_t unload) 6273 { 6274 metaslab_group_t *mg = msp->ms_group; 6275 spa_t *spa = mg->mg_vd->vdev_spa; 6276 6277 /* 6278 * Wait for the outstanding IO to be synced to prevent newly 6279 * allocated blocks from being overwritten. This used by 6280 * initialize and TRIM which are modifying unallocated space. 6281 */ 6282 if (sync) 6283 txg_wait_synced(spa_get_dsl(spa), 0); 6284 6285 mutex_enter(&mg->mg_ms_disabled_lock); 6286 mutex_enter(&msp->ms_lock); 6287 if (--msp->ms_disabled == 0) { 6288 mg->mg_ms_disabled--; 6289 cv_broadcast(&mg->mg_ms_disabled_cv); 6290 if (unload) 6291 metaslab_unload(msp); 6292 } 6293 mutex_exit(&msp->ms_lock); 6294 mutex_exit(&mg->mg_ms_disabled_lock); 6295 } 6296 6297 void 6298 metaslab_set_unflushed_dirty(metaslab_t *ms, boolean_t dirty) 6299 { 6300 ms->ms_unflushed_dirty = dirty; 6301 } 6302 6303 static void 6304 metaslab_update_ondisk_flush_data(metaslab_t *ms, dmu_tx_t *tx) 6305 { 6306 vdev_t *vd = ms->ms_group->mg_vd; 6307 spa_t *spa = vd->vdev_spa; 6308 objset_t *mos = spa_meta_objset(spa); 6309 6310 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 6311 6312 metaslab_unflushed_phys_t entry = { 6313 .msp_unflushed_txg = metaslab_unflushed_txg(ms), 6314 }; 6315 uint64_t entry_size = sizeof (entry); 6316 uint64_t entry_offset = ms->ms_id * entry_size; 6317 6318 uint64_t object = 0; 6319 int err = zap_lookup(mos, vd->vdev_top_zap, 6320 VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1, 6321 &object); 6322 if (err == ENOENT) { 6323 object = dmu_object_alloc(mos, DMU_OTN_UINT64_METADATA, 6324 SPA_OLD_MAXBLOCKSIZE, DMU_OT_NONE, 0, tx); 6325 VERIFY0(zap_add(mos, vd->vdev_top_zap, 6326 VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1, 6327 &object, tx)); 6328 } else { 6329 VERIFY0(err); 6330 } 6331 6332 dmu_write(spa_meta_objset(spa), object, entry_offset, entry_size, 6333 &entry, tx, DMU_READ_NO_PREFETCH); 6334 } 6335 6336 void 6337 metaslab_set_unflushed_txg(metaslab_t *ms, uint64_t txg, dmu_tx_t *tx) 6338 { 6339 ms->ms_unflushed_txg = txg; 6340 metaslab_update_ondisk_flush_data(ms, tx); 6341 } 6342 6343 boolean_t 6344 metaslab_unflushed_dirty(metaslab_t *ms) 6345 { 6346 return (ms->ms_unflushed_dirty); 6347 } 6348 6349 uint64_t 6350 metaslab_unflushed_txg(metaslab_t *ms) 6351 { 6352 return (ms->ms_unflushed_txg); 6353 } 6354 6355 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, aliquot, U64, ZMOD_RW, 6356 "Allocation granularity (a.k.a. stripe size)"); 6357 6358 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_load, INT, ZMOD_RW, 6359 "Load all metaslabs when pool is first opened"); 6360 6361 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_unload, INT, ZMOD_RW, 6362 "Prevent metaslabs from being unloaded"); 6363 6364 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_enabled, INT, ZMOD_RW, 6365 "Preload potential metaslabs during reassessment"); 6366 6367 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_limit, UINT, ZMOD_RW, 6368 "Max number of metaslabs per group to preload"); 6369 6370 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay, UINT, ZMOD_RW, 6371 "Delay in txgs after metaslab was last used before unloading"); 6372 6373 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay_ms, UINT, ZMOD_RW, 6374 "Delay in milliseconds after metaslab was last used before unloading"); 6375 6376 ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, noalloc_threshold, UINT, ZMOD_RW, 6377 "Percentage of metaslab group size that should be free to make it " 6378 "eligible for allocation"); 6379 6380 ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, fragmentation_threshold, UINT, ZMOD_RW, 6381 "Percentage of metaslab group size that should be considered eligible " 6382 "for allocations unless all metaslab groups within the metaslab class " 6383 "have also crossed this threshold"); 6384 6385 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, fragmentation_factor_enabled, INT, 6386 ZMOD_RW, 6387 "Use the fragmentation metric to prefer less fragmented metaslabs"); 6388 6389 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, fragmentation_threshold, UINT, 6390 ZMOD_RW, "Fragmentation for metaslab to allow allocation"); 6391 6392 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, lba_weighting_enabled, INT, ZMOD_RW, 6393 "Prefer metaslabs with lower LBAs"); 6394 6395 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, bias_enabled, INT, ZMOD_RW, 6396 "Enable space-based metaslab group biasing"); 6397 6398 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, perf_bias, INT, ZMOD_RW, 6399 "Enable performance-based metaslab group biasing"); 6400 6401 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, segment_weight_enabled, INT, 6402 ZMOD_RW, "Enable segment-based metaslab selection"); 6403 6404 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, switch_threshold, INT, ZMOD_RW, 6405 "Segment-based metaslab selection maximum buckets before switching"); 6406 6407 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging, U64, ZMOD_RW, 6408 "Blocks larger than this size are sometimes forced to be gang blocks"); 6409 6410 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging_pct, UINT, ZMOD_RW, 6411 "Percentage of large blocks that will be forced to be gang blocks"); 6412 6413 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_max_search, UINT, ZMOD_RW, 6414 "Max distance (bytes) to search forward before using size tree"); 6415 6416 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_use_largest_segment, INT, ZMOD_RW, 6417 "When looking in size tree, use largest segment instead of exact fit"); 6418 6419 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, max_size_cache_sec, U64, 6420 ZMOD_RW, "How long to trust the cached max chunk size of a metaslab"); 6421 6422 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, mem_limit, UINT, ZMOD_RW, 6423 "Percentage of memory that can be used to store metaslab range trees"); 6424 6425 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, try_hard_before_gang, INT, 6426 ZMOD_RW, "Try hard to allocate before ganging"); 6427 6428 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, find_max_tries, UINT, ZMOD_RW, 6429 "Normally only consider this many of the best metaslabs in each vdev"); 6430 6431 ZFS_MODULE_PARAM_CALL(zfs, zfs_, active_allocator, 6432 param_set_active_allocator, param_get_charp, ZMOD_RW, 6433 "SPA active allocator"); 6434