1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2019 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved. 27 * Copyright (c) 2017, Intel Corporation. 28 */ 29 30 #include <sys/zfs_context.h> 31 #include <sys/brt.h> 32 #include <sys/dmu.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/space_map.h> 35 #include <sys/metaslab_impl.h> 36 #include <sys/vdev_impl.h> 37 #include <sys/vdev_draid.h> 38 #include <sys/zio.h> 39 #include <sys/spa_impl.h> 40 #include <sys/zfeature.h> 41 #include <sys/vdev_indirect_mapping.h> 42 #include <sys/zap.h> 43 #include <sys/btree.h> 44 45 #define GANG_ALLOCATION(flags) \ 46 ((flags) & (METASLAB_GANG_CHILD | METASLAB_GANG_HEADER)) 47 48 /* 49 * Metaslab group's per child vdev granularity, in bytes. This is roughly 50 * similar to what would be referred to as the "stripe size" in traditional 51 * RAID arrays. In normal operation, we will try to write this amount of 52 * data to each disk before moving on to the next top-level vdev. 53 */ 54 static uint64_t metaslab_aliquot = 2 * 1024 * 1024; 55 56 /* 57 * For testing, make some blocks above a certain size be gang blocks. 58 */ 59 uint64_t metaslab_force_ganging = SPA_MAXBLOCKSIZE + 1; 60 61 /* 62 * Of blocks of size >= metaslab_force_ganging, actually gang them this often. 63 */ 64 uint_t metaslab_force_ganging_pct = 3; 65 66 /* 67 * In pools where the log space map feature is not enabled we touch 68 * multiple metaslabs (and their respective space maps) with each 69 * transaction group. Thus, we benefit from having a small space map 70 * block size since it allows us to issue more I/O operations scattered 71 * around the disk. So a sane default for the space map block size 72 * is 8~16K. 73 */ 74 int zfs_metaslab_sm_blksz_no_log = (1 << 14); 75 76 /* 77 * When the log space map feature is enabled, we accumulate a lot of 78 * changes per metaslab that are flushed once in a while so we benefit 79 * from a bigger block size like 128K for the metaslab space maps. 80 */ 81 int zfs_metaslab_sm_blksz_with_log = (1 << 17); 82 83 /* 84 * The in-core space map representation is more compact than its on-disk form. 85 * The zfs_condense_pct determines how much more compact the in-core 86 * space map representation must be before we compact it on-disk. 87 * Values should be greater than or equal to 100. 88 */ 89 uint_t zfs_condense_pct = 200; 90 91 /* 92 * Condensing a metaslab is not guaranteed to actually reduce the amount of 93 * space used on disk. In particular, a space map uses data in increments of 94 * MAX(1 << ashift, space_map_blksz), so a metaslab might use the 95 * same number of blocks after condensing. Since the goal of condensing is to 96 * reduce the number of IOPs required to read the space map, we only want to 97 * condense when we can be sure we will reduce the number of blocks used by the 98 * space map. Unfortunately, we cannot precisely compute whether or not this is 99 * the case in metaslab_should_condense since we are holding ms_lock. Instead, 100 * we apply the following heuristic: do not condense a spacemap unless the 101 * uncondensed size consumes greater than zfs_metaslab_condense_block_threshold 102 * blocks. 103 */ 104 static const int zfs_metaslab_condense_block_threshold = 4; 105 106 /* 107 * The zfs_mg_noalloc_threshold defines which metaslab groups should 108 * be eligible for allocation. The value is defined as a percentage of 109 * free space. Metaslab groups that have more free space than 110 * zfs_mg_noalloc_threshold are always eligible for allocations. Once 111 * a metaslab group's free space is less than or equal to the 112 * zfs_mg_noalloc_threshold the allocator will avoid allocating to that 113 * group unless all groups in the pool have reached zfs_mg_noalloc_threshold. 114 * Once all groups in the pool reach zfs_mg_noalloc_threshold then all 115 * groups are allowed to accept allocations. Gang blocks are always 116 * eligible to allocate on any metaslab group. The default value of 0 means 117 * no metaslab group will be excluded based on this criterion. 118 */ 119 static uint_t zfs_mg_noalloc_threshold = 0; 120 121 /* 122 * Metaslab groups are considered eligible for allocations if their 123 * fragmentation metric (measured as a percentage) is less than or 124 * equal to zfs_mg_fragmentation_threshold. If a metaslab group 125 * exceeds this threshold then it will be skipped unless all metaslab 126 * groups within the metaslab class have also crossed this threshold. 127 * 128 * This tunable was introduced to avoid edge cases where we continue 129 * allocating from very fragmented disks in our pool while other, less 130 * fragmented disks, exists. On the other hand, if all disks in the 131 * pool are uniformly approaching the threshold, the threshold can 132 * be a speed bump in performance, where we keep switching the disks 133 * that we allocate from (e.g. we allocate some segments from disk A 134 * making it bypassing the threshold while freeing segments from disk 135 * B getting its fragmentation below the threshold). 136 * 137 * Empirically, we've seen that our vdev selection for allocations is 138 * good enough that fragmentation increases uniformly across all vdevs 139 * the majority of the time. Thus we set the threshold percentage high 140 * enough to avoid hitting the speed bump on pools that are being pushed 141 * to the edge. 142 */ 143 static uint_t zfs_mg_fragmentation_threshold = 95; 144 145 /* 146 * Allow metaslabs to keep their active state as long as their fragmentation 147 * percentage is less than or equal to zfs_metaslab_fragmentation_threshold. An 148 * active metaslab that exceeds this threshold will no longer keep its active 149 * status allowing better metaslabs to be selected. 150 */ 151 static uint_t zfs_metaslab_fragmentation_threshold = 77; 152 153 /* 154 * When set will load all metaslabs when pool is first opened. 155 */ 156 int metaslab_debug_load = B_FALSE; 157 158 /* 159 * When set will prevent metaslabs from being unloaded. 160 */ 161 static int metaslab_debug_unload = B_FALSE; 162 163 /* 164 * Minimum size which forces the dynamic allocator to change 165 * it's allocation strategy. Once the space map cannot satisfy 166 * an allocation of this size then it switches to using more 167 * aggressive strategy (i.e search by size rather than offset). 168 */ 169 uint64_t metaslab_df_alloc_threshold = SPA_OLD_MAXBLOCKSIZE; 170 171 /* 172 * The minimum free space, in percent, which must be available 173 * in a space map to continue allocations in a first-fit fashion. 174 * Once the space map's free space drops below this level we dynamically 175 * switch to using best-fit allocations. 176 */ 177 uint_t metaslab_df_free_pct = 4; 178 179 /* 180 * Maximum distance to search forward from the last offset. Without this 181 * limit, fragmented pools can see >100,000 iterations and 182 * metaslab_block_picker() becomes the performance limiting factor on 183 * high-performance storage. 184 * 185 * With the default setting of 16MB, we typically see less than 500 186 * iterations, even with very fragmented, ashift=9 pools. The maximum number 187 * of iterations possible is: 188 * metaslab_df_max_search / (2 * (1<<ashift)) 189 * With the default setting of 16MB this is 16*1024 (with ashift=9) or 190 * 2048 (with ashift=12). 191 */ 192 static uint_t metaslab_df_max_search = 16 * 1024 * 1024; 193 194 /* 195 * Forces the metaslab_block_picker function to search for at least this many 196 * segments forwards until giving up on finding a segment that the allocation 197 * will fit into. 198 */ 199 static const uint32_t metaslab_min_search_count = 100; 200 201 /* 202 * If we are not searching forward (due to metaslab_df_max_search, 203 * metaslab_df_free_pct, or metaslab_df_alloc_threshold), this tunable 204 * controls what segment is used. If it is set, we will use the largest free 205 * segment. If it is not set, we will use a segment of exactly the requested 206 * size (or larger). 207 */ 208 static int metaslab_df_use_largest_segment = B_FALSE; 209 210 /* 211 * These tunables control how long a metaslab will remain loaded after the 212 * last allocation from it. A metaslab can't be unloaded until at least 213 * metaslab_unload_delay TXG's and metaslab_unload_delay_ms milliseconds 214 * have elapsed. However, zfs_metaslab_mem_limit may cause it to be 215 * unloaded sooner. These settings are intended to be generous -- to keep 216 * metaslabs loaded for a long time, reducing the rate of metaslab loading. 217 */ 218 static uint_t metaslab_unload_delay = 32; 219 static uint_t metaslab_unload_delay_ms = 10 * 60 * 1000; /* ten minutes */ 220 221 /* 222 * Max number of metaslabs per group to preload. 223 */ 224 uint_t metaslab_preload_limit = 10; 225 226 /* 227 * Enable/disable preloading of metaslab. 228 */ 229 static int metaslab_preload_enabled = B_TRUE; 230 231 /* 232 * Enable/disable fragmentation weighting on metaslabs. 233 */ 234 static int metaslab_fragmentation_factor_enabled = B_TRUE; 235 236 /* 237 * Enable/disable lba weighting (i.e. outer tracks are given preference). 238 */ 239 static int metaslab_lba_weighting_enabled = B_TRUE; 240 241 /* 242 * Enable/disable space-based metaslab group biasing. 243 */ 244 static int metaslab_bias_enabled = B_TRUE; 245 246 /* 247 * Control performance-based metaslab group biasing. 248 */ 249 static int metaslab_perf_bias = 1; 250 251 /* 252 * Enable/disable remapping of indirect DVAs to their concrete vdevs. 253 */ 254 static const boolean_t zfs_remap_blkptr_enable = B_TRUE; 255 256 /* 257 * Enable/disable segment-based metaslab selection. 258 */ 259 static int zfs_metaslab_segment_weight_enabled = B_TRUE; 260 261 /* 262 * When using segment-based metaslab selection, we will continue 263 * allocating from the active metaslab until we have exhausted 264 * zfs_metaslab_switch_threshold of its buckets. 265 */ 266 static int zfs_metaslab_switch_threshold = 2; 267 268 /* 269 * Internal switch to enable/disable the metaslab allocation tracing 270 * facility. 271 */ 272 static const boolean_t metaslab_trace_enabled = B_FALSE; 273 274 /* 275 * Maximum entries that the metaslab allocation tracing facility will keep 276 * in a given list when running in non-debug mode. We limit the number 277 * of entries in non-debug mode to prevent us from using up too much memory. 278 * The limit should be sufficiently large that we don't expect any allocation 279 * to every exceed this value. In debug mode, the system will panic if this 280 * limit is ever reached allowing for further investigation. 281 */ 282 static const uint64_t metaslab_trace_max_entries = 5000; 283 284 /* 285 * Maximum number of metaslabs per group that can be disabled 286 * simultaneously. 287 */ 288 static const int max_disabled_ms = 3; 289 290 /* 291 * Time (in seconds) to respect ms_max_size when the metaslab is not loaded. 292 * To avoid 64-bit overflow, don't set above UINT32_MAX. 293 */ 294 static uint64_t zfs_metaslab_max_size_cache_sec = 1 * 60 * 60; /* 1 hour */ 295 296 /* 297 * Maximum percentage of memory to use on storing loaded metaslabs. If loading 298 * a metaslab would take it over this percentage, the oldest selected metaslab 299 * is automatically unloaded. 300 */ 301 static uint_t zfs_metaslab_mem_limit = 25; 302 303 /* 304 * Force the per-metaslab range trees to use 64-bit integers to store 305 * segments. Used for debugging purposes. 306 */ 307 static const boolean_t zfs_metaslab_force_large_segs = B_FALSE; 308 309 /* 310 * By default we only store segments over a certain size in the size-sorted 311 * metaslab trees (ms_allocatable_by_size and 312 * ms_unflushed_frees_by_size). This dramatically reduces memory usage and 313 * improves load and unload times at the cost of causing us to use slightly 314 * larger segments than we would otherwise in some cases. 315 */ 316 static const uint32_t metaslab_by_size_min_shift = 14; 317 318 /* 319 * If not set, we will first try normal allocation. If that fails then 320 * we will do a gang allocation. If that fails then we will do a "try hard" 321 * gang allocation. If that fails then we will have a multi-layer gang 322 * block. 323 * 324 * If set, we will first try normal allocation. If that fails then 325 * we will do a "try hard" allocation. If that fails we will do a gang 326 * allocation. If that fails we will do a "try hard" gang allocation. If 327 * that fails then we will have a multi-layer gang block. 328 */ 329 static int zfs_metaslab_try_hard_before_gang = B_FALSE; 330 331 /* 332 * When not trying hard, we only consider the best zfs_metaslab_find_max_tries 333 * metaslabs. This improves performance, especially when there are many 334 * metaslabs per vdev and the allocation can't actually be satisfied (so we 335 * would otherwise iterate all the metaslabs). If there is a metaslab with a 336 * worse weight but it can actually satisfy the allocation, we won't find it 337 * until trying hard. This may happen if the worse metaslab is not loaded 338 * (and the true weight is better than we have calculated), or due to weight 339 * bucketization. E.g. we are looking for a 60K segment, and the best 340 * metaslabs all have free segments in the 32-63K bucket, but the best 341 * zfs_metaslab_find_max_tries metaslabs have ms_max_size <60KB, and a 342 * subsequent metaslab has ms_max_size >60KB (but fewer segments in this 343 * bucket, and therefore a lower weight). 344 */ 345 static uint_t zfs_metaslab_find_max_tries = 100; 346 347 static uint64_t metaslab_weight(metaslab_t *, boolean_t); 348 static void metaslab_set_fragmentation(metaslab_t *, boolean_t); 349 static void metaslab_free_impl(vdev_t *, uint64_t, uint64_t, boolean_t); 350 static void metaslab_check_free_impl(vdev_t *, uint64_t, uint64_t); 351 352 static void metaslab_passivate(metaslab_t *msp, uint64_t weight); 353 static uint64_t metaslab_weight_from_range_tree(metaslab_t *msp); 354 static void metaslab_flush_update(metaslab_t *, dmu_tx_t *); 355 static unsigned int metaslab_idx_func(multilist_t *, void *); 356 static void metaslab_evict(metaslab_t *, uint64_t); 357 static void metaslab_rt_add(zfs_range_tree_t *rt, zfs_range_seg_t *rs, 358 void *arg); 359 kmem_cache_t *metaslab_alloc_trace_cache; 360 361 typedef struct metaslab_stats { 362 kstat_named_t metaslabstat_trace_over_limit; 363 kstat_named_t metaslabstat_reload_tree; 364 kstat_named_t metaslabstat_too_many_tries; 365 kstat_named_t metaslabstat_try_hard; 366 } metaslab_stats_t; 367 368 static metaslab_stats_t metaslab_stats = { 369 { "trace_over_limit", KSTAT_DATA_UINT64 }, 370 { "reload_tree", KSTAT_DATA_UINT64 }, 371 { "too_many_tries", KSTAT_DATA_UINT64 }, 372 { "try_hard", KSTAT_DATA_UINT64 }, 373 }; 374 375 #define METASLABSTAT_BUMP(stat) \ 376 atomic_inc_64(&metaslab_stats.stat.value.ui64); 377 378 char * 379 metaslab_rt_name(metaslab_group_t *mg, metaslab_t *ms, const char *name) 380 { 381 return (kmem_asprintf("{spa=%s vdev_guid=%llu ms_id=%llu %s}", 382 spa_name(mg->mg_vd->vdev_spa), 383 (u_longlong_t)mg->mg_vd->vdev_guid, 384 (u_longlong_t)ms->ms_id, 385 name)); 386 } 387 388 389 static kstat_t *metaslab_ksp; 390 391 void 392 metaslab_stat_init(void) 393 { 394 ASSERT0P(metaslab_alloc_trace_cache); 395 metaslab_alloc_trace_cache = kmem_cache_create( 396 "metaslab_alloc_trace_cache", sizeof (metaslab_alloc_trace_t), 397 0, NULL, NULL, NULL, NULL, NULL, 0); 398 metaslab_ksp = kstat_create("zfs", 0, "metaslab_stats", 399 "misc", KSTAT_TYPE_NAMED, sizeof (metaslab_stats) / 400 sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 401 if (metaslab_ksp != NULL) { 402 metaslab_ksp->ks_data = &metaslab_stats; 403 kstat_install(metaslab_ksp); 404 } 405 } 406 407 void 408 metaslab_stat_fini(void) 409 { 410 if (metaslab_ksp != NULL) { 411 kstat_delete(metaslab_ksp); 412 metaslab_ksp = NULL; 413 } 414 415 kmem_cache_destroy(metaslab_alloc_trace_cache); 416 metaslab_alloc_trace_cache = NULL; 417 } 418 419 /* 420 * ========================================================================== 421 * Metaslab classes 422 * ========================================================================== 423 */ 424 metaslab_class_t * 425 metaslab_class_create(spa_t *spa, const char *name, 426 const metaslab_ops_t *ops, boolean_t is_log) 427 { 428 metaslab_class_t *mc; 429 430 mc = kmem_zalloc(offsetof(metaslab_class_t, 431 mc_allocator[spa->spa_alloc_count]), KM_SLEEP); 432 433 mc->mc_spa = spa; 434 mc->mc_name = name; 435 mc->mc_ops = ops; 436 mc->mc_is_log = is_log; 437 mc->mc_alloc_io_size = SPA_OLD_MAXBLOCKSIZE; 438 mc->mc_alloc_max = UINT64_MAX; 439 mutex_init(&mc->mc_lock, NULL, MUTEX_DEFAULT, NULL); 440 multilist_create(&mc->mc_metaslab_txg_list, sizeof (metaslab_t), 441 offsetof(metaslab_t, ms_class_txg_node), metaslab_idx_func); 442 for (int i = 0; i < spa->spa_alloc_count; i++) { 443 metaslab_class_allocator_t *mca = &mc->mc_allocator[i]; 444 mutex_init(&mca->mca_lock, NULL, MUTEX_DEFAULT, NULL); 445 avl_create(&mca->mca_tree, zio_bookmark_compare, 446 sizeof (zio_t), offsetof(zio_t, io_queue_node.a)); 447 mca->mca_rotor = NULL; 448 mca->mca_reserved = 0; 449 } 450 451 return (mc); 452 } 453 454 void 455 metaslab_class_destroy(metaslab_class_t *mc) 456 { 457 spa_t *spa = mc->mc_spa; 458 459 ASSERT0(mc->mc_alloc); 460 ASSERT0(mc->mc_deferred); 461 ASSERT0(mc->mc_space); 462 ASSERT0(mc->mc_dspace); 463 464 for (int i = 0; i < spa->spa_alloc_count; i++) { 465 metaslab_class_allocator_t *mca = &mc->mc_allocator[i]; 466 avl_destroy(&mca->mca_tree); 467 mutex_destroy(&mca->mca_lock); 468 ASSERT0P(mca->mca_rotor); 469 ASSERT0(mca->mca_reserved); 470 } 471 mutex_destroy(&mc->mc_lock); 472 multilist_destroy(&mc->mc_metaslab_txg_list); 473 kmem_free(mc, offsetof(metaslab_class_t, 474 mc_allocator[spa->spa_alloc_count])); 475 } 476 477 void 478 metaslab_class_validate(metaslab_class_t *mc) 479 { 480 #ifdef ZFS_DEBUG 481 spa_t *spa = mc->mc_spa; 482 483 /* 484 * Must hold one of the spa_config locks. 485 */ 486 ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) || 487 spa_config_held(spa, SCL_ALL, RW_WRITER)); 488 489 for (int i = 0; i < spa->spa_alloc_count; i++) { 490 metaslab_class_allocator_t *mca = &mc->mc_allocator[i]; 491 metaslab_group_t *mg, *rotor; 492 493 ASSERT0(avl_numnodes(&mca->mca_tree)); 494 ASSERT0(mca->mca_reserved); 495 496 if ((mg = rotor = mca->mca_rotor) == NULL) 497 continue; 498 do { 499 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 500 vdev_t *vd = mg->mg_vd; 501 502 ASSERT3P(vd->vdev_top, ==, vd); 503 ASSERT(vd->vdev_mg == mg || vd->vdev_log_mg == mg); 504 ASSERT3P(mg->mg_class, ==, mc); 505 ASSERT3P(vd->vdev_ops, !=, &vdev_hole_ops); 506 ASSERT0(zfs_refcount_count(&mga->mga_queue_depth)); 507 } while ((mg = mg->mg_next) != rotor); 508 } 509 #endif 510 } 511 512 /* 513 * For each metaslab group in a class pre-calculate allocation quota and 514 * target queue depth to balance their space usage and write performance. 515 * Based on those pre-calculate class allocation throttle threshold for 516 * optimal saturation. onsync is true once per TXG to enable/disable 517 * allocation throttling and update moving average of maximum I/O size. 518 */ 519 void 520 metaslab_class_balance(metaslab_class_t *mc, boolean_t onsync) 521 { 522 metaslab_group_t *mg, *first; 523 524 /* 525 * Must hold one of the spa_config locks. 526 */ 527 ASSERT(spa_config_held(mc->mc_spa, SCL_ALL, RW_READER) || 528 spa_config_held(mc->mc_spa, SCL_ALL, RW_WRITER)); 529 530 if (onsync) 531 metaslab_class_validate(mc); 532 533 if (mc->mc_groups == 0) { 534 if (onsync) 535 mc->mc_alloc_throttle_enabled = B_FALSE; 536 mc->mc_alloc_max = UINT64_MAX; 537 return; 538 } 539 540 if (onsync) { 541 /* 542 * Moving average of maximum allocation size, in absence of 543 * large allocations shrinking to 1/8 of metaslab_aliquot. 544 */ 545 mc->mc_alloc_io_size = (3 * mc->mc_alloc_io_size + 546 metaslab_aliquot / 8) / 4; 547 mc->mc_alloc_throttle_enabled = mc->mc_is_log ? 0 : 548 zio_dva_throttle_enabled; 549 } 550 551 mg = first = mc->mc_allocator[0].mca_rotor; 552 uint64_t children = 0; 553 do { 554 children += vdev_get_ndisks(mg->mg_vd) - 555 vdev_get_nparity(mg->mg_vd); 556 } while ((mg = mg->mg_next) != first); 557 558 uint64_t sum_aliquot = 0; 559 do { 560 vdev_stat_t *vs = &mg->mg_vd->vdev_stat; 561 uint_t ratio; 562 563 /* 564 * Scale allocations per iteration with average number of 565 * children. Wider vdevs need more sequential allocations 566 * to keep decent per-child I/O size. 567 */ 568 uint64_t mg_aliquot = MAX(metaslab_aliquot * children / 569 mc->mc_groups, mc->mc_alloc_io_size * 4); 570 571 /* 572 * Scale allocations per iteration with the vdev capacity, 573 * relative to average. Bigger vdevs should get more to 574 * fill up at the same time as smaller ones. 575 */ 576 if (mc->mc_space > 0 && vs->vs_space > 0) { 577 ratio = vs->vs_space / (mc->mc_space / (mc->mc_groups * 578 256) + 1); 579 mg_aliquot = mg_aliquot * ratio / 256; 580 } 581 582 /* 583 * Scale allocations per iteration with the vdev's free space 584 * fraction, relative to average. Despite the above, vdevs free 585 * space fractions may get imbalanced, for example due to new 586 * vdev addition or different performance. We want free space 587 * fractions to be similar to postpone fragmentation. 588 * 589 * But same time we don't want to throttle vdevs still having 590 * plenty of free space, that appear faster than others, even 591 * if that cause temporary imbalance. Allow them to allocate 592 * more by keeping their allocation queue depth equivalent to 593 * 2.5 full iteration, even if they repeatedly drain it. Later 594 * with the free space reduction gradually reduce the target 595 * queue depth, stronger enforcing the free space balance. 596 */ 597 if (metaslab_bias_enabled && 598 mc->mc_space > 0 && vs->vs_space > 0) { 599 uint64_t vs_free = vs->vs_space > vs->vs_alloc ? 600 vs->vs_space - vs->vs_alloc : 0; 601 uint64_t mc_free = mc->mc_space > mc->mc_alloc ? 602 mc->mc_space - mc->mc_alloc : 0; 603 /* 604 * vs_fr is 16 bit fixed-point free space fraction. 605 * mc_fr is 8 bit fixed-point free space fraction. 606 * ratio as their quotient is 8 bit fixed-point. 607 */ 608 uint_t vs_fr = vs_free / (vs->vs_space / 65536 + 1); 609 uint_t mc_fr = mc_free / (mc->mc_space / 256 + 1); 610 ratio = vs_fr / (mc_fr + 1); 611 mg->mg_aliquot = mg_aliquot * ratio / 256; 612 /* From 2.5x at 25% full to 1x at 75%. */ 613 ratio = MIN(163840, vs_fr * 3 + 16384); 614 mg->mg_queue_target = MAX(mg->mg_aliquot, 615 mg->mg_aliquot * ratio / 65536); 616 } else { 617 mg->mg_aliquot = mg_aliquot; 618 mg->mg_queue_target = mg->mg_aliquot * 2; 619 } 620 sum_aliquot += mg->mg_aliquot; 621 } while ((mg = mg->mg_next) != first); 622 623 /* 624 * Set per-class allocation throttle threshold to 4 iterations through 625 * all the vdevs. This should keep all vdevs busy even if some are 626 * allocating more than we planned for them due to bigger blocks or 627 * better performance. 628 */ 629 mc->mc_alloc_max = sum_aliquot * 4; 630 } 631 632 static void 633 metaslab_class_rotate(metaslab_group_t *mg, int allocator, uint64_t psize, 634 boolean_t success) 635 { 636 metaslab_class_t *mc = mg->mg_class; 637 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 638 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 639 640 /* 641 * Exit fast if there is nothing to rotate, we are not following 642 * the rotor (copies, gangs, etc) or somebody already rotated it. 643 */ 644 if (mc->mc_groups < 2 || mca->mca_rotor != mg) 645 return; 646 647 /* 648 * Always rotate in case of allocation error or a log class. 649 */ 650 if (!success || mc->mc_is_log) 651 goto rotate; 652 653 /* 654 * Allocate from this group if we expect next I/O of the same size to 655 * mostly fit within the allocation quota. Rotate if we expect it to 656 * mostly go over the target queue depth. Meanwhile, to stripe between 657 * groups in configured amounts per child even if we can't reach the 658 * target queue depth, i.e. can't saturate the group write performance, 659 * always rotate after allocating the queue target bytes. 660 */ 661 uint64_t naq = atomic_add_64_nv(&mca->mca_aliquot, psize) + psize / 2; 662 if (naq < mg->mg_aliquot) 663 return; 664 if (naq >= mg->mg_queue_target) 665 goto rotate; 666 if (zfs_refcount_count(&mga->mga_queue_depth) + psize + psize / 2 >= 667 mg->mg_queue_target) 668 goto rotate; 669 670 /* 671 * When the pool is not too busy, prefer restoring the vdev free space 672 * balance instead of getting maximum speed we might not need, so that 673 * we could have more flexibility during more busy times later. 674 */ 675 if (metaslab_perf_bias <= 0) 676 goto rotate; 677 if (metaslab_perf_bias >= 2) 678 return; 679 spa_t *spa = mc->mc_spa; 680 dsl_pool_t *dp = spa_get_dsl(spa); 681 if (dp == NULL) 682 return; 683 uint64_t busy_thresh = zfs_dirty_data_max * 684 (zfs_vdev_async_write_active_min_dirty_percent + 685 zfs_vdev_async_write_active_max_dirty_percent) / 200; 686 if (dp->dp_dirty_total > busy_thresh || spa_has_pending_synctask(spa)) 687 return; 688 689 rotate: 690 mca->mca_rotor = mg->mg_next; 691 mca->mca_aliquot = 0; 692 } 693 694 static void 695 metaslab_class_space_update(metaslab_class_t *mc, int64_t alloc_delta, 696 int64_t defer_delta, int64_t space_delta, int64_t dspace_delta) 697 { 698 atomic_add_64(&mc->mc_alloc, alloc_delta); 699 atomic_add_64(&mc->mc_deferred, defer_delta); 700 atomic_add_64(&mc->mc_space, space_delta); 701 atomic_add_64(&mc->mc_dspace, dspace_delta); 702 } 703 704 const char * 705 metaslab_class_get_name(metaslab_class_t *mc) 706 { 707 return (mc->mc_name); 708 } 709 710 uint64_t 711 metaslab_class_get_alloc(metaslab_class_t *mc) 712 { 713 return (mc->mc_alloc); 714 } 715 716 uint64_t 717 metaslab_class_get_deferred(metaslab_class_t *mc) 718 { 719 return (mc->mc_deferred); 720 } 721 722 uint64_t 723 metaslab_class_get_space(metaslab_class_t *mc) 724 { 725 return (mc->mc_space); 726 } 727 728 uint64_t 729 metaslab_class_get_dspace(metaslab_class_t *mc) 730 { 731 return (spa_deflate(mc->mc_spa) ? mc->mc_dspace : mc->mc_space); 732 } 733 734 void 735 metaslab_class_histogram_verify(metaslab_class_t *mc) 736 { 737 spa_t *spa = mc->mc_spa; 738 vdev_t *rvd = spa->spa_root_vdev; 739 uint64_t *mc_hist; 740 int i; 741 742 if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0) 743 return; 744 745 mc_hist = kmem_zalloc(sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE, 746 KM_SLEEP); 747 748 mutex_enter(&mc->mc_lock); 749 for (int c = 0; c < rvd->vdev_children; c++) { 750 vdev_t *tvd = rvd->vdev_child[c]; 751 metaslab_group_t *mg = vdev_get_mg(tvd, mc); 752 753 /* 754 * Skip any holes, uninitialized top-levels, or 755 * vdevs that are not in this metalab class. 756 */ 757 if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 || 758 mg->mg_class != mc) { 759 continue; 760 } 761 762 IMPLY(mg == mg->mg_vd->vdev_log_mg, 763 mc == spa_embedded_log_class(mg->mg_vd->vdev_spa) || 764 mc == spa_special_embedded_log_class(mg->mg_vd->vdev_spa)); 765 766 for (i = 0; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i++) 767 mc_hist[i] += mg->mg_histogram[i]; 768 } 769 770 for (i = 0; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i++) { 771 VERIFY3U(mc_hist[i], ==, mc->mc_histogram[i]); 772 } 773 774 mutex_exit(&mc->mc_lock); 775 kmem_free(mc_hist, sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE); 776 } 777 778 /* 779 * Calculate the metaslab class's fragmentation metric. The metric 780 * is weighted based on the space contribution of each metaslab group. 781 * The return value will be a number between 0 and 100 (inclusive), or 782 * ZFS_FRAG_INVALID if the metric has not been set. See comment above the 783 * zfs_frag_table for more information about the metric. 784 */ 785 uint64_t 786 metaslab_class_fragmentation(metaslab_class_t *mc) 787 { 788 vdev_t *rvd = mc->mc_spa->spa_root_vdev; 789 uint64_t fragmentation = 0; 790 791 spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER); 792 793 for (int c = 0; c < rvd->vdev_children; c++) { 794 vdev_t *tvd = rvd->vdev_child[c]; 795 metaslab_group_t *mg = tvd->vdev_mg; 796 797 /* 798 * Skip any holes, uninitialized top-levels, 799 * or vdevs that are not in this metalab class. 800 */ 801 if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 || 802 mg->mg_class != mc) { 803 continue; 804 } 805 806 /* 807 * If a metaslab group does not contain a fragmentation 808 * metric then just bail out. 809 */ 810 if (mg->mg_fragmentation == ZFS_FRAG_INVALID) { 811 spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG); 812 return (ZFS_FRAG_INVALID); 813 } 814 815 /* 816 * Determine how much this metaslab_group is contributing 817 * to the overall pool fragmentation metric. 818 */ 819 fragmentation += mg->mg_fragmentation * 820 metaslab_group_get_space(mg); 821 } 822 fragmentation /= metaslab_class_get_space(mc); 823 824 ASSERT3U(fragmentation, <=, 100); 825 spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG); 826 return (fragmentation); 827 } 828 829 /* 830 * Calculate the amount of expandable space that is available in 831 * this metaslab class. If a device is expanded then its expandable 832 * space will be the amount of allocatable space that is currently not 833 * part of this metaslab class. 834 */ 835 uint64_t 836 metaslab_class_expandable_space(metaslab_class_t *mc) 837 { 838 vdev_t *rvd = mc->mc_spa->spa_root_vdev; 839 uint64_t space = 0; 840 841 spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER); 842 for (int c = 0; c < rvd->vdev_children; c++) { 843 vdev_t *tvd = rvd->vdev_child[c]; 844 metaslab_group_t *mg = tvd->vdev_mg; 845 846 if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 || 847 mg->mg_class != mc) { 848 continue; 849 } 850 851 /* 852 * Calculate if we have enough space to add additional 853 * metaslabs. We report the expandable space in terms 854 * of the metaslab size since that's the unit of expansion. 855 */ 856 space += P2ALIGN_TYPED(tvd->vdev_max_asize - tvd->vdev_asize, 857 1ULL << tvd->vdev_ms_shift, uint64_t); 858 } 859 spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG); 860 return (space); 861 } 862 863 void 864 metaslab_class_evict_old(metaslab_class_t *mc, uint64_t txg) 865 { 866 multilist_t *ml = &mc->mc_metaslab_txg_list; 867 uint64_t now = gethrestime_sec(); 868 /* Round delay up to next second. */ 869 uint_t delay = (metaslab_unload_delay_ms + 999) / 1000; 870 for (int i = 0; i < multilist_get_num_sublists(ml); i++) { 871 multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i); 872 metaslab_t *msp = multilist_sublist_head(mls); 873 multilist_sublist_unlock(mls); 874 while (msp != NULL) { 875 mutex_enter(&msp->ms_lock); 876 877 /* 878 * If the metaslab has been removed from the list 879 * (which could happen if we were at the memory limit 880 * and it was evicted during this loop), then we can't 881 * proceed and we should restart the sublist. 882 */ 883 if (!multilist_link_active(&msp->ms_class_txg_node)) { 884 mutex_exit(&msp->ms_lock); 885 i--; 886 break; 887 } 888 mls = multilist_sublist_lock_idx(ml, i); 889 metaslab_t *next_msp = multilist_sublist_next(mls, msp); 890 multilist_sublist_unlock(mls); 891 if (txg > 892 msp->ms_selected_txg + metaslab_unload_delay && 893 now > msp->ms_selected_time + delay && 894 (msp->ms_allocator == -1 || 895 !metaslab_preload_enabled)) { 896 metaslab_evict(msp, txg); 897 } else { 898 /* 899 * Once we've hit a metaslab selected too 900 * recently to evict, we're done evicting for 901 * now. 902 */ 903 mutex_exit(&msp->ms_lock); 904 break; 905 } 906 mutex_exit(&msp->ms_lock); 907 msp = next_msp; 908 } 909 } 910 } 911 912 static int 913 metaslab_compare(const void *x1, const void *x2) 914 { 915 const metaslab_t *m1 = (const metaslab_t *)x1; 916 const metaslab_t *m2 = (const metaslab_t *)x2; 917 918 int sort1 = 0; 919 int sort2 = 0; 920 if (m1->ms_allocator != -1 && m1->ms_primary) 921 sort1 = 1; 922 else if (m1->ms_allocator != -1 && !m1->ms_primary) 923 sort1 = 2; 924 if (m2->ms_allocator != -1 && m2->ms_primary) 925 sort2 = 1; 926 else if (m2->ms_allocator != -1 && !m2->ms_primary) 927 sort2 = 2; 928 929 /* 930 * Sort inactive metaslabs first, then primaries, then secondaries. When 931 * selecting a metaslab to allocate from, an allocator first tries its 932 * primary, then secondary active metaslab. If it doesn't have active 933 * metaslabs, or can't allocate from them, it searches for an inactive 934 * metaslab to activate. If it can't find a suitable one, it will steal 935 * a primary or secondary metaslab from another allocator. 936 */ 937 if (sort1 < sort2) 938 return (-1); 939 if (sort1 > sort2) 940 return (1); 941 942 int cmp = TREE_CMP(m2->ms_weight, m1->ms_weight); 943 if (likely(cmp)) 944 return (cmp); 945 946 IMPLY(TREE_CMP(m1->ms_start, m2->ms_start) == 0, m1 == m2); 947 948 return (TREE_CMP(m1->ms_start, m2->ms_start)); 949 } 950 951 /* 952 * ========================================================================== 953 * Metaslab groups 954 * ========================================================================== 955 */ 956 /* 957 * Update the allocatable flag and the metaslab group's capacity. 958 * The allocatable flag is set to true if the capacity is below 959 * the zfs_mg_noalloc_threshold or has a fragmentation value that is 960 * greater than zfs_mg_fragmentation_threshold. If a metaslab group 961 * transitions from allocatable to non-allocatable or vice versa then the 962 * metaslab group's class is updated to reflect the transition. 963 */ 964 static void 965 metaslab_group_alloc_update(metaslab_group_t *mg) 966 { 967 vdev_t *vd = mg->mg_vd; 968 metaslab_class_t *mc = mg->mg_class; 969 vdev_stat_t *vs = &vd->vdev_stat; 970 boolean_t was_allocatable; 971 boolean_t was_initialized; 972 973 ASSERT(vd == vd->vdev_top); 974 ASSERT3U(spa_config_held(mc->mc_spa, SCL_ALLOC, RW_READER), ==, 975 SCL_ALLOC); 976 977 mutex_enter(&mg->mg_lock); 978 was_allocatable = mg->mg_allocatable; 979 was_initialized = mg->mg_initialized; 980 981 uint64_t free_capacity = ((vs->vs_space - vs->vs_alloc) * 100) / 982 (vs->vs_space + 1); 983 984 mutex_enter(&mc->mc_lock); 985 986 /* 987 * If the metaslab group was just added then it won't 988 * have any space until we finish syncing out this txg. 989 * At that point we will consider it initialized and available 990 * for allocations. We also don't consider non-activated 991 * metaslab groups (e.g. vdevs that are in the middle of being removed) 992 * to be initialized, because they can't be used for allocation. 993 */ 994 mg->mg_initialized = metaslab_group_initialized(mg); 995 if (!was_initialized && mg->mg_initialized) { 996 mc->mc_groups++; 997 } else if (was_initialized && !mg->mg_initialized) { 998 ASSERT3U(mc->mc_groups, >, 0); 999 mc->mc_groups--; 1000 } 1001 if (mg->mg_initialized) 1002 mg->mg_no_free_space = B_FALSE; 1003 1004 /* 1005 * A metaslab group is considered allocatable if it has plenty 1006 * of free space or is not heavily fragmented. We only take 1007 * fragmentation into account if the metaslab group has a valid 1008 * fragmentation metric (i.e. a value between 0 and 100). 1009 */ 1010 mg->mg_allocatable = (mg->mg_activation_count > 0 && 1011 free_capacity > zfs_mg_noalloc_threshold && 1012 (mg->mg_fragmentation == ZFS_FRAG_INVALID || 1013 mg->mg_fragmentation <= zfs_mg_fragmentation_threshold)); 1014 1015 /* 1016 * The mc_alloc_groups maintains a count of the number of 1017 * groups in this metaslab class that are still above the 1018 * zfs_mg_noalloc_threshold. This is used by the allocating 1019 * threads to determine if they should avoid allocations to 1020 * a given group. The allocator will avoid allocations to a group 1021 * if that group has reached or is below the zfs_mg_noalloc_threshold 1022 * and there are still other groups that are above the threshold. 1023 * When a group transitions from allocatable to non-allocatable or 1024 * vice versa we update the metaslab class to reflect that change. 1025 * When the mc_alloc_groups value drops to 0 that means that all 1026 * groups have reached the zfs_mg_noalloc_threshold making all groups 1027 * eligible for allocations. This effectively means that all devices 1028 * are balanced again. 1029 */ 1030 if (was_allocatable && !mg->mg_allocatable) 1031 mc->mc_alloc_groups--; 1032 else if (!was_allocatable && mg->mg_allocatable) 1033 mc->mc_alloc_groups++; 1034 mutex_exit(&mc->mc_lock); 1035 1036 mutex_exit(&mg->mg_lock); 1037 } 1038 1039 int 1040 metaslab_sort_by_flushed(const void *va, const void *vb) 1041 { 1042 const metaslab_t *a = va; 1043 const metaslab_t *b = vb; 1044 1045 int cmp = TREE_CMP(a->ms_unflushed_txg, b->ms_unflushed_txg); 1046 if (likely(cmp)) 1047 return (cmp); 1048 1049 uint64_t a_vdev_id = a->ms_group->mg_vd->vdev_id; 1050 uint64_t b_vdev_id = b->ms_group->mg_vd->vdev_id; 1051 cmp = TREE_CMP(a_vdev_id, b_vdev_id); 1052 if (cmp) 1053 return (cmp); 1054 1055 return (TREE_CMP(a->ms_id, b->ms_id)); 1056 } 1057 1058 metaslab_group_t * 1059 metaslab_group_create(metaslab_class_t *mc, vdev_t *vd) 1060 { 1061 spa_t *spa = mc->mc_spa; 1062 metaslab_group_t *mg; 1063 1064 mg = kmem_zalloc(offsetof(metaslab_group_t, 1065 mg_allocator[spa->spa_alloc_count]), KM_SLEEP); 1066 mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL); 1067 mutex_init(&mg->mg_ms_disabled_lock, NULL, MUTEX_DEFAULT, NULL); 1068 cv_init(&mg->mg_ms_disabled_cv, NULL, CV_DEFAULT, NULL); 1069 avl_create(&mg->mg_metaslab_tree, metaslab_compare, 1070 sizeof (metaslab_t), offsetof(metaslab_t, ms_group_node)); 1071 mg->mg_vd = vd; 1072 mg->mg_class = mc; 1073 mg->mg_activation_count = 0; 1074 mg->mg_initialized = B_FALSE; 1075 mg->mg_no_free_space = B_TRUE; 1076 1077 for (int i = 0; i < spa->spa_alloc_count; i++) { 1078 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 1079 zfs_refcount_create_tracked(&mga->mga_queue_depth); 1080 } 1081 1082 return (mg); 1083 } 1084 1085 void 1086 metaslab_group_destroy(metaslab_group_t *mg) 1087 { 1088 spa_t *spa = mg->mg_class->mc_spa; 1089 1090 ASSERT0P(mg->mg_prev); 1091 ASSERT0P(mg->mg_next); 1092 /* 1093 * We may have gone below zero with the activation count 1094 * either because we never activated in the first place or 1095 * because we're done, and possibly removing the vdev. 1096 */ 1097 ASSERT(mg->mg_activation_count <= 0); 1098 1099 avl_destroy(&mg->mg_metaslab_tree); 1100 mutex_destroy(&mg->mg_lock); 1101 mutex_destroy(&mg->mg_ms_disabled_lock); 1102 cv_destroy(&mg->mg_ms_disabled_cv); 1103 1104 for (int i = 0; i < spa->spa_alloc_count; i++) { 1105 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 1106 zfs_refcount_destroy(&mga->mga_queue_depth); 1107 } 1108 kmem_free(mg, offsetof(metaslab_group_t, 1109 mg_allocator[spa->spa_alloc_count])); 1110 } 1111 1112 void 1113 metaslab_group_activate(metaslab_group_t *mg) 1114 { 1115 metaslab_class_t *mc = mg->mg_class; 1116 spa_t *spa = mc->mc_spa; 1117 metaslab_group_t *mgprev, *mgnext; 1118 1119 ASSERT3U(spa_config_held(spa, SCL_ALLOC, RW_WRITER), !=, 0); 1120 1121 ASSERT0P(mg->mg_prev); 1122 ASSERT0P(mg->mg_next); 1123 ASSERT(mg->mg_activation_count <= 0); 1124 1125 if (++mg->mg_activation_count <= 0) 1126 return; 1127 1128 metaslab_group_alloc_update(mg); 1129 1130 if ((mgprev = mc->mc_allocator[0].mca_rotor) == NULL) { 1131 mg->mg_prev = mg; 1132 mg->mg_next = mg; 1133 } else { 1134 mgnext = mgprev->mg_next; 1135 mg->mg_prev = mgprev; 1136 mg->mg_next = mgnext; 1137 mgprev->mg_next = mg; 1138 mgnext->mg_prev = mg; 1139 } 1140 for (int i = 0; i < spa->spa_alloc_count; i++) { 1141 mc->mc_allocator[i].mca_rotor = mg; 1142 mg = mg->mg_next; 1143 } 1144 metaslab_class_balance(mc, B_FALSE); 1145 } 1146 1147 /* 1148 * Passivate a metaslab group and remove it from the allocation rotor. 1149 * Callers must hold both the SCL_ALLOC and SCL_ZIO lock prior to passivating 1150 * a metaslab group. This function will momentarily drop spa_config_locks 1151 * that are lower than the SCL_ALLOC lock (see comment below). 1152 */ 1153 void 1154 metaslab_group_passivate(metaslab_group_t *mg) 1155 { 1156 metaslab_class_t *mc = mg->mg_class; 1157 spa_t *spa = mc->mc_spa; 1158 metaslab_group_t *mgprev, *mgnext; 1159 int locks = spa_config_held(spa, SCL_ALL, RW_WRITER); 1160 1161 ASSERT3U(spa_config_held(spa, SCL_ALLOC | SCL_ZIO, RW_WRITER), ==, 1162 (SCL_ALLOC | SCL_ZIO)); 1163 1164 if (--mg->mg_activation_count != 0) { 1165 for (int i = 0; i < spa->spa_alloc_count; i++) 1166 ASSERT(mc->mc_allocator[i].mca_rotor != mg); 1167 ASSERT0P(mg->mg_prev); 1168 ASSERT0P(mg->mg_next); 1169 ASSERT(mg->mg_activation_count < 0); 1170 return; 1171 } 1172 1173 /* 1174 * The spa_config_lock is an array of rwlocks, ordered as 1175 * follows (from highest to lowest): 1176 * SCL_CONFIG > SCL_STATE > SCL_L2ARC > SCL_ALLOC > 1177 * SCL_ZIO > SCL_FREE > SCL_VDEV 1178 * (For more information about the spa_config_lock see spa_misc.c) 1179 * The higher the lock, the broader its coverage. When we passivate 1180 * a metaslab group, we must hold both the SCL_ALLOC and the SCL_ZIO 1181 * config locks. However, the metaslab group's taskq might be trying 1182 * to preload metaslabs so we must drop the SCL_ZIO lock and any 1183 * lower locks to allow the I/O to complete. At a minimum, 1184 * we continue to hold the SCL_ALLOC lock, which prevents any future 1185 * allocations from taking place and any changes to the vdev tree. 1186 */ 1187 spa_config_exit(spa, locks & ~(SCL_ZIO - 1), spa); 1188 taskq_wait_outstanding(spa->spa_metaslab_taskq, 0); 1189 spa_config_enter(spa, locks & ~(SCL_ZIO - 1), spa, RW_WRITER); 1190 metaslab_group_alloc_update(mg); 1191 for (int i = 0; i < spa->spa_alloc_count; i++) { 1192 metaslab_group_allocator_t *mga = &mg->mg_allocator[i]; 1193 metaslab_t *msp = mga->mga_primary; 1194 if (msp != NULL) { 1195 mutex_enter(&msp->ms_lock); 1196 metaslab_passivate(msp, 1197 metaslab_weight(msp, B_TRUE) & 1198 ~METASLAB_ACTIVE_MASK); 1199 mutex_exit(&msp->ms_lock); 1200 } 1201 msp = mga->mga_secondary; 1202 if (msp != NULL) { 1203 mutex_enter(&msp->ms_lock); 1204 metaslab_passivate(msp, 1205 metaslab_weight(msp, B_TRUE) & 1206 ~METASLAB_ACTIVE_MASK); 1207 mutex_exit(&msp->ms_lock); 1208 } 1209 } 1210 1211 mgprev = mg->mg_prev; 1212 mgnext = mg->mg_next; 1213 1214 if (mg == mgnext) { 1215 mgnext = NULL; 1216 } else { 1217 mgprev->mg_next = mgnext; 1218 mgnext->mg_prev = mgprev; 1219 } 1220 for (int i = 0; i < spa->spa_alloc_count; i++) { 1221 if (mc->mc_allocator[i].mca_rotor == mg) 1222 mc->mc_allocator[i].mca_rotor = mgnext; 1223 } 1224 1225 mg->mg_prev = NULL; 1226 mg->mg_next = NULL; 1227 metaslab_class_balance(mc, B_FALSE); 1228 } 1229 1230 boolean_t 1231 metaslab_group_initialized(metaslab_group_t *mg) 1232 { 1233 vdev_t *vd = mg->mg_vd; 1234 vdev_stat_t *vs = &vd->vdev_stat; 1235 1236 return (vs->vs_space != 0 && mg->mg_activation_count > 0); 1237 } 1238 1239 uint64_t 1240 metaslab_group_get_space(metaslab_group_t *mg) 1241 { 1242 /* 1243 * Note that the number of nodes in mg_metaslab_tree may be one less 1244 * than vdev_ms_count, due to the embedded log metaslab. 1245 */ 1246 mutex_enter(&mg->mg_lock); 1247 uint64_t ms_count = avl_numnodes(&mg->mg_metaslab_tree); 1248 mutex_exit(&mg->mg_lock); 1249 return ((1ULL << mg->mg_vd->vdev_ms_shift) * ms_count); 1250 } 1251 1252 void 1253 metaslab_group_histogram_verify(metaslab_group_t *mg) 1254 { 1255 uint64_t *mg_hist; 1256 avl_tree_t *t = &mg->mg_metaslab_tree; 1257 uint64_t ashift = mg->mg_vd->vdev_ashift; 1258 1259 if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0) 1260 return; 1261 1262 mg_hist = kmem_zalloc(sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE, 1263 KM_SLEEP); 1264 1265 ASSERT3U(ZFS_RANGE_TREE_HISTOGRAM_SIZE, >=, 1266 SPACE_MAP_HISTOGRAM_SIZE + ashift); 1267 1268 mutex_enter(&mg->mg_lock); 1269 for (metaslab_t *msp = avl_first(t); 1270 msp != NULL; msp = AVL_NEXT(t, msp)) { 1271 VERIFY3P(msp->ms_group, ==, mg); 1272 /* skip if not active */ 1273 if (msp->ms_sm == NULL) 1274 continue; 1275 1276 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 1277 mg_hist[i + ashift] += 1278 msp->ms_sm->sm_phys->smp_histogram[i]; 1279 } 1280 } 1281 1282 for (int i = 0; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i ++) 1283 VERIFY3U(mg_hist[i], ==, mg->mg_histogram[i]); 1284 1285 mutex_exit(&mg->mg_lock); 1286 1287 kmem_free(mg_hist, sizeof (uint64_t) * ZFS_RANGE_TREE_HISTOGRAM_SIZE); 1288 } 1289 1290 static void 1291 metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp) 1292 { 1293 metaslab_class_t *mc = mg->mg_class; 1294 uint64_t ashift = mg->mg_vd->vdev_ashift; 1295 1296 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1297 if (msp->ms_sm == NULL) 1298 return; 1299 1300 mutex_enter(&mg->mg_lock); 1301 mutex_enter(&mc->mc_lock); 1302 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 1303 IMPLY(mg == mg->mg_vd->vdev_log_mg, 1304 mc == spa_embedded_log_class(mg->mg_vd->vdev_spa) || 1305 mc == spa_special_embedded_log_class(mg->mg_vd->vdev_spa)); 1306 mg->mg_histogram[i + ashift] += 1307 msp->ms_sm->sm_phys->smp_histogram[i]; 1308 mc->mc_histogram[i + ashift] += 1309 msp->ms_sm->sm_phys->smp_histogram[i]; 1310 } 1311 mutex_exit(&mc->mc_lock); 1312 mutex_exit(&mg->mg_lock); 1313 } 1314 1315 void 1316 metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp) 1317 { 1318 metaslab_class_t *mc = mg->mg_class; 1319 uint64_t ashift = mg->mg_vd->vdev_ashift; 1320 1321 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1322 if (msp->ms_sm == NULL) 1323 return; 1324 1325 mutex_enter(&mg->mg_lock); 1326 mutex_enter(&mc->mc_lock); 1327 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 1328 ASSERT3U(mg->mg_histogram[i + ashift], >=, 1329 msp->ms_sm->sm_phys->smp_histogram[i]); 1330 ASSERT3U(mc->mc_histogram[i + ashift], >=, 1331 msp->ms_sm->sm_phys->smp_histogram[i]); 1332 IMPLY(mg == mg->mg_vd->vdev_log_mg, 1333 mc == spa_embedded_log_class(mg->mg_vd->vdev_spa) || 1334 mc == spa_special_embedded_log_class(mg->mg_vd->vdev_spa)); 1335 1336 mg->mg_histogram[i + ashift] -= 1337 msp->ms_sm->sm_phys->smp_histogram[i]; 1338 mc->mc_histogram[i + ashift] -= 1339 msp->ms_sm->sm_phys->smp_histogram[i]; 1340 } 1341 mutex_exit(&mc->mc_lock); 1342 mutex_exit(&mg->mg_lock); 1343 } 1344 1345 static void 1346 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp) 1347 { 1348 ASSERT0P(msp->ms_group); 1349 mutex_enter(&mg->mg_lock); 1350 msp->ms_group = mg; 1351 msp->ms_weight = 0; 1352 avl_add(&mg->mg_metaslab_tree, msp); 1353 mutex_exit(&mg->mg_lock); 1354 1355 mutex_enter(&msp->ms_lock); 1356 metaslab_group_histogram_add(mg, msp); 1357 mutex_exit(&msp->ms_lock); 1358 } 1359 1360 static void 1361 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp) 1362 { 1363 mutex_enter(&msp->ms_lock); 1364 metaslab_group_histogram_remove(mg, msp); 1365 mutex_exit(&msp->ms_lock); 1366 1367 mutex_enter(&mg->mg_lock); 1368 ASSERT(msp->ms_group == mg); 1369 avl_remove(&mg->mg_metaslab_tree, msp); 1370 1371 metaslab_class_t *mc = msp->ms_group->mg_class; 1372 multilist_sublist_t *mls = 1373 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 1374 if (multilist_link_active(&msp->ms_class_txg_node)) 1375 multilist_sublist_remove(mls, msp); 1376 multilist_sublist_unlock(mls); 1377 1378 msp->ms_group = NULL; 1379 mutex_exit(&mg->mg_lock); 1380 } 1381 1382 static void 1383 metaslab_group_sort_impl(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) 1384 { 1385 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1386 ASSERT(MUTEX_HELD(&mg->mg_lock)); 1387 ASSERT(msp->ms_group == mg); 1388 1389 avl_remove(&mg->mg_metaslab_tree, msp); 1390 msp->ms_weight = weight; 1391 avl_add(&mg->mg_metaslab_tree, msp); 1392 1393 } 1394 1395 static void 1396 metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight) 1397 { 1398 /* 1399 * Although in principle the weight can be any value, in 1400 * practice we do not use values in the range [1, 511]. 1401 */ 1402 ASSERT(weight >= SPA_MINBLOCKSIZE || weight == 0); 1403 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1404 1405 mutex_enter(&mg->mg_lock); 1406 metaslab_group_sort_impl(mg, msp, weight); 1407 mutex_exit(&mg->mg_lock); 1408 } 1409 1410 /* 1411 * Calculate the fragmentation for a given metaslab group. Weight metaslabs 1412 * on the amount of free space. The return value will be between 0 and 100 1413 * (inclusive), or ZFS_FRAG_INVALID if less than half of the metaslab in this 1414 * group have a fragmentation metric. 1415 */ 1416 uint64_t 1417 metaslab_group_fragmentation(metaslab_group_t *mg) 1418 { 1419 vdev_t *vd = mg->mg_vd; 1420 uint64_t fragmentation = 0; 1421 uint64_t valid_ms = 0, total_ms = 0; 1422 uint64_t free, total_free = 0; 1423 1424 for (int m = 0; m < vd->vdev_ms_count; m++) { 1425 metaslab_t *msp = vd->vdev_ms[m]; 1426 1427 if (msp->ms_group != mg) 1428 continue; 1429 total_ms++; 1430 if (msp->ms_fragmentation == ZFS_FRAG_INVALID) 1431 continue; 1432 1433 valid_ms++; 1434 free = (msp->ms_size - metaslab_allocated_space(msp)) / 1435 SPA_MINBLOCKSIZE; /* To prevent overflows. */ 1436 total_free += free; 1437 fragmentation += msp->ms_fragmentation * free; 1438 } 1439 1440 if (valid_ms < (total_ms + 1) / 2 || total_free == 0) 1441 return (ZFS_FRAG_INVALID); 1442 1443 fragmentation /= total_free; 1444 ASSERT3U(fragmentation, <=, 100); 1445 return (fragmentation); 1446 } 1447 1448 /* 1449 * ========================================================================== 1450 * Range tree callbacks 1451 * ========================================================================== 1452 */ 1453 1454 /* 1455 * Comparison function for the private size-ordered tree using 32-bit 1456 * ranges. Tree is sorted by size, larger sizes at the end of the tree. 1457 */ 1458 __attribute__((always_inline)) inline 1459 static int 1460 metaslab_rangesize32_compare(const void *x1, const void *x2) 1461 { 1462 const zfs_range_seg32_t *r1 = x1; 1463 const zfs_range_seg32_t *r2 = x2; 1464 1465 uint64_t rs_size1 = r1->rs_end - r1->rs_start; 1466 uint64_t rs_size2 = r2->rs_end - r2->rs_start; 1467 1468 int cmp = TREE_CMP(rs_size1, rs_size2); 1469 1470 return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start)); 1471 } 1472 1473 /* 1474 * Comparison function for the private size-ordered tree using 64-bit 1475 * ranges. Tree is sorted by size, larger sizes at the end of the tree. 1476 */ 1477 __attribute__((always_inline)) inline 1478 static int 1479 metaslab_rangesize64_compare(const void *x1, const void *x2) 1480 { 1481 const zfs_range_seg64_t *r1 = x1; 1482 const zfs_range_seg64_t *r2 = x2; 1483 1484 uint64_t rs_size1 = r1->rs_end - r1->rs_start; 1485 uint64_t rs_size2 = r2->rs_end - r2->rs_start; 1486 1487 int cmp = TREE_CMP(rs_size1, rs_size2); 1488 1489 return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start)); 1490 } 1491 1492 typedef struct metaslab_rt_arg { 1493 zfs_btree_t *mra_bt; 1494 uint32_t mra_floor_shift; 1495 } metaslab_rt_arg_t; 1496 1497 struct mssa_arg { 1498 zfs_range_tree_t *rt; 1499 metaslab_rt_arg_t *mra; 1500 }; 1501 1502 static void 1503 metaslab_size_sorted_add(void *arg, uint64_t start, uint64_t size) 1504 { 1505 struct mssa_arg *mssap = arg; 1506 zfs_range_tree_t *rt = mssap->rt; 1507 metaslab_rt_arg_t *mrap = mssap->mra; 1508 zfs_range_seg_max_t seg = {0}; 1509 zfs_rs_set_start(&seg, rt, start); 1510 zfs_rs_set_end(&seg, rt, start + size); 1511 metaslab_rt_add(rt, &seg, mrap); 1512 } 1513 1514 static void 1515 metaslab_size_tree_full_load(zfs_range_tree_t *rt) 1516 { 1517 metaslab_rt_arg_t *mrap = rt->rt_arg; 1518 METASLABSTAT_BUMP(metaslabstat_reload_tree); 1519 ASSERT0(zfs_btree_numnodes(mrap->mra_bt)); 1520 mrap->mra_floor_shift = 0; 1521 struct mssa_arg arg = {0}; 1522 arg.rt = rt; 1523 arg.mra = mrap; 1524 zfs_range_tree_walk(rt, metaslab_size_sorted_add, &arg); 1525 } 1526 1527 1528 ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize32_in_buf, 1529 zfs_range_seg32_t, metaslab_rangesize32_compare) 1530 1531 ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize64_in_buf, 1532 zfs_range_seg64_t, metaslab_rangesize64_compare) 1533 1534 /* 1535 * Create any block allocator specific components. The current allocators 1536 * rely on using both a size-ordered zfs_range_tree_t and an array of 1537 * uint64_t's. 1538 */ 1539 static void 1540 metaslab_rt_create(zfs_range_tree_t *rt, void *arg) 1541 { 1542 metaslab_rt_arg_t *mrap = arg; 1543 zfs_btree_t *size_tree = mrap->mra_bt; 1544 1545 size_t size; 1546 int (*compare) (const void *, const void *); 1547 bt_find_in_buf_f bt_find; 1548 switch (rt->rt_type) { 1549 case ZFS_RANGE_SEG32: 1550 size = sizeof (zfs_range_seg32_t); 1551 compare = metaslab_rangesize32_compare; 1552 bt_find = metaslab_rt_find_rangesize32_in_buf; 1553 break; 1554 case ZFS_RANGE_SEG64: 1555 size = sizeof (zfs_range_seg64_t); 1556 compare = metaslab_rangesize64_compare; 1557 bt_find = metaslab_rt_find_rangesize64_in_buf; 1558 break; 1559 default: 1560 panic("Invalid range seg type %d", rt->rt_type); 1561 } 1562 zfs_btree_create(size_tree, compare, bt_find, size); 1563 mrap->mra_floor_shift = metaslab_by_size_min_shift; 1564 } 1565 1566 static void 1567 metaslab_rt_destroy(zfs_range_tree_t *rt, void *arg) 1568 { 1569 (void) rt; 1570 metaslab_rt_arg_t *mrap = arg; 1571 zfs_btree_t *size_tree = mrap->mra_bt; 1572 1573 zfs_btree_destroy(size_tree); 1574 kmem_free(mrap, sizeof (*mrap)); 1575 } 1576 1577 static void 1578 metaslab_rt_add(zfs_range_tree_t *rt, zfs_range_seg_t *rs, void *arg) 1579 { 1580 metaslab_rt_arg_t *mrap = arg; 1581 zfs_btree_t *size_tree = mrap->mra_bt; 1582 1583 if (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt) < 1584 (1ULL << mrap->mra_floor_shift)) 1585 return; 1586 1587 zfs_btree_add(size_tree, rs); 1588 } 1589 1590 static void 1591 metaslab_rt_remove(zfs_range_tree_t *rt, zfs_range_seg_t *rs, void *arg) 1592 { 1593 metaslab_rt_arg_t *mrap = arg; 1594 zfs_btree_t *size_tree = mrap->mra_bt; 1595 1596 if (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt) < (1ULL << 1597 mrap->mra_floor_shift)) 1598 return; 1599 1600 zfs_btree_remove(size_tree, rs); 1601 } 1602 1603 static void 1604 metaslab_rt_vacate(zfs_range_tree_t *rt, void *arg) 1605 { 1606 metaslab_rt_arg_t *mrap = arg; 1607 zfs_btree_t *size_tree = mrap->mra_bt; 1608 zfs_btree_clear(size_tree); 1609 zfs_btree_destroy(size_tree); 1610 1611 metaslab_rt_create(rt, arg); 1612 } 1613 1614 static const zfs_range_tree_ops_t metaslab_rt_ops = { 1615 .rtop_create = metaslab_rt_create, 1616 .rtop_destroy = metaslab_rt_destroy, 1617 .rtop_add = metaslab_rt_add, 1618 .rtop_remove = metaslab_rt_remove, 1619 .rtop_vacate = metaslab_rt_vacate 1620 }; 1621 1622 /* 1623 * ========================================================================== 1624 * Common allocator routines 1625 * ========================================================================== 1626 */ 1627 1628 /* 1629 * Return the maximum contiguous segment within the metaslab. 1630 */ 1631 uint64_t 1632 metaslab_largest_allocatable(metaslab_t *msp) 1633 { 1634 zfs_btree_t *t = &msp->ms_allocatable_by_size; 1635 zfs_range_seg_t *rs; 1636 1637 if (t == NULL) 1638 return (0); 1639 if (zfs_btree_numnodes(t) == 0) 1640 metaslab_size_tree_full_load(msp->ms_allocatable); 1641 1642 rs = zfs_btree_last(t, NULL); 1643 if (rs == NULL) 1644 return (0); 1645 1646 return (zfs_rs_get_end(rs, msp->ms_allocatable) - zfs_rs_get_start(rs, 1647 msp->ms_allocatable)); 1648 } 1649 1650 /* 1651 * Return the maximum contiguous segment within the unflushed frees of this 1652 * metaslab. 1653 */ 1654 static uint64_t 1655 metaslab_largest_unflushed_free(metaslab_t *msp) 1656 { 1657 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1658 1659 if (msp->ms_unflushed_frees == NULL) 1660 return (0); 1661 1662 if (zfs_btree_numnodes(&msp->ms_unflushed_frees_by_size) == 0) 1663 metaslab_size_tree_full_load(msp->ms_unflushed_frees); 1664 zfs_range_seg_t *rs = zfs_btree_last(&msp->ms_unflushed_frees_by_size, 1665 NULL); 1666 if (rs == NULL) 1667 return (0); 1668 1669 /* 1670 * When a range is freed from the metaslab, that range is added to 1671 * both the unflushed frees and the deferred frees. While the block 1672 * will eventually be usable, if the metaslab were loaded the range 1673 * would not be added to the ms_allocatable tree until TXG_DEFER_SIZE 1674 * txgs had passed. As a result, when attempting to estimate an upper 1675 * bound for the largest currently-usable free segment in the 1676 * metaslab, we need to not consider any ranges currently in the defer 1677 * trees. This algorithm approximates the largest available chunk in 1678 * the largest range in the unflushed_frees tree by taking the first 1679 * chunk. While this may be a poor estimate, it should only remain so 1680 * briefly and should eventually self-correct as frees are no longer 1681 * deferred. Similar logic applies to the ms_freed tree. See 1682 * metaslab_load() for more details. 1683 * 1684 * There are two primary sources of inaccuracy in this estimate. Both 1685 * are tolerated for performance reasons. The first source is that we 1686 * only check the largest segment for overlaps. Smaller segments may 1687 * have more favorable overlaps with the other trees, resulting in 1688 * larger usable chunks. Second, we only look at the first chunk in 1689 * the largest segment; there may be other usable chunks in the 1690 * largest segment, but we ignore them. 1691 */ 1692 uint64_t rstart = zfs_rs_get_start(rs, msp->ms_unflushed_frees); 1693 uint64_t rsize = zfs_rs_get_end(rs, msp->ms_unflushed_frees) - rstart; 1694 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 1695 uint64_t start = 0; 1696 uint64_t size = 0; 1697 boolean_t found = zfs_range_tree_find_in(msp->ms_defer[t], 1698 rstart, rsize, &start, &size); 1699 if (found) { 1700 if (rstart == start) 1701 return (0); 1702 rsize = start - rstart; 1703 } 1704 } 1705 1706 uint64_t start = 0; 1707 uint64_t size = 0; 1708 boolean_t found = zfs_range_tree_find_in(msp->ms_freed, rstart, 1709 rsize, &start, &size); 1710 if (found) 1711 rsize = start - rstart; 1712 1713 return (rsize); 1714 } 1715 1716 static zfs_range_seg_t * 1717 metaslab_block_find(zfs_btree_t *t, zfs_range_tree_t *rt, uint64_t start, 1718 uint64_t size, uint64_t max_size, zfs_btree_index_t *where) 1719 { 1720 zfs_range_seg_t *rs; 1721 zfs_range_seg_max_t rsearch; 1722 1723 zfs_rs_set_start(&rsearch, rt, start); 1724 zfs_rs_set_end(&rsearch, rt, start + max_size); 1725 1726 rs = zfs_btree_find(t, &rsearch, where); 1727 if (rs == NULL) { 1728 if (size == max_size) { 1729 rs = zfs_btree_next(t, where, where); 1730 } else { 1731 /* 1732 * If we're searching for a range, get the largest 1733 * segment in that range, or the smallest one bigger 1734 * than it. 1735 */ 1736 rs = zfs_btree_prev(t, where, where); 1737 if (rs == NULL || zfs_rs_get_end(rs, rt) - 1738 zfs_rs_get_start(rs, rt) < size) { 1739 rs = zfs_btree_next(t, where, where); 1740 } 1741 } 1742 } 1743 1744 return (rs); 1745 } 1746 1747 /* 1748 * This is a helper function that can be used by the allocator to find a 1749 * suitable block to allocate. This will search the specified B-tree looking 1750 * for a block that matches the specified criteria. 1751 */ 1752 static uint64_t 1753 metaslab_block_picker(zfs_range_tree_t *rt, uint64_t *cursor, uint64_t size, 1754 uint64_t max_size, uint64_t max_search, uint64_t *found_size) 1755 { 1756 if (*cursor == 0) 1757 *cursor = rt->rt_start; 1758 zfs_btree_t *bt = &rt->rt_root; 1759 zfs_btree_index_t where; 1760 zfs_range_seg_t *rs = metaslab_block_find(bt, rt, *cursor, size, 1761 max_size, &where); 1762 uint64_t first_found; 1763 int count_searched = 0; 1764 1765 if (rs != NULL) 1766 first_found = zfs_rs_get_start(rs, rt); 1767 1768 while (rs != NULL && (zfs_rs_get_start(rs, rt) - first_found <= 1769 max_search || count_searched < metaslab_min_search_count)) { 1770 uint64_t offset = zfs_rs_get_start(rs, rt); 1771 if (offset + size <= zfs_rs_get_end(rs, rt)) { 1772 *found_size = MIN(zfs_rs_get_end(rs, rt) - offset, 1773 max_size); 1774 *cursor = offset + *found_size; 1775 return (offset); 1776 } 1777 rs = zfs_btree_next(bt, &where, &where); 1778 count_searched++; 1779 } 1780 1781 *cursor = 0; 1782 *found_size = 0; 1783 return (-1ULL); 1784 } 1785 1786 static uint64_t metaslab_df_alloc(metaslab_t *msp, uint64_t size, 1787 uint64_t max_size, uint64_t *found_size); 1788 static uint64_t metaslab_cf_alloc(metaslab_t *msp, uint64_t size, 1789 uint64_t max_size, uint64_t *found_size); 1790 static uint64_t metaslab_ndf_alloc(metaslab_t *msp, uint64_t size, 1791 uint64_t max_size, uint64_t *found_size); 1792 metaslab_ops_t *metaslab_allocator(spa_t *spa); 1793 1794 static metaslab_ops_t metaslab_allocators[] = { 1795 { "dynamic", metaslab_df_alloc }, 1796 { "cursor", metaslab_cf_alloc }, 1797 { "new-dynamic", metaslab_ndf_alloc }, 1798 }; 1799 1800 static int 1801 spa_find_allocator_byname(const char *val) 1802 { 1803 int a = ARRAY_SIZE(metaslab_allocators) - 1; 1804 if (strcmp("new-dynamic", val) == 0) 1805 return (-1); /* remove when ndf is working */ 1806 for (; a >= 0; a--) { 1807 if (strcmp(val, metaslab_allocators[a].msop_name) == 0) 1808 return (a); 1809 } 1810 return (-1); 1811 } 1812 1813 void 1814 spa_set_allocator(spa_t *spa, const char *allocator) 1815 { 1816 int a = spa_find_allocator_byname(allocator); 1817 if (a < 0) a = 0; 1818 spa->spa_active_allocator = a; 1819 zfs_dbgmsg("spa allocator: %s", metaslab_allocators[a].msop_name); 1820 } 1821 1822 int 1823 spa_get_allocator(spa_t *spa) 1824 { 1825 return (spa->spa_active_allocator); 1826 } 1827 1828 #if defined(_KERNEL) 1829 int 1830 param_set_active_allocator_common(const char *val) 1831 { 1832 char *p; 1833 1834 if (val == NULL) 1835 return (SET_ERROR(EINVAL)); 1836 1837 if ((p = strchr(val, '\n')) != NULL) 1838 *p = '\0'; 1839 1840 int a = spa_find_allocator_byname(val); 1841 if (a < 0) 1842 return (SET_ERROR(EINVAL)); 1843 1844 zfs_active_allocator = metaslab_allocators[a].msop_name; 1845 return (0); 1846 } 1847 #endif 1848 1849 metaslab_ops_t * 1850 metaslab_allocator(spa_t *spa) 1851 { 1852 int allocator = spa_get_allocator(spa); 1853 return (&metaslab_allocators[allocator]); 1854 } 1855 1856 /* 1857 * ========================================================================== 1858 * Dynamic Fit (df) block allocator 1859 * 1860 * Search for a free chunk of at least this size, starting from the last 1861 * offset (for this alignment of block) looking for up to 1862 * metaslab_df_max_search bytes (16MB). If a large enough free chunk is not 1863 * found within 16MB, then return a free chunk of exactly the requested size (or 1864 * larger). 1865 * 1866 * If it seems like searching from the last offset will be unproductive, skip 1867 * that and just return a free chunk of exactly the requested size (or larger). 1868 * This is based on metaslab_df_alloc_threshold and metaslab_df_free_pct. This 1869 * mechanism is probably not very useful and may be removed in the future. 1870 * 1871 * The behavior when not searching can be changed to return the largest free 1872 * chunk, instead of a free chunk of exactly the requested size, by setting 1873 * metaslab_df_use_largest_segment. 1874 * ========================================================================== 1875 */ 1876 static uint64_t 1877 metaslab_df_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 1878 uint64_t *found_size) 1879 { 1880 /* 1881 * Find the largest power of 2 block size that evenly divides the 1882 * requested size. This is used to try to allocate blocks with similar 1883 * alignment from the same area of the metaslab (i.e. same cursor 1884 * bucket) but it does not guarantee that other allocations sizes 1885 * may exist in the same region. 1886 */ 1887 uint64_t align = max_size & -max_size; 1888 uint64_t *cursor = &msp->ms_lbas[highbit64(align) - 1]; 1889 zfs_range_tree_t *rt = msp->ms_allocatable; 1890 uint_t free_pct = zfs_range_tree_space(rt) * 100 / msp->ms_size; 1891 uint64_t offset; 1892 1893 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1894 1895 /* 1896 * If we're running low on space, find a segment based on size, 1897 * rather than iterating based on offset. 1898 */ 1899 if (metaslab_largest_allocatable(msp) < metaslab_df_alloc_threshold || 1900 free_pct < metaslab_df_free_pct) { 1901 align = size & -size; 1902 cursor = &msp->ms_lbas[highbit64(align) - 1]; 1903 offset = -1; 1904 } else { 1905 offset = metaslab_block_picker(rt, cursor, size, max_size, 1906 metaslab_df_max_search, found_size); 1907 if (max_size != size && offset == -1) { 1908 align = size & -size; 1909 cursor = &msp->ms_lbas[highbit64(align) - 1]; 1910 offset = metaslab_block_picker(rt, cursor, size, 1911 max_size, metaslab_df_max_search, found_size); 1912 } 1913 } 1914 1915 if (offset == -1) { 1916 zfs_range_seg_t *rs; 1917 if (zfs_btree_numnodes(&msp->ms_allocatable_by_size) == 0) 1918 metaslab_size_tree_full_load(msp->ms_allocatable); 1919 1920 if (metaslab_df_use_largest_segment) { 1921 /* use largest free segment */ 1922 rs = zfs_btree_last(&msp->ms_allocatable_by_size, NULL); 1923 } else { 1924 zfs_btree_index_t where; 1925 /* use segment of this size, or next largest */ 1926 rs = metaslab_block_find(&msp->ms_allocatable_by_size, 1927 rt, msp->ms_start, size, max_size, &where); 1928 } 1929 if (rs != NULL && zfs_rs_get_start(rs, rt) + size <= 1930 zfs_rs_get_end(rs, rt)) { 1931 offset = zfs_rs_get_start(rs, rt); 1932 *found_size = MIN(zfs_rs_get_end(rs, rt) - offset, 1933 max_size); 1934 *cursor = offset + *found_size; 1935 } 1936 } 1937 1938 return (offset); 1939 } 1940 1941 /* 1942 * ========================================================================== 1943 * Cursor fit block allocator - 1944 * Select the largest region in the metaslab, set the cursor to the beginning 1945 * of the range and the cursor_end to the end of the range. As allocations 1946 * are made advance the cursor. Continue allocating from the cursor until 1947 * the range is exhausted and then find a new range. 1948 * ========================================================================== 1949 */ 1950 static uint64_t 1951 metaslab_cf_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 1952 uint64_t *found_size) 1953 { 1954 zfs_range_tree_t *rt = msp->ms_allocatable; 1955 zfs_btree_t *t = &msp->ms_allocatable_by_size; 1956 uint64_t *cursor = &msp->ms_lbas[0]; 1957 uint64_t *cursor_end = &msp->ms_lbas[1]; 1958 uint64_t offset = 0; 1959 1960 ASSERT(MUTEX_HELD(&msp->ms_lock)); 1961 1962 ASSERT3U(*cursor_end, >=, *cursor); 1963 1964 if ((*cursor + size) > *cursor_end) { 1965 zfs_range_seg_t *rs; 1966 1967 if (zfs_btree_numnodes(t) == 0) 1968 metaslab_size_tree_full_load(msp->ms_allocatable); 1969 rs = zfs_btree_last(t, NULL); 1970 if (rs == NULL || (zfs_rs_get_end(rs, rt) - 1971 zfs_rs_get_start(rs, rt)) < size) 1972 return (-1ULL); 1973 1974 *cursor = zfs_rs_get_start(rs, rt); 1975 *cursor_end = zfs_rs_get_end(rs, rt); 1976 } 1977 1978 offset = *cursor; 1979 *found_size = MIN(*cursor_end - offset, max_size); 1980 *cursor = offset + *found_size; 1981 1982 return (offset); 1983 } 1984 1985 /* 1986 * ========================================================================== 1987 * New dynamic fit allocator - 1988 * Select a region that is large enough to allocate 2^metaslab_ndf_clump_shift 1989 * contiguous blocks. If no region is found then just use the largest segment 1990 * that remains. 1991 * ========================================================================== 1992 */ 1993 1994 /* 1995 * Determines desired number of contiguous blocks (2^metaslab_ndf_clump_shift) 1996 * to request from the allocator. 1997 */ 1998 uint64_t metaslab_ndf_clump_shift = 4; 1999 2000 static uint64_t 2001 metaslab_ndf_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 2002 uint64_t *found_size) 2003 { 2004 zfs_btree_t *t = &msp->ms_allocatable->rt_root; 2005 zfs_range_tree_t *rt = msp->ms_allocatable; 2006 zfs_btree_index_t where; 2007 zfs_range_seg_t *rs; 2008 zfs_range_seg_max_t rsearch; 2009 uint64_t hbit = highbit64(max_size); 2010 uint64_t *cursor = &msp->ms_lbas[hbit - 1]; 2011 uint64_t max_possible_size = metaslab_largest_allocatable(msp); 2012 2013 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2014 2015 if (max_possible_size < size) 2016 return (-1ULL); 2017 2018 zfs_rs_set_start(&rsearch, rt, *cursor); 2019 zfs_rs_set_end(&rsearch, rt, *cursor + max_size); 2020 2021 rs = zfs_btree_find(t, &rsearch, &where); 2022 if (rs == NULL || (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt)) < 2023 max_size) { 2024 hbit = highbit64(size); 2025 cursor = &msp->ms_lbas[hbit - 1]; 2026 zfs_rs_set_start(&rsearch, rt, *cursor); 2027 zfs_rs_set_end(&rsearch, rt, *cursor + size); 2028 2029 rs = zfs_btree_find(t, &rsearch, &where); 2030 } 2031 if (rs == NULL || (zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt)) < 2032 size) { 2033 t = &msp->ms_allocatable_by_size; 2034 2035 zfs_rs_set_start(&rsearch, rt, 0); 2036 zfs_rs_set_end(&rsearch, rt, MIN(max_possible_size, 2037 1ULL << (hbit + metaslab_ndf_clump_shift))); 2038 2039 rs = zfs_btree_find(t, &rsearch, &where); 2040 if (rs == NULL) 2041 rs = zfs_btree_next(t, &where, &where); 2042 ASSERT(rs != NULL); 2043 } 2044 2045 if ((zfs_rs_get_end(rs, rt) - zfs_rs_get_start(rs, rt)) >= size) { 2046 *found_size = MIN(zfs_rs_get_end(rs, rt) - 2047 zfs_rs_get_start(rs, rt), max_size); 2048 *cursor = zfs_rs_get_start(rs, rt) + *found_size; 2049 return (zfs_rs_get_start(rs, rt)); 2050 } 2051 return (-1ULL); 2052 } 2053 2054 /* 2055 * ========================================================================== 2056 * Metaslabs 2057 * ========================================================================== 2058 */ 2059 2060 /* 2061 * Wait for any in-progress metaslab loads to complete. 2062 */ 2063 static void 2064 metaslab_load_wait(metaslab_t *msp) 2065 { 2066 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2067 2068 while (msp->ms_loading) { 2069 ASSERT(!msp->ms_loaded); 2070 cv_wait(&msp->ms_load_cv, &msp->ms_lock); 2071 } 2072 } 2073 2074 /* 2075 * Wait for any in-progress flushing to complete. 2076 */ 2077 static void 2078 metaslab_flush_wait(metaslab_t *msp) 2079 { 2080 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2081 2082 while (msp->ms_flushing) 2083 cv_wait(&msp->ms_flush_cv, &msp->ms_lock); 2084 } 2085 2086 static unsigned int 2087 metaslab_idx_func(multilist_t *ml, void *arg) 2088 { 2089 metaslab_t *msp = arg; 2090 2091 /* 2092 * ms_id values are allocated sequentially, so full 64bit 2093 * division would be a waste of time, so limit it to 32 bits. 2094 */ 2095 return ((unsigned int)msp->ms_id % multilist_get_num_sublists(ml)); 2096 } 2097 2098 uint64_t 2099 metaslab_allocated_space(metaslab_t *msp) 2100 { 2101 return (msp->ms_allocated_space); 2102 } 2103 2104 /* 2105 * Verify that the space accounting on disk matches the in-core range_trees. 2106 */ 2107 static void 2108 metaslab_verify_space(metaslab_t *msp, uint64_t txg) 2109 { 2110 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2111 uint64_t allocating = 0; 2112 uint64_t sm_free_space, msp_free_space; 2113 2114 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2115 ASSERT(!msp->ms_condensing); 2116 2117 if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0) 2118 return; 2119 2120 /* 2121 * We can only verify the metaslab space when we're called 2122 * from syncing context with a loaded metaslab that has an 2123 * allocated space map. Calling this in non-syncing context 2124 * does not provide a consistent view of the metaslab since 2125 * we're performing allocations in the future. 2126 */ 2127 if (txg != spa_syncing_txg(spa) || msp->ms_sm == NULL || 2128 !msp->ms_loaded) 2129 return; 2130 2131 /* 2132 * Even though the smp_alloc field can get negative, 2133 * when it comes to a metaslab's space map, that should 2134 * never be the case. 2135 */ 2136 ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0); 2137 2138 ASSERT3U(space_map_allocated(msp->ms_sm), >=, 2139 zfs_range_tree_space(msp->ms_unflushed_frees)); 2140 2141 ASSERT3U(metaslab_allocated_space(msp), ==, 2142 space_map_allocated(msp->ms_sm) + 2143 zfs_range_tree_space(msp->ms_unflushed_allocs) - 2144 zfs_range_tree_space(msp->ms_unflushed_frees)); 2145 2146 sm_free_space = msp->ms_size - metaslab_allocated_space(msp); 2147 2148 /* 2149 * Account for future allocations since we would have 2150 * already deducted that space from the ms_allocatable. 2151 */ 2152 for (int t = 0; t < TXG_CONCURRENT_STATES; t++) { 2153 allocating += 2154 zfs_range_tree_space(msp->ms_allocating[(txg + t) & 2155 TXG_MASK]); 2156 } 2157 ASSERT3U(allocating + msp->ms_allocated_this_txg, ==, 2158 msp->ms_allocating_total); 2159 2160 ASSERT3U(msp->ms_deferspace, ==, 2161 zfs_range_tree_space(msp->ms_defer[0]) + 2162 zfs_range_tree_space(msp->ms_defer[1])); 2163 2164 msp_free_space = zfs_range_tree_space(msp->ms_allocatable) + 2165 allocating + msp->ms_deferspace + 2166 zfs_range_tree_space(msp->ms_freed); 2167 2168 VERIFY3U(sm_free_space, ==, msp_free_space); 2169 } 2170 2171 static void 2172 metaslab_aux_histograms_clear(metaslab_t *msp) 2173 { 2174 /* 2175 * Auxiliary histograms are only cleared when resetting them, 2176 * which can only happen while the metaslab is loaded. 2177 */ 2178 ASSERT(msp->ms_loaded); 2179 2180 memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist)); 2181 for (int t = 0; t < TXG_DEFER_SIZE; t++) 2182 memset(msp->ms_deferhist[t], 0, sizeof (msp->ms_deferhist[t])); 2183 } 2184 2185 static void 2186 metaslab_aux_histogram_add(uint64_t *histogram, uint64_t shift, 2187 zfs_range_tree_t *rt) 2188 { 2189 /* 2190 * This is modeled after space_map_histogram_add(), so refer to that 2191 * function for implementation details. We want this to work like 2192 * the space map histogram, and not the range tree histogram, as we 2193 * are essentially constructing a delta that will be later subtracted 2194 * from the space map histogram. 2195 */ 2196 int idx = 0; 2197 for (int i = shift; i < ZFS_RANGE_TREE_HISTOGRAM_SIZE; i++) { 2198 ASSERT3U(i, >=, idx + shift); 2199 histogram[idx] += rt->rt_histogram[i] << (i - idx - shift); 2200 2201 if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) { 2202 ASSERT3U(idx + shift, ==, i); 2203 idx++; 2204 ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE); 2205 } 2206 } 2207 } 2208 2209 /* 2210 * Called at every sync pass that the metaslab gets synced. 2211 * 2212 * The reason is that we want our auxiliary histograms to be updated 2213 * wherever the metaslab's space map histogram is updated. This way 2214 * we stay consistent on which parts of the metaslab space map's 2215 * histogram are currently not available for allocations (e.g because 2216 * they are in the defer, freed, and freeing trees). 2217 */ 2218 static void 2219 metaslab_aux_histograms_update(metaslab_t *msp) 2220 { 2221 space_map_t *sm = msp->ms_sm; 2222 ASSERT(sm != NULL); 2223 2224 /* 2225 * This is similar to the metaslab's space map histogram updates 2226 * that take place in metaslab_sync(). The only difference is that 2227 * we only care about segments that haven't made it into the 2228 * ms_allocatable tree yet. 2229 */ 2230 if (msp->ms_loaded) { 2231 metaslab_aux_histograms_clear(msp); 2232 2233 metaslab_aux_histogram_add(msp->ms_synchist, 2234 sm->sm_shift, msp->ms_freed); 2235 2236 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 2237 metaslab_aux_histogram_add(msp->ms_deferhist[t], 2238 sm->sm_shift, msp->ms_defer[t]); 2239 } 2240 } 2241 2242 metaslab_aux_histogram_add(msp->ms_synchist, 2243 sm->sm_shift, msp->ms_freeing); 2244 } 2245 2246 /* 2247 * Called every time we are done syncing (writing to) the metaslab, 2248 * i.e. at the end of each sync pass. 2249 * [see the comment in metaslab_impl.h for ms_synchist, ms_deferhist] 2250 */ 2251 static void 2252 metaslab_aux_histograms_update_done(metaslab_t *msp, boolean_t defer_allowed) 2253 { 2254 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2255 space_map_t *sm = msp->ms_sm; 2256 2257 if (sm == NULL) { 2258 /* 2259 * We came here from metaslab_init() when creating/opening a 2260 * pool, looking at a metaslab that hasn't had any allocations 2261 * yet. 2262 */ 2263 return; 2264 } 2265 2266 /* 2267 * This is similar to the actions that we take for the ms_freed 2268 * and ms_defer trees in metaslab_sync_done(). 2269 */ 2270 uint64_t hist_index = spa_syncing_txg(spa) % TXG_DEFER_SIZE; 2271 if (defer_allowed) { 2272 memcpy(msp->ms_deferhist[hist_index], msp->ms_synchist, 2273 sizeof (msp->ms_synchist)); 2274 } else { 2275 memset(msp->ms_deferhist[hist_index], 0, 2276 sizeof (msp->ms_deferhist[hist_index])); 2277 } 2278 memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist)); 2279 } 2280 2281 /* 2282 * Ensure that the metaslab's weight and fragmentation are consistent 2283 * with the contents of the histogram (either the range tree's histogram 2284 * or the space map's depending whether the metaslab is loaded). 2285 */ 2286 static void 2287 metaslab_verify_weight_and_frag(metaslab_t *msp) 2288 { 2289 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2290 2291 if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0) 2292 return; 2293 2294 /* 2295 * We can end up here from vdev_remove_complete(), in which case we 2296 * cannot do these assertions because we hold spa config locks and 2297 * thus we are not allowed to read from the DMU. 2298 * 2299 * We check if the metaslab group has been removed and if that's 2300 * the case we return immediately as that would mean that we are 2301 * here from the aforementioned code path. 2302 */ 2303 if (msp->ms_group == NULL) 2304 return; 2305 2306 /* 2307 * Devices being removed always return a weight of 0 and leave 2308 * fragmentation and ms_max_size as is - there is nothing for 2309 * us to verify here. 2310 */ 2311 vdev_t *vd = msp->ms_group->mg_vd; 2312 if (vd->vdev_removing) 2313 return; 2314 2315 /* 2316 * If the metaslab is dirty it probably means that we've done 2317 * some allocations or frees that have changed our histograms 2318 * and thus the weight. 2319 */ 2320 for (int t = 0; t < TXG_SIZE; t++) { 2321 if (txg_list_member(&vd->vdev_ms_list, msp, t)) 2322 return; 2323 } 2324 2325 /* 2326 * This verification checks that our in-memory state is consistent 2327 * with what's on disk. If the pool is read-only then there aren't 2328 * any changes and we just have the initially-loaded state. 2329 */ 2330 if (!spa_writeable(msp->ms_group->mg_vd->vdev_spa)) 2331 return; 2332 2333 /* some extra verification for in-core tree if you can */ 2334 if (msp->ms_loaded) { 2335 zfs_range_tree_stat_verify(msp->ms_allocatable); 2336 VERIFY(space_map_histogram_verify(msp->ms_sm, 2337 msp->ms_allocatable)); 2338 } 2339 2340 uint64_t weight = msp->ms_weight; 2341 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK; 2342 boolean_t space_based = WEIGHT_IS_SPACEBASED(msp->ms_weight); 2343 uint64_t frag = msp->ms_fragmentation; 2344 uint64_t max_segsize = msp->ms_max_size; 2345 2346 msp->ms_weight = 0; 2347 msp->ms_fragmentation = 0; 2348 2349 /* 2350 * This function is used for verification purposes and thus should 2351 * not introduce any side-effects/mutations on the system's state. 2352 * 2353 * Regardless of whether metaslab_weight() thinks this metaslab 2354 * should be active or not, we want to ensure that the actual weight 2355 * (and therefore the value of ms_weight) would be the same if it 2356 * was to be recalculated at this point. 2357 * 2358 * In addition we set the nodirty flag so metaslab_weight() does 2359 * not dirty the metaslab for future TXGs (e.g. when trying to 2360 * force condensing to upgrade the metaslab spacemaps). 2361 */ 2362 msp->ms_weight = metaslab_weight(msp, B_TRUE) | was_active; 2363 2364 VERIFY3U(max_segsize, ==, msp->ms_max_size); 2365 2366 /* 2367 * If the weight type changed then there is no point in doing 2368 * verification. Revert fields to their original values. 2369 */ 2370 if ((space_based && !WEIGHT_IS_SPACEBASED(msp->ms_weight)) || 2371 (!space_based && WEIGHT_IS_SPACEBASED(msp->ms_weight))) { 2372 msp->ms_fragmentation = frag; 2373 msp->ms_weight = weight; 2374 return; 2375 } 2376 2377 VERIFY3U(msp->ms_fragmentation, ==, frag); 2378 VERIFY3U(msp->ms_weight, ==, weight); 2379 } 2380 2381 /* 2382 * If we're over the zfs_metaslab_mem_limit, select the loaded metaslab from 2383 * this class that was used longest ago, and attempt to unload it. We don't 2384 * want to spend too much time in this loop to prevent performance 2385 * degradation, and we expect that most of the time this operation will 2386 * succeed. Between that and the normal unloading processing during txg sync, 2387 * we expect this to keep the metaslab memory usage under control. 2388 */ 2389 static void 2390 metaslab_potentially_evict(metaslab_class_t *mc) 2391 { 2392 #ifdef _KERNEL 2393 uint64_t allmem = arc_all_memory(); 2394 uint64_t inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache); 2395 uint64_t size = spl_kmem_cache_entry_size(zfs_btree_leaf_cache); 2396 uint_t tries = 0; 2397 for (; allmem * zfs_metaslab_mem_limit / 100 < inuse * size && 2398 tries < multilist_get_num_sublists(&mc->mc_metaslab_txg_list) * 2; 2399 tries++) { 2400 unsigned int idx = multilist_get_random_index( 2401 &mc->mc_metaslab_txg_list); 2402 multilist_sublist_t *mls = 2403 multilist_sublist_lock_idx(&mc->mc_metaslab_txg_list, idx); 2404 metaslab_t *msp = multilist_sublist_head(mls); 2405 multilist_sublist_unlock(mls); 2406 while (msp != NULL && allmem * zfs_metaslab_mem_limit / 100 < 2407 inuse * size) { 2408 VERIFY3P(mls, ==, multilist_sublist_lock_idx( 2409 &mc->mc_metaslab_txg_list, idx)); 2410 ASSERT3U(idx, ==, 2411 metaslab_idx_func(&mc->mc_metaslab_txg_list, msp)); 2412 2413 if (!multilist_link_active(&msp->ms_class_txg_node)) { 2414 multilist_sublist_unlock(mls); 2415 break; 2416 } 2417 metaslab_t *next_msp = multilist_sublist_next(mls, msp); 2418 multilist_sublist_unlock(mls); 2419 /* 2420 * If the metaslab is currently loading there are two 2421 * cases. If it's the metaslab we're evicting, we 2422 * can't continue on or we'll panic when we attempt to 2423 * recursively lock the mutex. If it's another 2424 * metaslab that's loading, it can be safely skipped, 2425 * since we know it's very new and therefore not a 2426 * good eviction candidate. We check later once the 2427 * lock is held that the metaslab is fully loaded 2428 * before actually unloading it. 2429 */ 2430 if (msp->ms_loading) { 2431 msp = next_msp; 2432 inuse = 2433 spl_kmem_cache_inuse(zfs_btree_leaf_cache); 2434 continue; 2435 } 2436 /* 2437 * We can't unload metaslabs with no spacemap because 2438 * they're not ready to be unloaded yet. We can't 2439 * unload metaslabs with outstanding allocations 2440 * because doing so could cause the metaslab's weight 2441 * to decrease while it's unloaded, which violates an 2442 * invariant that we use to prevent unnecessary 2443 * loading. We also don't unload metaslabs that are 2444 * currently active because they are high-weight 2445 * metaslabs that are likely to be used in the near 2446 * future. 2447 */ 2448 mutex_enter(&msp->ms_lock); 2449 if (msp->ms_allocator == -1 && msp->ms_sm != NULL && 2450 msp->ms_allocating_total == 0) { 2451 metaslab_unload(msp); 2452 } 2453 mutex_exit(&msp->ms_lock); 2454 msp = next_msp; 2455 inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache); 2456 } 2457 } 2458 #else 2459 (void) mc, (void) zfs_metaslab_mem_limit; 2460 #endif 2461 } 2462 2463 static int 2464 metaslab_load_impl(metaslab_t *msp) 2465 { 2466 int error = 0; 2467 2468 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2469 ASSERT(msp->ms_loading); 2470 ASSERT(!msp->ms_condensing); 2471 2472 /* 2473 * We temporarily drop the lock to unblock other operations while we 2474 * are reading the space map. Therefore, metaslab_sync() and 2475 * metaslab_sync_done() can run at the same time as we do. 2476 * 2477 * If we are using the log space maps, metaslab_sync() can't write to 2478 * the metaslab's space map while we are loading as we only write to 2479 * it when we are flushing the metaslab, and that can't happen while 2480 * we are loading it. 2481 * 2482 * If we are not using log space maps though, metaslab_sync() can 2483 * append to the space map while we are loading. Therefore we load 2484 * only entries that existed when we started the load. Additionally, 2485 * metaslab_sync_done() has to wait for the load to complete because 2486 * there are potential races like metaslab_load() loading parts of the 2487 * space map that are currently being appended by metaslab_sync(). If 2488 * we didn't, the ms_allocatable would have entries that 2489 * metaslab_sync_done() would try to re-add later. 2490 * 2491 * That's why before dropping the lock we remember the synced length 2492 * of the metaslab and read up to that point of the space map, 2493 * ignoring entries appended by metaslab_sync() that happen after we 2494 * drop the lock. 2495 */ 2496 uint64_t length = msp->ms_synced_length; 2497 mutex_exit(&msp->ms_lock); 2498 2499 hrtime_t load_start = gethrtime(); 2500 metaslab_rt_arg_t *mrap; 2501 if (msp->ms_allocatable->rt_arg == NULL) { 2502 mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP); 2503 } else { 2504 mrap = msp->ms_allocatable->rt_arg; 2505 msp->ms_allocatable->rt_ops = NULL; 2506 msp->ms_allocatable->rt_arg = NULL; 2507 } 2508 mrap->mra_bt = &msp->ms_allocatable_by_size; 2509 mrap->mra_floor_shift = metaslab_by_size_min_shift; 2510 2511 if (msp->ms_sm != NULL) { 2512 error = space_map_load_length(msp->ms_sm, msp->ms_allocatable, 2513 SM_FREE, length); 2514 2515 /* Now, populate the size-sorted tree. */ 2516 metaslab_rt_create(msp->ms_allocatable, mrap); 2517 msp->ms_allocatable->rt_ops = &metaslab_rt_ops; 2518 msp->ms_allocatable->rt_arg = mrap; 2519 2520 struct mssa_arg arg = {0}; 2521 arg.rt = msp->ms_allocatable; 2522 arg.mra = mrap; 2523 zfs_range_tree_walk(msp->ms_allocatable, 2524 metaslab_size_sorted_add, &arg); 2525 } else { 2526 /* 2527 * Add the size-sorted tree first, since we don't need to load 2528 * the metaslab from the spacemap. 2529 */ 2530 metaslab_rt_create(msp->ms_allocatable, mrap); 2531 msp->ms_allocatable->rt_ops = &metaslab_rt_ops; 2532 msp->ms_allocatable->rt_arg = mrap; 2533 /* 2534 * The space map has not been allocated yet, so treat 2535 * all the space in the metaslab as free and add it to the 2536 * ms_allocatable tree. 2537 */ 2538 zfs_range_tree_add(msp->ms_allocatable, 2539 msp->ms_start, msp->ms_size); 2540 2541 if (msp->ms_new) { 2542 /* 2543 * If the ms_sm doesn't exist, this means that this 2544 * metaslab hasn't gone through metaslab_sync() and 2545 * thus has never been dirtied. So we shouldn't 2546 * expect any unflushed allocs or frees from previous 2547 * TXGs. 2548 */ 2549 ASSERT(zfs_range_tree_is_empty( 2550 msp->ms_unflushed_allocs)); 2551 ASSERT(zfs_range_tree_is_empty( 2552 msp->ms_unflushed_frees)); 2553 } 2554 } 2555 2556 /* 2557 * We need to grab the ms_sync_lock to prevent metaslab_sync() from 2558 * changing the ms_sm (or log_sm) and the metaslab's range trees 2559 * while we are about to use them and populate the ms_allocatable. 2560 * The ms_lock is insufficient for this because metaslab_sync() doesn't 2561 * hold the ms_lock while writing the ms_checkpointing tree to disk. 2562 */ 2563 mutex_enter(&msp->ms_sync_lock); 2564 mutex_enter(&msp->ms_lock); 2565 2566 ASSERT(!msp->ms_condensing); 2567 ASSERT(!msp->ms_flushing); 2568 2569 if (error != 0) { 2570 mutex_exit(&msp->ms_sync_lock); 2571 return (error); 2572 } 2573 2574 ASSERT3P(msp->ms_group, !=, NULL); 2575 msp->ms_loaded = B_TRUE; 2576 2577 /* 2578 * Apply all the unflushed changes to ms_allocatable right 2579 * away so any manipulations we do below have a clear view 2580 * of what is allocated and what is free. 2581 */ 2582 zfs_range_tree_walk(msp->ms_unflushed_allocs, 2583 zfs_range_tree_remove, msp->ms_allocatable); 2584 zfs_range_tree_walk(msp->ms_unflushed_frees, 2585 zfs_range_tree_add, msp->ms_allocatable); 2586 2587 ASSERT3P(msp->ms_group, !=, NULL); 2588 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2589 if (spa_syncing_log_sm(spa) != NULL) { 2590 ASSERT(spa_feature_is_enabled(spa, 2591 SPA_FEATURE_LOG_SPACEMAP)); 2592 2593 /* 2594 * If we use a log space map we add all the segments 2595 * that are in ms_unflushed_frees so they are available 2596 * for allocation. 2597 * 2598 * ms_allocatable needs to contain all free segments 2599 * that are ready for allocations (thus not segments 2600 * from ms_freeing, ms_freed, and the ms_defer trees). 2601 * But if we grab the lock in this code path at a sync 2602 * pass later that 1, then it also contains the 2603 * segments of ms_freed (they were added to it earlier 2604 * in this path through ms_unflushed_frees). So we 2605 * need to remove all the segments that exist in 2606 * ms_freed from ms_allocatable as they will be added 2607 * later in metaslab_sync_done(). 2608 * 2609 * When there's no log space map, the ms_allocatable 2610 * correctly doesn't contain any segments that exist 2611 * in ms_freed [see ms_synced_length]. 2612 */ 2613 zfs_range_tree_walk(msp->ms_freed, 2614 zfs_range_tree_remove, msp->ms_allocatable); 2615 } 2616 2617 /* 2618 * If we are not using the log space map, ms_allocatable 2619 * contains the segments that exist in the ms_defer trees 2620 * [see ms_synced_length]. Thus we need to remove them 2621 * from ms_allocatable as they will be added again in 2622 * metaslab_sync_done(). 2623 * 2624 * If we are using the log space map, ms_allocatable still 2625 * contains the segments that exist in the ms_defer trees. 2626 * Not because it read them through the ms_sm though. But 2627 * because these segments are part of ms_unflushed_frees 2628 * whose segments we add to ms_allocatable earlier in this 2629 * code path. 2630 */ 2631 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 2632 zfs_range_tree_walk(msp->ms_defer[t], 2633 zfs_range_tree_remove, msp->ms_allocatable); 2634 } 2635 2636 /* 2637 * Call metaslab_recalculate_weight_and_sort() now that the 2638 * metaslab is loaded so we get the metaslab's real weight. 2639 * 2640 * Unless this metaslab was created with older software and 2641 * has not yet been converted to use segment-based weight, we 2642 * expect the new weight to be better or equal to the weight 2643 * that the metaslab had while it was not loaded. This is 2644 * because the old weight does not take into account the 2645 * consolidation of adjacent segments between TXGs. [see 2646 * comment for ms_synchist and ms_deferhist[] for more info] 2647 */ 2648 uint64_t weight = msp->ms_weight; 2649 uint64_t max_size = msp->ms_max_size; 2650 metaslab_recalculate_weight_and_sort(msp); 2651 if (!WEIGHT_IS_SPACEBASED(weight)) 2652 ASSERT3U(weight, <=, msp->ms_weight); 2653 msp->ms_max_size = metaslab_largest_allocatable(msp); 2654 ASSERT3U(max_size, <=, msp->ms_max_size); 2655 hrtime_t load_end = gethrtime(); 2656 msp->ms_load_time = load_end; 2657 zfs_dbgmsg("metaslab_load: txg %llu, spa %s, class %s, vdev_id %llu, " 2658 "ms_id %llu, smp_length %llu, " 2659 "unflushed_allocs %llu, unflushed_frees %llu, " 2660 "freed %llu, defer %llu + %llu, unloaded time %llu ms, " 2661 "loading_time %lld ms, ms_max_size %llu, " 2662 "max size error %lld, " 2663 "old_weight %llx, new_weight %llx", 2664 (u_longlong_t)spa_syncing_txg(spa), spa_name(spa), 2665 msp->ms_group->mg_class->mc_name, 2666 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 2667 (u_longlong_t)msp->ms_id, 2668 (u_longlong_t)space_map_length(msp->ms_sm), 2669 (u_longlong_t)zfs_range_tree_space(msp->ms_unflushed_allocs), 2670 (u_longlong_t)zfs_range_tree_space(msp->ms_unflushed_frees), 2671 (u_longlong_t)zfs_range_tree_space(msp->ms_freed), 2672 (u_longlong_t)zfs_range_tree_space(msp->ms_defer[0]), 2673 (u_longlong_t)zfs_range_tree_space(msp->ms_defer[1]), 2674 (longlong_t)((load_start - msp->ms_unload_time) / 1000000), 2675 (longlong_t)((load_end - load_start) / 1000000), 2676 (u_longlong_t)msp->ms_max_size, 2677 (u_longlong_t)msp->ms_max_size - max_size, 2678 (u_longlong_t)weight, (u_longlong_t)msp->ms_weight); 2679 2680 metaslab_verify_space(msp, spa_syncing_txg(spa)); 2681 mutex_exit(&msp->ms_sync_lock); 2682 return (0); 2683 } 2684 2685 int 2686 metaslab_load(metaslab_t *msp) 2687 { 2688 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2689 2690 /* 2691 * There may be another thread loading the same metaslab, if that's 2692 * the case just wait until the other thread is done and return. 2693 */ 2694 metaslab_load_wait(msp); 2695 if (msp->ms_loaded) 2696 return (0); 2697 VERIFY(!msp->ms_loading); 2698 ASSERT(!msp->ms_condensing); 2699 2700 /* 2701 * We set the loading flag BEFORE potentially dropping the lock to 2702 * wait for an ongoing flush (see ms_flushing below). This way other 2703 * threads know that there is already a thread that is loading this 2704 * metaslab. 2705 */ 2706 msp->ms_loading = B_TRUE; 2707 2708 /* 2709 * Wait for any in-progress flushing to finish as we drop the ms_lock 2710 * both here (during space_map_load()) and in metaslab_flush() (when 2711 * we flush our changes to the ms_sm). 2712 */ 2713 if (msp->ms_flushing) 2714 metaslab_flush_wait(msp); 2715 2716 /* 2717 * In the possibility that we were waiting for the metaslab to be 2718 * flushed (where we temporarily dropped the ms_lock), ensure that 2719 * no one else loaded the metaslab somehow. 2720 */ 2721 ASSERT(!msp->ms_loaded); 2722 2723 /* 2724 * If we're loading a metaslab in the normal class, consider evicting 2725 * another one to keep our memory usage under the limit defined by the 2726 * zfs_metaslab_mem_limit tunable. 2727 */ 2728 if (spa_normal_class(msp->ms_group->mg_class->mc_spa) == 2729 msp->ms_group->mg_class) { 2730 metaslab_potentially_evict(msp->ms_group->mg_class); 2731 } 2732 2733 int error = metaslab_load_impl(msp); 2734 2735 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2736 msp->ms_loading = B_FALSE; 2737 cv_broadcast(&msp->ms_load_cv); 2738 2739 return (error); 2740 } 2741 2742 void 2743 metaslab_unload(metaslab_t *msp) 2744 { 2745 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2746 2747 /* 2748 * This can happen if a metaslab is selected for eviction (in 2749 * metaslab_potentially_evict) and then unloaded during spa_sync (via 2750 * metaslab_class_evict_old). 2751 */ 2752 if (!msp->ms_loaded) 2753 return; 2754 2755 zfs_range_tree_vacate(msp->ms_allocatable, NULL, NULL); 2756 msp->ms_loaded = B_FALSE; 2757 msp->ms_unload_time = gethrtime(); 2758 2759 msp->ms_activation_weight = 0; 2760 msp->ms_weight &= ~METASLAB_ACTIVE_MASK; 2761 2762 if (msp->ms_group != NULL) { 2763 metaslab_class_t *mc = msp->ms_group->mg_class; 2764 multilist_sublist_t *mls = 2765 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 2766 if (multilist_link_active(&msp->ms_class_txg_node)) 2767 multilist_sublist_remove(mls, msp); 2768 multilist_sublist_unlock(mls); 2769 2770 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2771 zfs_dbgmsg("metaslab_unload: txg %llu, spa %s, class %s, " 2772 "vdev_id %llu, ms_id %llu, weight %llx, " 2773 "selected txg %llu (%llu s ago), alloc_txg %llu, " 2774 "loaded %llu ms ago, max_size %llu", 2775 (u_longlong_t)spa_syncing_txg(spa), spa_name(spa), 2776 msp->ms_group->mg_class->mc_name, 2777 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 2778 (u_longlong_t)msp->ms_id, 2779 (u_longlong_t)msp->ms_weight, 2780 (u_longlong_t)msp->ms_selected_txg, 2781 (u_longlong_t)(NSEC2SEC(msp->ms_unload_time) - 2782 msp->ms_selected_time), 2783 (u_longlong_t)msp->ms_alloc_txg, 2784 (u_longlong_t)(msp->ms_unload_time - 2785 msp->ms_load_time) / 1000 / 1000, 2786 (u_longlong_t)msp->ms_max_size); 2787 } 2788 2789 /* 2790 * We explicitly recalculate the metaslab's weight based on its space 2791 * map (as it is now not loaded). We want unload metaslabs to always 2792 * have their weights calculated from the space map histograms, while 2793 * loaded ones have it calculated from their in-core range tree 2794 * [see metaslab_load()]. This way, the weight reflects the information 2795 * available in-core, whether it is loaded or not. 2796 * 2797 * If ms_group == NULL means that we came here from metaslab_fini(), 2798 * at which point it doesn't make sense for us to do the recalculation 2799 * and the sorting. 2800 */ 2801 if (msp->ms_group != NULL) 2802 metaslab_recalculate_weight_and_sort(msp); 2803 } 2804 2805 /* 2806 * We want to optimize the memory use of the per-metaslab range 2807 * trees. To do this, we store the segments in the range trees in 2808 * units of sectors, zero-indexing from the start of the metaslab. If 2809 * the vdev_ms_shift - the vdev_ashift is less than 32, we can store 2810 * the ranges using two uint32_ts, rather than two uint64_ts. 2811 */ 2812 zfs_range_seg_type_t 2813 metaslab_calculate_range_tree_type(vdev_t *vdev, metaslab_t *msp, 2814 uint64_t *start, uint64_t *shift) 2815 { 2816 if (vdev->vdev_ms_shift - vdev->vdev_ashift < 32 && 2817 !zfs_metaslab_force_large_segs) { 2818 *shift = vdev->vdev_ashift; 2819 *start = msp->ms_start; 2820 return (ZFS_RANGE_SEG32); 2821 } else { 2822 *shift = 0; 2823 *start = 0; 2824 return (ZFS_RANGE_SEG64); 2825 } 2826 } 2827 2828 void 2829 metaslab_set_selected_txg(metaslab_t *msp, uint64_t txg) 2830 { 2831 ASSERT(MUTEX_HELD(&msp->ms_lock)); 2832 metaslab_class_t *mc = msp->ms_group->mg_class; 2833 multilist_sublist_t *mls = 2834 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 2835 if (multilist_link_active(&msp->ms_class_txg_node)) 2836 multilist_sublist_remove(mls, msp); 2837 msp->ms_selected_txg = txg; 2838 msp->ms_selected_time = gethrestime_sec(); 2839 multilist_sublist_insert_tail(mls, msp); 2840 multilist_sublist_unlock(mls); 2841 } 2842 2843 void 2844 metaslab_space_update(vdev_t *vd, metaslab_class_t *mc, int64_t alloc_delta, 2845 int64_t defer_delta, int64_t space_delta) 2846 { 2847 vdev_space_update(vd, alloc_delta, defer_delta, space_delta); 2848 2849 ASSERT3P(vd->vdev_spa->spa_root_vdev, ==, vd->vdev_parent); 2850 ASSERT(vd->vdev_ms_count != 0); 2851 2852 metaslab_class_space_update(mc, alloc_delta, defer_delta, space_delta, 2853 vdev_deflated_space(vd, space_delta)); 2854 } 2855 2856 int 2857 metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object, 2858 uint64_t txg, metaslab_t **msp) 2859 { 2860 vdev_t *vd = mg->mg_vd; 2861 spa_t *spa = vd->vdev_spa; 2862 objset_t *mos = spa->spa_meta_objset; 2863 metaslab_t *ms; 2864 int error; 2865 2866 ms = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP); 2867 mutex_init(&ms->ms_lock, NULL, MUTEX_DEFAULT, NULL); 2868 mutex_init(&ms->ms_sync_lock, NULL, MUTEX_DEFAULT, NULL); 2869 cv_init(&ms->ms_load_cv, NULL, CV_DEFAULT, NULL); 2870 cv_init(&ms->ms_flush_cv, NULL, CV_DEFAULT, NULL); 2871 multilist_link_init(&ms->ms_class_txg_node); 2872 2873 ms->ms_id = id; 2874 ms->ms_start = id << vd->vdev_ms_shift; 2875 ms->ms_size = 1ULL << vd->vdev_ms_shift; 2876 ms->ms_allocator = -1; 2877 ms->ms_new = B_TRUE; 2878 2879 vdev_ops_t *ops = vd->vdev_ops; 2880 if (ops->vdev_op_metaslab_init != NULL) 2881 ops->vdev_op_metaslab_init(vd, &ms->ms_start, &ms->ms_size); 2882 2883 /* 2884 * We only open space map objects that already exist. All others 2885 * will be opened when we finally allocate an object for it. For 2886 * readonly pools there is no need to open the space map object. 2887 * 2888 * Note: 2889 * When called from vdev_expand(), we can't call into the DMU as 2890 * we are holding the spa_config_lock as a writer and we would 2891 * deadlock [see relevant comment in vdev_metaslab_init()]. in 2892 * that case, the object parameter is zero though, so we won't 2893 * call into the DMU. 2894 */ 2895 if (object != 0 && !(spa->spa_mode == SPA_MODE_READ && 2896 !spa->spa_read_spacemaps)) { 2897 error = space_map_open(&ms->ms_sm, mos, object, ms->ms_start, 2898 ms->ms_size, vd->vdev_ashift); 2899 2900 if (error != 0) { 2901 kmem_free(ms, sizeof (metaslab_t)); 2902 return (error); 2903 } 2904 2905 ASSERT(ms->ms_sm != NULL); 2906 ms->ms_allocated_space = space_map_allocated(ms->ms_sm); 2907 } 2908 2909 uint64_t shift, start; 2910 zfs_range_seg_type_t type = 2911 metaslab_calculate_range_tree_type(vd, ms, &start, &shift); 2912 2913 ms->ms_allocatable = zfs_range_tree_create_flags( 2914 NULL, type, NULL, start, shift, 2915 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_allocatable")); 2916 for (int t = 0; t < TXG_SIZE; t++) { 2917 ms->ms_allocating[t] = zfs_range_tree_create_flags( 2918 NULL, type, NULL, start, shift, 2919 ZFS_RT_F_DYN_NAME, 2920 metaslab_rt_name(mg, ms, "ms_allocating")); 2921 } 2922 ms->ms_freeing = zfs_range_tree_create_flags( 2923 NULL, type, NULL, start, shift, 2924 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_freeing")); 2925 ms->ms_freed = zfs_range_tree_create_flags( 2926 NULL, type, NULL, start, shift, 2927 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_freed")); 2928 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 2929 ms->ms_defer[t] = zfs_range_tree_create_flags( 2930 NULL, type, NULL, start, shift, 2931 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_defer")); 2932 } 2933 ms->ms_checkpointing = zfs_range_tree_create_flags( 2934 NULL, type, NULL, start, shift, 2935 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_checkpointing")); 2936 ms->ms_unflushed_allocs = zfs_range_tree_create_flags( 2937 NULL, type, NULL, start, shift, 2938 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_unflushed_allocs")); 2939 2940 metaslab_rt_arg_t *mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP); 2941 mrap->mra_bt = &ms->ms_unflushed_frees_by_size; 2942 mrap->mra_floor_shift = metaslab_by_size_min_shift; 2943 ms->ms_unflushed_frees = zfs_range_tree_create_flags( 2944 &metaslab_rt_ops, type, mrap, start, shift, 2945 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_unflushed_frees")); 2946 2947 ms->ms_trim = zfs_range_tree_create_flags( 2948 NULL, type, NULL, start, shift, 2949 ZFS_RT_F_DYN_NAME, metaslab_rt_name(mg, ms, "ms_trim")); 2950 2951 metaslab_group_add(mg, ms); 2952 metaslab_set_fragmentation(ms, B_FALSE); 2953 2954 /* 2955 * If we're opening an existing pool (txg == 0) or creating 2956 * a new one (txg == TXG_INITIAL), all space is available now. 2957 * If we're adding space to an existing pool, the new space 2958 * does not become available until after this txg has synced. 2959 * The metaslab's weight will also be initialized when we sync 2960 * out this txg. This ensures that we don't attempt to allocate 2961 * from it before we have initialized it completely. 2962 */ 2963 if (txg <= TXG_INITIAL) { 2964 metaslab_sync_done(ms, 0); 2965 metaslab_space_update(vd, mg->mg_class, 2966 metaslab_allocated_space(ms), 0, 0); 2967 } 2968 2969 if (txg != 0) { 2970 vdev_dirty(vd, 0, NULL, txg); 2971 vdev_dirty(vd, VDD_METASLAB, ms, txg); 2972 } 2973 2974 *msp = ms; 2975 2976 return (0); 2977 } 2978 2979 static void 2980 metaslab_fini_flush_data(metaslab_t *msp) 2981 { 2982 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 2983 2984 if (metaslab_unflushed_txg(msp) == 0) { 2985 ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), 2986 ==, NULL); 2987 return; 2988 } 2989 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 2990 2991 mutex_enter(&spa->spa_flushed_ms_lock); 2992 avl_remove(&spa->spa_metaslabs_by_flushed, msp); 2993 mutex_exit(&spa->spa_flushed_ms_lock); 2994 2995 spa_log_sm_decrement_mscount(spa, metaslab_unflushed_txg(msp)); 2996 spa_log_summary_decrement_mscount(spa, metaslab_unflushed_txg(msp), 2997 metaslab_unflushed_dirty(msp)); 2998 } 2999 3000 uint64_t 3001 metaslab_unflushed_changes_memused(metaslab_t *ms) 3002 { 3003 return ((zfs_range_tree_numsegs(ms->ms_unflushed_allocs) + 3004 zfs_range_tree_numsegs(ms->ms_unflushed_frees)) * 3005 ms->ms_unflushed_allocs->rt_root.bt_elem_size); 3006 } 3007 3008 void 3009 metaslab_fini(metaslab_t *msp) 3010 { 3011 metaslab_group_t *mg = msp->ms_group; 3012 vdev_t *vd = mg->mg_vd; 3013 spa_t *spa = vd->vdev_spa; 3014 3015 metaslab_fini_flush_data(msp); 3016 3017 metaslab_group_remove(mg, msp); 3018 3019 mutex_enter(&msp->ms_lock); 3020 VERIFY0P(msp->ms_group); 3021 3022 /* 3023 * If this metaslab hasn't been through metaslab_sync_done() yet its 3024 * space hasn't been accounted for in its vdev and doesn't need to be 3025 * subtracted. 3026 */ 3027 if (!msp->ms_new) { 3028 metaslab_space_update(vd, mg->mg_class, 3029 -metaslab_allocated_space(msp), 0, -msp->ms_size); 3030 3031 } 3032 space_map_close(msp->ms_sm); 3033 msp->ms_sm = NULL; 3034 3035 metaslab_unload(msp); 3036 3037 zfs_range_tree_destroy(msp->ms_allocatable); 3038 zfs_range_tree_destroy(msp->ms_freeing); 3039 zfs_range_tree_destroy(msp->ms_freed); 3040 3041 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 3042 metaslab_unflushed_changes_memused(msp)); 3043 spa->spa_unflushed_stats.sus_memused -= 3044 metaslab_unflushed_changes_memused(msp); 3045 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); 3046 zfs_range_tree_destroy(msp->ms_unflushed_allocs); 3047 zfs_range_tree_destroy(msp->ms_checkpointing); 3048 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); 3049 zfs_range_tree_destroy(msp->ms_unflushed_frees); 3050 3051 for (int t = 0; t < TXG_SIZE; t++) { 3052 zfs_range_tree_destroy(msp->ms_allocating[t]); 3053 } 3054 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 3055 zfs_range_tree_destroy(msp->ms_defer[t]); 3056 } 3057 ASSERT0(msp->ms_deferspace); 3058 3059 for (int t = 0; t < TXG_SIZE; t++) 3060 ASSERT(!txg_list_member(&vd->vdev_ms_list, msp, t)); 3061 3062 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); 3063 zfs_range_tree_destroy(msp->ms_trim); 3064 3065 mutex_exit(&msp->ms_lock); 3066 cv_destroy(&msp->ms_load_cv); 3067 cv_destroy(&msp->ms_flush_cv); 3068 mutex_destroy(&msp->ms_lock); 3069 mutex_destroy(&msp->ms_sync_lock); 3070 ASSERT3U(msp->ms_allocator, ==, -1); 3071 3072 kmem_free(msp, sizeof (metaslab_t)); 3073 } 3074 3075 /* 3076 * This table defines a segment size based fragmentation metric that will 3077 * allow each metaslab to derive its own fragmentation value. This is done 3078 * by calculating the space in each bucket of the spacemap histogram and 3079 * multiplying that by the fragmentation metric in this table. Doing 3080 * this for all buckets and dividing it by the total amount of free 3081 * space in this metaslab (i.e. the total free space in all buckets) gives 3082 * us the fragmentation metric. This means that a high fragmentation metric 3083 * equates to most of the free space being comprised of small segments. 3084 * Conversely, if the metric is low, then most of the free space is in 3085 * large segments. 3086 * 3087 * This table defines 0% fragmented space using 512M segments. Using this value, 3088 * we derive the rest of the table. This table originally went up to 16MB, but 3089 * with larger recordsizes, larger ashifts, and use of raidz3, it is possible 3090 * to have significantly larger allocations than were previously possible. 3091 * Since the fragmentation value is never stored on disk, it is possible to 3092 * change these calculations in the future. 3093 */ 3094 static const int zfs_frag_table[] = { 3095 100, /* 512B */ 3096 99, /* 1K */ 3097 97, /* 2K */ 3098 93, /* 4K */ 3099 88, /* 8K */ 3100 83, /* 16K */ 3101 77, /* 32K */ 3102 71, /* 64K */ 3103 64, /* 128K */ 3104 57, /* 256K */ 3105 50, /* 512K */ 3106 43, /* 1M */ 3107 36, /* 2M */ 3108 29, /* 4M */ 3109 23, /* 8M */ 3110 17, /* 16M */ 3111 12, /* 32M */ 3112 7, /* 64M */ 3113 3, /* 128M */ 3114 1, /* 256M */ 3115 0, /* 512M */ 3116 }; 3117 #define FRAGMENTATION_TABLE_SIZE \ 3118 (sizeof (zfs_frag_table)/(sizeof (zfs_frag_table[0]))) 3119 3120 /* 3121 * Calculate the metaslab's fragmentation metric and set ms_fragmentation. 3122 * Setting this value to ZFS_FRAG_INVALID means that the metaslab has not 3123 * been upgraded and does not support this metric. Otherwise, the return 3124 * value should be in the range [0, 100]. 3125 */ 3126 static void 3127 metaslab_set_fragmentation(metaslab_t *msp, boolean_t nodirty) 3128 { 3129 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 3130 uint64_t fragmentation = 0; 3131 uint64_t total = 0; 3132 boolean_t feature_enabled = spa_feature_is_enabled(spa, 3133 SPA_FEATURE_SPACEMAP_HISTOGRAM); 3134 3135 if (!feature_enabled) { 3136 msp->ms_fragmentation = ZFS_FRAG_INVALID; 3137 return; 3138 } 3139 3140 /* 3141 * A null space map means that the entire metaslab is free 3142 * and thus is not fragmented. 3143 */ 3144 if (msp->ms_sm == NULL) { 3145 msp->ms_fragmentation = 0; 3146 return; 3147 } 3148 3149 /* 3150 * If this metaslab's space map has not been upgraded, flag it 3151 * so that we upgrade next time we encounter it. 3152 */ 3153 if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) { 3154 uint64_t txg = spa_syncing_txg(spa); 3155 vdev_t *vd = msp->ms_group->mg_vd; 3156 3157 /* 3158 * If we've reached the final dirty txg, then we must 3159 * be shutting down the pool. We don't want to dirty 3160 * any data past this point so skip setting the condense 3161 * flag. We can retry this action the next time the pool 3162 * is imported. We also skip marking this metaslab for 3163 * condensing if the caller has explicitly set nodirty. 3164 */ 3165 if (!nodirty && 3166 spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) { 3167 msp->ms_condense_wanted = B_TRUE; 3168 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); 3169 zfs_dbgmsg("txg %llu, requesting force condense: " 3170 "ms_id %llu, vdev_id %llu", (u_longlong_t)txg, 3171 (u_longlong_t)msp->ms_id, 3172 (u_longlong_t)vd->vdev_id); 3173 } 3174 msp->ms_fragmentation = ZFS_FRAG_INVALID; 3175 return; 3176 } 3177 3178 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 3179 uint64_t space = 0; 3180 uint8_t shift = msp->ms_sm->sm_shift; 3181 3182 int idx = MIN(shift - SPA_MINBLOCKSHIFT + i, 3183 FRAGMENTATION_TABLE_SIZE - 1); 3184 3185 if (msp->ms_sm->sm_phys->smp_histogram[i] == 0) 3186 continue; 3187 3188 space = msp->ms_sm->sm_phys->smp_histogram[i] << (i + shift); 3189 total += space; 3190 3191 ASSERT3U(idx, <, FRAGMENTATION_TABLE_SIZE); 3192 fragmentation += space * zfs_frag_table[idx]; 3193 } 3194 3195 if (total > 0) 3196 fragmentation /= total; 3197 ASSERT3U(fragmentation, <=, 100); 3198 3199 msp->ms_fragmentation = fragmentation; 3200 } 3201 3202 /* 3203 * Compute a weight -- a selection preference value -- for the given metaslab. 3204 * This is based on the amount of free space, the level of fragmentation, 3205 * the LBA range, and whether the metaslab is loaded. 3206 */ 3207 static uint64_t 3208 metaslab_space_weight(metaslab_t *msp) 3209 { 3210 metaslab_group_t *mg = msp->ms_group; 3211 vdev_t *vd = mg->mg_vd; 3212 uint64_t weight, space; 3213 3214 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3215 3216 /* 3217 * The baseline weight is the metaslab's free space. 3218 */ 3219 space = msp->ms_size - metaslab_allocated_space(msp); 3220 3221 if (metaslab_fragmentation_factor_enabled && 3222 msp->ms_fragmentation != ZFS_FRAG_INVALID) { 3223 /* 3224 * Use the fragmentation information to inversely scale 3225 * down the baseline weight. We need to ensure that we 3226 * don't exclude this metaslab completely when it's 100% 3227 * fragmented. To avoid this we reduce the fragmented value 3228 * by 1. 3229 */ 3230 space = (space * (100 - (msp->ms_fragmentation - 1))) / 100; 3231 3232 /* 3233 * If space < SPA_MINBLOCKSIZE, then we will not allocate from 3234 * this metaslab again. The fragmentation metric may have 3235 * decreased the space to something smaller than 3236 * SPA_MINBLOCKSIZE, so reset the space to SPA_MINBLOCKSIZE 3237 * so that we can consume any remaining space. 3238 */ 3239 if (space > 0 && space < SPA_MINBLOCKSIZE) 3240 space = SPA_MINBLOCKSIZE; 3241 } 3242 weight = space; 3243 3244 /* 3245 * Modern disks have uniform bit density and constant angular velocity. 3246 * Therefore, the outer recording zones are faster (higher bandwidth) 3247 * than the inner zones by the ratio of outer to inner track diameter, 3248 * which is typically around 2:1. We account for this by assigning 3249 * higher weight to lower metaslabs (multiplier ranging from 2x to 1x). 3250 * In effect, this means that we'll select the metaslab with the most 3251 * free bandwidth rather than simply the one with the most free space. 3252 */ 3253 if (!vd->vdev_nonrot && metaslab_lba_weighting_enabled) { 3254 weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count; 3255 ASSERT(weight >= space && weight <= 2 * space); 3256 } 3257 3258 /* 3259 * If this metaslab is one we're actively using, adjust its 3260 * weight to make it preferable to any inactive metaslab so 3261 * we'll polish it off. If the fragmentation on this metaslab 3262 * has exceed our threshold, then don't mark it active. 3263 */ 3264 if (msp->ms_loaded && msp->ms_fragmentation != ZFS_FRAG_INVALID && 3265 msp->ms_fragmentation <= zfs_metaslab_fragmentation_threshold) { 3266 weight |= (msp->ms_weight & METASLAB_ACTIVE_MASK); 3267 } 3268 3269 WEIGHT_SET_SPACEBASED(weight); 3270 return (weight); 3271 } 3272 3273 /* 3274 * Return the weight of the specified metaslab, according to the segment-based 3275 * weighting algorithm. The metaslab must be loaded. This function can 3276 * be called within a sync pass since it relies only on the metaslab's 3277 * range tree which is always accurate when the metaslab is loaded. 3278 */ 3279 static uint64_t 3280 metaslab_weight_from_range_tree(metaslab_t *msp) 3281 { 3282 uint64_t weight = 0; 3283 uint32_t segments = 0; 3284 3285 ASSERT(msp->ms_loaded); 3286 3287 for (int i = ZFS_RANGE_TREE_HISTOGRAM_SIZE - 1; i >= SPA_MINBLOCKSHIFT; 3288 i--) { 3289 uint8_t shift = msp->ms_group->mg_vd->vdev_ashift; 3290 int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1; 3291 3292 segments <<= 1; 3293 segments += msp->ms_allocatable->rt_histogram[i]; 3294 3295 /* 3296 * The range tree provides more precision than the space map 3297 * and must be downgraded so that all values fit within the 3298 * space map's histogram. This allows us to compare loaded 3299 * vs. unloaded metaslabs to determine which metaslab is 3300 * considered "best". 3301 */ 3302 if (i > max_idx) 3303 continue; 3304 3305 if (segments != 0) { 3306 WEIGHT_SET_COUNT(weight, segments); 3307 WEIGHT_SET_INDEX(weight, i); 3308 WEIGHT_SET_ACTIVE(weight, 0); 3309 break; 3310 } 3311 } 3312 return (weight); 3313 } 3314 3315 /* 3316 * Calculate the weight based on the on-disk histogram. Should be applied 3317 * only to unloaded metaslabs (i.e no incoming allocations) in-order to 3318 * give results consistent with the on-disk state 3319 */ 3320 static uint64_t 3321 metaslab_weight_from_spacemap(metaslab_t *msp) 3322 { 3323 space_map_t *sm = msp->ms_sm; 3324 ASSERT(!msp->ms_loaded); 3325 ASSERT(sm != NULL); 3326 ASSERT3U(space_map_object(sm), !=, 0); 3327 ASSERT3U(sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t)); 3328 3329 /* 3330 * Create a joint histogram from all the segments that have made 3331 * it to the metaslab's space map histogram, that are not yet 3332 * available for allocation because they are still in the freeing 3333 * pipeline (e.g. freeing, freed, and defer trees). Then subtract 3334 * these segments from the space map's histogram to get a more 3335 * accurate weight. 3336 */ 3337 uint64_t deferspace_histogram[SPACE_MAP_HISTOGRAM_SIZE] = {0}; 3338 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) 3339 deferspace_histogram[i] += msp->ms_synchist[i]; 3340 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 3341 for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) { 3342 deferspace_histogram[i] += msp->ms_deferhist[t][i]; 3343 } 3344 } 3345 3346 uint64_t weight = 0; 3347 for (int i = SPACE_MAP_HISTOGRAM_SIZE - 1; i >= 0; i--) { 3348 ASSERT3U(sm->sm_phys->smp_histogram[i], >=, 3349 deferspace_histogram[i]); 3350 uint64_t count = 3351 sm->sm_phys->smp_histogram[i] - deferspace_histogram[i]; 3352 if (count != 0) { 3353 WEIGHT_SET_COUNT(weight, count); 3354 WEIGHT_SET_INDEX(weight, i + sm->sm_shift); 3355 WEIGHT_SET_ACTIVE(weight, 0); 3356 break; 3357 } 3358 } 3359 return (weight); 3360 } 3361 3362 /* 3363 * Compute a segment-based weight for the specified metaslab. The weight 3364 * is determined by highest bucket in the histogram. The information 3365 * for the highest bucket is encoded into the weight value. 3366 */ 3367 static uint64_t 3368 metaslab_segment_weight(metaslab_t *msp) 3369 { 3370 metaslab_group_t *mg = msp->ms_group; 3371 uint64_t weight = 0; 3372 uint8_t shift = mg->mg_vd->vdev_ashift; 3373 3374 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3375 3376 /* 3377 * The metaslab is completely free. 3378 */ 3379 if (metaslab_allocated_space(msp) == 0) { 3380 int idx = highbit64(msp->ms_size) - 1; 3381 int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1; 3382 3383 if (idx < max_idx) { 3384 WEIGHT_SET_COUNT(weight, 1ULL); 3385 WEIGHT_SET_INDEX(weight, idx); 3386 } else { 3387 WEIGHT_SET_COUNT(weight, 1ULL << (idx - max_idx)); 3388 WEIGHT_SET_INDEX(weight, max_idx); 3389 } 3390 WEIGHT_SET_ACTIVE(weight, 0); 3391 ASSERT(!WEIGHT_IS_SPACEBASED(weight)); 3392 return (weight); 3393 } 3394 3395 ASSERT3U(msp->ms_sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t)); 3396 3397 /* 3398 * If the metaslab is fully allocated then just make the weight 0. 3399 */ 3400 if (metaslab_allocated_space(msp) == msp->ms_size) 3401 return (0); 3402 /* 3403 * If the metaslab is already loaded, then use the range tree to 3404 * determine the weight. Otherwise, we rely on the space map information 3405 * to generate the weight. 3406 */ 3407 if (msp->ms_loaded) { 3408 weight = metaslab_weight_from_range_tree(msp); 3409 } else { 3410 weight = metaslab_weight_from_spacemap(msp); 3411 } 3412 3413 /* 3414 * If the metaslab was active the last time we calculated its weight 3415 * then keep it active. We want to consume the entire region that 3416 * is associated with this weight. 3417 */ 3418 if (msp->ms_activation_weight != 0 && weight != 0) 3419 WEIGHT_SET_ACTIVE(weight, WEIGHT_GET_ACTIVE(msp->ms_weight)); 3420 return (weight); 3421 } 3422 3423 /* 3424 * Determine if we should attempt to allocate from this metaslab. If the 3425 * metaslab is loaded, then we can determine if the desired allocation 3426 * can be satisfied by looking at the size of the maximum free segment 3427 * on that metaslab. Otherwise, we make our decision based on the metaslab's 3428 * weight. For segment-based weighting we can determine the maximum 3429 * allocation based on the index encoded in its value. For space-based 3430 * weights we rely on the entire weight (excluding the weight-type bit). 3431 */ 3432 static boolean_t 3433 metaslab_should_allocate(metaslab_t *msp, uint64_t asize, boolean_t try_hard) 3434 { 3435 /* 3436 * This case will usually but not always get caught by the checks below; 3437 * metaslabs can be loaded by various means, including the trim and 3438 * initialize code. Once that happens, without this check they are 3439 * allocatable even before they finish their first txg sync. 3440 */ 3441 if (unlikely(msp->ms_new)) 3442 return (B_FALSE); 3443 3444 /* 3445 * If the metaslab is loaded, ms_max_size is definitive and we can use 3446 * the fast check. If it's not, the ms_max_size is a lower bound (once 3447 * set), and we should use the fast check as long as we're not in 3448 * try_hard and it's been less than zfs_metaslab_max_size_cache_sec 3449 * seconds since the metaslab was unloaded. 3450 */ 3451 if (msp->ms_loaded || 3452 (msp->ms_max_size != 0 && !try_hard && gethrtime() < 3453 msp->ms_unload_time + SEC2NSEC(zfs_metaslab_max_size_cache_sec))) 3454 return (msp->ms_max_size >= asize); 3455 3456 boolean_t should_allocate; 3457 if (!WEIGHT_IS_SPACEBASED(msp->ms_weight)) { 3458 /* 3459 * The metaslab segment weight indicates segments in the 3460 * range [2^i, 2^(i+1)), where i is the index in the weight. 3461 * Since the asize might be in the middle of the range, we 3462 * should attempt the allocation if asize < 2^(i+1). 3463 */ 3464 should_allocate = (asize < 3465 1ULL << (WEIGHT_GET_INDEX(msp->ms_weight) + 1)); 3466 } else { 3467 should_allocate = (asize <= 3468 (msp->ms_weight & ~METASLAB_WEIGHT_TYPE)); 3469 } 3470 3471 return (should_allocate); 3472 } 3473 3474 static uint64_t 3475 metaslab_weight(metaslab_t *msp, boolean_t nodirty) 3476 { 3477 vdev_t *vd = msp->ms_group->mg_vd; 3478 spa_t *spa = vd->vdev_spa; 3479 uint64_t weight; 3480 3481 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3482 3483 metaslab_set_fragmentation(msp, nodirty); 3484 3485 /* 3486 * Update the maximum size. If the metaslab is loaded, this will 3487 * ensure that we get an accurate maximum size if newly freed space 3488 * has been added back into the free tree. If the metaslab is 3489 * unloaded, we check if there's a larger free segment in the 3490 * unflushed frees. This is a lower bound on the largest allocatable 3491 * segment size. Coalescing of adjacent entries may reveal larger 3492 * allocatable segments, but we aren't aware of those until loading 3493 * the space map into a range tree. 3494 */ 3495 if (msp->ms_loaded) { 3496 msp->ms_max_size = metaslab_largest_allocatable(msp); 3497 } else { 3498 msp->ms_max_size = MAX(msp->ms_max_size, 3499 metaslab_largest_unflushed_free(msp)); 3500 } 3501 3502 /* 3503 * Segment-based weighting requires space map histogram support. 3504 */ 3505 if (zfs_metaslab_segment_weight_enabled && 3506 spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) && 3507 (msp->ms_sm == NULL || msp->ms_sm->sm_dbuf->db_size == 3508 sizeof (space_map_phys_t))) { 3509 weight = metaslab_segment_weight(msp); 3510 } else { 3511 weight = metaslab_space_weight(msp); 3512 } 3513 return (weight); 3514 } 3515 3516 void 3517 metaslab_recalculate_weight_and_sort(metaslab_t *msp) 3518 { 3519 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3520 3521 /* note: we preserve the mask (e.g. indication of primary, etc..) */ 3522 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK; 3523 metaslab_group_sort(msp->ms_group, msp, 3524 metaslab_weight(msp, B_FALSE) | was_active); 3525 } 3526 3527 static int 3528 metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp, 3529 int allocator, uint64_t activation_weight) 3530 { 3531 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 3532 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3533 3534 /* 3535 * If we're activating for the claim code, we don't want to actually 3536 * set the metaslab up for a specific allocator. 3537 */ 3538 if (activation_weight == METASLAB_WEIGHT_CLAIM) { 3539 ASSERT0(msp->ms_activation_weight); 3540 msp->ms_activation_weight = msp->ms_weight; 3541 metaslab_group_sort(mg, msp, msp->ms_weight | 3542 activation_weight); 3543 return (0); 3544 } 3545 3546 metaslab_t **mspp = (activation_weight == METASLAB_WEIGHT_PRIMARY ? 3547 &mga->mga_primary : &mga->mga_secondary); 3548 3549 mutex_enter(&mg->mg_lock); 3550 if (*mspp != NULL) { 3551 mutex_exit(&mg->mg_lock); 3552 return (EEXIST); 3553 } 3554 3555 *mspp = msp; 3556 ASSERT3S(msp->ms_allocator, ==, -1); 3557 msp->ms_allocator = allocator; 3558 msp->ms_primary = (activation_weight == METASLAB_WEIGHT_PRIMARY); 3559 3560 ASSERT0(msp->ms_activation_weight); 3561 msp->ms_activation_weight = msp->ms_weight; 3562 metaslab_group_sort_impl(mg, msp, 3563 msp->ms_weight | activation_weight); 3564 mutex_exit(&mg->mg_lock); 3565 3566 return (0); 3567 } 3568 3569 static int 3570 metaslab_activate(metaslab_t *msp, int allocator, uint64_t activation_weight) 3571 { 3572 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3573 3574 /* 3575 * The current metaslab is already activated for us so there 3576 * is nothing to do. Already activated though, doesn't mean 3577 * that this metaslab is activated for our allocator nor our 3578 * requested activation weight. The metaslab could have started 3579 * as an active one for our allocator but changed allocators 3580 * while we were waiting to grab its ms_lock or we stole it 3581 * [see find_valid_metaslab()]. This means that there is a 3582 * possibility of passivating a metaslab of another allocator 3583 * or from a different activation mask, from this thread. 3584 */ 3585 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) { 3586 ASSERT(msp->ms_loaded); 3587 return (0); 3588 } 3589 3590 int error = metaslab_load(msp); 3591 if (error != 0) { 3592 metaslab_group_sort(msp->ms_group, msp, 0); 3593 return (error); 3594 } 3595 3596 /* 3597 * When entering metaslab_load() we may have dropped the 3598 * ms_lock because we were loading this metaslab, or we 3599 * were waiting for another thread to load it for us. In 3600 * that scenario, we recheck the weight of the metaslab 3601 * to see if it was activated by another thread. 3602 * 3603 * If the metaslab was activated for another allocator or 3604 * it was activated with a different activation weight (e.g. 3605 * we wanted to make it a primary but it was activated as 3606 * secondary) we return error (EBUSY). 3607 * 3608 * If the metaslab was activated for the same allocator 3609 * and requested activation mask, skip activating it. 3610 */ 3611 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) { 3612 if (msp->ms_allocator != allocator) 3613 return (EBUSY); 3614 3615 if ((msp->ms_weight & activation_weight) == 0) 3616 return (SET_ERROR(EBUSY)); 3617 3618 EQUIV((activation_weight == METASLAB_WEIGHT_PRIMARY), 3619 msp->ms_primary); 3620 return (0); 3621 } 3622 3623 /* 3624 * If the metaslab has literally 0 space, it will have weight 0. In 3625 * that case, don't bother activating it. This can happen if the 3626 * metaslab had space during find_valid_metaslab, but another thread 3627 * loaded it and used all that space while we were waiting to grab the 3628 * lock. 3629 */ 3630 if (msp->ms_weight == 0) { 3631 ASSERT0(zfs_range_tree_space(msp->ms_allocatable)); 3632 return (SET_ERROR(ENOSPC)); 3633 } 3634 3635 if ((error = metaslab_activate_allocator(msp->ms_group, msp, 3636 allocator, activation_weight)) != 0) { 3637 return (error); 3638 } 3639 3640 ASSERT(msp->ms_loaded); 3641 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 3642 3643 return (0); 3644 } 3645 3646 static void 3647 metaslab_passivate_allocator(metaslab_group_t *mg, metaslab_t *msp, 3648 uint64_t weight) 3649 { 3650 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3651 ASSERT(msp->ms_loaded); 3652 3653 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) { 3654 metaslab_group_sort(mg, msp, weight); 3655 return; 3656 } 3657 3658 mutex_enter(&mg->mg_lock); 3659 ASSERT3P(msp->ms_group, ==, mg); 3660 ASSERT3S(0, <=, msp->ms_allocator); 3661 ASSERT3U(msp->ms_allocator, <, mg->mg_class->mc_spa->spa_alloc_count); 3662 3663 metaslab_group_allocator_t *mga = &mg->mg_allocator[msp->ms_allocator]; 3664 if (msp->ms_primary) { 3665 ASSERT3P(mga->mga_primary, ==, msp); 3666 ASSERT(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); 3667 mga->mga_primary = NULL; 3668 } else { 3669 ASSERT3P(mga->mga_secondary, ==, msp); 3670 ASSERT(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); 3671 mga->mga_secondary = NULL; 3672 } 3673 msp->ms_allocator = -1; 3674 metaslab_group_sort_impl(mg, msp, weight); 3675 mutex_exit(&mg->mg_lock); 3676 } 3677 3678 static void 3679 metaslab_passivate(metaslab_t *msp, uint64_t weight) 3680 { 3681 uint64_t size __maybe_unused = weight & ~METASLAB_WEIGHT_TYPE; 3682 3683 /* 3684 * If size < SPA_MINBLOCKSIZE, then we will not allocate from 3685 * this metaslab again. In that case, it had better be empty, 3686 * or we would be leaving space on the table. 3687 */ 3688 ASSERT(!WEIGHT_IS_SPACEBASED(msp->ms_weight) || 3689 size >= SPA_MINBLOCKSIZE || 3690 zfs_range_tree_space(msp->ms_allocatable) == 0); 3691 ASSERT0(weight & METASLAB_ACTIVE_MASK); 3692 3693 ASSERT(msp->ms_activation_weight != 0); 3694 msp->ms_activation_weight = 0; 3695 metaslab_passivate_allocator(msp->ms_group, msp, weight); 3696 ASSERT0(msp->ms_weight & METASLAB_ACTIVE_MASK); 3697 } 3698 3699 /* 3700 * Segment-based metaslabs are activated once and remain active until 3701 * we either fail an allocation attempt (similar to space-based metaslabs) 3702 * or have exhausted the free space in zfs_metaslab_switch_threshold 3703 * buckets since the metaslab was activated. This function checks to see 3704 * if we've exhausted the zfs_metaslab_switch_threshold buckets in the 3705 * metaslab and passivates it proactively. This will allow us to select a 3706 * metaslab with a larger contiguous region, if any, remaining within this 3707 * metaslab group. If we're in sync pass > 1, then we continue using this 3708 * metaslab so that we don't dirty more block and cause more sync passes. 3709 */ 3710 static void 3711 metaslab_segment_may_passivate(metaslab_t *msp) 3712 { 3713 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 3714 3715 if (WEIGHT_IS_SPACEBASED(msp->ms_weight) || spa_sync_pass(spa) > 1) 3716 return; 3717 3718 /* 3719 * As long as a single largest free segment covers majorioty of free 3720 * space, don't consider the metaslab fragmented. It should allow 3721 * us to fill new unfragmented metaslabs full before switching. 3722 */ 3723 if (metaslab_largest_allocatable(msp) > 3724 zfs_range_tree_space(msp->ms_allocatable) * 15 / 16) 3725 return; 3726 3727 /* 3728 * Since we are in the middle of a sync pass, the most accurate 3729 * information that is accessible to us is the in-core range tree 3730 * histogram; calculate the new weight based on that information. 3731 */ 3732 uint64_t weight = metaslab_weight_from_range_tree(msp); 3733 int activation_idx = WEIGHT_GET_INDEX(msp->ms_activation_weight); 3734 int current_idx = WEIGHT_GET_INDEX(weight); 3735 3736 if (current_idx <= activation_idx - zfs_metaslab_switch_threshold) 3737 metaslab_passivate(msp, weight); 3738 } 3739 3740 static void 3741 metaslab_preload(void *arg) 3742 { 3743 metaslab_t *msp = arg; 3744 metaslab_class_t *mc = msp->ms_group->mg_class; 3745 spa_t *spa = mc->mc_spa; 3746 fstrans_cookie_t cookie = spl_fstrans_mark(); 3747 3748 ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock)); 3749 3750 mutex_enter(&msp->ms_lock); 3751 (void) metaslab_load(msp); 3752 metaslab_set_selected_txg(msp, spa_syncing_txg(spa)); 3753 mutex_exit(&msp->ms_lock); 3754 spl_fstrans_unmark(cookie); 3755 } 3756 3757 static void 3758 metaslab_group_preload(metaslab_group_t *mg) 3759 { 3760 spa_t *spa = mg->mg_vd->vdev_spa; 3761 metaslab_t *msp; 3762 avl_tree_t *t = &mg->mg_metaslab_tree; 3763 int m = 0; 3764 3765 if (spa_shutting_down(spa) || !metaslab_preload_enabled) 3766 return; 3767 3768 mutex_enter(&mg->mg_lock); 3769 3770 /* 3771 * Load the next potential metaslabs 3772 */ 3773 for (msp = avl_first(t); msp != NULL; msp = AVL_NEXT(t, msp)) { 3774 ASSERT3P(msp->ms_group, ==, mg); 3775 3776 /* 3777 * We preload only the maximum number of metaslabs specified 3778 * by metaslab_preload_limit. If a metaslab is being forced 3779 * to condense then we preload it too. This will ensure 3780 * that force condensing happens in the next txg. 3781 */ 3782 if (++m > metaslab_preload_limit && !msp->ms_condense_wanted) { 3783 continue; 3784 } 3785 3786 VERIFY(taskq_dispatch(spa->spa_metaslab_taskq, metaslab_preload, 3787 msp, TQ_SLEEP | (m <= spa->spa_alloc_count ? TQ_FRONT : 0)) 3788 != TASKQID_INVALID); 3789 } 3790 mutex_exit(&mg->mg_lock); 3791 } 3792 3793 /* 3794 * Determine if the space map's on-disk footprint is past our tolerance for 3795 * inefficiency. We would like to use the following criteria to make our 3796 * decision: 3797 * 3798 * 1. Do not condense if the size of the space map object would dramatically 3799 * increase as a result of writing out the free space range tree. 3800 * 3801 * 2. Condense if the on on-disk space map representation is at least 3802 * zfs_condense_pct/100 times the size of the optimal representation 3803 * (i.e. zfs_condense_pct = 110 and in-core = 1MB, optimal = 1.1MB). 3804 * 3805 * 3. Do not condense if the on-disk size of the space map does not actually 3806 * decrease. 3807 * 3808 * Unfortunately, we cannot compute the on-disk size of the space map in this 3809 * context because we cannot accurately compute the effects of compression, etc. 3810 * Instead, we apply the heuristic described in the block comment for 3811 * zfs_metaslab_condense_block_threshold - we only condense if the space used 3812 * is greater than a threshold number of blocks. 3813 */ 3814 static boolean_t 3815 metaslab_should_condense(metaslab_t *msp) 3816 { 3817 space_map_t *sm = msp->ms_sm; 3818 vdev_t *vd = msp->ms_group->mg_vd; 3819 uint64_t vdev_blocksize = 1ULL << vd->vdev_ashift; 3820 3821 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3822 ASSERT(msp->ms_loaded); 3823 ASSERT(sm != NULL); 3824 ASSERT3U(spa_sync_pass(vd->vdev_spa), ==, 1); 3825 3826 /* 3827 * We always condense metaslabs that are empty and metaslabs for 3828 * which a condense request has been made. 3829 */ 3830 if (zfs_range_tree_numsegs(msp->ms_allocatable) == 0 || 3831 msp->ms_condense_wanted) 3832 return (B_TRUE); 3833 3834 uint64_t record_size = MAX(sm->sm_blksz, vdev_blocksize); 3835 uint64_t object_size = space_map_length(sm); 3836 uint64_t optimal_size = space_map_estimate_optimal_size(sm, 3837 msp->ms_allocatable, SM_NO_VDEVID); 3838 3839 return (object_size >= (optimal_size * zfs_condense_pct / 100) && 3840 object_size > zfs_metaslab_condense_block_threshold * record_size); 3841 } 3842 3843 /* 3844 * Condense the on-disk space map representation to its minimized form. 3845 * The minimized form consists of a small number of allocations followed 3846 * by the entries of the free range tree (ms_allocatable). The condensed 3847 * spacemap contains all the entries of previous TXGs (including those in 3848 * the pool-wide log spacemaps; thus this is effectively a superset of 3849 * metaslab_flush()), but this TXG's entries still need to be written. 3850 */ 3851 static void 3852 metaslab_condense(metaslab_t *msp, dmu_tx_t *tx) 3853 { 3854 zfs_range_tree_t *condense_tree; 3855 space_map_t *sm = msp->ms_sm; 3856 uint64_t txg = dmu_tx_get_txg(tx); 3857 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 3858 3859 ASSERT(MUTEX_HELD(&msp->ms_lock)); 3860 ASSERT(msp->ms_loaded); 3861 ASSERT(msp->ms_sm != NULL); 3862 3863 /* 3864 * In order to condense the space map, we need to change it so it 3865 * only describes which segments are currently allocated and free. 3866 * 3867 * All the current free space resides in the ms_allocatable, all 3868 * the ms_defer trees, and all the ms_allocating trees. We ignore 3869 * ms_freed because it is empty because we're in sync pass 1. We 3870 * ignore ms_freeing because these changes are not yet reflected 3871 * in the spacemap (they will be written later this txg). 3872 * 3873 * So to truncate the space map to represent all the entries of 3874 * previous TXGs we do the following: 3875 * 3876 * 1] We create a range tree (condense tree) that is 100% empty. 3877 * 2] We add to it all segments found in the ms_defer trees 3878 * as those segments are marked as free in the original space 3879 * map. We do the same with the ms_allocating trees for the same 3880 * reason. Adding these segments should be a relatively 3881 * inexpensive operation since we expect these trees to have a 3882 * small number of nodes. 3883 * 3] We vacate any unflushed allocs, since they are not frees we 3884 * need to add to the condense tree. Then we vacate any 3885 * unflushed frees as they should already be part of ms_allocatable. 3886 * 4] At this point, we would ideally like to add all segments 3887 * in the ms_allocatable tree from the condense tree. This way 3888 * we would write all the entries of the condense tree as the 3889 * condensed space map, which would only contain freed 3890 * segments with everything else assumed to be allocated. 3891 * 3892 * Doing so can be prohibitively expensive as ms_allocatable can 3893 * be large, and therefore computationally expensive to add to 3894 * the condense_tree. Instead we first sync out an entry marking 3895 * everything as allocated, then the condense_tree and then the 3896 * ms_allocatable, in the condensed space map. While this is not 3897 * optimal, it is typically close to optimal and more importantly 3898 * much cheaper to compute. 3899 * 3900 * 5] Finally, as both of the unflushed trees were written to our 3901 * new and condensed metaslab space map, we basically flushed 3902 * all the unflushed changes to disk, thus we call 3903 * metaslab_flush_update(). 3904 */ 3905 ASSERT3U(spa_sync_pass(spa), ==, 1); 3906 ASSERT(zfs_range_tree_is_empty(msp->ms_freed)); /* since it is pass 1 */ 3907 3908 zfs_dbgmsg("condensing: txg %llu, msp[%llu] %px, vdev id %llu, " 3909 "spa %s, smp size %llu, segments %llu, forcing condense=%s", 3910 (u_longlong_t)txg, (u_longlong_t)msp->ms_id, msp, 3911 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 3912 spa->spa_name, (u_longlong_t)space_map_length(msp->ms_sm), 3913 (u_longlong_t)zfs_range_tree_numsegs(msp->ms_allocatable), 3914 msp->ms_condense_wanted ? "TRUE" : "FALSE"); 3915 3916 msp->ms_condense_wanted = B_FALSE; 3917 3918 zfs_range_seg_type_t type; 3919 uint64_t shift, start; 3920 type = metaslab_calculate_range_tree_type(msp->ms_group->mg_vd, msp, 3921 &start, &shift); 3922 3923 condense_tree = zfs_range_tree_create_flags( 3924 NULL, type, NULL, start, shift, 3925 ZFS_RT_F_DYN_NAME, 3926 metaslab_rt_name(msp->ms_group, msp, "condense_tree")); 3927 3928 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 3929 zfs_range_tree_walk(msp->ms_defer[t], 3930 zfs_range_tree_add, condense_tree); 3931 } 3932 3933 for (int t = 0; t < TXG_CONCURRENT_STATES; t++) { 3934 zfs_range_tree_walk(msp->ms_allocating[(txg + t) & TXG_MASK], 3935 zfs_range_tree_add, condense_tree); 3936 } 3937 3938 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 3939 metaslab_unflushed_changes_memused(msp)); 3940 spa->spa_unflushed_stats.sus_memused -= 3941 metaslab_unflushed_changes_memused(msp); 3942 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); 3943 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); 3944 3945 /* 3946 * We're about to drop the metaslab's lock thus allowing other 3947 * consumers to change it's content. Set the metaslab's ms_condensing 3948 * flag to ensure that allocations on this metaslab do not occur 3949 * while we're in the middle of committing it to disk. This is only 3950 * critical for ms_allocatable as all other range trees use per TXG 3951 * views of their content. 3952 */ 3953 msp->ms_condensing = B_TRUE; 3954 3955 mutex_exit(&msp->ms_lock); 3956 uint64_t object = space_map_object(msp->ms_sm); 3957 space_map_truncate(sm, 3958 spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ? 3959 zfs_metaslab_sm_blksz_with_log : zfs_metaslab_sm_blksz_no_log, tx); 3960 3961 /* 3962 * space_map_truncate() may have reallocated the spacemap object. 3963 * If so, update the vdev_ms_array. 3964 */ 3965 if (space_map_object(msp->ms_sm) != object) { 3966 object = space_map_object(msp->ms_sm); 3967 dmu_write(spa->spa_meta_objset, 3968 msp->ms_group->mg_vd->vdev_ms_array, sizeof (uint64_t) * 3969 msp->ms_id, sizeof (uint64_t), &object, tx); 3970 } 3971 3972 /* 3973 * Note: 3974 * When the log space map feature is enabled, each space map will 3975 * always have ALLOCS followed by FREES for each sync pass. This is 3976 * typically true even when the log space map feature is disabled, 3977 * except from the case where a metaslab goes through metaslab_sync() 3978 * and gets condensed. In that case the metaslab's space map will have 3979 * ALLOCS followed by FREES (due to condensing) followed by ALLOCS 3980 * followed by FREES (due to space_map_write() in metaslab_sync()) for 3981 * sync pass 1. 3982 */ 3983 zfs_range_tree_t *tmp_tree = zfs_range_tree_create_flags( 3984 NULL, type, NULL, start, shift, 3985 ZFS_RT_F_DYN_NAME, 3986 metaslab_rt_name(msp->ms_group, msp, "tmp_tree")); 3987 zfs_range_tree_add(tmp_tree, msp->ms_start, msp->ms_size); 3988 space_map_write(sm, tmp_tree, SM_ALLOC, SM_NO_VDEVID, tx); 3989 space_map_write(sm, msp->ms_allocatable, SM_FREE, SM_NO_VDEVID, tx); 3990 space_map_write(sm, condense_tree, SM_FREE, SM_NO_VDEVID, tx); 3991 3992 zfs_range_tree_vacate(condense_tree, NULL, NULL); 3993 zfs_range_tree_destroy(condense_tree); 3994 zfs_range_tree_vacate(tmp_tree, NULL, NULL); 3995 zfs_range_tree_destroy(tmp_tree); 3996 mutex_enter(&msp->ms_lock); 3997 3998 msp->ms_condensing = B_FALSE; 3999 metaslab_flush_update(msp, tx); 4000 } 4001 4002 static void 4003 metaslab_unflushed_add(metaslab_t *msp, dmu_tx_t *tx) 4004 { 4005 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 4006 ASSERT(spa_syncing_log_sm(spa) != NULL); 4007 ASSERT(msp->ms_sm != NULL); 4008 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); 4009 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); 4010 4011 mutex_enter(&spa->spa_flushed_ms_lock); 4012 metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx); 4013 metaslab_set_unflushed_dirty(msp, B_TRUE); 4014 avl_add(&spa->spa_metaslabs_by_flushed, msp); 4015 mutex_exit(&spa->spa_flushed_ms_lock); 4016 4017 spa_log_sm_increment_current_mscount(spa); 4018 spa_log_summary_add_flushed_metaslab(spa, B_TRUE); 4019 } 4020 4021 void 4022 metaslab_unflushed_bump(metaslab_t *msp, dmu_tx_t *tx, boolean_t dirty) 4023 { 4024 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 4025 ASSERT(spa_syncing_log_sm(spa) != NULL); 4026 ASSERT(msp->ms_sm != NULL); 4027 ASSERT(metaslab_unflushed_txg(msp) != 0); 4028 ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), ==, msp); 4029 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); 4030 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); 4031 4032 VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(spa)); 4033 4034 /* update metaslab's position in our flushing tree */ 4035 uint64_t ms_prev_flushed_txg = metaslab_unflushed_txg(msp); 4036 boolean_t ms_prev_flushed_dirty = metaslab_unflushed_dirty(msp); 4037 mutex_enter(&spa->spa_flushed_ms_lock); 4038 avl_remove(&spa->spa_metaslabs_by_flushed, msp); 4039 metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx); 4040 metaslab_set_unflushed_dirty(msp, dirty); 4041 avl_add(&spa->spa_metaslabs_by_flushed, msp); 4042 mutex_exit(&spa->spa_flushed_ms_lock); 4043 4044 /* update metaslab counts of spa_log_sm_t nodes */ 4045 spa_log_sm_decrement_mscount(spa, ms_prev_flushed_txg); 4046 spa_log_sm_increment_current_mscount(spa); 4047 4048 /* update log space map summary */ 4049 spa_log_summary_decrement_mscount(spa, ms_prev_flushed_txg, 4050 ms_prev_flushed_dirty); 4051 spa_log_summary_add_flushed_metaslab(spa, dirty); 4052 4053 /* cleanup obsolete logs if any */ 4054 spa_cleanup_old_sm_logs(spa, tx); 4055 } 4056 4057 /* 4058 * Called when the metaslab has been flushed (its own spacemap now reflects 4059 * all the contents of the pool-wide spacemap log). Updates the metaslab's 4060 * metadata and any pool-wide related log space map data (e.g. summary, 4061 * obsolete logs, etc..) to reflect that. 4062 */ 4063 static void 4064 metaslab_flush_update(metaslab_t *msp, dmu_tx_t *tx) 4065 { 4066 metaslab_group_t *mg = msp->ms_group; 4067 spa_t *spa = mg->mg_vd->vdev_spa; 4068 4069 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4070 4071 ASSERT3U(spa_sync_pass(spa), ==, 1); 4072 4073 /* 4074 * Just because a metaslab got flushed, that doesn't mean that 4075 * it will pass through metaslab_sync_done(). Thus, make sure to 4076 * update ms_synced_length here in case it doesn't. 4077 */ 4078 msp->ms_synced_length = space_map_length(msp->ms_sm); 4079 4080 /* 4081 * We may end up here from metaslab_condense() without the 4082 * feature being active. In that case this is a no-op. 4083 */ 4084 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP) || 4085 metaslab_unflushed_txg(msp) == 0) 4086 return; 4087 4088 metaslab_unflushed_bump(msp, tx, B_FALSE); 4089 } 4090 4091 boolean_t 4092 metaslab_flush(metaslab_t *msp, dmu_tx_t *tx) 4093 { 4094 spa_t *spa = msp->ms_group->mg_vd->vdev_spa; 4095 4096 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4097 ASSERT3U(spa_sync_pass(spa), ==, 1); 4098 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 4099 4100 ASSERT(msp->ms_sm != NULL); 4101 ASSERT(metaslab_unflushed_txg(msp) != 0); 4102 ASSERT(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL) != NULL); 4103 4104 /* 4105 * There is nothing wrong with flushing the same metaslab twice, as 4106 * this codepath should work on that case. However, the current 4107 * flushing scheme makes sure to avoid this situation as we would be 4108 * making all these calls without having anything meaningful to write 4109 * to disk. We assert this behavior here. 4110 */ 4111 ASSERT3U(metaslab_unflushed_txg(msp), <, dmu_tx_get_txg(tx)); 4112 4113 /* 4114 * We can not flush while loading, because then we would 4115 * not load the ms_unflushed_{allocs,frees}. 4116 */ 4117 if (msp->ms_loading) 4118 return (B_FALSE); 4119 4120 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4121 metaslab_verify_weight_and_frag(msp); 4122 4123 /* 4124 * Metaslab condensing is effectively flushing. Therefore if the 4125 * metaslab can be condensed we can just condense it instead of 4126 * flushing it. 4127 * 4128 * Note that metaslab_condense() does call metaslab_flush_update() 4129 * so we can just return immediately after condensing. We also 4130 * don't need to care about setting ms_flushing or broadcasting 4131 * ms_flush_cv, even if we temporarily drop the ms_lock in 4132 * metaslab_condense(), as the metaslab is already loaded. 4133 */ 4134 if (msp->ms_loaded && metaslab_should_condense(msp)) { 4135 metaslab_group_t *mg = msp->ms_group; 4136 4137 /* 4138 * For all histogram operations below refer to the 4139 * comments of metaslab_sync() where we follow a 4140 * similar procedure. 4141 */ 4142 metaslab_group_histogram_verify(mg); 4143 metaslab_class_histogram_verify(mg->mg_class); 4144 metaslab_group_histogram_remove(mg, msp); 4145 4146 metaslab_condense(msp, tx); 4147 4148 space_map_histogram_clear(msp->ms_sm); 4149 space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx); 4150 ASSERT(zfs_range_tree_is_empty(msp->ms_freed)); 4151 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 4152 space_map_histogram_add(msp->ms_sm, 4153 msp->ms_defer[t], tx); 4154 } 4155 metaslab_aux_histograms_update(msp); 4156 4157 metaslab_group_histogram_add(mg, msp); 4158 metaslab_group_histogram_verify(mg); 4159 metaslab_class_histogram_verify(mg->mg_class); 4160 4161 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4162 4163 /* 4164 * Since we recreated the histogram (and potentially 4165 * the ms_sm too while condensing) ensure that the 4166 * weight is updated too because we are not guaranteed 4167 * that this metaslab is dirty and will go through 4168 * metaslab_sync_done(). 4169 */ 4170 metaslab_recalculate_weight_and_sort(msp); 4171 return (B_TRUE); 4172 } 4173 4174 msp->ms_flushing = B_TRUE; 4175 uint64_t sm_len_before = space_map_length(msp->ms_sm); 4176 4177 mutex_exit(&msp->ms_lock); 4178 space_map_write(msp->ms_sm, msp->ms_unflushed_allocs, SM_ALLOC, 4179 SM_NO_VDEVID, tx); 4180 space_map_write(msp->ms_sm, msp->ms_unflushed_frees, SM_FREE, 4181 SM_NO_VDEVID, tx); 4182 mutex_enter(&msp->ms_lock); 4183 4184 uint64_t sm_len_after = space_map_length(msp->ms_sm); 4185 if (zfs_flags & ZFS_DEBUG_LOG_SPACEMAP) { 4186 zfs_dbgmsg("flushing: txg %llu, spa %s, vdev_id %llu, " 4187 "ms_id %llu, unflushed_allocs %llu, unflushed_frees %llu, " 4188 "appended %llu bytes", (u_longlong_t)dmu_tx_get_txg(tx), 4189 spa_name(spa), 4190 (u_longlong_t)msp->ms_group->mg_vd->vdev_id, 4191 (u_longlong_t)msp->ms_id, 4192 (u_longlong_t)zfs_range_tree_space( 4193 msp->ms_unflushed_allocs), 4194 (u_longlong_t)zfs_range_tree_space( 4195 msp->ms_unflushed_frees), 4196 (u_longlong_t)(sm_len_after - sm_len_before)); 4197 } 4198 4199 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 4200 metaslab_unflushed_changes_memused(msp)); 4201 spa->spa_unflushed_stats.sus_memused -= 4202 metaslab_unflushed_changes_memused(msp); 4203 zfs_range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL); 4204 zfs_range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL); 4205 4206 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4207 metaslab_verify_weight_and_frag(msp); 4208 4209 metaslab_flush_update(msp, tx); 4210 4211 metaslab_verify_space(msp, dmu_tx_get_txg(tx)); 4212 metaslab_verify_weight_and_frag(msp); 4213 4214 msp->ms_flushing = B_FALSE; 4215 cv_broadcast(&msp->ms_flush_cv); 4216 return (B_TRUE); 4217 } 4218 4219 /* 4220 * Write a metaslab to disk in the context of the specified transaction group. 4221 */ 4222 void 4223 metaslab_sync(metaslab_t *msp, uint64_t txg) 4224 { 4225 metaslab_group_t *mg = msp->ms_group; 4226 vdev_t *vd = mg->mg_vd; 4227 spa_t *spa = vd->vdev_spa; 4228 objset_t *mos = spa_meta_objset(spa); 4229 zfs_range_tree_t *alloctree = msp->ms_allocating[txg & TXG_MASK]; 4230 dmu_tx_t *tx; 4231 4232 ASSERT(!vd->vdev_ishole); 4233 4234 /* 4235 * This metaslab has just been added so there's no work to do now. 4236 */ 4237 if (msp->ms_new) { 4238 ASSERT0(zfs_range_tree_space(alloctree)); 4239 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4240 ASSERT0(zfs_range_tree_space(msp->ms_freed)); 4241 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4242 ASSERT0(zfs_range_tree_space(msp->ms_trim)); 4243 return; 4244 } 4245 4246 /* 4247 * Normally, we don't want to process a metaslab if there are no 4248 * allocations or frees to perform. However, if the metaslab is being 4249 * forced to condense, it's loaded and we're not beyond the final 4250 * dirty txg, we need to let it through. Not condensing beyond the 4251 * final dirty txg prevents an issue where metaslabs that need to be 4252 * condensed but were loaded for other reasons could cause a panic 4253 * here. By only checking the txg in that branch of the conditional, 4254 * we preserve the utility of the VERIFY statements in all other 4255 * cases. 4256 */ 4257 if (zfs_range_tree_is_empty(alloctree) && 4258 zfs_range_tree_is_empty(msp->ms_freeing) && 4259 zfs_range_tree_is_empty(msp->ms_checkpointing) && 4260 !(msp->ms_loaded && msp->ms_condense_wanted && 4261 txg <= spa_final_dirty_txg(spa))) 4262 return; 4263 4264 4265 VERIFY3U(txg, <=, spa_final_dirty_txg(spa)); 4266 4267 /* 4268 * The only state that can actually be changing concurrently 4269 * with metaslab_sync() is the metaslab's ms_allocatable. No 4270 * other thread can be modifying this txg's alloc, freeing, 4271 * freed, or space_map_phys_t. We drop ms_lock whenever we 4272 * could call into the DMU, because the DMU can call down to 4273 * us (e.g. via zio_free()) at any time. 4274 * 4275 * The spa_vdev_remove_thread() can be reading metaslab state 4276 * concurrently, and it is locked out by the ms_sync_lock. 4277 * Note that the ms_lock is insufficient for this, because it 4278 * is dropped by space_map_write(). 4279 */ 4280 tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg); 4281 4282 /* 4283 * Generate a log space map if one doesn't exist already. 4284 */ 4285 spa_generate_syncing_log_sm(spa, tx); 4286 4287 if (msp->ms_sm == NULL) { 4288 uint64_t new_object = space_map_alloc(mos, 4289 spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ? 4290 zfs_metaslab_sm_blksz_with_log : 4291 zfs_metaslab_sm_blksz_no_log, tx); 4292 VERIFY3U(new_object, !=, 0); 4293 4294 dmu_write(mos, vd->vdev_ms_array, sizeof (uint64_t) * 4295 msp->ms_id, sizeof (uint64_t), &new_object, tx); 4296 4297 VERIFY0(space_map_open(&msp->ms_sm, mos, new_object, 4298 msp->ms_start, msp->ms_size, vd->vdev_ashift)); 4299 ASSERT(msp->ms_sm != NULL); 4300 4301 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_allocs)); 4302 ASSERT(zfs_range_tree_is_empty(msp->ms_unflushed_frees)); 4303 ASSERT0(metaslab_allocated_space(msp)); 4304 } 4305 4306 if (!zfs_range_tree_is_empty(msp->ms_checkpointing) && 4307 vd->vdev_checkpoint_sm == NULL) { 4308 ASSERT(spa_has_checkpoint(spa)); 4309 4310 uint64_t new_object = space_map_alloc(mos, 4311 zfs_vdev_standard_sm_blksz, tx); 4312 VERIFY3U(new_object, !=, 0); 4313 4314 VERIFY0(space_map_open(&vd->vdev_checkpoint_sm, 4315 mos, new_object, 0, vd->vdev_asize, vd->vdev_ashift)); 4316 ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL); 4317 4318 /* 4319 * We save the space map object as an entry in vdev_top_zap 4320 * so it can be retrieved when the pool is reopened after an 4321 * export or through zdb. 4322 */ 4323 VERIFY0(zap_add(vd->vdev_spa->spa_meta_objset, 4324 vd->vdev_top_zap, VDEV_TOP_ZAP_POOL_CHECKPOINT_SM, 4325 sizeof (new_object), 1, &new_object, tx)); 4326 } 4327 4328 mutex_enter(&msp->ms_sync_lock); 4329 mutex_enter(&msp->ms_lock); 4330 4331 /* 4332 * Note: metaslab_condense() clears the space map's histogram. 4333 * Therefore we must verify and remove this histogram before 4334 * condensing. 4335 */ 4336 metaslab_group_histogram_verify(mg); 4337 metaslab_class_histogram_verify(mg->mg_class); 4338 metaslab_group_histogram_remove(mg, msp); 4339 4340 if (spa->spa_sync_pass == 1 && msp->ms_loaded && 4341 metaslab_should_condense(msp)) 4342 metaslab_condense(msp, tx); 4343 4344 /* 4345 * We'll be going to disk to sync our space accounting, thus we 4346 * drop the ms_lock during that time so allocations coming from 4347 * open-context (ZIL) for future TXGs do not block. 4348 */ 4349 mutex_exit(&msp->ms_lock); 4350 space_map_t *log_sm = spa_syncing_log_sm(spa); 4351 if (log_sm != NULL) { 4352 ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP)); 4353 if (metaslab_unflushed_txg(msp) == 0) 4354 metaslab_unflushed_add(msp, tx); 4355 else if (!metaslab_unflushed_dirty(msp)) 4356 metaslab_unflushed_bump(msp, tx, B_TRUE); 4357 4358 space_map_write(log_sm, alloctree, SM_ALLOC, 4359 vd->vdev_id, tx); 4360 space_map_write(log_sm, msp->ms_freeing, SM_FREE, 4361 vd->vdev_id, tx); 4362 mutex_enter(&msp->ms_lock); 4363 4364 ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=, 4365 metaslab_unflushed_changes_memused(msp)); 4366 spa->spa_unflushed_stats.sus_memused -= 4367 metaslab_unflushed_changes_memused(msp); 4368 zfs_range_tree_remove_xor_add(alloctree, 4369 msp->ms_unflushed_frees, msp->ms_unflushed_allocs); 4370 zfs_range_tree_remove_xor_add(msp->ms_freeing, 4371 msp->ms_unflushed_allocs, msp->ms_unflushed_frees); 4372 spa->spa_unflushed_stats.sus_memused += 4373 metaslab_unflushed_changes_memused(msp); 4374 } else { 4375 ASSERT(!spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP)); 4376 4377 space_map_write(msp->ms_sm, alloctree, SM_ALLOC, 4378 SM_NO_VDEVID, tx); 4379 space_map_write(msp->ms_sm, msp->ms_freeing, SM_FREE, 4380 SM_NO_VDEVID, tx); 4381 mutex_enter(&msp->ms_lock); 4382 } 4383 4384 msp->ms_allocated_space += zfs_range_tree_space(alloctree); 4385 ASSERT3U(msp->ms_allocated_space, >=, 4386 zfs_range_tree_space(msp->ms_freeing)); 4387 msp->ms_allocated_space -= zfs_range_tree_space(msp->ms_freeing); 4388 4389 if (!zfs_range_tree_is_empty(msp->ms_checkpointing)) { 4390 ASSERT(spa_has_checkpoint(spa)); 4391 ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL); 4392 4393 /* 4394 * Since we are doing writes to disk and the ms_checkpointing 4395 * tree won't be changing during that time, we drop the 4396 * ms_lock while writing to the checkpoint space map, for the 4397 * same reason mentioned above. 4398 */ 4399 mutex_exit(&msp->ms_lock); 4400 space_map_write(vd->vdev_checkpoint_sm, 4401 msp->ms_checkpointing, SM_FREE, SM_NO_VDEVID, tx); 4402 mutex_enter(&msp->ms_lock); 4403 4404 spa->spa_checkpoint_info.sci_dspace += 4405 zfs_range_tree_space(msp->ms_checkpointing); 4406 vd->vdev_stat.vs_checkpoint_space += 4407 zfs_range_tree_space(msp->ms_checkpointing); 4408 ASSERT3U(vd->vdev_stat.vs_checkpoint_space, ==, 4409 -space_map_allocated(vd->vdev_checkpoint_sm)); 4410 4411 zfs_range_tree_vacate(msp->ms_checkpointing, NULL, NULL); 4412 } 4413 4414 if (msp->ms_loaded) { 4415 /* 4416 * When the space map is loaded, we have an accurate 4417 * histogram in the range tree. This gives us an opportunity 4418 * to bring the space map's histogram up-to-date so we clear 4419 * it first before updating it. 4420 */ 4421 space_map_histogram_clear(msp->ms_sm); 4422 space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx); 4423 4424 /* 4425 * Since we've cleared the histogram we need to add back 4426 * any free space that has already been processed, plus 4427 * any deferred space. This allows the on-disk histogram 4428 * to accurately reflect all free space even if some space 4429 * is not yet available for allocation (i.e. deferred). 4430 */ 4431 space_map_histogram_add(msp->ms_sm, msp->ms_freed, tx); 4432 4433 /* 4434 * Add back any deferred free space that has not been 4435 * added back into the in-core free tree yet. This will 4436 * ensure that we don't end up with a space map histogram 4437 * that is completely empty unless the metaslab is fully 4438 * allocated. 4439 */ 4440 for (int t = 0; t < TXG_DEFER_SIZE; t++) { 4441 space_map_histogram_add(msp->ms_sm, 4442 msp->ms_defer[t], tx); 4443 } 4444 } 4445 4446 /* 4447 * Always add the free space from this sync pass to the space 4448 * map histogram. We want to make sure that the on-disk histogram 4449 * accounts for all free space. If the space map is not loaded, 4450 * then we will lose some accuracy but will correct it the next 4451 * time we load the space map. 4452 */ 4453 space_map_histogram_add(msp->ms_sm, msp->ms_freeing, tx); 4454 metaslab_aux_histograms_update(msp); 4455 4456 metaslab_group_histogram_add(mg, msp); 4457 metaslab_group_histogram_verify(mg); 4458 metaslab_class_histogram_verify(mg->mg_class); 4459 4460 /* 4461 * For sync pass 1, we avoid traversing this txg's free range tree 4462 * and instead will just swap the pointers for freeing and freed. 4463 * We can safely do this since the freed_tree is guaranteed to be 4464 * empty on the initial pass. 4465 * 4466 * Keep in mind that even if we are currently using a log spacemap 4467 * we want current frees to end up in the ms_allocatable (but not 4468 * get appended to the ms_sm) so their ranges can be reused as usual. 4469 */ 4470 if (spa_sync_pass(spa) == 1) { 4471 zfs_range_tree_swap(&msp->ms_freeing, &msp->ms_freed); 4472 ASSERT0(msp->ms_allocated_this_txg); 4473 } else { 4474 zfs_range_tree_vacate(msp->ms_freeing, 4475 zfs_range_tree_add, msp->ms_freed); 4476 } 4477 msp->ms_allocated_this_txg += zfs_range_tree_space(alloctree); 4478 zfs_range_tree_vacate(alloctree, NULL, NULL); 4479 4480 ASSERT0(zfs_range_tree_space(msp->ms_allocating[txg & TXG_MASK])); 4481 ASSERT0(zfs_range_tree_space(msp->ms_allocating[TXG_CLEAN(txg) 4482 & TXG_MASK])); 4483 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4484 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4485 4486 mutex_exit(&msp->ms_lock); 4487 4488 /* 4489 * Verify that the space map object ID has been recorded in the 4490 * vdev_ms_array. 4491 */ 4492 uint64_t object; 4493 VERIFY0(dmu_read(mos, vd->vdev_ms_array, 4494 msp->ms_id * sizeof (uint64_t), sizeof (uint64_t), &object, 0)); 4495 VERIFY3U(object, ==, space_map_object(msp->ms_sm)); 4496 4497 mutex_exit(&msp->ms_sync_lock); 4498 dmu_tx_commit(tx); 4499 } 4500 4501 static void 4502 metaslab_evict(metaslab_t *msp, uint64_t txg) 4503 { 4504 if (!msp->ms_loaded || msp->ms_disabled != 0) 4505 return; 4506 4507 for (int t = 1; t < TXG_CONCURRENT_STATES; t++) { 4508 VERIFY0(zfs_range_tree_space( 4509 msp->ms_allocating[(txg + t) & TXG_MASK])); 4510 } 4511 if (msp->ms_allocator != -1) 4512 metaslab_passivate(msp, msp->ms_weight & ~METASLAB_ACTIVE_MASK); 4513 4514 if (!metaslab_debug_unload) 4515 metaslab_unload(msp); 4516 } 4517 4518 /* 4519 * Called after a transaction group has completely synced to mark 4520 * all of the metaslab's free space as usable. 4521 */ 4522 void 4523 metaslab_sync_done(metaslab_t *msp, uint64_t txg) 4524 { 4525 metaslab_group_t *mg = msp->ms_group; 4526 vdev_t *vd = mg->mg_vd; 4527 spa_t *spa = vd->vdev_spa; 4528 zfs_range_tree_t **defer_tree; 4529 int64_t alloc_delta, defer_delta; 4530 boolean_t defer_allowed = B_TRUE; 4531 4532 ASSERT(!vd->vdev_ishole); 4533 4534 mutex_enter(&msp->ms_lock); 4535 4536 if (msp->ms_new) { 4537 /* this is a new metaslab, add its capacity to the vdev */ 4538 metaslab_space_update(vd, mg->mg_class, 0, 0, msp->ms_size); 4539 4540 /* there should be no allocations nor frees at this point */ 4541 VERIFY0(msp->ms_allocated_this_txg); 4542 VERIFY0(zfs_range_tree_space(msp->ms_freed)); 4543 } 4544 4545 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4546 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4547 4548 defer_tree = &msp->ms_defer[txg % TXG_DEFER_SIZE]; 4549 4550 uint64_t free_space = metaslab_class_get_space(spa_normal_class(spa)) - 4551 metaslab_class_get_alloc(spa_normal_class(spa)); 4552 if (free_space <= spa_get_slop_space(spa) || vd->vdev_removing || 4553 vd->vdev_rz_expanding) { 4554 defer_allowed = B_FALSE; 4555 } 4556 4557 defer_delta = 0; 4558 alloc_delta = msp->ms_allocated_this_txg - 4559 zfs_range_tree_space(msp->ms_freed); 4560 4561 if (defer_allowed) { 4562 defer_delta = zfs_range_tree_space(msp->ms_freed) - 4563 zfs_range_tree_space(*defer_tree); 4564 } else { 4565 defer_delta -= zfs_range_tree_space(*defer_tree); 4566 } 4567 metaslab_space_update(vd, mg->mg_class, alloc_delta + defer_delta, 4568 defer_delta, 0); 4569 4570 if (spa_syncing_log_sm(spa) == NULL) { 4571 /* 4572 * If there's a metaslab_load() in progress and we don't have 4573 * a log space map, it means that we probably wrote to the 4574 * metaslab's space map. If this is the case, we need to 4575 * make sure that we wait for the load to complete so that we 4576 * have a consistent view at the in-core side of the metaslab. 4577 */ 4578 metaslab_load_wait(msp); 4579 } else { 4580 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 4581 } 4582 4583 /* 4584 * When auto-trimming is enabled, free ranges which are added to 4585 * ms_allocatable are also be added to ms_trim. The ms_trim tree is 4586 * periodically consumed by the vdev_autotrim_thread() which issues 4587 * trims for all ranges and then vacates the tree. The ms_trim tree 4588 * can be discarded at any time with the sole consequence of recent 4589 * frees not being trimmed. 4590 */ 4591 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) { 4592 zfs_range_tree_walk(*defer_tree, zfs_range_tree_add, 4593 msp->ms_trim); 4594 if (!defer_allowed) { 4595 zfs_range_tree_walk(msp->ms_freed, zfs_range_tree_add, 4596 msp->ms_trim); 4597 } 4598 } else { 4599 zfs_range_tree_vacate(msp->ms_trim, NULL, NULL); 4600 } 4601 4602 /* 4603 * Move the frees from the defer_tree back to the free 4604 * range tree (if it's loaded). Swap the freed_tree and 4605 * the defer_tree -- this is safe to do because we've 4606 * just emptied out the defer_tree. 4607 */ 4608 zfs_range_tree_vacate(*defer_tree, 4609 msp->ms_loaded ? zfs_range_tree_add : NULL, msp->ms_allocatable); 4610 if (defer_allowed) { 4611 zfs_range_tree_swap(&msp->ms_freed, defer_tree); 4612 } else { 4613 zfs_range_tree_vacate(msp->ms_freed, 4614 msp->ms_loaded ? zfs_range_tree_add : NULL, 4615 msp->ms_allocatable); 4616 } 4617 4618 msp->ms_synced_length = space_map_length(msp->ms_sm); 4619 4620 msp->ms_deferspace += defer_delta; 4621 ASSERT3S(msp->ms_deferspace, >=, 0); 4622 ASSERT3S(msp->ms_deferspace, <=, msp->ms_size); 4623 if (msp->ms_deferspace != 0) { 4624 /* 4625 * Keep syncing this metaslab until all deferred frees 4626 * are back in circulation. 4627 */ 4628 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1); 4629 } 4630 metaslab_aux_histograms_update_done(msp, defer_allowed); 4631 4632 if (msp->ms_new) { 4633 msp->ms_new = B_FALSE; 4634 mutex_enter(&mg->mg_lock); 4635 mg->mg_ms_ready++; 4636 mutex_exit(&mg->mg_lock); 4637 } 4638 4639 /* 4640 * Re-sort metaslab within its group now that we've adjusted 4641 * its allocatable space. 4642 */ 4643 metaslab_recalculate_weight_and_sort(msp); 4644 4645 ASSERT0(zfs_range_tree_space(msp->ms_allocating[txg & TXG_MASK])); 4646 ASSERT0(zfs_range_tree_space(msp->ms_freeing)); 4647 ASSERT0(zfs_range_tree_space(msp->ms_freed)); 4648 ASSERT0(zfs_range_tree_space(msp->ms_checkpointing)); 4649 msp->ms_allocating_total -= msp->ms_allocated_this_txg; 4650 msp->ms_allocated_this_txg = 0; 4651 mutex_exit(&msp->ms_lock); 4652 } 4653 4654 void 4655 metaslab_sync_reassess(metaslab_group_t *mg) 4656 { 4657 spa_t *spa = mg->mg_class->mc_spa; 4658 4659 spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER); 4660 mg->mg_fragmentation = metaslab_group_fragmentation(mg); 4661 metaslab_group_alloc_update(mg); 4662 4663 /* 4664 * Preload the next potential metaslabs but only on active 4665 * metaslab groups. We can get into a state where the metaslab 4666 * is no longer active since we dirty metaslabs as we remove a 4667 * a device, thus potentially making the metaslab group eligible 4668 * for preloading. 4669 */ 4670 if (mg->mg_activation_count > 0) { 4671 metaslab_group_preload(mg); 4672 } 4673 spa_config_exit(spa, SCL_ALLOC, FTAG); 4674 } 4675 4676 /* 4677 * When writing a ditto block (i.e. more than one DVA for a given BP) on 4678 * the same vdev as an existing DVA of this BP, then try to allocate it 4679 * on a different metaslab than existing DVAs (i.e. a unique metaslab). 4680 */ 4681 static boolean_t 4682 metaslab_is_unique(metaslab_t *msp, dva_t *dva) 4683 { 4684 uint64_t dva_ms_id; 4685 4686 if (DVA_GET_ASIZE(dva) == 0) 4687 return (B_TRUE); 4688 4689 if (msp->ms_group->mg_vd->vdev_id != DVA_GET_VDEV(dva)) 4690 return (B_TRUE); 4691 4692 dva_ms_id = DVA_GET_OFFSET(dva) >> msp->ms_group->mg_vd->vdev_ms_shift; 4693 4694 return (msp->ms_id != dva_ms_id); 4695 } 4696 4697 /* 4698 * ========================================================================== 4699 * Metaslab allocation tracing facility 4700 * ========================================================================== 4701 */ 4702 4703 /* 4704 * Add an allocation trace element to the allocation tracing list. 4705 */ 4706 static void 4707 metaslab_trace_add(zio_alloc_list_t *zal, metaslab_group_t *mg, 4708 metaslab_t *msp, uint64_t psize, uint32_t dva_id, uint64_t offset, 4709 int allocator) 4710 { 4711 metaslab_alloc_trace_t *mat; 4712 4713 if (!metaslab_trace_enabled) 4714 return; 4715 4716 /* 4717 * When the tracing list reaches its maximum we remove 4718 * the second element in the list before adding a new one. 4719 * By removing the second element we preserve the original 4720 * entry as a clue to what allocations steps have already been 4721 * performed. 4722 */ 4723 if (zal->zal_size == metaslab_trace_max_entries) { 4724 metaslab_alloc_trace_t *mat_next; 4725 #ifdef ZFS_DEBUG 4726 panic("too many entries in allocation list"); 4727 #endif 4728 METASLABSTAT_BUMP(metaslabstat_trace_over_limit); 4729 zal->zal_size--; 4730 mat_next = list_next(&zal->zal_list, list_head(&zal->zal_list)); 4731 list_remove(&zal->zal_list, mat_next); 4732 kmem_cache_free(metaslab_alloc_trace_cache, mat_next); 4733 } 4734 4735 mat = kmem_cache_alloc(metaslab_alloc_trace_cache, KM_SLEEP); 4736 list_link_init(&mat->mat_list_node); 4737 mat->mat_mg = mg; 4738 mat->mat_msp = msp; 4739 mat->mat_size = psize; 4740 mat->mat_dva_id = dva_id; 4741 mat->mat_offset = offset; 4742 mat->mat_weight = 0; 4743 mat->mat_allocator = allocator; 4744 4745 if (msp != NULL) 4746 mat->mat_weight = msp->ms_weight; 4747 4748 /* 4749 * The list is part of the zio so locking is not required. Only 4750 * a single thread will perform allocations for a given zio. 4751 */ 4752 list_insert_tail(&zal->zal_list, mat); 4753 zal->zal_size++; 4754 4755 ASSERT3U(zal->zal_size, <=, metaslab_trace_max_entries); 4756 } 4757 4758 void 4759 metaslab_trace_move(zio_alloc_list_t *old, zio_alloc_list_t *new) 4760 { 4761 ASSERT0(new->zal_size); 4762 list_move_tail(&new->zal_list, &old->zal_list); 4763 new->zal_size = old->zal_size; 4764 list_destroy(&old->zal_list); 4765 } 4766 4767 void 4768 metaslab_trace_init(zio_alloc_list_t *zal) 4769 { 4770 list_create(&zal->zal_list, sizeof (metaslab_alloc_trace_t), 4771 offsetof(metaslab_alloc_trace_t, mat_list_node)); 4772 zal->zal_size = 0; 4773 } 4774 4775 void 4776 metaslab_trace_fini(zio_alloc_list_t *zal) 4777 { 4778 metaslab_alloc_trace_t *mat; 4779 4780 while ((mat = list_remove_head(&zal->zal_list)) != NULL) 4781 kmem_cache_free(metaslab_alloc_trace_cache, mat); 4782 list_destroy(&zal->zal_list); 4783 zal->zal_size = 0; 4784 } 4785 4786 /* 4787 * ========================================================================== 4788 * Metaslab block operations 4789 * ========================================================================== 4790 */ 4791 4792 static void 4793 metaslab_group_alloc_increment(spa_t *spa, uint64_t vdev, int allocator, 4794 int flags, uint64_t psize, const void *tag) 4795 { 4796 if (!(flags & METASLAB_ASYNC_ALLOC) || tag == NULL) 4797 return; 4798 4799 metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; 4800 if (!mg->mg_class->mc_alloc_throttle_enabled) 4801 return; 4802 4803 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 4804 (void) zfs_refcount_add_many(&mga->mga_queue_depth, psize, tag); 4805 } 4806 4807 void 4808 metaslab_group_alloc_increment_all(spa_t *spa, blkptr_t *bp, int allocator, 4809 int flags, uint64_t psize, const void *tag) 4810 { 4811 for (int d = 0; d < BP_GET_NDVAS(bp); d++) { 4812 uint64_t vdev = DVA_GET_VDEV(&bp->blk_dva[d]); 4813 metaslab_group_alloc_increment(spa, vdev, allocator, flags, 4814 psize, tag); 4815 } 4816 } 4817 4818 void 4819 metaslab_group_alloc_decrement(spa_t *spa, uint64_t vdev, int allocator, 4820 int flags, uint64_t psize, const void *tag) 4821 { 4822 if (!(flags & METASLAB_ASYNC_ALLOC) || tag == NULL) 4823 return; 4824 4825 metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg; 4826 if (!mg->mg_class->mc_alloc_throttle_enabled) 4827 return; 4828 4829 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 4830 (void) zfs_refcount_remove_many(&mga->mga_queue_depth, psize, tag); 4831 } 4832 4833 static uint64_t 4834 metaslab_block_alloc(metaslab_t *msp, uint64_t size, uint64_t max_size, 4835 uint64_t txg, uint64_t *actual_size) 4836 { 4837 uint64_t start; 4838 zfs_range_tree_t *rt = msp->ms_allocatable; 4839 metaslab_class_t *mc = msp->ms_group->mg_class; 4840 4841 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4842 VERIFY(!msp->ms_condensing); 4843 VERIFY0(msp->ms_disabled); 4844 VERIFY0(msp->ms_new); 4845 4846 start = mc->mc_ops->msop_alloc(msp, size, max_size, actual_size); 4847 if (start != -1ULL) { 4848 size = *actual_size; 4849 metaslab_group_t *mg = msp->ms_group; 4850 vdev_t *vd = mg->mg_vd; 4851 4852 VERIFY0(P2PHASE(start, 1ULL << vd->vdev_ashift)); 4853 VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 4854 VERIFY3U(zfs_range_tree_space(rt) - size, <=, msp->ms_size); 4855 zfs_range_tree_remove(rt, start, size); 4856 zfs_range_tree_clear(msp->ms_trim, start, size); 4857 4858 if (zfs_range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK])) 4859 vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg); 4860 4861 zfs_range_tree_add(msp->ms_allocating[txg & TXG_MASK], start, 4862 size); 4863 msp->ms_allocating_total += size; 4864 4865 /* Track the last successful allocation */ 4866 msp->ms_alloc_txg = txg; 4867 metaslab_verify_space(msp, txg); 4868 } 4869 4870 /* 4871 * Now that we've attempted the allocation we need to update the 4872 * metaslab's maximum block size since it may have changed. 4873 */ 4874 msp->ms_max_size = metaslab_largest_allocatable(msp); 4875 return (start); 4876 } 4877 4878 /* 4879 * Find the metaslab with the highest weight that is less than what we've 4880 * already tried. In the common case, this means that we will examine each 4881 * metaslab at most once. Note that concurrent callers could reorder metaslabs 4882 * by activation/passivation once we have dropped the mg_lock. If a metaslab is 4883 * activated by another thread, and we fail to allocate from the metaslab we 4884 * have selected, we may not try the newly-activated metaslab, and instead 4885 * activate another metaslab. This is not optimal, but generally does not cause 4886 * any problems (a possible exception being if every metaslab is completely full 4887 * except for the newly-activated metaslab which we fail to examine). 4888 */ 4889 static metaslab_t * 4890 find_valid_metaslab(metaslab_group_t *mg, uint64_t activation_weight, 4891 dva_t *dva, int d, uint64_t asize, int allocator, 4892 boolean_t try_hard, zio_alloc_list_t *zal, metaslab_t *search, 4893 boolean_t *was_active) 4894 { 4895 avl_index_t idx; 4896 avl_tree_t *t = &mg->mg_metaslab_tree; 4897 metaslab_t *msp = avl_find(t, search, &idx); 4898 if (msp == NULL) 4899 msp = avl_nearest(t, idx, AVL_AFTER); 4900 4901 uint_t tries = 0; 4902 for (; msp != NULL; msp = AVL_NEXT(t, msp)) { 4903 int i; 4904 4905 if (!try_hard && tries > zfs_metaslab_find_max_tries) { 4906 METASLABSTAT_BUMP(metaslabstat_too_many_tries); 4907 return (NULL); 4908 } 4909 tries++; 4910 4911 if (!metaslab_should_allocate(msp, asize, try_hard)) { 4912 metaslab_trace_add(zal, mg, msp, asize, d, 4913 TRACE_TOO_SMALL, allocator); 4914 continue; 4915 } 4916 4917 /* 4918 * If the selected metaslab is condensing or disabled, or 4919 * hasn't gone through a metaslab_sync_done(), then skip it. 4920 */ 4921 if (msp->ms_condensing || msp->ms_disabled > 0 || msp->ms_new) 4922 continue; 4923 4924 *was_active = msp->ms_allocator != -1; 4925 /* 4926 * If we're activating as primary, this is our first allocation 4927 * from this disk, so we don't need to check how close we are. 4928 * If the metaslab under consideration was already active, 4929 * we're getting desperate enough to steal another allocator's 4930 * metaslab, so we still don't care about distances. 4931 */ 4932 if (activation_weight == METASLAB_WEIGHT_PRIMARY || *was_active) 4933 break; 4934 4935 if (!try_hard) { 4936 for (i = 0; i < d; i++) { 4937 if (!metaslab_is_unique(msp, &dva[i])) 4938 break; /* try another metaslab */ 4939 } 4940 if (i == d) 4941 break; 4942 } 4943 } 4944 4945 if (msp != NULL) { 4946 search->ms_weight = msp->ms_weight; 4947 search->ms_start = msp->ms_start + 1; 4948 search->ms_allocator = msp->ms_allocator; 4949 search->ms_primary = msp->ms_primary; 4950 } 4951 return (msp); 4952 } 4953 4954 static void 4955 metaslab_active_mask_verify(metaslab_t *msp) 4956 { 4957 ASSERT(MUTEX_HELD(&msp->ms_lock)); 4958 4959 if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0) 4960 return; 4961 4962 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0) 4963 return; 4964 4965 if (msp->ms_weight & METASLAB_WEIGHT_PRIMARY) { 4966 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); 4967 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM); 4968 VERIFY3S(msp->ms_allocator, !=, -1); 4969 VERIFY(msp->ms_primary); 4970 return; 4971 } 4972 4973 if (msp->ms_weight & METASLAB_WEIGHT_SECONDARY) { 4974 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); 4975 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM); 4976 VERIFY3S(msp->ms_allocator, !=, -1); 4977 VERIFY(!msp->ms_primary); 4978 return; 4979 } 4980 4981 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) { 4982 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY); 4983 VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY); 4984 VERIFY3S(msp->ms_allocator, ==, -1); 4985 return; 4986 } 4987 } 4988 4989 static uint64_t 4990 metaslab_group_alloc(metaslab_group_t *mg, zio_alloc_list_t *zal, 4991 uint64_t asize, uint64_t max_asize, uint64_t txg, 4992 dva_t *dva, int d, int allocator, boolean_t try_hard, 4993 uint64_t *actual_asize) 4994 { 4995 metaslab_t *msp = NULL; 4996 uint64_t offset = -1ULL; 4997 4998 uint64_t activation_weight = METASLAB_WEIGHT_PRIMARY; 4999 for (int i = 0; i < d; i++) { 5000 if (activation_weight == METASLAB_WEIGHT_PRIMARY && 5001 DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) { 5002 activation_weight = METASLAB_WEIGHT_SECONDARY; 5003 } else if (activation_weight == METASLAB_WEIGHT_SECONDARY && 5004 DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) { 5005 activation_weight = METASLAB_WEIGHT_CLAIM; 5006 break; 5007 } 5008 } 5009 5010 /* 5011 * If we don't have enough metaslabs active, we just use the 0th slot. 5012 */ 5013 if (allocator >= mg->mg_ms_ready / 3) 5014 allocator = 0; 5015 metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator]; 5016 5017 ASSERT3U(mg->mg_vd->vdev_ms_count, >=, 2); 5018 5019 metaslab_t *search = kmem_alloc(sizeof (*search), KM_SLEEP); 5020 search->ms_weight = UINT64_MAX; 5021 search->ms_start = 0; 5022 /* 5023 * At the end of the metaslab tree are the already-active metaslabs, 5024 * first the primaries, then the secondaries. When we resume searching 5025 * through the tree, we need to consider ms_allocator and ms_primary so 5026 * we start in the location right after where we left off, and don't 5027 * accidentally loop forever considering the same metaslabs. 5028 */ 5029 search->ms_allocator = -1; 5030 search->ms_primary = B_TRUE; 5031 for (;;) { 5032 boolean_t was_active = B_FALSE; 5033 5034 mutex_enter(&mg->mg_lock); 5035 5036 if (activation_weight == METASLAB_WEIGHT_PRIMARY && 5037 mga->mga_primary != NULL) { 5038 msp = mga->mga_primary; 5039 5040 /* 5041 * Even though we don't hold the ms_lock for the 5042 * primary metaslab, those fields should not 5043 * change while we hold the mg_lock. Thus it is 5044 * safe to make assertions on them. 5045 */ 5046 ASSERT(msp->ms_primary); 5047 ASSERT3S(msp->ms_allocator, ==, allocator); 5048 ASSERT(msp->ms_loaded); 5049 5050 was_active = B_TRUE; 5051 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 5052 } else if (activation_weight == METASLAB_WEIGHT_SECONDARY && 5053 mga->mga_secondary != NULL) { 5054 msp = mga->mga_secondary; 5055 5056 /* 5057 * See comment above about the similar assertions 5058 * for the primary metaslab. 5059 */ 5060 ASSERT(!msp->ms_primary); 5061 ASSERT3S(msp->ms_allocator, ==, allocator); 5062 ASSERT(msp->ms_loaded); 5063 5064 was_active = B_TRUE; 5065 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 5066 } else { 5067 msp = find_valid_metaslab(mg, activation_weight, dva, d, 5068 asize, allocator, try_hard, zal, search, 5069 &was_active); 5070 } 5071 5072 mutex_exit(&mg->mg_lock); 5073 if (msp == NULL) 5074 break; 5075 mutex_enter(&msp->ms_lock); 5076 5077 metaslab_active_mask_verify(msp); 5078 5079 /* 5080 * This code is disabled out because of issues with 5081 * tracepoints in non-gpl kernel modules. 5082 */ 5083 #if 0 5084 DTRACE_PROBE3(ms__activation__attempt, 5085 metaslab_t *, msp, uint64_t, activation_weight, 5086 boolean_t, was_active); 5087 #endif 5088 5089 /* 5090 * Ensure that the metaslab we have selected is still 5091 * capable of handling our request. It's possible that 5092 * another thread may have changed the weight while we 5093 * were blocked on the metaslab lock. We check the 5094 * active status first to see if we need to set_selected_txg 5095 * a new metaslab. 5096 */ 5097 if (was_active && !(msp->ms_weight & METASLAB_ACTIVE_MASK)) { 5098 ASSERT3S(msp->ms_allocator, ==, -1); 5099 mutex_exit(&msp->ms_lock); 5100 continue; 5101 } 5102 5103 /* 5104 * If the metaslab was activated for another allocator 5105 * while we were waiting in the ms_lock above, or it's 5106 * a primary and we're seeking a secondary (or vice versa), 5107 * we go back and select a new metaslab. 5108 */ 5109 if (!was_active && (msp->ms_weight & METASLAB_ACTIVE_MASK) && 5110 (msp->ms_allocator != -1) && 5111 (msp->ms_allocator != allocator || ((activation_weight == 5112 METASLAB_WEIGHT_PRIMARY) != msp->ms_primary))) { 5113 ASSERT(msp->ms_loaded); 5114 ASSERT((msp->ms_weight & METASLAB_WEIGHT_CLAIM) || 5115 msp->ms_allocator != -1); 5116 mutex_exit(&msp->ms_lock); 5117 continue; 5118 } 5119 5120 /* 5121 * This metaslab was used for claiming regions allocated 5122 * by the ZIL during pool import. Once these regions are 5123 * claimed we don't need to keep the CLAIM bit set 5124 * anymore. Passivate this metaslab to zero its activation 5125 * mask. 5126 */ 5127 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM && 5128 activation_weight != METASLAB_WEIGHT_CLAIM) { 5129 ASSERT(msp->ms_loaded); 5130 ASSERT3S(msp->ms_allocator, ==, -1); 5131 metaslab_passivate(msp, msp->ms_weight & 5132 ~METASLAB_WEIGHT_CLAIM); 5133 mutex_exit(&msp->ms_lock); 5134 continue; 5135 } 5136 5137 metaslab_set_selected_txg(msp, txg); 5138 5139 int activation_error = 5140 metaslab_activate(msp, allocator, activation_weight); 5141 metaslab_active_mask_verify(msp); 5142 5143 /* 5144 * If the metaslab was activated by another thread for 5145 * another allocator or activation_weight (EBUSY), or it 5146 * failed because another metaslab was assigned as primary 5147 * for this allocator (EEXIST) we continue using this 5148 * metaslab for our allocation, rather than going on to a 5149 * worse metaslab (we waited for that metaslab to be loaded 5150 * after all). 5151 * 5152 * If the activation failed due to an I/O error or ENOSPC we 5153 * skip to the next metaslab. 5154 */ 5155 boolean_t activated; 5156 if (activation_error == 0) { 5157 activated = B_TRUE; 5158 } else if (activation_error == EBUSY || 5159 activation_error == EEXIST) { 5160 activated = B_FALSE; 5161 } else { 5162 mutex_exit(&msp->ms_lock); 5163 continue; 5164 } 5165 ASSERT(msp->ms_loaded); 5166 5167 /* 5168 * Now that we have the lock, recheck to see if we should 5169 * continue to use this metaslab for this allocation. The 5170 * the metaslab is now loaded so metaslab_should_allocate() 5171 * can accurately determine if the allocation attempt should 5172 * proceed. 5173 */ 5174 if (!metaslab_should_allocate(msp, asize, try_hard)) { 5175 /* Passivate this metaslab and select a new one. */ 5176 metaslab_trace_add(zal, mg, msp, asize, d, 5177 TRACE_TOO_SMALL, allocator); 5178 goto next; 5179 } 5180 5181 /* 5182 * If this metaslab is currently condensing then pick again 5183 * as we can't manipulate this metaslab until it's committed 5184 * to disk. If this metaslab is being initialized, we shouldn't 5185 * allocate from it since the allocated region might be 5186 * overwritten after allocation. 5187 */ 5188 if (msp->ms_condensing) { 5189 metaslab_trace_add(zal, mg, msp, asize, d, 5190 TRACE_CONDENSING, allocator); 5191 if (activated) { 5192 metaslab_passivate(msp, msp->ms_weight & 5193 ~METASLAB_ACTIVE_MASK); 5194 } 5195 mutex_exit(&msp->ms_lock); 5196 continue; 5197 } else if (msp->ms_disabled > 0) { 5198 metaslab_trace_add(zal, mg, msp, asize, d, 5199 TRACE_DISABLED, allocator); 5200 if (activated) { 5201 metaslab_passivate(msp, msp->ms_weight & 5202 ~METASLAB_ACTIVE_MASK); 5203 } 5204 mutex_exit(&msp->ms_lock); 5205 continue; 5206 } 5207 5208 offset = metaslab_block_alloc(msp, asize, max_asize, txg, 5209 actual_asize); 5210 5211 if (offset != -1ULL) { 5212 metaslab_trace_add(zal, mg, msp, *actual_asize, d, 5213 offset, allocator); 5214 /* Proactively passivate the metaslab, if needed */ 5215 if (activated) 5216 metaslab_segment_may_passivate(msp); 5217 mutex_exit(&msp->ms_lock); 5218 break; 5219 } 5220 metaslab_trace_add(zal, mg, msp, asize, d, offset, allocator); 5221 next: 5222 ASSERT(msp->ms_loaded); 5223 5224 /* 5225 * This code is disabled out because of issues with 5226 * tracepoints in non-gpl kernel modules. 5227 */ 5228 #if 0 5229 DTRACE_PROBE2(ms__alloc__failure, metaslab_t *, msp, 5230 uint64_t, asize); 5231 #endif 5232 5233 /* 5234 * We were unable to allocate from this metaslab so determine 5235 * a new weight for this metaslab. The weight was last 5236 * recalculated either when we loaded it (if this is the first 5237 * TXG it's been loaded in), or the last time a txg was synced 5238 * out. 5239 */ 5240 uint64_t weight; 5241 if (WEIGHT_IS_SPACEBASED(msp->ms_weight)) { 5242 metaslab_set_fragmentation(msp, B_TRUE); 5243 weight = metaslab_space_weight(msp) & 5244 ~METASLAB_ACTIVE_MASK; 5245 } else { 5246 weight = metaslab_weight_from_range_tree(msp); 5247 } 5248 5249 if (activated) { 5250 metaslab_passivate(msp, weight); 5251 } else { 5252 /* 5253 * For the case where we use the metaslab that is 5254 * active for another allocator we want to make 5255 * sure that we retain the activation mask. 5256 */ 5257 weight |= msp->ms_weight & METASLAB_ACTIVE_MASK; 5258 metaslab_group_sort(mg, msp, weight); 5259 } 5260 metaslab_active_mask_verify(msp); 5261 5262 /* 5263 * We have just failed an allocation attempt, check 5264 * that metaslab_should_allocate() agrees. Otherwise, 5265 * we may end up in an infinite loop retrying the same 5266 * metaslab. 5267 */ 5268 ASSERT(!metaslab_should_allocate(msp, asize, try_hard)); 5269 5270 mutex_exit(&msp->ms_lock); 5271 } 5272 kmem_free(search, sizeof (*search)); 5273 5274 if (offset == -1ULL) { 5275 metaslab_trace_add(zal, mg, NULL, asize, d, 5276 TRACE_GROUP_FAILURE, allocator); 5277 if (asize <= vdev_get_min_alloc(mg->mg_vd)) { 5278 /* 5279 * This metaslab group was unable to allocate 5280 * the minimum block size so it must be out of 5281 * space. Notify the allocation throttle to 5282 * skip allocation attempts to this group until 5283 * more space becomes available. 5284 */ 5285 mg->mg_no_free_space = B_TRUE; 5286 } 5287 } 5288 return (offset); 5289 } 5290 5291 static boolean_t 5292 metaslab_group_allocatable(spa_t *spa, metaslab_group_t *mg, uint64_t psize, 5293 int d, int flags, boolean_t try_hard, zio_alloc_list_t *zal, int allocator) 5294 { 5295 metaslab_class_t *mc = mg->mg_class; 5296 vdev_t *vd = mg->mg_vd; 5297 boolean_t allocatable; 5298 5299 /* 5300 * Don't allocate from faulted devices. 5301 */ 5302 if (try_hard) 5303 spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER); 5304 allocatable = vdev_allocatable(vd); 5305 if (try_hard) 5306 spa_config_exit(spa, SCL_ZIO, FTAG); 5307 if (!allocatable) { 5308 metaslab_trace_add(zal, mg, NULL, psize, d, 5309 TRACE_NOT_ALLOCATABLE, allocator); 5310 return (B_FALSE); 5311 } 5312 5313 if (!try_hard) { 5314 /* 5315 * Avoid vdevs with too little space or too fragmented. 5316 */ 5317 if (!GANG_ALLOCATION(flags) && (mg->mg_no_free_space || 5318 (!mg->mg_allocatable && mc->mc_alloc_groups > 0))) { 5319 metaslab_trace_add(zal, mg, NULL, psize, d, 5320 TRACE_NOT_ALLOCATABLE, allocator); 5321 return (B_FALSE); 5322 } 5323 5324 /* 5325 * Avoid writing single-copy data to an unhealthy, 5326 * non-redundant vdev. 5327 */ 5328 if (d == 0 && vd->vdev_state < VDEV_STATE_HEALTHY && 5329 vd->vdev_children == 0) { 5330 metaslab_trace_add(zal, mg, NULL, psize, d, 5331 TRACE_VDEV_ERROR, allocator); 5332 return (B_FALSE); 5333 } 5334 } 5335 5336 return (B_TRUE); 5337 } 5338 5339 static int 5340 metaslab_alloc_dva_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize, 5341 uint64_t max_psize, dva_t *dva, int d, const dva_t *hintdva, uint64_t txg, 5342 int flags, zio_alloc_list_t *zal, int allocator, uint64_t *actual_psize) 5343 { 5344 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 5345 metaslab_group_t *mg = NULL, *rotor; 5346 vdev_t *vd; 5347 boolean_t try_hard = B_FALSE; 5348 5349 ASSERT(!DVA_IS_VALID(&dva[d])); 5350 5351 /* 5352 * For testing, make some blocks above a certain size be gang blocks. 5353 * This will result in more split blocks when using device removal, 5354 * and a large number of split blocks coupled with ztest-induced 5355 * damage can result in extremely long reconstruction times. This 5356 * will also test spilling from special to normal. 5357 */ 5358 if (psize >= metaslab_force_ganging && 5359 metaslab_force_ganging_pct > 0 && 5360 (random_in_range(100) < MIN(metaslab_force_ganging_pct, 100))) { 5361 metaslab_trace_add(zal, NULL, NULL, psize, d, TRACE_FORCE_GANG, 5362 allocator); 5363 return (SET_ERROR(ENOSPC)); 5364 } 5365 if (max_psize > psize && max_psize >= metaslab_force_ganging && 5366 metaslab_force_ganging_pct > 0 && 5367 (random_in_range(100) < MIN(metaslab_force_ganging_pct, 100))) { 5368 max_psize = MAX((psize + max_psize) / 2, 5369 metaslab_force_ganging); 5370 } 5371 ASSERT3U(psize, <=, max_psize); 5372 5373 /* 5374 * Start at the rotor and loop through all mgs until we find something. 5375 * Note that there's no locking on mca_rotor or mca_aliquot because 5376 * nothing actually breaks if we miss a few updates -- we just won't 5377 * allocate quite as evenly. It all balances out over time. 5378 * 5379 * If we are doing ditto or log blocks, try to spread them across 5380 * consecutive vdevs. If we're forced to reuse a vdev before we've 5381 * allocated all of our ditto blocks, then try and spread them out on 5382 * that vdev as much as possible. If it turns out to not be possible, 5383 * gradually lower our standards until anything becomes acceptable. 5384 * Also, allocating on consecutive vdevs (as opposed to random vdevs) 5385 * gives us hope of containing our fault domains to something we're 5386 * able to reason about. Otherwise, any two top-level vdev failures 5387 * will guarantee the loss of data. With consecutive allocation, 5388 * only two adjacent top-level vdev failures will result in data loss. 5389 * 5390 * If we are doing gang blocks (hintdva is non-NULL), try to keep 5391 * ourselves on the same vdev as our gang block header. It makes our 5392 * fault domains something tractable. 5393 */ 5394 if (hintdva && DVA_IS_VALID(&hintdva[d])) { 5395 vd = vdev_lookup_top(spa, DVA_GET_VDEV(&hintdva[d])); 5396 mg = vdev_get_mg(vd, mc); 5397 } 5398 if (mg == NULL && d != 0) { 5399 vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[d - 1])); 5400 mg = vdev_get_mg(vd, mc)->mg_next; 5401 } 5402 if (mg == NULL || mg->mg_class != mc || mg->mg_activation_count <= 0) { 5403 ASSERT(mca->mca_rotor != NULL); 5404 mg = mca->mca_rotor; 5405 } 5406 5407 rotor = mg; 5408 top: 5409 do { 5410 ASSERT(mg->mg_activation_count == 1); 5411 ASSERT(mg->mg_class == mc); 5412 5413 if (!metaslab_group_allocatable(spa, mg, psize, d, flags, 5414 try_hard, zal, allocator)) 5415 goto next; 5416 5417 vd = mg->mg_vd; 5418 uint64_t asize = vdev_psize_to_asize_txg(vd, psize, txg); 5419 ASSERT0(P2PHASE(asize, 1ULL << vd->vdev_ashift)); 5420 uint64_t max_asize = vdev_psize_to_asize_txg(vd, max_psize, 5421 txg); 5422 ASSERT0(P2PHASE(max_asize, 1ULL << vd->vdev_ashift)); 5423 uint64_t offset = metaslab_group_alloc(mg, zal, asize, 5424 max_asize, txg, dva, d, allocator, try_hard, 5425 &asize); 5426 5427 if (offset != -1ULL) { 5428 if (actual_psize) 5429 *actual_psize = vdev_asize_to_psize_txg(vd, 5430 asize, txg); 5431 metaslab_class_rotate(mg, allocator, psize, B_TRUE); 5432 5433 DVA_SET_VDEV(&dva[d], vd->vdev_id); 5434 DVA_SET_OFFSET(&dva[d], offset); 5435 DVA_SET_GANG(&dva[d], 5436 ((flags & METASLAB_GANG_HEADER) ? 1 : 0)); 5437 DVA_SET_ASIZE(&dva[d], asize); 5438 return (0); 5439 } 5440 next: 5441 metaslab_class_rotate(mg, allocator, psize, B_FALSE); 5442 } while ((mg = mg->mg_next) != rotor); 5443 5444 /* 5445 * If we haven't tried hard, perhaps do so now. 5446 */ 5447 if (!try_hard && (zfs_metaslab_try_hard_before_gang || 5448 GANG_ALLOCATION(flags) || (flags & METASLAB_ZIL) != 0 || 5449 psize <= spa->spa_min_alloc)) { 5450 METASLABSTAT_BUMP(metaslabstat_try_hard); 5451 try_hard = B_TRUE; 5452 goto top; 5453 } 5454 5455 memset(&dva[d], 0, sizeof (dva_t)); 5456 5457 metaslab_trace_add(zal, rotor, NULL, psize, d, TRACE_ENOSPC, allocator); 5458 return (SET_ERROR(ENOSPC)); 5459 } 5460 5461 /* 5462 * Allocate a block for the specified i/o. 5463 */ 5464 int 5465 metaslab_alloc_dva(spa_t *spa, metaslab_class_t *mc, uint64_t psize, 5466 dva_t *dva, int d, const dva_t *hintdva, uint64_t txg, int flags, 5467 zio_alloc_list_t *zal, int allocator) 5468 { 5469 return (metaslab_alloc_dva_range(spa, mc, psize, psize, dva, d, hintdva, 5470 txg, flags, zal, allocator, NULL)); 5471 } 5472 5473 void 5474 metaslab_free_concrete(vdev_t *vd, uint64_t offset, uint64_t asize, 5475 boolean_t checkpoint) 5476 { 5477 metaslab_t *msp; 5478 spa_t *spa = vd->vdev_spa; 5479 int m = offset >> vd->vdev_ms_shift; 5480 5481 ASSERT(vdev_is_concrete(vd)); 5482 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5483 VERIFY3U(m, <, vd->vdev_ms_count); 5484 5485 msp = vd->vdev_ms[m]; 5486 5487 VERIFY(!msp->ms_condensing); 5488 VERIFY3U(offset, >=, msp->ms_start); 5489 VERIFY3U(offset + asize, <=, msp->ms_start + msp->ms_size); 5490 VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 5491 VERIFY0(P2PHASE(asize, 1ULL << vd->vdev_ashift)); 5492 5493 metaslab_check_free_impl(vd, offset, asize); 5494 5495 mutex_enter(&msp->ms_lock); 5496 if (zfs_range_tree_is_empty(msp->ms_freeing) && 5497 zfs_range_tree_is_empty(msp->ms_checkpointing)) { 5498 vdev_dirty(vd, VDD_METASLAB, msp, spa_syncing_txg(spa)); 5499 } 5500 5501 if (checkpoint) { 5502 ASSERT(spa_has_checkpoint(spa)); 5503 zfs_range_tree_add(msp->ms_checkpointing, offset, asize); 5504 } else { 5505 zfs_range_tree_add(msp->ms_freeing, offset, asize); 5506 } 5507 mutex_exit(&msp->ms_lock); 5508 } 5509 5510 void 5511 metaslab_free_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, 5512 uint64_t size, void *arg) 5513 { 5514 (void) inner_offset; 5515 boolean_t *checkpoint = arg; 5516 5517 ASSERT3P(checkpoint, !=, NULL); 5518 5519 if (vd->vdev_ops->vdev_op_remap != NULL) 5520 vdev_indirect_mark_obsolete(vd, offset, size); 5521 else 5522 metaslab_free_impl(vd, offset, size, *checkpoint); 5523 } 5524 5525 static void 5526 metaslab_free_impl(vdev_t *vd, uint64_t offset, uint64_t size, 5527 boolean_t checkpoint) 5528 { 5529 spa_t *spa = vd->vdev_spa; 5530 5531 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5532 5533 if (spa_syncing_txg(spa) > spa_freeze_txg(spa)) 5534 return; 5535 5536 if (spa->spa_vdev_removal != NULL && 5537 spa->spa_vdev_removal->svr_vdev_id == vd->vdev_id && 5538 vdev_is_concrete(vd)) { 5539 /* 5540 * Note: we check if the vdev is concrete because when 5541 * we complete the removal, we first change the vdev to be 5542 * an indirect vdev (in open context), and then (in syncing 5543 * context) clear spa_vdev_removal. 5544 */ 5545 free_from_removing_vdev(vd, offset, size); 5546 } else if (vd->vdev_ops->vdev_op_remap != NULL) { 5547 vdev_indirect_mark_obsolete(vd, offset, size); 5548 vd->vdev_ops->vdev_op_remap(vd, offset, size, 5549 metaslab_free_impl_cb, &checkpoint); 5550 } else { 5551 metaslab_free_concrete(vd, offset, size, checkpoint); 5552 } 5553 } 5554 5555 typedef struct remap_blkptr_cb_arg { 5556 blkptr_t *rbca_bp; 5557 spa_remap_cb_t rbca_cb; 5558 vdev_t *rbca_remap_vd; 5559 uint64_t rbca_remap_offset; 5560 void *rbca_cb_arg; 5561 } remap_blkptr_cb_arg_t; 5562 5563 static void 5564 remap_blkptr_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, 5565 uint64_t size, void *arg) 5566 { 5567 remap_blkptr_cb_arg_t *rbca = arg; 5568 blkptr_t *bp = rbca->rbca_bp; 5569 5570 /* We can not remap split blocks. */ 5571 if (size != DVA_GET_ASIZE(&bp->blk_dva[0])) 5572 return; 5573 ASSERT0(inner_offset); 5574 5575 if (rbca->rbca_cb != NULL) { 5576 /* 5577 * At this point we know that we are not handling split 5578 * blocks and we invoke the callback on the previous 5579 * vdev which must be indirect. 5580 */ 5581 ASSERT3P(rbca->rbca_remap_vd->vdev_ops, ==, &vdev_indirect_ops); 5582 5583 rbca->rbca_cb(rbca->rbca_remap_vd->vdev_id, 5584 rbca->rbca_remap_offset, size, rbca->rbca_cb_arg); 5585 5586 /* set up remap_blkptr_cb_arg for the next call */ 5587 rbca->rbca_remap_vd = vd; 5588 rbca->rbca_remap_offset = offset; 5589 } 5590 5591 /* 5592 * The phys birth time is that of dva[0]. This ensures that we know 5593 * when each dva was written, so that resilver can determine which 5594 * blocks need to be scrubbed (i.e. those written during the time 5595 * the vdev was offline). It also ensures that the key used in 5596 * the ARC hash table is unique (i.e. dva[0] + phys_birth). If 5597 * we didn't change the phys_birth, a lookup in the ARC for a 5598 * remapped BP could find the data that was previously stored at 5599 * this vdev + offset. 5600 */ 5601 vdev_t *oldvd = vdev_lookup_top(vd->vdev_spa, 5602 DVA_GET_VDEV(&bp->blk_dva[0])); 5603 vdev_indirect_births_t *vib = oldvd->vdev_indirect_births; 5604 uint64_t physical_birth = vdev_indirect_births_physbirth(vib, 5605 DVA_GET_OFFSET(&bp->blk_dva[0]), DVA_GET_ASIZE(&bp->blk_dva[0])); 5606 5607 /* 5608 * For rewritten blocks, use the old physical birth as the new logical 5609 * birth (representing when the space was allocated) and the removal 5610 * time as the new physical birth (representing when it was actually 5611 * written). 5612 */ 5613 if (BP_GET_REWRITE(bp)) { 5614 uint64_t old_physical_birth = BP_GET_PHYSICAL_BIRTH(bp); 5615 ASSERT3U(old_physical_birth, <, physical_birth); 5616 BP_SET_BIRTH(bp, old_physical_birth, physical_birth); 5617 BP_SET_REWRITE(bp, 0); 5618 } else { 5619 BP_SET_PHYSICAL_BIRTH(bp, physical_birth); 5620 } 5621 5622 DVA_SET_VDEV(&bp->blk_dva[0], vd->vdev_id); 5623 DVA_SET_OFFSET(&bp->blk_dva[0], offset); 5624 } 5625 5626 /* 5627 * If the block pointer contains any indirect DVAs, modify them to refer to 5628 * concrete DVAs. Note that this will sometimes not be possible, leaving 5629 * the indirect DVA in place. This happens if the indirect DVA spans multiple 5630 * segments in the mapping (i.e. it is a "split block"). 5631 * 5632 * If the BP was remapped, calls the callback on the original dva (note the 5633 * callback can be called multiple times if the original indirect DVA refers 5634 * to another indirect DVA, etc). 5635 * 5636 * Returns TRUE if the BP was remapped. 5637 */ 5638 boolean_t 5639 spa_remap_blkptr(spa_t *spa, blkptr_t *bp, spa_remap_cb_t callback, void *arg) 5640 { 5641 remap_blkptr_cb_arg_t rbca; 5642 5643 if (!zfs_remap_blkptr_enable) 5644 return (B_FALSE); 5645 5646 if (!spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)) 5647 return (B_FALSE); 5648 5649 /* 5650 * Dedup BP's can not be remapped, because ddt_phys_select() depends 5651 * on DVA[0] being the same in the BP as in the DDT (dedup table). 5652 */ 5653 if (BP_GET_DEDUP(bp)) 5654 return (B_FALSE); 5655 5656 /* 5657 * Gang blocks can not be remapped, because 5658 * zio_checksum_gang_verifier() depends on the DVA[0] that's in 5659 * the BP used to read the gang block header (GBH) being the same 5660 * as the DVA[0] that we allocated for the GBH. 5661 */ 5662 if (BP_IS_GANG(bp)) 5663 return (B_FALSE); 5664 5665 /* 5666 * Embedded BP's have no DVA to remap. 5667 */ 5668 if (BP_GET_NDVAS(bp) < 1) 5669 return (B_FALSE); 5670 5671 /* 5672 * Cloned blocks can not be remapped since BRT depends on specific 5673 * vdev id and offset in the DVA[0] for its reference counting. 5674 */ 5675 if (!BP_IS_METADATA(bp) && brt_maybe_exists(spa, bp)) 5676 return (B_FALSE); 5677 5678 /* 5679 * Note: we only remap dva[0]. If we remapped other dvas, we 5680 * would no longer know what their phys birth txg is. 5681 */ 5682 dva_t *dva = &bp->blk_dva[0]; 5683 5684 uint64_t offset = DVA_GET_OFFSET(dva); 5685 uint64_t size = DVA_GET_ASIZE(dva); 5686 vdev_t *vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva)); 5687 5688 if (vd->vdev_ops->vdev_op_remap == NULL) 5689 return (B_FALSE); 5690 5691 rbca.rbca_bp = bp; 5692 rbca.rbca_cb = callback; 5693 rbca.rbca_remap_vd = vd; 5694 rbca.rbca_remap_offset = offset; 5695 rbca.rbca_cb_arg = arg; 5696 5697 /* 5698 * remap_blkptr_cb() will be called in order for each level of 5699 * indirection, until a concrete vdev is reached or a split block is 5700 * encountered. old_vd and old_offset are updated within the callback 5701 * as we go from the one indirect vdev to the next one (either concrete 5702 * or indirect again) in that order. 5703 */ 5704 vd->vdev_ops->vdev_op_remap(vd, offset, size, remap_blkptr_cb, &rbca); 5705 5706 /* Check if the DVA wasn't remapped because it is a split block */ 5707 if (DVA_GET_VDEV(&rbca.rbca_bp->blk_dva[0]) == vd->vdev_id) 5708 return (B_FALSE); 5709 5710 return (B_TRUE); 5711 } 5712 5713 /* 5714 * Undo the allocation of a DVA which happened in the given transaction group. 5715 */ 5716 void 5717 metaslab_unalloc_dva(spa_t *spa, const dva_t *dva, uint64_t txg) 5718 { 5719 metaslab_t *msp; 5720 vdev_t *vd; 5721 uint64_t vdev = DVA_GET_VDEV(dva); 5722 uint64_t offset = DVA_GET_OFFSET(dva); 5723 uint64_t size = DVA_GET_ASIZE(dva); 5724 5725 ASSERT(DVA_IS_VALID(dva)); 5726 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5727 5728 if (txg > spa_freeze_txg(spa)) 5729 return; 5730 5731 if ((vd = vdev_lookup_top(spa, vdev)) == NULL || !DVA_IS_VALID(dva) || 5732 (offset >> vd->vdev_ms_shift) >= vd->vdev_ms_count) { 5733 zfs_panic_recover("metaslab_free_dva(): bad DVA %llu:%llu:%llu", 5734 (u_longlong_t)vdev, (u_longlong_t)offset, 5735 (u_longlong_t)size); 5736 return; 5737 } 5738 5739 ASSERT(!vd->vdev_removing); 5740 ASSERT(vdev_is_concrete(vd)); 5741 ASSERT0(vd->vdev_indirect_config.vic_mapping_object); 5742 ASSERT0P(vd->vdev_indirect_mapping); 5743 5744 if (DVA_GET_GANG(dva)) 5745 size = vdev_gang_header_asize(vd); 5746 5747 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; 5748 5749 mutex_enter(&msp->ms_lock); 5750 zfs_range_tree_remove(msp->ms_allocating[txg & TXG_MASK], 5751 offset, size); 5752 msp->ms_allocating_total -= size; 5753 5754 VERIFY(!msp->ms_condensing); 5755 VERIFY3U(offset, >=, msp->ms_start); 5756 VERIFY3U(offset + size, <=, msp->ms_start + msp->ms_size); 5757 VERIFY3U(zfs_range_tree_space(msp->ms_allocatable) + size, <=, 5758 msp->ms_size); 5759 VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 5760 VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 5761 zfs_range_tree_add(msp->ms_allocatable, offset, size); 5762 mutex_exit(&msp->ms_lock); 5763 } 5764 5765 /* 5766 * Free the block represented by the given DVA. 5767 */ 5768 void 5769 metaslab_free_dva(spa_t *spa, const dva_t *dva, boolean_t checkpoint) 5770 { 5771 uint64_t vdev = DVA_GET_VDEV(dva); 5772 uint64_t offset = DVA_GET_OFFSET(dva); 5773 uint64_t size = DVA_GET_ASIZE(dva); 5774 vdev_t *vd = vdev_lookup_top(spa, vdev); 5775 5776 ASSERT(DVA_IS_VALID(dva)); 5777 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 5778 5779 if (DVA_GET_GANG(dva)) { 5780 size = vdev_gang_header_asize(vd); 5781 } 5782 5783 metaslab_free_impl(vd, offset, size, checkpoint); 5784 } 5785 5786 /* 5787 * Reserve some space for a future allocation. The reservation system must be 5788 * called before we call into the allocator. If there aren't enough space 5789 * available, the calling I/O will be throttled until another I/O completes and 5790 * its reservation is released. The function returns true if it was successful 5791 * in placing the reservation. 5792 */ 5793 boolean_t 5794 metaslab_class_throttle_reserve(metaslab_class_t *mc, int allocator, 5795 int copies, uint64_t io_size, boolean_t must, boolean_t *more) 5796 { 5797 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 5798 5799 ASSERT(mc->mc_alloc_throttle_enabled); 5800 if (mc->mc_alloc_io_size < io_size) { 5801 mc->mc_alloc_io_size = io_size; 5802 metaslab_class_balance(mc, B_FALSE); 5803 } 5804 if (must || mca->mca_reserved <= mc->mc_alloc_max) { 5805 /* 5806 * The potential race between compare and add is covered by the 5807 * allocator lock in most cases, or irrelevant due to must set. 5808 * But even if we assume some other non-existing scenario, the 5809 * worst that can happen is few more I/Os get to allocation 5810 * earlier, that is not a problem. 5811 */ 5812 int64_t delta = copies * io_size; 5813 *more = (atomic_add_64_nv(&mca->mca_reserved, delta) <= 5814 mc->mc_alloc_max); 5815 return (B_TRUE); 5816 } 5817 *more = B_FALSE; 5818 return (B_FALSE); 5819 } 5820 5821 boolean_t 5822 metaslab_class_throttle_unreserve(metaslab_class_t *mc, int allocator, 5823 int copies, uint64_t io_size) 5824 { 5825 metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator]; 5826 5827 ASSERT(mc->mc_alloc_throttle_enabled); 5828 int64_t delta = copies * io_size; 5829 return (atomic_add_64_nv(&mca->mca_reserved, -delta) <= 5830 mc->mc_alloc_max); 5831 } 5832 5833 static int 5834 metaslab_claim_concrete(vdev_t *vd, uint64_t offset, uint64_t size, 5835 uint64_t txg) 5836 { 5837 metaslab_t *msp; 5838 spa_t *spa = vd->vdev_spa; 5839 int error = 0; 5840 5841 if (offset >> vd->vdev_ms_shift >= vd->vdev_ms_count) 5842 return (SET_ERROR(ENXIO)); 5843 5844 ASSERT3P(vd->vdev_ms, !=, NULL); 5845 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; 5846 5847 mutex_enter(&msp->ms_lock); 5848 5849 if ((txg != 0 && spa_writeable(spa)) || !msp->ms_loaded) { 5850 error = metaslab_activate(msp, 0, METASLAB_WEIGHT_CLAIM); 5851 if (error == EBUSY) { 5852 ASSERT(msp->ms_loaded); 5853 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK); 5854 error = 0; 5855 } 5856 } 5857 5858 if (error == 0 && 5859 !zfs_range_tree_contains(msp->ms_allocatable, offset, size)) 5860 error = SET_ERROR(ENOENT); 5861 5862 if (error || txg == 0) { /* txg == 0 indicates dry run */ 5863 mutex_exit(&msp->ms_lock); 5864 return (error); 5865 } 5866 5867 VERIFY(!msp->ms_condensing); 5868 VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 5869 VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 5870 VERIFY3U(zfs_range_tree_space(msp->ms_allocatable) - size, <=, 5871 msp->ms_size); 5872 zfs_range_tree_remove(msp->ms_allocatable, offset, size); 5873 zfs_range_tree_clear(msp->ms_trim, offset, size); 5874 5875 if (spa_writeable(spa)) { /* don't dirty if we're zdb(8) */ 5876 metaslab_class_t *mc = msp->ms_group->mg_class; 5877 multilist_sublist_t *mls = 5878 multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp); 5879 if (!multilist_link_active(&msp->ms_class_txg_node)) { 5880 msp->ms_selected_txg = txg; 5881 multilist_sublist_insert_head(mls, msp); 5882 } 5883 multilist_sublist_unlock(mls); 5884 5885 if (zfs_range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK])) 5886 vdev_dirty(vd, VDD_METASLAB, msp, txg); 5887 zfs_range_tree_add(msp->ms_allocating[txg & TXG_MASK], 5888 offset, size); 5889 msp->ms_allocating_total += size; 5890 } 5891 5892 mutex_exit(&msp->ms_lock); 5893 5894 return (0); 5895 } 5896 5897 typedef struct metaslab_claim_cb_arg_t { 5898 uint64_t mcca_txg; 5899 int mcca_error; 5900 } metaslab_claim_cb_arg_t; 5901 5902 static void 5903 metaslab_claim_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset, 5904 uint64_t size, void *arg) 5905 { 5906 (void) inner_offset; 5907 metaslab_claim_cb_arg_t *mcca_arg = arg; 5908 5909 if (mcca_arg->mcca_error == 0) { 5910 mcca_arg->mcca_error = metaslab_claim_concrete(vd, offset, 5911 size, mcca_arg->mcca_txg); 5912 } 5913 } 5914 5915 int 5916 metaslab_claim_impl(vdev_t *vd, uint64_t offset, uint64_t size, uint64_t txg) 5917 { 5918 if (vd->vdev_ops->vdev_op_remap != NULL) { 5919 metaslab_claim_cb_arg_t arg; 5920 5921 /* 5922 * Only zdb(8) can claim on indirect vdevs. This is used 5923 * to detect leaks of mapped space (that are not accounted 5924 * for in the obsolete counts, spacemap, or bpobj). 5925 */ 5926 ASSERT(!spa_writeable(vd->vdev_spa)); 5927 arg.mcca_error = 0; 5928 arg.mcca_txg = txg; 5929 5930 vd->vdev_ops->vdev_op_remap(vd, offset, size, 5931 metaslab_claim_impl_cb, &arg); 5932 5933 if (arg.mcca_error == 0) { 5934 arg.mcca_error = metaslab_claim_concrete(vd, 5935 offset, size, txg); 5936 } 5937 return (arg.mcca_error); 5938 } else { 5939 return (metaslab_claim_concrete(vd, offset, size, txg)); 5940 } 5941 } 5942 5943 /* 5944 * Intent log support: upon opening the pool after a crash, notify the SPA 5945 * of blocks that the intent log has allocated for immediate write, but 5946 * which are still considered free by the SPA because the last transaction 5947 * group didn't commit yet. 5948 */ 5949 static int 5950 metaslab_claim_dva(spa_t *spa, const dva_t *dva, uint64_t txg) 5951 { 5952 uint64_t vdev = DVA_GET_VDEV(dva); 5953 uint64_t offset = DVA_GET_OFFSET(dva); 5954 uint64_t size = DVA_GET_ASIZE(dva); 5955 vdev_t *vd; 5956 5957 if ((vd = vdev_lookup_top(spa, vdev)) == NULL) { 5958 return (SET_ERROR(ENXIO)); 5959 } 5960 5961 ASSERT(DVA_IS_VALID(dva)); 5962 5963 if (DVA_GET_GANG(dva)) 5964 size = vdev_gang_header_asize(vd); 5965 5966 return (metaslab_claim_impl(vd, offset, size, txg)); 5967 } 5968 5969 int 5970 metaslab_alloc(spa_t *spa, metaslab_class_t *mc, uint64_t psize, blkptr_t *bp, 5971 int ndvas, uint64_t txg, const blkptr_t *hintbp, int flags, 5972 zio_alloc_list_t *zal, int allocator, const void *tag) 5973 { 5974 return (metaslab_alloc_range(spa, mc, psize, psize, bp, ndvas, txg, 5975 hintbp, flags, zal, allocator, tag, NULL)); 5976 } 5977 5978 int 5979 metaslab_alloc_range(spa_t *spa, metaslab_class_t *mc, uint64_t psize, 5980 uint64_t max_psize, blkptr_t *bp, int ndvas, uint64_t txg, 5981 const blkptr_t *hintbp, int flags, zio_alloc_list_t *zal, int allocator, 5982 const void *tag, uint64_t *actual_psize) 5983 { 5984 dva_t *dva = bp->blk_dva; 5985 const dva_t *hintdva = (hintbp != NULL) ? hintbp->blk_dva : NULL; 5986 int error = 0; 5987 5988 ASSERT0(BP_GET_LOGICAL_BIRTH(bp)); 5989 ASSERT0(BP_GET_RAW_PHYSICAL_BIRTH(bp)); 5990 5991 spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER); 5992 5993 if (mc->mc_allocator[allocator].mca_rotor == NULL) { 5994 /* no vdevs in this class */ 5995 spa_config_exit(spa, SCL_ALLOC, FTAG); 5996 return (SET_ERROR(ENOSPC)); 5997 } 5998 5999 ASSERT(ndvas > 0 && ndvas <= spa_max_replication(spa)); 6000 ASSERT0(BP_GET_NDVAS(bp)); 6001 ASSERT(hintbp == NULL || ndvas <= BP_GET_NDVAS(hintbp)); 6002 ASSERT3P(zal, !=, NULL); 6003 6004 uint64_t smallest_psize = UINT64_MAX; 6005 for (int d = 0; d < ndvas; d++) { 6006 uint64_t cur_psize = 0; 6007 error = metaslab_alloc_dva_range(spa, mc, psize, 6008 MIN(smallest_psize, max_psize), dva, d, hintdva, txg, 6009 flags, zal, allocator, actual_psize ? &cur_psize : NULL); 6010 if (error != 0) { 6011 for (d--; d >= 0; d--) { 6012 metaslab_unalloc_dva(spa, &dva[d], txg); 6013 metaslab_group_alloc_decrement(spa, 6014 DVA_GET_VDEV(&dva[d]), allocator, flags, 6015 psize, tag); 6016 memset(&dva[d], 0, sizeof (dva_t)); 6017 } 6018 spa_config_exit(spa, SCL_ALLOC, FTAG); 6019 return (error); 6020 } else { 6021 /* 6022 * Update the metaslab group's queue depth 6023 * based on the newly allocated dva. 6024 */ 6025 metaslab_group_alloc_increment(spa, 6026 DVA_GET_VDEV(&dva[d]), allocator, flags, psize, 6027 tag); 6028 if (actual_psize) 6029 smallest_psize = MIN(cur_psize, smallest_psize); 6030 } 6031 } 6032 ASSERT0(error); 6033 ASSERT(BP_GET_NDVAS(bp) == ndvas); 6034 if (actual_psize) 6035 *actual_psize = smallest_psize; 6036 6037 spa_config_exit(spa, SCL_ALLOC, FTAG); 6038 6039 BP_SET_BIRTH(bp, txg, 0); 6040 6041 return (0); 6042 } 6043 6044 void 6045 metaslab_free(spa_t *spa, const blkptr_t *bp, uint64_t txg, boolean_t now) 6046 { 6047 const dva_t *dva = bp->blk_dva; 6048 int ndvas = BP_GET_NDVAS(bp); 6049 6050 ASSERT(!BP_IS_HOLE(bp)); 6051 ASSERT(!now || BP_GET_BIRTH(bp) >= spa_syncing_txg(spa)); 6052 6053 /* 6054 * If we have a checkpoint for the pool we need to make sure that 6055 * the blocks that we free that are part of the checkpoint won't be 6056 * reused until the checkpoint is discarded or we revert to it. 6057 * 6058 * The checkpoint flag is passed down the metaslab_free code path 6059 * and is set whenever we want to add a block to the checkpoint's 6060 * accounting. That is, we "checkpoint" blocks that existed at the 6061 * time the checkpoint was created and are therefore referenced by 6062 * the checkpointed uberblock. 6063 * 6064 * Note that, we don't checkpoint any blocks if the current 6065 * syncing txg <= spa_checkpoint_txg. We want these frees to sync 6066 * normally as they will be referenced by the checkpointed uberblock. 6067 */ 6068 boolean_t checkpoint = B_FALSE; 6069 if (BP_GET_BIRTH(bp) <= spa->spa_checkpoint_txg && 6070 spa_syncing_txg(spa) > spa->spa_checkpoint_txg) { 6071 /* 6072 * At this point, if the block is part of the checkpoint 6073 * there is no way it was created in the current txg. 6074 */ 6075 ASSERT(!now); 6076 ASSERT3U(spa_syncing_txg(spa), ==, txg); 6077 checkpoint = B_TRUE; 6078 } 6079 6080 spa_config_enter(spa, SCL_FREE, FTAG, RW_READER); 6081 6082 for (int d = 0; d < ndvas; d++) { 6083 if (now) { 6084 metaslab_unalloc_dva(spa, &dva[d], txg); 6085 } else { 6086 ASSERT3U(txg, ==, spa_syncing_txg(spa)); 6087 metaslab_free_dva(spa, &dva[d], checkpoint); 6088 } 6089 } 6090 6091 spa_config_exit(spa, SCL_FREE, FTAG); 6092 } 6093 6094 int 6095 metaslab_claim(spa_t *spa, const blkptr_t *bp, uint64_t txg) 6096 { 6097 const dva_t *dva = bp->blk_dva; 6098 int ndvas = BP_GET_NDVAS(bp); 6099 int error = 0; 6100 6101 ASSERT(!BP_IS_HOLE(bp)); 6102 6103 if (txg != 0) { 6104 /* 6105 * First do a dry run to make sure all DVAs are claimable, 6106 * so we don't have to unwind from partial failures below. 6107 */ 6108 if ((error = metaslab_claim(spa, bp, 0)) != 0) 6109 return (error); 6110 } 6111 6112 spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER); 6113 6114 for (int d = 0; d < ndvas; d++) { 6115 error = metaslab_claim_dva(spa, &dva[d], txg); 6116 if (error != 0) 6117 break; 6118 } 6119 6120 spa_config_exit(spa, SCL_ALLOC, FTAG); 6121 6122 ASSERT(error == 0 || txg == 0); 6123 6124 return (error); 6125 } 6126 6127 static void 6128 metaslab_check_free_impl_cb(uint64_t inner, vdev_t *vd, uint64_t offset, 6129 uint64_t size, void *arg) 6130 { 6131 (void) inner, (void) arg; 6132 6133 if (vd->vdev_ops == &vdev_indirect_ops) 6134 return; 6135 6136 metaslab_check_free_impl(vd, offset, size); 6137 } 6138 6139 static void 6140 metaslab_check_free_impl(vdev_t *vd, uint64_t offset, uint64_t size) 6141 { 6142 metaslab_t *msp; 6143 spa_t *spa __maybe_unused = vd->vdev_spa; 6144 6145 if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0) 6146 return; 6147 6148 if (vd->vdev_ops->vdev_op_remap != NULL) { 6149 vd->vdev_ops->vdev_op_remap(vd, offset, size, 6150 metaslab_check_free_impl_cb, NULL); 6151 return; 6152 } 6153 6154 ASSERT(vdev_is_concrete(vd)); 6155 ASSERT3U(offset >> vd->vdev_ms_shift, <, vd->vdev_ms_count); 6156 ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0); 6157 6158 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift]; 6159 6160 mutex_enter(&msp->ms_lock); 6161 if (msp->ms_loaded) { 6162 zfs_range_tree_verify_not_present(msp->ms_allocatable, 6163 offset, size); 6164 } 6165 6166 /* 6167 * Check all segments that currently exist in the freeing pipeline. 6168 * 6169 * It would intuitively make sense to also check the current allocating 6170 * tree since metaslab_unalloc_dva() exists for extents that are 6171 * allocated and freed in the same sync pass within the same txg. 6172 * Unfortunately there are places (e.g. the ZIL) where we allocate a 6173 * segment but then we free part of it within the same txg 6174 * [see zil_sync()]. Thus, we don't call zfs_range_tree_verify() in the 6175 * current allocating tree. 6176 */ 6177 zfs_range_tree_verify_not_present(msp->ms_freeing, offset, size); 6178 zfs_range_tree_verify_not_present(msp->ms_checkpointing, offset, size); 6179 zfs_range_tree_verify_not_present(msp->ms_freed, offset, size); 6180 for (int j = 0; j < TXG_DEFER_SIZE; j++) 6181 zfs_range_tree_verify_not_present(msp->ms_defer[j], offset, 6182 size); 6183 zfs_range_tree_verify_not_present(msp->ms_trim, offset, size); 6184 mutex_exit(&msp->ms_lock); 6185 } 6186 6187 void 6188 metaslab_check_free(spa_t *spa, const blkptr_t *bp) 6189 { 6190 if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0) 6191 return; 6192 6193 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER); 6194 for (int i = 0; i < BP_GET_NDVAS(bp); i++) { 6195 uint64_t vdev = DVA_GET_VDEV(&bp->blk_dva[i]); 6196 vdev_t *vd = vdev_lookup_top(spa, vdev); 6197 uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]); 6198 uint64_t size = DVA_GET_ASIZE(&bp->blk_dva[i]); 6199 6200 if (DVA_GET_GANG(&bp->blk_dva[i])) 6201 size = vdev_gang_header_asize(vd); 6202 6203 ASSERT3P(vd, !=, NULL); 6204 6205 metaslab_check_free_impl(vd, offset, size); 6206 } 6207 spa_config_exit(spa, SCL_VDEV, FTAG); 6208 } 6209 6210 static void 6211 metaslab_group_disable_wait(metaslab_group_t *mg) 6212 { 6213 ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock)); 6214 while (mg->mg_disabled_updating) { 6215 cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock); 6216 } 6217 } 6218 6219 static void 6220 metaslab_group_disabled_increment(metaslab_group_t *mg) 6221 { 6222 ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock)); 6223 ASSERT(mg->mg_disabled_updating); 6224 6225 while (mg->mg_ms_disabled >= max_disabled_ms) { 6226 cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock); 6227 } 6228 mg->mg_ms_disabled++; 6229 ASSERT3U(mg->mg_ms_disabled, <=, max_disabled_ms); 6230 } 6231 6232 /* 6233 * Mark the metaslab as disabled to prevent any allocations on this metaslab. 6234 * We must also track how many metaslabs are currently disabled within a 6235 * metaslab group and limit them to prevent allocation failures from 6236 * occurring because all metaslabs are disabled. 6237 */ 6238 void 6239 metaslab_disable(metaslab_t *msp) 6240 { 6241 ASSERT(!MUTEX_HELD(&msp->ms_lock)); 6242 metaslab_group_t *mg = msp->ms_group; 6243 6244 mutex_enter(&mg->mg_ms_disabled_lock); 6245 6246 /* 6247 * To keep an accurate count of how many threads have disabled 6248 * a specific metaslab group, we only allow one thread to mark 6249 * the metaslab group at a time. This ensures that the value of 6250 * ms_disabled will be accurate when we decide to mark a metaslab 6251 * group as disabled. To do this we force all other threads 6252 * to wait till the metaslab's mg_disabled_updating flag is no 6253 * longer set. 6254 */ 6255 metaslab_group_disable_wait(mg); 6256 mg->mg_disabled_updating = B_TRUE; 6257 if (msp->ms_disabled == 0) { 6258 metaslab_group_disabled_increment(mg); 6259 } 6260 mutex_enter(&msp->ms_lock); 6261 msp->ms_disabled++; 6262 mutex_exit(&msp->ms_lock); 6263 6264 mg->mg_disabled_updating = B_FALSE; 6265 cv_broadcast(&mg->mg_ms_disabled_cv); 6266 mutex_exit(&mg->mg_ms_disabled_lock); 6267 } 6268 6269 void 6270 metaslab_enable(metaslab_t *msp, boolean_t sync, boolean_t unload) 6271 { 6272 metaslab_group_t *mg = msp->ms_group; 6273 spa_t *spa = mg->mg_vd->vdev_spa; 6274 6275 /* 6276 * Wait for the outstanding IO to be synced to prevent newly 6277 * allocated blocks from being overwritten. This used by 6278 * initialize and TRIM which are modifying unallocated space. 6279 */ 6280 if (sync) 6281 txg_wait_synced(spa_get_dsl(spa), 0); 6282 6283 mutex_enter(&mg->mg_ms_disabled_lock); 6284 mutex_enter(&msp->ms_lock); 6285 if (--msp->ms_disabled == 0) { 6286 mg->mg_ms_disabled--; 6287 cv_broadcast(&mg->mg_ms_disabled_cv); 6288 if (unload) 6289 metaslab_unload(msp); 6290 } 6291 mutex_exit(&msp->ms_lock); 6292 mutex_exit(&mg->mg_ms_disabled_lock); 6293 } 6294 6295 void 6296 metaslab_set_unflushed_dirty(metaslab_t *ms, boolean_t dirty) 6297 { 6298 ms->ms_unflushed_dirty = dirty; 6299 } 6300 6301 static void 6302 metaslab_update_ondisk_flush_data(metaslab_t *ms, dmu_tx_t *tx) 6303 { 6304 vdev_t *vd = ms->ms_group->mg_vd; 6305 spa_t *spa = vd->vdev_spa; 6306 objset_t *mos = spa_meta_objset(spa); 6307 6308 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)); 6309 6310 metaslab_unflushed_phys_t entry = { 6311 .msp_unflushed_txg = metaslab_unflushed_txg(ms), 6312 }; 6313 uint64_t entry_size = sizeof (entry); 6314 uint64_t entry_offset = ms->ms_id * entry_size; 6315 6316 uint64_t object = 0; 6317 int err = zap_lookup(mos, vd->vdev_top_zap, 6318 VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1, 6319 &object); 6320 if (err == ENOENT) { 6321 object = dmu_object_alloc(mos, DMU_OTN_UINT64_METADATA, 6322 SPA_OLD_MAXBLOCKSIZE, DMU_OT_NONE, 0, tx); 6323 VERIFY0(zap_add(mos, vd->vdev_top_zap, 6324 VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1, 6325 &object, tx)); 6326 } else { 6327 VERIFY0(err); 6328 } 6329 6330 dmu_write(spa_meta_objset(spa), object, entry_offset, entry_size, 6331 &entry, tx); 6332 } 6333 6334 void 6335 metaslab_set_unflushed_txg(metaslab_t *ms, uint64_t txg, dmu_tx_t *tx) 6336 { 6337 ms->ms_unflushed_txg = txg; 6338 metaslab_update_ondisk_flush_data(ms, tx); 6339 } 6340 6341 boolean_t 6342 metaslab_unflushed_dirty(metaslab_t *ms) 6343 { 6344 return (ms->ms_unflushed_dirty); 6345 } 6346 6347 uint64_t 6348 metaslab_unflushed_txg(metaslab_t *ms) 6349 { 6350 return (ms->ms_unflushed_txg); 6351 } 6352 6353 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, aliquot, U64, ZMOD_RW, 6354 "Allocation granularity (a.k.a. stripe size)"); 6355 6356 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_load, INT, ZMOD_RW, 6357 "Load all metaslabs when pool is first opened"); 6358 6359 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_unload, INT, ZMOD_RW, 6360 "Prevent metaslabs from being unloaded"); 6361 6362 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_enabled, INT, ZMOD_RW, 6363 "Preload potential metaslabs during reassessment"); 6364 6365 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_limit, UINT, ZMOD_RW, 6366 "Max number of metaslabs per group to preload"); 6367 6368 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay, UINT, ZMOD_RW, 6369 "Delay in txgs after metaslab was last used before unloading"); 6370 6371 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay_ms, UINT, ZMOD_RW, 6372 "Delay in milliseconds after metaslab was last used before unloading"); 6373 6374 ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, noalloc_threshold, UINT, ZMOD_RW, 6375 "Percentage of metaslab group size that should be free to make it " 6376 "eligible for allocation"); 6377 6378 ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, fragmentation_threshold, UINT, ZMOD_RW, 6379 "Percentage of metaslab group size that should be considered eligible " 6380 "for allocations unless all metaslab groups within the metaslab class " 6381 "have also crossed this threshold"); 6382 6383 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, fragmentation_factor_enabled, INT, 6384 ZMOD_RW, 6385 "Use the fragmentation metric to prefer less fragmented metaslabs"); 6386 6387 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, fragmentation_threshold, UINT, 6388 ZMOD_RW, "Fragmentation for metaslab to allow allocation"); 6389 6390 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, lba_weighting_enabled, INT, ZMOD_RW, 6391 "Prefer metaslabs with lower LBAs"); 6392 6393 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, bias_enabled, INT, ZMOD_RW, 6394 "Enable space-based metaslab group biasing"); 6395 6396 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, perf_bias, INT, ZMOD_RW, 6397 "Enable performance-based metaslab group biasing"); 6398 6399 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, segment_weight_enabled, INT, 6400 ZMOD_RW, "Enable segment-based metaslab selection"); 6401 6402 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, switch_threshold, INT, ZMOD_RW, 6403 "Segment-based metaslab selection maximum buckets before switching"); 6404 6405 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging, U64, ZMOD_RW, 6406 "Blocks larger than this size are sometimes forced to be gang blocks"); 6407 6408 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging_pct, UINT, ZMOD_RW, 6409 "Percentage of large blocks that will be forced to be gang blocks"); 6410 6411 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_max_search, UINT, ZMOD_RW, 6412 "Max distance (bytes) to search forward before using size tree"); 6413 6414 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_use_largest_segment, INT, ZMOD_RW, 6415 "When looking in size tree, use largest segment instead of exact fit"); 6416 6417 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, max_size_cache_sec, U64, 6418 ZMOD_RW, "How long to trust the cached max chunk size of a metaslab"); 6419 6420 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, mem_limit, UINT, ZMOD_RW, 6421 "Percentage of memory that can be used to store metaslab range trees"); 6422 6423 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, try_hard_before_gang, INT, 6424 ZMOD_RW, "Try hard to allocate before ganging"); 6425 6426 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, find_max_tries, UINT, ZMOD_RW, 6427 "Normally only consider this many of the best metaslabs in each vdev"); 6428 6429 ZFS_MODULE_PARAM_CALL(zfs, zfs_, active_allocator, 6430 param_set_active_allocator, param_get_charp, ZMOD_RW, 6431 "SPA active allocator"); 6432