1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 25 * Copyright (c) 2011, 2024 by Delphix. All rights reserved. 26 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 27 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 28 * Copyright 2013 Saso Kiselkov. All rights reserved. 29 * Copyright (c) 2014 Integros [integros.com] 30 * Copyright 2016 Toomas Soome <tsoome@me.com> 31 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 32 * Copyright 2018 Joyent, Inc. 33 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 34 * Copyright 2017 Joyent, Inc. 35 * Copyright (c) 2017, Intel Corporation. 36 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 37 * Copyright (c) 2023 Hewlett Packard Enterprise Development LP. 38 * Copyright (c) 2023, 2024, Klara Inc. 39 */ 40 41 /* 42 * SPA: Storage Pool Allocator 43 * 44 * This file contains all the routines used when modifying on-disk SPA state. 45 * This includes opening, importing, destroying, exporting a pool, and syncing a 46 * pool. 47 */ 48 49 #include <sys/zfs_context.h> 50 #include <sys/fm/fs/zfs.h> 51 #include <sys/spa_impl.h> 52 #include <sys/zio.h> 53 #include <sys/zio_checksum.h> 54 #include <sys/dmu.h> 55 #include <sys/dmu_tx.h> 56 #include <sys/zap.h> 57 #include <sys/zil.h> 58 #include <sys/brt.h> 59 #include <sys/ddt.h> 60 #include <sys/vdev_impl.h> 61 #include <sys/vdev_removal.h> 62 #include <sys/vdev_indirect_mapping.h> 63 #include <sys/vdev_indirect_births.h> 64 #include <sys/vdev_initialize.h> 65 #include <sys/vdev_rebuild.h> 66 #include <sys/vdev_trim.h> 67 #include <sys/vdev_disk.h> 68 #include <sys/vdev_raidz.h> 69 #include <sys/vdev_draid.h> 70 #include <sys/metaslab.h> 71 #include <sys/metaslab_impl.h> 72 #include <sys/mmp.h> 73 #include <sys/uberblock_impl.h> 74 #include <sys/txg.h> 75 #include <sys/avl.h> 76 #include <sys/bpobj.h> 77 #include <sys/dmu_traverse.h> 78 #include <sys/dmu_objset.h> 79 #include <sys/unique.h> 80 #include <sys/dsl_pool.h> 81 #include <sys/dsl_dataset.h> 82 #include <sys/dsl_dir.h> 83 #include <sys/dsl_prop.h> 84 #include <sys/dsl_synctask.h> 85 #include <sys/fs/zfs.h> 86 #include <sys/arc.h> 87 #include <sys/callb.h> 88 #include <sys/systeminfo.h> 89 #include <sys/zfs_ioctl.h> 90 #include <sys/dsl_scan.h> 91 #include <sys/zfeature.h> 92 #include <sys/dsl_destroy.h> 93 #include <sys/zvol.h> 94 95 #ifdef _KERNEL 96 #include <sys/fm/protocol.h> 97 #include <sys/fm/util.h> 98 #include <sys/callb.h> 99 #include <sys/zone.h> 100 #include <sys/vmsystm.h> 101 #endif /* _KERNEL */ 102 103 #include "zfs_prop.h" 104 #include "zfs_comutil.h" 105 #include <cityhash.h> 106 107 /* 108 * spa_thread() existed on Illumos as a parent thread for the various worker 109 * threads that actually run the pool, as a way to both reference the entire 110 * pool work as a single object, and to share properties like scheduling 111 * options. It has not yet been adapted to Linux or FreeBSD. This define is 112 * used to mark related parts of the code to make things easier for the reader, 113 * and to compile this code out. It can be removed when someone implements it, 114 * moves it to some Illumos-specific place, or removes it entirely. 115 */ 116 #undef HAVE_SPA_THREAD 117 118 /* 119 * The "System Duty Cycle" scheduling class is an Illumos feature to help 120 * prevent CPU-intensive kernel threads from affecting latency on interactive 121 * threads. It doesn't exist on Linux or FreeBSD, so the supporting code is 122 * gated behind a define. On Illumos SDC depends on spa_thread(), but 123 * spa_thread() also has other uses, so this is a separate define. 124 */ 125 #undef HAVE_SYSDC 126 127 /* 128 * The interval, in seconds, at which failed configuration cache file writes 129 * should be retried. 130 */ 131 int zfs_ccw_retry_interval = 300; 132 133 typedef enum zti_modes { 134 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 135 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 136 ZTI_MODE_SYNC, /* sync thread assigned */ 137 ZTI_MODE_NULL, /* don't create a taskq */ 138 ZTI_NMODES 139 } zti_modes_t; 140 141 #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 142 #define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 143 #define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 } 144 #define ZTI_SYNC { ZTI_MODE_SYNC, 0, 1 } 145 #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 146 147 #define ZTI_N(n) ZTI_P(n, 1) 148 #define ZTI_ONE ZTI_N(1) 149 150 typedef struct zio_taskq_info { 151 zti_modes_t zti_mode; 152 uint_t zti_value; 153 uint_t zti_count; 154 } zio_taskq_info_t; 155 156 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 157 "iss", "iss_h", "int", "int_h" 158 }; 159 160 /* 161 * This table defines the taskq settings for each ZFS I/O type. When 162 * initializing a pool, we use this table to create an appropriately sized 163 * taskq. Some operations are low volume and therefore have a small, static 164 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 165 * macros. Other operations process a large amount of data; the ZTI_SCALE 166 * macro causes us to create a taskq oriented for throughput. Some operations 167 * are so high frequency and short-lived that the taskq itself can become a 168 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 169 * additional degree of parallelism specified by the number of threads per- 170 * taskq and the number of taskqs; when dispatching an event in this case, the 171 * particular taskq is chosen at random. ZTI_SCALE uses a number of taskqs 172 * that scales with the number of CPUs. 173 * 174 * The different taskq priorities are to handle the different contexts (issue 175 * and interrupt) and then to reserve threads for high priority I/Os that 176 * need to be handled with minimum delay. Illumos taskq has unfair TQ_FRONT 177 * implementation, so separate high priority threads are used there. 178 */ 179 static zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 180 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 181 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 182 { ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */ 183 #ifdef illumos 184 { ZTI_SYNC, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */ 185 #else 186 { ZTI_SYNC, ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* WRITE */ 187 #endif 188 { ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 189 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 190 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FLUSH */ 191 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 192 }; 193 194 static void spa_sync_version(void *arg, dmu_tx_t *tx); 195 static void spa_sync_props(void *arg, dmu_tx_t *tx); 196 static boolean_t spa_has_active_shared_spare(spa_t *spa); 197 static int spa_load_impl(spa_t *spa, spa_import_type_t type, 198 const char **ereport); 199 static void spa_vdev_resilver_done(spa_t *spa); 200 201 /* 202 * Percentage of all CPUs that can be used by the metaslab preload taskq. 203 */ 204 static uint_t metaslab_preload_pct = 50; 205 206 static uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 207 static uint_t zio_taskq_batch_tpq; /* threads per taskq */ 208 209 #ifdef HAVE_SYSDC 210 static const boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 211 static const uint_t zio_taskq_basedc = 80; /* base duty cycle */ 212 #endif 213 214 #ifdef HAVE_SPA_THREAD 215 static const boolean_t spa_create_process = B_TRUE; /* no process => no sysdc */ 216 #endif 217 218 static uint_t zio_taskq_write_tpq = 16; 219 220 /* 221 * Report any spa_load_verify errors found, but do not fail spa_load. 222 * This is used by zdb to analyze non-idle pools. 223 */ 224 boolean_t spa_load_verify_dryrun = B_FALSE; 225 226 /* 227 * Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ). 228 * This is used by zdb for spacemaps verification. 229 */ 230 boolean_t spa_mode_readable_spacemaps = B_FALSE; 231 232 /* 233 * This (illegal) pool name is used when temporarily importing a spa_t in order 234 * to get the vdev stats associated with the imported devices. 235 */ 236 #define TRYIMPORT_NAME "$import" 237 238 /* 239 * For debugging purposes: print out vdev tree during pool import. 240 */ 241 static int spa_load_print_vdev_tree = B_FALSE; 242 243 /* 244 * A non-zero value for zfs_max_missing_tvds means that we allow importing 245 * pools with missing top-level vdevs. This is strictly intended for advanced 246 * pool recovery cases since missing data is almost inevitable. Pools with 247 * missing devices can only be imported read-only for safety reasons, and their 248 * fail-mode will be automatically set to "continue". 249 * 250 * With 1 missing vdev we should be able to import the pool and mount all 251 * datasets. User data that was not modified after the missing device has been 252 * added should be recoverable. This means that snapshots created prior to the 253 * addition of that device should be completely intact. 254 * 255 * With 2 missing vdevs, some datasets may fail to mount since there are 256 * dataset statistics that are stored as regular metadata. Some data might be 257 * recoverable if those vdevs were added recently. 258 * 259 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 260 * may be missing entirely. Chances of data recovery are very low. Note that 261 * there are also risks of performing an inadvertent rewind as we might be 262 * missing all the vdevs with the latest uberblocks. 263 */ 264 uint64_t zfs_max_missing_tvds = 0; 265 266 /* 267 * The parameters below are similar to zfs_max_missing_tvds but are only 268 * intended for a preliminary open of the pool with an untrusted config which 269 * might be incomplete or out-dated. 270 * 271 * We are more tolerant for pools opened from a cachefile since we could have 272 * an out-dated cachefile where a device removal was not registered. 273 * We could have set the limit arbitrarily high but in the case where devices 274 * are really missing we would want to return the proper error codes; we chose 275 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 276 * and we get a chance to retrieve the trusted config. 277 */ 278 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 279 280 /* 281 * In the case where config was assembled by scanning device paths (/dev/dsks 282 * by default) we are less tolerant since all the existing devices should have 283 * been detected and we want spa_load to return the right error codes. 284 */ 285 uint64_t zfs_max_missing_tvds_scan = 0; 286 287 /* 288 * Debugging aid that pauses spa_sync() towards the end. 289 */ 290 static const boolean_t zfs_pause_spa_sync = B_FALSE; 291 292 /* 293 * Variables to indicate the livelist condense zthr func should wait at certain 294 * points for the livelist to be removed - used to test condense/destroy races 295 */ 296 static int zfs_livelist_condense_zthr_pause = 0; 297 static int zfs_livelist_condense_sync_pause = 0; 298 299 /* 300 * Variables to track whether or not condense cancellation has been 301 * triggered in testing. 302 */ 303 static int zfs_livelist_condense_sync_cancel = 0; 304 static int zfs_livelist_condense_zthr_cancel = 0; 305 306 /* 307 * Variable to track whether or not extra ALLOC blkptrs were added to a 308 * livelist entry while it was being condensed (caused by the way we track 309 * remapped blkptrs in dbuf_remap_impl) 310 */ 311 static int zfs_livelist_condense_new_alloc = 0; 312 313 /* 314 * ========================================================================== 315 * SPA properties routines 316 * ========================================================================== 317 */ 318 319 /* 320 * Add a (source=src, propname=propval) list to an nvlist. 321 */ 322 static void 323 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, const char *strval, 324 uint64_t intval, zprop_source_t src) 325 { 326 const char *propname = zpool_prop_to_name(prop); 327 nvlist_t *propval; 328 329 propval = fnvlist_alloc(); 330 fnvlist_add_uint64(propval, ZPROP_SOURCE, src); 331 332 if (strval != NULL) 333 fnvlist_add_string(propval, ZPROP_VALUE, strval); 334 else 335 fnvlist_add_uint64(propval, ZPROP_VALUE, intval); 336 337 fnvlist_add_nvlist(nvl, propname, propval); 338 nvlist_free(propval); 339 } 340 341 static int 342 spa_prop_add(spa_t *spa, const char *propname, nvlist_t *outnvl) 343 { 344 zpool_prop_t prop = zpool_name_to_prop(propname); 345 zprop_source_t src = ZPROP_SRC_NONE; 346 uint64_t intval; 347 int err; 348 349 /* 350 * NB: Not all properties lookups via this API require 351 * the spa props lock, so they must explicitly grab it here. 352 */ 353 switch (prop) { 354 case ZPOOL_PROP_DEDUPCACHED: 355 err = ddt_get_pool_dedup_cached(spa, &intval); 356 if (err != 0) 357 return (SET_ERROR(err)); 358 break; 359 default: 360 return (SET_ERROR(EINVAL)); 361 } 362 363 spa_prop_add_list(outnvl, prop, NULL, intval, src); 364 365 return (0); 366 } 367 368 int 369 spa_prop_get_nvlist(spa_t *spa, char **props, unsigned int n_props, 370 nvlist_t *outnvl) 371 { 372 int err = 0; 373 374 if (props == NULL) 375 return (0); 376 377 for (unsigned int i = 0; i < n_props && err == 0; i++) { 378 err = spa_prop_add(spa, props[i], outnvl); 379 } 380 381 return (err); 382 } 383 384 /* 385 * Add a user property (source=src, propname=propval) to an nvlist. 386 */ 387 static void 388 spa_prop_add_user(nvlist_t *nvl, const char *propname, char *strval, 389 zprop_source_t src) 390 { 391 nvlist_t *propval; 392 393 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); 394 VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); 395 VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0); 396 VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0); 397 nvlist_free(propval); 398 } 399 400 /* 401 * Get property values from the spa configuration. 402 */ 403 static void 404 spa_prop_get_config(spa_t *spa, nvlist_t *nv) 405 { 406 vdev_t *rvd = spa->spa_root_vdev; 407 dsl_pool_t *pool = spa->spa_dsl_pool; 408 uint64_t size, alloc, cap, version; 409 const zprop_source_t src = ZPROP_SRC_NONE; 410 spa_config_dirent_t *dp; 411 metaslab_class_t *mc = spa_normal_class(spa); 412 413 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 414 415 if (rvd != NULL) { 416 alloc = metaslab_class_get_alloc(mc); 417 alloc += metaslab_class_get_alloc(spa_special_class(spa)); 418 alloc += metaslab_class_get_alloc(spa_dedup_class(spa)); 419 alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa)); 420 421 size = metaslab_class_get_space(mc); 422 size += metaslab_class_get_space(spa_special_class(spa)); 423 size += metaslab_class_get_space(spa_dedup_class(spa)); 424 size += metaslab_class_get_space(spa_embedded_log_class(spa)); 425 426 spa_prop_add_list(nv, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 427 spa_prop_add_list(nv, ZPOOL_PROP_SIZE, NULL, size, src); 428 spa_prop_add_list(nv, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 429 spa_prop_add_list(nv, ZPOOL_PROP_FREE, NULL, 430 size - alloc, src); 431 spa_prop_add_list(nv, ZPOOL_PROP_CHECKPOINT, NULL, 432 spa->spa_checkpoint_info.sci_dspace, src); 433 434 spa_prop_add_list(nv, ZPOOL_PROP_FRAGMENTATION, NULL, 435 metaslab_class_fragmentation(mc), src); 436 spa_prop_add_list(nv, ZPOOL_PROP_EXPANDSZ, NULL, 437 metaslab_class_expandable_space(mc), src); 438 spa_prop_add_list(nv, ZPOOL_PROP_READONLY, NULL, 439 (spa_mode(spa) == SPA_MODE_READ), src); 440 441 cap = (size == 0) ? 0 : (alloc * 100 / size); 442 spa_prop_add_list(nv, ZPOOL_PROP_CAPACITY, NULL, cap, src); 443 444 spa_prop_add_list(nv, ZPOOL_PROP_DEDUPRATIO, NULL, 445 ddt_get_pool_dedup_ratio(spa), src); 446 spa_prop_add_list(nv, ZPOOL_PROP_BCLONEUSED, NULL, 447 brt_get_used(spa), src); 448 spa_prop_add_list(nv, ZPOOL_PROP_BCLONESAVED, NULL, 449 brt_get_saved(spa), src); 450 spa_prop_add_list(nv, ZPOOL_PROP_BCLONERATIO, NULL, 451 brt_get_ratio(spa), src); 452 453 spa_prop_add_list(nv, ZPOOL_PROP_DEDUP_TABLE_SIZE, NULL, 454 ddt_get_ddt_dsize(spa), src); 455 spa_prop_add_list(nv, ZPOOL_PROP_HEALTH, NULL, 456 rvd->vdev_state, src); 457 spa_prop_add_list(nv, ZPOOL_PROP_LAST_SCRUBBED_TXG, NULL, 458 spa_get_last_scrubbed_txg(spa), src); 459 460 version = spa_version(spa); 461 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 462 spa_prop_add_list(nv, ZPOOL_PROP_VERSION, NULL, 463 version, ZPROP_SRC_DEFAULT); 464 } else { 465 spa_prop_add_list(nv, ZPOOL_PROP_VERSION, NULL, 466 version, ZPROP_SRC_LOCAL); 467 } 468 spa_prop_add_list(nv, ZPOOL_PROP_LOAD_GUID, 469 NULL, spa_load_guid(spa), src); 470 } 471 472 if (pool != NULL) { 473 /* 474 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 475 * when opening pools before this version freedir will be NULL. 476 */ 477 if (pool->dp_free_dir != NULL) { 478 spa_prop_add_list(nv, ZPOOL_PROP_FREEING, NULL, 479 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 480 src); 481 } else { 482 spa_prop_add_list(nv, ZPOOL_PROP_FREEING, 483 NULL, 0, src); 484 } 485 486 if (pool->dp_leak_dir != NULL) { 487 spa_prop_add_list(nv, ZPOOL_PROP_LEAKED, NULL, 488 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 489 src); 490 } else { 491 spa_prop_add_list(nv, ZPOOL_PROP_LEAKED, 492 NULL, 0, src); 493 } 494 } 495 496 spa_prop_add_list(nv, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 497 498 if (spa->spa_comment != NULL) { 499 spa_prop_add_list(nv, ZPOOL_PROP_COMMENT, spa->spa_comment, 500 0, ZPROP_SRC_LOCAL); 501 } 502 503 if (spa->spa_compatibility != NULL) { 504 spa_prop_add_list(nv, ZPOOL_PROP_COMPATIBILITY, 505 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 506 } 507 508 if (spa->spa_root != NULL) 509 spa_prop_add_list(nv, ZPOOL_PROP_ALTROOT, spa->spa_root, 510 0, ZPROP_SRC_LOCAL); 511 512 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 513 spa_prop_add_list(nv, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 514 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 515 } else { 516 spa_prop_add_list(nv, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 517 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 518 } 519 520 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 521 spa_prop_add_list(nv, ZPOOL_PROP_MAXDNODESIZE, NULL, 522 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 523 } else { 524 spa_prop_add_list(nv, ZPOOL_PROP_MAXDNODESIZE, NULL, 525 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 526 } 527 528 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 529 if (dp->scd_path == NULL) { 530 spa_prop_add_list(nv, ZPOOL_PROP_CACHEFILE, 531 "none", 0, ZPROP_SRC_LOCAL); 532 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 533 spa_prop_add_list(nv, ZPOOL_PROP_CACHEFILE, 534 dp->scd_path, 0, ZPROP_SRC_LOCAL); 535 } 536 } 537 } 538 539 /* 540 * Get zpool property values. 541 */ 542 int 543 spa_prop_get(spa_t *spa, nvlist_t *nv) 544 { 545 objset_t *mos = spa->spa_meta_objset; 546 zap_cursor_t zc; 547 zap_attribute_t *za; 548 dsl_pool_t *dp; 549 int err = 0; 550 551 dp = spa_get_dsl(spa); 552 dsl_pool_config_enter(dp, FTAG); 553 za = zap_attribute_alloc(); 554 mutex_enter(&spa->spa_props_lock); 555 556 /* 557 * Get properties from the spa config. 558 */ 559 spa_prop_get_config(spa, nv); 560 561 /* If no pool property object, no more prop to get. */ 562 if (mos == NULL || spa->spa_pool_props_object == 0) 563 goto out; 564 565 /* 566 * Get properties from the MOS pool property object. 567 */ 568 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 569 (err = zap_cursor_retrieve(&zc, za)) == 0; 570 zap_cursor_advance(&zc)) { 571 uint64_t intval = 0; 572 char *strval = NULL; 573 zprop_source_t src = ZPROP_SRC_DEFAULT; 574 zpool_prop_t prop; 575 576 if ((prop = zpool_name_to_prop(za->za_name)) == 577 ZPOOL_PROP_INVAL && !zfs_prop_user(za->za_name)) 578 continue; 579 580 switch (za->za_integer_length) { 581 case 8: 582 /* integer property */ 583 if (za->za_first_integer != 584 zpool_prop_default_numeric(prop)) 585 src = ZPROP_SRC_LOCAL; 586 587 if (prop == ZPOOL_PROP_BOOTFS) { 588 dsl_dataset_t *ds = NULL; 589 590 err = dsl_dataset_hold_obj(dp, 591 za->za_first_integer, FTAG, &ds); 592 if (err != 0) 593 break; 594 595 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 596 KM_SLEEP); 597 dsl_dataset_name(ds, strval); 598 dsl_dataset_rele(ds, FTAG); 599 } else { 600 strval = NULL; 601 intval = za->za_first_integer; 602 } 603 604 spa_prop_add_list(nv, prop, strval, intval, src); 605 606 if (strval != NULL) 607 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 608 609 break; 610 611 case 1: 612 /* string property */ 613 strval = kmem_alloc(za->za_num_integers, KM_SLEEP); 614 err = zap_lookup(mos, spa->spa_pool_props_object, 615 za->za_name, 1, za->za_num_integers, strval); 616 if (err) { 617 kmem_free(strval, za->za_num_integers); 618 break; 619 } 620 if (prop != ZPOOL_PROP_INVAL) { 621 spa_prop_add_list(nv, prop, strval, 0, src); 622 } else { 623 src = ZPROP_SRC_LOCAL; 624 spa_prop_add_user(nv, za->za_name, strval, 625 src); 626 } 627 kmem_free(strval, za->za_num_integers); 628 break; 629 630 default: 631 break; 632 } 633 } 634 zap_cursor_fini(&zc); 635 out: 636 mutex_exit(&spa->spa_props_lock); 637 dsl_pool_config_exit(dp, FTAG); 638 zap_attribute_free(za); 639 640 if (err && err != ENOENT) 641 return (err); 642 643 return (0); 644 } 645 646 /* 647 * Validate the given pool properties nvlist and modify the list 648 * for the property values to be set. 649 */ 650 static int 651 spa_prop_validate(spa_t *spa, nvlist_t *props) 652 { 653 nvpair_t *elem; 654 int error = 0, reset_bootfs = 0; 655 uint64_t objnum = 0; 656 boolean_t has_feature = B_FALSE; 657 658 elem = NULL; 659 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 660 uint64_t intval; 661 const char *strval, *slash, *check, *fname; 662 const char *propname = nvpair_name(elem); 663 zpool_prop_t prop = zpool_name_to_prop(propname); 664 665 switch (prop) { 666 case ZPOOL_PROP_INVAL: 667 /* 668 * Sanitize the input. 669 */ 670 if (zfs_prop_user(propname)) { 671 if (strlen(propname) >= ZAP_MAXNAMELEN) { 672 error = SET_ERROR(ENAMETOOLONG); 673 break; 674 } 675 676 if (strlen(fnvpair_value_string(elem)) >= 677 ZAP_MAXVALUELEN) { 678 error = SET_ERROR(E2BIG); 679 break; 680 } 681 } else if (zpool_prop_feature(propname)) { 682 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 683 error = SET_ERROR(EINVAL); 684 break; 685 } 686 687 if (nvpair_value_uint64(elem, &intval) != 0) { 688 error = SET_ERROR(EINVAL); 689 break; 690 } 691 692 if (intval != 0) { 693 error = SET_ERROR(EINVAL); 694 break; 695 } 696 697 fname = strchr(propname, '@') + 1; 698 if (zfeature_lookup_name(fname, NULL) != 0) { 699 error = SET_ERROR(EINVAL); 700 break; 701 } 702 703 has_feature = B_TRUE; 704 } else { 705 error = SET_ERROR(EINVAL); 706 break; 707 } 708 break; 709 710 case ZPOOL_PROP_VERSION: 711 error = nvpair_value_uint64(elem, &intval); 712 if (!error && 713 (intval < spa_version(spa) || 714 intval > SPA_VERSION_BEFORE_FEATURES || 715 has_feature)) 716 error = SET_ERROR(EINVAL); 717 break; 718 719 case ZPOOL_PROP_DEDUP_TABLE_QUOTA: 720 error = nvpair_value_uint64(elem, &intval); 721 break; 722 723 case ZPOOL_PROP_DELEGATION: 724 case ZPOOL_PROP_AUTOREPLACE: 725 case ZPOOL_PROP_LISTSNAPS: 726 case ZPOOL_PROP_AUTOEXPAND: 727 case ZPOOL_PROP_AUTOTRIM: 728 error = nvpair_value_uint64(elem, &intval); 729 if (!error && intval > 1) 730 error = SET_ERROR(EINVAL); 731 break; 732 733 case ZPOOL_PROP_MULTIHOST: 734 error = nvpair_value_uint64(elem, &intval); 735 if (!error && intval > 1) 736 error = SET_ERROR(EINVAL); 737 738 if (!error) { 739 uint32_t hostid = zone_get_hostid(NULL); 740 if (hostid) 741 spa->spa_hostid = hostid; 742 else 743 error = SET_ERROR(ENOTSUP); 744 } 745 746 break; 747 748 case ZPOOL_PROP_BOOTFS: 749 /* 750 * If the pool version is less than SPA_VERSION_BOOTFS, 751 * or the pool is still being created (version == 0), 752 * the bootfs property cannot be set. 753 */ 754 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 755 error = SET_ERROR(ENOTSUP); 756 break; 757 } 758 759 /* 760 * Make sure the vdev config is bootable 761 */ 762 if (!vdev_is_bootable(spa->spa_root_vdev)) { 763 error = SET_ERROR(ENOTSUP); 764 break; 765 } 766 767 reset_bootfs = 1; 768 769 error = nvpair_value_string(elem, &strval); 770 771 if (!error) { 772 objset_t *os; 773 774 if (strval == NULL || strval[0] == '\0') { 775 objnum = zpool_prop_default_numeric( 776 ZPOOL_PROP_BOOTFS); 777 break; 778 } 779 780 error = dmu_objset_hold(strval, FTAG, &os); 781 if (error != 0) 782 break; 783 784 /* Must be ZPL. */ 785 if (dmu_objset_type(os) != DMU_OST_ZFS) { 786 error = SET_ERROR(ENOTSUP); 787 } else { 788 objnum = dmu_objset_id(os); 789 } 790 dmu_objset_rele(os, FTAG); 791 } 792 break; 793 794 case ZPOOL_PROP_FAILUREMODE: 795 error = nvpair_value_uint64(elem, &intval); 796 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 797 error = SET_ERROR(EINVAL); 798 799 /* 800 * This is a special case which only occurs when 801 * the pool has completely failed. This allows 802 * the user to change the in-core failmode property 803 * without syncing it out to disk (I/Os might 804 * currently be blocked). We do this by returning 805 * EIO to the caller (spa_prop_set) to trick it 806 * into thinking we encountered a property validation 807 * error. 808 */ 809 if (!error && spa_suspended(spa)) { 810 spa->spa_failmode = intval; 811 error = SET_ERROR(EIO); 812 } 813 break; 814 815 case ZPOOL_PROP_CACHEFILE: 816 if ((error = nvpair_value_string(elem, &strval)) != 0) 817 break; 818 819 if (strval[0] == '\0') 820 break; 821 822 if (strcmp(strval, "none") == 0) 823 break; 824 825 if (strval[0] != '/') { 826 error = SET_ERROR(EINVAL); 827 break; 828 } 829 830 slash = strrchr(strval, '/'); 831 ASSERT(slash != NULL); 832 833 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 834 strcmp(slash, "/..") == 0) 835 error = SET_ERROR(EINVAL); 836 break; 837 838 case ZPOOL_PROP_COMMENT: 839 if ((error = nvpair_value_string(elem, &strval)) != 0) 840 break; 841 for (check = strval; *check != '\0'; check++) { 842 if (!isprint(*check)) { 843 error = SET_ERROR(EINVAL); 844 break; 845 } 846 } 847 if (strlen(strval) > ZPROP_MAX_COMMENT) 848 error = SET_ERROR(E2BIG); 849 break; 850 851 default: 852 break; 853 } 854 855 if (error) 856 break; 857 } 858 859 (void) nvlist_remove_all(props, 860 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 861 862 if (!error && reset_bootfs) { 863 error = nvlist_remove(props, 864 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 865 866 if (!error) { 867 error = nvlist_add_uint64(props, 868 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 869 } 870 } 871 872 return (error); 873 } 874 875 void 876 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 877 { 878 const char *cachefile; 879 spa_config_dirent_t *dp; 880 881 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 882 &cachefile) != 0) 883 return; 884 885 dp = kmem_alloc(sizeof (spa_config_dirent_t), 886 KM_SLEEP); 887 888 if (cachefile[0] == '\0') 889 dp->scd_path = spa_strdup(spa_config_path); 890 else if (strcmp(cachefile, "none") == 0) 891 dp->scd_path = NULL; 892 else 893 dp->scd_path = spa_strdup(cachefile); 894 895 list_insert_head(&spa->spa_config_list, dp); 896 if (need_sync) 897 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 898 } 899 900 int 901 spa_prop_set(spa_t *spa, nvlist_t *nvp) 902 { 903 int error; 904 nvpair_t *elem = NULL; 905 boolean_t need_sync = B_FALSE; 906 907 if ((error = spa_prop_validate(spa, nvp)) != 0) 908 return (error); 909 910 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 911 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 912 913 if (prop == ZPOOL_PROP_CACHEFILE || 914 prop == ZPOOL_PROP_ALTROOT || 915 prop == ZPOOL_PROP_READONLY) 916 continue; 917 918 if (prop == ZPOOL_PROP_INVAL && 919 zfs_prop_user(nvpair_name(elem))) { 920 need_sync = B_TRUE; 921 break; 922 } 923 924 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 925 uint64_t ver = 0; 926 927 if (prop == ZPOOL_PROP_VERSION) { 928 VERIFY(nvpair_value_uint64(elem, &ver) == 0); 929 } else { 930 ASSERT(zpool_prop_feature(nvpair_name(elem))); 931 ver = SPA_VERSION_FEATURES; 932 need_sync = B_TRUE; 933 } 934 935 /* Save time if the version is already set. */ 936 if (ver == spa_version(spa)) 937 continue; 938 939 /* 940 * In addition to the pool directory object, we might 941 * create the pool properties object, the features for 942 * read object, the features for write object, or the 943 * feature descriptions object. 944 */ 945 error = dsl_sync_task(spa->spa_name, NULL, 946 spa_sync_version, &ver, 947 6, ZFS_SPACE_CHECK_RESERVED); 948 if (error) 949 return (error); 950 continue; 951 } 952 953 need_sync = B_TRUE; 954 break; 955 } 956 957 if (need_sync) { 958 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 959 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 960 } 961 962 return (0); 963 } 964 965 /* 966 * If the bootfs property value is dsobj, clear it. 967 */ 968 void 969 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 970 { 971 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 972 VERIFY(zap_remove(spa->spa_meta_objset, 973 spa->spa_pool_props_object, 974 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 975 spa->spa_bootfs = 0; 976 } 977 } 978 979 static int 980 spa_change_guid_check(void *arg, dmu_tx_t *tx) 981 { 982 uint64_t *newguid __maybe_unused = arg; 983 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 984 vdev_t *rvd = spa->spa_root_vdev; 985 uint64_t vdev_state; 986 987 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 988 int error = (spa_has_checkpoint(spa)) ? 989 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 990 return (SET_ERROR(error)); 991 } 992 993 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 994 vdev_state = rvd->vdev_state; 995 spa_config_exit(spa, SCL_STATE, FTAG); 996 997 if (vdev_state != VDEV_STATE_HEALTHY) 998 return (SET_ERROR(ENXIO)); 999 1000 ASSERT3U(spa_guid(spa), !=, *newguid); 1001 1002 return (0); 1003 } 1004 1005 static void 1006 spa_change_guid_sync(void *arg, dmu_tx_t *tx) 1007 { 1008 uint64_t *newguid = arg; 1009 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 1010 uint64_t oldguid; 1011 vdev_t *rvd = spa->spa_root_vdev; 1012 1013 oldguid = spa_guid(spa); 1014 1015 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 1016 rvd->vdev_guid = *newguid; 1017 rvd->vdev_guid_sum += (*newguid - oldguid); 1018 vdev_config_dirty(rvd); 1019 spa_config_exit(spa, SCL_STATE, FTAG); 1020 1021 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 1022 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 1023 } 1024 1025 /* 1026 * Change the GUID for the pool. This is done so that we can later 1027 * re-import a pool built from a clone of our own vdevs. We will modify 1028 * the root vdev's guid, our own pool guid, and then mark all of our 1029 * vdevs dirty. Note that we must make sure that all our vdevs are 1030 * online when we do this, or else any vdevs that weren't present 1031 * would be orphaned from our pool. We are also going to issue a 1032 * sysevent to update any watchers. 1033 * 1034 * The GUID of the pool will be changed to the value pointed to by guidp. 1035 * The GUID may not be set to the reserverd value of 0. 1036 * The new GUID will be generated if guidp is NULL. 1037 */ 1038 int 1039 spa_change_guid(spa_t *spa, const uint64_t *guidp) 1040 { 1041 uint64_t guid; 1042 int error; 1043 1044 mutex_enter(&spa->spa_vdev_top_lock); 1045 mutex_enter(&spa_namespace_lock); 1046 1047 if (guidp != NULL) { 1048 guid = *guidp; 1049 if (guid == 0) { 1050 error = SET_ERROR(EINVAL); 1051 goto out; 1052 } 1053 1054 if (spa_guid_exists(guid, 0)) { 1055 error = SET_ERROR(EEXIST); 1056 goto out; 1057 } 1058 } else { 1059 guid = spa_generate_guid(NULL); 1060 } 1061 1062 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 1063 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 1064 1065 if (error == 0) { 1066 /* 1067 * Clear the kobj flag from all the vdevs to allow 1068 * vdev_cache_process_kobj_evt() to post events to all the 1069 * vdevs since GUID is updated. 1070 */ 1071 vdev_clear_kobj_evt(spa->spa_root_vdev); 1072 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 1073 vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]); 1074 1075 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 1076 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 1077 } 1078 1079 out: 1080 mutex_exit(&spa_namespace_lock); 1081 mutex_exit(&spa->spa_vdev_top_lock); 1082 1083 return (error); 1084 } 1085 1086 /* 1087 * ========================================================================== 1088 * SPA state manipulation (open/create/destroy/import/export) 1089 * ========================================================================== 1090 */ 1091 1092 static int 1093 spa_error_entry_compare(const void *a, const void *b) 1094 { 1095 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 1096 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 1097 int ret; 1098 1099 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 1100 sizeof (zbookmark_phys_t)); 1101 1102 return (TREE_ISIGN(ret)); 1103 } 1104 1105 /* 1106 * Utility function which retrieves copies of the current logs and 1107 * re-initializes them in the process. 1108 */ 1109 void 1110 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 1111 { 1112 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 1113 1114 memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t)); 1115 memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t)); 1116 1117 avl_create(&spa->spa_errlist_scrub, 1118 spa_error_entry_compare, sizeof (spa_error_entry_t), 1119 offsetof(spa_error_entry_t, se_avl)); 1120 avl_create(&spa->spa_errlist_last, 1121 spa_error_entry_compare, sizeof (spa_error_entry_t), 1122 offsetof(spa_error_entry_t, se_avl)); 1123 } 1124 1125 static void 1126 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1127 { 1128 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 1129 enum zti_modes mode = ztip->zti_mode; 1130 uint_t value = ztip->zti_value; 1131 uint_t count = ztip->zti_count; 1132 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1133 uint_t cpus, flags = TASKQ_DYNAMIC; 1134 1135 switch (mode) { 1136 case ZTI_MODE_FIXED: 1137 ASSERT3U(value, >, 0); 1138 break; 1139 1140 case ZTI_MODE_SYNC: 1141 1142 /* 1143 * Create one wr_iss taskq for every 'zio_taskq_write_tpq' CPUs, 1144 * not to exceed the number of spa allocators, and align to it. 1145 */ 1146 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 1147 count = MAX(1, cpus / MAX(1, zio_taskq_write_tpq)); 1148 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1149 count = MIN(count, spa->spa_alloc_count); 1150 while (spa->spa_alloc_count % count != 0 && 1151 spa->spa_alloc_count < count * 2) 1152 count--; 1153 1154 /* 1155 * zio_taskq_batch_pct is unbounded and may exceed 100%, but no 1156 * single taskq may have more threads than 100% of online cpus. 1157 */ 1158 value = (zio_taskq_batch_pct + count / 2) / count; 1159 value = MIN(value, 100); 1160 flags |= TASKQ_THREADS_CPU_PCT; 1161 break; 1162 1163 case ZTI_MODE_SCALE: 1164 flags |= TASKQ_THREADS_CPU_PCT; 1165 /* 1166 * We want more taskqs to reduce lock contention, but we want 1167 * less for better request ordering and CPU utilization. 1168 */ 1169 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 1170 if (zio_taskq_batch_tpq > 0) { 1171 count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) / 1172 zio_taskq_batch_tpq); 1173 } else { 1174 /* 1175 * Prefer 6 threads per taskq, but no more taskqs 1176 * than threads in them on large systems. For 80%: 1177 * 1178 * taskq taskq total 1179 * cpus taskqs percent threads threads 1180 * ------- ------- ------- ------- ------- 1181 * 1 1 80% 1 1 1182 * 2 1 80% 1 1 1183 * 4 1 80% 3 3 1184 * 8 2 40% 3 6 1185 * 16 3 27% 4 12 1186 * 32 5 16% 5 25 1187 * 64 7 11% 7 49 1188 * 128 10 8% 10 100 1189 * 256 14 6% 15 210 1190 */ 1191 count = 1 + cpus / 6; 1192 while (count * count > cpus) 1193 count--; 1194 } 1195 /* Limit each taskq within 100% to not trigger assertion. */ 1196 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1197 value = (zio_taskq_batch_pct + count / 2) / count; 1198 break; 1199 1200 case ZTI_MODE_NULL: 1201 tqs->stqs_count = 0; 1202 tqs->stqs_taskq = NULL; 1203 return; 1204 1205 default: 1206 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1207 "spa_taskqs_init()", 1208 zio_type_name[t], zio_taskq_types[q], mode, value); 1209 break; 1210 } 1211 1212 ASSERT3U(count, >, 0); 1213 tqs->stqs_count = count; 1214 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1215 1216 for (uint_t i = 0; i < count; i++) { 1217 taskq_t *tq; 1218 char name[32]; 1219 1220 if (count > 1) 1221 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1222 zio_type_name[t], zio_taskq_types[q], i); 1223 else 1224 (void) snprintf(name, sizeof (name), "%s_%s", 1225 zio_type_name[t], zio_taskq_types[q]); 1226 1227 #ifdef HAVE_SYSDC 1228 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1229 (void) zio_taskq_basedc; 1230 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1231 spa->spa_proc, zio_taskq_basedc, flags); 1232 } else { 1233 #endif 1234 pri_t pri = maxclsyspri; 1235 /* 1236 * The write issue taskq can be extremely CPU 1237 * intensive. Run it at slightly less important 1238 * priority than the other taskqs. 1239 * 1240 * Under Linux and FreeBSD this means incrementing 1241 * the priority value as opposed to platforms like 1242 * illumos where it should be decremented. 1243 * 1244 * On FreeBSD, if priorities divided by four (RQ_PPQ) 1245 * are equal then a difference between them is 1246 * insignificant. 1247 */ 1248 if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) { 1249 #if defined(__linux__) 1250 pri++; 1251 #elif defined(__FreeBSD__) 1252 pri += 4; 1253 #else 1254 #error "unknown OS" 1255 #endif 1256 } 1257 tq = taskq_create_proc(name, value, pri, 50, 1258 INT_MAX, spa->spa_proc, flags); 1259 #ifdef HAVE_SYSDC 1260 } 1261 #endif 1262 1263 tqs->stqs_taskq[i] = tq; 1264 } 1265 } 1266 1267 static void 1268 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1269 { 1270 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1271 1272 if (tqs->stqs_taskq == NULL) { 1273 ASSERT3U(tqs->stqs_count, ==, 0); 1274 return; 1275 } 1276 1277 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1278 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1279 taskq_destroy(tqs->stqs_taskq[i]); 1280 } 1281 1282 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1283 tqs->stqs_taskq = NULL; 1284 } 1285 1286 #ifdef _KERNEL 1287 /* 1288 * The READ and WRITE rows of zio_taskqs are configurable at module load time 1289 * by setting zio_taskq_read or zio_taskq_write. 1290 * 1291 * Example (the defaults for READ and WRITE) 1292 * zio_taskq_read='fixed,1,8 null scale null' 1293 * zio_taskq_write='sync null scale null' 1294 * 1295 * Each sets the entire row at a time. 1296 * 1297 * 'fixed' is parameterised: fixed,Q,T where Q is number of taskqs, T is number 1298 * of threads per taskq. 1299 * 1300 * 'null' can only be set on the high-priority queues (queue selection for 1301 * high-priority queues will fall back to the regular queue if the high-pri 1302 * is NULL. 1303 */ 1304 static const char *const modes[ZTI_NMODES] = { 1305 "fixed", "scale", "sync", "null" 1306 }; 1307 1308 /* Parse the incoming config string. Modifies cfg */ 1309 static int 1310 spa_taskq_param_set(zio_type_t t, char *cfg) 1311 { 1312 int err = 0; 1313 1314 zio_taskq_info_t row[ZIO_TASKQ_TYPES] = {{0}}; 1315 1316 char *next = cfg, *tok, *c; 1317 1318 /* 1319 * Parse out each element from the string and fill `row`. The entire 1320 * row has to be set at once, so any errors are flagged by just 1321 * breaking out of this loop early. 1322 */ 1323 uint_t q; 1324 for (q = 0; q < ZIO_TASKQ_TYPES; q++) { 1325 /* `next` is the start of the config */ 1326 if (next == NULL) 1327 break; 1328 1329 /* Eat up leading space */ 1330 while (isspace(*next)) 1331 next++; 1332 if (*next == '\0') 1333 break; 1334 1335 /* Mode ends at space or end of string */ 1336 tok = next; 1337 next = strchr(tok, ' '); 1338 if (next != NULL) *next++ = '\0'; 1339 1340 /* Parameters start after a comma */ 1341 c = strchr(tok, ','); 1342 if (c != NULL) *c++ = '\0'; 1343 1344 /* Match mode string */ 1345 uint_t mode; 1346 for (mode = 0; mode < ZTI_NMODES; mode++) 1347 if (strcmp(tok, modes[mode]) == 0) 1348 break; 1349 if (mode == ZTI_NMODES) 1350 break; 1351 1352 /* Invalid canary */ 1353 row[q].zti_mode = ZTI_NMODES; 1354 1355 /* Per-mode setup */ 1356 switch (mode) { 1357 1358 /* 1359 * FIXED is parameterised: number of queues, and number of 1360 * threads per queue. 1361 */ 1362 case ZTI_MODE_FIXED: { 1363 /* No parameters? */ 1364 if (c == NULL || *c == '\0') 1365 break; 1366 1367 /* Find next parameter */ 1368 tok = c; 1369 c = strchr(tok, ','); 1370 if (c == NULL) 1371 break; 1372 1373 /* Take digits and convert */ 1374 unsigned long long nq; 1375 if (!(isdigit(*tok))) 1376 break; 1377 err = ddi_strtoull(tok, &tok, 10, &nq); 1378 /* Must succeed and also end at the next param sep */ 1379 if (err != 0 || tok != c) 1380 break; 1381 1382 /* Move past the comma */ 1383 tok++; 1384 /* Need another number */ 1385 if (!(isdigit(*tok))) 1386 break; 1387 /* Remember start to make sure we moved */ 1388 c = tok; 1389 1390 /* Take digits */ 1391 unsigned long long ntpq; 1392 err = ddi_strtoull(tok, &tok, 10, &ntpq); 1393 /* Must succeed, and moved forward */ 1394 if (err != 0 || tok == c || *tok != '\0') 1395 break; 1396 1397 /* 1398 * sanity; zero queues/threads make no sense, and 1399 * 16K is almost certainly more than anyone will ever 1400 * need and avoids silly numbers like UINT32_MAX 1401 */ 1402 if (nq == 0 || nq >= 16384 || 1403 ntpq == 0 || ntpq >= 16384) 1404 break; 1405 1406 const zio_taskq_info_t zti = ZTI_P(ntpq, nq); 1407 row[q] = zti; 1408 break; 1409 } 1410 1411 case ZTI_MODE_SCALE: { 1412 const zio_taskq_info_t zti = ZTI_SCALE; 1413 row[q] = zti; 1414 break; 1415 } 1416 1417 case ZTI_MODE_SYNC: { 1418 const zio_taskq_info_t zti = ZTI_SYNC; 1419 row[q] = zti; 1420 break; 1421 } 1422 1423 case ZTI_MODE_NULL: { 1424 /* 1425 * Can only null the high-priority queues; the general- 1426 * purpose ones have to exist. 1427 */ 1428 if (q != ZIO_TASKQ_ISSUE_HIGH && 1429 q != ZIO_TASKQ_INTERRUPT_HIGH) 1430 break; 1431 1432 const zio_taskq_info_t zti = ZTI_NULL; 1433 row[q] = zti; 1434 break; 1435 } 1436 1437 default: 1438 break; 1439 } 1440 1441 /* Ensure we set a mode */ 1442 if (row[q].zti_mode == ZTI_NMODES) 1443 break; 1444 } 1445 1446 /* Didn't get a full row, fail */ 1447 if (q < ZIO_TASKQ_TYPES) 1448 return (SET_ERROR(EINVAL)); 1449 1450 /* Eat trailing space */ 1451 if (next != NULL) 1452 while (isspace(*next)) 1453 next++; 1454 1455 /* If there's anything left over then fail */ 1456 if (next != NULL && *next != '\0') 1457 return (SET_ERROR(EINVAL)); 1458 1459 /* Success! Copy it into the real config */ 1460 for (q = 0; q < ZIO_TASKQ_TYPES; q++) 1461 zio_taskqs[t][q] = row[q]; 1462 1463 return (0); 1464 } 1465 1466 static int 1467 spa_taskq_param_get(zio_type_t t, char *buf, boolean_t add_newline) 1468 { 1469 int pos = 0; 1470 1471 /* Build paramater string from live config */ 1472 const char *sep = ""; 1473 for (uint_t q = 0; q < ZIO_TASKQ_TYPES; q++) { 1474 const zio_taskq_info_t *zti = &zio_taskqs[t][q]; 1475 if (zti->zti_mode == ZTI_MODE_FIXED) 1476 pos += sprintf(&buf[pos], "%s%s,%u,%u", sep, 1477 modes[zti->zti_mode], zti->zti_count, 1478 zti->zti_value); 1479 else 1480 pos += sprintf(&buf[pos], "%s%s", sep, 1481 modes[zti->zti_mode]); 1482 sep = " "; 1483 } 1484 1485 if (add_newline) 1486 buf[pos++] = '\n'; 1487 buf[pos] = '\0'; 1488 1489 return (pos); 1490 } 1491 1492 #ifdef __linux__ 1493 static int 1494 spa_taskq_read_param_set(const char *val, zfs_kernel_param_t *kp) 1495 { 1496 char *cfg = kmem_strdup(val); 1497 int err = spa_taskq_param_set(ZIO_TYPE_READ, cfg); 1498 kmem_free(cfg, strlen(val)+1); 1499 return (-err); 1500 } 1501 static int 1502 spa_taskq_read_param_get(char *buf, zfs_kernel_param_t *kp) 1503 { 1504 return (spa_taskq_param_get(ZIO_TYPE_READ, buf, TRUE)); 1505 } 1506 1507 static int 1508 spa_taskq_write_param_set(const char *val, zfs_kernel_param_t *kp) 1509 { 1510 char *cfg = kmem_strdup(val); 1511 int err = spa_taskq_param_set(ZIO_TYPE_WRITE, cfg); 1512 kmem_free(cfg, strlen(val)+1); 1513 return (-err); 1514 } 1515 static int 1516 spa_taskq_write_param_get(char *buf, zfs_kernel_param_t *kp) 1517 { 1518 return (spa_taskq_param_get(ZIO_TYPE_WRITE, buf, TRUE)); 1519 } 1520 #else 1521 /* 1522 * On FreeBSD load-time parameters can be set up before malloc() is available, 1523 * so we have to do all the parsing work on the stack. 1524 */ 1525 #define SPA_TASKQ_PARAM_MAX (128) 1526 1527 static int 1528 spa_taskq_read_param(ZFS_MODULE_PARAM_ARGS) 1529 { 1530 char buf[SPA_TASKQ_PARAM_MAX]; 1531 int err; 1532 1533 (void) spa_taskq_param_get(ZIO_TYPE_READ, buf, FALSE); 1534 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 1535 if (err || req->newptr == NULL) 1536 return (err); 1537 return (spa_taskq_param_set(ZIO_TYPE_READ, buf)); 1538 } 1539 1540 static int 1541 spa_taskq_write_param(ZFS_MODULE_PARAM_ARGS) 1542 { 1543 char buf[SPA_TASKQ_PARAM_MAX]; 1544 int err; 1545 1546 (void) spa_taskq_param_get(ZIO_TYPE_WRITE, buf, FALSE); 1547 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 1548 if (err || req->newptr == NULL) 1549 return (err); 1550 return (spa_taskq_param_set(ZIO_TYPE_WRITE, buf)); 1551 } 1552 #endif 1553 #endif /* _KERNEL */ 1554 1555 /* 1556 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1557 * Note that a type may have multiple discrete taskqs to avoid lock contention 1558 * on the taskq itself. 1559 */ 1560 void 1561 spa_taskq_dispatch(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1562 task_func_t *func, zio_t *zio, boolean_t cutinline) 1563 { 1564 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1565 taskq_t *tq; 1566 1567 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1568 ASSERT3U(tqs->stqs_count, !=, 0); 1569 1570 /* 1571 * NB: We are assuming that the zio can only be dispatched 1572 * to a single taskq at a time. It would be a grievous error 1573 * to dispatch the zio to another taskq at the same time. 1574 */ 1575 ASSERT(zio); 1576 ASSERT(taskq_empty_ent(&zio->io_tqent)); 1577 1578 if (tqs->stqs_count == 1) { 1579 tq = tqs->stqs_taskq[0]; 1580 } else if ((t == ZIO_TYPE_WRITE) && (q == ZIO_TASKQ_ISSUE) && 1581 ZIO_HAS_ALLOCATOR(zio)) { 1582 tq = tqs->stqs_taskq[zio->io_allocator % tqs->stqs_count]; 1583 } else { 1584 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1585 } 1586 1587 taskq_dispatch_ent(tq, func, zio, cutinline ? TQ_FRONT : 0, 1588 &zio->io_tqent); 1589 } 1590 1591 static void 1592 spa_create_zio_taskqs(spa_t *spa) 1593 { 1594 for (int t = 0; t < ZIO_TYPES; t++) { 1595 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1596 spa_taskqs_init(spa, t, q); 1597 } 1598 } 1599 } 1600 1601 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1602 static void 1603 spa_thread(void *arg) 1604 { 1605 psetid_t zio_taskq_psrset_bind = PS_NONE; 1606 callb_cpr_t cprinfo; 1607 1608 spa_t *spa = arg; 1609 user_t *pu = PTOU(curproc); 1610 1611 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1612 spa->spa_name); 1613 1614 ASSERT(curproc != &p0); 1615 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1616 "zpool-%s", spa->spa_name); 1617 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1618 1619 /* bind this thread to the requested psrset */ 1620 if (zio_taskq_psrset_bind != PS_NONE) { 1621 pool_lock(); 1622 mutex_enter(&cpu_lock); 1623 mutex_enter(&pidlock); 1624 mutex_enter(&curproc->p_lock); 1625 1626 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1627 0, NULL, NULL) == 0) { 1628 curthread->t_bind_pset = zio_taskq_psrset_bind; 1629 } else { 1630 cmn_err(CE_WARN, 1631 "Couldn't bind process for zfs pool \"%s\" to " 1632 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1633 } 1634 1635 mutex_exit(&curproc->p_lock); 1636 mutex_exit(&pidlock); 1637 mutex_exit(&cpu_lock); 1638 pool_unlock(); 1639 } 1640 1641 #ifdef HAVE_SYSDC 1642 if (zio_taskq_sysdc) { 1643 sysdc_thread_enter(curthread, 100, 0); 1644 } 1645 #endif 1646 1647 spa->spa_proc = curproc; 1648 spa->spa_did = curthread->t_did; 1649 1650 spa_create_zio_taskqs(spa); 1651 1652 mutex_enter(&spa->spa_proc_lock); 1653 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1654 1655 spa->spa_proc_state = SPA_PROC_ACTIVE; 1656 cv_broadcast(&spa->spa_proc_cv); 1657 1658 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1659 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1660 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1661 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1662 1663 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1664 spa->spa_proc_state = SPA_PROC_GONE; 1665 spa->spa_proc = &p0; 1666 cv_broadcast(&spa->spa_proc_cv); 1667 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1668 1669 mutex_enter(&curproc->p_lock); 1670 lwp_exit(); 1671 } 1672 #endif 1673 1674 extern metaslab_ops_t *metaslab_allocator(spa_t *spa); 1675 1676 /* 1677 * Activate an uninitialized pool. 1678 */ 1679 static void 1680 spa_activate(spa_t *spa, spa_mode_t mode) 1681 { 1682 metaslab_ops_t *msp = metaslab_allocator(spa); 1683 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1684 1685 spa->spa_state = POOL_STATE_ACTIVE; 1686 spa->spa_mode = mode; 1687 spa->spa_read_spacemaps = spa_mode_readable_spacemaps; 1688 1689 spa->spa_normal_class = metaslab_class_create(spa, msp, B_FALSE); 1690 spa->spa_log_class = metaslab_class_create(spa, msp, B_TRUE); 1691 spa->spa_embedded_log_class = metaslab_class_create(spa, msp, B_TRUE); 1692 spa->spa_special_class = metaslab_class_create(spa, msp, B_FALSE); 1693 spa->spa_dedup_class = metaslab_class_create(spa, msp, B_FALSE); 1694 1695 /* Try to create a covering process */ 1696 mutex_enter(&spa->spa_proc_lock); 1697 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1698 ASSERT(spa->spa_proc == &p0); 1699 spa->spa_did = 0; 1700 1701 #ifdef HAVE_SPA_THREAD 1702 /* Only create a process if we're going to be around a while. */ 1703 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1704 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1705 NULL, 0) == 0) { 1706 spa->spa_proc_state = SPA_PROC_CREATED; 1707 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1708 cv_wait(&spa->spa_proc_cv, 1709 &spa->spa_proc_lock); 1710 } 1711 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1712 ASSERT(spa->spa_proc != &p0); 1713 ASSERT(spa->spa_did != 0); 1714 } else { 1715 #ifdef _KERNEL 1716 cmn_err(CE_WARN, 1717 "Couldn't create process for zfs pool \"%s\"\n", 1718 spa->spa_name); 1719 #endif 1720 } 1721 } 1722 #endif /* HAVE_SPA_THREAD */ 1723 mutex_exit(&spa->spa_proc_lock); 1724 1725 /* If we didn't create a process, we need to create our taskqs. */ 1726 if (spa->spa_proc == &p0) { 1727 spa_create_zio_taskqs(spa); 1728 } 1729 1730 for (size_t i = 0; i < TXG_SIZE; i++) { 1731 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1732 ZIO_FLAG_CANFAIL); 1733 } 1734 1735 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1736 offsetof(vdev_t, vdev_config_dirty_node)); 1737 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1738 offsetof(objset_t, os_evicting_node)); 1739 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1740 offsetof(vdev_t, vdev_state_dirty_node)); 1741 1742 txg_list_create(&spa->spa_vdev_txg_list, spa, 1743 offsetof(struct vdev, vdev_txg_node)); 1744 1745 avl_create(&spa->spa_errlist_scrub, 1746 spa_error_entry_compare, sizeof (spa_error_entry_t), 1747 offsetof(spa_error_entry_t, se_avl)); 1748 avl_create(&spa->spa_errlist_last, 1749 spa_error_entry_compare, sizeof (spa_error_entry_t), 1750 offsetof(spa_error_entry_t, se_avl)); 1751 avl_create(&spa->spa_errlist_healed, 1752 spa_error_entry_compare, sizeof (spa_error_entry_t), 1753 offsetof(spa_error_entry_t, se_avl)); 1754 1755 spa_activate_os(spa); 1756 1757 spa_keystore_init(&spa->spa_keystore); 1758 1759 /* 1760 * This taskq is used to perform zvol-minor-related tasks 1761 * asynchronously. This has several advantages, including easy 1762 * resolution of various deadlocks. 1763 * 1764 * The taskq must be single threaded to ensure tasks are always 1765 * processed in the order in which they were dispatched. 1766 * 1767 * A taskq per pool allows one to keep the pools independent. 1768 * This way if one pool is suspended, it will not impact another. 1769 * 1770 * The preferred location to dispatch a zvol minor task is a sync 1771 * task. In this context, there is easy access to the spa_t and minimal 1772 * error handling is required because the sync task must succeed. 1773 */ 1774 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1775 1, INT_MAX, 0); 1776 1777 /* 1778 * The taskq to preload metaslabs. 1779 */ 1780 spa->spa_metaslab_taskq = taskq_create("z_metaslab", 1781 metaslab_preload_pct, maxclsyspri, 1, INT_MAX, 1782 TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1783 1784 /* 1785 * Taskq dedicated to prefetcher threads: this is used to prevent the 1786 * pool traverse code from monopolizing the global (and limited) 1787 * system_taskq by inappropriately scheduling long running tasks on it. 1788 */ 1789 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1790 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1791 1792 /* 1793 * The taskq to upgrade datasets in this pool. Currently used by 1794 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1795 */ 1796 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1797 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1798 } 1799 1800 /* 1801 * Opposite of spa_activate(). 1802 */ 1803 static void 1804 spa_deactivate(spa_t *spa) 1805 { 1806 ASSERT(spa->spa_sync_on == B_FALSE); 1807 ASSERT(spa->spa_dsl_pool == NULL); 1808 ASSERT(spa->spa_root_vdev == NULL); 1809 ASSERT(spa->spa_async_zio_root == NULL); 1810 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1811 1812 spa_evicting_os_wait(spa); 1813 1814 if (spa->spa_zvol_taskq) { 1815 taskq_destroy(spa->spa_zvol_taskq); 1816 spa->spa_zvol_taskq = NULL; 1817 } 1818 1819 if (spa->spa_metaslab_taskq) { 1820 taskq_destroy(spa->spa_metaslab_taskq); 1821 spa->spa_metaslab_taskq = NULL; 1822 } 1823 1824 if (spa->spa_prefetch_taskq) { 1825 taskq_destroy(spa->spa_prefetch_taskq); 1826 spa->spa_prefetch_taskq = NULL; 1827 } 1828 1829 if (spa->spa_upgrade_taskq) { 1830 taskq_destroy(spa->spa_upgrade_taskq); 1831 spa->spa_upgrade_taskq = NULL; 1832 } 1833 1834 txg_list_destroy(&spa->spa_vdev_txg_list); 1835 1836 list_destroy(&spa->spa_config_dirty_list); 1837 list_destroy(&spa->spa_evicting_os_list); 1838 list_destroy(&spa->spa_state_dirty_list); 1839 1840 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 1841 1842 for (int t = 0; t < ZIO_TYPES; t++) { 1843 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1844 spa_taskqs_fini(spa, t, q); 1845 } 1846 } 1847 1848 for (size_t i = 0; i < TXG_SIZE; i++) { 1849 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1850 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1851 spa->spa_txg_zio[i] = NULL; 1852 } 1853 1854 metaslab_class_destroy(spa->spa_normal_class); 1855 spa->spa_normal_class = NULL; 1856 1857 metaslab_class_destroy(spa->spa_log_class); 1858 spa->spa_log_class = NULL; 1859 1860 metaslab_class_destroy(spa->spa_embedded_log_class); 1861 spa->spa_embedded_log_class = NULL; 1862 1863 metaslab_class_destroy(spa->spa_special_class); 1864 spa->spa_special_class = NULL; 1865 1866 metaslab_class_destroy(spa->spa_dedup_class); 1867 spa->spa_dedup_class = NULL; 1868 1869 /* 1870 * If this was part of an import or the open otherwise failed, we may 1871 * still have errors left in the queues. Empty them just in case. 1872 */ 1873 spa_errlog_drain(spa); 1874 avl_destroy(&spa->spa_errlist_scrub); 1875 avl_destroy(&spa->spa_errlist_last); 1876 avl_destroy(&spa->spa_errlist_healed); 1877 1878 spa_keystore_fini(&spa->spa_keystore); 1879 1880 spa->spa_state = POOL_STATE_UNINITIALIZED; 1881 1882 mutex_enter(&spa->spa_proc_lock); 1883 if (spa->spa_proc_state != SPA_PROC_NONE) { 1884 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1885 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 1886 cv_broadcast(&spa->spa_proc_cv); 1887 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 1888 ASSERT(spa->spa_proc != &p0); 1889 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1890 } 1891 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 1892 spa->spa_proc_state = SPA_PROC_NONE; 1893 } 1894 ASSERT(spa->spa_proc == &p0); 1895 mutex_exit(&spa->spa_proc_lock); 1896 1897 /* 1898 * We want to make sure spa_thread() has actually exited the ZFS 1899 * module, so that the module can't be unloaded out from underneath 1900 * it. 1901 */ 1902 if (spa->spa_did != 0) { 1903 thread_join(spa->spa_did); 1904 spa->spa_did = 0; 1905 } 1906 1907 spa_deactivate_os(spa); 1908 1909 } 1910 1911 /* 1912 * Verify a pool configuration, and construct the vdev tree appropriately. This 1913 * will create all the necessary vdevs in the appropriate layout, with each vdev 1914 * in the CLOSED state. This will prep the pool before open/creation/import. 1915 * All vdev validation is done by the vdev_alloc() routine. 1916 */ 1917 int 1918 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 1919 uint_t id, int atype) 1920 { 1921 nvlist_t **child; 1922 uint_t children; 1923 int error; 1924 1925 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 1926 return (error); 1927 1928 if ((*vdp)->vdev_ops->vdev_op_leaf) 1929 return (0); 1930 1931 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 1932 &child, &children); 1933 1934 if (error == ENOENT) 1935 return (0); 1936 1937 if (error) { 1938 vdev_free(*vdp); 1939 *vdp = NULL; 1940 return (SET_ERROR(EINVAL)); 1941 } 1942 1943 for (int c = 0; c < children; c++) { 1944 vdev_t *vd; 1945 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 1946 atype)) != 0) { 1947 vdev_free(*vdp); 1948 *vdp = NULL; 1949 return (error); 1950 } 1951 } 1952 1953 ASSERT(*vdp != NULL); 1954 1955 return (0); 1956 } 1957 1958 static boolean_t 1959 spa_should_flush_logs_on_unload(spa_t *spa) 1960 { 1961 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 1962 return (B_FALSE); 1963 1964 if (!spa_writeable(spa)) 1965 return (B_FALSE); 1966 1967 if (!spa->spa_sync_on) 1968 return (B_FALSE); 1969 1970 if (spa_state(spa) != POOL_STATE_EXPORTED) 1971 return (B_FALSE); 1972 1973 if (zfs_keep_log_spacemaps_at_export) 1974 return (B_FALSE); 1975 1976 return (B_TRUE); 1977 } 1978 1979 /* 1980 * Opens a transaction that will set the flag that will instruct 1981 * spa_sync to attempt to flush all the metaslabs for that txg. 1982 */ 1983 static void 1984 spa_unload_log_sm_flush_all(spa_t *spa) 1985 { 1986 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 1987 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT)); 1988 1989 ASSERT3U(spa->spa_log_flushall_txg, ==, 0); 1990 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); 1991 1992 dmu_tx_commit(tx); 1993 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 1994 } 1995 1996 static void 1997 spa_unload_log_sm_metadata(spa_t *spa) 1998 { 1999 void *cookie = NULL; 2000 spa_log_sm_t *sls; 2001 log_summary_entry_t *e; 2002 2003 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 2004 &cookie)) != NULL) { 2005 VERIFY0(sls->sls_mscount); 2006 kmem_free(sls, sizeof (spa_log_sm_t)); 2007 } 2008 2009 while ((e = list_remove_head(&spa->spa_log_summary)) != NULL) { 2010 VERIFY0(e->lse_mscount); 2011 kmem_free(e, sizeof (log_summary_entry_t)); 2012 } 2013 2014 spa->spa_unflushed_stats.sus_nblocks = 0; 2015 spa->spa_unflushed_stats.sus_memused = 0; 2016 spa->spa_unflushed_stats.sus_blocklimit = 0; 2017 } 2018 2019 static void 2020 spa_destroy_aux_threads(spa_t *spa) 2021 { 2022 if (spa->spa_condense_zthr != NULL) { 2023 zthr_destroy(spa->spa_condense_zthr); 2024 spa->spa_condense_zthr = NULL; 2025 } 2026 if (spa->spa_checkpoint_discard_zthr != NULL) { 2027 zthr_destroy(spa->spa_checkpoint_discard_zthr); 2028 spa->spa_checkpoint_discard_zthr = NULL; 2029 } 2030 if (spa->spa_livelist_delete_zthr != NULL) { 2031 zthr_destroy(spa->spa_livelist_delete_zthr); 2032 spa->spa_livelist_delete_zthr = NULL; 2033 } 2034 if (spa->spa_livelist_condense_zthr != NULL) { 2035 zthr_destroy(spa->spa_livelist_condense_zthr); 2036 spa->spa_livelist_condense_zthr = NULL; 2037 } 2038 if (spa->spa_raidz_expand_zthr != NULL) { 2039 zthr_destroy(spa->spa_raidz_expand_zthr); 2040 spa->spa_raidz_expand_zthr = NULL; 2041 } 2042 } 2043 2044 /* 2045 * Opposite of spa_load(). 2046 */ 2047 static void 2048 spa_unload(spa_t *spa) 2049 { 2050 ASSERT(MUTEX_HELD(&spa_namespace_lock) || 2051 spa->spa_export_thread == curthread); 2052 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 2053 2054 spa_import_progress_remove(spa_guid(spa)); 2055 spa_load_note(spa, "UNLOADING"); 2056 2057 spa_wake_waiters(spa); 2058 2059 /* 2060 * If we have set the spa_final_txg, we have already performed the 2061 * tasks below in spa_export_common(). We should not redo it here since 2062 * we delay the final TXGs beyond what spa_final_txg is set at. 2063 */ 2064 if (spa->spa_final_txg == UINT64_MAX) { 2065 /* 2066 * If the log space map feature is enabled and the pool is 2067 * getting exported (but not destroyed), we want to spend some 2068 * time flushing as many metaslabs as we can in an attempt to 2069 * destroy log space maps and save import time. 2070 */ 2071 if (spa_should_flush_logs_on_unload(spa)) 2072 spa_unload_log_sm_flush_all(spa); 2073 2074 /* 2075 * Stop async tasks. 2076 */ 2077 spa_async_suspend(spa); 2078 2079 if (spa->spa_root_vdev) { 2080 vdev_t *root_vdev = spa->spa_root_vdev; 2081 vdev_initialize_stop_all(root_vdev, 2082 VDEV_INITIALIZE_ACTIVE); 2083 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 2084 vdev_autotrim_stop_all(spa); 2085 vdev_rebuild_stop_all(spa); 2086 l2arc_spa_rebuild_stop(spa); 2087 } 2088 } 2089 2090 /* 2091 * Stop syncing. 2092 */ 2093 if (spa->spa_sync_on) { 2094 txg_sync_stop(spa->spa_dsl_pool); 2095 spa->spa_sync_on = B_FALSE; 2096 } 2097 2098 /* 2099 * This ensures that there is no async metaslab prefetching 2100 * while we attempt to unload the spa. 2101 */ 2102 taskq_wait(spa->spa_metaslab_taskq); 2103 2104 if (spa->spa_mmp.mmp_thread) 2105 mmp_thread_stop(spa); 2106 2107 /* 2108 * Wait for any outstanding async I/O to complete. 2109 */ 2110 if (spa->spa_async_zio_root != NULL) { 2111 for (int i = 0; i < max_ncpus; i++) 2112 (void) zio_wait(spa->spa_async_zio_root[i]); 2113 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 2114 spa->spa_async_zio_root = NULL; 2115 } 2116 2117 if (spa->spa_vdev_removal != NULL) { 2118 spa_vdev_removal_destroy(spa->spa_vdev_removal); 2119 spa->spa_vdev_removal = NULL; 2120 } 2121 2122 spa_destroy_aux_threads(spa); 2123 2124 spa_condense_fini(spa); 2125 2126 bpobj_close(&spa->spa_deferred_bpobj); 2127 2128 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 2129 2130 /* 2131 * Close all vdevs. 2132 */ 2133 if (spa->spa_root_vdev) 2134 vdev_free(spa->spa_root_vdev); 2135 ASSERT(spa->spa_root_vdev == NULL); 2136 2137 /* 2138 * Close the dsl pool. 2139 */ 2140 if (spa->spa_dsl_pool) { 2141 dsl_pool_close(spa->spa_dsl_pool); 2142 spa->spa_dsl_pool = NULL; 2143 spa->spa_meta_objset = NULL; 2144 } 2145 2146 ddt_unload(spa); 2147 brt_unload(spa); 2148 spa_unload_log_sm_metadata(spa); 2149 2150 /* 2151 * Drop and purge level 2 cache 2152 */ 2153 spa_l2cache_drop(spa); 2154 2155 if (spa->spa_spares.sav_vdevs) { 2156 for (int i = 0; i < spa->spa_spares.sav_count; i++) 2157 vdev_free(spa->spa_spares.sav_vdevs[i]); 2158 kmem_free(spa->spa_spares.sav_vdevs, 2159 spa->spa_spares.sav_count * sizeof (void *)); 2160 spa->spa_spares.sav_vdevs = NULL; 2161 } 2162 if (spa->spa_spares.sav_config) { 2163 nvlist_free(spa->spa_spares.sav_config); 2164 spa->spa_spares.sav_config = NULL; 2165 } 2166 spa->spa_spares.sav_count = 0; 2167 2168 if (spa->spa_l2cache.sav_vdevs) { 2169 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 2170 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 2171 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 2172 } 2173 kmem_free(spa->spa_l2cache.sav_vdevs, 2174 spa->spa_l2cache.sav_count * sizeof (void *)); 2175 spa->spa_l2cache.sav_vdevs = NULL; 2176 } 2177 if (spa->spa_l2cache.sav_config) { 2178 nvlist_free(spa->spa_l2cache.sav_config); 2179 spa->spa_l2cache.sav_config = NULL; 2180 } 2181 spa->spa_l2cache.sav_count = 0; 2182 2183 spa->spa_async_suspended = 0; 2184 2185 spa->spa_indirect_vdevs_loaded = B_FALSE; 2186 2187 if (spa->spa_comment != NULL) { 2188 spa_strfree(spa->spa_comment); 2189 spa->spa_comment = NULL; 2190 } 2191 if (spa->spa_compatibility != NULL) { 2192 spa_strfree(spa->spa_compatibility); 2193 spa->spa_compatibility = NULL; 2194 } 2195 2196 spa->spa_raidz_expand = NULL; 2197 2198 spa_config_exit(spa, SCL_ALL, spa); 2199 } 2200 2201 /* 2202 * Load (or re-load) the current list of vdevs describing the active spares for 2203 * this pool. When this is called, we have some form of basic information in 2204 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 2205 * then re-generate a more complete list including status information. 2206 */ 2207 void 2208 spa_load_spares(spa_t *spa) 2209 { 2210 nvlist_t **spares; 2211 uint_t nspares; 2212 int i; 2213 vdev_t *vd, *tvd; 2214 2215 #ifndef _KERNEL 2216 /* 2217 * zdb opens both the current state of the pool and the 2218 * checkpointed state (if present), with a different spa_t. 2219 * 2220 * As spare vdevs are shared among open pools, we skip loading 2221 * them when we load the checkpointed state of the pool. 2222 */ 2223 if (!spa_writeable(spa)) 2224 return; 2225 #endif 2226 2227 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 2228 2229 /* 2230 * First, close and free any existing spare vdevs. 2231 */ 2232 if (spa->spa_spares.sav_vdevs) { 2233 for (i = 0; i < spa->spa_spares.sav_count; i++) { 2234 vd = spa->spa_spares.sav_vdevs[i]; 2235 2236 /* Undo the call to spa_activate() below */ 2237 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 2238 B_FALSE)) != NULL && tvd->vdev_isspare) 2239 spa_spare_remove(tvd); 2240 vdev_close(vd); 2241 vdev_free(vd); 2242 } 2243 2244 kmem_free(spa->spa_spares.sav_vdevs, 2245 spa->spa_spares.sav_count * sizeof (void *)); 2246 } 2247 2248 if (spa->spa_spares.sav_config == NULL) 2249 nspares = 0; 2250 else 2251 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 2252 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 2253 2254 spa->spa_spares.sav_count = (int)nspares; 2255 spa->spa_spares.sav_vdevs = NULL; 2256 2257 if (nspares == 0) 2258 return; 2259 2260 /* 2261 * Construct the array of vdevs, opening them to get status in the 2262 * process. For each spare, there is potentially two different vdev_t 2263 * structures associated with it: one in the list of spares (used only 2264 * for basic validation purposes) and one in the active vdev 2265 * configuration (if it's spared in). During this phase we open and 2266 * validate each vdev on the spare list. If the vdev also exists in the 2267 * active configuration, then we also mark this vdev as an active spare. 2268 */ 2269 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 2270 KM_SLEEP); 2271 for (i = 0; i < spa->spa_spares.sav_count; i++) { 2272 VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, 2273 VDEV_ALLOC_SPARE) == 0); 2274 ASSERT(vd != NULL); 2275 2276 spa->spa_spares.sav_vdevs[i] = vd; 2277 2278 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 2279 B_FALSE)) != NULL) { 2280 if (!tvd->vdev_isspare) 2281 spa_spare_add(tvd); 2282 2283 /* 2284 * We only mark the spare active if we were successfully 2285 * able to load the vdev. Otherwise, importing a pool 2286 * with a bad active spare would result in strange 2287 * behavior, because multiple pool would think the spare 2288 * is actively in use. 2289 * 2290 * There is a vulnerability here to an equally bizarre 2291 * circumstance, where a dead active spare is later 2292 * brought back to life (onlined or otherwise). Given 2293 * the rarity of this scenario, and the extra complexity 2294 * it adds, we ignore the possibility. 2295 */ 2296 if (!vdev_is_dead(tvd)) 2297 spa_spare_activate(tvd); 2298 } 2299 2300 vd->vdev_top = vd; 2301 vd->vdev_aux = &spa->spa_spares; 2302 2303 if (vdev_open(vd) != 0) 2304 continue; 2305 2306 if (vdev_validate_aux(vd) == 0) 2307 spa_spare_add(vd); 2308 } 2309 2310 /* 2311 * Recompute the stashed list of spares, with status information 2312 * this time. 2313 */ 2314 fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES); 2315 2316 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 2317 KM_SLEEP); 2318 for (i = 0; i < spa->spa_spares.sav_count; i++) 2319 spares[i] = vdev_config_generate(spa, 2320 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 2321 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 2322 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 2323 spa->spa_spares.sav_count); 2324 for (i = 0; i < spa->spa_spares.sav_count; i++) 2325 nvlist_free(spares[i]); 2326 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 2327 } 2328 2329 /* 2330 * Load (or re-load) the current list of vdevs describing the active l2cache for 2331 * this pool. When this is called, we have some form of basic information in 2332 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 2333 * then re-generate a more complete list including status information. 2334 * Devices which are already active have their details maintained, and are 2335 * not re-opened. 2336 */ 2337 void 2338 spa_load_l2cache(spa_t *spa) 2339 { 2340 nvlist_t **l2cache = NULL; 2341 uint_t nl2cache; 2342 int i, j, oldnvdevs; 2343 uint64_t guid; 2344 vdev_t *vd, **oldvdevs, **newvdevs; 2345 spa_aux_vdev_t *sav = &spa->spa_l2cache; 2346 2347 #ifndef _KERNEL 2348 /* 2349 * zdb opens both the current state of the pool and the 2350 * checkpointed state (if present), with a different spa_t. 2351 * 2352 * As L2 caches are part of the ARC which is shared among open 2353 * pools, we skip loading them when we load the checkpointed 2354 * state of the pool. 2355 */ 2356 if (!spa_writeable(spa)) 2357 return; 2358 #endif 2359 2360 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 2361 2362 oldvdevs = sav->sav_vdevs; 2363 oldnvdevs = sav->sav_count; 2364 sav->sav_vdevs = NULL; 2365 sav->sav_count = 0; 2366 2367 if (sav->sav_config == NULL) { 2368 nl2cache = 0; 2369 newvdevs = NULL; 2370 goto out; 2371 } 2372 2373 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, 2374 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 2375 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 2376 2377 /* 2378 * Process new nvlist of vdevs. 2379 */ 2380 for (i = 0; i < nl2cache; i++) { 2381 guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID); 2382 2383 newvdevs[i] = NULL; 2384 for (j = 0; j < oldnvdevs; j++) { 2385 vd = oldvdevs[j]; 2386 if (vd != NULL && guid == vd->vdev_guid) { 2387 /* 2388 * Retain previous vdev for add/remove ops. 2389 */ 2390 newvdevs[i] = vd; 2391 oldvdevs[j] = NULL; 2392 break; 2393 } 2394 } 2395 2396 if (newvdevs[i] == NULL) { 2397 /* 2398 * Create new vdev 2399 */ 2400 VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 2401 VDEV_ALLOC_L2CACHE) == 0); 2402 ASSERT(vd != NULL); 2403 newvdevs[i] = vd; 2404 2405 /* 2406 * Commit this vdev as an l2cache device, 2407 * even if it fails to open. 2408 */ 2409 spa_l2cache_add(vd); 2410 2411 vd->vdev_top = vd; 2412 vd->vdev_aux = sav; 2413 2414 spa_l2cache_activate(vd); 2415 2416 if (vdev_open(vd) != 0) 2417 continue; 2418 2419 (void) vdev_validate_aux(vd); 2420 2421 if (!vdev_is_dead(vd)) 2422 l2arc_add_vdev(spa, vd); 2423 2424 /* 2425 * Upon cache device addition to a pool or pool 2426 * creation with a cache device or if the header 2427 * of the device is invalid we issue an async 2428 * TRIM command for the whole device which will 2429 * execute if l2arc_trim_ahead > 0. 2430 */ 2431 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 2432 } 2433 } 2434 2435 sav->sav_vdevs = newvdevs; 2436 sav->sav_count = (int)nl2cache; 2437 2438 /* 2439 * Recompute the stashed list of l2cache devices, with status 2440 * information this time. 2441 */ 2442 fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE); 2443 2444 if (sav->sav_count > 0) 2445 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 2446 KM_SLEEP); 2447 for (i = 0; i < sav->sav_count; i++) 2448 l2cache[i] = vdev_config_generate(spa, 2449 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 2450 fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 2451 (const nvlist_t * const *)l2cache, sav->sav_count); 2452 2453 out: 2454 /* 2455 * Purge vdevs that were dropped 2456 */ 2457 if (oldvdevs) { 2458 for (i = 0; i < oldnvdevs; i++) { 2459 uint64_t pool; 2460 2461 vd = oldvdevs[i]; 2462 if (vd != NULL) { 2463 ASSERT(vd->vdev_isl2cache); 2464 2465 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 2466 pool != 0ULL && l2arc_vdev_present(vd)) 2467 l2arc_remove_vdev(vd); 2468 vdev_clear_stats(vd); 2469 vdev_free(vd); 2470 } 2471 } 2472 2473 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2474 } 2475 2476 for (i = 0; i < sav->sav_count; i++) 2477 nvlist_free(l2cache[i]); 2478 if (sav->sav_count) 2479 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2480 } 2481 2482 static int 2483 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2484 { 2485 dmu_buf_t *db; 2486 char *packed = NULL; 2487 size_t nvsize = 0; 2488 int error; 2489 *value = NULL; 2490 2491 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2492 if (error) 2493 return (error); 2494 2495 nvsize = *(uint64_t *)db->db_data; 2496 dmu_buf_rele(db, FTAG); 2497 2498 packed = vmem_alloc(nvsize, KM_SLEEP); 2499 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2500 DMU_READ_PREFETCH); 2501 if (error == 0) 2502 error = nvlist_unpack(packed, nvsize, value, 0); 2503 vmem_free(packed, nvsize); 2504 2505 return (error); 2506 } 2507 2508 /* 2509 * Concrete top-level vdevs that are not missing and are not logs. At every 2510 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2511 */ 2512 static uint64_t 2513 spa_healthy_core_tvds(spa_t *spa) 2514 { 2515 vdev_t *rvd = spa->spa_root_vdev; 2516 uint64_t tvds = 0; 2517 2518 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2519 vdev_t *vd = rvd->vdev_child[i]; 2520 if (vd->vdev_islog) 2521 continue; 2522 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2523 tvds++; 2524 } 2525 2526 return (tvds); 2527 } 2528 2529 /* 2530 * Checks to see if the given vdev could not be opened, in which case we post a 2531 * sysevent to notify the autoreplace code that the device has been removed. 2532 */ 2533 static void 2534 spa_check_removed(vdev_t *vd) 2535 { 2536 for (uint64_t c = 0; c < vd->vdev_children; c++) 2537 spa_check_removed(vd->vdev_child[c]); 2538 2539 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2540 vdev_is_concrete(vd)) { 2541 zfs_post_autoreplace(vd->vdev_spa, vd); 2542 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2543 } 2544 } 2545 2546 static int 2547 spa_check_for_missing_logs(spa_t *spa) 2548 { 2549 vdev_t *rvd = spa->spa_root_vdev; 2550 2551 /* 2552 * If we're doing a normal import, then build up any additional 2553 * diagnostic information about missing log devices. 2554 * We'll pass this up to the user for further processing. 2555 */ 2556 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2557 nvlist_t **child, *nv; 2558 uint64_t idx = 0; 2559 2560 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2561 KM_SLEEP); 2562 nv = fnvlist_alloc(); 2563 2564 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2565 vdev_t *tvd = rvd->vdev_child[c]; 2566 2567 /* 2568 * We consider a device as missing only if it failed 2569 * to open (i.e. offline or faulted is not considered 2570 * as missing). 2571 */ 2572 if (tvd->vdev_islog && 2573 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2574 child[idx++] = vdev_config_generate(spa, tvd, 2575 B_FALSE, VDEV_CONFIG_MISSING); 2576 } 2577 } 2578 2579 if (idx > 0) { 2580 fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 2581 (const nvlist_t * const *)child, idx); 2582 fnvlist_add_nvlist(spa->spa_load_info, 2583 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2584 2585 for (uint64_t i = 0; i < idx; i++) 2586 nvlist_free(child[i]); 2587 } 2588 nvlist_free(nv); 2589 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2590 2591 if (idx > 0) { 2592 spa_load_failed(spa, "some log devices are missing"); 2593 vdev_dbgmsg_print_tree(rvd, 2); 2594 return (SET_ERROR(ENXIO)); 2595 } 2596 } else { 2597 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2598 vdev_t *tvd = rvd->vdev_child[c]; 2599 2600 if (tvd->vdev_islog && 2601 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2602 spa_set_log_state(spa, SPA_LOG_CLEAR); 2603 spa_load_note(spa, "some log devices are " 2604 "missing, ZIL is dropped."); 2605 vdev_dbgmsg_print_tree(rvd, 2); 2606 break; 2607 } 2608 } 2609 } 2610 2611 return (0); 2612 } 2613 2614 /* 2615 * Check for missing log devices 2616 */ 2617 static boolean_t 2618 spa_check_logs(spa_t *spa) 2619 { 2620 boolean_t rv = B_FALSE; 2621 dsl_pool_t *dp = spa_get_dsl(spa); 2622 2623 switch (spa->spa_log_state) { 2624 default: 2625 break; 2626 case SPA_LOG_MISSING: 2627 /* need to recheck in case slog has been restored */ 2628 case SPA_LOG_UNKNOWN: 2629 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2630 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2631 if (rv) 2632 spa_set_log_state(spa, SPA_LOG_MISSING); 2633 break; 2634 } 2635 return (rv); 2636 } 2637 2638 /* 2639 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2640 */ 2641 static boolean_t 2642 spa_passivate_log(spa_t *spa) 2643 { 2644 vdev_t *rvd = spa->spa_root_vdev; 2645 boolean_t slog_found = B_FALSE; 2646 2647 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2648 2649 for (int c = 0; c < rvd->vdev_children; c++) { 2650 vdev_t *tvd = rvd->vdev_child[c]; 2651 2652 if (tvd->vdev_islog) { 2653 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2654 metaslab_group_passivate(tvd->vdev_mg); 2655 slog_found = B_TRUE; 2656 } 2657 } 2658 2659 return (slog_found); 2660 } 2661 2662 /* 2663 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2664 */ 2665 static void 2666 spa_activate_log(spa_t *spa) 2667 { 2668 vdev_t *rvd = spa->spa_root_vdev; 2669 2670 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2671 2672 for (int c = 0; c < rvd->vdev_children; c++) { 2673 vdev_t *tvd = rvd->vdev_child[c]; 2674 2675 if (tvd->vdev_islog) { 2676 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2677 metaslab_group_activate(tvd->vdev_mg); 2678 } 2679 } 2680 } 2681 2682 int 2683 spa_reset_logs(spa_t *spa) 2684 { 2685 int error; 2686 2687 error = dmu_objset_find(spa_name(spa), zil_reset, 2688 NULL, DS_FIND_CHILDREN); 2689 if (error == 0) { 2690 /* 2691 * We successfully offlined the log device, sync out the 2692 * current txg so that the "stubby" block can be removed 2693 * by zil_sync(). 2694 */ 2695 txg_wait_synced(spa->spa_dsl_pool, 0); 2696 } 2697 return (error); 2698 } 2699 2700 static void 2701 spa_aux_check_removed(spa_aux_vdev_t *sav) 2702 { 2703 for (int i = 0; i < sav->sav_count; i++) 2704 spa_check_removed(sav->sav_vdevs[i]); 2705 } 2706 2707 void 2708 spa_claim_notify(zio_t *zio) 2709 { 2710 spa_t *spa = zio->io_spa; 2711 2712 if (zio->io_error) 2713 return; 2714 2715 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2716 if (spa->spa_claim_max_txg < BP_GET_LOGICAL_BIRTH(zio->io_bp)) 2717 spa->spa_claim_max_txg = BP_GET_LOGICAL_BIRTH(zio->io_bp); 2718 mutex_exit(&spa->spa_props_lock); 2719 } 2720 2721 typedef struct spa_load_error { 2722 boolean_t sle_verify_data; 2723 uint64_t sle_meta_count; 2724 uint64_t sle_data_count; 2725 } spa_load_error_t; 2726 2727 static void 2728 spa_load_verify_done(zio_t *zio) 2729 { 2730 blkptr_t *bp = zio->io_bp; 2731 spa_load_error_t *sle = zio->io_private; 2732 dmu_object_type_t type = BP_GET_TYPE(bp); 2733 int error = zio->io_error; 2734 spa_t *spa = zio->io_spa; 2735 2736 abd_free(zio->io_abd); 2737 if (error) { 2738 if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && 2739 type != DMU_OT_INTENT_LOG) 2740 atomic_inc_64(&sle->sle_meta_count); 2741 else 2742 atomic_inc_64(&sle->sle_data_count); 2743 } 2744 2745 mutex_enter(&spa->spa_scrub_lock); 2746 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 2747 cv_broadcast(&spa->spa_scrub_io_cv); 2748 mutex_exit(&spa->spa_scrub_lock); 2749 } 2750 2751 /* 2752 * Maximum number of inflight bytes is the log2 fraction of the arc size. 2753 * By default, we set it to 1/16th of the arc. 2754 */ 2755 static uint_t spa_load_verify_shift = 4; 2756 static int spa_load_verify_metadata = B_TRUE; 2757 static int spa_load_verify_data = B_TRUE; 2758 2759 static int 2760 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 2761 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 2762 { 2763 zio_t *rio = arg; 2764 spa_load_error_t *sle = rio->io_private; 2765 2766 (void) zilog, (void) dnp; 2767 2768 /* 2769 * Note: normally this routine will not be called if 2770 * spa_load_verify_metadata is not set. However, it may be useful 2771 * to manually set the flag after the traversal has begun. 2772 */ 2773 if (!spa_load_verify_metadata) 2774 return (0); 2775 2776 /* 2777 * Sanity check the block pointer in order to detect obvious damage 2778 * before using the contents in subsequent checks or in zio_read(). 2779 * When damaged consider it to be a metadata error since we cannot 2780 * trust the BP_GET_TYPE and BP_GET_LEVEL values. 2781 */ 2782 if (zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) { 2783 atomic_inc_64(&sle->sle_meta_count); 2784 return (0); 2785 } 2786 2787 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 2788 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 2789 return (0); 2790 2791 if (!BP_IS_METADATA(bp) && 2792 (!spa_load_verify_data || !sle->sle_verify_data)) 2793 return (0); 2794 2795 uint64_t maxinflight_bytes = 2796 arc_target_bytes() >> spa_load_verify_shift; 2797 size_t size = BP_GET_PSIZE(bp); 2798 2799 mutex_enter(&spa->spa_scrub_lock); 2800 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 2801 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 2802 spa->spa_load_verify_bytes += size; 2803 mutex_exit(&spa->spa_scrub_lock); 2804 2805 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 2806 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 2807 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 2808 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 2809 return (0); 2810 } 2811 2812 static int 2813 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 2814 { 2815 (void) dp, (void) arg; 2816 2817 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 2818 return (SET_ERROR(ENAMETOOLONG)); 2819 2820 return (0); 2821 } 2822 2823 static int 2824 spa_load_verify(spa_t *spa) 2825 { 2826 zio_t *rio; 2827 spa_load_error_t sle = { 0 }; 2828 zpool_load_policy_t policy; 2829 boolean_t verify_ok = B_FALSE; 2830 int error = 0; 2831 2832 zpool_get_load_policy(spa->spa_config, &policy); 2833 2834 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND || 2835 policy.zlp_maxmeta == UINT64_MAX) 2836 return (0); 2837 2838 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 2839 error = dmu_objset_find_dp(spa->spa_dsl_pool, 2840 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 2841 DS_FIND_CHILDREN); 2842 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 2843 if (error != 0) 2844 return (error); 2845 2846 /* 2847 * Verify data only if we are rewinding or error limit was set. 2848 * Otherwise nothing except dbgmsg care about it to waste time. 2849 */ 2850 sle.sle_verify_data = (policy.zlp_rewind & ZPOOL_REWIND_MASK) || 2851 (policy.zlp_maxdata < UINT64_MAX); 2852 2853 rio = zio_root(spa, NULL, &sle, 2854 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 2855 2856 if (spa_load_verify_metadata) { 2857 if (spa->spa_extreme_rewind) { 2858 spa_load_note(spa, "performing a complete scan of the " 2859 "pool since extreme rewind is on. This may take " 2860 "a very long time.\n (spa_load_verify_data=%u, " 2861 "spa_load_verify_metadata=%u)", 2862 spa_load_verify_data, spa_load_verify_metadata); 2863 } 2864 2865 error = traverse_pool(spa, spa->spa_verify_min_txg, 2866 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 2867 TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio); 2868 } 2869 2870 (void) zio_wait(rio); 2871 ASSERT0(spa->spa_load_verify_bytes); 2872 2873 spa->spa_load_meta_errors = sle.sle_meta_count; 2874 spa->spa_load_data_errors = sle.sle_data_count; 2875 2876 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 2877 spa_load_note(spa, "spa_load_verify found %llu metadata errors " 2878 "and %llu data errors", (u_longlong_t)sle.sle_meta_count, 2879 (u_longlong_t)sle.sle_data_count); 2880 } 2881 2882 if (spa_load_verify_dryrun || 2883 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 2884 sle.sle_data_count <= policy.zlp_maxdata)) { 2885 int64_t loss = 0; 2886 2887 verify_ok = B_TRUE; 2888 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 2889 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 2890 2891 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; 2892 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME, 2893 spa->spa_load_txg_ts); 2894 fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME, 2895 loss); 2896 fnvlist_add_uint64(spa->spa_load_info, 2897 ZPOOL_CONFIG_LOAD_META_ERRORS, sle.sle_meta_count); 2898 fnvlist_add_uint64(spa->spa_load_info, 2899 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count); 2900 } else { 2901 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 2902 } 2903 2904 if (spa_load_verify_dryrun) 2905 return (0); 2906 2907 if (error) { 2908 if (error != ENXIO && error != EIO) 2909 error = SET_ERROR(EIO); 2910 return (error); 2911 } 2912 2913 return (verify_ok ? 0 : EIO); 2914 } 2915 2916 /* 2917 * Find a value in the pool props object. 2918 */ 2919 static void 2920 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 2921 { 2922 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 2923 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 2924 } 2925 2926 /* 2927 * Find a value in the pool directory object. 2928 */ 2929 static int 2930 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 2931 { 2932 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2933 name, sizeof (uint64_t), 1, val); 2934 2935 if (error != 0 && (error != ENOENT || log_enoent)) { 2936 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 2937 "[error=%d]", name, error); 2938 } 2939 2940 return (error); 2941 } 2942 2943 static int 2944 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 2945 { 2946 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 2947 return (SET_ERROR(err)); 2948 } 2949 2950 boolean_t 2951 spa_livelist_delete_check(spa_t *spa) 2952 { 2953 return (spa->spa_livelists_to_delete != 0); 2954 } 2955 2956 static boolean_t 2957 spa_livelist_delete_cb_check(void *arg, zthr_t *z) 2958 { 2959 (void) z; 2960 spa_t *spa = arg; 2961 return (spa_livelist_delete_check(spa)); 2962 } 2963 2964 static int 2965 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 2966 { 2967 spa_t *spa = arg; 2968 zio_free(spa, tx->tx_txg, bp); 2969 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 2970 -bp_get_dsize_sync(spa, bp), 2971 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 2972 return (0); 2973 } 2974 2975 static int 2976 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 2977 { 2978 int err; 2979 zap_cursor_t zc; 2980 zap_attribute_t *za = zap_attribute_alloc(); 2981 zap_cursor_init(&zc, os, zap_obj); 2982 err = zap_cursor_retrieve(&zc, za); 2983 zap_cursor_fini(&zc); 2984 if (err == 0) 2985 *llp = za->za_first_integer; 2986 zap_attribute_free(za); 2987 return (err); 2988 } 2989 2990 /* 2991 * Components of livelist deletion that must be performed in syncing 2992 * context: freeing block pointers and updating the pool-wide data 2993 * structures to indicate how much work is left to do 2994 */ 2995 typedef struct sublist_delete_arg { 2996 spa_t *spa; 2997 dsl_deadlist_t *ll; 2998 uint64_t key; 2999 bplist_t *to_free; 3000 } sublist_delete_arg_t; 3001 3002 static void 3003 sublist_delete_sync(void *arg, dmu_tx_t *tx) 3004 { 3005 sublist_delete_arg_t *sda = arg; 3006 spa_t *spa = sda->spa; 3007 dsl_deadlist_t *ll = sda->ll; 3008 uint64_t key = sda->key; 3009 bplist_t *to_free = sda->to_free; 3010 3011 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 3012 dsl_deadlist_remove_entry(ll, key, tx); 3013 } 3014 3015 typedef struct livelist_delete_arg { 3016 spa_t *spa; 3017 uint64_t ll_obj; 3018 uint64_t zap_obj; 3019 } livelist_delete_arg_t; 3020 3021 static void 3022 livelist_delete_sync(void *arg, dmu_tx_t *tx) 3023 { 3024 livelist_delete_arg_t *lda = arg; 3025 spa_t *spa = lda->spa; 3026 uint64_t ll_obj = lda->ll_obj; 3027 uint64_t zap_obj = lda->zap_obj; 3028 objset_t *mos = spa->spa_meta_objset; 3029 uint64_t count; 3030 3031 /* free the livelist and decrement the feature count */ 3032 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 3033 dsl_deadlist_free(mos, ll_obj, tx); 3034 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 3035 VERIFY0(zap_count(mos, zap_obj, &count)); 3036 if (count == 0) { 3037 /* no more livelists to delete */ 3038 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 3039 DMU_POOL_DELETED_CLONES, tx)); 3040 VERIFY0(zap_destroy(mos, zap_obj, tx)); 3041 spa->spa_livelists_to_delete = 0; 3042 spa_notify_waiters(spa); 3043 } 3044 } 3045 3046 /* 3047 * Load in the value for the livelist to be removed and open it. Then, 3048 * load its first sublist and determine which block pointers should actually 3049 * be freed. Then, call a synctask which performs the actual frees and updates 3050 * the pool-wide livelist data. 3051 */ 3052 static void 3053 spa_livelist_delete_cb(void *arg, zthr_t *z) 3054 { 3055 spa_t *spa = arg; 3056 uint64_t ll_obj = 0, count; 3057 objset_t *mos = spa->spa_meta_objset; 3058 uint64_t zap_obj = spa->spa_livelists_to_delete; 3059 /* 3060 * Determine the next livelist to delete. This function should only 3061 * be called if there is at least one deleted clone. 3062 */ 3063 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 3064 VERIFY0(zap_count(mos, ll_obj, &count)); 3065 if (count > 0) { 3066 dsl_deadlist_t *ll; 3067 dsl_deadlist_entry_t *dle; 3068 bplist_t to_free; 3069 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 3070 VERIFY0(dsl_deadlist_open(ll, mos, ll_obj)); 3071 dle = dsl_deadlist_first(ll); 3072 ASSERT3P(dle, !=, NULL); 3073 bplist_create(&to_free); 3074 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 3075 z, NULL); 3076 if (err == 0) { 3077 sublist_delete_arg_t sync_arg = { 3078 .spa = spa, 3079 .ll = ll, 3080 .key = dle->dle_mintxg, 3081 .to_free = &to_free 3082 }; 3083 zfs_dbgmsg("deleting sublist (id %llu) from" 3084 " livelist %llu, %lld remaining", 3085 (u_longlong_t)dle->dle_bpobj.bpo_object, 3086 (u_longlong_t)ll_obj, (longlong_t)count - 1); 3087 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 3088 sublist_delete_sync, &sync_arg, 0, 3089 ZFS_SPACE_CHECK_DESTROY)); 3090 } else { 3091 VERIFY3U(err, ==, EINTR); 3092 } 3093 bplist_clear(&to_free); 3094 bplist_destroy(&to_free); 3095 dsl_deadlist_close(ll); 3096 kmem_free(ll, sizeof (dsl_deadlist_t)); 3097 } else { 3098 livelist_delete_arg_t sync_arg = { 3099 .spa = spa, 3100 .ll_obj = ll_obj, 3101 .zap_obj = zap_obj 3102 }; 3103 zfs_dbgmsg("deletion of livelist %llu completed", 3104 (u_longlong_t)ll_obj); 3105 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 3106 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 3107 } 3108 } 3109 3110 static void 3111 spa_start_livelist_destroy_thread(spa_t *spa) 3112 { 3113 ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL); 3114 spa->spa_livelist_delete_zthr = 3115 zthr_create("z_livelist_destroy", 3116 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa, 3117 minclsyspri); 3118 } 3119 3120 typedef struct livelist_new_arg { 3121 bplist_t *allocs; 3122 bplist_t *frees; 3123 } livelist_new_arg_t; 3124 3125 static int 3126 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 3127 dmu_tx_t *tx) 3128 { 3129 ASSERT(tx == NULL); 3130 livelist_new_arg_t *lna = arg; 3131 if (bp_freed) { 3132 bplist_append(lna->frees, bp); 3133 } else { 3134 bplist_append(lna->allocs, bp); 3135 zfs_livelist_condense_new_alloc++; 3136 } 3137 return (0); 3138 } 3139 3140 typedef struct livelist_condense_arg { 3141 spa_t *spa; 3142 bplist_t to_keep; 3143 uint64_t first_size; 3144 uint64_t next_size; 3145 } livelist_condense_arg_t; 3146 3147 static void 3148 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 3149 { 3150 livelist_condense_arg_t *lca = arg; 3151 spa_t *spa = lca->spa; 3152 bplist_t new_frees; 3153 dsl_dataset_t *ds = spa->spa_to_condense.ds; 3154 3155 /* Have we been cancelled? */ 3156 if (spa->spa_to_condense.cancelled) { 3157 zfs_livelist_condense_sync_cancel++; 3158 goto out; 3159 } 3160 3161 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 3162 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 3163 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 3164 3165 /* 3166 * It's possible that the livelist was changed while the zthr was 3167 * running. Therefore, we need to check for new blkptrs in the two 3168 * entries being condensed and continue to track them in the livelist. 3169 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 3170 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 3171 * we need to sort them into two different bplists. 3172 */ 3173 uint64_t first_obj = first->dle_bpobj.bpo_object; 3174 uint64_t next_obj = next->dle_bpobj.bpo_object; 3175 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 3176 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 3177 3178 bplist_create(&new_frees); 3179 livelist_new_arg_t new_bps = { 3180 .allocs = &lca->to_keep, 3181 .frees = &new_frees, 3182 }; 3183 3184 if (cur_first_size > lca->first_size) { 3185 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 3186 livelist_track_new_cb, &new_bps, lca->first_size)); 3187 } 3188 if (cur_next_size > lca->next_size) { 3189 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 3190 livelist_track_new_cb, &new_bps, lca->next_size)); 3191 } 3192 3193 dsl_deadlist_clear_entry(first, ll, tx); 3194 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 3195 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 3196 3197 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 3198 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 3199 bplist_destroy(&new_frees); 3200 3201 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 3202 dsl_dataset_name(ds, dsname); 3203 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 3204 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 3205 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, 3206 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, 3207 (u_longlong_t)cur_first_size, (u_longlong_t)next_obj, 3208 (u_longlong_t)cur_next_size, 3209 (u_longlong_t)first->dle_bpobj.bpo_object, 3210 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 3211 out: 3212 dmu_buf_rele(ds->ds_dbuf, spa); 3213 spa->spa_to_condense.ds = NULL; 3214 bplist_clear(&lca->to_keep); 3215 bplist_destroy(&lca->to_keep); 3216 kmem_free(lca, sizeof (livelist_condense_arg_t)); 3217 spa->spa_to_condense.syncing = B_FALSE; 3218 } 3219 3220 static void 3221 spa_livelist_condense_cb(void *arg, zthr_t *t) 3222 { 3223 while (zfs_livelist_condense_zthr_pause && 3224 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 3225 delay(1); 3226 3227 spa_t *spa = arg; 3228 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 3229 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 3230 uint64_t first_size, next_size; 3231 3232 livelist_condense_arg_t *lca = 3233 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 3234 bplist_create(&lca->to_keep); 3235 3236 /* 3237 * Process the livelists (matching FREEs and ALLOCs) in open context 3238 * so we have minimal work in syncing context to condense. 3239 * 3240 * We save bpobj sizes (first_size and next_size) to use later in 3241 * syncing context to determine if entries were added to these sublists 3242 * while in open context. This is possible because the clone is still 3243 * active and open for normal writes and we want to make sure the new, 3244 * unprocessed blockpointers are inserted into the livelist normally. 3245 * 3246 * Note that dsl_process_sub_livelist() both stores the size number of 3247 * blockpointers and iterates over them while the bpobj's lock held, so 3248 * the sizes returned to us are consistent which what was actually 3249 * processed. 3250 */ 3251 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 3252 &first_size); 3253 if (err == 0) 3254 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 3255 t, &next_size); 3256 3257 if (err == 0) { 3258 while (zfs_livelist_condense_sync_pause && 3259 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 3260 delay(1); 3261 3262 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 3263 dmu_tx_mark_netfree(tx); 3264 dmu_tx_hold_space(tx, 1); 3265 err = dmu_tx_assign(tx, DMU_TX_NOWAIT | DMU_TX_NOTHROTTLE); 3266 if (err == 0) { 3267 /* 3268 * Prevent the condense zthr restarting before 3269 * the synctask completes. 3270 */ 3271 spa->spa_to_condense.syncing = B_TRUE; 3272 lca->spa = spa; 3273 lca->first_size = first_size; 3274 lca->next_size = next_size; 3275 dsl_sync_task_nowait(spa_get_dsl(spa), 3276 spa_livelist_condense_sync, lca, tx); 3277 dmu_tx_commit(tx); 3278 return; 3279 } 3280 } 3281 /* 3282 * Condensing can not continue: either it was externally stopped or 3283 * we were unable to assign to a tx because the pool has run out of 3284 * space. In the second case, we'll just end up trying to condense 3285 * again in a later txg. 3286 */ 3287 ASSERT(err != 0); 3288 bplist_clear(&lca->to_keep); 3289 bplist_destroy(&lca->to_keep); 3290 kmem_free(lca, sizeof (livelist_condense_arg_t)); 3291 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 3292 spa->spa_to_condense.ds = NULL; 3293 if (err == EINTR) 3294 zfs_livelist_condense_zthr_cancel++; 3295 } 3296 3297 /* 3298 * Check that there is something to condense but that a condense is not 3299 * already in progress and that condensing has not been cancelled. 3300 */ 3301 static boolean_t 3302 spa_livelist_condense_cb_check(void *arg, zthr_t *z) 3303 { 3304 (void) z; 3305 spa_t *spa = arg; 3306 if ((spa->spa_to_condense.ds != NULL) && 3307 (spa->spa_to_condense.syncing == B_FALSE) && 3308 (spa->spa_to_condense.cancelled == B_FALSE)) { 3309 return (B_TRUE); 3310 } 3311 return (B_FALSE); 3312 } 3313 3314 static void 3315 spa_start_livelist_condensing_thread(spa_t *spa) 3316 { 3317 spa->spa_to_condense.ds = NULL; 3318 spa->spa_to_condense.first = NULL; 3319 spa->spa_to_condense.next = NULL; 3320 spa->spa_to_condense.syncing = B_FALSE; 3321 spa->spa_to_condense.cancelled = B_FALSE; 3322 3323 ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL); 3324 spa->spa_livelist_condense_zthr = 3325 zthr_create("z_livelist_condense", 3326 spa_livelist_condense_cb_check, 3327 spa_livelist_condense_cb, spa, minclsyspri); 3328 } 3329 3330 static void 3331 spa_spawn_aux_threads(spa_t *spa) 3332 { 3333 ASSERT(spa_writeable(spa)); 3334 3335 spa_start_raidz_expansion_thread(spa); 3336 spa_start_indirect_condensing_thread(spa); 3337 spa_start_livelist_destroy_thread(spa); 3338 spa_start_livelist_condensing_thread(spa); 3339 3340 ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL); 3341 spa->spa_checkpoint_discard_zthr = 3342 zthr_create("z_checkpoint_discard", 3343 spa_checkpoint_discard_thread_check, 3344 spa_checkpoint_discard_thread, spa, minclsyspri); 3345 } 3346 3347 /* 3348 * Fix up config after a partly-completed split. This is done with the 3349 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 3350 * pool have that entry in their config, but only the splitting one contains 3351 * a list of all the guids of the vdevs that are being split off. 3352 * 3353 * This function determines what to do with that list: either rejoin 3354 * all the disks to the pool, or complete the splitting process. To attempt 3355 * the rejoin, each disk that is offlined is marked online again, and 3356 * we do a reopen() call. If the vdev label for every disk that was 3357 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 3358 * then we call vdev_split() on each disk, and complete the split. 3359 * 3360 * Otherwise we leave the config alone, with all the vdevs in place in 3361 * the original pool. 3362 */ 3363 static void 3364 spa_try_repair(spa_t *spa, nvlist_t *config) 3365 { 3366 uint_t extracted; 3367 uint64_t *glist; 3368 uint_t i, gcount; 3369 nvlist_t *nvl; 3370 vdev_t **vd; 3371 boolean_t attempt_reopen; 3372 3373 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 3374 return; 3375 3376 /* check that the config is complete */ 3377 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 3378 &glist, &gcount) != 0) 3379 return; 3380 3381 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 3382 3383 /* attempt to online all the vdevs & validate */ 3384 attempt_reopen = B_TRUE; 3385 for (i = 0; i < gcount; i++) { 3386 if (glist[i] == 0) /* vdev is hole */ 3387 continue; 3388 3389 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 3390 if (vd[i] == NULL) { 3391 /* 3392 * Don't bother attempting to reopen the disks; 3393 * just do the split. 3394 */ 3395 attempt_reopen = B_FALSE; 3396 } else { 3397 /* attempt to re-online it */ 3398 vd[i]->vdev_offline = B_FALSE; 3399 } 3400 } 3401 3402 if (attempt_reopen) { 3403 vdev_reopen(spa->spa_root_vdev); 3404 3405 /* check each device to see what state it's in */ 3406 for (extracted = 0, i = 0; i < gcount; i++) { 3407 if (vd[i] != NULL && 3408 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 3409 break; 3410 ++extracted; 3411 } 3412 } 3413 3414 /* 3415 * If every disk has been moved to the new pool, or if we never 3416 * even attempted to look at them, then we split them off for 3417 * good. 3418 */ 3419 if (!attempt_reopen || gcount == extracted) { 3420 for (i = 0; i < gcount; i++) 3421 if (vd[i] != NULL) 3422 vdev_split(vd[i]); 3423 vdev_reopen(spa->spa_root_vdev); 3424 } 3425 3426 kmem_free(vd, gcount * sizeof (vdev_t *)); 3427 } 3428 3429 static int 3430 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 3431 { 3432 const char *ereport = FM_EREPORT_ZFS_POOL; 3433 int error; 3434 3435 spa->spa_load_state = state; 3436 (void) spa_import_progress_set_state(spa_guid(spa), 3437 spa_load_state(spa)); 3438 spa_import_progress_set_notes(spa, "spa_load()"); 3439 3440 gethrestime(&spa->spa_loaded_ts); 3441 error = spa_load_impl(spa, type, &ereport); 3442 3443 /* 3444 * Don't count references from objsets that are already closed 3445 * and are making their way through the eviction process. 3446 */ 3447 spa_evicting_os_wait(spa); 3448 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 3449 if (error) { 3450 if (error != EEXIST) { 3451 spa->spa_loaded_ts.tv_sec = 0; 3452 spa->spa_loaded_ts.tv_nsec = 0; 3453 } 3454 if (error != EBADF) { 3455 (void) zfs_ereport_post(ereport, spa, 3456 NULL, NULL, NULL, 0); 3457 } 3458 } 3459 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 3460 spa->spa_ena = 0; 3461 3462 (void) spa_import_progress_set_state(spa_guid(spa), 3463 spa_load_state(spa)); 3464 3465 return (error); 3466 } 3467 3468 #ifdef ZFS_DEBUG 3469 /* 3470 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 3471 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 3472 * spa's per-vdev ZAP list. 3473 */ 3474 static uint64_t 3475 vdev_count_verify_zaps(vdev_t *vd) 3476 { 3477 spa_t *spa = vd->vdev_spa; 3478 uint64_t total = 0; 3479 3480 if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) && 3481 vd->vdev_root_zap != 0) { 3482 total++; 3483 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3484 spa->spa_all_vdev_zaps, vd->vdev_root_zap)); 3485 } 3486 if (vd->vdev_top_zap != 0) { 3487 total++; 3488 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3489 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 3490 } 3491 if (vd->vdev_leaf_zap != 0) { 3492 total++; 3493 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3494 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 3495 } 3496 3497 for (uint64_t i = 0; i < vd->vdev_children; i++) { 3498 total += vdev_count_verify_zaps(vd->vdev_child[i]); 3499 } 3500 3501 return (total); 3502 } 3503 #else 3504 #define vdev_count_verify_zaps(vd) ((void) sizeof (vd), 0) 3505 #endif 3506 3507 /* 3508 * Determine whether the activity check is required. 3509 */ 3510 static boolean_t 3511 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label, 3512 nvlist_t *config) 3513 { 3514 uint64_t state = 0; 3515 uint64_t hostid = 0; 3516 uint64_t tryconfig_txg = 0; 3517 uint64_t tryconfig_timestamp = 0; 3518 uint16_t tryconfig_mmp_seq = 0; 3519 nvlist_t *nvinfo; 3520 3521 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3522 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 3523 (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3524 &tryconfig_txg); 3525 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3526 &tryconfig_timestamp); 3527 (void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3528 &tryconfig_mmp_seq); 3529 } 3530 3531 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); 3532 3533 /* 3534 * Disable the MMP activity check - This is used by zdb which 3535 * is intended to be used on potentially active pools. 3536 */ 3537 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) 3538 return (B_FALSE); 3539 3540 /* 3541 * Skip the activity check when the MMP feature is disabled. 3542 */ 3543 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) 3544 return (B_FALSE); 3545 3546 /* 3547 * If the tryconfig_ values are nonzero, they are the results of an 3548 * earlier tryimport. If they all match the uberblock we just found, 3549 * then the pool has not changed and we return false so we do not test 3550 * a second time. 3551 */ 3552 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && 3553 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && 3554 tryconfig_mmp_seq && tryconfig_mmp_seq == 3555 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) 3556 return (B_FALSE); 3557 3558 /* 3559 * Allow the activity check to be skipped when importing the pool 3560 * on the same host which last imported it. Since the hostid from 3561 * configuration may be stale use the one read from the label. 3562 */ 3563 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3564 hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID); 3565 3566 if (hostid == spa_get_hostid(spa)) 3567 return (B_FALSE); 3568 3569 /* 3570 * Skip the activity test when the pool was cleanly exported. 3571 */ 3572 if (state != POOL_STATE_ACTIVE) 3573 return (B_FALSE); 3574 3575 return (B_TRUE); 3576 } 3577 3578 /* 3579 * Nanoseconds the activity check must watch for changes on-disk. 3580 */ 3581 static uint64_t 3582 spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3583 { 3584 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 3585 uint64_t multihost_interval = MSEC2NSEC( 3586 MMP_INTERVAL_OK(zfs_multihost_interval)); 3587 uint64_t import_delay = MAX(NANOSEC, import_intervals * 3588 multihost_interval); 3589 3590 /* 3591 * Local tunables determine a minimum duration except for the case 3592 * where we know when the remote host will suspend the pool if MMP 3593 * writes do not land. 3594 * 3595 * See Big Theory comment at the top of mmp.c for the reasoning behind 3596 * these cases and times. 3597 */ 3598 3599 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 3600 3601 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3602 MMP_FAIL_INT(ub) > 0) { 3603 3604 /* MMP on remote host will suspend pool after failed writes */ 3605 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 3606 MMP_IMPORT_SAFETY_FACTOR / 100; 3607 3608 zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp " 3609 "mmp_fails=%llu ub_mmp mmp_interval=%llu " 3610 "import_intervals=%llu", (u_longlong_t)import_delay, 3611 (u_longlong_t)MMP_FAIL_INT(ub), 3612 (u_longlong_t)MMP_INTERVAL(ub), 3613 (u_longlong_t)import_intervals); 3614 3615 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3616 MMP_FAIL_INT(ub) == 0) { 3617 3618 /* MMP on remote host will never suspend pool */ 3619 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 3620 ub->ub_mmp_delay) * import_intervals); 3621 3622 zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp " 3623 "mmp_interval=%llu ub_mmp_delay=%llu " 3624 "import_intervals=%llu", (u_longlong_t)import_delay, 3625 (u_longlong_t)MMP_INTERVAL(ub), 3626 (u_longlong_t)ub->ub_mmp_delay, 3627 (u_longlong_t)import_intervals); 3628 3629 } else if (MMP_VALID(ub)) { 3630 /* 3631 * zfs-0.7 compatibility case 3632 */ 3633 3634 import_delay = MAX(import_delay, (multihost_interval + 3635 ub->ub_mmp_delay) * import_intervals); 3636 3637 zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu " 3638 "import_intervals=%llu leaves=%u", 3639 (u_longlong_t)import_delay, 3640 (u_longlong_t)ub->ub_mmp_delay, 3641 (u_longlong_t)import_intervals, 3642 vdev_count_leaves(spa)); 3643 } else { 3644 /* Using local tunings is the only reasonable option */ 3645 zfs_dbgmsg("pool last imported on non-MMP aware " 3646 "host using import_delay=%llu multihost_interval=%llu " 3647 "import_intervals=%llu", (u_longlong_t)import_delay, 3648 (u_longlong_t)multihost_interval, 3649 (u_longlong_t)import_intervals); 3650 } 3651 3652 return (import_delay); 3653 } 3654 3655 /* 3656 * Remote host activity check. 3657 * 3658 * error results: 3659 * 0 - no activity detected 3660 * EREMOTEIO - remote activity detected 3661 * EINTR - user canceled the operation 3662 */ 3663 static int 3664 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config, 3665 boolean_t importing) 3666 { 3667 uint64_t txg = ub->ub_txg; 3668 uint64_t timestamp = ub->ub_timestamp; 3669 uint64_t mmp_config = ub->ub_mmp_config; 3670 uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0; 3671 uint64_t import_delay; 3672 hrtime_t import_expire, now; 3673 nvlist_t *mmp_label = NULL; 3674 vdev_t *rvd = spa->spa_root_vdev; 3675 kcondvar_t cv; 3676 kmutex_t mtx; 3677 int error = 0; 3678 3679 cv_init(&cv, NULL, CV_DEFAULT, NULL); 3680 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 3681 mutex_enter(&mtx); 3682 3683 /* 3684 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3685 * during the earlier tryimport. If the txg recorded there is 0 then 3686 * the pool is known to be active on another host. 3687 * 3688 * Otherwise, the pool might be in use on another host. Check for 3689 * changes in the uberblocks on disk if necessary. 3690 */ 3691 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3692 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 3693 ZPOOL_CONFIG_LOAD_INFO); 3694 3695 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && 3696 fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { 3697 vdev_uberblock_load(rvd, ub, &mmp_label); 3698 error = SET_ERROR(EREMOTEIO); 3699 goto out; 3700 } 3701 } 3702 3703 import_delay = spa_activity_check_duration(spa, ub); 3704 3705 /* Add a small random factor in case of simultaneous imports (0-25%) */ 3706 import_delay += import_delay * random_in_range(250) / 1000; 3707 3708 import_expire = gethrtime() + import_delay; 3709 3710 if (importing) { 3711 spa_import_progress_set_notes(spa, "Checking MMP activity, " 3712 "waiting %llu ms", (u_longlong_t)NSEC2MSEC(import_delay)); 3713 } 3714 3715 int iterations = 0; 3716 while ((now = gethrtime()) < import_expire) { 3717 if (importing && iterations++ % 30 == 0) { 3718 spa_import_progress_set_notes(spa, "Checking MMP " 3719 "activity, %llu ms remaining", 3720 (u_longlong_t)NSEC2MSEC(import_expire - now)); 3721 } 3722 3723 if (importing) { 3724 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 3725 NSEC2SEC(import_expire - gethrtime())); 3726 } 3727 3728 vdev_uberblock_load(rvd, ub, &mmp_label); 3729 3730 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || 3731 mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) { 3732 zfs_dbgmsg("multihost activity detected " 3733 "txg %llu ub_txg %llu " 3734 "timestamp %llu ub_timestamp %llu " 3735 "mmp_config %#llx ub_mmp_config %#llx", 3736 (u_longlong_t)txg, (u_longlong_t)ub->ub_txg, 3737 (u_longlong_t)timestamp, 3738 (u_longlong_t)ub->ub_timestamp, 3739 (u_longlong_t)mmp_config, 3740 (u_longlong_t)ub->ub_mmp_config); 3741 3742 error = SET_ERROR(EREMOTEIO); 3743 break; 3744 } 3745 3746 if (mmp_label) { 3747 nvlist_free(mmp_label); 3748 mmp_label = NULL; 3749 } 3750 3751 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 3752 if (error != -1) { 3753 error = SET_ERROR(EINTR); 3754 break; 3755 } 3756 error = 0; 3757 } 3758 3759 out: 3760 mutex_exit(&mtx); 3761 mutex_destroy(&mtx); 3762 cv_destroy(&cv); 3763 3764 /* 3765 * If the pool is determined to be active store the status in the 3766 * spa->spa_load_info nvlist. If the remote hostname or hostid are 3767 * available from configuration read from disk store them as well. 3768 * This allows 'zpool import' to generate a more useful message. 3769 * 3770 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) 3771 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 3772 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 3773 */ 3774 if (error == EREMOTEIO) { 3775 const char *hostname = "<unknown>"; 3776 uint64_t hostid = 0; 3777 3778 if (mmp_label) { 3779 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { 3780 hostname = fnvlist_lookup_string(mmp_label, 3781 ZPOOL_CONFIG_HOSTNAME); 3782 fnvlist_add_string(spa->spa_load_info, 3783 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 3784 } 3785 3786 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { 3787 hostid = fnvlist_lookup_uint64(mmp_label, 3788 ZPOOL_CONFIG_HOSTID); 3789 fnvlist_add_uint64(spa->spa_load_info, 3790 ZPOOL_CONFIG_MMP_HOSTID, hostid); 3791 } 3792 } 3793 3794 fnvlist_add_uint64(spa->spa_load_info, 3795 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); 3796 fnvlist_add_uint64(spa->spa_load_info, 3797 ZPOOL_CONFIG_MMP_TXG, 0); 3798 3799 error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); 3800 } 3801 3802 if (mmp_label) 3803 nvlist_free(mmp_label); 3804 3805 return (error); 3806 } 3807 3808 /* 3809 * Called from zfs_ioc_clear for a pool that was suspended 3810 * after failing mmp write checks. 3811 */ 3812 boolean_t 3813 spa_mmp_remote_host_activity(spa_t *spa) 3814 { 3815 ASSERT(spa_multihost(spa) && spa_suspended(spa)); 3816 3817 nvlist_t *best_label; 3818 uberblock_t best_ub; 3819 3820 /* 3821 * Locate the best uberblock on disk 3822 */ 3823 vdev_uberblock_load(spa->spa_root_vdev, &best_ub, &best_label); 3824 if (best_label) { 3825 /* 3826 * confirm that the best hostid matches our hostid 3827 */ 3828 if (nvlist_exists(best_label, ZPOOL_CONFIG_HOSTID) && 3829 spa_get_hostid(spa) != 3830 fnvlist_lookup_uint64(best_label, ZPOOL_CONFIG_HOSTID)) { 3831 nvlist_free(best_label); 3832 return (B_TRUE); 3833 } 3834 nvlist_free(best_label); 3835 } else { 3836 return (B_TRUE); 3837 } 3838 3839 if (!MMP_VALID(&best_ub) || 3840 !MMP_FAIL_INT_VALID(&best_ub) || 3841 MMP_FAIL_INT(&best_ub) == 0) { 3842 return (B_TRUE); 3843 } 3844 3845 if (best_ub.ub_txg != spa->spa_uberblock.ub_txg || 3846 best_ub.ub_timestamp != spa->spa_uberblock.ub_timestamp) { 3847 zfs_dbgmsg("txg mismatch detected during pool clear " 3848 "txg %llu ub_txg %llu timestamp %llu ub_timestamp %llu", 3849 (u_longlong_t)spa->spa_uberblock.ub_txg, 3850 (u_longlong_t)best_ub.ub_txg, 3851 (u_longlong_t)spa->spa_uberblock.ub_timestamp, 3852 (u_longlong_t)best_ub.ub_timestamp); 3853 return (B_TRUE); 3854 } 3855 3856 /* 3857 * Perform an activity check looking for any remote writer 3858 */ 3859 return (spa_activity_check(spa, &spa->spa_uberblock, spa->spa_config, 3860 B_FALSE) != 0); 3861 } 3862 3863 static int 3864 spa_verify_host(spa_t *spa, nvlist_t *mos_config) 3865 { 3866 uint64_t hostid; 3867 const char *hostname; 3868 uint64_t myhostid = 0; 3869 3870 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 3871 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 3872 hostname = fnvlist_lookup_string(mos_config, 3873 ZPOOL_CONFIG_HOSTNAME); 3874 3875 myhostid = zone_get_hostid(NULL); 3876 3877 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 3878 cmn_err(CE_WARN, "pool '%s' could not be " 3879 "loaded as it was last accessed by " 3880 "another system (host: %s hostid: 0x%llx). " 3881 "See: https://openzfs.github.io/openzfs-docs/msg/" 3882 "ZFS-8000-EY", 3883 spa_name(spa), hostname, (u_longlong_t)hostid); 3884 spa_load_failed(spa, "hostid verification failed: pool " 3885 "last accessed by host: %s (hostid: 0x%llx)", 3886 hostname, (u_longlong_t)hostid); 3887 return (SET_ERROR(EBADF)); 3888 } 3889 } 3890 3891 return (0); 3892 } 3893 3894 static int 3895 spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 3896 { 3897 int error = 0; 3898 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 3899 int parse; 3900 vdev_t *rvd; 3901 uint64_t pool_guid; 3902 const char *comment; 3903 const char *compatibility; 3904 3905 /* 3906 * Versioning wasn't explicitly added to the label until later, so if 3907 * it's not present treat it as the initial version. 3908 */ 3909 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 3910 &spa->spa_ubsync.ub_version) != 0) 3911 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 3912 3913 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 3914 spa_load_failed(spa, "invalid config provided: '%s' missing", 3915 ZPOOL_CONFIG_POOL_GUID); 3916 return (SET_ERROR(EINVAL)); 3917 } 3918 3919 /* 3920 * If we are doing an import, ensure that the pool is not already 3921 * imported by checking if its pool guid already exists in the 3922 * spa namespace. 3923 * 3924 * The only case that we allow an already imported pool to be 3925 * imported again, is when the pool is checkpointed and we want to 3926 * look at its checkpointed state from userland tools like zdb. 3927 */ 3928 #ifdef _KERNEL 3929 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3930 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3931 spa_guid_exists(pool_guid, 0)) { 3932 #else 3933 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3934 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3935 spa_guid_exists(pool_guid, 0) && 3936 !spa_importing_readonly_checkpoint(spa)) { 3937 #endif 3938 spa_load_failed(spa, "a pool with guid %llu is already open", 3939 (u_longlong_t)pool_guid); 3940 return (SET_ERROR(EEXIST)); 3941 } 3942 3943 spa->spa_config_guid = pool_guid; 3944 3945 nvlist_free(spa->spa_load_info); 3946 spa->spa_load_info = fnvlist_alloc(); 3947 3948 ASSERT(spa->spa_comment == NULL); 3949 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 3950 spa->spa_comment = spa_strdup(comment); 3951 3952 ASSERT(spa->spa_compatibility == NULL); 3953 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 3954 &compatibility) == 0) 3955 spa->spa_compatibility = spa_strdup(compatibility); 3956 3957 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 3958 &spa->spa_config_txg); 3959 3960 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 3961 spa->spa_config_splitting = fnvlist_dup(nvl); 3962 3963 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 3964 spa_load_failed(spa, "invalid config provided: '%s' missing", 3965 ZPOOL_CONFIG_VDEV_TREE); 3966 return (SET_ERROR(EINVAL)); 3967 } 3968 3969 /* 3970 * Create "The Godfather" zio to hold all async IOs 3971 */ 3972 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 3973 KM_SLEEP); 3974 for (int i = 0; i < max_ncpus; i++) { 3975 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 3976 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3977 ZIO_FLAG_GODFATHER); 3978 } 3979 3980 /* 3981 * Parse the configuration into a vdev tree. We explicitly set the 3982 * value that will be returned by spa_version() since parsing the 3983 * configuration requires knowing the version number. 3984 */ 3985 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3986 parse = (type == SPA_IMPORT_EXISTING ? 3987 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 3988 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 3989 spa_config_exit(spa, SCL_ALL, FTAG); 3990 3991 if (error != 0) { 3992 spa_load_failed(spa, "unable to parse config [error=%d]", 3993 error); 3994 return (error); 3995 } 3996 3997 ASSERT(spa->spa_root_vdev == rvd); 3998 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 3999 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 4000 4001 if (type != SPA_IMPORT_ASSEMBLE) { 4002 ASSERT(spa_guid(spa) == pool_guid); 4003 } 4004 4005 return (0); 4006 } 4007 4008 /* 4009 * Recursively open all vdevs in the vdev tree. This function is called twice: 4010 * first with the untrusted config, then with the trusted config. 4011 */ 4012 static int 4013 spa_ld_open_vdevs(spa_t *spa) 4014 { 4015 int error = 0; 4016 4017 /* 4018 * spa_missing_tvds_allowed defines how many top-level vdevs can be 4019 * missing/unopenable for the root vdev to be still considered openable. 4020 */ 4021 if (spa->spa_trust_config) { 4022 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 4023 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 4024 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 4025 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 4026 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 4027 } else { 4028 spa->spa_missing_tvds_allowed = 0; 4029 } 4030 4031 spa->spa_missing_tvds_allowed = 4032 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 4033 4034 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4035 error = vdev_open(spa->spa_root_vdev); 4036 spa_config_exit(spa, SCL_ALL, FTAG); 4037 4038 if (spa->spa_missing_tvds != 0) { 4039 spa_load_note(spa, "vdev tree has %lld missing top-level " 4040 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 4041 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 4042 /* 4043 * Although theoretically we could allow users to open 4044 * incomplete pools in RW mode, we'd need to add a lot 4045 * of extra logic (e.g. adjust pool space to account 4046 * for missing vdevs). 4047 * This limitation also prevents users from accidentally 4048 * opening the pool in RW mode during data recovery and 4049 * damaging it further. 4050 */ 4051 spa_load_note(spa, "pools with missing top-level " 4052 "vdevs can only be opened in read-only mode."); 4053 error = SET_ERROR(ENXIO); 4054 } else { 4055 spa_load_note(spa, "current settings allow for maximum " 4056 "%lld missing top-level vdevs at this stage.", 4057 (u_longlong_t)spa->spa_missing_tvds_allowed); 4058 } 4059 } 4060 if (error != 0) { 4061 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 4062 error); 4063 } 4064 if (spa->spa_missing_tvds != 0 || error != 0) 4065 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 4066 4067 return (error); 4068 } 4069 4070 /* 4071 * We need to validate the vdev labels against the configuration that 4072 * we have in hand. This function is called twice: first with an untrusted 4073 * config, then with a trusted config. The validation is more strict when the 4074 * config is trusted. 4075 */ 4076 static int 4077 spa_ld_validate_vdevs(spa_t *spa) 4078 { 4079 int error = 0; 4080 vdev_t *rvd = spa->spa_root_vdev; 4081 4082 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4083 error = vdev_validate(rvd); 4084 spa_config_exit(spa, SCL_ALL, FTAG); 4085 4086 if (error != 0) { 4087 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 4088 return (error); 4089 } 4090 4091 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 4092 spa_load_failed(spa, "cannot open vdev tree after invalidating " 4093 "some vdevs"); 4094 vdev_dbgmsg_print_tree(rvd, 2); 4095 return (SET_ERROR(ENXIO)); 4096 } 4097 4098 return (0); 4099 } 4100 4101 static void 4102 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 4103 { 4104 spa->spa_state = POOL_STATE_ACTIVE; 4105 spa->spa_ubsync = spa->spa_uberblock; 4106 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 4107 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 4108 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 4109 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 4110 spa->spa_claim_max_txg = spa->spa_first_txg; 4111 spa->spa_prev_software_version = ub->ub_software_version; 4112 } 4113 4114 static int 4115 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 4116 { 4117 vdev_t *rvd = spa->spa_root_vdev; 4118 nvlist_t *label; 4119 uberblock_t *ub = &spa->spa_uberblock; 4120 boolean_t activity_check = B_FALSE; 4121 4122 /* 4123 * If we are opening the checkpointed state of the pool by 4124 * rewinding to it, at this point we will have written the 4125 * checkpointed uberblock to the vdev labels, so searching 4126 * the labels will find the right uberblock. However, if 4127 * we are opening the checkpointed state read-only, we have 4128 * not modified the labels. Therefore, we must ignore the 4129 * labels and continue using the spa_uberblock that was set 4130 * by spa_ld_checkpoint_rewind. 4131 * 4132 * Note that it would be fine to ignore the labels when 4133 * rewinding (opening writeable) as well. However, if we 4134 * crash just after writing the labels, we will end up 4135 * searching the labels. Doing so in the common case means 4136 * that this code path gets exercised normally, rather than 4137 * just in the edge case. 4138 */ 4139 if (ub->ub_checkpoint_txg != 0 && 4140 spa_importing_readonly_checkpoint(spa)) { 4141 spa_ld_select_uberblock_done(spa, ub); 4142 return (0); 4143 } 4144 4145 /* 4146 * Find the best uberblock. 4147 */ 4148 vdev_uberblock_load(rvd, ub, &label); 4149 4150 /* 4151 * If we weren't able to find a single valid uberblock, return failure. 4152 */ 4153 if (ub->ub_txg == 0) { 4154 nvlist_free(label); 4155 spa_load_failed(spa, "no valid uberblock found"); 4156 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 4157 } 4158 4159 if (spa->spa_load_max_txg != UINT64_MAX) { 4160 (void) spa_import_progress_set_max_txg(spa_guid(spa), 4161 (u_longlong_t)spa->spa_load_max_txg); 4162 } 4163 spa_load_note(spa, "using uberblock with txg=%llu", 4164 (u_longlong_t)ub->ub_txg); 4165 if (ub->ub_raidz_reflow_info != 0) { 4166 spa_load_note(spa, "uberblock raidz_reflow_info: " 4167 "state=%u offset=%llu", 4168 (int)RRSS_GET_STATE(ub), 4169 (u_longlong_t)RRSS_GET_OFFSET(ub)); 4170 } 4171 4172 4173 /* 4174 * For pools which have the multihost property on determine if the 4175 * pool is truly inactive and can be safely imported. Prevent 4176 * hosts which don't have a hostid set from importing the pool. 4177 */ 4178 activity_check = spa_activity_check_required(spa, ub, label, 4179 spa->spa_config); 4180 if (activity_check) { 4181 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 4182 spa_get_hostid(spa) == 0) { 4183 nvlist_free(label); 4184 fnvlist_add_uint64(spa->spa_load_info, 4185 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4186 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4187 } 4188 4189 int error = 4190 spa_activity_check(spa, ub, spa->spa_config, B_TRUE); 4191 if (error) { 4192 nvlist_free(label); 4193 return (error); 4194 } 4195 4196 fnvlist_add_uint64(spa->spa_load_info, 4197 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); 4198 fnvlist_add_uint64(spa->spa_load_info, 4199 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); 4200 fnvlist_add_uint16(spa->spa_load_info, 4201 ZPOOL_CONFIG_MMP_SEQ, 4202 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)); 4203 } 4204 4205 /* 4206 * If the pool has an unsupported version we can't open it. 4207 */ 4208 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 4209 nvlist_free(label); 4210 spa_load_failed(spa, "version %llu is not supported", 4211 (u_longlong_t)ub->ub_version); 4212 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 4213 } 4214 4215 if (ub->ub_version >= SPA_VERSION_FEATURES) { 4216 nvlist_t *features; 4217 4218 /* 4219 * If we weren't able to find what's necessary for reading the 4220 * MOS in the label, return failure. 4221 */ 4222 if (label == NULL) { 4223 spa_load_failed(spa, "label config unavailable"); 4224 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4225 ENXIO)); 4226 } 4227 4228 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 4229 &features) != 0) { 4230 nvlist_free(label); 4231 spa_load_failed(spa, "invalid label: '%s' missing", 4232 ZPOOL_CONFIG_FEATURES_FOR_READ); 4233 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4234 ENXIO)); 4235 } 4236 4237 /* 4238 * Update our in-core representation with the definitive values 4239 * from the label. 4240 */ 4241 nvlist_free(spa->spa_label_features); 4242 spa->spa_label_features = fnvlist_dup(features); 4243 } 4244 4245 nvlist_free(label); 4246 4247 /* 4248 * Look through entries in the label nvlist's features_for_read. If 4249 * there is a feature listed there which we don't understand then we 4250 * cannot open a pool. 4251 */ 4252 if (ub->ub_version >= SPA_VERSION_FEATURES) { 4253 nvlist_t *unsup_feat; 4254 4255 unsup_feat = fnvlist_alloc(); 4256 4257 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 4258 NULL); nvp != NULL; 4259 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 4260 if (!zfeature_is_supported(nvpair_name(nvp))) { 4261 fnvlist_add_string(unsup_feat, 4262 nvpair_name(nvp), ""); 4263 } 4264 } 4265 4266 if (!nvlist_empty(unsup_feat)) { 4267 fnvlist_add_nvlist(spa->spa_load_info, 4268 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 4269 nvlist_free(unsup_feat); 4270 spa_load_failed(spa, "some features are unsupported"); 4271 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 4272 ENOTSUP)); 4273 } 4274 4275 nvlist_free(unsup_feat); 4276 } 4277 4278 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 4279 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4280 spa_try_repair(spa, spa->spa_config); 4281 spa_config_exit(spa, SCL_ALL, FTAG); 4282 nvlist_free(spa->spa_config_splitting); 4283 spa->spa_config_splitting = NULL; 4284 } 4285 4286 /* 4287 * Initialize internal SPA structures. 4288 */ 4289 spa_ld_select_uberblock_done(spa, ub); 4290 4291 return (0); 4292 } 4293 4294 static int 4295 spa_ld_open_rootbp(spa_t *spa) 4296 { 4297 int error = 0; 4298 vdev_t *rvd = spa->spa_root_vdev; 4299 4300 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 4301 if (error != 0) { 4302 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 4303 "[error=%d]", error); 4304 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4305 } 4306 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 4307 4308 return (0); 4309 } 4310 4311 static int 4312 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 4313 boolean_t reloading) 4314 { 4315 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 4316 nvlist_t *nv, *mos_config, *policy; 4317 int error = 0, copy_error; 4318 uint64_t healthy_tvds, healthy_tvds_mos; 4319 uint64_t mos_config_txg; 4320 4321 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 4322 != 0) 4323 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4324 4325 /* 4326 * If we're assembling a pool from a split, the config provided is 4327 * already trusted so there is nothing to do. 4328 */ 4329 if (type == SPA_IMPORT_ASSEMBLE) 4330 return (0); 4331 4332 healthy_tvds = spa_healthy_core_tvds(spa); 4333 4334 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 4335 != 0) { 4336 spa_load_failed(spa, "unable to retrieve MOS config"); 4337 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4338 } 4339 4340 /* 4341 * If we are doing an open, pool owner wasn't verified yet, thus do 4342 * the verification here. 4343 */ 4344 if (spa->spa_load_state == SPA_LOAD_OPEN) { 4345 error = spa_verify_host(spa, mos_config); 4346 if (error != 0) { 4347 nvlist_free(mos_config); 4348 return (error); 4349 } 4350 } 4351 4352 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 4353 4354 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4355 4356 /* 4357 * Build a new vdev tree from the trusted config 4358 */ 4359 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 4360 if (error != 0) { 4361 nvlist_free(mos_config); 4362 spa_config_exit(spa, SCL_ALL, FTAG); 4363 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 4364 error); 4365 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4366 } 4367 4368 /* 4369 * Vdev paths in the MOS may be obsolete. If the untrusted config was 4370 * obtained by scanning /dev/dsk, then it will have the right vdev 4371 * paths. We update the trusted MOS config with this information. 4372 * We first try to copy the paths with vdev_copy_path_strict, which 4373 * succeeds only when both configs have exactly the same vdev tree. 4374 * If that fails, we fall back to a more flexible method that has a 4375 * best effort policy. 4376 */ 4377 copy_error = vdev_copy_path_strict(rvd, mrvd); 4378 if (copy_error != 0 || spa_load_print_vdev_tree) { 4379 spa_load_note(spa, "provided vdev tree:"); 4380 vdev_dbgmsg_print_tree(rvd, 2); 4381 spa_load_note(spa, "MOS vdev tree:"); 4382 vdev_dbgmsg_print_tree(mrvd, 2); 4383 } 4384 if (copy_error != 0) { 4385 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 4386 "back to vdev_copy_path_relaxed"); 4387 vdev_copy_path_relaxed(rvd, mrvd); 4388 } 4389 4390 vdev_close(rvd); 4391 vdev_free(rvd); 4392 spa->spa_root_vdev = mrvd; 4393 rvd = mrvd; 4394 spa_config_exit(spa, SCL_ALL, FTAG); 4395 4396 /* 4397 * If 'zpool import' used a cached config, then the on-disk hostid and 4398 * hostname may be different to the cached config in ways that should 4399 * prevent import. Userspace can't discover this without a scan, but 4400 * we know, so we add these values to LOAD_INFO so the caller can know 4401 * the difference. 4402 * 4403 * Note that we have to do this before the config is regenerated, 4404 * because the new config will have the hostid and hostname for this 4405 * host, in readiness for import. 4406 */ 4407 if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTID)) 4408 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_HOSTID, 4409 fnvlist_lookup_uint64(mos_config, ZPOOL_CONFIG_HOSTID)); 4410 if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTNAME)) 4411 fnvlist_add_string(spa->spa_load_info, ZPOOL_CONFIG_HOSTNAME, 4412 fnvlist_lookup_string(mos_config, ZPOOL_CONFIG_HOSTNAME)); 4413 4414 /* 4415 * We will use spa_config if we decide to reload the spa or if spa_load 4416 * fails and we rewind. We must thus regenerate the config using the 4417 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 4418 * pass settings on how to load the pool and is not stored in the MOS. 4419 * We copy it over to our new, trusted config. 4420 */ 4421 mos_config_txg = fnvlist_lookup_uint64(mos_config, 4422 ZPOOL_CONFIG_POOL_TXG); 4423 nvlist_free(mos_config); 4424 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 4425 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 4426 &policy) == 0) 4427 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 4428 spa_config_set(spa, mos_config); 4429 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 4430 4431 /* 4432 * Now that we got the config from the MOS, we should be more strict 4433 * in checking blkptrs and can make assumptions about the consistency 4434 * of the vdev tree. spa_trust_config must be set to true before opening 4435 * vdevs in order for them to be writeable. 4436 */ 4437 spa->spa_trust_config = B_TRUE; 4438 4439 /* 4440 * Open and validate the new vdev tree 4441 */ 4442 error = spa_ld_open_vdevs(spa); 4443 if (error != 0) 4444 return (error); 4445 4446 error = spa_ld_validate_vdevs(spa); 4447 if (error != 0) 4448 return (error); 4449 4450 if (copy_error != 0 || spa_load_print_vdev_tree) { 4451 spa_load_note(spa, "final vdev tree:"); 4452 vdev_dbgmsg_print_tree(rvd, 2); 4453 } 4454 4455 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 4456 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 4457 /* 4458 * Sanity check to make sure that we are indeed loading the 4459 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 4460 * in the config provided and they happened to be the only ones 4461 * to have the latest uberblock, we could involuntarily perform 4462 * an extreme rewind. 4463 */ 4464 healthy_tvds_mos = spa_healthy_core_tvds(spa); 4465 if (healthy_tvds_mos - healthy_tvds >= 4466 SPA_SYNC_MIN_VDEVS) { 4467 spa_load_note(spa, "config provided misses too many " 4468 "top-level vdevs compared to MOS (%lld vs %lld). ", 4469 (u_longlong_t)healthy_tvds, 4470 (u_longlong_t)healthy_tvds_mos); 4471 spa_load_note(spa, "vdev tree:"); 4472 vdev_dbgmsg_print_tree(rvd, 2); 4473 if (reloading) { 4474 spa_load_failed(spa, "config was already " 4475 "provided from MOS. Aborting."); 4476 return (spa_vdev_err(rvd, 4477 VDEV_AUX_CORRUPT_DATA, EIO)); 4478 } 4479 spa_load_note(spa, "spa must be reloaded using MOS " 4480 "config"); 4481 return (SET_ERROR(EAGAIN)); 4482 } 4483 } 4484 4485 error = spa_check_for_missing_logs(spa); 4486 if (error != 0) 4487 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 4488 4489 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 4490 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 4491 "guid sum (%llu != %llu)", 4492 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 4493 (u_longlong_t)rvd->vdev_guid_sum); 4494 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 4495 ENXIO)); 4496 } 4497 4498 return (0); 4499 } 4500 4501 static int 4502 spa_ld_open_indirect_vdev_metadata(spa_t *spa) 4503 { 4504 int error = 0; 4505 vdev_t *rvd = spa->spa_root_vdev; 4506 4507 /* 4508 * Everything that we read before spa_remove_init() must be stored 4509 * on concreted vdevs. Therefore we do this as early as possible. 4510 */ 4511 error = spa_remove_init(spa); 4512 if (error != 0) { 4513 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 4514 error); 4515 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4516 } 4517 4518 /* 4519 * Retrieve information needed to condense indirect vdev mappings. 4520 */ 4521 error = spa_condense_init(spa); 4522 if (error != 0) { 4523 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 4524 error); 4525 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4526 } 4527 4528 return (0); 4529 } 4530 4531 static int 4532 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 4533 { 4534 int error = 0; 4535 vdev_t *rvd = spa->spa_root_vdev; 4536 4537 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 4538 boolean_t missing_feat_read = B_FALSE; 4539 nvlist_t *unsup_feat, *enabled_feat; 4540 4541 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 4542 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 4543 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4544 } 4545 4546 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 4547 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 4548 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4549 } 4550 4551 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 4552 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 4553 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4554 } 4555 4556 enabled_feat = fnvlist_alloc(); 4557 unsup_feat = fnvlist_alloc(); 4558 4559 if (!spa_features_check(spa, B_FALSE, 4560 unsup_feat, enabled_feat)) 4561 missing_feat_read = B_TRUE; 4562 4563 if (spa_writeable(spa) || 4564 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 4565 if (!spa_features_check(spa, B_TRUE, 4566 unsup_feat, enabled_feat)) { 4567 *missing_feat_writep = B_TRUE; 4568 } 4569 } 4570 4571 fnvlist_add_nvlist(spa->spa_load_info, 4572 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 4573 4574 if (!nvlist_empty(unsup_feat)) { 4575 fnvlist_add_nvlist(spa->spa_load_info, 4576 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 4577 } 4578 4579 fnvlist_free(enabled_feat); 4580 fnvlist_free(unsup_feat); 4581 4582 if (!missing_feat_read) { 4583 fnvlist_add_boolean(spa->spa_load_info, 4584 ZPOOL_CONFIG_CAN_RDONLY); 4585 } 4586 4587 /* 4588 * If the state is SPA_LOAD_TRYIMPORT, our objective is 4589 * twofold: to determine whether the pool is available for 4590 * import in read-write mode and (if it is not) whether the 4591 * pool is available for import in read-only mode. If the pool 4592 * is available for import in read-write mode, it is displayed 4593 * as available in userland; if it is not available for import 4594 * in read-only mode, it is displayed as unavailable in 4595 * userland. If the pool is available for import in read-only 4596 * mode but not read-write mode, it is displayed as unavailable 4597 * in userland with a special note that the pool is actually 4598 * available for open in read-only mode. 4599 * 4600 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 4601 * missing a feature for write, we must first determine whether 4602 * the pool can be opened read-only before returning to 4603 * userland in order to know whether to display the 4604 * abovementioned note. 4605 */ 4606 if (missing_feat_read || (*missing_feat_writep && 4607 spa_writeable(spa))) { 4608 spa_load_failed(spa, "pool uses unsupported features"); 4609 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 4610 ENOTSUP)); 4611 } 4612 4613 /* 4614 * Load refcounts for ZFS features from disk into an in-memory 4615 * cache during SPA initialization. 4616 */ 4617 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 4618 uint64_t refcount; 4619 4620 error = feature_get_refcount_from_disk(spa, 4621 &spa_feature_table[i], &refcount); 4622 if (error == 0) { 4623 spa->spa_feat_refcount_cache[i] = refcount; 4624 } else if (error == ENOTSUP) { 4625 spa->spa_feat_refcount_cache[i] = 4626 SPA_FEATURE_DISABLED; 4627 } else { 4628 spa_load_failed(spa, "error getting refcount " 4629 "for feature %s [error=%d]", 4630 spa_feature_table[i].fi_guid, error); 4631 return (spa_vdev_err(rvd, 4632 VDEV_AUX_CORRUPT_DATA, EIO)); 4633 } 4634 } 4635 } 4636 4637 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 4638 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 4639 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 4640 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4641 } 4642 4643 /* 4644 * Encryption was added before bookmark_v2, even though bookmark_v2 4645 * is now a dependency. If this pool has encryption enabled without 4646 * bookmark_v2, trigger an errata message. 4647 */ 4648 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 4649 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 4650 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 4651 } 4652 4653 return (0); 4654 } 4655 4656 static int 4657 spa_ld_load_special_directories(spa_t *spa) 4658 { 4659 int error = 0; 4660 vdev_t *rvd = spa->spa_root_vdev; 4661 4662 spa->spa_is_initializing = B_TRUE; 4663 error = dsl_pool_open(spa->spa_dsl_pool); 4664 spa->spa_is_initializing = B_FALSE; 4665 if (error != 0) { 4666 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 4667 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4668 } 4669 4670 return (0); 4671 } 4672 4673 static int 4674 spa_ld_get_props(spa_t *spa) 4675 { 4676 int error = 0; 4677 uint64_t obj; 4678 vdev_t *rvd = spa->spa_root_vdev; 4679 4680 /* Grab the checksum salt from the MOS. */ 4681 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4682 DMU_POOL_CHECKSUM_SALT, 1, 4683 sizeof (spa->spa_cksum_salt.zcs_bytes), 4684 spa->spa_cksum_salt.zcs_bytes); 4685 if (error == ENOENT) { 4686 /* Generate a new salt for subsequent use */ 4687 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 4688 sizeof (spa->spa_cksum_salt.zcs_bytes)); 4689 } else if (error != 0) { 4690 spa_load_failed(spa, "unable to retrieve checksum salt from " 4691 "MOS [error=%d]", error); 4692 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4693 } 4694 4695 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 4696 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4697 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 4698 if (error != 0) { 4699 spa_load_failed(spa, "error opening deferred-frees bpobj " 4700 "[error=%d]", error); 4701 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4702 } 4703 4704 /* 4705 * Load the bit that tells us to use the new accounting function 4706 * (raid-z deflation). If we have an older pool, this will not 4707 * be present. 4708 */ 4709 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 4710 if (error != 0 && error != ENOENT) 4711 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4712 4713 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 4714 &spa->spa_creation_version, B_FALSE); 4715 if (error != 0 && error != ENOENT) 4716 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4717 4718 /* 4719 * Load the persistent error log. If we have an older pool, this will 4720 * not be present. 4721 */ 4722 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 4723 B_FALSE); 4724 if (error != 0 && error != ENOENT) 4725 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4726 4727 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 4728 &spa->spa_errlog_scrub, B_FALSE); 4729 if (error != 0 && error != ENOENT) 4730 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4731 4732 /* Load the last scrubbed txg. */ 4733 error = spa_dir_prop(spa, DMU_POOL_LAST_SCRUBBED_TXG, 4734 &spa->spa_scrubbed_last_txg, B_FALSE); 4735 if (error != 0 && error != ENOENT) 4736 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4737 4738 /* 4739 * Load the livelist deletion field. If a livelist is queued for 4740 * deletion, indicate that in the spa 4741 */ 4742 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 4743 &spa->spa_livelists_to_delete, B_FALSE); 4744 if (error != 0 && error != ENOENT) 4745 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4746 4747 /* 4748 * Load the history object. If we have an older pool, this 4749 * will not be present. 4750 */ 4751 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 4752 if (error != 0 && error != ENOENT) 4753 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4754 4755 /* 4756 * Load the per-vdev ZAP map. If we have an older pool, this will not 4757 * be present; in this case, defer its creation to a later time to 4758 * avoid dirtying the MOS this early / out of sync context. See 4759 * spa_sync_config_object. 4760 */ 4761 4762 /* The sentinel is only available in the MOS config. */ 4763 nvlist_t *mos_config; 4764 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 4765 spa_load_failed(spa, "unable to retrieve MOS config"); 4766 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4767 } 4768 4769 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 4770 &spa->spa_all_vdev_zaps, B_FALSE); 4771 4772 if (error == ENOENT) { 4773 VERIFY(!nvlist_exists(mos_config, 4774 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 4775 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 4776 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4777 } else if (error != 0) { 4778 nvlist_free(mos_config); 4779 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4780 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 4781 /* 4782 * An older version of ZFS overwrote the sentinel value, so 4783 * we have orphaned per-vdev ZAPs in the MOS. Defer their 4784 * destruction to later; see spa_sync_config_object. 4785 */ 4786 spa->spa_avz_action = AVZ_ACTION_DESTROY; 4787 /* 4788 * We're assuming that no vdevs have had their ZAPs created 4789 * before this. Better be sure of it. 4790 */ 4791 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4792 } 4793 nvlist_free(mos_config); 4794 4795 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 4796 4797 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 4798 B_FALSE); 4799 if (error && error != ENOENT) 4800 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4801 4802 if (error == 0) { 4803 uint64_t autoreplace = 0; 4804 4805 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 4806 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 4807 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 4808 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 4809 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 4810 spa_prop_find(spa, ZPOOL_PROP_DEDUP_TABLE_QUOTA, 4811 &spa->spa_dedup_table_quota); 4812 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 4813 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 4814 spa->spa_autoreplace = (autoreplace != 0); 4815 } 4816 4817 /* 4818 * If we are importing a pool with missing top-level vdevs, 4819 * we enforce that the pool doesn't panic or get suspended on 4820 * error since the likelihood of missing data is extremely high. 4821 */ 4822 if (spa->spa_missing_tvds > 0 && 4823 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 4824 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4825 spa_load_note(spa, "forcing failmode to 'continue' " 4826 "as some top level vdevs are missing"); 4827 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 4828 } 4829 4830 return (0); 4831 } 4832 4833 static int 4834 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 4835 { 4836 int error = 0; 4837 vdev_t *rvd = spa->spa_root_vdev; 4838 4839 /* 4840 * If we're assembling the pool from the split-off vdevs of 4841 * an existing pool, we don't want to attach the spares & cache 4842 * devices. 4843 */ 4844 4845 /* 4846 * Load any hot spares for this pool. 4847 */ 4848 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 4849 B_FALSE); 4850 if (error != 0 && error != ENOENT) 4851 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4852 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4853 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 4854 if (load_nvlist(spa, spa->spa_spares.sav_object, 4855 &spa->spa_spares.sav_config) != 0) { 4856 spa_load_failed(spa, "error loading spares nvlist"); 4857 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4858 } 4859 4860 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4861 spa_load_spares(spa); 4862 spa_config_exit(spa, SCL_ALL, FTAG); 4863 } else if (error == 0) { 4864 spa->spa_spares.sav_sync = B_TRUE; 4865 } 4866 4867 /* 4868 * Load any level 2 ARC devices for this pool. 4869 */ 4870 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 4871 &spa->spa_l2cache.sav_object, B_FALSE); 4872 if (error != 0 && error != ENOENT) 4873 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4874 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4875 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 4876 if (load_nvlist(spa, spa->spa_l2cache.sav_object, 4877 &spa->spa_l2cache.sav_config) != 0) { 4878 spa_load_failed(spa, "error loading l2cache nvlist"); 4879 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4880 } 4881 4882 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4883 spa_load_l2cache(spa); 4884 spa_config_exit(spa, SCL_ALL, FTAG); 4885 } else if (error == 0) { 4886 spa->spa_l2cache.sav_sync = B_TRUE; 4887 } 4888 4889 return (0); 4890 } 4891 4892 static int 4893 spa_ld_load_vdev_metadata(spa_t *spa) 4894 { 4895 int error = 0; 4896 vdev_t *rvd = spa->spa_root_vdev; 4897 4898 /* 4899 * If the 'multihost' property is set, then never allow a pool to 4900 * be imported when the system hostid is zero. The exception to 4901 * this rule is zdb which is always allowed to access pools. 4902 */ 4903 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 4904 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 4905 fnvlist_add_uint64(spa->spa_load_info, 4906 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4907 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4908 } 4909 4910 /* 4911 * If the 'autoreplace' property is set, then post a resource notifying 4912 * the ZFS DE that it should not issue any faults for unopenable 4913 * devices. We also iterate over the vdevs, and post a sysevent for any 4914 * unopenable vdevs so that the normal autoreplace handler can take 4915 * over. 4916 */ 4917 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4918 spa_check_removed(spa->spa_root_vdev); 4919 /* 4920 * For the import case, this is done in spa_import(), because 4921 * at this point we're using the spare definitions from 4922 * the MOS config, not necessarily from the userland config. 4923 */ 4924 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 4925 spa_aux_check_removed(&spa->spa_spares); 4926 spa_aux_check_removed(&spa->spa_l2cache); 4927 } 4928 } 4929 4930 /* 4931 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 4932 */ 4933 error = vdev_load(rvd); 4934 if (error != 0) { 4935 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 4936 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4937 } 4938 4939 error = spa_ld_log_spacemaps(spa); 4940 if (error != 0) { 4941 spa_load_failed(spa, "spa_ld_log_spacemaps failed [error=%d]", 4942 error); 4943 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4944 } 4945 4946 /* 4947 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 4948 */ 4949 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4950 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 4951 spa_config_exit(spa, SCL_ALL, FTAG); 4952 4953 return (0); 4954 } 4955 4956 static int 4957 spa_ld_load_dedup_tables(spa_t *spa) 4958 { 4959 int error = 0; 4960 vdev_t *rvd = spa->spa_root_vdev; 4961 4962 error = ddt_load(spa); 4963 if (error != 0) { 4964 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 4965 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4966 } 4967 4968 return (0); 4969 } 4970 4971 static int 4972 spa_ld_load_brt(spa_t *spa) 4973 { 4974 int error = 0; 4975 vdev_t *rvd = spa->spa_root_vdev; 4976 4977 error = brt_load(spa); 4978 if (error != 0) { 4979 spa_load_failed(spa, "brt_load failed [error=%d]", error); 4980 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4981 } 4982 4983 return (0); 4984 } 4985 4986 static int 4987 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, const char **ereport) 4988 { 4989 vdev_t *rvd = spa->spa_root_vdev; 4990 4991 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 4992 boolean_t missing = spa_check_logs(spa); 4993 if (missing) { 4994 if (spa->spa_missing_tvds != 0) { 4995 spa_load_note(spa, "spa_check_logs failed " 4996 "so dropping the logs"); 4997 } else { 4998 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 4999 spa_load_failed(spa, "spa_check_logs failed"); 5000 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 5001 ENXIO)); 5002 } 5003 } 5004 } 5005 5006 return (0); 5007 } 5008 5009 static int 5010 spa_ld_verify_pool_data(spa_t *spa) 5011 { 5012 int error = 0; 5013 vdev_t *rvd = spa->spa_root_vdev; 5014 5015 /* 5016 * We've successfully opened the pool, verify that we're ready 5017 * to start pushing transactions. 5018 */ 5019 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 5020 error = spa_load_verify(spa); 5021 if (error != 0) { 5022 spa_load_failed(spa, "spa_load_verify failed " 5023 "[error=%d]", error); 5024 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 5025 error)); 5026 } 5027 } 5028 5029 return (0); 5030 } 5031 5032 static void 5033 spa_ld_claim_log_blocks(spa_t *spa) 5034 { 5035 dmu_tx_t *tx; 5036 dsl_pool_t *dp = spa_get_dsl(spa); 5037 5038 /* 5039 * Claim log blocks that haven't been committed yet. 5040 * This must all happen in a single txg. 5041 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 5042 * invoked from zil_claim_log_block()'s i/o done callback. 5043 * Price of rollback is that we abandon the log. 5044 */ 5045 spa->spa_claiming = B_TRUE; 5046 5047 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 5048 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 5049 zil_claim, tx, DS_FIND_CHILDREN); 5050 dmu_tx_commit(tx); 5051 5052 spa->spa_claiming = B_FALSE; 5053 5054 spa_set_log_state(spa, SPA_LOG_GOOD); 5055 } 5056 5057 static void 5058 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 5059 boolean_t update_config_cache) 5060 { 5061 vdev_t *rvd = spa->spa_root_vdev; 5062 int need_update = B_FALSE; 5063 5064 /* 5065 * If the config cache is stale, or we have uninitialized 5066 * metaslabs (see spa_vdev_add()), then update the config. 5067 * 5068 * If this is a verbatim import, trust the current 5069 * in-core spa_config and update the disk labels. 5070 */ 5071 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 5072 spa->spa_load_state == SPA_LOAD_IMPORT || 5073 spa->spa_load_state == SPA_LOAD_RECOVER || 5074 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 5075 need_update = B_TRUE; 5076 5077 for (int c = 0; c < rvd->vdev_children; c++) 5078 if (rvd->vdev_child[c]->vdev_ms_array == 0) 5079 need_update = B_TRUE; 5080 5081 /* 5082 * Update the config cache asynchronously in case we're the 5083 * root pool, in which case the config cache isn't writable yet. 5084 */ 5085 if (need_update) 5086 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 5087 } 5088 5089 static void 5090 spa_ld_prepare_for_reload(spa_t *spa) 5091 { 5092 spa_mode_t mode = spa->spa_mode; 5093 int async_suspended = spa->spa_async_suspended; 5094 5095 spa_unload(spa); 5096 spa_deactivate(spa); 5097 spa_activate(spa, mode); 5098 5099 /* 5100 * We save the value of spa_async_suspended as it gets reset to 0 by 5101 * spa_unload(). We want to restore it back to the original value before 5102 * returning as we might be calling spa_async_resume() later. 5103 */ 5104 spa->spa_async_suspended = async_suspended; 5105 } 5106 5107 static int 5108 spa_ld_read_checkpoint_txg(spa_t *spa) 5109 { 5110 uberblock_t checkpoint; 5111 int error = 0; 5112 5113 ASSERT0(spa->spa_checkpoint_txg); 5114 ASSERT(MUTEX_HELD(&spa_namespace_lock) || 5115 spa->spa_load_thread == curthread); 5116 5117 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 5118 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 5119 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 5120 5121 if (error == ENOENT) 5122 return (0); 5123 5124 if (error != 0) 5125 return (error); 5126 5127 ASSERT3U(checkpoint.ub_txg, !=, 0); 5128 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 5129 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 5130 spa->spa_checkpoint_txg = checkpoint.ub_txg; 5131 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 5132 5133 return (0); 5134 } 5135 5136 static int 5137 spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 5138 { 5139 int error = 0; 5140 5141 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 5142 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 5143 5144 /* 5145 * Never trust the config that is provided unless we are assembling 5146 * a pool following a split. 5147 * This means don't trust blkptrs and the vdev tree in general. This 5148 * also effectively puts the spa in read-only mode since 5149 * spa_writeable() checks for spa_trust_config to be true. 5150 * We will later load a trusted config from the MOS. 5151 */ 5152 if (type != SPA_IMPORT_ASSEMBLE) 5153 spa->spa_trust_config = B_FALSE; 5154 5155 /* 5156 * Parse the config provided to create a vdev tree. 5157 */ 5158 error = spa_ld_parse_config(spa, type); 5159 if (error != 0) 5160 return (error); 5161 5162 spa_import_progress_add(spa); 5163 5164 /* 5165 * Now that we have the vdev tree, try to open each vdev. This involves 5166 * opening the underlying physical device, retrieving its geometry and 5167 * probing the vdev with a dummy I/O. The state of each vdev will be set 5168 * based on the success of those operations. After this we'll be ready 5169 * to read from the vdevs. 5170 */ 5171 error = spa_ld_open_vdevs(spa); 5172 if (error != 0) 5173 return (error); 5174 5175 /* 5176 * Read the label of each vdev and make sure that the GUIDs stored 5177 * there match the GUIDs in the config provided. 5178 * If we're assembling a new pool that's been split off from an 5179 * existing pool, the labels haven't yet been updated so we skip 5180 * validation for now. 5181 */ 5182 if (type != SPA_IMPORT_ASSEMBLE) { 5183 error = spa_ld_validate_vdevs(spa); 5184 if (error != 0) 5185 return (error); 5186 } 5187 5188 /* 5189 * Read all vdev labels to find the best uberblock (i.e. latest, 5190 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 5191 * get the list of features required to read blkptrs in the MOS from 5192 * the vdev label with the best uberblock and verify that our version 5193 * of zfs supports them all. 5194 */ 5195 error = spa_ld_select_uberblock(spa, type); 5196 if (error != 0) 5197 return (error); 5198 5199 /* 5200 * Pass that uberblock to the dsl_pool layer which will open the root 5201 * blkptr. This blkptr points to the latest version of the MOS and will 5202 * allow us to read its contents. 5203 */ 5204 error = spa_ld_open_rootbp(spa); 5205 if (error != 0) 5206 return (error); 5207 5208 return (0); 5209 } 5210 5211 static int 5212 spa_ld_checkpoint_rewind(spa_t *spa) 5213 { 5214 uberblock_t checkpoint; 5215 int error = 0; 5216 5217 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 5218 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5219 5220 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 5221 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 5222 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 5223 5224 if (error != 0) { 5225 spa_load_failed(spa, "unable to retrieve checkpointed " 5226 "uberblock from the MOS config [error=%d]", error); 5227 5228 if (error == ENOENT) 5229 error = ZFS_ERR_NO_CHECKPOINT; 5230 5231 return (error); 5232 } 5233 5234 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 5235 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 5236 5237 /* 5238 * We need to update the txg and timestamp of the checkpointed 5239 * uberblock to be higher than the latest one. This ensures that 5240 * the checkpointed uberblock is selected if we were to close and 5241 * reopen the pool right after we've written it in the vdev labels. 5242 * (also see block comment in vdev_uberblock_compare) 5243 */ 5244 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 5245 checkpoint.ub_timestamp = gethrestime_sec(); 5246 5247 /* 5248 * Set current uberblock to be the checkpointed uberblock. 5249 */ 5250 spa->spa_uberblock = checkpoint; 5251 5252 /* 5253 * If we are doing a normal rewind, then the pool is open for 5254 * writing and we sync the "updated" checkpointed uberblock to 5255 * disk. Once this is done, we've basically rewound the whole 5256 * pool and there is no way back. 5257 * 5258 * There are cases when we don't want to attempt and sync the 5259 * checkpointed uberblock to disk because we are opening a 5260 * pool as read-only. Specifically, verifying the checkpointed 5261 * state with zdb, and importing the checkpointed state to get 5262 * a "preview" of its content. 5263 */ 5264 if (spa_writeable(spa)) { 5265 vdev_t *rvd = spa->spa_root_vdev; 5266 5267 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5268 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 5269 int svdcount = 0; 5270 int children = rvd->vdev_children; 5271 int c0 = random_in_range(children); 5272 5273 for (int c = 0; c < children; c++) { 5274 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 5275 5276 /* Stop when revisiting the first vdev */ 5277 if (c > 0 && svd[0] == vd) 5278 break; 5279 5280 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 5281 !vdev_is_concrete(vd)) 5282 continue; 5283 5284 svd[svdcount++] = vd; 5285 if (svdcount == SPA_SYNC_MIN_VDEVS) 5286 break; 5287 } 5288 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg); 5289 if (error == 0) 5290 spa->spa_last_synced_guid = rvd->vdev_guid; 5291 spa_config_exit(spa, SCL_ALL, FTAG); 5292 5293 if (error != 0) { 5294 spa_load_failed(spa, "failed to write checkpointed " 5295 "uberblock to the vdev labels [error=%d]", error); 5296 return (error); 5297 } 5298 } 5299 5300 return (0); 5301 } 5302 5303 static int 5304 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 5305 boolean_t *update_config_cache) 5306 { 5307 int error; 5308 5309 /* 5310 * Parse the config for pool, open and validate vdevs, 5311 * select an uberblock, and use that uberblock to open 5312 * the MOS. 5313 */ 5314 error = spa_ld_mos_init(spa, type); 5315 if (error != 0) 5316 return (error); 5317 5318 /* 5319 * Retrieve the trusted config stored in the MOS and use it to create 5320 * a new, exact version of the vdev tree, then reopen all vdevs. 5321 */ 5322 error = spa_ld_trusted_config(spa, type, B_FALSE); 5323 if (error == EAGAIN) { 5324 if (update_config_cache != NULL) 5325 *update_config_cache = B_TRUE; 5326 5327 /* 5328 * Redo the loading process with the trusted config if it is 5329 * too different from the untrusted config. 5330 */ 5331 spa_ld_prepare_for_reload(spa); 5332 spa_load_note(spa, "RELOADING"); 5333 error = spa_ld_mos_init(spa, type); 5334 if (error != 0) 5335 return (error); 5336 5337 error = spa_ld_trusted_config(spa, type, B_TRUE); 5338 if (error != 0) 5339 return (error); 5340 5341 } else if (error != 0) { 5342 return (error); 5343 } 5344 5345 return (0); 5346 } 5347 5348 /* 5349 * Load an existing storage pool, using the config provided. This config 5350 * describes which vdevs are part of the pool and is later validated against 5351 * partial configs present in each vdev's label and an entire copy of the 5352 * config stored in the MOS. 5353 */ 5354 static int 5355 spa_load_impl(spa_t *spa, spa_import_type_t type, const char **ereport) 5356 { 5357 int error = 0; 5358 boolean_t missing_feat_write = B_FALSE; 5359 boolean_t checkpoint_rewind = 5360 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5361 boolean_t update_config_cache = B_FALSE; 5362 hrtime_t load_start = gethrtime(); 5363 5364 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 5365 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 5366 5367 spa_load_note(spa, "LOADING"); 5368 5369 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 5370 if (error != 0) 5371 return (error); 5372 5373 /* 5374 * If we are rewinding to the checkpoint then we need to repeat 5375 * everything we've done so far in this function but this time 5376 * selecting the checkpointed uberblock and using that to open 5377 * the MOS. 5378 */ 5379 if (checkpoint_rewind) { 5380 /* 5381 * If we are rewinding to the checkpoint update config cache 5382 * anyway. 5383 */ 5384 update_config_cache = B_TRUE; 5385 5386 /* 5387 * Extract the checkpointed uberblock from the current MOS 5388 * and use this as the pool's uberblock from now on. If the 5389 * pool is imported as writeable we also write the checkpoint 5390 * uberblock to the labels, making the rewind permanent. 5391 */ 5392 error = spa_ld_checkpoint_rewind(spa); 5393 if (error != 0) 5394 return (error); 5395 5396 /* 5397 * Redo the loading process again with the 5398 * checkpointed uberblock. 5399 */ 5400 spa_ld_prepare_for_reload(spa); 5401 spa_load_note(spa, "LOADING checkpointed uberblock"); 5402 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 5403 if (error != 0) 5404 return (error); 5405 } 5406 5407 /* 5408 * Drop the namespace lock for the rest of the function. 5409 */ 5410 spa->spa_load_thread = curthread; 5411 mutex_exit(&spa_namespace_lock); 5412 5413 /* 5414 * Retrieve the checkpoint txg if the pool has a checkpoint. 5415 */ 5416 spa_import_progress_set_notes(spa, "Loading checkpoint txg"); 5417 error = spa_ld_read_checkpoint_txg(spa); 5418 if (error != 0) 5419 goto fail; 5420 5421 /* 5422 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 5423 * from the pool and their contents were re-mapped to other vdevs. Note 5424 * that everything that we read before this step must have been 5425 * rewritten on concrete vdevs after the last device removal was 5426 * initiated. Otherwise we could be reading from indirect vdevs before 5427 * we have loaded their mappings. 5428 */ 5429 spa_import_progress_set_notes(spa, "Loading indirect vdev metadata"); 5430 error = spa_ld_open_indirect_vdev_metadata(spa); 5431 if (error != 0) 5432 goto fail; 5433 5434 /* 5435 * Retrieve the full list of active features from the MOS and check if 5436 * they are all supported. 5437 */ 5438 spa_import_progress_set_notes(spa, "Checking feature flags"); 5439 error = spa_ld_check_features(spa, &missing_feat_write); 5440 if (error != 0) 5441 goto fail; 5442 5443 /* 5444 * Load several special directories from the MOS needed by the dsl_pool 5445 * layer. 5446 */ 5447 spa_import_progress_set_notes(spa, "Loading special MOS directories"); 5448 error = spa_ld_load_special_directories(spa); 5449 if (error != 0) 5450 goto fail; 5451 5452 /* 5453 * Retrieve pool properties from the MOS. 5454 */ 5455 spa_import_progress_set_notes(spa, "Loading properties"); 5456 error = spa_ld_get_props(spa); 5457 if (error != 0) 5458 goto fail; 5459 5460 /* 5461 * Retrieve the list of auxiliary devices - cache devices and spares - 5462 * and open them. 5463 */ 5464 spa_import_progress_set_notes(spa, "Loading AUX vdevs"); 5465 error = spa_ld_open_aux_vdevs(spa, type); 5466 if (error != 0) 5467 goto fail; 5468 5469 /* 5470 * Load the metadata for all vdevs. Also check if unopenable devices 5471 * should be autoreplaced. 5472 */ 5473 spa_import_progress_set_notes(spa, "Loading vdev metadata"); 5474 error = spa_ld_load_vdev_metadata(spa); 5475 if (error != 0) 5476 goto fail; 5477 5478 spa_import_progress_set_notes(spa, "Loading dedup tables"); 5479 error = spa_ld_load_dedup_tables(spa); 5480 if (error != 0) 5481 goto fail; 5482 5483 spa_import_progress_set_notes(spa, "Loading BRT"); 5484 error = spa_ld_load_brt(spa); 5485 if (error != 0) 5486 goto fail; 5487 5488 /* 5489 * Verify the logs now to make sure we don't have any unexpected errors 5490 * when we claim log blocks later. 5491 */ 5492 spa_import_progress_set_notes(spa, "Verifying Log Devices"); 5493 error = spa_ld_verify_logs(spa, type, ereport); 5494 if (error != 0) 5495 goto fail; 5496 5497 if (missing_feat_write) { 5498 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 5499 5500 /* 5501 * At this point, we know that we can open the pool in 5502 * read-only mode but not read-write mode. We now have enough 5503 * information and can return to userland. 5504 */ 5505 error = spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 5506 ENOTSUP); 5507 goto fail; 5508 } 5509 5510 /* 5511 * Traverse the last txgs to make sure the pool was left off in a safe 5512 * state. When performing an extreme rewind, we verify the whole pool, 5513 * which can take a very long time. 5514 */ 5515 spa_import_progress_set_notes(spa, "Verifying pool data"); 5516 error = spa_ld_verify_pool_data(spa); 5517 if (error != 0) 5518 goto fail; 5519 5520 /* 5521 * Calculate the deflated space for the pool. This must be done before 5522 * we write anything to the pool because we'd need to update the space 5523 * accounting using the deflated sizes. 5524 */ 5525 spa_import_progress_set_notes(spa, "Calculating deflated space"); 5526 spa_update_dspace(spa); 5527 5528 /* 5529 * We have now retrieved all the information we needed to open the 5530 * pool. If we are importing the pool in read-write mode, a few 5531 * additional steps must be performed to finish the import. 5532 */ 5533 spa_import_progress_set_notes(spa, "Starting import"); 5534 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 5535 spa->spa_load_max_txg == UINT64_MAX)) { 5536 uint64_t config_cache_txg = spa->spa_config_txg; 5537 5538 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 5539 5540 /* 5541 * Before we do any zio_write's, complete the raidz expansion 5542 * scratch space copying, if necessary. 5543 */ 5544 if (RRSS_GET_STATE(&spa->spa_uberblock) == RRSS_SCRATCH_VALID) 5545 vdev_raidz_reflow_copy_scratch(spa); 5546 5547 /* 5548 * In case of a checkpoint rewind, log the original txg 5549 * of the checkpointed uberblock. 5550 */ 5551 if (checkpoint_rewind) { 5552 spa_history_log_internal(spa, "checkpoint rewind", 5553 NULL, "rewound state to txg=%llu", 5554 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 5555 } 5556 5557 spa_import_progress_set_notes(spa, "Claiming ZIL blocks"); 5558 /* 5559 * Traverse the ZIL and claim all blocks. 5560 */ 5561 spa_ld_claim_log_blocks(spa); 5562 5563 /* 5564 * Kick-off the syncing thread. 5565 */ 5566 spa->spa_sync_on = B_TRUE; 5567 txg_sync_start(spa->spa_dsl_pool); 5568 mmp_thread_start(spa); 5569 5570 /* 5571 * Wait for all claims to sync. We sync up to the highest 5572 * claimed log block birth time so that claimed log blocks 5573 * don't appear to be from the future. spa_claim_max_txg 5574 * will have been set for us by ZIL traversal operations 5575 * performed above. 5576 */ 5577 spa_import_progress_set_notes(spa, "Syncing ZIL claims"); 5578 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 5579 5580 /* 5581 * Check if we need to request an update of the config. On the 5582 * next sync, we would update the config stored in vdev labels 5583 * and the cachefile (by default /etc/zfs/zpool.cache). 5584 */ 5585 spa_import_progress_set_notes(spa, "Updating configs"); 5586 spa_ld_check_for_config_update(spa, config_cache_txg, 5587 update_config_cache); 5588 5589 /* 5590 * Check if a rebuild was in progress and if so resume it. 5591 * Then check all DTLs to see if anything needs resilvering. 5592 * The resilver will be deferred if a rebuild was started. 5593 */ 5594 spa_import_progress_set_notes(spa, "Starting resilvers"); 5595 if (vdev_rebuild_active(spa->spa_root_vdev)) { 5596 vdev_rebuild_restart(spa); 5597 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 5598 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 5599 spa_async_request(spa, SPA_ASYNC_RESILVER); 5600 } 5601 5602 /* 5603 * Log the fact that we booted up (so that we can detect if 5604 * we rebooted in the middle of an operation). 5605 */ 5606 spa_history_log_version(spa, "open", NULL); 5607 5608 spa_import_progress_set_notes(spa, 5609 "Restarting device removals"); 5610 spa_restart_removal(spa); 5611 spa_spawn_aux_threads(spa); 5612 5613 /* 5614 * Delete any inconsistent datasets. 5615 * 5616 * Note: 5617 * Since we may be issuing deletes for clones here, 5618 * we make sure to do so after we've spawned all the 5619 * auxiliary threads above (from which the livelist 5620 * deletion zthr is part of). 5621 */ 5622 spa_import_progress_set_notes(spa, 5623 "Cleaning up inconsistent objsets"); 5624 (void) dmu_objset_find(spa_name(spa), 5625 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 5626 5627 /* 5628 * Clean up any stale temporary dataset userrefs. 5629 */ 5630 spa_import_progress_set_notes(spa, 5631 "Cleaning up temporary userrefs"); 5632 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 5633 5634 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5635 spa_import_progress_set_notes(spa, "Restarting initialize"); 5636 vdev_initialize_restart(spa->spa_root_vdev); 5637 spa_import_progress_set_notes(spa, "Restarting TRIM"); 5638 vdev_trim_restart(spa->spa_root_vdev); 5639 vdev_autotrim_restart(spa); 5640 spa_config_exit(spa, SCL_CONFIG, FTAG); 5641 spa_import_progress_set_notes(spa, "Finished importing"); 5642 } 5643 zio_handle_import_delay(spa, gethrtime() - load_start); 5644 5645 spa_import_progress_remove(spa_guid(spa)); 5646 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 5647 5648 spa_load_note(spa, "LOADED"); 5649 fail: 5650 mutex_enter(&spa_namespace_lock); 5651 spa->spa_load_thread = NULL; 5652 cv_broadcast(&spa_namespace_cv); 5653 5654 return (error); 5655 5656 } 5657 5658 static int 5659 spa_load_retry(spa_t *spa, spa_load_state_t state) 5660 { 5661 spa_mode_t mode = spa->spa_mode; 5662 5663 spa_unload(spa); 5664 spa_deactivate(spa); 5665 5666 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 5667 5668 spa_activate(spa, mode); 5669 spa_async_suspend(spa); 5670 5671 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 5672 (u_longlong_t)spa->spa_load_max_txg); 5673 5674 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 5675 } 5676 5677 /* 5678 * If spa_load() fails this function will try loading prior txg's. If 5679 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 5680 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 5681 * function will not rewind the pool and will return the same error as 5682 * spa_load(). 5683 */ 5684 static int 5685 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 5686 int rewind_flags) 5687 { 5688 nvlist_t *loadinfo = NULL; 5689 nvlist_t *config = NULL; 5690 int load_error, rewind_error; 5691 uint64_t safe_rewind_txg; 5692 uint64_t min_txg; 5693 5694 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 5695 spa->spa_load_max_txg = spa->spa_load_txg; 5696 spa_set_log_state(spa, SPA_LOG_CLEAR); 5697 } else { 5698 spa->spa_load_max_txg = max_request; 5699 if (max_request != UINT64_MAX) 5700 spa->spa_extreme_rewind = B_TRUE; 5701 } 5702 5703 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 5704 if (load_error == 0) 5705 return (0); 5706 if (load_error == ZFS_ERR_NO_CHECKPOINT) { 5707 /* 5708 * When attempting checkpoint-rewind on a pool with no 5709 * checkpoint, we should not attempt to load uberblocks 5710 * from previous txgs when spa_load fails. 5711 */ 5712 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5713 spa_import_progress_remove(spa_guid(spa)); 5714 return (load_error); 5715 } 5716 5717 if (spa->spa_root_vdev != NULL) 5718 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5719 5720 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 5721 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 5722 5723 if (rewind_flags & ZPOOL_NEVER_REWIND) { 5724 nvlist_free(config); 5725 spa_import_progress_remove(spa_guid(spa)); 5726 return (load_error); 5727 } 5728 5729 if (state == SPA_LOAD_RECOVER) { 5730 /* Price of rolling back is discarding txgs, including log */ 5731 spa_set_log_state(spa, SPA_LOG_CLEAR); 5732 } else { 5733 /* 5734 * If we aren't rolling back save the load info from our first 5735 * import attempt so that we can restore it after attempting 5736 * to rewind. 5737 */ 5738 loadinfo = spa->spa_load_info; 5739 spa->spa_load_info = fnvlist_alloc(); 5740 } 5741 5742 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 5743 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 5744 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 5745 TXG_INITIAL : safe_rewind_txg; 5746 5747 /* 5748 * Continue as long as we're finding errors, we're still within 5749 * the acceptable rewind range, and we're still finding uberblocks 5750 */ 5751 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 5752 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 5753 if (spa->spa_load_max_txg < safe_rewind_txg) 5754 spa->spa_extreme_rewind = B_TRUE; 5755 rewind_error = spa_load_retry(spa, state); 5756 } 5757 5758 spa->spa_extreme_rewind = B_FALSE; 5759 spa->spa_load_max_txg = UINT64_MAX; 5760 5761 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 5762 spa_config_set(spa, config); 5763 else 5764 nvlist_free(config); 5765 5766 if (state == SPA_LOAD_RECOVER) { 5767 ASSERT3P(loadinfo, ==, NULL); 5768 spa_import_progress_remove(spa_guid(spa)); 5769 return (rewind_error); 5770 } else { 5771 /* Store the rewind info as part of the initial load info */ 5772 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 5773 spa->spa_load_info); 5774 5775 /* Restore the initial load info */ 5776 fnvlist_free(spa->spa_load_info); 5777 spa->spa_load_info = loadinfo; 5778 5779 spa_import_progress_remove(spa_guid(spa)); 5780 return (load_error); 5781 } 5782 } 5783 5784 /* 5785 * Pool Open/Import 5786 * 5787 * The import case is identical to an open except that the configuration is sent 5788 * down from userland, instead of grabbed from the configuration cache. For the 5789 * case of an open, the pool configuration will exist in the 5790 * POOL_STATE_UNINITIALIZED state. 5791 * 5792 * The stats information (gen/count/ustats) is used to gather vdev statistics at 5793 * the same time open the pool, without having to keep around the spa_t in some 5794 * ambiguous state. 5795 */ 5796 static int 5797 spa_open_common(const char *pool, spa_t **spapp, const void *tag, 5798 nvlist_t *nvpolicy, nvlist_t **config) 5799 { 5800 spa_t *spa; 5801 spa_load_state_t state = SPA_LOAD_OPEN; 5802 int error; 5803 int locked = B_FALSE; 5804 int firstopen = B_FALSE; 5805 5806 *spapp = NULL; 5807 5808 /* 5809 * As disgusting as this is, we need to support recursive calls to this 5810 * function because dsl_dir_open() is called during spa_load(), and ends 5811 * up calling spa_open() again. The real fix is to figure out how to 5812 * avoid dsl_dir_open() calling this in the first place. 5813 */ 5814 if (MUTEX_NOT_HELD(&spa_namespace_lock)) { 5815 mutex_enter(&spa_namespace_lock); 5816 locked = B_TRUE; 5817 } 5818 5819 if ((spa = spa_lookup(pool)) == NULL) { 5820 if (locked) 5821 mutex_exit(&spa_namespace_lock); 5822 return (SET_ERROR(ENOENT)); 5823 } 5824 5825 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 5826 zpool_load_policy_t policy; 5827 5828 firstopen = B_TRUE; 5829 5830 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 5831 &policy); 5832 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 5833 state = SPA_LOAD_RECOVER; 5834 5835 spa_activate(spa, spa_mode_global); 5836 5837 if (state != SPA_LOAD_RECOVER) 5838 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 5839 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 5840 5841 zfs_dbgmsg("spa_open_common: opening %s", pool); 5842 error = spa_load_best(spa, state, policy.zlp_txg, 5843 policy.zlp_rewind); 5844 5845 if (error == EBADF) { 5846 /* 5847 * If vdev_validate() returns failure (indicated by 5848 * EBADF), it indicates that one of the vdevs indicates 5849 * that the pool has been exported or destroyed. If 5850 * this is the case, the config cache is out of sync and 5851 * we should remove the pool from the namespace. 5852 */ 5853 spa_unload(spa); 5854 spa_deactivate(spa); 5855 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 5856 spa_remove(spa); 5857 if (locked) 5858 mutex_exit(&spa_namespace_lock); 5859 return (SET_ERROR(ENOENT)); 5860 } 5861 5862 if (error) { 5863 /* 5864 * We can't open the pool, but we still have useful 5865 * information: the state of each vdev after the 5866 * attempted vdev_open(). Return this to the user. 5867 */ 5868 if (config != NULL && spa->spa_config) { 5869 *config = fnvlist_dup(spa->spa_config); 5870 fnvlist_add_nvlist(*config, 5871 ZPOOL_CONFIG_LOAD_INFO, 5872 spa->spa_load_info); 5873 } 5874 spa_unload(spa); 5875 spa_deactivate(spa); 5876 spa->spa_last_open_failed = error; 5877 if (locked) 5878 mutex_exit(&spa_namespace_lock); 5879 *spapp = NULL; 5880 return (error); 5881 } 5882 } 5883 5884 spa_open_ref(spa, tag); 5885 5886 if (config != NULL) 5887 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5888 5889 /* 5890 * If we've recovered the pool, pass back any information we 5891 * gathered while doing the load. 5892 */ 5893 if (state == SPA_LOAD_RECOVER && config != NULL) { 5894 fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 5895 spa->spa_load_info); 5896 } 5897 5898 if (locked) { 5899 spa->spa_last_open_failed = 0; 5900 spa->spa_last_ubsync_txg = 0; 5901 spa->spa_load_txg = 0; 5902 mutex_exit(&spa_namespace_lock); 5903 } 5904 5905 if (firstopen) 5906 zvol_create_minors_recursive(spa_name(spa)); 5907 5908 *spapp = spa; 5909 5910 return (0); 5911 } 5912 5913 int 5914 spa_open_rewind(const char *name, spa_t **spapp, const void *tag, 5915 nvlist_t *policy, nvlist_t **config) 5916 { 5917 return (spa_open_common(name, spapp, tag, policy, config)); 5918 } 5919 5920 int 5921 spa_open(const char *name, spa_t **spapp, const void *tag) 5922 { 5923 return (spa_open_common(name, spapp, tag, NULL, NULL)); 5924 } 5925 5926 /* 5927 * Lookup the given spa_t, incrementing the inject count in the process, 5928 * preventing it from being exported or destroyed. 5929 */ 5930 spa_t * 5931 spa_inject_addref(char *name) 5932 { 5933 spa_t *spa; 5934 5935 mutex_enter(&spa_namespace_lock); 5936 if ((spa = spa_lookup(name)) == NULL) { 5937 mutex_exit(&spa_namespace_lock); 5938 return (NULL); 5939 } 5940 spa->spa_inject_ref++; 5941 mutex_exit(&spa_namespace_lock); 5942 5943 return (spa); 5944 } 5945 5946 void 5947 spa_inject_delref(spa_t *spa) 5948 { 5949 mutex_enter(&spa_namespace_lock); 5950 spa->spa_inject_ref--; 5951 mutex_exit(&spa_namespace_lock); 5952 } 5953 5954 /* 5955 * Add spares device information to the nvlist. 5956 */ 5957 static void 5958 spa_add_spares(spa_t *spa, nvlist_t *config) 5959 { 5960 nvlist_t **spares; 5961 uint_t i, nspares; 5962 nvlist_t *nvroot; 5963 uint64_t guid; 5964 vdev_stat_t *vs; 5965 uint_t vsc; 5966 uint64_t pool; 5967 5968 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5969 5970 if (spa->spa_spares.sav_count == 0) 5971 return; 5972 5973 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 5974 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 5975 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 5976 if (nspares != 0) { 5977 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5978 (const nvlist_t * const *)spares, nspares); 5979 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5980 &spares, &nspares)); 5981 5982 /* 5983 * Go through and find any spares which have since been 5984 * repurposed as an active spare. If this is the case, update 5985 * their status appropriately. 5986 */ 5987 for (i = 0; i < nspares; i++) { 5988 guid = fnvlist_lookup_uint64(spares[i], 5989 ZPOOL_CONFIG_GUID); 5990 VERIFY0(nvlist_lookup_uint64_array(spares[i], 5991 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 5992 if (spa_spare_exists(guid, &pool, NULL) && 5993 pool != 0ULL) { 5994 vs->vs_state = VDEV_STATE_CANT_OPEN; 5995 vs->vs_aux = VDEV_AUX_SPARED; 5996 } else { 5997 vs->vs_state = 5998 spa->spa_spares.sav_vdevs[i]->vdev_state; 5999 } 6000 } 6001 } 6002 } 6003 6004 /* 6005 * Add l2cache device information to the nvlist, including vdev stats. 6006 */ 6007 static void 6008 spa_add_l2cache(spa_t *spa, nvlist_t *config) 6009 { 6010 nvlist_t **l2cache; 6011 uint_t i, j, nl2cache; 6012 nvlist_t *nvroot; 6013 uint64_t guid; 6014 vdev_t *vd; 6015 vdev_stat_t *vs; 6016 uint_t vsc; 6017 6018 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 6019 6020 if (spa->spa_l2cache.sav_count == 0) 6021 return; 6022 6023 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 6024 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 6025 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 6026 if (nl2cache != 0) { 6027 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6028 (const nvlist_t * const *)l2cache, nl2cache); 6029 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6030 &l2cache, &nl2cache)); 6031 6032 /* 6033 * Update level 2 cache device stats. 6034 */ 6035 6036 for (i = 0; i < nl2cache; i++) { 6037 guid = fnvlist_lookup_uint64(l2cache[i], 6038 ZPOOL_CONFIG_GUID); 6039 6040 vd = NULL; 6041 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 6042 if (guid == 6043 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 6044 vd = spa->spa_l2cache.sav_vdevs[j]; 6045 break; 6046 } 6047 } 6048 ASSERT(vd != NULL); 6049 6050 VERIFY0(nvlist_lookup_uint64_array(l2cache[i], 6051 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 6052 vdev_get_stats(vd, vs); 6053 vdev_config_generate_stats(vd, l2cache[i]); 6054 6055 } 6056 } 6057 } 6058 6059 static void 6060 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 6061 { 6062 zap_cursor_t zc; 6063 zap_attribute_t *za = zap_attribute_alloc(); 6064 6065 if (spa->spa_feat_for_read_obj != 0) { 6066 for (zap_cursor_init(&zc, spa->spa_meta_objset, 6067 spa->spa_feat_for_read_obj); 6068 zap_cursor_retrieve(&zc, za) == 0; 6069 zap_cursor_advance(&zc)) { 6070 ASSERT(za->za_integer_length == sizeof (uint64_t) && 6071 za->za_num_integers == 1); 6072 VERIFY0(nvlist_add_uint64(features, za->za_name, 6073 za->za_first_integer)); 6074 } 6075 zap_cursor_fini(&zc); 6076 } 6077 6078 if (spa->spa_feat_for_write_obj != 0) { 6079 for (zap_cursor_init(&zc, spa->spa_meta_objset, 6080 spa->spa_feat_for_write_obj); 6081 zap_cursor_retrieve(&zc, za) == 0; 6082 zap_cursor_advance(&zc)) { 6083 ASSERT(za->za_integer_length == sizeof (uint64_t) && 6084 za->za_num_integers == 1); 6085 VERIFY0(nvlist_add_uint64(features, za->za_name, 6086 za->za_first_integer)); 6087 } 6088 zap_cursor_fini(&zc); 6089 } 6090 zap_attribute_free(za); 6091 } 6092 6093 static void 6094 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 6095 { 6096 int i; 6097 6098 for (i = 0; i < SPA_FEATURES; i++) { 6099 zfeature_info_t feature = spa_feature_table[i]; 6100 uint64_t refcount; 6101 6102 if (feature_get_refcount(spa, &feature, &refcount) != 0) 6103 continue; 6104 6105 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 6106 } 6107 } 6108 6109 /* 6110 * Store a list of pool features and their reference counts in the 6111 * config. 6112 * 6113 * The first time this is called on a spa, allocate a new nvlist, fetch 6114 * the pool features and reference counts from disk, then save the list 6115 * in the spa. In subsequent calls on the same spa use the saved nvlist 6116 * and refresh its values from the cached reference counts. This 6117 * ensures we don't block here on I/O on a suspended pool so 'zpool 6118 * clear' can resume the pool. 6119 */ 6120 static void 6121 spa_add_feature_stats(spa_t *spa, nvlist_t *config) 6122 { 6123 nvlist_t *features; 6124 6125 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 6126 6127 mutex_enter(&spa->spa_feat_stats_lock); 6128 features = spa->spa_feat_stats; 6129 6130 if (features != NULL) { 6131 spa_feature_stats_from_cache(spa, features); 6132 } else { 6133 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 6134 spa->spa_feat_stats = features; 6135 spa_feature_stats_from_disk(spa, features); 6136 } 6137 6138 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 6139 features)); 6140 6141 mutex_exit(&spa->spa_feat_stats_lock); 6142 } 6143 6144 int 6145 spa_get_stats(const char *name, nvlist_t **config, 6146 char *altroot, size_t buflen) 6147 { 6148 int error; 6149 spa_t *spa; 6150 6151 *config = NULL; 6152 error = spa_open_common(name, &spa, FTAG, NULL, config); 6153 6154 if (spa != NULL) { 6155 /* 6156 * This still leaves a window of inconsistency where the spares 6157 * or l2cache devices could change and the config would be 6158 * self-inconsistent. 6159 */ 6160 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6161 6162 if (*config != NULL) { 6163 uint64_t loadtimes[2]; 6164 6165 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 6166 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 6167 fnvlist_add_uint64_array(*config, 6168 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2); 6169 6170 fnvlist_add_uint64(*config, 6171 ZPOOL_CONFIG_ERRCOUNT, 6172 spa_approx_errlog_size(spa)); 6173 6174 if (spa_suspended(spa)) { 6175 fnvlist_add_uint64(*config, 6176 ZPOOL_CONFIG_SUSPENDED, 6177 spa->spa_failmode); 6178 fnvlist_add_uint64(*config, 6179 ZPOOL_CONFIG_SUSPENDED_REASON, 6180 spa->spa_suspended); 6181 } 6182 6183 spa_add_spares(spa, *config); 6184 spa_add_l2cache(spa, *config); 6185 spa_add_feature_stats(spa, *config); 6186 } 6187 } 6188 6189 /* 6190 * We want to get the alternate root even for faulted pools, so we cheat 6191 * and call spa_lookup() directly. 6192 */ 6193 if (altroot) { 6194 if (spa == NULL) { 6195 mutex_enter(&spa_namespace_lock); 6196 spa = spa_lookup(name); 6197 if (spa) 6198 spa_altroot(spa, altroot, buflen); 6199 else 6200 altroot[0] = '\0'; 6201 spa = NULL; 6202 mutex_exit(&spa_namespace_lock); 6203 } else { 6204 spa_altroot(spa, altroot, buflen); 6205 } 6206 } 6207 6208 if (spa != NULL) { 6209 spa_config_exit(spa, SCL_CONFIG, FTAG); 6210 spa_close(spa, FTAG); 6211 } 6212 6213 return (error); 6214 } 6215 6216 /* 6217 * Validate that the auxiliary device array is well formed. We must have an 6218 * array of nvlists, each which describes a valid leaf vdev. If this is an 6219 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 6220 * specified, as long as they are well-formed. 6221 */ 6222 static int 6223 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 6224 spa_aux_vdev_t *sav, const char *config, uint64_t version, 6225 vdev_labeltype_t label) 6226 { 6227 nvlist_t **dev; 6228 uint_t i, ndev; 6229 vdev_t *vd; 6230 int error; 6231 6232 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 6233 6234 /* 6235 * It's acceptable to have no devs specified. 6236 */ 6237 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 6238 return (0); 6239 6240 if (ndev == 0) 6241 return (SET_ERROR(EINVAL)); 6242 6243 /* 6244 * Make sure the pool is formatted with a version that supports this 6245 * device type. 6246 */ 6247 if (spa_version(spa) < version) 6248 return (SET_ERROR(ENOTSUP)); 6249 6250 /* 6251 * Set the pending device list so we correctly handle device in-use 6252 * checking. 6253 */ 6254 sav->sav_pending = dev; 6255 sav->sav_npending = ndev; 6256 6257 for (i = 0; i < ndev; i++) { 6258 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 6259 mode)) != 0) 6260 goto out; 6261 6262 if (!vd->vdev_ops->vdev_op_leaf) { 6263 vdev_free(vd); 6264 error = SET_ERROR(EINVAL); 6265 goto out; 6266 } 6267 6268 vd->vdev_top = vd; 6269 6270 if ((error = vdev_open(vd)) == 0 && 6271 (error = vdev_label_init(vd, crtxg, label)) == 0) { 6272 fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 6273 vd->vdev_guid); 6274 } 6275 6276 vdev_free(vd); 6277 6278 if (error && 6279 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 6280 goto out; 6281 else 6282 error = 0; 6283 } 6284 6285 out: 6286 sav->sav_pending = NULL; 6287 sav->sav_npending = 0; 6288 return (error); 6289 } 6290 6291 static int 6292 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 6293 { 6294 int error; 6295 6296 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 6297 6298 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 6299 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 6300 VDEV_LABEL_SPARE)) != 0) { 6301 return (error); 6302 } 6303 6304 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 6305 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 6306 VDEV_LABEL_L2CACHE)); 6307 } 6308 6309 static void 6310 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 6311 const char *config) 6312 { 6313 int i; 6314 6315 if (sav->sav_config != NULL) { 6316 nvlist_t **olddevs; 6317 uint_t oldndevs; 6318 nvlist_t **newdevs; 6319 6320 /* 6321 * Generate new dev list by concatenating with the 6322 * current dev list. 6323 */ 6324 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config, 6325 &olddevs, &oldndevs)); 6326 6327 newdevs = kmem_alloc(sizeof (void *) * 6328 (ndevs + oldndevs), KM_SLEEP); 6329 for (i = 0; i < oldndevs; i++) 6330 newdevs[i] = fnvlist_dup(olddevs[i]); 6331 for (i = 0; i < ndevs; i++) 6332 newdevs[i + oldndevs] = fnvlist_dup(devs[i]); 6333 6334 fnvlist_remove(sav->sav_config, config); 6335 6336 fnvlist_add_nvlist_array(sav->sav_config, config, 6337 (const nvlist_t * const *)newdevs, ndevs + oldndevs); 6338 for (i = 0; i < oldndevs + ndevs; i++) 6339 nvlist_free(newdevs[i]); 6340 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 6341 } else { 6342 /* 6343 * Generate a new dev list. 6344 */ 6345 sav->sav_config = fnvlist_alloc(); 6346 fnvlist_add_nvlist_array(sav->sav_config, config, 6347 (const nvlist_t * const *)devs, ndevs); 6348 } 6349 } 6350 6351 /* 6352 * Stop and drop level 2 ARC devices 6353 */ 6354 void 6355 spa_l2cache_drop(spa_t *spa) 6356 { 6357 vdev_t *vd; 6358 int i; 6359 spa_aux_vdev_t *sav = &spa->spa_l2cache; 6360 6361 for (i = 0; i < sav->sav_count; i++) { 6362 uint64_t pool; 6363 6364 vd = sav->sav_vdevs[i]; 6365 ASSERT(vd != NULL); 6366 6367 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 6368 pool != 0ULL && l2arc_vdev_present(vd)) 6369 l2arc_remove_vdev(vd); 6370 } 6371 } 6372 6373 /* 6374 * Verify encryption parameters for spa creation. If we are encrypting, we must 6375 * have the encryption feature flag enabled. 6376 */ 6377 static int 6378 spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 6379 boolean_t has_encryption) 6380 { 6381 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 6382 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 6383 !has_encryption) 6384 return (SET_ERROR(ENOTSUP)); 6385 6386 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 6387 } 6388 6389 /* 6390 * Pool Creation 6391 */ 6392 int 6393 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 6394 nvlist_t *zplprops, dsl_crypto_params_t *dcp) 6395 { 6396 spa_t *spa; 6397 const char *altroot = NULL; 6398 vdev_t *rvd; 6399 dsl_pool_t *dp; 6400 dmu_tx_t *tx; 6401 int error = 0; 6402 uint64_t txg = TXG_INITIAL; 6403 nvlist_t **spares, **l2cache; 6404 uint_t nspares, nl2cache; 6405 uint64_t version, obj, ndraid = 0; 6406 boolean_t has_features; 6407 boolean_t has_encryption; 6408 boolean_t has_allocclass; 6409 spa_feature_t feat; 6410 const char *feat_name; 6411 const char *poolname; 6412 nvlist_t *nvl; 6413 6414 if (props == NULL || 6415 nvlist_lookup_string(props, 6416 zpool_prop_to_name(ZPOOL_PROP_TNAME), &poolname) != 0) 6417 poolname = (char *)pool; 6418 6419 /* 6420 * If this pool already exists, return failure. 6421 */ 6422 mutex_enter(&spa_namespace_lock); 6423 if (spa_lookup(poolname) != NULL) { 6424 mutex_exit(&spa_namespace_lock); 6425 return (SET_ERROR(EEXIST)); 6426 } 6427 6428 /* 6429 * Allocate a new spa_t structure. 6430 */ 6431 nvl = fnvlist_alloc(); 6432 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 6433 (void) nvlist_lookup_string(props, 6434 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6435 spa = spa_add(poolname, nvl, altroot); 6436 fnvlist_free(nvl); 6437 spa_activate(spa, spa_mode_global); 6438 6439 if (props && (error = spa_prop_validate(spa, props))) { 6440 spa_deactivate(spa); 6441 spa_remove(spa); 6442 mutex_exit(&spa_namespace_lock); 6443 return (error); 6444 } 6445 6446 /* 6447 * Temporary pool names should never be written to disk. 6448 */ 6449 if (poolname != pool) 6450 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 6451 6452 has_features = B_FALSE; 6453 has_encryption = B_FALSE; 6454 has_allocclass = B_FALSE; 6455 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 6456 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 6457 if (zpool_prop_feature(nvpair_name(elem))) { 6458 has_features = B_TRUE; 6459 6460 feat_name = strchr(nvpair_name(elem), '@') + 1; 6461 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 6462 if (feat == SPA_FEATURE_ENCRYPTION) 6463 has_encryption = B_TRUE; 6464 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 6465 has_allocclass = B_TRUE; 6466 } 6467 } 6468 6469 /* verify encryption params, if they were provided */ 6470 if (dcp != NULL) { 6471 error = spa_create_check_encryption_params(dcp, has_encryption); 6472 if (error != 0) { 6473 spa_deactivate(spa); 6474 spa_remove(spa); 6475 mutex_exit(&spa_namespace_lock); 6476 return (error); 6477 } 6478 } 6479 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 6480 spa_deactivate(spa); 6481 spa_remove(spa); 6482 mutex_exit(&spa_namespace_lock); 6483 return (ENOTSUP); 6484 } 6485 6486 if (has_features || nvlist_lookup_uint64(props, 6487 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 6488 version = SPA_VERSION; 6489 } 6490 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 6491 6492 spa->spa_first_txg = txg; 6493 spa->spa_uberblock.ub_txg = txg - 1; 6494 spa->spa_uberblock.ub_version = version; 6495 spa->spa_ubsync = spa->spa_uberblock; 6496 spa->spa_load_state = SPA_LOAD_CREATE; 6497 spa->spa_removing_phys.sr_state = DSS_NONE; 6498 spa->spa_removing_phys.sr_removing_vdev = -1; 6499 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 6500 spa->spa_indirect_vdevs_loaded = B_TRUE; 6501 6502 /* 6503 * Create "The Godfather" zio to hold all async IOs 6504 */ 6505 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 6506 KM_SLEEP); 6507 for (int i = 0; i < max_ncpus; i++) { 6508 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 6509 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 6510 ZIO_FLAG_GODFATHER); 6511 } 6512 6513 /* 6514 * Create the root vdev. 6515 */ 6516 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6517 6518 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 6519 6520 ASSERT(error != 0 || rvd != NULL); 6521 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 6522 6523 if (error == 0 && !zfs_allocatable_devs(nvroot)) 6524 error = SET_ERROR(EINVAL); 6525 6526 if (error == 0 && 6527 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 6528 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 && 6529 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 6530 /* 6531 * instantiate the metaslab groups (this will dirty the vdevs) 6532 * we can no longer error exit past this point 6533 */ 6534 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 6535 vdev_t *vd = rvd->vdev_child[c]; 6536 6537 vdev_metaslab_set_size(vd); 6538 vdev_expand(vd, txg); 6539 } 6540 } 6541 6542 spa_config_exit(spa, SCL_ALL, FTAG); 6543 6544 if (error != 0) { 6545 spa_unload(spa); 6546 spa_deactivate(spa); 6547 spa_remove(spa); 6548 mutex_exit(&spa_namespace_lock); 6549 return (error); 6550 } 6551 6552 /* 6553 * Get the list of spares, if specified. 6554 */ 6555 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6556 &spares, &nspares) == 0) { 6557 spa->spa_spares.sav_config = fnvlist_alloc(); 6558 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 6559 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 6560 nspares); 6561 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6562 spa_load_spares(spa); 6563 spa_config_exit(spa, SCL_ALL, FTAG); 6564 spa->spa_spares.sav_sync = B_TRUE; 6565 } 6566 6567 /* 6568 * Get the list of level 2 cache devices, if specified. 6569 */ 6570 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6571 &l2cache, &nl2cache) == 0) { 6572 VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config, 6573 NV_UNIQUE_NAME, KM_SLEEP)); 6574 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6575 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 6576 nl2cache); 6577 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6578 spa_load_l2cache(spa); 6579 spa_config_exit(spa, SCL_ALL, FTAG); 6580 spa->spa_l2cache.sav_sync = B_TRUE; 6581 } 6582 6583 spa->spa_is_initializing = B_TRUE; 6584 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 6585 spa->spa_is_initializing = B_FALSE; 6586 6587 /* 6588 * Create DDTs (dedup tables). 6589 */ 6590 ddt_create(spa); 6591 /* 6592 * Create BRT table and BRT table object. 6593 */ 6594 brt_create(spa); 6595 6596 spa_update_dspace(spa); 6597 6598 tx = dmu_tx_create_assigned(dp, txg); 6599 6600 /* 6601 * Create the pool's history object. 6602 */ 6603 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 6604 spa_history_create_obj(spa, tx); 6605 6606 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 6607 spa_history_log_version(spa, "create", tx); 6608 6609 /* 6610 * Create the pool config object. 6611 */ 6612 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 6613 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 6614 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 6615 6616 if (zap_add(spa->spa_meta_objset, 6617 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 6618 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 6619 cmn_err(CE_PANIC, "failed to add pool config"); 6620 } 6621 6622 if (zap_add(spa->spa_meta_objset, 6623 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 6624 sizeof (uint64_t), 1, &version, tx) != 0) { 6625 cmn_err(CE_PANIC, "failed to add pool version"); 6626 } 6627 6628 /* Newly created pools with the right version are always deflated. */ 6629 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 6630 spa->spa_deflate = TRUE; 6631 if (zap_add(spa->spa_meta_objset, 6632 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 6633 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 6634 cmn_err(CE_PANIC, "failed to add deflate"); 6635 } 6636 } 6637 6638 /* 6639 * Create the deferred-free bpobj. Turn off compression 6640 * because sync-to-convergence takes longer if the blocksize 6641 * keeps changing. 6642 */ 6643 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 6644 dmu_object_set_compress(spa->spa_meta_objset, obj, 6645 ZIO_COMPRESS_OFF, tx); 6646 if (zap_add(spa->spa_meta_objset, 6647 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 6648 sizeof (uint64_t), 1, &obj, tx) != 0) { 6649 cmn_err(CE_PANIC, "failed to add bpobj"); 6650 } 6651 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 6652 spa->spa_meta_objset, obj)); 6653 6654 /* 6655 * Generate some random noise for salted checksums to operate on. 6656 */ 6657 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 6658 sizeof (spa->spa_cksum_salt.zcs_bytes)); 6659 6660 /* 6661 * Set pool properties. 6662 */ 6663 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 6664 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 6665 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 6666 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 6667 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 6668 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 6669 spa->spa_dedup_table_quota = 6670 zpool_prop_default_numeric(ZPOOL_PROP_DEDUP_TABLE_QUOTA); 6671 6672 if (props != NULL) { 6673 spa_configfile_set(spa, props, B_FALSE); 6674 spa_sync_props(props, tx); 6675 } 6676 6677 for (int i = 0; i < ndraid; i++) 6678 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6679 6680 dmu_tx_commit(tx); 6681 6682 spa->spa_sync_on = B_TRUE; 6683 txg_sync_start(dp); 6684 mmp_thread_start(spa); 6685 txg_wait_synced(dp, txg); 6686 6687 spa_spawn_aux_threads(spa); 6688 6689 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 6690 6691 /* 6692 * Don't count references from objsets that are already closed 6693 * and are making their way through the eviction process. 6694 */ 6695 spa_evicting_os_wait(spa); 6696 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 6697 spa->spa_load_state = SPA_LOAD_NONE; 6698 6699 spa_import_os(spa); 6700 6701 mutex_exit(&spa_namespace_lock); 6702 6703 return (0); 6704 } 6705 6706 /* 6707 * Import a non-root pool into the system. 6708 */ 6709 int 6710 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 6711 { 6712 spa_t *spa; 6713 const char *altroot = NULL; 6714 spa_load_state_t state = SPA_LOAD_IMPORT; 6715 zpool_load_policy_t policy; 6716 spa_mode_t mode = spa_mode_global; 6717 uint64_t readonly = B_FALSE; 6718 int error; 6719 nvlist_t *nvroot; 6720 nvlist_t **spares, **l2cache; 6721 uint_t nspares, nl2cache; 6722 6723 /* 6724 * If a pool with this name exists, return failure. 6725 */ 6726 mutex_enter(&spa_namespace_lock); 6727 if (spa_lookup(pool) != NULL) { 6728 mutex_exit(&spa_namespace_lock); 6729 return (SET_ERROR(EEXIST)); 6730 } 6731 6732 /* 6733 * Create and initialize the spa structure. 6734 */ 6735 (void) nvlist_lookup_string(props, 6736 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6737 (void) nvlist_lookup_uint64(props, 6738 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 6739 if (readonly) 6740 mode = SPA_MODE_READ; 6741 spa = spa_add(pool, config, altroot); 6742 spa->spa_import_flags = flags; 6743 6744 /* 6745 * Verbatim import - Take a pool and insert it into the namespace 6746 * as if it had been loaded at boot. 6747 */ 6748 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 6749 if (props != NULL) 6750 spa_configfile_set(spa, props, B_FALSE); 6751 6752 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE); 6753 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6754 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 6755 mutex_exit(&spa_namespace_lock); 6756 return (0); 6757 } 6758 6759 spa_activate(spa, mode); 6760 6761 /* 6762 * Don't start async tasks until we know everything is healthy. 6763 */ 6764 spa_async_suspend(spa); 6765 6766 zpool_get_load_policy(config, &policy); 6767 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6768 state = SPA_LOAD_RECOVER; 6769 6770 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 6771 6772 if (state != SPA_LOAD_RECOVER) { 6773 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6774 zfs_dbgmsg("spa_import: importing %s", pool); 6775 } else { 6776 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 6777 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 6778 } 6779 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 6780 6781 /* 6782 * Propagate anything learned while loading the pool and pass it 6783 * back to caller (i.e. rewind info, missing devices, etc). 6784 */ 6785 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info); 6786 6787 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6788 /* 6789 * Toss any existing sparelist, as it doesn't have any validity 6790 * anymore, and conflicts with spa_has_spare(). 6791 */ 6792 if (spa->spa_spares.sav_config) { 6793 nvlist_free(spa->spa_spares.sav_config); 6794 spa->spa_spares.sav_config = NULL; 6795 spa_load_spares(spa); 6796 } 6797 if (spa->spa_l2cache.sav_config) { 6798 nvlist_free(spa->spa_l2cache.sav_config); 6799 spa->spa_l2cache.sav_config = NULL; 6800 spa_load_l2cache(spa); 6801 } 6802 6803 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 6804 spa_config_exit(spa, SCL_ALL, FTAG); 6805 6806 if (props != NULL) 6807 spa_configfile_set(spa, props, B_FALSE); 6808 6809 if (error != 0 || (props && spa_writeable(spa) && 6810 (error = spa_prop_set(spa, props)))) { 6811 spa_unload(spa); 6812 spa_deactivate(spa); 6813 spa_remove(spa); 6814 mutex_exit(&spa_namespace_lock); 6815 return (error); 6816 } 6817 6818 spa_async_resume(spa); 6819 6820 /* 6821 * Override any spares and level 2 cache devices as specified by 6822 * the user, as these may have correct device names/devids, etc. 6823 */ 6824 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6825 &spares, &nspares) == 0) { 6826 if (spa->spa_spares.sav_config) 6827 fnvlist_remove(spa->spa_spares.sav_config, 6828 ZPOOL_CONFIG_SPARES); 6829 else 6830 spa->spa_spares.sav_config = fnvlist_alloc(); 6831 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 6832 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 6833 nspares); 6834 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6835 spa_load_spares(spa); 6836 spa_config_exit(spa, SCL_ALL, FTAG); 6837 spa->spa_spares.sav_sync = B_TRUE; 6838 spa->spa_spares.sav_label_sync = B_TRUE; 6839 } 6840 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6841 &l2cache, &nl2cache) == 0) { 6842 if (spa->spa_l2cache.sav_config) 6843 fnvlist_remove(spa->spa_l2cache.sav_config, 6844 ZPOOL_CONFIG_L2CACHE); 6845 else 6846 spa->spa_l2cache.sav_config = fnvlist_alloc(); 6847 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6848 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 6849 nl2cache); 6850 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6851 spa_load_l2cache(spa); 6852 spa_config_exit(spa, SCL_ALL, FTAG); 6853 spa->spa_l2cache.sav_sync = B_TRUE; 6854 spa->spa_l2cache.sav_label_sync = B_TRUE; 6855 } 6856 6857 /* 6858 * Check for any removed devices. 6859 */ 6860 if (spa->spa_autoreplace) { 6861 spa_aux_check_removed(&spa->spa_spares); 6862 spa_aux_check_removed(&spa->spa_l2cache); 6863 } 6864 6865 if (spa_writeable(spa)) { 6866 /* 6867 * Update the config cache to include the newly-imported pool. 6868 */ 6869 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6870 } 6871 6872 /* 6873 * It's possible that the pool was expanded while it was exported. 6874 * We kick off an async task to handle this for us. 6875 */ 6876 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 6877 6878 spa_history_log_version(spa, "import", NULL); 6879 6880 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6881 6882 mutex_exit(&spa_namespace_lock); 6883 6884 zvol_create_minors_recursive(pool); 6885 6886 spa_import_os(spa); 6887 6888 return (0); 6889 } 6890 6891 nvlist_t * 6892 spa_tryimport(nvlist_t *tryconfig) 6893 { 6894 nvlist_t *config = NULL; 6895 const char *poolname, *cachefile; 6896 spa_t *spa; 6897 uint64_t state; 6898 int error; 6899 zpool_load_policy_t policy; 6900 6901 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 6902 return (NULL); 6903 6904 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 6905 return (NULL); 6906 6907 /* 6908 * Create and initialize the spa structure. 6909 */ 6910 char *name = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6911 (void) snprintf(name, MAXPATHLEN, "%s-%llx-%s", 6912 TRYIMPORT_NAME, (u_longlong_t)(uintptr_t)curthread, poolname); 6913 6914 mutex_enter(&spa_namespace_lock); 6915 spa = spa_add(name, tryconfig, NULL); 6916 spa_activate(spa, SPA_MODE_READ); 6917 kmem_free(name, MAXPATHLEN); 6918 6919 /* 6920 * Rewind pool if a max txg was provided. 6921 */ 6922 zpool_get_load_policy(spa->spa_config, &policy); 6923 if (policy.zlp_txg != UINT64_MAX) { 6924 spa->spa_load_max_txg = policy.zlp_txg; 6925 spa->spa_extreme_rewind = B_TRUE; 6926 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 6927 poolname, (longlong_t)policy.zlp_txg); 6928 } else { 6929 zfs_dbgmsg("spa_tryimport: importing %s", poolname); 6930 } 6931 6932 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 6933 == 0) { 6934 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 6935 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6936 } else { 6937 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 6938 } 6939 6940 /* 6941 * spa_import() relies on a pool config fetched by spa_try_import() 6942 * for spare/cache devices. Import flags are not passed to 6943 * spa_tryimport(), which makes it return early due to a missing log 6944 * device and missing retrieving the cache device and spare eventually. 6945 * Passing ZFS_IMPORT_MISSING_LOG to spa_tryimport() makes it fetch 6946 * the correct configuration regardless of the missing log device. 6947 */ 6948 spa->spa_import_flags |= ZFS_IMPORT_MISSING_LOG; 6949 6950 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 6951 6952 /* 6953 * If 'tryconfig' was at least parsable, return the current config. 6954 */ 6955 if (spa->spa_root_vdev != NULL) { 6956 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6957 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname); 6958 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state); 6959 fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 6960 spa->spa_uberblock.ub_timestamp); 6961 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6962 spa->spa_load_info); 6963 fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 6964 spa->spa_errata); 6965 6966 /* 6967 * If the bootfs property exists on this pool then we 6968 * copy it out so that external consumers can tell which 6969 * pools are bootable. 6970 */ 6971 if ((!error || error == EEXIST) && spa->spa_bootfs) { 6972 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6973 6974 /* 6975 * We have to play games with the name since the 6976 * pool was opened as TRYIMPORT_NAME. 6977 */ 6978 if (dsl_dsobj_to_dsname(spa_name(spa), 6979 spa->spa_bootfs, tmpname) == 0) { 6980 char *cp; 6981 char *dsname; 6982 6983 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6984 6985 cp = strchr(tmpname, '/'); 6986 if (cp == NULL) { 6987 (void) strlcpy(dsname, tmpname, 6988 MAXPATHLEN); 6989 } else { 6990 (void) snprintf(dsname, MAXPATHLEN, 6991 "%s/%s", poolname, ++cp); 6992 } 6993 fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS, 6994 dsname); 6995 kmem_free(dsname, MAXPATHLEN); 6996 } 6997 kmem_free(tmpname, MAXPATHLEN); 6998 } 6999 7000 /* 7001 * Add the list of hot spares and level 2 cache devices. 7002 */ 7003 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 7004 spa_add_spares(spa, config); 7005 spa_add_l2cache(spa, config); 7006 spa_config_exit(spa, SCL_CONFIG, FTAG); 7007 } 7008 7009 spa_unload(spa); 7010 spa_deactivate(spa); 7011 spa_remove(spa); 7012 mutex_exit(&spa_namespace_lock); 7013 7014 return (config); 7015 } 7016 7017 /* 7018 * Pool export/destroy 7019 * 7020 * The act of destroying or exporting a pool is very simple. We make sure there 7021 * is no more pending I/O and any references to the pool are gone. Then, we 7022 * update the pool state and sync all the labels to disk, removing the 7023 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 7024 * we don't sync the labels or remove the configuration cache. 7025 */ 7026 static int 7027 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 7028 boolean_t force, boolean_t hardforce) 7029 { 7030 int error = 0; 7031 spa_t *spa; 7032 hrtime_t export_start = gethrtime(); 7033 7034 if (oldconfig) 7035 *oldconfig = NULL; 7036 7037 if (!(spa_mode_global & SPA_MODE_WRITE)) 7038 return (SET_ERROR(EROFS)); 7039 7040 mutex_enter(&spa_namespace_lock); 7041 if ((spa = spa_lookup(pool)) == NULL) { 7042 mutex_exit(&spa_namespace_lock); 7043 return (SET_ERROR(ENOENT)); 7044 } 7045 7046 if (spa->spa_is_exporting) { 7047 /* the pool is being exported by another thread */ 7048 mutex_exit(&spa_namespace_lock); 7049 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 7050 } 7051 spa->spa_is_exporting = B_TRUE; 7052 7053 /* 7054 * Put a hold on the pool, drop the namespace lock, stop async tasks 7055 * and see if we can export. 7056 */ 7057 spa_open_ref(spa, FTAG); 7058 mutex_exit(&spa_namespace_lock); 7059 spa_async_suspend(spa); 7060 if (spa->spa_zvol_taskq) { 7061 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 7062 taskq_wait(spa->spa_zvol_taskq); 7063 } 7064 mutex_enter(&spa_namespace_lock); 7065 spa->spa_export_thread = curthread; 7066 spa_close(spa, FTAG); 7067 7068 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 7069 mutex_exit(&spa_namespace_lock); 7070 goto export_spa; 7071 } 7072 7073 /* 7074 * The pool will be in core if it's openable, in which case we can 7075 * modify its state. Objsets may be open only because they're dirty, 7076 * so we have to force it to sync before checking spa_refcnt. 7077 */ 7078 if (spa->spa_sync_on) { 7079 txg_wait_synced(spa->spa_dsl_pool, 0); 7080 spa_evicting_os_wait(spa); 7081 } 7082 7083 /* 7084 * A pool cannot be exported or destroyed if there are active 7085 * references. If we are resetting a pool, allow references by 7086 * fault injection handlers. 7087 */ 7088 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 7089 error = SET_ERROR(EBUSY); 7090 goto fail; 7091 } 7092 7093 mutex_exit(&spa_namespace_lock); 7094 /* 7095 * At this point we no longer hold the spa_namespace_lock and 7096 * there were no references on the spa. Future spa_lookups will 7097 * notice the spa->spa_export_thread and wait until we signal 7098 * that we are finshed. 7099 */ 7100 7101 if (spa->spa_sync_on) { 7102 vdev_t *rvd = spa->spa_root_vdev; 7103 /* 7104 * A pool cannot be exported if it has an active shared spare. 7105 * This is to prevent other pools stealing the active spare 7106 * from an exported pool. At user's own will, such pool can 7107 * be forcedly exported. 7108 */ 7109 if (!force && new_state == POOL_STATE_EXPORTED && 7110 spa_has_active_shared_spare(spa)) { 7111 error = SET_ERROR(EXDEV); 7112 mutex_enter(&spa_namespace_lock); 7113 goto fail; 7114 } 7115 7116 /* 7117 * We're about to export or destroy this pool. Make sure 7118 * we stop all initialization and trim activity here before 7119 * we set the spa_final_txg. This will ensure that all 7120 * dirty data resulting from the initialization is 7121 * committed to disk before we unload the pool. 7122 */ 7123 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 7124 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 7125 vdev_autotrim_stop_all(spa); 7126 vdev_rebuild_stop_all(spa); 7127 l2arc_spa_rebuild_stop(spa); 7128 7129 /* 7130 * We want this to be reflected on every label, 7131 * so mark them all dirty. spa_unload() will do the 7132 * final sync that pushes these changes out. 7133 */ 7134 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 7135 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7136 spa->spa_state = new_state; 7137 vdev_config_dirty(rvd); 7138 spa_config_exit(spa, SCL_ALL, FTAG); 7139 } 7140 7141 /* 7142 * If the log space map feature is enabled and the pool is 7143 * getting exported (but not destroyed), we want to spend some 7144 * time flushing as many metaslabs as we can in an attempt to 7145 * destroy log space maps and save import time. This has to be 7146 * done before we set the spa_final_txg, otherwise 7147 * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs. 7148 * spa_should_flush_logs_on_unload() should be called after 7149 * spa_state has been set to the new_state. 7150 */ 7151 if (spa_should_flush_logs_on_unload(spa)) 7152 spa_unload_log_sm_flush_all(spa); 7153 7154 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 7155 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7156 spa->spa_final_txg = spa_last_synced_txg(spa) + 7157 TXG_DEFER_SIZE + 1; 7158 spa_config_exit(spa, SCL_ALL, FTAG); 7159 } 7160 } 7161 7162 export_spa: 7163 spa_export_os(spa); 7164 7165 if (new_state == POOL_STATE_DESTROYED) 7166 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 7167 else if (new_state == POOL_STATE_EXPORTED) 7168 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 7169 7170 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 7171 spa_unload(spa); 7172 spa_deactivate(spa); 7173 } 7174 7175 if (oldconfig && spa->spa_config) 7176 *oldconfig = fnvlist_dup(spa->spa_config); 7177 7178 if (new_state == POOL_STATE_EXPORTED) 7179 zio_handle_export_delay(spa, gethrtime() - export_start); 7180 7181 /* 7182 * Take the namespace lock for the actual spa_t removal 7183 */ 7184 mutex_enter(&spa_namespace_lock); 7185 if (new_state != POOL_STATE_UNINITIALIZED) { 7186 if (!hardforce) 7187 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 7188 spa_remove(spa); 7189 } else { 7190 /* 7191 * If spa_remove() is not called for this spa_t and 7192 * there is any possibility that it can be reused, 7193 * we make sure to reset the exporting flag. 7194 */ 7195 spa->spa_is_exporting = B_FALSE; 7196 spa->spa_export_thread = NULL; 7197 } 7198 7199 /* 7200 * Wake up any waiters in spa_lookup() 7201 */ 7202 cv_broadcast(&spa_namespace_cv); 7203 mutex_exit(&spa_namespace_lock); 7204 return (0); 7205 7206 fail: 7207 spa->spa_is_exporting = B_FALSE; 7208 spa->spa_export_thread = NULL; 7209 7210 spa_async_resume(spa); 7211 /* 7212 * Wake up any waiters in spa_lookup() 7213 */ 7214 cv_broadcast(&spa_namespace_cv); 7215 mutex_exit(&spa_namespace_lock); 7216 return (error); 7217 } 7218 7219 /* 7220 * Destroy a storage pool. 7221 */ 7222 int 7223 spa_destroy(const char *pool) 7224 { 7225 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 7226 B_FALSE, B_FALSE)); 7227 } 7228 7229 /* 7230 * Export a storage pool. 7231 */ 7232 int 7233 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 7234 boolean_t hardforce) 7235 { 7236 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 7237 force, hardforce)); 7238 } 7239 7240 /* 7241 * Similar to spa_export(), this unloads the spa_t without actually removing it 7242 * from the namespace in any way. 7243 */ 7244 int 7245 spa_reset(const char *pool) 7246 { 7247 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 7248 B_FALSE, B_FALSE)); 7249 } 7250 7251 /* 7252 * ========================================================================== 7253 * Device manipulation 7254 * ========================================================================== 7255 */ 7256 7257 /* 7258 * This is called as a synctask to increment the draid feature flag 7259 */ 7260 static void 7261 spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 7262 { 7263 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 7264 int draid = (int)(uintptr_t)arg; 7265 7266 for (int c = 0; c < draid; c++) 7267 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 7268 } 7269 7270 /* 7271 * Add a device to a storage pool. 7272 */ 7273 int 7274 spa_vdev_add(spa_t *spa, nvlist_t *nvroot, boolean_t check_ashift) 7275 { 7276 uint64_t txg, ndraid = 0; 7277 int error; 7278 vdev_t *rvd = spa->spa_root_vdev; 7279 vdev_t *vd, *tvd; 7280 nvlist_t **spares, **l2cache; 7281 uint_t nspares, nl2cache; 7282 7283 ASSERT(spa_writeable(spa)); 7284 7285 txg = spa_vdev_enter(spa); 7286 7287 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 7288 VDEV_ALLOC_ADD)) != 0) 7289 return (spa_vdev_exit(spa, NULL, txg, error)); 7290 7291 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 7292 7293 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 7294 &nspares) != 0) 7295 nspares = 0; 7296 7297 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 7298 &nl2cache) != 0) 7299 nl2cache = 0; 7300 7301 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 7302 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 7303 7304 if (vd->vdev_children != 0 && 7305 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 7306 return (spa_vdev_exit(spa, vd, txg, error)); 7307 } 7308 7309 /* 7310 * The virtual dRAID spares must be added after vdev tree is created 7311 * and the vdev guids are generated. The guid of their associated 7312 * dRAID is stored in the config and used when opening the spare. 7313 */ 7314 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 7315 rvd->vdev_children)) == 0) { 7316 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 7317 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 7318 nspares = 0; 7319 } else { 7320 return (spa_vdev_exit(spa, vd, txg, error)); 7321 } 7322 7323 /* 7324 * We must validate the spares and l2cache devices after checking the 7325 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 7326 */ 7327 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 7328 return (spa_vdev_exit(spa, vd, txg, error)); 7329 7330 /* 7331 * If we are in the middle of a device removal, we can only add 7332 * devices which match the existing devices in the pool. 7333 * If we are in the middle of a removal, or have some indirect 7334 * vdevs, we can not add raidz or dRAID top levels. 7335 */ 7336 if (spa->spa_vdev_removal != NULL || 7337 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 7338 for (int c = 0; c < vd->vdev_children; c++) { 7339 tvd = vd->vdev_child[c]; 7340 if (spa->spa_vdev_removal != NULL && 7341 tvd->vdev_ashift != spa->spa_max_ashift) { 7342 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 7343 } 7344 /* Fail if top level vdev is raidz or a dRAID */ 7345 if (vdev_get_nparity(tvd) != 0) 7346 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 7347 7348 /* 7349 * Need the top level mirror to be 7350 * a mirror of leaf vdevs only 7351 */ 7352 if (tvd->vdev_ops == &vdev_mirror_ops) { 7353 for (uint64_t cid = 0; 7354 cid < tvd->vdev_children; cid++) { 7355 vdev_t *cvd = tvd->vdev_child[cid]; 7356 if (!cvd->vdev_ops->vdev_op_leaf) { 7357 return (spa_vdev_exit(spa, vd, 7358 txg, EINVAL)); 7359 } 7360 } 7361 } 7362 } 7363 } 7364 7365 if (check_ashift && spa->spa_max_ashift == spa->spa_min_ashift) { 7366 for (int c = 0; c < vd->vdev_children; c++) { 7367 tvd = vd->vdev_child[c]; 7368 if (tvd->vdev_ashift != spa->spa_max_ashift) { 7369 return (spa_vdev_exit(spa, vd, txg, 7370 ZFS_ERR_ASHIFT_MISMATCH)); 7371 } 7372 } 7373 } 7374 7375 for (int c = 0; c < vd->vdev_children; c++) { 7376 tvd = vd->vdev_child[c]; 7377 vdev_remove_child(vd, tvd); 7378 tvd->vdev_id = rvd->vdev_children; 7379 vdev_add_child(rvd, tvd); 7380 vdev_config_dirty(tvd); 7381 } 7382 7383 if (nspares != 0) { 7384 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 7385 ZPOOL_CONFIG_SPARES); 7386 spa_load_spares(spa); 7387 spa->spa_spares.sav_sync = B_TRUE; 7388 } 7389 7390 if (nl2cache != 0) { 7391 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 7392 ZPOOL_CONFIG_L2CACHE); 7393 spa_load_l2cache(spa); 7394 spa->spa_l2cache.sav_sync = B_TRUE; 7395 } 7396 7397 /* 7398 * We can't increment a feature while holding spa_vdev so we 7399 * have to do it in a synctask. 7400 */ 7401 if (ndraid != 0) { 7402 dmu_tx_t *tx; 7403 7404 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 7405 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 7406 (void *)(uintptr_t)ndraid, tx); 7407 dmu_tx_commit(tx); 7408 } 7409 7410 /* 7411 * We have to be careful when adding new vdevs to an existing pool. 7412 * If other threads start allocating from these vdevs before we 7413 * sync the config cache, and we lose power, then upon reboot we may 7414 * fail to open the pool because there are DVAs that the config cache 7415 * can't translate. Therefore, we first add the vdevs without 7416 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 7417 * and then let spa_config_update() initialize the new metaslabs. 7418 * 7419 * spa_load() checks for added-but-not-initialized vdevs, so that 7420 * if we lose power at any point in this sequence, the remaining 7421 * steps will be completed the next time we load the pool. 7422 */ 7423 (void) spa_vdev_exit(spa, vd, txg, 0); 7424 7425 mutex_enter(&spa_namespace_lock); 7426 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 7427 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 7428 mutex_exit(&spa_namespace_lock); 7429 7430 return (0); 7431 } 7432 7433 /* 7434 * Attach a device to a vdev specified by its guid. The vdev type can be 7435 * a mirror, a raidz, or a leaf device that is also a top-level (e.g. a 7436 * single device). When the vdev is a single device, a mirror vdev will be 7437 * automatically inserted. 7438 * 7439 * If 'replacing' is specified, the new device is intended to replace the 7440 * existing device; in this case the two devices are made into their own 7441 * mirror using the 'replacing' vdev, which is functionally identical to 7442 * the mirror vdev (it actually reuses all the same ops) but has a few 7443 * extra rules: you can't attach to it after it's been created, and upon 7444 * completion of resilvering, the first disk (the one being replaced) 7445 * is automatically detached. 7446 * 7447 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 7448 * should be performed instead of traditional healing reconstruction. From 7449 * an administrators perspective these are both resilver operations. 7450 */ 7451 int 7452 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 7453 int rebuild) 7454 { 7455 uint64_t txg, dtl_max_txg; 7456 vdev_t *rvd = spa->spa_root_vdev; 7457 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 7458 vdev_ops_t *pvops; 7459 char *oldvdpath, *newvdpath; 7460 int newvd_isspare = B_FALSE; 7461 int error; 7462 7463 ASSERT(spa_writeable(spa)); 7464 7465 txg = spa_vdev_enter(spa); 7466 7467 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 7468 7469 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7470 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7471 error = (spa_has_checkpoint(spa)) ? 7472 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7473 return (spa_vdev_exit(spa, NULL, txg, error)); 7474 } 7475 7476 if (rebuild) { 7477 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 7478 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7479 7480 if (dsl_scan_resilvering(spa_get_dsl(spa)) || 7481 dsl_scan_resilver_scheduled(spa_get_dsl(spa))) { 7482 return (spa_vdev_exit(spa, NULL, txg, 7483 ZFS_ERR_RESILVER_IN_PROGRESS)); 7484 } 7485 } else { 7486 if (vdev_rebuild_active(rvd)) 7487 return (spa_vdev_exit(spa, NULL, txg, 7488 ZFS_ERR_REBUILD_IN_PROGRESS)); 7489 } 7490 7491 if (spa->spa_vdev_removal != NULL) { 7492 return (spa_vdev_exit(spa, NULL, txg, 7493 ZFS_ERR_DEVRM_IN_PROGRESS)); 7494 } 7495 7496 if (oldvd == NULL) 7497 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 7498 7499 boolean_t raidz = oldvd->vdev_ops == &vdev_raidz_ops; 7500 7501 if (raidz) { 7502 if (!spa_feature_is_enabled(spa, SPA_FEATURE_RAIDZ_EXPANSION)) 7503 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7504 7505 /* 7506 * Can't expand a raidz while prior expand is in progress. 7507 */ 7508 if (spa->spa_raidz_expand != NULL) { 7509 return (spa_vdev_exit(spa, NULL, txg, 7510 ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS)); 7511 } 7512 } else if (!oldvd->vdev_ops->vdev_op_leaf) { 7513 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7514 } 7515 7516 if (raidz) 7517 pvd = oldvd; 7518 else 7519 pvd = oldvd->vdev_parent; 7520 7521 if (spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 7522 VDEV_ALLOC_ATTACH) != 0) 7523 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7524 7525 if (newrootvd->vdev_children != 1) 7526 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 7527 7528 newvd = newrootvd->vdev_child[0]; 7529 7530 if (!newvd->vdev_ops->vdev_op_leaf) 7531 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 7532 7533 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 7534 return (spa_vdev_exit(spa, newrootvd, txg, error)); 7535 7536 /* 7537 * log, dedup and special vdevs should not be replaced by spares. 7538 */ 7539 if ((oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE || 7540 oldvd->vdev_top->vdev_islog) && newvd->vdev_isspare) { 7541 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7542 } 7543 7544 /* 7545 * A dRAID spare can only replace a child of its parent dRAID vdev. 7546 */ 7547 if (newvd->vdev_ops == &vdev_draid_spare_ops && 7548 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 7549 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7550 } 7551 7552 if (rebuild) { 7553 /* 7554 * For rebuilds, the top vdev must support reconstruction 7555 * using only space maps. This means the only allowable 7556 * vdevs types are the root vdev, a mirror, or dRAID. 7557 */ 7558 tvd = pvd; 7559 if (pvd->vdev_top != NULL) 7560 tvd = pvd->vdev_top; 7561 7562 if (tvd->vdev_ops != &vdev_mirror_ops && 7563 tvd->vdev_ops != &vdev_root_ops && 7564 tvd->vdev_ops != &vdev_draid_ops) { 7565 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7566 } 7567 } 7568 7569 if (!replacing) { 7570 /* 7571 * For attach, the only allowable parent is a mirror or 7572 * the root vdev. A raidz vdev can be attached to, but 7573 * you cannot attach to a raidz child. 7574 */ 7575 if (pvd->vdev_ops != &vdev_mirror_ops && 7576 pvd->vdev_ops != &vdev_root_ops && 7577 !raidz) 7578 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7579 7580 pvops = &vdev_mirror_ops; 7581 } else { 7582 /* 7583 * Active hot spares can only be replaced by inactive hot 7584 * spares. 7585 */ 7586 if (pvd->vdev_ops == &vdev_spare_ops && 7587 oldvd->vdev_isspare && 7588 !spa_has_spare(spa, newvd->vdev_guid)) 7589 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7590 7591 /* 7592 * If the source is a hot spare, and the parent isn't already a 7593 * spare, then we want to create a new hot spare. Otherwise, we 7594 * want to create a replacing vdev. The user is not allowed to 7595 * attach to a spared vdev child unless the 'isspare' state is 7596 * the same (spare replaces spare, non-spare replaces 7597 * non-spare). 7598 */ 7599 if (pvd->vdev_ops == &vdev_replacing_ops && 7600 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 7601 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7602 } else if (pvd->vdev_ops == &vdev_spare_ops && 7603 newvd->vdev_isspare != oldvd->vdev_isspare) { 7604 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7605 } 7606 7607 if (newvd->vdev_isspare) 7608 pvops = &vdev_spare_ops; 7609 else 7610 pvops = &vdev_replacing_ops; 7611 } 7612 7613 /* 7614 * Make sure the new device is big enough. 7615 */ 7616 vdev_t *min_vdev = raidz ? oldvd->vdev_child[0] : oldvd; 7617 if (newvd->vdev_asize < vdev_get_min_asize(min_vdev)) 7618 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 7619 7620 /* 7621 * The new device cannot have a higher alignment requirement 7622 * than the top-level vdev. 7623 */ 7624 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) { 7625 return (spa_vdev_exit(spa, newrootvd, txg, 7626 ZFS_ERR_ASHIFT_MISMATCH)); 7627 } 7628 7629 /* 7630 * RAIDZ-expansion-specific checks. 7631 */ 7632 if (raidz) { 7633 if (vdev_raidz_attach_check(newvd) != 0) 7634 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7635 7636 /* 7637 * Fail early if a child is not healthy or being replaced 7638 */ 7639 for (int i = 0; i < oldvd->vdev_children; i++) { 7640 if (vdev_is_dead(oldvd->vdev_child[i]) || 7641 !oldvd->vdev_child[i]->vdev_ops->vdev_op_leaf) { 7642 return (spa_vdev_exit(spa, newrootvd, txg, 7643 ENXIO)); 7644 } 7645 /* Also fail if reserved boot area is in-use */ 7646 if (vdev_check_boot_reserve(spa, oldvd->vdev_child[i]) 7647 != 0) { 7648 return (spa_vdev_exit(spa, newrootvd, txg, 7649 EADDRINUSE)); 7650 } 7651 } 7652 } 7653 7654 if (raidz) { 7655 /* 7656 * Note: oldvdpath is freed by spa_strfree(), but 7657 * kmem_asprintf() is freed by kmem_strfree(), so we have to 7658 * move it to a spa_strdup-ed string. 7659 */ 7660 char *tmp = kmem_asprintf("raidz%u-%u", 7661 (uint_t)vdev_get_nparity(oldvd), (uint_t)oldvd->vdev_id); 7662 oldvdpath = spa_strdup(tmp); 7663 kmem_strfree(tmp); 7664 } else { 7665 oldvdpath = spa_strdup(oldvd->vdev_path); 7666 } 7667 newvdpath = spa_strdup(newvd->vdev_path); 7668 7669 /* 7670 * If this is an in-place replacement, update oldvd's path and devid 7671 * to make it distinguishable from newvd, and unopenable from now on. 7672 */ 7673 if (strcmp(oldvdpath, newvdpath) == 0) { 7674 spa_strfree(oldvd->vdev_path); 7675 oldvd->vdev_path = kmem_alloc(strlen(newvdpath) + 5, 7676 KM_SLEEP); 7677 (void) sprintf(oldvd->vdev_path, "%s/old", 7678 newvdpath); 7679 if (oldvd->vdev_devid != NULL) { 7680 spa_strfree(oldvd->vdev_devid); 7681 oldvd->vdev_devid = NULL; 7682 } 7683 spa_strfree(oldvdpath); 7684 oldvdpath = spa_strdup(oldvd->vdev_path); 7685 } 7686 7687 /* 7688 * If the parent is not a mirror, or if we're replacing, insert the new 7689 * mirror/replacing/spare vdev above oldvd. 7690 */ 7691 if (!raidz && pvd->vdev_ops != pvops) { 7692 pvd = vdev_add_parent(oldvd, pvops); 7693 ASSERT(pvd->vdev_ops == pvops); 7694 ASSERT(oldvd->vdev_parent == pvd); 7695 } 7696 7697 ASSERT(pvd->vdev_top->vdev_parent == rvd); 7698 7699 /* 7700 * Extract the new device from its root and add it to pvd. 7701 */ 7702 vdev_remove_child(newrootvd, newvd); 7703 newvd->vdev_id = pvd->vdev_children; 7704 newvd->vdev_crtxg = oldvd->vdev_crtxg; 7705 vdev_add_child(pvd, newvd); 7706 7707 /* 7708 * Reevaluate the parent vdev state. 7709 */ 7710 vdev_propagate_state(pvd); 7711 7712 tvd = newvd->vdev_top; 7713 ASSERT(pvd->vdev_top == tvd); 7714 ASSERT(tvd->vdev_parent == rvd); 7715 7716 vdev_config_dirty(tvd); 7717 7718 /* 7719 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 7720 * for any dmu_sync-ed blocks. It will propagate upward when 7721 * spa_vdev_exit() calls vdev_dtl_reassess(). 7722 */ 7723 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 7724 7725 if (raidz) { 7726 /* 7727 * Wait for the youngest allocations and frees to sync, 7728 * and then wait for the deferral of those frees to finish. 7729 */ 7730 spa_vdev_config_exit(spa, NULL, 7731 txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG); 7732 7733 vdev_initialize_stop_all(tvd, VDEV_INITIALIZE_ACTIVE); 7734 vdev_trim_stop_all(tvd, VDEV_TRIM_ACTIVE); 7735 vdev_autotrim_stop_wait(tvd); 7736 7737 dtl_max_txg = spa_vdev_config_enter(spa); 7738 7739 tvd->vdev_rz_expanding = B_TRUE; 7740 7741 vdev_dirty_leaves(tvd, VDD_DTL, dtl_max_txg); 7742 vdev_config_dirty(tvd); 7743 7744 dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool, 7745 dtl_max_txg); 7746 dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_raidz_attach_sync, 7747 newvd, tx); 7748 dmu_tx_commit(tx); 7749 } else { 7750 vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL, 7751 dtl_max_txg - TXG_INITIAL); 7752 7753 if (newvd->vdev_isspare) { 7754 spa_spare_activate(newvd); 7755 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 7756 } 7757 7758 newvd_isspare = newvd->vdev_isspare; 7759 7760 /* 7761 * Mark newvd's DTL dirty in this txg. 7762 */ 7763 vdev_dirty(tvd, VDD_DTL, newvd, txg); 7764 7765 /* 7766 * Schedule the resilver or rebuild to restart in the future. 7767 * We do this to ensure that dmu_sync-ed blocks have been 7768 * stitched into the respective datasets. 7769 */ 7770 if (rebuild) { 7771 newvd->vdev_rebuild_txg = txg; 7772 7773 vdev_rebuild(tvd); 7774 } else { 7775 newvd->vdev_resilver_txg = txg; 7776 7777 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 7778 spa_feature_is_enabled(spa, 7779 SPA_FEATURE_RESILVER_DEFER)) { 7780 vdev_defer_resilver(newvd); 7781 } else { 7782 dsl_scan_restart_resilver(spa->spa_dsl_pool, 7783 dtl_max_txg); 7784 } 7785 } 7786 } 7787 7788 if (spa->spa_bootfs) 7789 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 7790 7791 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 7792 7793 /* 7794 * Commit the config 7795 */ 7796 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 7797 7798 spa_history_log_internal(spa, "vdev attach", NULL, 7799 "%s vdev=%s %s vdev=%s", 7800 replacing && newvd_isspare ? "spare in" : 7801 replacing ? "replace" : "attach", newvdpath, 7802 replacing ? "for" : "to", oldvdpath); 7803 7804 spa_strfree(oldvdpath); 7805 spa_strfree(newvdpath); 7806 7807 return (0); 7808 } 7809 7810 /* 7811 * Detach a device from a mirror or replacing vdev. 7812 * 7813 * If 'replace_done' is specified, only detach if the parent 7814 * is a replacing or a spare vdev. 7815 */ 7816 int 7817 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 7818 { 7819 uint64_t txg; 7820 int error; 7821 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 7822 vdev_t *vd, *pvd, *cvd, *tvd; 7823 boolean_t unspare = B_FALSE; 7824 uint64_t unspare_guid = 0; 7825 char *vdpath; 7826 7827 ASSERT(spa_writeable(spa)); 7828 7829 txg = spa_vdev_detach_enter(spa, guid); 7830 7831 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7832 7833 /* 7834 * Besides being called directly from the userland through the 7835 * ioctl interface, spa_vdev_detach() can be potentially called 7836 * at the end of spa_vdev_resilver_done(). 7837 * 7838 * In the regular case, when we have a checkpoint this shouldn't 7839 * happen as we never empty the DTLs of a vdev during the scrub 7840 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 7841 * should never get here when we have a checkpoint. 7842 * 7843 * That said, even in a case when we checkpoint the pool exactly 7844 * as spa_vdev_resilver_done() calls this function everything 7845 * should be fine as the resilver will return right away. 7846 */ 7847 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7848 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7849 error = (spa_has_checkpoint(spa)) ? 7850 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7851 return (spa_vdev_exit(spa, NULL, txg, error)); 7852 } 7853 7854 if (vd == NULL) 7855 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 7856 7857 if (!vd->vdev_ops->vdev_op_leaf) 7858 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7859 7860 pvd = vd->vdev_parent; 7861 7862 /* 7863 * If the parent/child relationship is not as expected, don't do it. 7864 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 7865 * vdev that's replacing B with C. The user's intent in replacing 7866 * is to go from M(A,B) to M(A,C). If the user decides to cancel 7867 * the replace by detaching C, the expected behavior is to end up 7868 * M(A,B). But suppose that right after deciding to detach C, 7869 * the replacement of B completes. We would have M(A,C), and then 7870 * ask to detach C, which would leave us with just A -- not what 7871 * the user wanted. To prevent this, we make sure that the 7872 * parent/child relationship hasn't changed -- in this example, 7873 * that C's parent is still the replacing vdev R. 7874 */ 7875 if (pvd->vdev_guid != pguid && pguid != 0) 7876 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7877 7878 /* 7879 * Only 'replacing' or 'spare' vdevs can be replaced. 7880 */ 7881 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 7882 pvd->vdev_ops != &vdev_spare_ops) 7883 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7884 7885 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 7886 spa_version(spa) >= SPA_VERSION_SPARES); 7887 7888 /* 7889 * Only mirror, replacing, and spare vdevs support detach. 7890 */ 7891 if (pvd->vdev_ops != &vdev_replacing_ops && 7892 pvd->vdev_ops != &vdev_mirror_ops && 7893 pvd->vdev_ops != &vdev_spare_ops) 7894 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7895 7896 /* 7897 * If this device has the only valid copy of some data, 7898 * we cannot safely detach it. 7899 */ 7900 if (vdev_dtl_required(vd)) 7901 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7902 7903 ASSERT(pvd->vdev_children >= 2); 7904 7905 /* 7906 * If we are detaching the second disk from a replacing vdev, then 7907 * check to see if we changed the original vdev's path to have "/old" 7908 * at the end in spa_vdev_attach(). If so, undo that change now. 7909 */ 7910 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 7911 vd->vdev_path != NULL) { 7912 size_t len = strlen(vd->vdev_path); 7913 7914 for (int c = 0; c < pvd->vdev_children; c++) { 7915 cvd = pvd->vdev_child[c]; 7916 7917 if (cvd == vd || cvd->vdev_path == NULL) 7918 continue; 7919 7920 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 7921 strcmp(cvd->vdev_path + len, "/old") == 0) { 7922 spa_strfree(cvd->vdev_path); 7923 cvd->vdev_path = spa_strdup(vd->vdev_path); 7924 break; 7925 } 7926 } 7927 } 7928 7929 /* 7930 * If we are detaching the original disk from a normal spare, then it 7931 * implies that the spare should become a real disk, and be removed 7932 * from the active spare list for the pool. dRAID spares on the 7933 * other hand are coupled to the pool and thus should never be removed 7934 * from the spares list. 7935 */ 7936 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 7937 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7938 7939 if (last_cvd->vdev_isspare && 7940 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 7941 unspare = B_TRUE; 7942 } 7943 } 7944 7945 /* 7946 * Erase the disk labels so the disk can be used for other things. 7947 * This must be done after all other error cases are handled, 7948 * but before we disembowel vd (so we can still do I/O to it). 7949 * But if we can't do it, don't treat the error as fatal -- 7950 * it may be that the unwritability of the disk is the reason 7951 * it's being detached! 7952 */ 7953 (void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 7954 7955 /* 7956 * Remove vd from its parent and compact the parent's children. 7957 */ 7958 vdev_remove_child(pvd, vd); 7959 vdev_compact_children(pvd); 7960 7961 /* 7962 * Remember one of the remaining children so we can get tvd below. 7963 */ 7964 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7965 7966 /* 7967 * If we need to remove the remaining child from the list of hot spares, 7968 * do it now, marking the vdev as no longer a spare in the process. 7969 * We must do this before vdev_remove_parent(), because that can 7970 * change the GUID if it creates a new toplevel GUID. For a similar 7971 * reason, we must remove the spare now, in the same txg as the detach; 7972 * otherwise someone could attach a new sibling, change the GUID, and 7973 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 7974 */ 7975 if (unspare) { 7976 ASSERT(cvd->vdev_isspare); 7977 spa_spare_remove(cvd); 7978 unspare_guid = cvd->vdev_guid; 7979 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 7980 cvd->vdev_unspare = B_TRUE; 7981 } 7982 7983 /* 7984 * If the parent mirror/replacing vdev only has one child, 7985 * the parent is no longer needed. Remove it from the tree. 7986 */ 7987 if (pvd->vdev_children == 1) { 7988 if (pvd->vdev_ops == &vdev_spare_ops) 7989 cvd->vdev_unspare = B_FALSE; 7990 vdev_remove_parent(cvd); 7991 } 7992 7993 /* 7994 * We don't set tvd until now because the parent we just removed 7995 * may have been the previous top-level vdev. 7996 */ 7997 tvd = cvd->vdev_top; 7998 ASSERT(tvd->vdev_parent == rvd); 7999 8000 /* 8001 * Reevaluate the parent vdev state. 8002 */ 8003 vdev_propagate_state(cvd); 8004 8005 /* 8006 * If the 'autoexpand' property is set on the pool then automatically 8007 * try to expand the size of the pool. For example if the device we 8008 * just detached was smaller than the others, it may be possible to 8009 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 8010 * first so that we can obtain the updated sizes of the leaf vdevs. 8011 */ 8012 if (spa->spa_autoexpand) { 8013 vdev_reopen(tvd); 8014 vdev_expand(tvd, txg); 8015 } 8016 8017 vdev_config_dirty(tvd); 8018 8019 /* 8020 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 8021 * vd->vdev_detached is set and free vd's DTL object in syncing context. 8022 * But first make sure we're not on any *other* txg's DTL list, to 8023 * prevent vd from being accessed after it's freed. 8024 */ 8025 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 8026 for (int t = 0; t < TXG_SIZE; t++) 8027 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 8028 vd->vdev_detached = B_TRUE; 8029 vdev_dirty(tvd, VDD_DTL, vd, txg); 8030 8031 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 8032 spa_notify_waiters(spa); 8033 8034 /* hang on to the spa before we release the lock */ 8035 spa_open_ref(spa, FTAG); 8036 8037 error = spa_vdev_exit(spa, vd, txg, 0); 8038 8039 spa_history_log_internal(spa, "detach", NULL, 8040 "vdev=%s", vdpath); 8041 spa_strfree(vdpath); 8042 8043 /* 8044 * If this was the removal of the original device in a hot spare vdev, 8045 * then we want to go through and remove the device from the hot spare 8046 * list of every other pool. 8047 */ 8048 if (unspare) { 8049 spa_t *altspa = NULL; 8050 8051 mutex_enter(&spa_namespace_lock); 8052 while ((altspa = spa_next(altspa)) != NULL) { 8053 if (altspa->spa_state != POOL_STATE_ACTIVE || 8054 altspa == spa) 8055 continue; 8056 8057 spa_open_ref(altspa, FTAG); 8058 mutex_exit(&spa_namespace_lock); 8059 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 8060 mutex_enter(&spa_namespace_lock); 8061 spa_close(altspa, FTAG); 8062 } 8063 mutex_exit(&spa_namespace_lock); 8064 8065 /* search the rest of the vdevs for spares to remove */ 8066 spa_vdev_resilver_done(spa); 8067 } 8068 8069 /* all done with the spa; OK to release */ 8070 mutex_enter(&spa_namespace_lock); 8071 spa_close(spa, FTAG); 8072 mutex_exit(&spa_namespace_lock); 8073 8074 return (error); 8075 } 8076 8077 static int 8078 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 8079 list_t *vd_list) 8080 { 8081 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 8082 8083 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 8084 8085 /* Look up vdev and ensure it's a leaf. */ 8086 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 8087 if (vd == NULL || vd->vdev_detached) { 8088 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8089 return (SET_ERROR(ENODEV)); 8090 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 8091 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8092 return (SET_ERROR(EINVAL)); 8093 } else if (!vdev_writeable(vd)) { 8094 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8095 return (SET_ERROR(EROFS)); 8096 } 8097 mutex_enter(&vd->vdev_initialize_lock); 8098 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8099 8100 /* 8101 * When we activate an initialize action we check to see 8102 * if the vdev_initialize_thread is NULL. We do this instead 8103 * of using the vdev_initialize_state since there might be 8104 * a previous initialization process which has completed but 8105 * the thread is not exited. 8106 */ 8107 if (cmd_type == POOL_INITIALIZE_START && 8108 (vd->vdev_initialize_thread != NULL || 8109 vd->vdev_top->vdev_removing || vd->vdev_top->vdev_rz_expanding)) { 8110 mutex_exit(&vd->vdev_initialize_lock); 8111 return (SET_ERROR(EBUSY)); 8112 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 8113 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 8114 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 8115 mutex_exit(&vd->vdev_initialize_lock); 8116 return (SET_ERROR(ESRCH)); 8117 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 8118 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 8119 mutex_exit(&vd->vdev_initialize_lock); 8120 return (SET_ERROR(ESRCH)); 8121 } else if (cmd_type == POOL_INITIALIZE_UNINIT && 8122 vd->vdev_initialize_thread != NULL) { 8123 mutex_exit(&vd->vdev_initialize_lock); 8124 return (SET_ERROR(EBUSY)); 8125 } 8126 8127 switch (cmd_type) { 8128 case POOL_INITIALIZE_START: 8129 vdev_initialize(vd); 8130 break; 8131 case POOL_INITIALIZE_CANCEL: 8132 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 8133 break; 8134 case POOL_INITIALIZE_SUSPEND: 8135 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 8136 break; 8137 case POOL_INITIALIZE_UNINIT: 8138 vdev_uninitialize(vd); 8139 break; 8140 default: 8141 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 8142 } 8143 mutex_exit(&vd->vdev_initialize_lock); 8144 8145 return (0); 8146 } 8147 8148 int 8149 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 8150 nvlist_t *vdev_errlist) 8151 { 8152 int total_errors = 0; 8153 list_t vd_list; 8154 8155 list_create(&vd_list, sizeof (vdev_t), 8156 offsetof(vdev_t, vdev_initialize_node)); 8157 8158 /* 8159 * We hold the namespace lock through the whole function 8160 * to prevent any changes to the pool while we're starting or 8161 * stopping initialization. The config and state locks are held so that 8162 * we can properly assess the vdev state before we commit to 8163 * the initializing operation. 8164 */ 8165 mutex_enter(&spa_namespace_lock); 8166 8167 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 8168 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 8169 uint64_t vdev_guid = fnvpair_value_uint64(pair); 8170 8171 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 8172 &vd_list); 8173 if (error != 0) { 8174 char guid_as_str[MAXNAMELEN]; 8175 8176 (void) snprintf(guid_as_str, sizeof (guid_as_str), 8177 "%llu", (unsigned long long)vdev_guid); 8178 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 8179 total_errors++; 8180 } 8181 } 8182 8183 /* Wait for all initialize threads to stop. */ 8184 vdev_initialize_stop_wait(spa, &vd_list); 8185 8186 /* Sync out the initializing state */ 8187 txg_wait_synced(spa->spa_dsl_pool, 0); 8188 mutex_exit(&spa_namespace_lock); 8189 8190 list_destroy(&vd_list); 8191 8192 return (total_errors); 8193 } 8194 8195 static int 8196 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 8197 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 8198 { 8199 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 8200 8201 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 8202 8203 /* Look up vdev and ensure it's a leaf. */ 8204 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 8205 if (vd == NULL || vd->vdev_detached) { 8206 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8207 return (SET_ERROR(ENODEV)); 8208 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 8209 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8210 return (SET_ERROR(EINVAL)); 8211 } else if (!vdev_writeable(vd)) { 8212 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8213 return (SET_ERROR(EROFS)); 8214 } else if (!vd->vdev_has_trim) { 8215 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8216 return (SET_ERROR(EOPNOTSUPP)); 8217 } else if (secure && !vd->vdev_has_securetrim) { 8218 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8219 return (SET_ERROR(EOPNOTSUPP)); 8220 } 8221 mutex_enter(&vd->vdev_trim_lock); 8222 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 8223 8224 /* 8225 * When we activate a TRIM action we check to see if the 8226 * vdev_trim_thread is NULL. We do this instead of using the 8227 * vdev_trim_state since there might be a previous TRIM process 8228 * which has completed but the thread is not exited. 8229 */ 8230 if (cmd_type == POOL_TRIM_START && 8231 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing || 8232 vd->vdev_top->vdev_rz_expanding)) { 8233 mutex_exit(&vd->vdev_trim_lock); 8234 return (SET_ERROR(EBUSY)); 8235 } else if (cmd_type == POOL_TRIM_CANCEL && 8236 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 8237 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 8238 mutex_exit(&vd->vdev_trim_lock); 8239 return (SET_ERROR(ESRCH)); 8240 } else if (cmd_type == POOL_TRIM_SUSPEND && 8241 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 8242 mutex_exit(&vd->vdev_trim_lock); 8243 return (SET_ERROR(ESRCH)); 8244 } 8245 8246 switch (cmd_type) { 8247 case POOL_TRIM_START: 8248 vdev_trim(vd, rate, partial, secure); 8249 break; 8250 case POOL_TRIM_CANCEL: 8251 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 8252 break; 8253 case POOL_TRIM_SUSPEND: 8254 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 8255 break; 8256 default: 8257 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 8258 } 8259 mutex_exit(&vd->vdev_trim_lock); 8260 8261 return (0); 8262 } 8263 8264 /* 8265 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 8266 * TRIM threads for each child vdev. These threads pass over all of the free 8267 * space in the vdev's metaslabs and issues TRIM commands for that space. 8268 */ 8269 int 8270 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 8271 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 8272 { 8273 int total_errors = 0; 8274 list_t vd_list; 8275 8276 list_create(&vd_list, sizeof (vdev_t), 8277 offsetof(vdev_t, vdev_trim_node)); 8278 8279 /* 8280 * We hold the namespace lock through the whole function 8281 * to prevent any changes to the pool while we're starting or 8282 * stopping TRIM. The config and state locks are held so that 8283 * we can properly assess the vdev state before we commit to 8284 * the TRIM operation. 8285 */ 8286 mutex_enter(&spa_namespace_lock); 8287 8288 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 8289 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 8290 uint64_t vdev_guid = fnvpair_value_uint64(pair); 8291 8292 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 8293 rate, partial, secure, &vd_list); 8294 if (error != 0) { 8295 char guid_as_str[MAXNAMELEN]; 8296 8297 (void) snprintf(guid_as_str, sizeof (guid_as_str), 8298 "%llu", (unsigned long long)vdev_guid); 8299 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 8300 total_errors++; 8301 } 8302 } 8303 8304 /* Wait for all TRIM threads to stop. */ 8305 vdev_trim_stop_wait(spa, &vd_list); 8306 8307 /* Sync out the TRIM state */ 8308 txg_wait_synced(spa->spa_dsl_pool, 0); 8309 mutex_exit(&spa_namespace_lock); 8310 8311 list_destroy(&vd_list); 8312 8313 return (total_errors); 8314 } 8315 8316 /* 8317 * Split a set of devices from their mirrors, and create a new pool from them. 8318 */ 8319 int 8320 spa_vdev_split_mirror(spa_t *spa, const char *newname, nvlist_t *config, 8321 nvlist_t *props, boolean_t exp) 8322 { 8323 int error = 0; 8324 uint64_t txg, *glist; 8325 spa_t *newspa; 8326 uint_t c, children, lastlog; 8327 nvlist_t **child, *nvl, *tmp; 8328 dmu_tx_t *tx; 8329 const char *altroot = NULL; 8330 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 8331 boolean_t activate_slog; 8332 8333 ASSERT(spa_writeable(spa)); 8334 8335 txg = spa_vdev_enter(spa); 8336 8337 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 8338 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 8339 error = (spa_has_checkpoint(spa)) ? 8340 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 8341 return (spa_vdev_exit(spa, NULL, txg, error)); 8342 } 8343 8344 /* clear the log and flush everything up to now */ 8345 activate_slog = spa_passivate_log(spa); 8346 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 8347 error = spa_reset_logs(spa); 8348 txg = spa_vdev_config_enter(spa); 8349 8350 if (activate_slog) 8351 spa_activate_log(spa); 8352 8353 if (error != 0) 8354 return (spa_vdev_exit(spa, NULL, txg, error)); 8355 8356 /* check new spa name before going any further */ 8357 if (spa_lookup(newname) != NULL) 8358 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 8359 8360 /* 8361 * scan through all the children to ensure they're all mirrors 8362 */ 8363 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 8364 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 8365 &children) != 0) 8366 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 8367 8368 /* first, check to ensure we've got the right child count */ 8369 rvd = spa->spa_root_vdev; 8370 lastlog = 0; 8371 for (c = 0; c < rvd->vdev_children; c++) { 8372 vdev_t *vd = rvd->vdev_child[c]; 8373 8374 /* don't count the holes & logs as children */ 8375 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 8376 !vdev_is_concrete(vd))) { 8377 if (lastlog == 0) 8378 lastlog = c; 8379 continue; 8380 } 8381 8382 lastlog = 0; 8383 } 8384 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 8385 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 8386 8387 /* next, ensure no spare or cache devices are part of the split */ 8388 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 8389 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 8390 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 8391 8392 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 8393 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 8394 8395 /* then, loop over each vdev and validate it */ 8396 for (c = 0; c < children; c++) { 8397 uint64_t is_hole = 0; 8398 8399 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 8400 &is_hole); 8401 8402 if (is_hole != 0) { 8403 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 8404 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 8405 continue; 8406 } else { 8407 error = SET_ERROR(EINVAL); 8408 break; 8409 } 8410 } 8411 8412 /* deal with indirect vdevs */ 8413 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 8414 &vdev_indirect_ops) 8415 continue; 8416 8417 /* which disk is going to be split? */ 8418 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 8419 &glist[c]) != 0) { 8420 error = SET_ERROR(EINVAL); 8421 break; 8422 } 8423 8424 /* look it up in the spa */ 8425 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 8426 if (vml[c] == NULL) { 8427 error = SET_ERROR(ENODEV); 8428 break; 8429 } 8430 8431 /* make sure there's nothing stopping the split */ 8432 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 8433 vml[c]->vdev_islog || 8434 !vdev_is_concrete(vml[c]) || 8435 vml[c]->vdev_isspare || 8436 vml[c]->vdev_isl2cache || 8437 !vdev_writeable(vml[c]) || 8438 vml[c]->vdev_children != 0 || 8439 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 8440 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 8441 error = SET_ERROR(EINVAL); 8442 break; 8443 } 8444 8445 if (vdev_dtl_required(vml[c]) || 8446 vdev_resilver_needed(vml[c], NULL, NULL)) { 8447 error = SET_ERROR(EBUSY); 8448 break; 8449 } 8450 8451 /* we need certain info from the top level */ 8452 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 8453 vml[c]->vdev_top->vdev_ms_array); 8454 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 8455 vml[c]->vdev_top->vdev_ms_shift); 8456 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 8457 vml[c]->vdev_top->vdev_asize); 8458 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 8459 vml[c]->vdev_top->vdev_ashift); 8460 8461 /* transfer per-vdev ZAPs */ 8462 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 8463 VERIFY0(nvlist_add_uint64(child[c], 8464 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 8465 8466 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 8467 VERIFY0(nvlist_add_uint64(child[c], 8468 ZPOOL_CONFIG_VDEV_TOP_ZAP, 8469 vml[c]->vdev_parent->vdev_top_zap)); 8470 } 8471 8472 if (error != 0) { 8473 kmem_free(vml, children * sizeof (vdev_t *)); 8474 kmem_free(glist, children * sizeof (uint64_t)); 8475 return (spa_vdev_exit(spa, NULL, txg, error)); 8476 } 8477 8478 /* stop writers from using the disks */ 8479 for (c = 0; c < children; c++) { 8480 if (vml[c] != NULL) 8481 vml[c]->vdev_offline = B_TRUE; 8482 } 8483 vdev_reopen(spa->spa_root_vdev); 8484 8485 /* 8486 * Temporarily record the splitting vdevs in the spa config. This 8487 * will disappear once the config is regenerated. 8488 */ 8489 nvl = fnvlist_alloc(); 8490 fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children); 8491 kmem_free(glist, children * sizeof (uint64_t)); 8492 8493 mutex_enter(&spa->spa_props_lock); 8494 fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl); 8495 mutex_exit(&spa->spa_props_lock); 8496 spa->spa_config_splitting = nvl; 8497 vdev_config_dirty(spa->spa_root_vdev); 8498 8499 /* configure and create the new pool */ 8500 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname); 8501 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 8502 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE); 8503 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)); 8504 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg); 8505 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 8506 spa_generate_guid(NULL)); 8507 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 8508 (void) nvlist_lookup_string(props, 8509 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 8510 8511 /* add the new pool to the namespace */ 8512 newspa = spa_add(newname, config, altroot); 8513 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 8514 newspa->spa_config_txg = spa->spa_config_txg; 8515 spa_set_log_state(newspa, SPA_LOG_CLEAR); 8516 8517 /* release the spa config lock, retaining the namespace lock */ 8518 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 8519 8520 if (zio_injection_enabled) 8521 zio_handle_panic_injection(spa, FTAG, 1); 8522 8523 spa_activate(newspa, spa_mode_global); 8524 spa_async_suspend(newspa); 8525 8526 /* 8527 * Temporarily stop the initializing and TRIM activity. We set the 8528 * state to ACTIVE so that we know to resume initializing or TRIM 8529 * once the split has completed. 8530 */ 8531 list_t vd_initialize_list; 8532 list_create(&vd_initialize_list, sizeof (vdev_t), 8533 offsetof(vdev_t, vdev_initialize_node)); 8534 8535 list_t vd_trim_list; 8536 list_create(&vd_trim_list, sizeof (vdev_t), 8537 offsetof(vdev_t, vdev_trim_node)); 8538 8539 for (c = 0; c < children; c++) { 8540 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 8541 mutex_enter(&vml[c]->vdev_initialize_lock); 8542 vdev_initialize_stop(vml[c], 8543 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 8544 mutex_exit(&vml[c]->vdev_initialize_lock); 8545 8546 mutex_enter(&vml[c]->vdev_trim_lock); 8547 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 8548 mutex_exit(&vml[c]->vdev_trim_lock); 8549 } 8550 } 8551 8552 vdev_initialize_stop_wait(spa, &vd_initialize_list); 8553 vdev_trim_stop_wait(spa, &vd_trim_list); 8554 8555 list_destroy(&vd_initialize_list); 8556 list_destroy(&vd_trim_list); 8557 8558 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 8559 newspa->spa_is_splitting = B_TRUE; 8560 8561 /* create the new pool from the disks of the original pool */ 8562 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 8563 if (error) 8564 goto out; 8565 8566 /* if that worked, generate a real config for the new pool */ 8567 if (newspa->spa_root_vdev != NULL) { 8568 newspa->spa_config_splitting = fnvlist_alloc(); 8569 fnvlist_add_uint64(newspa->spa_config_splitting, 8570 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)); 8571 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 8572 B_TRUE)); 8573 } 8574 8575 /* set the props */ 8576 if (props != NULL) { 8577 spa_configfile_set(newspa, props, B_FALSE); 8578 error = spa_prop_set(newspa, props); 8579 if (error) 8580 goto out; 8581 } 8582 8583 /* flush everything */ 8584 txg = spa_vdev_config_enter(newspa); 8585 vdev_config_dirty(newspa->spa_root_vdev); 8586 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 8587 8588 if (zio_injection_enabled) 8589 zio_handle_panic_injection(spa, FTAG, 2); 8590 8591 spa_async_resume(newspa); 8592 8593 /* finally, update the original pool's config */ 8594 txg = spa_vdev_config_enter(spa); 8595 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 8596 error = dmu_tx_assign(tx, DMU_TX_WAIT); 8597 if (error != 0) 8598 dmu_tx_abort(tx); 8599 for (c = 0; c < children; c++) { 8600 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 8601 vdev_t *tvd = vml[c]->vdev_top; 8602 8603 /* 8604 * Need to be sure the detachable VDEV is not 8605 * on any *other* txg's DTL list to prevent it 8606 * from being accessed after it's freed. 8607 */ 8608 for (int t = 0; t < TXG_SIZE; t++) { 8609 (void) txg_list_remove_this( 8610 &tvd->vdev_dtl_list, vml[c], t); 8611 } 8612 8613 vdev_split(vml[c]); 8614 if (error == 0) 8615 spa_history_log_internal(spa, "detach", tx, 8616 "vdev=%s", vml[c]->vdev_path); 8617 8618 vdev_free(vml[c]); 8619 } 8620 } 8621 spa->spa_avz_action = AVZ_ACTION_REBUILD; 8622 vdev_config_dirty(spa->spa_root_vdev); 8623 spa->spa_config_splitting = NULL; 8624 nvlist_free(nvl); 8625 if (error == 0) 8626 dmu_tx_commit(tx); 8627 (void) spa_vdev_exit(spa, NULL, txg, 0); 8628 8629 if (zio_injection_enabled) 8630 zio_handle_panic_injection(spa, FTAG, 3); 8631 8632 /* split is complete; log a history record */ 8633 spa_history_log_internal(newspa, "split", NULL, 8634 "from pool %s", spa_name(spa)); 8635 8636 newspa->spa_is_splitting = B_FALSE; 8637 kmem_free(vml, children * sizeof (vdev_t *)); 8638 8639 /* if we're not going to mount the filesystems in userland, export */ 8640 if (exp) 8641 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 8642 B_FALSE, B_FALSE); 8643 8644 return (error); 8645 8646 out: 8647 spa_unload(newspa); 8648 spa_deactivate(newspa); 8649 spa_remove(newspa); 8650 8651 txg = spa_vdev_config_enter(spa); 8652 8653 /* re-online all offlined disks */ 8654 for (c = 0; c < children; c++) { 8655 if (vml[c] != NULL) 8656 vml[c]->vdev_offline = B_FALSE; 8657 } 8658 8659 /* restart initializing or trimming disks as necessary */ 8660 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 8661 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 8662 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 8663 8664 vdev_reopen(spa->spa_root_vdev); 8665 8666 nvlist_free(spa->spa_config_splitting); 8667 spa->spa_config_splitting = NULL; 8668 (void) spa_vdev_exit(spa, NULL, txg, error); 8669 8670 kmem_free(vml, children * sizeof (vdev_t *)); 8671 return (error); 8672 } 8673 8674 /* 8675 * Find any device that's done replacing, or a vdev marked 'unspare' that's 8676 * currently spared, so we can detach it. 8677 */ 8678 static vdev_t * 8679 spa_vdev_resilver_done_hunt(vdev_t *vd) 8680 { 8681 vdev_t *newvd, *oldvd; 8682 8683 for (int c = 0; c < vd->vdev_children; c++) { 8684 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 8685 if (oldvd != NULL) 8686 return (oldvd); 8687 } 8688 8689 /* 8690 * Check for a completed replacement. We always consider the first 8691 * vdev in the list to be the oldest vdev, and the last one to be 8692 * the newest (see spa_vdev_attach() for how that works). In 8693 * the case where the newest vdev is faulted, we will not automatically 8694 * remove it after a resilver completes. This is OK as it will require 8695 * user intervention to determine which disk the admin wishes to keep. 8696 */ 8697 if (vd->vdev_ops == &vdev_replacing_ops) { 8698 ASSERT(vd->vdev_children > 1); 8699 8700 newvd = vd->vdev_child[vd->vdev_children - 1]; 8701 oldvd = vd->vdev_child[0]; 8702 8703 if (vdev_dtl_empty(newvd, DTL_MISSING) && 8704 vdev_dtl_empty(newvd, DTL_OUTAGE) && 8705 !vdev_dtl_required(oldvd)) 8706 return (oldvd); 8707 } 8708 8709 /* 8710 * Check for a completed resilver with the 'unspare' flag set. 8711 * Also potentially update faulted state. 8712 */ 8713 if (vd->vdev_ops == &vdev_spare_ops) { 8714 vdev_t *first = vd->vdev_child[0]; 8715 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 8716 8717 if (last->vdev_unspare) { 8718 oldvd = first; 8719 newvd = last; 8720 } else if (first->vdev_unspare) { 8721 oldvd = last; 8722 newvd = first; 8723 } else { 8724 oldvd = NULL; 8725 } 8726 8727 if (oldvd != NULL && 8728 vdev_dtl_empty(newvd, DTL_MISSING) && 8729 vdev_dtl_empty(newvd, DTL_OUTAGE) && 8730 !vdev_dtl_required(oldvd)) 8731 return (oldvd); 8732 8733 vdev_propagate_state(vd); 8734 8735 /* 8736 * If there are more than two spares attached to a disk, 8737 * and those spares are not required, then we want to 8738 * attempt to free them up now so that they can be used 8739 * by other pools. Once we're back down to a single 8740 * disk+spare, we stop removing them. 8741 */ 8742 if (vd->vdev_children > 2) { 8743 newvd = vd->vdev_child[1]; 8744 8745 if (newvd->vdev_isspare && last->vdev_isspare && 8746 vdev_dtl_empty(last, DTL_MISSING) && 8747 vdev_dtl_empty(last, DTL_OUTAGE) && 8748 !vdev_dtl_required(newvd)) 8749 return (newvd); 8750 } 8751 } 8752 8753 return (NULL); 8754 } 8755 8756 static void 8757 spa_vdev_resilver_done(spa_t *spa) 8758 { 8759 vdev_t *vd, *pvd, *ppvd; 8760 uint64_t guid, sguid, pguid, ppguid; 8761 8762 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 8763 8764 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 8765 pvd = vd->vdev_parent; 8766 ppvd = pvd->vdev_parent; 8767 guid = vd->vdev_guid; 8768 pguid = pvd->vdev_guid; 8769 ppguid = ppvd->vdev_guid; 8770 sguid = 0; 8771 /* 8772 * If we have just finished replacing a hot spared device, then 8773 * we need to detach the parent's first child (the original hot 8774 * spare) as well. 8775 */ 8776 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 8777 ppvd->vdev_children == 2) { 8778 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 8779 sguid = ppvd->vdev_child[1]->vdev_guid; 8780 } 8781 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 8782 8783 spa_config_exit(spa, SCL_ALL, FTAG); 8784 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 8785 return; 8786 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 8787 return; 8788 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 8789 } 8790 8791 spa_config_exit(spa, SCL_ALL, FTAG); 8792 8793 /* 8794 * If a detach was not performed above replace waiters will not have 8795 * been notified. In which case we must do so now. 8796 */ 8797 spa_notify_waiters(spa); 8798 } 8799 8800 /* 8801 * Update the stored path or FRU for this vdev. 8802 */ 8803 static int 8804 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 8805 boolean_t ispath) 8806 { 8807 vdev_t *vd; 8808 boolean_t sync = B_FALSE; 8809 8810 ASSERT(spa_writeable(spa)); 8811 8812 spa_vdev_state_enter(spa, SCL_ALL); 8813 8814 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 8815 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 8816 8817 if (!vd->vdev_ops->vdev_op_leaf) 8818 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 8819 8820 if (ispath) { 8821 if (strcmp(value, vd->vdev_path) != 0) { 8822 spa_strfree(vd->vdev_path); 8823 vd->vdev_path = spa_strdup(value); 8824 sync = B_TRUE; 8825 } 8826 } else { 8827 if (vd->vdev_fru == NULL) { 8828 vd->vdev_fru = spa_strdup(value); 8829 sync = B_TRUE; 8830 } else if (strcmp(value, vd->vdev_fru) != 0) { 8831 spa_strfree(vd->vdev_fru); 8832 vd->vdev_fru = spa_strdup(value); 8833 sync = B_TRUE; 8834 } 8835 } 8836 8837 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 8838 } 8839 8840 int 8841 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 8842 { 8843 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 8844 } 8845 8846 int 8847 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 8848 { 8849 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 8850 } 8851 8852 /* 8853 * ========================================================================== 8854 * SPA Scanning 8855 * ========================================================================== 8856 */ 8857 int 8858 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 8859 { 8860 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8861 8862 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8863 return (SET_ERROR(EBUSY)); 8864 8865 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 8866 } 8867 8868 int 8869 spa_scan_stop(spa_t *spa) 8870 { 8871 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8872 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8873 return (SET_ERROR(EBUSY)); 8874 8875 return (dsl_scan_cancel(spa->spa_dsl_pool)); 8876 } 8877 8878 int 8879 spa_scan(spa_t *spa, pool_scan_func_t func) 8880 { 8881 return (spa_scan_range(spa, func, 0, 0)); 8882 } 8883 8884 int 8885 spa_scan_range(spa_t *spa, pool_scan_func_t func, uint64_t txgstart, 8886 uint64_t txgend) 8887 { 8888 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8889 8890 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 8891 return (SET_ERROR(ENOTSUP)); 8892 8893 if (func == POOL_SCAN_RESILVER && 8894 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 8895 return (SET_ERROR(ENOTSUP)); 8896 8897 if (func != POOL_SCAN_SCRUB && (txgstart != 0 || txgend != 0)) 8898 return (SET_ERROR(ENOTSUP)); 8899 8900 /* 8901 * If a resilver was requested, but there is no DTL on a 8902 * writeable leaf device, we have nothing to do. 8903 */ 8904 if (func == POOL_SCAN_RESILVER && 8905 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 8906 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 8907 return (0); 8908 } 8909 8910 if (func == POOL_SCAN_ERRORSCRUB && 8911 !spa_feature_is_enabled(spa, SPA_FEATURE_HEAD_ERRLOG)) 8912 return (SET_ERROR(ENOTSUP)); 8913 8914 return (dsl_scan(spa->spa_dsl_pool, func, txgstart, txgend)); 8915 } 8916 8917 /* 8918 * ========================================================================== 8919 * SPA async task processing 8920 * ========================================================================== 8921 */ 8922 8923 static void 8924 spa_async_remove(spa_t *spa, vdev_t *vd) 8925 { 8926 if (vd->vdev_remove_wanted) { 8927 vd->vdev_remove_wanted = B_FALSE; 8928 vd->vdev_delayed_close = B_FALSE; 8929 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 8930 8931 /* 8932 * We want to clear the stats, but we don't want to do a full 8933 * vdev_clear() as that will cause us to throw away 8934 * degraded/faulted state as well as attempt to reopen the 8935 * device, all of which is a waste. 8936 */ 8937 vd->vdev_stat.vs_read_errors = 0; 8938 vd->vdev_stat.vs_write_errors = 0; 8939 vd->vdev_stat.vs_checksum_errors = 0; 8940 8941 vdev_state_dirty(vd->vdev_top); 8942 8943 /* Tell userspace that the vdev is gone. */ 8944 zfs_post_remove(spa, vd); 8945 } 8946 8947 for (int c = 0; c < vd->vdev_children; c++) 8948 spa_async_remove(spa, vd->vdev_child[c]); 8949 } 8950 8951 static void 8952 spa_async_fault_vdev(vdev_t *vd, boolean_t *suspend) 8953 { 8954 if (vd->vdev_fault_wanted) { 8955 vdev_state_t newstate = VDEV_STATE_FAULTED; 8956 vd->vdev_fault_wanted = B_FALSE; 8957 8958 /* 8959 * If this device has the only valid copy of the data, then 8960 * back off and simply mark the vdev as degraded instead. 8961 */ 8962 if (!vd->vdev_top->vdev_islog && vd->vdev_aux == NULL && 8963 vdev_dtl_required(vd)) { 8964 newstate = VDEV_STATE_DEGRADED; 8965 /* A required disk is missing so suspend the pool */ 8966 *suspend = B_TRUE; 8967 } 8968 vdev_set_state(vd, B_TRUE, newstate, VDEV_AUX_ERR_EXCEEDED); 8969 } 8970 for (int c = 0; c < vd->vdev_children; c++) 8971 spa_async_fault_vdev(vd->vdev_child[c], suspend); 8972 } 8973 8974 static void 8975 spa_async_autoexpand(spa_t *spa, vdev_t *vd) 8976 { 8977 if (!spa->spa_autoexpand) 8978 return; 8979 8980 for (int c = 0; c < vd->vdev_children; c++) { 8981 vdev_t *cvd = vd->vdev_child[c]; 8982 spa_async_autoexpand(spa, cvd); 8983 } 8984 8985 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 8986 return; 8987 8988 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 8989 } 8990 8991 static __attribute__((noreturn)) void 8992 spa_async_thread(void *arg) 8993 { 8994 spa_t *spa = (spa_t *)arg; 8995 dsl_pool_t *dp = spa->spa_dsl_pool; 8996 int tasks; 8997 8998 ASSERT(spa->spa_sync_on); 8999 9000 mutex_enter(&spa->spa_async_lock); 9001 tasks = spa->spa_async_tasks; 9002 spa->spa_async_tasks = 0; 9003 mutex_exit(&spa->spa_async_lock); 9004 9005 /* 9006 * See if the config needs to be updated. 9007 */ 9008 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 9009 uint64_t old_space, new_space; 9010 9011 mutex_enter(&spa_namespace_lock); 9012 old_space = metaslab_class_get_space(spa_normal_class(spa)); 9013 old_space += metaslab_class_get_space(spa_special_class(spa)); 9014 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 9015 old_space += metaslab_class_get_space( 9016 spa_embedded_log_class(spa)); 9017 9018 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 9019 9020 new_space = metaslab_class_get_space(spa_normal_class(spa)); 9021 new_space += metaslab_class_get_space(spa_special_class(spa)); 9022 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 9023 new_space += metaslab_class_get_space( 9024 spa_embedded_log_class(spa)); 9025 mutex_exit(&spa_namespace_lock); 9026 9027 /* 9028 * If the pool grew as a result of the config update, 9029 * then log an internal history event. 9030 */ 9031 if (new_space != old_space) { 9032 spa_history_log_internal(spa, "vdev online", NULL, 9033 "pool '%s' size: %llu(+%llu)", 9034 spa_name(spa), (u_longlong_t)new_space, 9035 (u_longlong_t)(new_space - old_space)); 9036 } 9037 } 9038 9039 /* 9040 * See if any devices need to be marked REMOVED. 9041 */ 9042 if (tasks & SPA_ASYNC_REMOVE) { 9043 spa_vdev_state_enter(spa, SCL_NONE); 9044 spa_async_remove(spa, spa->spa_root_vdev); 9045 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 9046 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); 9047 for (int i = 0; i < spa->spa_spares.sav_count; i++) 9048 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); 9049 (void) spa_vdev_state_exit(spa, NULL, 0); 9050 } 9051 9052 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 9053 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9054 spa_async_autoexpand(spa, spa->spa_root_vdev); 9055 spa_config_exit(spa, SCL_CONFIG, FTAG); 9056 } 9057 9058 /* 9059 * See if any devices need to be marked faulted. 9060 */ 9061 if (tasks & SPA_ASYNC_FAULT_VDEV) { 9062 spa_vdev_state_enter(spa, SCL_NONE); 9063 boolean_t suspend = B_FALSE; 9064 spa_async_fault_vdev(spa->spa_root_vdev, &suspend); 9065 (void) spa_vdev_state_exit(spa, NULL, 0); 9066 if (suspend) 9067 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 9068 } 9069 9070 /* 9071 * If any devices are done replacing, detach them. 9072 */ 9073 if (tasks & SPA_ASYNC_RESILVER_DONE || 9074 tasks & SPA_ASYNC_REBUILD_DONE || 9075 tasks & SPA_ASYNC_DETACH_SPARE) { 9076 spa_vdev_resilver_done(spa); 9077 } 9078 9079 /* 9080 * Kick off a resilver. 9081 */ 9082 if (tasks & SPA_ASYNC_RESILVER && 9083 !vdev_rebuild_active(spa->spa_root_vdev) && 9084 (!dsl_scan_resilvering(dp) || 9085 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 9086 dsl_scan_restart_resilver(dp, 0); 9087 9088 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 9089 mutex_enter(&spa_namespace_lock); 9090 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9091 vdev_initialize_restart(spa->spa_root_vdev); 9092 spa_config_exit(spa, SCL_CONFIG, FTAG); 9093 mutex_exit(&spa_namespace_lock); 9094 } 9095 9096 if (tasks & SPA_ASYNC_TRIM_RESTART) { 9097 mutex_enter(&spa_namespace_lock); 9098 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9099 vdev_trim_restart(spa->spa_root_vdev); 9100 spa_config_exit(spa, SCL_CONFIG, FTAG); 9101 mutex_exit(&spa_namespace_lock); 9102 } 9103 9104 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 9105 mutex_enter(&spa_namespace_lock); 9106 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9107 vdev_autotrim_restart(spa); 9108 spa_config_exit(spa, SCL_CONFIG, FTAG); 9109 mutex_exit(&spa_namespace_lock); 9110 } 9111 9112 /* 9113 * Kick off L2 cache whole device TRIM. 9114 */ 9115 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 9116 mutex_enter(&spa_namespace_lock); 9117 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9118 vdev_trim_l2arc(spa); 9119 spa_config_exit(spa, SCL_CONFIG, FTAG); 9120 mutex_exit(&spa_namespace_lock); 9121 } 9122 9123 /* 9124 * Kick off L2 cache rebuilding. 9125 */ 9126 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 9127 mutex_enter(&spa_namespace_lock); 9128 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 9129 l2arc_spa_rebuild_start(spa); 9130 spa_config_exit(spa, SCL_L2ARC, FTAG); 9131 mutex_exit(&spa_namespace_lock); 9132 } 9133 9134 /* 9135 * Let the world know that we're done. 9136 */ 9137 mutex_enter(&spa->spa_async_lock); 9138 spa->spa_async_thread = NULL; 9139 cv_broadcast(&spa->spa_async_cv); 9140 mutex_exit(&spa->spa_async_lock); 9141 thread_exit(); 9142 } 9143 9144 void 9145 spa_async_suspend(spa_t *spa) 9146 { 9147 mutex_enter(&spa->spa_async_lock); 9148 spa->spa_async_suspended++; 9149 while (spa->spa_async_thread != NULL) 9150 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 9151 mutex_exit(&spa->spa_async_lock); 9152 9153 spa_vdev_remove_suspend(spa); 9154 9155 zthr_t *condense_thread = spa->spa_condense_zthr; 9156 if (condense_thread != NULL) 9157 zthr_cancel(condense_thread); 9158 9159 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr; 9160 if (raidz_expand_thread != NULL) 9161 zthr_cancel(raidz_expand_thread); 9162 9163 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 9164 if (discard_thread != NULL) 9165 zthr_cancel(discard_thread); 9166 9167 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 9168 if (ll_delete_thread != NULL) 9169 zthr_cancel(ll_delete_thread); 9170 9171 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 9172 if (ll_condense_thread != NULL) 9173 zthr_cancel(ll_condense_thread); 9174 } 9175 9176 void 9177 spa_async_resume(spa_t *spa) 9178 { 9179 mutex_enter(&spa->spa_async_lock); 9180 ASSERT(spa->spa_async_suspended != 0); 9181 spa->spa_async_suspended--; 9182 mutex_exit(&spa->spa_async_lock); 9183 spa_restart_removal(spa); 9184 9185 zthr_t *condense_thread = spa->spa_condense_zthr; 9186 if (condense_thread != NULL) 9187 zthr_resume(condense_thread); 9188 9189 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr; 9190 if (raidz_expand_thread != NULL) 9191 zthr_resume(raidz_expand_thread); 9192 9193 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 9194 if (discard_thread != NULL) 9195 zthr_resume(discard_thread); 9196 9197 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 9198 if (ll_delete_thread != NULL) 9199 zthr_resume(ll_delete_thread); 9200 9201 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 9202 if (ll_condense_thread != NULL) 9203 zthr_resume(ll_condense_thread); 9204 } 9205 9206 static boolean_t 9207 spa_async_tasks_pending(spa_t *spa) 9208 { 9209 uint_t non_config_tasks; 9210 uint_t config_task; 9211 boolean_t config_task_suspended; 9212 9213 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 9214 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 9215 if (spa->spa_ccw_fail_time == 0) { 9216 config_task_suspended = B_FALSE; 9217 } else { 9218 config_task_suspended = 9219 (gethrtime() - spa->spa_ccw_fail_time) < 9220 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 9221 } 9222 9223 return (non_config_tasks || (config_task && !config_task_suspended)); 9224 } 9225 9226 static void 9227 spa_async_dispatch(spa_t *spa) 9228 { 9229 mutex_enter(&spa->spa_async_lock); 9230 if (spa_async_tasks_pending(spa) && 9231 !spa->spa_async_suspended && 9232 spa->spa_async_thread == NULL) 9233 spa->spa_async_thread = thread_create(NULL, 0, 9234 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 9235 mutex_exit(&spa->spa_async_lock); 9236 } 9237 9238 void 9239 spa_async_request(spa_t *spa, int task) 9240 { 9241 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); 9242 mutex_enter(&spa->spa_async_lock); 9243 spa->spa_async_tasks |= task; 9244 mutex_exit(&spa->spa_async_lock); 9245 } 9246 9247 int 9248 spa_async_tasks(spa_t *spa) 9249 { 9250 return (spa->spa_async_tasks); 9251 } 9252 9253 /* 9254 * ========================================================================== 9255 * SPA syncing routines 9256 * ========================================================================== 9257 */ 9258 9259 9260 static int 9261 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 9262 dmu_tx_t *tx) 9263 { 9264 bpobj_t *bpo = arg; 9265 bpobj_enqueue(bpo, bp, bp_freed, tx); 9266 return (0); 9267 } 9268 9269 int 9270 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 9271 { 9272 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 9273 } 9274 9275 int 9276 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 9277 { 9278 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 9279 } 9280 9281 static int 9282 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 9283 { 9284 zio_t *pio = arg; 9285 9286 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 9287 pio->io_flags)); 9288 return (0); 9289 } 9290 9291 static int 9292 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 9293 dmu_tx_t *tx) 9294 { 9295 ASSERT(!bp_freed); 9296 return (spa_free_sync_cb(arg, bp, tx)); 9297 } 9298 9299 /* 9300 * Note: this simple function is not inlined to make it easier to dtrace the 9301 * amount of time spent syncing frees. 9302 */ 9303 static void 9304 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 9305 { 9306 zio_t *zio = zio_root(spa, NULL, NULL, 0); 9307 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 9308 VERIFY(zio_wait(zio) == 0); 9309 } 9310 9311 /* 9312 * Note: this simple function is not inlined to make it easier to dtrace the 9313 * amount of time spent syncing deferred frees. 9314 */ 9315 static void 9316 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 9317 { 9318 if (spa_sync_pass(spa) != 1) 9319 return; 9320 9321 /* 9322 * Note: 9323 * If the log space map feature is active, we stop deferring 9324 * frees to the next TXG and therefore running this function 9325 * would be considered a no-op as spa_deferred_bpobj should 9326 * not have any entries. 9327 * 9328 * That said we run this function anyway (instead of returning 9329 * immediately) for the edge-case scenario where we just 9330 * activated the log space map feature in this TXG but we have 9331 * deferred frees from the previous TXG. 9332 */ 9333 zio_t *zio = zio_root(spa, NULL, NULL, 0); 9334 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 9335 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 9336 VERIFY0(zio_wait(zio)); 9337 } 9338 9339 static void 9340 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 9341 { 9342 char *packed = NULL; 9343 size_t bufsize; 9344 size_t nvsize = 0; 9345 dmu_buf_t *db; 9346 9347 VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); 9348 9349 /* 9350 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 9351 * information. This avoids the dmu_buf_will_dirty() path and 9352 * saves us a pre-read to get data we don't actually care about. 9353 */ 9354 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 9355 packed = vmem_alloc(bufsize, KM_SLEEP); 9356 9357 VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 9358 KM_SLEEP) == 0); 9359 memset(packed + nvsize, 0, bufsize - nvsize); 9360 9361 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); 9362 9363 vmem_free(packed, bufsize); 9364 9365 VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 9366 dmu_buf_will_dirty(db, tx); 9367 *(uint64_t *)db->db_data = nvsize; 9368 dmu_buf_rele(db, FTAG); 9369 } 9370 9371 static void 9372 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 9373 const char *config, const char *entry) 9374 { 9375 nvlist_t *nvroot; 9376 nvlist_t **list; 9377 int i; 9378 9379 if (!sav->sav_sync) 9380 return; 9381 9382 /* 9383 * Update the MOS nvlist describing the list of available devices. 9384 * spa_validate_aux() will have already made sure this nvlist is 9385 * valid and the vdevs are labeled appropriately. 9386 */ 9387 if (sav->sav_object == 0) { 9388 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 9389 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 9390 sizeof (uint64_t), tx); 9391 VERIFY(zap_update(spa->spa_meta_objset, 9392 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 9393 &sav->sav_object, tx) == 0); 9394 } 9395 9396 nvroot = fnvlist_alloc(); 9397 if (sav->sav_count == 0) { 9398 fnvlist_add_nvlist_array(nvroot, config, 9399 (const nvlist_t * const *)NULL, 0); 9400 } else { 9401 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 9402 for (i = 0; i < sav->sav_count; i++) 9403 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 9404 B_FALSE, VDEV_CONFIG_L2CACHE); 9405 fnvlist_add_nvlist_array(nvroot, config, 9406 (const nvlist_t * const *)list, sav->sav_count); 9407 for (i = 0; i < sav->sav_count; i++) 9408 nvlist_free(list[i]); 9409 kmem_free(list, sav->sav_count * sizeof (void *)); 9410 } 9411 9412 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 9413 nvlist_free(nvroot); 9414 9415 sav->sav_sync = B_FALSE; 9416 } 9417 9418 /* 9419 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 9420 * The all-vdev ZAP must be empty. 9421 */ 9422 static void 9423 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 9424 { 9425 spa_t *spa = vd->vdev_spa; 9426 9427 if (vd->vdev_root_zap != 0 && 9428 spa_feature_is_active(spa, SPA_FEATURE_AVZ_V2)) { 9429 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 9430 vd->vdev_root_zap, tx)); 9431 } 9432 if (vd->vdev_top_zap != 0) { 9433 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 9434 vd->vdev_top_zap, tx)); 9435 } 9436 if (vd->vdev_leaf_zap != 0) { 9437 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 9438 vd->vdev_leaf_zap, tx)); 9439 } 9440 for (uint64_t i = 0; i < vd->vdev_children; i++) { 9441 spa_avz_build(vd->vdev_child[i], avz, tx); 9442 } 9443 } 9444 9445 static void 9446 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 9447 { 9448 nvlist_t *config; 9449 9450 /* 9451 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 9452 * its config may not be dirty but we still need to build per-vdev ZAPs. 9453 * Similarly, if the pool is being assembled (e.g. after a split), we 9454 * need to rebuild the AVZ although the config may not be dirty. 9455 */ 9456 if (list_is_empty(&spa->spa_config_dirty_list) && 9457 spa->spa_avz_action == AVZ_ACTION_NONE) 9458 return; 9459 9460 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9461 9462 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 9463 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 9464 spa->spa_all_vdev_zaps != 0); 9465 9466 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 9467 /* Make and build the new AVZ */ 9468 uint64_t new_avz = zap_create(spa->spa_meta_objset, 9469 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 9470 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 9471 9472 /* Diff old AVZ with new one */ 9473 zap_cursor_t zc; 9474 zap_attribute_t *za = zap_attribute_alloc(); 9475 9476 for (zap_cursor_init(&zc, spa->spa_meta_objset, 9477 spa->spa_all_vdev_zaps); 9478 zap_cursor_retrieve(&zc, za) == 0; 9479 zap_cursor_advance(&zc)) { 9480 uint64_t vdzap = za->za_first_integer; 9481 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 9482 vdzap) == ENOENT) { 9483 /* 9484 * ZAP is listed in old AVZ but not in new one; 9485 * destroy it 9486 */ 9487 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 9488 tx)); 9489 } 9490 } 9491 9492 zap_cursor_fini(&zc); 9493 zap_attribute_free(za); 9494 9495 /* Destroy the old AVZ */ 9496 VERIFY0(zap_destroy(spa->spa_meta_objset, 9497 spa->spa_all_vdev_zaps, tx)); 9498 9499 /* Replace the old AVZ in the dir obj with the new one */ 9500 VERIFY0(zap_update(spa->spa_meta_objset, 9501 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 9502 sizeof (new_avz), 1, &new_avz, tx)); 9503 9504 spa->spa_all_vdev_zaps = new_avz; 9505 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 9506 zap_cursor_t zc; 9507 zap_attribute_t *za = zap_attribute_alloc(); 9508 9509 /* Walk through the AVZ and destroy all listed ZAPs */ 9510 for (zap_cursor_init(&zc, spa->spa_meta_objset, 9511 spa->spa_all_vdev_zaps); 9512 zap_cursor_retrieve(&zc, za) == 0; 9513 zap_cursor_advance(&zc)) { 9514 uint64_t zap = za->za_first_integer; 9515 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 9516 } 9517 9518 zap_cursor_fini(&zc); 9519 zap_attribute_free(za); 9520 9521 /* Destroy and unlink the AVZ itself */ 9522 VERIFY0(zap_destroy(spa->spa_meta_objset, 9523 spa->spa_all_vdev_zaps, tx)); 9524 VERIFY0(zap_remove(spa->spa_meta_objset, 9525 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 9526 spa->spa_all_vdev_zaps = 0; 9527 } 9528 9529 if (spa->spa_all_vdev_zaps == 0) { 9530 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 9531 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 9532 DMU_POOL_VDEV_ZAP_MAP, tx); 9533 } 9534 spa->spa_avz_action = AVZ_ACTION_NONE; 9535 9536 /* Create ZAPs for vdevs that don't have them. */ 9537 vdev_construct_zaps(spa->spa_root_vdev, tx); 9538 9539 config = spa_config_generate(spa, spa->spa_root_vdev, 9540 dmu_tx_get_txg(tx), B_FALSE); 9541 9542 /* 9543 * If we're upgrading the spa version then make sure that 9544 * the config object gets updated with the correct version. 9545 */ 9546 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 9547 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 9548 spa->spa_uberblock.ub_version); 9549 9550 spa_config_exit(spa, SCL_STATE, FTAG); 9551 9552 nvlist_free(spa->spa_config_syncing); 9553 spa->spa_config_syncing = config; 9554 9555 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 9556 } 9557 9558 static void 9559 spa_sync_version(void *arg, dmu_tx_t *tx) 9560 { 9561 uint64_t *versionp = arg; 9562 uint64_t version = *versionp; 9563 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 9564 9565 /* 9566 * Setting the version is special cased when first creating the pool. 9567 */ 9568 ASSERT(tx->tx_txg != TXG_INITIAL); 9569 9570 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 9571 ASSERT(version >= spa_version(spa)); 9572 9573 spa->spa_uberblock.ub_version = version; 9574 vdev_config_dirty(spa->spa_root_vdev); 9575 spa_history_log_internal(spa, "set", tx, "version=%lld", 9576 (longlong_t)version); 9577 } 9578 9579 /* 9580 * Set zpool properties. 9581 */ 9582 static void 9583 spa_sync_props(void *arg, dmu_tx_t *tx) 9584 { 9585 nvlist_t *nvp = arg; 9586 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 9587 objset_t *mos = spa->spa_meta_objset; 9588 nvpair_t *elem = NULL; 9589 9590 mutex_enter(&spa->spa_props_lock); 9591 9592 while ((elem = nvlist_next_nvpair(nvp, elem))) { 9593 uint64_t intval; 9594 const char *strval, *fname; 9595 zpool_prop_t prop; 9596 const char *propname; 9597 const char *elemname = nvpair_name(elem); 9598 zprop_type_t proptype; 9599 spa_feature_t fid; 9600 9601 switch (prop = zpool_name_to_prop(elemname)) { 9602 case ZPOOL_PROP_VERSION: 9603 intval = fnvpair_value_uint64(elem); 9604 /* 9605 * The version is synced separately before other 9606 * properties and should be correct by now. 9607 */ 9608 ASSERT3U(spa_version(spa), >=, intval); 9609 break; 9610 9611 case ZPOOL_PROP_ALTROOT: 9612 /* 9613 * 'altroot' is a non-persistent property. It should 9614 * have been set temporarily at creation or import time. 9615 */ 9616 ASSERT(spa->spa_root != NULL); 9617 break; 9618 9619 case ZPOOL_PROP_READONLY: 9620 case ZPOOL_PROP_CACHEFILE: 9621 /* 9622 * 'readonly' and 'cachefile' are also non-persistent 9623 * properties. 9624 */ 9625 break; 9626 case ZPOOL_PROP_COMMENT: 9627 strval = fnvpair_value_string(elem); 9628 if (spa->spa_comment != NULL) 9629 spa_strfree(spa->spa_comment); 9630 spa->spa_comment = spa_strdup(strval); 9631 /* 9632 * We need to dirty the configuration on all the vdevs 9633 * so that their labels get updated. We also need to 9634 * update the cache file to keep it in sync with the 9635 * MOS version. It's unnecessary to do this for pool 9636 * creation since the vdev's configuration has already 9637 * been dirtied. 9638 */ 9639 if (tx->tx_txg != TXG_INITIAL) { 9640 vdev_config_dirty(spa->spa_root_vdev); 9641 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 9642 } 9643 spa_history_log_internal(spa, "set", tx, 9644 "%s=%s", elemname, strval); 9645 break; 9646 case ZPOOL_PROP_COMPATIBILITY: 9647 strval = fnvpair_value_string(elem); 9648 if (spa->spa_compatibility != NULL) 9649 spa_strfree(spa->spa_compatibility); 9650 spa->spa_compatibility = spa_strdup(strval); 9651 /* 9652 * Dirty the configuration on vdevs as above. 9653 */ 9654 if (tx->tx_txg != TXG_INITIAL) { 9655 vdev_config_dirty(spa->spa_root_vdev); 9656 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 9657 } 9658 9659 spa_history_log_internal(spa, "set", tx, 9660 "%s=%s", nvpair_name(elem), strval); 9661 break; 9662 9663 case ZPOOL_PROP_INVAL: 9664 if (zpool_prop_feature(elemname)) { 9665 fname = strchr(elemname, '@') + 1; 9666 VERIFY0(zfeature_lookup_name(fname, &fid)); 9667 9668 spa_feature_enable(spa, fid, tx); 9669 spa_history_log_internal(spa, "set", tx, 9670 "%s=enabled", elemname); 9671 break; 9672 } else if (!zfs_prop_user(elemname)) { 9673 ASSERT(zpool_prop_feature(elemname)); 9674 break; 9675 } 9676 zfs_fallthrough; 9677 default: 9678 /* 9679 * Set pool property values in the poolprops mos object. 9680 */ 9681 if (spa->spa_pool_props_object == 0) { 9682 spa->spa_pool_props_object = 9683 zap_create_link(mos, DMU_OT_POOL_PROPS, 9684 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 9685 tx); 9686 } 9687 9688 /* normalize the property name */ 9689 if (prop == ZPOOL_PROP_INVAL) { 9690 propname = elemname; 9691 proptype = PROP_TYPE_STRING; 9692 } else { 9693 propname = zpool_prop_to_name(prop); 9694 proptype = zpool_prop_get_type(prop); 9695 } 9696 9697 if (nvpair_type(elem) == DATA_TYPE_STRING) { 9698 ASSERT(proptype == PROP_TYPE_STRING); 9699 strval = fnvpair_value_string(elem); 9700 if (strlen(strval) == 0) { 9701 /* remove the property if value == "" */ 9702 (void) zap_remove(mos, 9703 spa->spa_pool_props_object, 9704 propname, tx); 9705 } else { 9706 VERIFY0(zap_update(mos, 9707 spa->spa_pool_props_object, 9708 propname, 1, strlen(strval) + 1, 9709 strval, tx)); 9710 } 9711 spa_history_log_internal(spa, "set", tx, 9712 "%s=%s", elemname, strval); 9713 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 9714 intval = fnvpair_value_uint64(elem); 9715 9716 if (proptype == PROP_TYPE_INDEX) { 9717 const char *unused; 9718 VERIFY0(zpool_prop_index_to_string( 9719 prop, intval, &unused)); 9720 } 9721 VERIFY0(zap_update(mos, 9722 spa->spa_pool_props_object, propname, 9723 8, 1, &intval, tx)); 9724 spa_history_log_internal(spa, "set", tx, 9725 "%s=%lld", elemname, 9726 (longlong_t)intval); 9727 9728 switch (prop) { 9729 case ZPOOL_PROP_DELEGATION: 9730 spa->spa_delegation = intval; 9731 break; 9732 case ZPOOL_PROP_BOOTFS: 9733 spa->spa_bootfs = intval; 9734 break; 9735 case ZPOOL_PROP_FAILUREMODE: 9736 spa->spa_failmode = intval; 9737 break; 9738 case ZPOOL_PROP_AUTOTRIM: 9739 spa->spa_autotrim = intval; 9740 spa_async_request(spa, 9741 SPA_ASYNC_AUTOTRIM_RESTART); 9742 break; 9743 case ZPOOL_PROP_AUTOEXPAND: 9744 spa->spa_autoexpand = intval; 9745 if (tx->tx_txg != TXG_INITIAL) 9746 spa_async_request(spa, 9747 SPA_ASYNC_AUTOEXPAND); 9748 break; 9749 case ZPOOL_PROP_MULTIHOST: 9750 spa->spa_multihost = intval; 9751 break; 9752 case ZPOOL_PROP_DEDUP_TABLE_QUOTA: 9753 spa->spa_dedup_table_quota = intval; 9754 break; 9755 default: 9756 break; 9757 } 9758 } else { 9759 ASSERT(0); /* not allowed */ 9760 } 9761 } 9762 9763 } 9764 9765 mutex_exit(&spa->spa_props_lock); 9766 } 9767 9768 /* 9769 * Perform one-time upgrade on-disk changes. spa_version() does not 9770 * reflect the new version this txg, so there must be no changes this 9771 * txg to anything that the upgrade code depends on after it executes. 9772 * Therefore this must be called after dsl_pool_sync() does the sync 9773 * tasks. 9774 */ 9775 static void 9776 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 9777 { 9778 if (spa_sync_pass(spa) != 1) 9779 return; 9780 9781 dsl_pool_t *dp = spa->spa_dsl_pool; 9782 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 9783 9784 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && 9785 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { 9786 dsl_pool_create_origin(dp, tx); 9787 9788 /* Keeping the origin open increases spa_minref */ 9789 spa->spa_minref += 3; 9790 } 9791 9792 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && 9793 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { 9794 dsl_pool_upgrade_clones(dp, tx); 9795 } 9796 9797 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && 9798 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { 9799 dsl_pool_upgrade_dir_clones(dp, tx); 9800 9801 /* Keeping the freedir open increases spa_minref */ 9802 spa->spa_minref += 3; 9803 } 9804 9805 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && 9806 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 9807 spa_feature_create_zap_objects(spa, tx); 9808 } 9809 9810 /* 9811 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 9812 * when possibility to use lz4 compression for metadata was added 9813 * Old pools that have this feature enabled must be upgraded to have 9814 * this feature active 9815 */ 9816 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 9817 boolean_t lz4_en = spa_feature_is_enabled(spa, 9818 SPA_FEATURE_LZ4_COMPRESS); 9819 boolean_t lz4_ac = spa_feature_is_active(spa, 9820 SPA_FEATURE_LZ4_COMPRESS); 9821 9822 if (lz4_en && !lz4_ac) 9823 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 9824 } 9825 9826 /* 9827 * If we haven't written the salt, do so now. Note that the 9828 * feature may not be activated yet, but that's fine since 9829 * the presence of this ZAP entry is backwards compatible. 9830 */ 9831 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 9832 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 9833 VERIFY0(zap_add(spa->spa_meta_objset, 9834 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 9835 sizeof (spa->spa_cksum_salt.zcs_bytes), 9836 spa->spa_cksum_salt.zcs_bytes, tx)); 9837 } 9838 9839 rrw_exit(&dp->dp_config_rwlock, FTAG); 9840 } 9841 9842 static void 9843 vdev_indirect_state_sync_verify(vdev_t *vd) 9844 { 9845 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 9846 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 9847 9848 if (vd->vdev_ops == &vdev_indirect_ops) { 9849 ASSERT(vim != NULL); 9850 ASSERT(vib != NULL); 9851 } 9852 9853 uint64_t obsolete_sm_object = 0; 9854 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 9855 if (obsolete_sm_object != 0) { 9856 ASSERT(vd->vdev_obsolete_sm != NULL); 9857 ASSERT(vd->vdev_removing || 9858 vd->vdev_ops == &vdev_indirect_ops); 9859 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 9860 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 9861 ASSERT3U(obsolete_sm_object, ==, 9862 space_map_object(vd->vdev_obsolete_sm)); 9863 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 9864 space_map_allocated(vd->vdev_obsolete_sm)); 9865 } 9866 ASSERT(vd->vdev_obsolete_segments != NULL); 9867 9868 /* 9869 * Since frees / remaps to an indirect vdev can only 9870 * happen in syncing context, the obsolete segments 9871 * tree must be empty when we start syncing. 9872 */ 9873 ASSERT0(zfs_range_tree_space(vd->vdev_obsolete_segments)); 9874 } 9875 9876 /* 9877 * Set the top-level vdev's max queue depth. Evaluate each top-level's 9878 * async write queue depth in case it changed. The max queue depth will 9879 * not change in the middle of syncing out this txg. 9880 */ 9881 static void 9882 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 9883 { 9884 ASSERT(spa_writeable(spa)); 9885 9886 metaslab_class_balance(spa_normal_class(spa), B_TRUE); 9887 metaslab_class_balance(spa_special_class(spa), B_TRUE); 9888 metaslab_class_balance(spa_dedup_class(spa), B_TRUE); 9889 } 9890 9891 static void 9892 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 9893 { 9894 ASSERT(spa_writeable(spa)); 9895 9896 vdev_t *rvd = spa->spa_root_vdev; 9897 for (int c = 0; c < rvd->vdev_children; c++) { 9898 vdev_t *vd = rvd->vdev_child[c]; 9899 vdev_indirect_state_sync_verify(vd); 9900 9901 if (vdev_indirect_should_condense(vd)) { 9902 spa_condense_indirect_start_sync(vd, tx); 9903 break; 9904 } 9905 } 9906 } 9907 9908 static void 9909 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 9910 { 9911 objset_t *mos = spa->spa_meta_objset; 9912 dsl_pool_t *dp = spa->spa_dsl_pool; 9913 uint64_t txg = tx->tx_txg; 9914 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 9915 9916 do { 9917 int pass = ++spa->spa_sync_pass; 9918 9919 spa_sync_config_object(spa, tx); 9920 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 9921 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 9922 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 9923 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 9924 spa_errlog_sync(spa, txg); 9925 dsl_pool_sync(dp, txg); 9926 9927 if (pass < zfs_sync_pass_deferred_free || 9928 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 9929 /* 9930 * If the log space map feature is active we don't 9931 * care about deferred frees and the deferred bpobj 9932 * as the log space map should effectively have the 9933 * same results (i.e. appending only to one object). 9934 */ 9935 spa_sync_frees(spa, free_bpl, tx); 9936 } else { 9937 /* 9938 * We can not defer frees in pass 1, because 9939 * we sync the deferred frees later in pass 1. 9940 */ 9941 ASSERT3U(pass, >, 1); 9942 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 9943 &spa->spa_deferred_bpobj, tx); 9944 } 9945 9946 brt_sync(spa, txg); 9947 ddt_sync(spa, txg); 9948 dsl_scan_sync(dp, tx); 9949 dsl_errorscrub_sync(dp, tx); 9950 svr_sync(spa, tx); 9951 spa_sync_upgrades(spa, tx); 9952 9953 spa_flush_metaslabs(spa, tx); 9954 9955 vdev_t *vd = NULL; 9956 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 9957 != NULL) 9958 vdev_sync(vd, txg); 9959 9960 if (pass == 1) { 9961 /* 9962 * dsl_pool_sync() -> dp_sync_tasks may have dirtied 9963 * the config. If that happens, this txg should not 9964 * be a no-op. So we must sync the config to the MOS 9965 * before checking for no-op. 9966 * 9967 * Note that when the config is dirty, it will 9968 * be written to the MOS (i.e. the MOS will be 9969 * dirtied) every time we call spa_sync_config_object() 9970 * in this txg. Therefore we can't call this after 9971 * dsl_pool_sync() every pass, because it would 9972 * prevent us from converging, since we'd dirty 9973 * the MOS every pass. 9974 * 9975 * Sync tasks can only be processed in pass 1, so 9976 * there's no need to do this in later passes. 9977 */ 9978 spa_sync_config_object(spa, tx); 9979 } 9980 9981 /* 9982 * Note: We need to check if the MOS is dirty because we could 9983 * have marked the MOS dirty without updating the uberblock 9984 * (e.g. if we have sync tasks but no dirty user data). We need 9985 * to check the uberblock's rootbp because it is updated if we 9986 * have synced out dirty data (though in this case the MOS will 9987 * most likely also be dirty due to second order effects, we 9988 * don't want to rely on that here). 9989 */ 9990 if (pass == 1 && 9991 BP_GET_LOGICAL_BIRTH(&spa->spa_uberblock.ub_rootbp) < txg && 9992 !dmu_objset_is_dirty(mos, txg)) { 9993 /* 9994 * Nothing changed on the first pass, therefore this 9995 * TXG is a no-op. Avoid syncing deferred frees, so 9996 * that we can keep this TXG as a no-op. 9997 */ 9998 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9999 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 10000 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 10001 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 10002 break; 10003 } 10004 10005 spa_sync_deferred_frees(spa, tx); 10006 } while (dmu_objset_is_dirty(mos, txg)); 10007 } 10008 10009 /* 10010 * Rewrite the vdev configuration (which includes the uberblock) to 10011 * commit the transaction group. 10012 * 10013 * If there are no dirty vdevs, we sync the uberblock to a few random 10014 * top-level vdevs that are known to be visible in the config cache 10015 * (see spa_vdev_add() for a complete description). If there *are* dirty 10016 * vdevs, sync the uberblock to all vdevs. 10017 */ 10018 static void 10019 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 10020 { 10021 vdev_t *rvd = spa->spa_root_vdev; 10022 uint64_t txg = tx->tx_txg; 10023 10024 for (;;) { 10025 int error = 0; 10026 10027 /* 10028 * We hold SCL_STATE to prevent vdev open/close/etc. 10029 * while we're attempting to write the vdev labels. 10030 */ 10031 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 10032 10033 if (list_is_empty(&spa->spa_config_dirty_list)) { 10034 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 10035 int svdcount = 0; 10036 int children = rvd->vdev_children; 10037 int c0 = random_in_range(children); 10038 10039 for (int c = 0; c < children; c++) { 10040 vdev_t *vd = 10041 rvd->vdev_child[(c0 + c) % children]; 10042 10043 /* Stop when revisiting the first vdev */ 10044 if (c > 0 && svd[0] == vd) 10045 break; 10046 10047 if (vd->vdev_ms_array == 0 || 10048 vd->vdev_islog || 10049 !vdev_is_concrete(vd)) 10050 continue; 10051 10052 svd[svdcount++] = vd; 10053 if (svdcount == SPA_SYNC_MIN_VDEVS) 10054 break; 10055 } 10056 error = vdev_config_sync(svd, svdcount, txg); 10057 } else { 10058 error = vdev_config_sync(rvd->vdev_child, 10059 rvd->vdev_children, txg); 10060 } 10061 10062 if (error == 0) 10063 spa->spa_last_synced_guid = rvd->vdev_guid; 10064 10065 spa_config_exit(spa, SCL_STATE, FTAG); 10066 10067 if (error == 0) 10068 break; 10069 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 10070 zio_resume_wait(spa); 10071 } 10072 } 10073 10074 /* 10075 * Sync the specified transaction group. New blocks may be dirtied as 10076 * part of the process, so we iterate until it converges. 10077 */ 10078 void 10079 spa_sync(spa_t *spa, uint64_t txg) 10080 { 10081 vdev_t *vd = NULL; 10082 10083 VERIFY(spa_writeable(spa)); 10084 10085 /* 10086 * Wait for i/os issued in open context that need to complete 10087 * before this txg syncs. 10088 */ 10089 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 10090 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 10091 ZIO_FLAG_CANFAIL); 10092 10093 /* 10094 * Now that there can be no more cloning in this transaction group, 10095 * but we are still before issuing frees, we can process pending BRT 10096 * updates. 10097 */ 10098 brt_pending_apply(spa, txg); 10099 10100 /* 10101 * Lock out configuration changes. 10102 */ 10103 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 10104 10105 spa->spa_syncing_txg = txg; 10106 spa->spa_sync_pass = 0; 10107 10108 /* 10109 * If there are any pending vdev state changes, convert them 10110 * into config changes that go out with this transaction group. 10111 */ 10112 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 10113 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 10114 /* Avoid holding the write lock unless actually necessary */ 10115 if (vd->vdev_aux == NULL) { 10116 vdev_state_clean(vd); 10117 vdev_config_dirty(vd); 10118 continue; 10119 } 10120 /* 10121 * We need the write lock here because, for aux vdevs, 10122 * calling vdev_config_dirty() modifies sav_config. 10123 * This is ugly and will become unnecessary when we 10124 * eliminate the aux vdev wart by integrating all vdevs 10125 * into the root vdev tree. 10126 */ 10127 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10128 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 10129 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 10130 vdev_state_clean(vd); 10131 vdev_config_dirty(vd); 10132 } 10133 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10134 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 10135 } 10136 spa_config_exit(spa, SCL_STATE, FTAG); 10137 10138 dsl_pool_t *dp = spa->spa_dsl_pool; 10139 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 10140 10141 spa->spa_sync_starttime = gethrtime(); 10142 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 10143 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 10144 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 10145 NSEC_TO_TICK(spa->spa_deadman_synctime)); 10146 10147 /* 10148 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 10149 * set spa_deflate if we have no raid-z vdevs. 10150 */ 10151 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 10152 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 10153 vdev_t *rvd = spa->spa_root_vdev; 10154 10155 int i; 10156 for (i = 0; i < rvd->vdev_children; i++) { 10157 vd = rvd->vdev_child[i]; 10158 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 10159 break; 10160 } 10161 if (i == rvd->vdev_children) { 10162 spa->spa_deflate = TRUE; 10163 VERIFY0(zap_add(spa->spa_meta_objset, 10164 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 10165 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 10166 } 10167 } 10168 10169 spa_sync_adjust_vdev_max_queue_depth(spa); 10170 10171 spa_sync_condense_indirect(spa, tx); 10172 10173 spa_sync_iterate_to_convergence(spa, tx); 10174 10175 #ifdef ZFS_DEBUG 10176 if (!list_is_empty(&spa->spa_config_dirty_list)) { 10177 /* 10178 * Make sure that the number of ZAPs for all the vdevs matches 10179 * the number of ZAPs in the per-vdev ZAP list. This only gets 10180 * called if the config is dirty; otherwise there may be 10181 * outstanding AVZ operations that weren't completed in 10182 * spa_sync_config_object. 10183 */ 10184 uint64_t all_vdev_zap_entry_count; 10185 ASSERT0(zap_count(spa->spa_meta_objset, 10186 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 10187 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 10188 all_vdev_zap_entry_count); 10189 } 10190 #endif 10191 10192 if (spa->spa_vdev_removal != NULL) { 10193 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 10194 } 10195 10196 spa_sync_rewrite_vdev_config(spa, tx); 10197 dmu_tx_commit(tx); 10198 10199 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 10200 spa->spa_deadman_tqid = 0; 10201 10202 /* 10203 * Clear the dirty config list. 10204 */ 10205 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 10206 vdev_config_clean(vd); 10207 10208 /* 10209 * Now that the new config has synced transactionally, 10210 * let it become visible to the config cache. 10211 */ 10212 if (spa->spa_config_syncing != NULL) { 10213 spa_config_set(spa, spa->spa_config_syncing); 10214 spa->spa_config_txg = txg; 10215 spa->spa_config_syncing = NULL; 10216 } 10217 10218 dsl_pool_sync_done(dp, txg); 10219 10220 /* 10221 * Update usable space statistics. 10222 */ 10223 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 10224 != NULL) 10225 vdev_sync_done(vd, txg); 10226 10227 metaslab_class_evict_old(spa->spa_normal_class, txg); 10228 metaslab_class_evict_old(spa->spa_log_class, txg); 10229 /* spa_embedded_log_class has only one metaslab per vdev. */ 10230 metaslab_class_evict_old(spa->spa_special_class, txg); 10231 metaslab_class_evict_old(spa->spa_dedup_class, txg); 10232 10233 spa_sync_close_syncing_log_sm(spa); 10234 10235 spa_update_dspace(spa); 10236 10237 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) 10238 vdev_autotrim_kick(spa); 10239 10240 /* 10241 * It had better be the case that we didn't dirty anything 10242 * since vdev_config_sync(). 10243 */ 10244 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 10245 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 10246 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 10247 10248 while (zfs_pause_spa_sync) 10249 delay(1); 10250 10251 spa->spa_sync_pass = 0; 10252 10253 /* 10254 * Update the last synced uberblock here. We want to do this at 10255 * the end of spa_sync() so that consumers of spa_last_synced_txg() 10256 * will be guaranteed that all the processing associated with 10257 * that txg has been completed. 10258 */ 10259 spa->spa_ubsync = spa->spa_uberblock; 10260 spa_config_exit(spa, SCL_CONFIG, FTAG); 10261 10262 spa_handle_ignored_writes(spa); 10263 10264 /* 10265 * If any async tasks have been requested, kick them off. 10266 */ 10267 spa_async_dispatch(spa); 10268 } 10269 10270 /* 10271 * Sync all pools. We don't want to hold the namespace lock across these 10272 * operations, so we take a reference on the spa_t and drop the lock during the 10273 * sync. 10274 */ 10275 void 10276 spa_sync_allpools(void) 10277 { 10278 spa_t *spa = NULL; 10279 mutex_enter(&spa_namespace_lock); 10280 while ((spa = spa_next(spa)) != NULL) { 10281 if (spa_state(spa) != POOL_STATE_ACTIVE || 10282 !spa_writeable(spa) || spa_suspended(spa)) 10283 continue; 10284 spa_open_ref(spa, FTAG); 10285 mutex_exit(&spa_namespace_lock); 10286 txg_wait_synced(spa_get_dsl(spa), 0); 10287 mutex_enter(&spa_namespace_lock); 10288 spa_close(spa, FTAG); 10289 } 10290 mutex_exit(&spa_namespace_lock); 10291 } 10292 10293 taskq_t * 10294 spa_sync_tq_create(spa_t *spa, const char *name) 10295 { 10296 kthread_t **kthreads; 10297 10298 ASSERT(spa->spa_sync_tq == NULL); 10299 ASSERT3S(spa->spa_alloc_count, <=, boot_ncpus); 10300 10301 /* 10302 * - do not allow more allocators than cpus. 10303 * - there may be more cpus than allocators. 10304 * - do not allow more sync taskq threads than allocators or cpus. 10305 */ 10306 int nthreads = spa->spa_alloc_count; 10307 spa->spa_syncthreads = kmem_zalloc(sizeof (spa_syncthread_info_t) * 10308 nthreads, KM_SLEEP); 10309 10310 spa->spa_sync_tq = taskq_create_synced(name, nthreads, minclsyspri, 10311 nthreads, INT_MAX, TASKQ_PREPOPULATE, &kthreads); 10312 VERIFY(spa->spa_sync_tq != NULL); 10313 VERIFY(kthreads != NULL); 10314 10315 spa_syncthread_info_t *ti = spa->spa_syncthreads; 10316 for (int i = 0; i < nthreads; i++, ti++) { 10317 ti->sti_thread = kthreads[i]; 10318 ti->sti_allocator = i; 10319 } 10320 10321 kmem_free(kthreads, sizeof (*kthreads) * nthreads); 10322 return (spa->spa_sync_tq); 10323 } 10324 10325 void 10326 spa_sync_tq_destroy(spa_t *spa) 10327 { 10328 ASSERT(spa->spa_sync_tq != NULL); 10329 10330 taskq_wait(spa->spa_sync_tq); 10331 taskq_destroy(spa->spa_sync_tq); 10332 kmem_free(spa->spa_syncthreads, 10333 sizeof (spa_syncthread_info_t) * spa->spa_alloc_count); 10334 spa->spa_sync_tq = NULL; 10335 } 10336 10337 uint_t 10338 spa_acq_allocator(spa_t *spa) 10339 { 10340 int i; 10341 10342 if (spa->spa_alloc_count == 1) 10343 return (0); 10344 10345 mutex_enter(&spa->spa_allocs_use->sau_lock); 10346 uint_t r = spa->spa_allocs_use->sau_rotor; 10347 do { 10348 if (++r == spa->spa_alloc_count) 10349 r = 0; 10350 } while (spa->spa_allocs_use->sau_inuse[r]); 10351 spa->spa_allocs_use->sau_inuse[r] = B_TRUE; 10352 spa->spa_allocs_use->sau_rotor = r; 10353 mutex_exit(&spa->spa_allocs_use->sau_lock); 10354 10355 spa_syncthread_info_t *ti = spa->spa_syncthreads; 10356 for (i = 0; i < spa->spa_alloc_count; i++, ti++) { 10357 if (ti->sti_thread == curthread) { 10358 ti->sti_allocator = r; 10359 break; 10360 } 10361 } 10362 ASSERT3S(i, <, spa->spa_alloc_count); 10363 return (r); 10364 } 10365 10366 void 10367 spa_rel_allocator(spa_t *spa, uint_t allocator) 10368 { 10369 if (spa->spa_alloc_count > 1) 10370 spa->spa_allocs_use->sau_inuse[allocator] = B_FALSE; 10371 } 10372 10373 void 10374 spa_select_allocator(zio_t *zio) 10375 { 10376 zbookmark_phys_t *bm = &zio->io_bookmark; 10377 spa_t *spa = zio->io_spa; 10378 10379 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 10380 10381 /* 10382 * A gang block (for example) may have inherited its parent's 10383 * allocator, in which case there is nothing further to do here. 10384 */ 10385 if (ZIO_HAS_ALLOCATOR(zio)) 10386 return; 10387 10388 ASSERT(spa != NULL); 10389 ASSERT(bm != NULL); 10390 10391 /* 10392 * First try to use an allocator assigned to the syncthread, and set 10393 * the corresponding write issue taskq for the allocator. 10394 * Note, we must have an open pool to do this. 10395 */ 10396 if (spa->spa_sync_tq != NULL) { 10397 spa_syncthread_info_t *ti = spa->spa_syncthreads; 10398 for (int i = 0; i < spa->spa_alloc_count; i++, ti++) { 10399 if (ti->sti_thread == curthread) { 10400 zio->io_allocator = ti->sti_allocator; 10401 return; 10402 } 10403 } 10404 } 10405 10406 /* 10407 * We want to try to use as many allocators as possible to help improve 10408 * performance, but we also want logically adjacent IOs to be physically 10409 * adjacent to improve sequential read performance. We chunk each object 10410 * into 2^20 block regions, and then hash based on the objset, object, 10411 * level, and region to accomplish both of these goals. 10412 */ 10413 uint64_t hv = cityhash4(bm->zb_objset, bm->zb_object, bm->zb_level, 10414 bm->zb_blkid >> 20); 10415 10416 zio->io_allocator = (uint_t)hv % spa->spa_alloc_count; 10417 } 10418 10419 /* 10420 * ========================================================================== 10421 * Miscellaneous routines 10422 * ========================================================================== 10423 */ 10424 10425 /* 10426 * Remove all pools in the system. 10427 */ 10428 void 10429 spa_evict_all(void) 10430 { 10431 spa_t *spa; 10432 10433 /* 10434 * Remove all cached state. All pools should be closed now, 10435 * so every spa in the AVL tree should be unreferenced. 10436 */ 10437 mutex_enter(&spa_namespace_lock); 10438 while ((spa = spa_next(NULL)) != NULL) { 10439 /* 10440 * Stop async tasks. The async thread may need to detach 10441 * a device that's been replaced, which requires grabbing 10442 * spa_namespace_lock, so we must drop it here. 10443 */ 10444 spa_open_ref(spa, FTAG); 10445 mutex_exit(&spa_namespace_lock); 10446 spa_async_suspend(spa); 10447 mutex_enter(&spa_namespace_lock); 10448 spa_close(spa, FTAG); 10449 10450 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 10451 spa_unload(spa); 10452 spa_deactivate(spa); 10453 } 10454 spa_remove(spa); 10455 } 10456 mutex_exit(&spa_namespace_lock); 10457 } 10458 10459 vdev_t * 10460 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 10461 { 10462 vdev_t *vd; 10463 int i; 10464 10465 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 10466 return (vd); 10467 10468 if (aux) { 10469 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 10470 vd = spa->spa_l2cache.sav_vdevs[i]; 10471 if (vd->vdev_guid == guid) 10472 return (vd); 10473 } 10474 10475 for (i = 0; i < spa->spa_spares.sav_count; i++) { 10476 vd = spa->spa_spares.sav_vdevs[i]; 10477 if (vd->vdev_guid == guid) 10478 return (vd); 10479 } 10480 } 10481 10482 return (NULL); 10483 } 10484 10485 void 10486 spa_upgrade(spa_t *spa, uint64_t version) 10487 { 10488 ASSERT(spa_writeable(spa)); 10489 10490 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 10491 10492 /* 10493 * This should only be called for a non-faulted pool, and since a 10494 * future version would result in an unopenable pool, this shouldn't be 10495 * possible. 10496 */ 10497 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 10498 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 10499 10500 spa->spa_uberblock.ub_version = version; 10501 vdev_config_dirty(spa->spa_root_vdev); 10502 10503 spa_config_exit(spa, SCL_ALL, FTAG); 10504 10505 txg_wait_synced(spa_get_dsl(spa), 0); 10506 } 10507 10508 static boolean_t 10509 spa_has_aux_vdev(spa_t *spa, uint64_t guid, spa_aux_vdev_t *sav) 10510 { 10511 (void) spa; 10512 int i; 10513 uint64_t vdev_guid; 10514 10515 for (i = 0; i < sav->sav_count; i++) 10516 if (sav->sav_vdevs[i]->vdev_guid == guid) 10517 return (B_TRUE); 10518 10519 for (i = 0; i < sav->sav_npending; i++) { 10520 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 10521 &vdev_guid) == 0 && vdev_guid == guid) 10522 return (B_TRUE); 10523 } 10524 10525 return (B_FALSE); 10526 } 10527 10528 boolean_t 10529 spa_has_l2cache(spa_t *spa, uint64_t guid) 10530 { 10531 return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache)); 10532 } 10533 10534 boolean_t 10535 spa_has_spare(spa_t *spa, uint64_t guid) 10536 { 10537 return (spa_has_aux_vdev(spa, guid, &spa->spa_spares)); 10538 } 10539 10540 /* 10541 * Check if a pool has an active shared spare device. 10542 * Note: reference count of an active spare is 2, as a spare and as a replace 10543 */ 10544 static boolean_t 10545 spa_has_active_shared_spare(spa_t *spa) 10546 { 10547 int i, refcnt; 10548 uint64_t pool; 10549 spa_aux_vdev_t *sav = &spa->spa_spares; 10550 10551 for (i = 0; i < sav->sav_count; i++) { 10552 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 10553 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 10554 refcnt > 2) 10555 return (B_TRUE); 10556 } 10557 10558 return (B_FALSE); 10559 } 10560 10561 uint64_t 10562 spa_total_metaslabs(spa_t *spa) 10563 { 10564 vdev_t *rvd = spa->spa_root_vdev; 10565 10566 uint64_t m = 0; 10567 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 10568 vdev_t *vd = rvd->vdev_child[c]; 10569 if (!vdev_is_concrete(vd)) 10570 continue; 10571 m += vd->vdev_ms_count; 10572 } 10573 return (m); 10574 } 10575 10576 /* 10577 * Notify any waiting threads that some activity has switched from being in- 10578 * progress to not-in-progress so that the thread can wake up and determine 10579 * whether it is finished waiting. 10580 */ 10581 void 10582 spa_notify_waiters(spa_t *spa) 10583 { 10584 /* 10585 * Acquiring spa_activities_lock here prevents the cv_broadcast from 10586 * happening between the waiting thread's check and cv_wait. 10587 */ 10588 mutex_enter(&spa->spa_activities_lock); 10589 cv_broadcast(&spa->spa_activities_cv); 10590 mutex_exit(&spa->spa_activities_lock); 10591 } 10592 10593 /* 10594 * Notify any waiting threads that the pool is exporting, and then block until 10595 * they are finished using the spa_t. 10596 */ 10597 void 10598 spa_wake_waiters(spa_t *spa) 10599 { 10600 mutex_enter(&spa->spa_activities_lock); 10601 spa->spa_waiters_cancel = B_TRUE; 10602 cv_broadcast(&spa->spa_activities_cv); 10603 while (spa->spa_waiters != 0) 10604 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 10605 spa->spa_waiters_cancel = B_FALSE; 10606 mutex_exit(&spa->spa_activities_lock); 10607 } 10608 10609 /* Whether the vdev or any of its descendants are being initialized/trimmed. */ 10610 static boolean_t 10611 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 10612 { 10613 spa_t *spa = vd->vdev_spa; 10614 10615 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 10616 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 10617 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 10618 activity == ZPOOL_WAIT_TRIM); 10619 10620 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 10621 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 10622 10623 mutex_exit(&spa->spa_activities_lock); 10624 mutex_enter(lock); 10625 mutex_enter(&spa->spa_activities_lock); 10626 10627 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 10628 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) : 10629 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE); 10630 mutex_exit(lock); 10631 10632 if (in_progress) 10633 return (B_TRUE); 10634 10635 for (int i = 0; i < vd->vdev_children; i++) { 10636 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 10637 activity)) 10638 return (B_TRUE); 10639 } 10640 10641 return (B_FALSE); 10642 } 10643 10644 /* 10645 * If use_guid is true, this checks whether the vdev specified by guid is 10646 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 10647 * is being initialized/trimmed. The caller must hold the config lock and 10648 * spa_activities_lock. 10649 */ 10650 static int 10651 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 10652 zpool_wait_activity_t activity, boolean_t *in_progress) 10653 { 10654 mutex_exit(&spa->spa_activities_lock); 10655 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 10656 mutex_enter(&spa->spa_activities_lock); 10657 10658 vdev_t *vd; 10659 if (use_guid) { 10660 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 10661 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 10662 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10663 return (EINVAL); 10664 } 10665 } else { 10666 vd = spa->spa_root_vdev; 10667 } 10668 10669 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 10670 10671 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10672 return (0); 10673 } 10674 10675 /* 10676 * Locking for waiting threads 10677 * --------------------------- 10678 * 10679 * Waiting threads need a way to check whether a given activity is in progress, 10680 * and then, if it is, wait for it to complete. Each activity will have some 10681 * in-memory representation of the relevant on-disk state which can be used to 10682 * determine whether or not the activity is in progress. The in-memory state and 10683 * the locking used to protect it will be different for each activity, and may 10684 * not be suitable for use with a cvar (e.g., some state is protected by the 10685 * config lock). To allow waiting threads to wait without any races, another 10686 * lock, spa_activities_lock, is used. 10687 * 10688 * When the state is checked, both the activity-specific lock (if there is one) 10689 * and spa_activities_lock are held. In some cases, the activity-specific lock 10690 * is acquired explicitly (e.g. the config lock). In others, the locking is 10691 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 10692 * thread releases the activity-specific lock and, if the activity is in 10693 * progress, then cv_waits using spa_activities_lock. 10694 * 10695 * The waiting thread is woken when another thread, one completing some 10696 * activity, updates the state of the activity and then calls 10697 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 10698 * needs to hold its activity-specific lock when updating the state, and this 10699 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 10700 * 10701 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 10702 * and because it is held when the waiting thread checks the state of the 10703 * activity, it can never be the case that the completing thread both updates 10704 * the activity state and cv_broadcasts in between the waiting thread's check 10705 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 10706 * 10707 * In order to prevent deadlock, when the waiting thread does its check, in some 10708 * cases it will temporarily drop spa_activities_lock in order to acquire the 10709 * activity-specific lock. The order in which spa_activities_lock and the 10710 * activity specific lock are acquired in the waiting thread is determined by 10711 * the order in which they are acquired in the completing thread; if the 10712 * completing thread calls spa_notify_waiters with the activity-specific lock 10713 * held, then the waiting thread must also acquire the activity-specific lock 10714 * first. 10715 */ 10716 10717 static int 10718 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 10719 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 10720 { 10721 int error = 0; 10722 10723 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 10724 10725 switch (activity) { 10726 case ZPOOL_WAIT_CKPT_DISCARD: 10727 *in_progress = 10728 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 10729 zap_contains(spa_meta_objset(spa), 10730 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 10731 ENOENT); 10732 break; 10733 case ZPOOL_WAIT_FREE: 10734 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 10735 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 10736 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 10737 spa_livelist_delete_check(spa)); 10738 break; 10739 case ZPOOL_WAIT_INITIALIZE: 10740 case ZPOOL_WAIT_TRIM: 10741 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 10742 activity, in_progress); 10743 break; 10744 case ZPOOL_WAIT_REPLACE: 10745 mutex_exit(&spa->spa_activities_lock); 10746 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 10747 mutex_enter(&spa->spa_activities_lock); 10748 10749 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 10750 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10751 break; 10752 case ZPOOL_WAIT_REMOVE: 10753 *in_progress = (spa->spa_removing_phys.sr_state == 10754 DSS_SCANNING); 10755 break; 10756 case ZPOOL_WAIT_RESILVER: 10757 *in_progress = vdev_rebuild_active(spa->spa_root_vdev); 10758 if (*in_progress) 10759 break; 10760 zfs_fallthrough; 10761 case ZPOOL_WAIT_SCRUB: 10762 { 10763 boolean_t scanning, paused, is_scrub; 10764 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 10765 10766 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 10767 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 10768 paused = dsl_scan_is_paused_scrub(scn); 10769 *in_progress = (scanning && !paused && 10770 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 10771 break; 10772 } 10773 case ZPOOL_WAIT_RAIDZ_EXPAND: 10774 { 10775 vdev_raidz_expand_t *vre = spa->spa_raidz_expand; 10776 *in_progress = (vre != NULL && vre->vre_state == DSS_SCANNING); 10777 break; 10778 } 10779 default: 10780 panic("unrecognized value for activity %d", activity); 10781 } 10782 10783 return (error); 10784 } 10785 10786 static int 10787 spa_wait_common(const char *pool, zpool_wait_activity_t activity, 10788 boolean_t use_tag, uint64_t tag, boolean_t *waited) 10789 { 10790 /* 10791 * The tag is used to distinguish between instances of an activity. 10792 * 'initialize' and 'trim' are the only activities that we use this for. 10793 * The other activities can only have a single instance in progress in a 10794 * pool at one time, making the tag unnecessary. 10795 * 10796 * There can be multiple devices being replaced at once, but since they 10797 * all finish once resilvering finishes, we don't bother keeping track 10798 * of them individually, we just wait for them all to finish. 10799 */ 10800 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 10801 activity != ZPOOL_WAIT_TRIM) 10802 return (EINVAL); 10803 10804 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 10805 return (EINVAL); 10806 10807 spa_t *spa; 10808 int error = spa_open(pool, &spa, FTAG); 10809 if (error != 0) 10810 return (error); 10811 10812 /* 10813 * Increment the spa's waiter count so that we can call spa_close and 10814 * still ensure that the spa_t doesn't get freed before this thread is 10815 * finished with it when the pool is exported. We want to call spa_close 10816 * before we start waiting because otherwise the additional ref would 10817 * prevent the pool from being exported or destroyed throughout the 10818 * potentially long wait. 10819 */ 10820 mutex_enter(&spa->spa_activities_lock); 10821 spa->spa_waiters++; 10822 spa_close(spa, FTAG); 10823 10824 *waited = B_FALSE; 10825 for (;;) { 10826 boolean_t in_progress; 10827 error = spa_activity_in_progress(spa, activity, use_tag, tag, 10828 &in_progress); 10829 10830 if (error || !in_progress || spa->spa_waiters_cancel) 10831 break; 10832 10833 *waited = B_TRUE; 10834 10835 if (cv_wait_sig(&spa->spa_activities_cv, 10836 &spa->spa_activities_lock) == 0) { 10837 error = EINTR; 10838 break; 10839 } 10840 } 10841 10842 spa->spa_waiters--; 10843 cv_signal(&spa->spa_waiters_cv); 10844 mutex_exit(&spa->spa_activities_lock); 10845 10846 return (error); 10847 } 10848 10849 /* 10850 * Wait for a particular instance of the specified activity to complete, where 10851 * the instance is identified by 'tag' 10852 */ 10853 int 10854 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 10855 boolean_t *waited) 10856 { 10857 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 10858 } 10859 10860 /* 10861 * Wait for all instances of the specified activity complete 10862 */ 10863 int 10864 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 10865 { 10866 10867 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 10868 } 10869 10870 sysevent_t * 10871 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 10872 { 10873 sysevent_t *ev = NULL; 10874 #ifdef _KERNEL 10875 nvlist_t *resource; 10876 10877 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 10878 if (resource) { 10879 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 10880 ev->resource = resource; 10881 } 10882 #else 10883 (void) spa, (void) vd, (void) hist_nvl, (void) name; 10884 #endif 10885 return (ev); 10886 } 10887 10888 void 10889 spa_event_post(sysevent_t *ev) 10890 { 10891 #ifdef _KERNEL 10892 if (ev) { 10893 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 10894 kmem_free(ev, sizeof (*ev)); 10895 } 10896 #else 10897 (void) ev; 10898 #endif 10899 } 10900 10901 /* 10902 * Post a zevent corresponding to the given sysevent. The 'name' must be one 10903 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 10904 * filled in from the spa and (optionally) the vdev. This doesn't do anything 10905 * in the userland libzpool, as we don't want consumers to misinterpret ztest 10906 * or zdb as real changes. 10907 */ 10908 void 10909 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 10910 { 10911 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 10912 } 10913 10914 /* state manipulation functions */ 10915 EXPORT_SYMBOL(spa_open); 10916 EXPORT_SYMBOL(spa_open_rewind); 10917 EXPORT_SYMBOL(spa_get_stats); 10918 EXPORT_SYMBOL(spa_create); 10919 EXPORT_SYMBOL(spa_import); 10920 EXPORT_SYMBOL(spa_tryimport); 10921 EXPORT_SYMBOL(spa_destroy); 10922 EXPORT_SYMBOL(spa_export); 10923 EXPORT_SYMBOL(spa_reset); 10924 EXPORT_SYMBOL(spa_async_request); 10925 EXPORT_SYMBOL(spa_async_suspend); 10926 EXPORT_SYMBOL(spa_async_resume); 10927 EXPORT_SYMBOL(spa_inject_addref); 10928 EXPORT_SYMBOL(spa_inject_delref); 10929 EXPORT_SYMBOL(spa_scan_stat_init); 10930 EXPORT_SYMBOL(spa_scan_get_stats); 10931 10932 /* device manipulation */ 10933 EXPORT_SYMBOL(spa_vdev_add); 10934 EXPORT_SYMBOL(spa_vdev_attach); 10935 EXPORT_SYMBOL(spa_vdev_detach); 10936 EXPORT_SYMBOL(spa_vdev_setpath); 10937 EXPORT_SYMBOL(spa_vdev_setfru); 10938 EXPORT_SYMBOL(spa_vdev_split_mirror); 10939 10940 /* spare statech is global across all pools) */ 10941 EXPORT_SYMBOL(spa_spare_add); 10942 EXPORT_SYMBOL(spa_spare_remove); 10943 EXPORT_SYMBOL(spa_spare_exists); 10944 EXPORT_SYMBOL(spa_spare_activate); 10945 10946 /* L2ARC statech is global across all pools) */ 10947 EXPORT_SYMBOL(spa_l2cache_add); 10948 EXPORT_SYMBOL(spa_l2cache_remove); 10949 EXPORT_SYMBOL(spa_l2cache_exists); 10950 EXPORT_SYMBOL(spa_l2cache_activate); 10951 EXPORT_SYMBOL(spa_l2cache_drop); 10952 10953 /* scanning */ 10954 EXPORT_SYMBOL(spa_scan); 10955 EXPORT_SYMBOL(spa_scan_range); 10956 EXPORT_SYMBOL(spa_scan_stop); 10957 10958 /* spa syncing */ 10959 EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 10960 EXPORT_SYMBOL(spa_sync_allpools); 10961 10962 /* properties */ 10963 EXPORT_SYMBOL(spa_prop_set); 10964 EXPORT_SYMBOL(spa_prop_get); 10965 EXPORT_SYMBOL(spa_prop_clear_bootfs); 10966 10967 /* asynchronous event notification */ 10968 EXPORT_SYMBOL(spa_event_notify); 10969 10970 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_pct, UINT, ZMOD_RW, 10971 "Percentage of CPUs to run a metaslab preload taskq"); 10972 10973 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, UINT, ZMOD_RW, 10974 "log2 fraction of arc that can be used by inflight I/Os when " 10975 "verifying pool during import"); 10976 10977 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 10978 "Set to traverse metadata on pool import"); 10979 10980 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 10981 "Set to traverse data on pool import"); 10982 10983 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 10984 "Print vdev tree to zfs_dbgmsg during pool import"); 10985 10986 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RW, 10987 "Percentage of CPUs to run an IO worker thread"); 10988 10989 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RW, 10990 "Number of threads per IO worker taskqueue"); 10991 10992 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, U64, ZMOD_RW, 10993 "Allow importing pool with up to this number of missing top-level " 10994 "vdevs (in read-only mode)"); 10995 10996 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, 10997 ZMOD_RW, "Set the livelist condense zthr to pause"); 10998 10999 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, 11000 ZMOD_RW, "Set the livelist condense synctask to pause"); 11001 11002 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, 11003 INT, ZMOD_RW, 11004 "Whether livelist condensing was canceled in the synctask"); 11005 11006 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, 11007 INT, ZMOD_RW, 11008 "Whether livelist condensing was canceled in the zthr function"); 11009 11010 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, 11011 ZMOD_RW, 11012 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 11013 "was being condensed"); 11014 11015 #ifdef _KERNEL 11016 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_read, 11017 spa_taskq_read_param_set, spa_taskq_read_param_get, ZMOD_RW, 11018 "Configure IO queues for read IO"); 11019 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_write, 11020 spa_taskq_write_param_set, spa_taskq_write_param_get, ZMOD_RW, 11021 "Configure IO queues for write IO"); 11022 #endif 11023 11024 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_write_tpq, UINT, ZMOD_RW, 11025 "Number of CPUs per write issue taskq"); 11026