1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 * Copyright 2016 Toomas Soome <tsoome@me.com> 30 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 31 * Copyright 2018 Joyent, Inc. 32 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 33 * Copyright 2017 Joyent, Inc. 34 * Copyright (c) 2017, Intel Corporation. 35 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 36 */ 37 38 /* 39 * SPA: Storage Pool Allocator 40 * 41 * This file contains all the routines used when modifying on-disk SPA state. 42 * This includes opening, importing, destroying, exporting a pool, and syncing a 43 * pool. 44 */ 45 46 #include <sys/zfs_context.h> 47 #include <sys/fm/fs/zfs.h> 48 #include <sys/spa_impl.h> 49 #include <sys/zio.h> 50 #include <sys/zio_checksum.h> 51 #include <sys/dmu.h> 52 #include <sys/dmu_tx.h> 53 #include <sys/zap.h> 54 #include <sys/zil.h> 55 #include <sys/brt.h> 56 #include <sys/ddt.h> 57 #include <sys/vdev_impl.h> 58 #include <sys/vdev_removal.h> 59 #include <sys/vdev_indirect_mapping.h> 60 #include <sys/vdev_indirect_births.h> 61 #include <sys/vdev_initialize.h> 62 #include <sys/vdev_rebuild.h> 63 #include <sys/vdev_trim.h> 64 #include <sys/vdev_disk.h> 65 #include <sys/vdev_draid.h> 66 #include <sys/metaslab.h> 67 #include <sys/metaslab_impl.h> 68 #include <sys/mmp.h> 69 #include <sys/uberblock_impl.h> 70 #include <sys/txg.h> 71 #include <sys/avl.h> 72 #include <sys/bpobj.h> 73 #include <sys/dmu_traverse.h> 74 #include <sys/dmu_objset.h> 75 #include <sys/unique.h> 76 #include <sys/dsl_pool.h> 77 #include <sys/dsl_dataset.h> 78 #include <sys/dsl_dir.h> 79 #include <sys/dsl_prop.h> 80 #include <sys/dsl_synctask.h> 81 #include <sys/fs/zfs.h> 82 #include <sys/arc.h> 83 #include <sys/callb.h> 84 #include <sys/systeminfo.h> 85 #include <sys/zfs_ioctl.h> 86 #include <sys/dsl_scan.h> 87 #include <sys/zfeature.h> 88 #include <sys/dsl_destroy.h> 89 #include <sys/zvol.h> 90 91 #ifdef _KERNEL 92 #include <sys/fm/protocol.h> 93 #include <sys/fm/util.h> 94 #include <sys/callb.h> 95 #include <sys/zone.h> 96 #include <sys/vmsystm.h> 97 #endif /* _KERNEL */ 98 99 #include "zfs_prop.h" 100 #include "zfs_comutil.h" 101 102 /* 103 * The interval, in seconds, at which failed configuration cache file writes 104 * should be retried. 105 */ 106 int zfs_ccw_retry_interval = 300; 107 108 typedef enum zti_modes { 109 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 110 ZTI_MODE_BATCH, /* cpu-intensive; value is ignored */ 111 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 112 ZTI_MODE_NULL, /* don't create a taskq */ 113 ZTI_NMODES 114 } zti_modes_t; 115 116 #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 117 #define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 118 #define ZTI_BATCH { ZTI_MODE_BATCH, 0, 1 } 119 #define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 } 120 #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 121 122 #define ZTI_N(n) ZTI_P(n, 1) 123 #define ZTI_ONE ZTI_N(1) 124 125 typedef struct zio_taskq_info { 126 zti_modes_t zti_mode; 127 uint_t zti_value; 128 uint_t zti_count; 129 } zio_taskq_info_t; 130 131 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 132 "iss", "iss_h", "int", "int_h" 133 }; 134 135 /* 136 * This table defines the taskq settings for each ZFS I/O type. When 137 * initializing a pool, we use this table to create an appropriately sized 138 * taskq. Some operations are low volume and therefore have a small, static 139 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 140 * macros. Other operations process a large amount of data; the ZTI_BATCH 141 * macro causes us to create a taskq oriented for throughput. Some operations 142 * are so high frequency and short-lived that the taskq itself can become a 143 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 144 * additional degree of parallelism specified by the number of threads per- 145 * taskq and the number of taskqs; when dispatching an event in this case, the 146 * particular taskq is chosen at random. ZTI_SCALE is similar to ZTI_BATCH, 147 * but with number of taskqs also scaling with number of CPUs. 148 * 149 * The different taskq priorities are to handle the different contexts (issue 150 * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that 151 * need to be handled with minimum delay. 152 */ 153 static const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 154 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 155 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 156 { ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */ 157 { ZTI_BATCH, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */ 158 { ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 159 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 160 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */ 161 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 162 }; 163 164 static void spa_sync_version(void *arg, dmu_tx_t *tx); 165 static void spa_sync_props(void *arg, dmu_tx_t *tx); 166 static boolean_t spa_has_active_shared_spare(spa_t *spa); 167 static int spa_load_impl(spa_t *spa, spa_import_type_t type, 168 const char **ereport); 169 static void spa_vdev_resilver_done(spa_t *spa); 170 171 static uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 172 static uint_t zio_taskq_batch_tpq; /* threads per taskq */ 173 static const boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 174 static const uint_t zio_taskq_basedc = 80; /* base duty cycle */ 175 176 static const boolean_t spa_create_process = B_TRUE; /* no process => no sysdc */ 177 178 /* 179 * Report any spa_load_verify errors found, but do not fail spa_load. 180 * This is used by zdb to analyze non-idle pools. 181 */ 182 boolean_t spa_load_verify_dryrun = B_FALSE; 183 184 /* 185 * Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ). 186 * This is used by zdb for spacemaps verification. 187 */ 188 boolean_t spa_mode_readable_spacemaps = B_FALSE; 189 190 /* 191 * This (illegal) pool name is used when temporarily importing a spa_t in order 192 * to get the vdev stats associated with the imported devices. 193 */ 194 #define TRYIMPORT_NAME "$import" 195 196 /* 197 * For debugging purposes: print out vdev tree during pool import. 198 */ 199 static int spa_load_print_vdev_tree = B_FALSE; 200 201 /* 202 * A non-zero value for zfs_max_missing_tvds means that we allow importing 203 * pools with missing top-level vdevs. This is strictly intended for advanced 204 * pool recovery cases since missing data is almost inevitable. Pools with 205 * missing devices can only be imported read-only for safety reasons, and their 206 * fail-mode will be automatically set to "continue". 207 * 208 * With 1 missing vdev we should be able to import the pool and mount all 209 * datasets. User data that was not modified after the missing device has been 210 * added should be recoverable. This means that snapshots created prior to the 211 * addition of that device should be completely intact. 212 * 213 * With 2 missing vdevs, some datasets may fail to mount since there are 214 * dataset statistics that are stored as regular metadata. Some data might be 215 * recoverable if those vdevs were added recently. 216 * 217 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 218 * may be missing entirely. Chances of data recovery are very low. Note that 219 * there are also risks of performing an inadvertent rewind as we might be 220 * missing all the vdevs with the latest uberblocks. 221 */ 222 uint64_t zfs_max_missing_tvds = 0; 223 224 /* 225 * The parameters below are similar to zfs_max_missing_tvds but are only 226 * intended for a preliminary open of the pool with an untrusted config which 227 * might be incomplete or out-dated. 228 * 229 * We are more tolerant for pools opened from a cachefile since we could have 230 * an out-dated cachefile where a device removal was not registered. 231 * We could have set the limit arbitrarily high but in the case where devices 232 * are really missing we would want to return the proper error codes; we chose 233 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 234 * and we get a chance to retrieve the trusted config. 235 */ 236 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 237 238 /* 239 * In the case where config was assembled by scanning device paths (/dev/dsks 240 * by default) we are less tolerant since all the existing devices should have 241 * been detected and we want spa_load to return the right error codes. 242 */ 243 uint64_t zfs_max_missing_tvds_scan = 0; 244 245 /* 246 * Debugging aid that pauses spa_sync() towards the end. 247 */ 248 static const boolean_t zfs_pause_spa_sync = B_FALSE; 249 250 /* 251 * Variables to indicate the livelist condense zthr func should wait at certain 252 * points for the livelist to be removed - used to test condense/destroy races 253 */ 254 static int zfs_livelist_condense_zthr_pause = 0; 255 static int zfs_livelist_condense_sync_pause = 0; 256 257 /* 258 * Variables to track whether or not condense cancellation has been 259 * triggered in testing. 260 */ 261 static int zfs_livelist_condense_sync_cancel = 0; 262 static int zfs_livelist_condense_zthr_cancel = 0; 263 264 /* 265 * Variable to track whether or not extra ALLOC blkptrs were added to a 266 * livelist entry while it was being condensed (caused by the way we track 267 * remapped blkptrs in dbuf_remap_impl) 268 */ 269 static int zfs_livelist_condense_new_alloc = 0; 270 271 /* 272 * ========================================================================== 273 * SPA properties routines 274 * ========================================================================== 275 */ 276 277 /* 278 * Add a (source=src, propname=propval) list to an nvlist. 279 */ 280 static void 281 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, const char *strval, 282 uint64_t intval, zprop_source_t src) 283 { 284 const char *propname = zpool_prop_to_name(prop); 285 nvlist_t *propval; 286 287 propval = fnvlist_alloc(); 288 fnvlist_add_uint64(propval, ZPROP_SOURCE, src); 289 290 if (strval != NULL) 291 fnvlist_add_string(propval, ZPROP_VALUE, strval); 292 else 293 fnvlist_add_uint64(propval, ZPROP_VALUE, intval); 294 295 fnvlist_add_nvlist(nvl, propname, propval); 296 nvlist_free(propval); 297 } 298 299 /* 300 * Get property values from the spa configuration. 301 */ 302 static void 303 spa_prop_get_config(spa_t *spa, nvlist_t **nvp) 304 { 305 vdev_t *rvd = spa->spa_root_vdev; 306 dsl_pool_t *pool = spa->spa_dsl_pool; 307 uint64_t size, alloc, cap, version; 308 const zprop_source_t src = ZPROP_SRC_NONE; 309 spa_config_dirent_t *dp; 310 metaslab_class_t *mc = spa_normal_class(spa); 311 312 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 313 314 if (rvd != NULL) { 315 alloc = metaslab_class_get_alloc(mc); 316 alloc += metaslab_class_get_alloc(spa_special_class(spa)); 317 alloc += metaslab_class_get_alloc(spa_dedup_class(spa)); 318 alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa)); 319 320 size = metaslab_class_get_space(mc); 321 size += metaslab_class_get_space(spa_special_class(spa)); 322 size += metaslab_class_get_space(spa_dedup_class(spa)); 323 size += metaslab_class_get_space(spa_embedded_log_class(spa)); 324 325 spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 326 spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src); 327 spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 328 spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL, 329 size - alloc, src); 330 spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL, 331 spa->spa_checkpoint_info.sci_dspace, src); 332 333 spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL, 334 metaslab_class_fragmentation(mc), src); 335 spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, 336 metaslab_class_expandable_space(mc), src); 337 spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL, 338 (spa_mode(spa) == SPA_MODE_READ), src); 339 340 cap = (size == 0) ? 0 : (alloc * 100 / size); 341 spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src); 342 343 spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL, 344 ddt_get_pool_dedup_ratio(spa), src); 345 spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONEUSED, NULL, 346 brt_get_used(spa), src); 347 spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONESAVED, NULL, 348 brt_get_saved(spa), src); 349 spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONERATIO, NULL, 350 brt_get_ratio(spa), src); 351 352 spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL, 353 rvd->vdev_state, src); 354 355 version = spa_version(spa); 356 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 357 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 358 version, ZPROP_SRC_DEFAULT); 359 } else { 360 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 361 version, ZPROP_SRC_LOCAL); 362 } 363 spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID, 364 NULL, spa_load_guid(spa), src); 365 } 366 367 if (pool != NULL) { 368 /* 369 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 370 * when opening pools before this version freedir will be NULL. 371 */ 372 if (pool->dp_free_dir != NULL) { 373 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, 374 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 375 src); 376 } else { 377 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, 378 NULL, 0, src); 379 } 380 381 if (pool->dp_leak_dir != NULL) { 382 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, 383 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 384 src); 385 } else { 386 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, 387 NULL, 0, src); 388 } 389 } 390 391 spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 392 393 if (spa->spa_comment != NULL) { 394 spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment, 395 0, ZPROP_SRC_LOCAL); 396 } 397 398 if (spa->spa_compatibility != NULL) { 399 spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY, 400 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 401 } 402 403 if (spa->spa_root != NULL) 404 spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root, 405 0, ZPROP_SRC_LOCAL); 406 407 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 408 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 409 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 410 } else { 411 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 412 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 413 } 414 415 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 416 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 417 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 418 } else { 419 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 420 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 421 } 422 423 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 424 if (dp->scd_path == NULL) { 425 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 426 "none", 0, ZPROP_SRC_LOCAL); 427 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 428 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 429 dp->scd_path, 0, ZPROP_SRC_LOCAL); 430 } 431 } 432 } 433 434 /* 435 * Get zpool property values. 436 */ 437 int 438 spa_prop_get(spa_t *spa, nvlist_t **nvp) 439 { 440 objset_t *mos = spa->spa_meta_objset; 441 zap_cursor_t zc; 442 zap_attribute_t za; 443 dsl_pool_t *dp; 444 int err; 445 446 err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); 447 if (err) 448 return (err); 449 450 dp = spa_get_dsl(spa); 451 dsl_pool_config_enter(dp, FTAG); 452 mutex_enter(&spa->spa_props_lock); 453 454 /* 455 * Get properties from the spa config. 456 */ 457 spa_prop_get_config(spa, nvp); 458 459 /* If no pool property object, no more prop to get. */ 460 if (mos == NULL || spa->spa_pool_props_object == 0) 461 goto out; 462 463 /* 464 * Get properties from the MOS pool property object. 465 */ 466 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 467 (err = zap_cursor_retrieve(&zc, &za)) == 0; 468 zap_cursor_advance(&zc)) { 469 uint64_t intval = 0; 470 char *strval = NULL; 471 zprop_source_t src = ZPROP_SRC_DEFAULT; 472 zpool_prop_t prop; 473 474 if ((prop = zpool_name_to_prop(za.za_name)) == ZPOOL_PROP_INVAL) 475 continue; 476 477 switch (za.za_integer_length) { 478 case 8: 479 /* integer property */ 480 if (za.za_first_integer != 481 zpool_prop_default_numeric(prop)) 482 src = ZPROP_SRC_LOCAL; 483 484 if (prop == ZPOOL_PROP_BOOTFS) { 485 dsl_dataset_t *ds = NULL; 486 487 err = dsl_dataset_hold_obj(dp, 488 za.za_first_integer, FTAG, &ds); 489 if (err != 0) 490 break; 491 492 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 493 KM_SLEEP); 494 dsl_dataset_name(ds, strval); 495 dsl_dataset_rele(ds, FTAG); 496 } else { 497 strval = NULL; 498 intval = za.za_first_integer; 499 } 500 501 spa_prop_add_list(*nvp, prop, strval, intval, src); 502 503 if (strval != NULL) 504 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 505 506 break; 507 508 case 1: 509 /* string property */ 510 strval = kmem_alloc(za.za_num_integers, KM_SLEEP); 511 err = zap_lookup(mos, spa->spa_pool_props_object, 512 za.za_name, 1, za.za_num_integers, strval); 513 if (err) { 514 kmem_free(strval, za.za_num_integers); 515 break; 516 } 517 spa_prop_add_list(*nvp, prop, strval, 0, src); 518 kmem_free(strval, za.za_num_integers); 519 break; 520 521 default: 522 break; 523 } 524 } 525 zap_cursor_fini(&zc); 526 out: 527 mutex_exit(&spa->spa_props_lock); 528 dsl_pool_config_exit(dp, FTAG); 529 if (err && err != ENOENT) { 530 nvlist_free(*nvp); 531 *nvp = NULL; 532 return (err); 533 } 534 535 return (0); 536 } 537 538 /* 539 * Validate the given pool properties nvlist and modify the list 540 * for the property values to be set. 541 */ 542 static int 543 spa_prop_validate(spa_t *spa, nvlist_t *props) 544 { 545 nvpair_t *elem; 546 int error = 0, reset_bootfs = 0; 547 uint64_t objnum = 0; 548 boolean_t has_feature = B_FALSE; 549 550 elem = NULL; 551 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 552 uint64_t intval; 553 const char *strval, *slash, *check, *fname; 554 const char *propname = nvpair_name(elem); 555 zpool_prop_t prop = zpool_name_to_prop(propname); 556 557 switch (prop) { 558 case ZPOOL_PROP_INVAL: 559 if (!zpool_prop_feature(propname)) { 560 error = SET_ERROR(EINVAL); 561 break; 562 } 563 564 /* 565 * Sanitize the input. 566 */ 567 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 568 error = SET_ERROR(EINVAL); 569 break; 570 } 571 572 if (nvpair_value_uint64(elem, &intval) != 0) { 573 error = SET_ERROR(EINVAL); 574 break; 575 } 576 577 if (intval != 0) { 578 error = SET_ERROR(EINVAL); 579 break; 580 } 581 582 fname = strchr(propname, '@') + 1; 583 if (zfeature_lookup_name(fname, NULL) != 0) { 584 error = SET_ERROR(EINVAL); 585 break; 586 } 587 588 has_feature = B_TRUE; 589 break; 590 591 case ZPOOL_PROP_VERSION: 592 error = nvpair_value_uint64(elem, &intval); 593 if (!error && 594 (intval < spa_version(spa) || 595 intval > SPA_VERSION_BEFORE_FEATURES || 596 has_feature)) 597 error = SET_ERROR(EINVAL); 598 break; 599 600 case ZPOOL_PROP_DELEGATION: 601 case ZPOOL_PROP_AUTOREPLACE: 602 case ZPOOL_PROP_LISTSNAPS: 603 case ZPOOL_PROP_AUTOEXPAND: 604 case ZPOOL_PROP_AUTOTRIM: 605 error = nvpair_value_uint64(elem, &intval); 606 if (!error && intval > 1) 607 error = SET_ERROR(EINVAL); 608 break; 609 610 case ZPOOL_PROP_MULTIHOST: 611 error = nvpair_value_uint64(elem, &intval); 612 if (!error && intval > 1) 613 error = SET_ERROR(EINVAL); 614 615 if (!error) { 616 uint32_t hostid = zone_get_hostid(NULL); 617 if (hostid) 618 spa->spa_hostid = hostid; 619 else 620 error = SET_ERROR(ENOTSUP); 621 } 622 623 break; 624 625 case ZPOOL_PROP_BOOTFS: 626 /* 627 * If the pool version is less than SPA_VERSION_BOOTFS, 628 * or the pool is still being created (version == 0), 629 * the bootfs property cannot be set. 630 */ 631 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 632 error = SET_ERROR(ENOTSUP); 633 break; 634 } 635 636 /* 637 * Make sure the vdev config is bootable 638 */ 639 if (!vdev_is_bootable(spa->spa_root_vdev)) { 640 error = SET_ERROR(ENOTSUP); 641 break; 642 } 643 644 reset_bootfs = 1; 645 646 error = nvpair_value_string(elem, &strval); 647 648 if (!error) { 649 objset_t *os; 650 651 if (strval == NULL || strval[0] == '\0') { 652 objnum = zpool_prop_default_numeric( 653 ZPOOL_PROP_BOOTFS); 654 break; 655 } 656 657 error = dmu_objset_hold(strval, FTAG, &os); 658 if (error != 0) 659 break; 660 661 /* Must be ZPL. */ 662 if (dmu_objset_type(os) != DMU_OST_ZFS) { 663 error = SET_ERROR(ENOTSUP); 664 } else { 665 objnum = dmu_objset_id(os); 666 } 667 dmu_objset_rele(os, FTAG); 668 } 669 break; 670 671 case ZPOOL_PROP_FAILUREMODE: 672 error = nvpair_value_uint64(elem, &intval); 673 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 674 error = SET_ERROR(EINVAL); 675 676 /* 677 * This is a special case which only occurs when 678 * the pool has completely failed. This allows 679 * the user to change the in-core failmode property 680 * without syncing it out to disk (I/Os might 681 * currently be blocked). We do this by returning 682 * EIO to the caller (spa_prop_set) to trick it 683 * into thinking we encountered a property validation 684 * error. 685 */ 686 if (!error && spa_suspended(spa)) { 687 spa->spa_failmode = intval; 688 error = SET_ERROR(EIO); 689 } 690 break; 691 692 case ZPOOL_PROP_CACHEFILE: 693 if ((error = nvpair_value_string(elem, &strval)) != 0) 694 break; 695 696 if (strval[0] == '\0') 697 break; 698 699 if (strcmp(strval, "none") == 0) 700 break; 701 702 if (strval[0] != '/') { 703 error = SET_ERROR(EINVAL); 704 break; 705 } 706 707 slash = strrchr(strval, '/'); 708 ASSERT(slash != NULL); 709 710 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 711 strcmp(slash, "/..") == 0) 712 error = SET_ERROR(EINVAL); 713 break; 714 715 case ZPOOL_PROP_COMMENT: 716 if ((error = nvpair_value_string(elem, &strval)) != 0) 717 break; 718 for (check = strval; *check != '\0'; check++) { 719 if (!isprint(*check)) { 720 error = SET_ERROR(EINVAL); 721 break; 722 } 723 } 724 if (strlen(strval) > ZPROP_MAX_COMMENT) 725 error = SET_ERROR(E2BIG); 726 break; 727 728 default: 729 break; 730 } 731 732 if (error) 733 break; 734 } 735 736 (void) nvlist_remove_all(props, 737 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 738 739 if (!error && reset_bootfs) { 740 error = nvlist_remove(props, 741 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 742 743 if (!error) { 744 error = nvlist_add_uint64(props, 745 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 746 } 747 } 748 749 return (error); 750 } 751 752 void 753 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 754 { 755 const char *cachefile; 756 spa_config_dirent_t *dp; 757 758 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 759 &cachefile) != 0) 760 return; 761 762 dp = kmem_alloc(sizeof (spa_config_dirent_t), 763 KM_SLEEP); 764 765 if (cachefile[0] == '\0') 766 dp->scd_path = spa_strdup(spa_config_path); 767 else if (strcmp(cachefile, "none") == 0) 768 dp->scd_path = NULL; 769 else 770 dp->scd_path = spa_strdup(cachefile); 771 772 list_insert_head(&spa->spa_config_list, dp); 773 if (need_sync) 774 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 775 } 776 777 int 778 spa_prop_set(spa_t *spa, nvlist_t *nvp) 779 { 780 int error; 781 nvpair_t *elem = NULL; 782 boolean_t need_sync = B_FALSE; 783 784 if ((error = spa_prop_validate(spa, nvp)) != 0) 785 return (error); 786 787 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 788 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 789 790 if (prop == ZPOOL_PROP_CACHEFILE || 791 prop == ZPOOL_PROP_ALTROOT || 792 prop == ZPOOL_PROP_READONLY) 793 continue; 794 795 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 796 uint64_t ver = 0; 797 798 if (prop == ZPOOL_PROP_VERSION) { 799 VERIFY(nvpair_value_uint64(elem, &ver) == 0); 800 } else { 801 ASSERT(zpool_prop_feature(nvpair_name(elem))); 802 ver = SPA_VERSION_FEATURES; 803 need_sync = B_TRUE; 804 } 805 806 /* Save time if the version is already set. */ 807 if (ver == spa_version(spa)) 808 continue; 809 810 /* 811 * In addition to the pool directory object, we might 812 * create the pool properties object, the features for 813 * read object, the features for write object, or the 814 * feature descriptions object. 815 */ 816 error = dsl_sync_task(spa->spa_name, NULL, 817 spa_sync_version, &ver, 818 6, ZFS_SPACE_CHECK_RESERVED); 819 if (error) 820 return (error); 821 continue; 822 } 823 824 need_sync = B_TRUE; 825 break; 826 } 827 828 if (need_sync) { 829 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 830 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 831 } 832 833 return (0); 834 } 835 836 /* 837 * If the bootfs property value is dsobj, clear it. 838 */ 839 void 840 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 841 { 842 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 843 VERIFY(zap_remove(spa->spa_meta_objset, 844 spa->spa_pool_props_object, 845 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 846 spa->spa_bootfs = 0; 847 } 848 } 849 850 static int 851 spa_change_guid_check(void *arg, dmu_tx_t *tx) 852 { 853 uint64_t *newguid __maybe_unused = arg; 854 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 855 vdev_t *rvd = spa->spa_root_vdev; 856 uint64_t vdev_state; 857 858 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 859 int error = (spa_has_checkpoint(spa)) ? 860 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 861 return (SET_ERROR(error)); 862 } 863 864 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 865 vdev_state = rvd->vdev_state; 866 spa_config_exit(spa, SCL_STATE, FTAG); 867 868 if (vdev_state != VDEV_STATE_HEALTHY) 869 return (SET_ERROR(ENXIO)); 870 871 ASSERT3U(spa_guid(spa), !=, *newguid); 872 873 return (0); 874 } 875 876 static void 877 spa_change_guid_sync(void *arg, dmu_tx_t *tx) 878 { 879 uint64_t *newguid = arg; 880 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 881 uint64_t oldguid; 882 vdev_t *rvd = spa->spa_root_vdev; 883 884 oldguid = spa_guid(spa); 885 886 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 887 rvd->vdev_guid = *newguid; 888 rvd->vdev_guid_sum += (*newguid - oldguid); 889 vdev_config_dirty(rvd); 890 spa_config_exit(spa, SCL_STATE, FTAG); 891 892 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 893 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 894 } 895 896 /* 897 * Change the GUID for the pool. This is done so that we can later 898 * re-import a pool built from a clone of our own vdevs. We will modify 899 * the root vdev's guid, our own pool guid, and then mark all of our 900 * vdevs dirty. Note that we must make sure that all our vdevs are 901 * online when we do this, or else any vdevs that weren't present 902 * would be orphaned from our pool. We are also going to issue a 903 * sysevent to update any watchers. 904 */ 905 int 906 spa_change_guid(spa_t *spa) 907 { 908 int error; 909 uint64_t guid; 910 911 mutex_enter(&spa->spa_vdev_top_lock); 912 mutex_enter(&spa_namespace_lock); 913 guid = spa_generate_guid(NULL); 914 915 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 916 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 917 918 if (error == 0) { 919 /* 920 * Clear the kobj flag from all the vdevs to allow 921 * vdev_cache_process_kobj_evt() to post events to all the 922 * vdevs since GUID is updated. 923 */ 924 vdev_clear_kobj_evt(spa->spa_root_vdev); 925 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 926 vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]); 927 928 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 929 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 930 } 931 932 mutex_exit(&spa_namespace_lock); 933 mutex_exit(&spa->spa_vdev_top_lock); 934 935 return (error); 936 } 937 938 /* 939 * ========================================================================== 940 * SPA state manipulation (open/create/destroy/import/export) 941 * ========================================================================== 942 */ 943 944 static int 945 spa_error_entry_compare(const void *a, const void *b) 946 { 947 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 948 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 949 int ret; 950 951 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 952 sizeof (zbookmark_phys_t)); 953 954 return (TREE_ISIGN(ret)); 955 } 956 957 /* 958 * Utility function which retrieves copies of the current logs and 959 * re-initializes them in the process. 960 */ 961 void 962 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 963 { 964 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 965 966 memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t)); 967 memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t)); 968 969 avl_create(&spa->spa_errlist_scrub, 970 spa_error_entry_compare, sizeof (spa_error_entry_t), 971 offsetof(spa_error_entry_t, se_avl)); 972 avl_create(&spa->spa_errlist_last, 973 spa_error_entry_compare, sizeof (spa_error_entry_t), 974 offsetof(spa_error_entry_t, se_avl)); 975 } 976 977 static void 978 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 979 { 980 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 981 enum zti_modes mode = ztip->zti_mode; 982 uint_t value = ztip->zti_value; 983 uint_t count = ztip->zti_count; 984 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 985 uint_t cpus, flags = TASKQ_DYNAMIC; 986 boolean_t batch = B_FALSE; 987 988 switch (mode) { 989 case ZTI_MODE_FIXED: 990 ASSERT3U(value, >, 0); 991 break; 992 993 case ZTI_MODE_BATCH: 994 batch = B_TRUE; 995 flags |= TASKQ_THREADS_CPU_PCT; 996 value = MIN(zio_taskq_batch_pct, 100); 997 break; 998 999 case ZTI_MODE_SCALE: 1000 flags |= TASKQ_THREADS_CPU_PCT; 1001 /* 1002 * We want more taskqs to reduce lock contention, but we want 1003 * less for better request ordering and CPU utilization. 1004 */ 1005 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 1006 if (zio_taskq_batch_tpq > 0) { 1007 count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) / 1008 zio_taskq_batch_tpq); 1009 } else { 1010 /* 1011 * Prefer 6 threads per taskq, but no more taskqs 1012 * than threads in them on large systems. For 80%: 1013 * 1014 * taskq taskq total 1015 * cpus taskqs percent threads threads 1016 * ------- ------- ------- ------- ------- 1017 * 1 1 80% 1 1 1018 * 2 1 80% 1 1 1019 * 4 1 80% 3 3 1020 * 8 2 40% 3 6 1021 * 16 3 27% 4 12 1022 * 32 5 16% 5 25 1023 * 64 7 11% 7 49 1024 * 128 10 8% 10 100 1025 * 256 14 6% 15 210 1026 */ 1027 count = 1 + cpus / 6; 1028 while (count * count > cpus) 1029 count--; 1030 } 1031 /* Limit each taskq within 100% to not trigger assertion. */ 1032 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1033 value = (zio_taskq_batch_pct + count / 2) / count; 1034 break; 1035 1036 case ZTI_MODE_NULL: 1037 tqs->stqs_count = 0; 1038 tqs->stqs_taskq = NULL; 1039 return; 1040 1041 default: 1042 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1043 "spa_activate()", 1044 zio_type_name[t], zio_taskq_types[q], mode, value); 1045 break; 1046 } 1047 1048 ASSERT3U(count, >, 0); 1049 tqs->stqs_count = count; 1050 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1051 1052 for (uint_t i = 0; i < count; i++) { 1053 taskq_t *tq; 1054 char name[32]; 1055 1056 if (count > 1) 1057 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1058 zio_type_name[t], zio_taskq_types[q], i); 1059 else 1060 (void) snprintf(name, sizeof (name), "%s_%s", 1061 zio_type_name[t], zio_taskq_types[q]); 1062 1063 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1064 if (batch) 1065 flags |= TASKQ_DC_BATCH; 1066 1067 (void) zio_taskq_basedc; 1068 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1069 spa->spa_proc, zio_taskq_basedc, flags); 1070 } else { 1071 pri_t pri = maxclsyspri; 1072 /* 1073 * The write issue taskq can be extremely CPU 1074 * intensive. Run it at slightly less important 1075 * priority than the other taskqs. 1076 * 1077 * Under Linux and FreeBSD this means incrementing 1078 * the priority value as opposed to platforms like 1079 * illumos where it should be decremented. 1080 * 1081 * On FreeBSD, if priorities divided by four (RQ_PPQ) 1082 * are equal then a difference between them is 1083 * insignificant. 1084 */ 1085 if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) { 1086 #if defined(__linux__) 1087 pri++; 1088 #elif defined(__FreeBSD__) 1089 pri += 4; 1090 #else 1091 #error "unknown OS" 1092 #endif 1093 } 1094 tq = taskq_create_proc(name, value, pri, 50, 1095 INT_MAX, spa->spa_proc, flags); 1096 } 1097 1098 tqs->stqs_taskq[i] = tq; 1099 } 1100 } 1101 1102 static void 1103 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1104 { 1105 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1106 1107 if (tqs->stqs_taskq == NULL) { 1108 ASSERT3U(tqs->stqs_count, ==, 0); 1109 return; 1110 } 1111 1112 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1113 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1114 taskq_destroy(tqs->stqs_taskq[i]); 1115 } 1116 1117 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1118 tqs->stqs_taskq = NULL; 1119 } 1120 1121 /* 1122 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1123 * Note that a type may have multiple discrete taskqs to avoid lock contention 1124 * on the taskq itself. In that case we choose which taskq at random by using 1125 * the low bits of gethrtime(). 1126 */ 1127 void 1128 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1129 task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent) 1130 { 1131 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1132 taskq_t *tq; 1133 1134 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1135 ASSERT3U(tqs->stqs_count, !=, 0); 1136 1137 if (tqs->stqs_count == 1) { 1138 tq = tqs->stqs_taskq[0]; 1139 } else { 1140 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1141 } 1142 1143 taskq_dispatch_ent(tq, func, arg, flags, ent); 1144 } 1145 1146 /* 1147 * Same as spa_taskq_dispatch_ent() but block on the task until completion. 1148 */ 1149 void 1150 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1151 task_func_t *func, void *arg, uint_t flags) 1152 { 1153 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1154 taskq_t *tq; 1155 taskqid_t id; 1156 1157 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1158 ASSERT3U(tqs->stqs_count, !=, 0); 1159 1160 if (tqs->stqs_count == 1) { 1161 tq = tqs->stqs_taskq[0]; 1162 } else { 1163 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1164 } 1165 1166 id = taskq_dispatch(tq, func, arg, flags); 1167 if (id) 1168 taskq_wait_id(tq, id); 1169 } 1170 1171 static void 1172 spa_create_zio_taskqs(spa_t *spa) 1173 { 1174 for (int t = 0; t < ZIO_TYPES; t++) { 1175 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1176 spa_taskqs_init(spa, t, q); 1177 } 1178 } 1179 } 1180 1181 /* 1182 * Disabled until spa_thread() can be adapted for Linux. 1183 */ 1184 #undef HAVE_SPA_THREAD 1185 1186 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1187 static void 1188 spa_thread(void *arg) 1189 { 1190 psetid_t zio_taskq_psrset_bind = PS_NONE; 1191 callb_cpr_t cprinfo; 1192 1193 spa_t *spa = arg; 1194 user_t *pu = PTOU(curproc); 1195 1196 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1197 spa->spa_name); 1198 1199 ASSERT(curproc != &p0); 1200 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1201 "zpool-%s", spa->spa_name); 1202 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1203 1204 /* bind this thread to the requested psrset */ 1205 if (zio_taskq_psrset_bind != PS_NONE) { 1206 pool_lock(); 1207 mutex_enter(&cpu_lock); 1208 mutex_enter(&pidlock); 1209 mutex_enter(&curproc->p_lock); 1210 1211 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1212 0, NULL, NULL) == 0) { 1213 curthread->t_bind_pset = zio_taskq_psrset_bind; 1214 } else { 1215 cmn_err(CE_WARN, 1216 "Couldn't bind process for zfs pool \"%s\" to " 1217 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1218 } 1219 1220 mutex_exit(&curproc->p_lock); 1221 mutex_exit(&pidlock); 1222 mutex_exit(&cpu_lock); 1223 pool_unlock(); 1224 } 1225 1226 if (zio_taskq_sysdc) { 1227 sysdc_thread_enter(curthread, 100, 0); 1228 } 1229 1230 spa->spa_proc = curproc; 1231 spa->spa_did = curthread->t_did; 1232 1233 spa_create_zio_taskqs(spa); 1234 1235 mutex_enter(&spa->spa_proc_lock); 1236 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1237 1238 spa->spa_proc_state = SPA_PROC_ACTIVE; 1239 cv_broadcast(&spa->spa_proc_cv); 1240 1241 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1242 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1243 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1244 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1245 1246 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1247 spa->spa_proc_state = SPA_PROC_GONE; 1248 spa->spa_proc = &p0; 1249 cv_broadcast(&spa->spa_proc_cv); 1250 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1251 1252 mutex_enter(&curproc->p_lock); 1253 lwp_exit(); 1254 } 1255 #endif 1256 1257 /* 1258 * Activate an uninitialized pool. 1259 */ 1260 static void 1261 spa_activate(spa_t *spa, spa_mode_t mode) 1262 { 1263 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1264 1265 spa->spa_state = POOL_STATE_ACTIVE; 1266 spa->spa_mode = mode; 1267 spa->spa_read_spacemaps = spa_mode_readable_spacemaps; 1268 1269 spa->spa_normal_class = metaslab_class_create(spa, &zfs_metaslab_ops); 1270 spa->spa_log_class = metaslab_class_create(spa, &zfs_metaslab_ops); 1271 spa->spa_embedded_log_class = 1272 metaslab_class_create(spa, &zfs_metaslab_ops); 1273 spa->spa_special_class = metaslab_class_create(spa, &zfs_metaslab_ops); 1274 spa->spa_dedup_class = metaslab_class_create(spa, &zfs_metaslab_ops); 1275 1276 /* Try to create a covering process */ 1277 mutex_enter(&spa->spa_proc_lock); 1278 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1279 ASSERT(spa->spa_proc == &p0); 1280 spa->spa_did = 0; 1281 1282 (void) spa_create_process; 1283 #ifdef HAVE_SPA_THREAD 1284 /* Only create a process if we're going to be around a while. */ 1285 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1286 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1287 NULL, 0) == 0) { 1288 spa->spa_proc_state = SPA_PROC_CREATED; 1289 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1290 cv_wait(&spa->spa_proc_cv, 1291 &spa->spa_proc_lock); 1292 } 1293 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1294 ASSERT(spa->spa_proc != &p0); 1295 ASSERT(spa->spa_did != 0); 1296 } else { 1297 #ifdef _KERNEL 1298 cmn_err(CE_WARN, 1299 "Couldn't create process for zfs pool \"%s\"\n", 1300 spa->spa_name); 1301 #endif 1302 } 1303 } 1304 #endif /* HAVE_SPA_THREAD */ 1305 mutex_exit(&spa->spa_proc_lock); 1306 1307 /* If we didn't create a process, we need to create our taskqs. */ 1308 if (spa->spa_proc == &p0) { 1309 spa_create_zio_taskqs(spa); 1310 } 1311 1312 for (size_t i = 0; i < TXG_SIZE; i++) { 1313 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1314 ZIO_FLAG_CANFAIL); 1315 } 1316 1317 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1318 offsetof(vdev_t, vdev_config_dirty_node)); 1319 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1320 offsetof(objset_t, os_evicting_node)); 1321 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1322 offsetof(vdev_t, vdev_state_dirty_node)); 1323 1324 txg_list_create(&spa->spa_vdev_txg_list, spa, 1325 offsetof(struct vdev, vdev_txg_node)); 1326 1327 avl_create(&spa->spa_errlist_scrub, 1328 spa_error_entry_compare, sizeof (spa_error_entry_t), 1329 offsetof(spa_error_entry_t, se_avl)); 1330 avl_create(&spa->spa_errlist_last, 1331 spa_error_entry_compare, sizeof (spa_error_entry_t), 1332 offsetof(spa_error_entry_t, se_avl)); 1333 avl_create(&spa->spa_errlist_healed, 1334 spa_error_entry_compare, sizeof (spa_error_entry_t), 1335 offsetof(spa_error_entry_t, se_avl)); 1336 1337 spa_activate_os(spa); 1338 1339 spa_keystore_init(&spa->spa_keystore); 1340 1341 /* 1342 * This taskq is used to perform zvol-minor-related tasks 1343 * asynchronously. This has several advantages, including easy 1344 * resolution of various deadlocks. 1345 * 1346 * The taskq must be single threaded to ensure tasks are always 1347 * processed in the order in which they were dispatched. 1348 * 1349 * A taskq per pool allows one to keep the pools independent. 1350 * This way if one pool is suspended, it will not impact another. 1351 * 1352 * The preferred location to dispatch a zvol minor task is a sync 1353 * task. In this context, there is easy access to the spa_t and minimal 1354 * error handling is required because the sync task must succeed. 1355 */ 1356 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1357 1, INT_MAX, 0); 1358 1359 /* 1360 * Taskq dedicated to prefetcher threads: this is used to prevent the 1361 * pool traverse code from monopolizing the global (and limited) 1362 * system_taskq by inappropriately scheduling long running tasks on it. 1363 */ 1364 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1365 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1366 1367 /* 1368 * The taskq to upgrade datasets in this pool. Currently used by 1369 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1370 */ 1371 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1372 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1373 } 1374 1375 /* 1376 * Opposite of spa_activate(). 1377 */ 1378 static void 1379 spa_deactivate(spa_t *spa) 1380 { 1381 ASSERT(spa->spa_sync_on == B_FALSE); 1382 ASSERT(spa->spa_dsl_pool == NULL); 1383 ASSERT(spa->spa_root_vdev == NULL); 1384 ASSERT(spa->spa_async_zio_root == NULL); 1385 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1386 1387 spa_evicting_os_wait(spa); 1388 1389 if (spa->spa_zvol_taskq) { 1390 taskq_destroy(spa->spa_zvol_taskq); 1391 spa->spa_zvol_taskq = NULL; 1392 } 1393 1394 if (spa->spa_prefetch_taskq) { 1395 taskq_destroy(spa->spa_prefetch_taskq); 1396 spa->spa_prefetch_taskq = NULL; 1397 } 1398 1399 if (spa->spa_upgrade_taskq) { 1400 taskq_destroy(spa->spa_upgrade_taskq); 1401 spa->spa_upgrade_taskq = NULL; 1402 } 1403 1404 txg_list_destroy(&spa->spa_vdev_txg_list); 1405 1406 list_destroy(&spa->spa_config_dirty_list); 1407 list_destroy(&spa->spa_evicting_os_list); 1408 list_destroy(&spa->spa_state_dirty_list); 1409 1410 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 1411 1412 for (int t = 0; t < ZIO_TYPES; t++) { 1413 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1414 spa_taskqs_fini(spa, t, q); 1415 } 1416 } 1417 1418 for (size_t i = 0; i < TXG_SIZE; i++) { 1419 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1420 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1421 spa->spa_txg_zio[i] = NULL; 1422 } 1423 1424 metaslab_class_destroy(spa->spa_normal_class); 1425 spa->spa_normal_class = NULL; 1426 1427 metaslab_class_destroy(spa->spa_log_class); 1428 spa->spa_log_class = NULL; 1429 1430 metaslab_class_destroy(spa->spa_embedded_log_class); 1431 spa->spa_embedded_log_class = NULL; 1432 1433 metaslab_class_destroy(spa->spa_special_class); 1434 spa->spa_special_class = NULL; 1435 1436 metaslab_class_destroy(spa->spa_dedup_class); 1437 spa->spa_dedup_class = NULL; 1438 1439 /* 1440 * If this was part of an import or the open otherwise failed, we may 1441 * still have errors left in the queues. Empty them just in case. 1442 */ 1443 spa_errlog_drain(spa); 1444 avl_destroy(&spa->spa_errlist_scrub); 1445 avl_destroy(&spa->spa_errlist_last); 1446 avl_destroy(&spa->spa_errlist_healed); 1447 1448 spa_keystore_fini(&spa->spa_keystore); 1449 1450 spa->spa_state = POOL_STATE_UNINITIALIZED; 1451 1452 mutex_enter(&spa->spa_proc_lock); 1453 if (spa->spa_proc_state != SPA_PROC_NONE) { 1454 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1455 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 1456 cv_broadcast(&spa->spa_proc_cv); 1457 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 1458 ASSERT(spa->spa_proc != &p0); 1459 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1460 } 1461 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 1462 spa->spa_proc_state = SPA_PROC_NONE; 1463 } 1464 ASSERT(spa->spa_proc == &p0); 1465 mutex_exit(&spa->spa_proc_lock); 1466 1467 /* 1468 * We want to make sure spa_thread() has actually exited the ZFS 1469 * module, so that the module can't be unloaded out from underneath 1470 * it. 1471 */ 1472 if (spa->spa_did != 0) { 1473 thread_join(spa->spa_did); 1474 spa->spa_did = 0; 1475 } 1476 1477 spa_deactivate_os(spa); 1478 1479 } 1480 1481 /* 1482 * Verify a pool configuration, and construct the vdev tree appropriately. This 1483 * will create all the necessary vdevs in the appropriate layout, with each vdev 1484 * in the CLOSED state. This will prep the pool before open/creation/import. 1485 * All vdev validation is done by the vdev_alloc() routine. 1486 */ 1487 int 1488 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 1489 uint_t id, int atype) 1490 { 1491 nvlist_t **child; 1492 uint_t children; 1493 int error; 1494 1495 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 1496 return (error); 1497 1498 if ((*vdp)->vdev_ops->vdev_op_leaf) 1499 return (0); 1500 1501 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 1502 &child, &children); 1503 1504 if (error == ENOENT) 1505 return (0); 1506 1507 if (error) { 1508 vdev_free(*vdp); 1509 *vdp = NULL; 1510 return (SET_ERROR(EINVAL)); 1511 } 1512 1513 for (int c = 0; c < children; c++) { 1514 vdev_t *vd; 1515 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 1516 atype)) != 0) { 1517 vdev_free(*vdp); 1518 *vdp = NULL; 1519 return (error); 1520 } 1521 } 1522 1523 ASSERT(*vdp != NULL); 1524 1525 return (0); 1526 } 1527 1528 static boolean_t 1529 spa_should_flush_logs_on_unload(spa_t *spa) 1530 { 1531 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 1532 return (B_FALSE); 1533 1534 if (!spa_writeable(spa)) 1535 return (B_FALSE); 1536 1537 if (!spa->spa_sync_on) 1538 return (B_FALSE); 1539 1540 if (spa_state(spa) != POOL_STATE_EXPORTED) 1541 return (B_FALSE); 1542 1543 if (zfs_keep_log_spacemaps_at_export) 1544 return (B_FALSE); 1545 1546 return (B_TRUE); 1547 } 1548 1549 /* 1550 * Opens a transaction that will set the flag that will instruct 1551 * spa_sync to attempt to flush all the metaslabs for that txg. 1552 */ 1553 static void 1554 spa_unload_log_sm_flush_all(spa_t *spa) 1555 { 1556 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 1557 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 1558 1559 ASSERT3U(spa->spa_log_flushall_txg, ==, 0); 1560 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); 1561 1562 dmu_tx_commit(tx); 1563 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 1564 } 1565 1566 static void 1567 spa_unload_log_sm_metadata(spa_t *spa) 1568 { 1569 void *cookie = NULL; 1570 spa_log_sm_t *sls; 1571 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 1572 &cookie)) != NULL) { 1573 VERIFY0(sls->sls_mscount); 1574 kmem_free(sls, sizeof (spa_log_sm_t)); 1575 } 1576 1577 for (log_summary_entry_t *e = list_head(&spa->spa_log_summary); 1578 e != NULL; e = list_head(&spa->spa_log_summary)) { 1579 VERIFY0(e->lse_mscount); 1580 list_remove(&spa->spa_log_summary, e); 1581 kmem_free(e, sizeof (log_summary_entry_t)); 1582 } 1583 1584 spa->spa_unflushed_stats.sus_nblocks = 0; 1585 spa->spa_unflushed_stats.sus_memused = 0; 1586 spa->spa_unflushed_stats.sus_blocklimit = 0; 1587 } 1588 1589 static void 1590 spa_destroy_aux_threads(spa_t *spa) 1591 { 1592 if (spa->spa_condense_zthr != NULL) { 1593 zthr_destroy(spa->spa_condense_zthr); 1594 spa->spa_condense_zthr = NULL; 1595 } 1596 if (spa->spa_checkpoint_discard_zthr != NULL) { 1597 zthr_destroy(spa->spa_checkpoint_discard_zthr); 1598 spa->spa_checkpoint_discard_zthr = NULL; 1599 } 1600 if (spa->spa_livelist_delete_zthr != NULL) { 1601 zthr_destroy(spa->spa_livelist_delete_zthr); 1602 spa->spa_livelist_delete_zthr = NULL; 1603 } 1604 if (spa->spa_livelist_condense_zthr != NULL) { 1605 zthr_destroy(spa->spa_livelist_condense_zthr); 1606 spa->spa_livelist_condense_zthr = NULL; 1607 } 1608 } 1609 1610 /* 1611 * Opposite of spa_load(). 1612 */ 1613 static void 1614 spa_unload(spa_t *spa) 1615 { 1616 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1617 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 1618 1619 spa_import_progress_remove(spa_guid(spa)); 1620 spa_load_note(spa, "UNLOADING"); 1621 1622 spa_wake_waiters(spa); 1623 1624 /* 1625 * If we have set the spa_final_txg, we have already performed the 1626 * tasks below in spa_export_common(). We should not redo it here since 1627 * we delay the final TXGs beyond what spa_final_txg is set at. 1628 */ 1629 if (spa->spa_final_txg == UINT64_MAX) { 1630 /* 1631 * If the log space map feature is enabled and the pool is 1632 * getting exported (but not destroyed), we want to spend some 1633 * time flushing as many metaslabs as we can in an attempt to 1634 * destroy log space maps and save import time. 1635 */ 1636 if (spa_should_flush_logs_on_unload(spa)) 1637 spa_unload_log_sm_flush_all(spa); 1638 1639 /* 1640 * Stop async tasks. 1641 */ 1642 spa_async_suspend(spa); 1643 1644 if (spa->spa_root_vdev) { 1645 vdev_t *root_vdev = spa->spa_root_vdev; 1646 vdev_initialize_stop_all(root_vdev, 1647 VDEV_INITIALIZE_ACTIVE); 1648 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 1649 vdev_autotrim_stop_all(spa); 1650 vdev_rebuild_stop_all(spa); 1651 } 1652 } 1653 1654 /* 1655 * Stop syncing. 1656 */ 1657 if (spa->spa_sync_on) { 1658 txg_sync_stop(spa->spa_dsl_pool); 1659 spa->spa_sync_on = B_FALSE; 1660 } 1661 1662 /* 1663 * This ensures that there is no async metaslab prefetching 1664 * while we attempt to unload the spa. 1665 */ 1666 if (spa->spa_root_vdev != NULL) { 1667 for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++) { 1668 vdev_t *vc = spa->spa_root_vdev->vdev_child[c]; 1669 if (vc->vdev_mg != NULL) 1670 taskq_wait(vc->vdev_mg->mg_taskq); 1671 } 1672 } 1673 1674 if (spa->spa_mmp.mmp_thread) 1675 mmp_thread_stop(spa); 1676 1677 /* 1678 * Wait for any outstanding async I/O to complete. 1679 */ 1680 if (spa->spa_async_zio_root != NULL) { 1681 for (int i = 0; i < max_ncpus; i++) 1682 (void) zio_wait(spa->spa_async_zio_root[i]); 1683 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 1684 spa->spa_async_zio_root = NULL; 1685 } 1686 1687 if (spa->spa_vdev_removal != NULL) { 1688 spa_vdev_removal_destroy(spa->spa_vdev_removal); 1689 spa->spa_vdev_removal = NULL; 1690 } 1691 1692 spa_destroy_aux_threads(spa); 1693 1694 spa_condense_fini(spa); 1695 1696 bpobj_close(&spa->spa_deferred_bpobj); 1697 1698 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 1699 1700 /* 1701 * Close all vdevs. 1702 */ 1703 if (spa->spa_root_vdev) 1704 vdev_free(spa->spa_root_vdev); 1705 ASSERT(spa->spa_root_vdev == NULL); 1706 1707 /* 1708 * Close the dsl pool. 1709 */ 1710 if (spa->spa_dsl_pool) { 1711 dsl_pool_close(spa->spa_dsl_pool); 1712 spa->spa_dsl_pool = NULL; 1713 spa->spa_meta_objset = NULL; 1714 } 1715 1716 ddt_unload(spa); 1717 brt_unload(spa); 1718 spa_unload_log_sm_metadata(spa); 1719 1720 /* 1721 * Drop and purge level 2 cache 1722 */ 1723 spa_l2cache_drop(spa); 1724 1725 if (spa->spa_spares.sav_vdevs) { 1726 for (int i = 0; i < spa->spa_spares.sav_count; i++) 1727 vdev_free(spa->spa_spares.sav_vdevs[i]); 1728 kmem_free(spa->spa_spares.sav_vdevs, 1729 spa->spa_spares.sav_count * sizeof (void *)); 1730 spa->spa_spares.sav_vdevs = NULL; 1731 } 1732 if (spa->spa_spares.sav_config) { 1733 nvlist_free(spa->spa_spares.sav_config); 1734 spa->spa_spares.sav_config = NULL; 1735 } 1736 spa->spa_spares.sav_count = 0; 1737 1738 if (spa->spa_l2cache.sav_vdevs) { 1739 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1740 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 1741 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 1742 } 1743 kmem_free(spa->spa_l2cache.sav_vdevs, 1744 spa->spa_l2cache.sav_count * sizeof (void *)); 1745 spa->spa_l2cache.sav_vdevs = NULL; 1746 } 1747 if (spa->spa_l2cache.sav_config) { 1748 nvlist_free(spa->spa_l2cache.sav_config); 1749 spa->spa_l2cache.sav_config = NULL; 1750 } 1751 spa->spa_l2cache.sav_count = 0; 1752 1753 spa->spa_async_suspended = 0; 1754 1755 spa->spa_indirect_vdevs_loaded = B_FALSE; 1756 1757 if (spa->spa_comment != NULL) { 1758 spa_strfree(spa->spa_comment); 1759 spa->spa_comment = NULL; 1760 } 1761 if (spa->spa_compatibility != NULL) { 1762 spa_strfree(spa->spa_compatibility); 1763 spa->spa_compatibility = NULL; 1764 } 1765 1766 spa_config_exit(spa, SCL_ALL, spa); 1767 } 1768 1769 /* 1770 * Load (or re-load) the current list of vdevs describing the active spares for 1771 * this pool. When this is called, we have some form of basic information in 1772 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 1773 * then re-generate a more complete list including status information. 1774 */ 1775 void 1776 spa_load_spares(spa_t *spa) 1777 { 1778 nvlist_t **spares; 1779 uint_t nspares; 1780 int i; 1781 vdev_t *vd, *tvd; 1782 1783 #ifndef _KERNEL 1784 /* 1785 * zdb opens both the current state of the pool and the 1786 * checkpointed state (if present), with a different spa_t. 1787 * 1788 * As spare vdevs are shared among open pools, we skip loading 1789 * them when we load the checkpointed state of the pool. 1790 */ 1791 if (!spa_writeable(spa)) 1792 return; 1793 #endif 1794 1795 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1796 1797 /* 1798 * First, close and free any existing spare vdevs. 1799 */ 1800 if (spa->spa_spares.sav_vdevs) { 1801 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1802 vd = spa->spa_spares.sav_vdevs[i]; 1803 1804 /* Undo the call to spa_activate() below */ 1805 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1806 B_FALSE)) != NULL && tvd->vdev_isspare) 1807 spa_spare_remove(tvd); 1808 vdev_close(vd); 1809 vdev_free(vd); 1810 } 1811 1812 kmem_free(spa->spa_spares.sav_vdevs, 1813 spa->spa_spares.sav_count * sizeof (void *)); 1814 } 1815 1816 if (spa->spa_spares.sav_config == NULL) 1817 nspares = 0; 1818 else 1819 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 1820 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 1821 1822 spa->spa_spares.sav_count = (int)nspares; 1823 spa->spa_spares.sav_vdevs = NULL; 1824 1825 if (nspares == 0) 1826 return; 1827 1828 /* 1829 * Construct the array of vdevs, opening them to get status in the 1830 * process. For each spare, there is potentially two different vdev_t 1831 * structures associated with it: one in the list of spares (used only 1832 * for basic validation purposes) and one in the active vdev 1833 * configuration (if it's spared in). During this phase we open and 1834 * validate each vdev on the spare list. If the vdev also exists in the 1835 * active configuration, then we also mark this vdev as an active spare. 1836 */ 1837 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 1838 KM_SLEEP); 1839 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1840 VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, 1841 VDEV_ALLOC_SPARE) == 0); 1842 ASSERT(vd != NULL); 1843 1844 spa->spa_spares.sav_vdevs[i] = vd; 1845 1846 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1847 B_FALSE)) != NULL) { 1848 if (!tvd->vdev_isspare) 1849 spa_spare_add(tvd); 1850 1851 /* 1852 * We only mark the spare active if we were successfully 1853 * able to load the vdev. Otherwise, importing a pool 1854 * with a bad active spare would result in strange 1855 * behavior, because multiple pool would think the spare 1856 * is actively in use. 1857 * 1858 * There is a vulnerability here to an equally bizarre 1859 * circumstance, where a dead active spare is later 1860 * brought back to life (onlined or otherwise). Given 1861 * the rarity of this scenario, and the extra complexity 1862 * it adds, we ignore the possibility. 1863 */ 1864 if (!vdev_is_dead(tvd)) 1865 spa_spare_activate(tvd); 1866 } 1867 1868 vd->vdev_top = vd; 1869 vd->vdev_aux = &spa->spa_spares; 1870 1871 if (vdev_open(vd) != 0) 1872 continue; 1873 1874 if (vdev_validate_aux(vd) == 0) 1875 spa_spare_add(vd); 1876 } 1877 1878 /* 1879 * Recompute the stashed list of spares, with status information 1880 * this time. 1881 */ 1882 fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES); 1883 1884 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 1885 KM_SLEEP); 1886 for (i = 0; i < spa->spa_spares.sav_count; i++) 1887 spares[i] = vdev_config_generate(spa, 1888 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 1889 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 1890 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 1891 spa->spa_spares.sav_count); 1892 for (i = 0; i < spa->spa_spares.sav_count; i++) 1893 nvlist_free(spares[i]); 1894 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 1895 } 1896 1897 /* 1898 * Load (or re-load) the current list of vdevs describing the active l2cache for 1899 * this pool. When this is called, we have some form of basic information in 1900 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 1901 * then re-generate a more complete list including status information. 1902 * Devices which are already active have their details maintained, and are 1903 * not re-opened. 1904 */ 1905 void 1906 spa_load_l2cache(spa_t *spa) 1907 { 1908 nvlist_t **l2cache = NULL; 1909 uint_t nl2cache; 1910 int i, j, oldnvdevs; 1911 uint64_t guid; 1912 vdev_t *vd, **oldvdevs, **newvdevs; 1913 spa_aux_vdev_t *sav = &spa->spa_l2cache; 1914 1915 #ifndef _KERNEL 1916 /* 1917 * zdb opens both the current state of the pool and the 1918 * checkpointed state (if present), with a different spa_t. 1919 * 1920 * As L2 caches are part of the ARC which is shared among open 1921 * pools, we skip loading them when we load the checkpointed 1922 * state of the pool. 1923 */ 1924 if (!spa_writeable(spa)) 1925 return; 1926 #endif 1927 1928 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1929 1930 oldvdevs = sav->sav_vdevs; 1931 oldnvdevs = sav->sav_count; 1932 sav->sav_vdevs = NULL; 1933 sav->sav_count = 0; 1934 1935 if (sav->sav_config == NULL) { 1936 nl2cache = 0; 1937 newvdevs = NULL; 1938 goto out; 1939 } 1940 1941 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, 1942 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 1943 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 1944 1945 /* 1946 * Process new nvlist of vdevs. 1947 */ 1948 for (i = 0; i < nl2cache; i++) { 1949 guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID); 1950 1951 newvdevs[i] = NULL; 1952 for (j = 0; j < oldnvdevs; j++) { 1953 vd = oldvdevs[j]; 1954 if (vd != NULL && guid == vd->vdev_guid) { 1955 /* 1956 * Retain previous vdev for add/remove ops. 1957 */ 1958 newvdevs[i] = vd; 1959 oldvdevs[j] = NULL; 1960 break; 1961 } 1962 } 1963 1964 if (newvdevs[i] == NULL) { 1965 /* 1966 * Create new vdev 1967 */ 1968 VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 1969 VDEV_ALLOC_L2CACHE) == 0); 1970 ASSERT(vd != NULL); 1971 newvdevs[i] = vd; 1972 1973 /* 1974 * Commit this vdev as an l2cache device, 1975 * even if it fails to open. 1976 */ 1977 spa_l2cache_add(vd); 1978 1979 vd->vdev_top = vd; 1980 vd->vdev_aux = sav; 1981 1982 spa_l2cache_activate(vd); 1983 1984 if (vdev_open(vd) != 0) 1985 continue; 1986 1987 (void) vdev_validate_aux(vd); 1988 1989 if (!vdev_is_dead(vd)) 1990 l2arc_add_vdev(spa, vd); 1991 1992 /* 1993 * Upon cache device addition to a pool or pool 1994 * creation with a cache device or if the header 1995 * of the device is invalid we issue an async 1996 * TRIM command for the whole device which will 1997 * execute if l2arc_trim_ahead > 0. 1998 */ 1999 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 2000 } 2001 } 2002 2003 sav->sav_vdevs = newvdevs; 2004 sav->sav_count = (int)nl2cache; 2005 2006 /* 2007 * Recompute the stashed list of l2cache devices, with status 2008 * information this time. 2009 */ 2010 fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE); 2011 2012 if (sav->sav_count > 0) 2013 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 2014 KM_SLEEP); 2015 for (i = 0; i < sav->sav_count; i++) 2016 l2cache[i] = vdev_config_generate(spa, 2017 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 2018 fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 2019 (const nvlist_t * const *)l2cache, sav->sav_count); 2020 2021 out: 2022 /* 2023 * Purge vdevs that were dropped 2024 */ 2025 if (oldvdevs) { 2026 for (i = 0; i < oldnvdevs; i++) { 2027 uint64_t pool; 2028 2029 vd = oldvdevs[i]; 2030 if (vd != NULL) { 2031 ASSERT(vd->vdev_isl2cache); 2032 2033 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 2034 pool != 0ULL && l2arc_vdev_present(vd)) 2035 l2arc_remove_vdev(vd); 2036 vdev_clear_stats(vd); 2037 vdev_free(vd); 2038 } 2039 } 2040 2041 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2042 } 2043 2044 for (i = 0; i < sav->sav_count; i++) 2045 nvlist_free(l2cache[i]); 2046 if (sav->sav_count) 2047 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2048 } 2049 2050 static int 2051 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2052 { 2053 dmu_buf_t *db; 2054 char *packed = NULL; 2055 size_t nvsize = 0; 2056 int error; 2057 *value = NULL; 2058 2059 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2060 if (error) 2061 return (error); 2062 2063 nvsize = *(uint64_t *)db->db_data; 2064 dmu_buf_rele(db, FTAG); 2065 2066 packed = vmem_alloc(nvsize, KM_SLEEP); 2067 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2068 DMU_READ_PREFETCH); 2069 if (error == 0) 2070 error = nvlist_unpack(packed, nvsize, value, 0); 2071 vmem_free(packed, nvsize); 2072 2073 return (error); 2074 } 2075 2076 /* 2077 * Concrete top-level vdevs that are not missing and are not logs. At every 2078 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2079 */ 2080 static uint64_t 2081 spa_healthy_core_tvds(spa_t *spa) 2082 { 2083 vdev_t *rvd = spa->spa_root_vdev; 2084 uint64_t tvds = 0; 2085 2086 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2087 vdev_t *vd = rvd->vdev_child[i]; 2088 if (vd->vdev_islog) 2089 continue; 2090 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2091 tvds++; 2092 } 2093 2094 return (tvds); 2095 } 2096 2097 /* 2098 * Checks to see if the given vdev could not be opened, in which case we post a 2099 * sysevent to notify the autoreplace code that the device has been removed. 2100 */ 2101 static void 2102 spa_check_removed(vdev_t *vd) 2103 { 2104 for (uint64_t c = 0; c < vd->vdev_children; c++) 2105 spa_check_removed(vd->vdev_child[c]); 2106 2107 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2108 vdev_is_concrete(vd)) { 2109 zfs_post_autoreplace(vd->vdev_spa, vd); 2110 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2111 } 2112 } 2113 2114 static int 2115 spa_check_for_missing_logs(spa_t *spa) 2116 { 2117 vdev_t *rvd = spa->spa_root_vdev; 2118 2119 /* 2120 * If we're doing a normal import, then build up any additional 2121 * diagnostic information about missing log devices. 2122 * We'll pass this up to the user for further processing. 2123 */ 2124 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2125 nvlist_t **child, *nv; 2126 uint64_t idx = 0; 2127 2128 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2129 KM_SLEEP); 2130 nv = fnvlist_alloc(); 2131 2132 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2133 vdev_t *tvd = rvd->vdev_child[c]; 2134 2135 /* 2136 * We consider a device as missing only if it failed 2137 * to open (i.e. offline or faulted is not considered 2138 * as missing). 2139 */ 2140 if (tvd->vdev_islog && 2141 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2142 child[idx++] = vdev_config_generate(spa, tvd, 2143 B_FALSE, VDEV_CONFIG_MISSING); 2144 } 2145 } 2146 2147 if (idx > 0) { 2148 fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 2149 (const nvlist_t * const *)child, idx); 2150 fnvlist_add_nvlist(spa->spa_load_info, 2151 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2152 2153 for (uint64_t i = 0; i < idx; i++) 2154 nvlist_free(child[i]); 2155 } 2156 nvlist_free(nv); 2157 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2158 2159 if (idx > 0) { 2160 spa_load_failed(spa, "some log devices are missing"); 2161 vdev_dbgmsg_print_tree(rvd, 2); 2162 return (SET_ERROR(ENXIO)); 2163 } 2164 } else { 2165 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2166 vdev_t *tvd = rvd->vdev_child[c]; 2167 2168 if (tvd->vdev_islog && 2169 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2170 spa_set_log_state(spa, SPA_LOG_CLEAR); 2171 spa_load_note(spa, "some log devices are " 2172 "missing, ZIL is dropped."); 2173 vdev_dbgmsg_print_tree(rvd, 2); 2174 break; 2175 } 2176 } 2177 } 2178 2179 return (0); 2180 } 2181 2182 /* 2183 * Check for missing log devices 2184 */ 2185 static boolean_t 2186 spa_check_logs(spa_t *spa) 2187 { 2188 boolean_t rv = B_FALSE; 2189 dsl_pool_t *dp = spa_get_dsl(spa); 2190 2191 switch (spa->spa_log_state) { 2192 default: 2193 break; 2194 case SPA_LOG_MISSING: 2195 /* need to recheck in case slog has been restored */ 2196 case SPA_LOG_UNKNOWN: 2197 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2198 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2199 if (rv) 2200 spa_set_log_state(spa, SPA_LOG_MISSING); 2201 break; 2202 } 2203 return (rv); 2204 } 2205 2206 /* 2207 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2208 */ 2209 static boolean_t 2210 spa_passivate_log(spa_t *spa) 2211 { 2212 vdev_t *rvd = spa->spa_root_vdev; 2213 boolean_t slog_found = B_FALSE; 2214 2215 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2216 2217 for (int c = 0; c < rvd->vdev_children; c++) { 2218 vdev_t *tvd = rvd->vdev_child[c]; 2219 2220 if (tvd->vdev_islog) { 2221 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2222 metaslab_group_passivate(tvd->vdev_mg); 2223 slog_found = B_TRUE; 2224 } 2225 } 2226 2227 return (slog_found); 2228 } 2229 2230 /* 2231 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2232 */ 2233 static void 2234 spa_activate_log(spa_t *spa) 2235 { 2236 vdev_t *rvd = spa->spa_root_vdev; 2237 2238 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2239 2240 for (int c = 0; c < rvd->vdev_children; c++) { 2241 vdev_t *tvd = rvd->vdev_child[c]; 2242 2243 if (tvd->vdev_islog) { 2244 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2245 metaslab_group_activate(tvd->vdev_mg); 2246 } 2247 } 2248 } 2249 2250 int 2251 spa_reset_logs(spa_t *spa) 2252 { 2253 int error; 2254 2255 error = dmu_objset_find(spa_name(spa), zil_reset, 2256 NULL, DS_FIND_CHILDREN); 2257 if (error == 0) { 2258 /* 2259 * We successfully offlined the log device, sync out the 2260 * current txg so that the "stubby" block can be removed 2261 * by zil_sync(). 2262 */ 2263 txg_wait_synced(spa->spa_dsl_pool, 0); 2264 } 2265 return (error); 2266 } 2267 2268 static void 2269 spa_aux_check_removed(spa_aux_vdev_t *sav) 2270 { 2271 for (int i = 0; i < sav->sav_count; i++) 2272 spa_check_removed(sav->sav_vdevs[i]); 2273 } 2274 2275 void 2276 spa_claim_notify(zio_t *zio) 2277 { 2278 spa_t *spa = zio->io_spa; 2279 2280 if (zio->io_error) 2281 return; 2282 2283 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2284 if (spa->spa_claim_max_txg < zio->io_bp->blk_birth) 2285 spa->spa_claim_max_txg = zio->io_bp->blk_birth; 2286 mutex_exit(&spa->spa_props_lock); 2287 } 2288 2289 typedef struct spa_load_error { 2290 boolean_t sle_verify_data; 2291 uint64_t sle_meta_count; 2292 uint64_t sle_data_count; 2293 } spa_load_error_t; 2294 2295 static void 2296 spa_load_verify_done(zio_t *zio) 2297 { 2298 blkptr_t *bp = zio->io_bp; 2299 spa_load_error_t *sle = zio->io_private; 2300 dmu_object_type_t type = BP_GET_TYPE(bp); 2301 int error = zio->io_error; 2302 spa_t *spa = zio->io_spa; 2303 2304 abd_free(zio->io_abd); 2305 if (error) { 2306 if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && 2307 type != DMU_OT_INTENT_LOG) 2308 atomic_inc_64(&sle->sle_meta_count); 2309 else 2310 atomic_inc_64(&sle->sle_data_count); 2311 } 2312 2313 mutex_enter(&spa->spa_scrub_lock); 2314 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 2315 cv_broadcast(&spa->spa_scrub_io_cv); 2316 mutex_exit(&spa->spa_scrub_lock); 2317 } 2318 2319 /* 2320 * Maximum number of inflight bytes is the log2 fraction of the arc size. 2321 * By default, we set it to 1/16th of the arc. 2322 */ 2323 static uint_t spa_load_verify_shift = 4; 2324 static int spa_load_verify_metadata = B_TRUE; 2325 static int spa_load_verify_data = B_TRUE; 2326 2327 static int 2328 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 2329 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 2330 { 2331 zio_t *rio = arg; 2332 spa_load_error_t *sle = rio->io_private; 2333 2334 (void) zilog, (void) dnp; 2335 2336 /* 2337 * Note: normally this routine will not be called if 2338 * spa_load_verify_metadata is not set. However, it may be useful 2339 * to manually set the flag after the traversal has begun. 2340 */ 2341 if (!spa_load_verify_metadata) 2342 return (0); 2343 2344 /* 2345 * Sanity check the block pointer in order to detect obvious damage 2346 * before using the contents in subsequent checks or in zio_read(). 2347 * When damaged consider it to be a metadata error since we cannot 2348 * trust the BP_GET_TYPE and BP_GET_LEVEL values. 2349 */ 2350 if (!zfs_blkptr_verify(spa, bp, B_FALSE, BLK_VERIFY_LOG)) { 2351 atomic_inc_64(&sle->sle_meta_count); 2352 return (0); 2353 } 2354 2355 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 2356 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 2357 return (0); 2358 2359 if (!BP_IS_METADATA(bp) && 2360 (!spa_load_verify_data || !sle->sle_verify_data)) 2361 return (0); 2362 2363 uint64_t maxinflight_bytes = 2364 arc_target_bytes() >> spa_load_verify_shift; 2365 size_t size = BP_GET_PSIZE(bp); 2366 2367 mutex_enter(&spa->spa_scrub_lock); 2368 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 2369 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 2370 spa->spa_load_verify_bytes += size; 2371 mutex_exit(&spa->spa_scrub_lock); 2372 2373 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 2374 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 2375 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 2376 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 2377 return (0); 2378 } 2379 2380 static int 2381 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 2382 { 2383 (void) dp, (void) arg; 2384 2385 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 2386 return (SET_ERROR(ENAMETOOLONG)); 2387 2388 return (0); 2389 } 2390 2391 static int 2392 spa_load_verify(spa_t *spa) 2393 { 2394 zio_t *rio; 2395 spa_load_error_t sle = { 0 }; 2396 zpool_load_policy_t policy; 2397 boolean_t verify_ok = B_FALSE; 2398 int error = 0; 2399 2400 zpool_get_load_policy(spa->spa_config, &policy); 2401 2402 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND || 2403 policy.zlp_maxmeta == UINT64_MAX) 2404 return (0); 2405 2406 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 2407 error = dmu_objset_find_dp(spa->spa_dsl_pool, 2408 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 2409 DS_FIND_CHILDREN); 2410 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 2411 if (error != 0) 2412 return (error); 2413 2414 /* 2415 * Verify data only if we are rewinding or error limit was set. 2416 * Otherwise nothing except dbgmsg care about it to waste time. 2417 */ 2418 sle.sle_verify_data = (policy.zlp_rewind & ZPOOL_REWIND_MASK) || 2419 (policy.zlp_maxdata < UINT64_MAX); 2420 2421 rio = zio_root(spa, NULL, &sle, 2422 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 2423 2424 if (spa_load_verify_metadata) { 2425 if (spa->spa_extreme_rewind) { 2426 spa_load_note(spa, "performing a complete scan of the " 2427 "pool since extreme rewind is on. This may take " 2428 "a very long time.\n (spa_load_verify_data=%u, " 2429 "spa_load_verify_metadata=%u)", 2430 spa_load_verify_data, spa_load_verify_metadata); 2431 } 2432 2433 error = traverse_pool(spa, spa->spa_verify_min_txg, 2434 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 2435 TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio); 2436 } 2437 2438 (void) zio_wait(rio); 2439 ASSERT0(spa->spa_load_verify_bytes); 2440 2441 spa->spa_load_meta_errors = sle.sle_meta_count; 2442 spa->spa_load_data_errors = sle.sle_data_count; 2443 2444 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 2445 spa_load_note(spa, "spa_load_verify found %llu metadata errors " 2446 "and %llu data errors", (u_longlong_t)sle.sle_meta_count, 2447 (u_longlong_t)sle.sle_data_count); 2448 } 2449 2450 if (spa_load_verify_dryrun || 2451 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 2452 sle.sle_data_count <= policy.zlp_maxdata)) { 2453 int64_t loss = 0; 2454 2455 verify_ok = B_TRUE; 2456 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 2457 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 2458 2459 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; 2460 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME, 2461 spa->spa_load_txg_ts); 2462 fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME, 2463 loss); 2464 fnvlist_add_uint64(spa->spa_load_info, 2465 ZPOOL_CONFIG_LOAD_META_ERRORS, sle.sle_meta_count); 2466 fnvlist_add_uint64(spa->spa_load_info, 2467 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count); 2468 } else { 2469 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 2470 } 2471 2472 if (spa_load_verify_dryrun) 2473 return (0); 2474 2475 if (error) { 2476 if (error != ENXIO && error != EIO) 2477 error = SET_ERROR(EIO); 2478 return (error); 2479 } 2480 2481 return (verify_ok ? 0 : EIO); 2482 } 2483 2484 /* 2485 * Find a value in the pool props object. 2486 */ 2487 static void 2488 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 2489 { 2490 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 2491 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 2492 } 2493 2494 /* 2495 * Find a value in the pool directory object. 2496 */ 2497 static int 2498 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 2499 { 2500 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2501 name, sizeof (uint64_t), 1, val); 2502 2503 if (error != 0 && (error != ENOENT || log_enoent)) { 2504 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 2505 "[error=%d]", name, error); 2506 } 2507 2508 return (error); 2509 } 2510 2511 static int 2512 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 2513 { 2514 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 2515 return (SET_ERROR(err)); 2516 } 2517 2518 boolean_t 2519 spa_livelist_delete_check(spa_t *spa) 2520 { 2521 return (spa->spa_livelists_to_delete != 0); 2522 } 2523 2524 static boolean_t 2525 spa_livelist_delete_cb_check(void *arg, zthr_t *z) 2526 { 2527 (void) z; 2528 spa_t *spa = arg; 2529 return (spa_livelist_delete_check(spa)); 2530 } 2531 2532 static int 2533 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 2534 { 2535 spa_t *spa = arg; 2536 zio_free(spa, tx->tx_txg, bp); 2537 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 2538 -bp_get_dsize_sync(spa, bp), 2539 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 2540 return (0); 2541 } 2542 2543 static int 2544 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 2545 { 2546 int err; 2547 zap_cursor_t zc; 2548 zap_attribute_t za; 2549 zap_cursor_init(&zc, os, zap_obj); 2550 err = zap_cursor_retrieve(&zc, &za); 2551 zap_cursor_fini(&zc); 2552 if (err == 0) 2553 *llp = za.za_first_integer; 2554 return (err); 2555 } 2556 2557 /* 2558 * Components of livelist deletion that must be performed in syncing 2559 * context: freeing block pointers and updating the pool-wide data 2560 * structures to indicate how much work is left to do 2561 */ 2562 typedef struct sublist_delete_arg { 2563 spa_t *spa; 2564 dsl_deadlist_t *ll; 2565 uint64_t key; 2566 bplist_t *to_free; 2567 } sublist_delete_arg_t; 2568 2569 static void 2570 sublist_delete_sync(void *arg, dmu_tx_t *tx) 2571 { 2572 sublist_delete_arg_t *sda = arg; 2573 spa_t *spa = sda->spa; 2574 dsl_deadlist_t *ll = sda->ll; 2575 uint64_t key = sda->key; 2576 bplist_t *to_free = sda->to_free; 2577 2578 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 2579 dsl_deadlist_remove_entry(ll, key, tx); 2580 } 2581 2582 typedef struct livelist_delete_arg { 2583 spa_t *spa; 2584 uint64_t ll_obj; 2585 uint64_t zap_obj; 2586 } livelist_delete_arg_t; 2587 2588 static void 2589 livelist_delete_sync(void *arg, dmu_tx_t *tx) 2590 { 2591 livelist_delete_arg_t *lda = arg; 2592 spa_t *spa = lda->spa; 2593 uint64_t ll_obj = lda->ll_obj; 2594 uint64_t zap_obj = lda->zap_obj; 2595 objset_t *mos = spa->spa_meta_objset; 2596 uint64_t count; 2597 2598 /* free the livelist and decrement the feature count */ 2599 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 2600 dsl_deadlist_free(mos, ll_obj, tx); 2601 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 2602 VERIFY0(zap_count(mos, zap_obj, &count)); 2603 if (count == 0) { 2604 /* no more livelists to delete */ 2605 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 2606 DMU_POOL_DELETED_CLONES, tx)); 2607 VERIFY0(zap_destroy(mos, zap_obj, tx)); 2608 spa->spa_livelists_to_delete = 0; 2609 spa_notify_waiters(spa); 2610 } 2611 } 2612 2613 /* 2614 * Load in the value for the livelist to be removed and open it. Then, 2615 * load its first sublist and determine which block pointers should actually 2616 * be freed. Then, call a synctask which performs the actual frees and updates 2617 * the pool-wide livelist data. 2618 */ 2619 static void 2620 spa_livelist_delete_cb(void *arg, zthr_t *z) 2621 { 2622 spa_t *spa = arg; 2623 uint64_t ll_obj = 0, count; 2624 objset_t *mos = spa->spa_meta_objset; 2625 uint64_t zap_obj = spa->spa_livelists_to_delete; 2626 /* 2627 * Determine the next livelist to delete. This function should only 2628 * be called if there is at least one deleted clone. 2629 */ 2630 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 2631 VERIFY0(zap_count(mos, ll_obj, &count)); 2632 if (count > 0) { 2633 dsl_deadlist_t *ll; 2634 dsl_deadlist_entry_t *dle; 2635 bplist_t to_free; 2636 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 2637 dsl_deadlist_open(ll, mos, ll_obj); 2638 dle = dsl_deadlist_first(ll); 2639 ASSERT3P(dle, !=, NULL); 2640 bplist_create(&to_free); 2641 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 2642 z, NULL); 2643 if (err == 0) { 2644 sublist_delete_arg_t sync_arg = { 2645 .spa = spa, 2646 .ll = ll, 2647 .key = dle->dle_mintxg, 2648 .to_free = &to_free 2649 }; 2650 zfs_dbgmsg("deleting sublist (id %llu) from" 2651 " livelist %llu, %lld remaining", 2652 (u_longlong_t)dle->dle_bpobj.bpo_object, 2653 (u_longlong_t)ll_obj, (longlong_t)count - 1); 2654 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 2655 sublist_delete_sync, &sync_arg, 0, 2656 ZFS_SPACE_CHECK_DESTROY)); 2657 } else { 2658 VERIFY3U(err, ==, EINTR); 2659 } 2660 bplist_clear(&to_free); 2661 bplist_destroy(&to_free); 2662 dsl_deadlist_close(ll); 2663 kmem_free(ll, sizeof (dsl_deadlist_t)); 2664 } else { 2665 livelist_delete_arg_t sync_arg = { 2666 .spa = spa, 2667 .ll_obj = ll_obj, 2668 .zap_obj = zap_obj 2669 }; 2670 zfs_dbgmsg("deletion of livelist %llu completed", 2671 (u_longlong_t)ll_obj); 2672 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 2673 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 2674 } 2675 } 2676 2677 static void 2678 spa_start_livelist_destroy_thread(spa_t *spa) 2679 { 2680 ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL); 2681 spa->spa_livelist_delete_zthr = 2682 zthr_create("z_livelist_destroy", 2683 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa, 2684 minclsyspri); 2685 } 2686 2687 typedef struct livelist_new_arg { 2688 bplist_t *allocs; 2689 bplist_t *frees; 2690 } livelist_new_arg_t; 2691 2692 static int 2693 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 2694 dmu_tx_t *tx) 2695 { 2696 ASSERT(tx == NULL); 2697 livelist_new_arg_t *lna = arg; 2698 if (bp_freed) { 2699 bplist_append(lna->frees, bp); 2700 } else { 2701 bplist_append(lna->allocs, bp); 2702 zfs_livelist_condense_new_alloc++; 2703 } 2704 return (0); 2705 } 2706 2707 typedef struct livelist_condense_arg { 2708 spa_t *spa; 2709 bplist_t to_keep; 2710 uint64_t first_size; 2711 uint64_t next_size; 2712 } livelist_condense_arg_t; 2713 2714 static void 2715 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 2716 { 2717 livelist_condense_arg_t *lca = arg; 2718 spa_t *spa = lca->spa; 2719 bplist_t new_frees; 2720 dsl_dataset_t *ds = spa->spa_to_condense.ds; 2721 2722 /* Have we been cancelled? */ 2723 if (spa->spa_to_condense.cancelled) { 2724 zfs_livelist_condense_sync_cancel++; 2725 goto out; 2726 } 2727 2728 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2729 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2730 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 2731 2732 /* 2733 * It's possible that the livelist was changed while the zthr was 2734 * running. Therefore, we need to check for new blkptrs in the two 2735 * entries being condensed and continue to track them in the livelist. 2736 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 2737 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 2738 * we need to sort them into two different bplists. 2739 */ 2740 uint64_t first_obj = first->dle_bpobj.bpo_object; 2741 uint64_t next_obj = next->dle_bpobj.bpo_object; 2742 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2743 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2744 2745 bplist_create(&new_frees); 2746 livelist_new_arg_t new_bps = { 2747 .allocs = &lca->to_keep, 2748 .frees = &new_frees, 2749 }; 2750 2751 if (cur_first_size > lca->first_size) { 2752 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 2753 livelist_track_new_cb, &new_bps, lca->first_size)); 2754 } 2755 if (cur_next_size > lca->next_size) { 2756 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 2757 livelist_track_new_cb, &new_bps, lca->next_size)); 2758 } 2759 2760 dsl_deadlist_clear_entry(first, ll, tx); 2761 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 2762 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 2763 2764 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 2765 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 2766 bplist_destroy(&new_frees); 2767 2768 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 2769 dsl_dataset_name(ds, dsname); 2770 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 2771 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 2772 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, 2773 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, 2774 (u_longlong_t)cur_first_size, (u_longlong_t)next_obj, 2775 (u_longlong_t)cur_next_size, 2776 (u_longlong_t)first->dle_bpobj.bpo_object, 2777 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 2778 out: 2779 dmu_buf_rele(ds->ds_dbuf, spa); 2780 spa->spa_to_condense.ds = NULL; 2781 bplist_clear(&lca->to_keep); 2782 bplist_destroy(&lca->to_keep); 2783 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2784 spa->spa_to_condense.syncing = B_FALSE; 2785 } 2786 2787 static void 2788 spa_livelist_condense_cb(void *arg, zthr_t *t) 2789 { 2790 while (zfs_livelist_condense_zthr_pause && 2791 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2792 delay(1); 2793 2794 spa_t *spa = arg; 2795 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2796 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2797 uint64_t first_size, next_size; 2798 2799 livelist_condense_arg_t *lca = 2800 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 2801 bplist_create(&lca->to_keep); 2802 2803 /* 2804 * Process the livelists (matching FREEs and ALLOCs) in open context 2805 * so we have minimal work in syncing context to condense. 2806 * 2807 * We save bpobj sizes (first_size and next_size) to use later in 2808 * syncing context to determine if entries were added to these sublists 2809 * while in open context. This is possible because the clone is still 2810 * active and open for normal writes and we want to make sure the new, 2811 * unprocessed blockpointers are inserted into the livelist normally. 2812 * 2813 * Note that dsl_process_sub_livelist() both stores the size number of 2814 * blockpointers and iterates over them while the bpobj's lock held, so 2815 * the sizes returned to us are consistent which what was actually 2816 * processed. 2817 */ 2818 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 2819 &first_size); 2820 if (err == 0) 2821 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 2822 t, &next_size); 2823 2824 if (err == 0) { 2825 while (zfs_livelist_condense_sync_pause && 2826 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2827 delay(1); 2828 2829 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2830 dmu_tx_mark_netfree(tx); 2831 dmu_tx_hold_space(tx, 1); 2832 err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE); 2833 if (err == 0) { 2834 /* 2835 * Prevent the condense zthr restarting before 2836 * the synctask completes. 2837 */ 2838 spa->spa_to_condense.syncing = B_TRUE; 2839 lca->spa = spa; 2840 lca->first_size = first_size; 2841 lca->next_size = next_size; 2842 dsl_sync_task_nowait(spa_get_dsl(spa), 2843 spa_livelist_condense_sync, lca, tx); 2844 dmu_tx_commit(tx); 2845 return; 2846 } 2847 } 2848 /* 2849 * Condensing can not continue: either it was externally stopped or 2850 * we were unable to assign to a tx because the pool has run out of 2851 * space. In the second case, we'll just end up trying to condense 2852 * again in a later txg. 2853 */ 2854 ASSERT(err != 0); 2855 bplist_clear(&lca->to_keep); 2856 bplist_destroy(&lca->to_keep); 2857 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2858 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 2859 spa->spa_to_condense.ds = NULL; 2860 if (err == EINTR) 2861 zfs_livelist_condense_zthr_cancel++; 2862 } 2863 2864 /* 2865 * Check that there is something to condense but that a condense is not 2866 * already in progress and that condensing has not been cancelled. 2867 */ 2868 static boolean_t 2869 spa_livelist_condense_cb_check(void *arg, zthr_t *z) 2870 { 2871 (void) z; 2872 spa_t *spa = arg; 2873 if ((spa->spa_to_condense.ds != NULL) && 2874 (spa->spa_to_condense.syncing == B_FALSE) && 2875 (spa->spa_to_condense.cancelled == B_FALSE)) { 2876 return (B_TRUE); 2877 } 2878 return (B_FALSE); 2879 } 2880 2881 static void 2882 spa_start_livelist_condensing_thread(spa_t *spa) 2883 { 2884 spa->spa_to_condense.ds = NULL; 2885 spa->spa_to_condense.first = NULL; 2886 spa->spa_to_condense.next = NULL; 2887 spa->spa_to_condense.syncing = B_FALSE; 2888 spa->spa_to_condense.cancelled = B_FALSE; 2889 2890 ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL); 2891 spa->spa_livelist_condense_zthr = 2892 zthr_create("z_livelist_condense", 2893 spa_livelist_condense_cb_check, 2894 spa_livelist_condense_cb, spa, minclsyspri); 2895 } 2896 2897 static void 2898 spa_spawn_aux_threads(spa_t *spa) 2899 { 2900 ASSERT(spa_writeable(spa)); 2901 2902 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 2903 2904 spa_start_indirect_condensing_thread(spa); 2905 spa_start_livelist_destroy_thread(spa); 2906 spa_start_livelist_condensing_thread(spa); 2907 2908 ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL); 2909 spa->spa_checkpoint_discard_zthr = 2910 zthr_create("z_checkpoint_discard", 2911 spa_checkpoint_discard_thread_check, 2912 spa_checkpoint_discard_thread, spa, minclsyspri); 2913 } 2914 2915 /* 2916 * Fix up config after a partly-completed split. This is done with the 2917 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 2918 * pool have that entry in their config, but only the splitting one contains 2919 * a list of all the guids of the vdevs that are being split off. 2920 * 2921 * This function determines what to do with that list: either rejoin 2922 * all the disks to the pool, or complete the splitting process. To attempt 2923 * the rejoin, each disk that is offlined is marked online again, and 2924 * we do a reopen() call. If the vdev label for every disk that was 2925 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 2926 * then we call vdev_split() on each disk, and complete the split. 2927 * 2928 * Otherwise we leave the config alone, with all the vdevs in place in 2929 * the original pool. 2930 */ 2931 static void 2932 spa_try_repair(spa_t *spa, nvlist_t *config) 2933 { 2934 uint_t extracted; 2935 uint64_t *glist; 2936 uint_t i, gcount; 2937 nvlist_t *nvl; 2938 vdev_t **vd; 2939 boolean_t attempt_reopen; 2940 2941 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 2942 return; 2943 2944 /* check that the config is complete */ 2945 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 2946 &glist, &gcount) != 0) 2947 return; 2948 2949 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 2950 2951 /* attempt to online all the vdevs & validate */ 2952 attempt_reopen = B_TRUE; 2953 for (i = 0; i < gcount; i++) { 2954 if (glist[i] == 0) /* vdev is hole */ 2955 continue; 2956 2957 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 2958 if (vd[i] == NULL) { 2959 /* 2960 * Don't bother attempting to reopen the disks; 2961 * just do the split. 2962 */ 2963 attempt_reopen = B_FALSE; 2964 } else { 2965 /* attempt to re-online it */ 2966 vd[i]->vdev_offline = B_FALSE; 2967 } 2968 } 2969 2970 if (attempt_reopen) { 2971 vdev_reopen(spa->spa_root_vdev); 2972 2973 /* check each device to see what state it's in */ 2974 for (extracted = 0, i = 0; i < gcount; i++) { 2975 if (vd[i] != NULL && 2976 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 2977 break; 2978 ++extracted; 2979 } 2980 } 2981 2982 /* 2983 * If every disk has been moved to the new pool, or if we never 2984 * even attempted to look at them, then we split them off for 2985 * good. 2986 */ 2987 if (!attempt_reopen || gcount == extracted) { 2988 for (i = 0; i < gcount; i++) 2989 if (vd[i] != NULL) 2990 vdev_split(vd[i]); 2991 vdev_reopen(spa->spa_root_vdev); 2992 } 2993 2994 kmem_free(vd, gcount * sizeof (vdev_t *)); 2995 } 2996 2997 static int 2998 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 2999 { 3000 const char *ereport = FM_EREPORT_ZFS_POOL; 3001 int error; 3002 3003 spa->spa_load_state = state; 3004 (void) spa_import_progress_set_state(spa_guid(spa), 3005 spa_load_state(spa)); 3006 3007 gethrestime(&spa->spa_loaded_ts); 3008 error = spa_load_impl(spa, type, &ereport); 3009 3010 /* 3011 * Don't count references from objsets that are already closed 3012 * and are making their way through the eviction process. 3013 */ 3014 spa_evicting_os_wait(spa); 3015 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 3016 if (error) { 3017 if (error != EEXIST) { 3018 spa->spa_loaded_ts.tv_sec = 0; 3019 spa->spa_loaded_ts.tv_nsec = 0; 3020 } 3021 if (error != EBADF) { 3022 (void) zfs_ereport_post(ereport, spa, 3023 NULL, NULL, NULL, 0); 3024 } 3025 } 3026 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 3027 spa->spa_ena = 0; 3028 3029 (void) spa_import_progress_set_state(spa_guid(spa), 3030 spa_load_state(spa)); 3031 3032 return (error); 3033 } 3034 3035 #ifdef ZFS_DEBUG 3036 /* 3037 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 3038 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 3039 * spa's per-vdev ZAP list. 3040 */ 3041 static uint64_t 3042 vdev_count_verify_zaps(vdev_t *vd) 3043 { 3044 spa_t *spa = vd->vdev_spa; 3045 uint64_t total = 0; 3046 3047 if (vd->vdev_top_zap != 0) { 3048 total++; 3049 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3050 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 3051 } 3052 if (vd->vdev_leaf_zap != 0) { 3053 total++; 3054 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3055 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 3056 } 3057 3058 for (uint64_t i = 0; i < vd->vdev_children; i++) { 3059 total += vdev_count_verify_zaps(vd->vdev_child[i]); 3060 } 3061 3062 return (total); 3063 } 3064 #else 3065 #define vdev_count_verify_zaps(vd) ((void) sizeof (vd), 0) 3066 #endif 3067 3068 /* 3069 * Determine whether the activity check is required. 3070 */ 3071 static boolean_t 3072 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label, 3073 nvlist_t *config) 3074 { 3075 uint64_t state = 0; 3076 uint64_t hostid = 0; 3077 uint64_t tryconfig_txg = 0; 3078 uint64_t tryconfig_timestamp = 0; 3079 uint16_t tryconfig_mmp_seq = 0; 3080 nvlist_t *nvinfo; 3081 3082 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3083 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 3084 (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3085 &tryconfig_txg); 3086 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3087 &tryconfig_timestamp); 3088 (void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3089 &tryconfig_mmp_seq); 3090 } 3091 3092 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); 3093 3094 /* 3095 * Disable the MMP activity check - This is used by zdb which 3096 * is intended to be used on potentially active pools. 3097 */ 3098 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) 3099 return (B_FALSE); 3100 3101 /* 3102 * Skip the activity check when the MMP feature is disabled. 3103 */ 3104 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) 3105 return (B_FALSE); 3106 3107 /* 3108 * If the tryconfig_ values are nonzero, they are the results of an 3109 * earlier tryimport. If they all match the uberblock we just found, 3110 * then the pool has not changed and we return false so we do not test 3111 * a second time. 3112 */ 3113 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && 3114 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && 3115 tryconfig_mmp_seq && tryconfig_mmp_seq == 3116 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) 3117 return (B_FALSE); 3118 3119 /* 3120 * Allow the activity check to be skipped when importing the pool 3121 * on the same host which last imported it. Since the hostid from 3122 * configuration may be stale use the one read from the label. 3123 */ 3124 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3125 hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID); 3126 3127 if (hostid == spa_get_hostid(spa)) 3128 return (B_FALSE); 3129 3130 /* 3131 * Skip the activity test when the pool was cleanly exported. 3132 */ 3133 if (state != POOL_STATE_ACTIVE) 3134 return (B_FALSE); 3135 3136 return (B_TRUE); 3137 } 3138 3139 /* 3140 * Nanoseconds the activity check must watch for changes on-disk. 3141 */ 3142 static uint64_t 3143 spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3144 { 3145 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 3146 uint64_t multihost_interval = MSEC2NSEC( 3147 MMP_INTERVAL_OK(zfs_multihost_interval)); 3148 uint64_t import_delay = MAX(NANOSEC, import_intervals * 3149 multihost_interval); 3150 3151 /* 3152 * Local tunables determine a minimum duration except for the case 3153 * where we know when the remote host will suspend the pool if MMP 3154 * writes do not land. 3155 * 3156 * See Big Theory comment at the top of mmp.c for the reasoning behind 3157 * these cases and times. 3158 */ 3159 3160 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 3161 3162 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3163 MMP_FAIL_INT(ub) > 0) { 3164 3165 /* MMP on remote host will suspend pool after failed writes */ 3166 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 3167 MMP_IMPORT_SAFETY_FACTOR / 100; 3168 3169 zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp " 3170 "mmp_fails=%llu ub_mmp mmp_interval=%llu " 3171 "import_intervals=%llu", (u_longlong_t)import_delay, 3172 (u_longlong_t)MMP_FAIL_INT(ub), 3173 (u_longlong_t)MMP_INTERVAL(ub), 3174 (u_longlong_t)import_intervals); 3175 3176 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3177 MMP_FAIL_INT(ub) == 0) { 3178 3179 /* MMP on remote host will never suspend pool */ 3180 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 3181 ub->ub_mmp_delay) * import_intervals); 3182 3183 zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp " 3184 "mmp_interval=%llu ub_mmp_delay=%llu " 3185 "import_intervals=%llu", (u_longlong_t)import_delay, 3186 (u_longlong_t)MMP_INTERVAL(ub), 3187 (u_longlong_t)ub->ub_mmp_delay, 3188 (u_longlong_t)import_intervals); 3189 3190 } else if (MMP_VALID(ub)) { 3191 /* 3192 * zfs-0.7 compatibility case 3193 */ 3194 3195 import_delay = MAX(import_delay, (multihost_interval + 3196 ub->ub_mmp_delay) * import_intervals); 3197 3198 zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu " 3199 "import_intervals=%llu leaves=%u", 3200 (u_longlong_t)import_delay, 3201 (u_longlong_t)ub->ub_mmp_delay, 3202 (u_longlong_t)import_intervals, 3203 vdev_count_leaves(spa)); 3204 } else { 3205 /* Using local tunings is the only reasonable option */ 3206 zfs_dbgmsg("pool last imported on non-MMP aware " 3207 "host using import_delay=%llu multihost_interval=%llu " 3208 "import_intervals=%llu", (u_longlong_t)import_delay, 3209 (u_longlong_t)multihost_interval, 3210 (u_longlong_t)import_intervals); 3211 } 3212 3213 return (import_delay); 3214 } 3215 3216 /* 3217 * Perform the import activity check. If the user canceled the import or 3218 * we detected activity then fail. 3219 */ 3220 static int 3221 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config) 3222 { 3223 uint64_t txg = ub->ub_txg; 3224 uint64_t timestamp = ub->ub_timestamp; 3225 uint64_t mmp_config = ub->ub_mmp_config; 3226 uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0; 3227 uint64_t import_delay; 3228 hrtime_t import_expire; 3229 nvlist_t *mmp_label = NULL; 3230 vdev_t *rvd = spa->spa_root_vdev; 3231 kcondvar_t cv; 3232 kmutex_t mtx; 3233 int error = 0; 3234 3235 cv_init(&cv, NULL, CV_DEFAULT, NULL); 3236 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 3237 mutex_enter(&mtx); 3238 3239 /* 3240 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3241 * during the earlier tryimport. If the txg recorded there is 0 then 3242 * the pool is known to be active on another host. 3243 * 3244 * Otherwise, the pool might be in use on another host. Check for 3245 * changes in the uberblocks on disk if necessary. 3246 */ 3247 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3248 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 3249 ZPOOL_CONFIG_LOAD_INFO); 3250 3251 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && 3252 fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { 3253 vdev_uberblock_load(rvd, ub, &mmp_label); 3254 error = SET_ERROR(EREMOTEIO); 3255 goto out; 3256 } 3257 } 3258 3259 import_delay = spa_activity_check_duration(spa, ub); 3260 3261 /* Add a small random factor in case of simultaneous imports (0-25%) */ 3262 import_delay += import_delay * random_in_range(250) / 1000; 3263 3264 import_expire = gethrtime() + import_delay; 3265 3266 while (gethrtime() < import_expire) { 3267 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 3268 NSEC2SEC(import_expire - gethrtime())); 3269 3270 vdev_uberblock_load(rvd, ub, &mmp_label); 3271 3272 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || 3273 mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) { 3274 zfs_dbgmsg("multihost activity detected " 3275 "txg %llu ub_txg %llu " 3276 "timestamp %llu ub_timestamp %llu " 3277 "mmp_config %#llx ub_mmp_config %#llx", 3278 (u_longlong_t)txg, (u_longlong_t)ub->ub_txg, 3279 (u_longlong_t)timestamp, 3280 (u_longlong_t)ub->ub_timestamp, 3281 (u_longlong_t)mmp_config, 3282 (u_longlong_t)ub->ub_mmp_config); 3283 3284 error = SET_ERROR(EREMOTEIO); 3285 break; 3286 } 3287 3288 if (mmp_label) { 3289 nvlist_free(mmp_label); 3290 mmp_label = NULL; 3291 } 3292 3293 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 3294 if (error != -1) { 3295 error = SET_ERROR(EINTR); 3296 break; 3297 } 3298 error = 0; 3299 } 3300 3301 out: 3302 mutex_exit(&mtx); 3303 mutex_destroy(&mtx); 3304 cv_destroy(&cv); 3305 3306 /* 3307 * If the pool is determined to be active store the status in the 3308 * spa->spa_load_info nvlist. If the remote hostname or hostid are 3309 * available from configuration read from disk store them as well. 3310 * This allows 'zpool import' to generate a more useful message. 3311 * 3312 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) 3313 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 3314 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 3315 */ 3316 if (error == EREMOTEIO) { 3317 const char *hostname = "<unknown>"; 3318 uint64_t hostid = 0; 3319 3320 if (mmp_label) { 3321 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { 3322 hostname = fnvlist_lookup_string(mmp_label, 3323 ZPOOL_CONFIG_HOSTNAME); 3324 fnvlist_add_string(spa->spa_load_info, 3325 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 3326 } 3327 3328 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { 3329 hostid = fnvlist_lookup_uint64(mmp_label, 3330 ZPOOL_CONFIG_HOSTID); 3331 fnvlist_add_uint64(spa->spa_load_info, 3332 ZPOOL_CONFIG_MMP_HOSTID, hostid); 3333 } 3334 } 3335 3336 fnvlist_add_uint64(spa->spa_load_info, 3337 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); 3338 fnvlist_add_uint64(spa->spa_load_info, 3339 ZPOOL_CONFIG_MMP_TXG, 0); 3340 3341 error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); 3342 } 3343 3344 if (mmp_label) 3345 nvlist_free(mmp_label); 3346 3347 return (error); 3348 } 3349 3350 static int 3351 spa_verify_host(spa_t *spa, nvlist_t *mos_config) 3352 { 3353 uint64_t hostid; 3354 const char *hostname; 3355 uint64_t myhostid = 0; 3356 3357 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 3358 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 3359 hostname = fnvlist_lookup_string(mos_config, 3360 ZPOOL_CONFIG_HOSTNAME); 3361 3362 myhostid = zone_get_hostid(NULL); 3363 3364 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 3365 cmn_err(CE_WARN, "pool '%s' could not be " 3366 "loaded as it was last accessed by " 3367 "another system (host: %s hostid: 0x%llx). " 3368 "See: https://openzfs.github.io/openzfs-docs/msg/" 3369 "ZFS-8000-EY", 3370 spa_name(spa), hostname, (u_longlong_t)hostid); 3371 spa_load_failed(spa, "hostid verification failed: pool " 3372 "last accessed by host: %s (hostid: 0x%llx)", 3373 hostname, (u_longlong_t)hostid); 3374 return (SET_ERROR(EBADF)); 3375 } 3376 } 3377 3378 return (0); 3379 } 3380 3381 static int 3382 spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 3383 { 3384 int error = 0; 3385 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 3386 int parse; 3387 vdev_t *rvd; 3388 uint64_t pool_guid; 3389 const char *comment; 3390 const char *compatibility; 3391 3392 /* 3393 * Versioning wasn't explicitly added to the label until later, so if 3394 * it's not present treat it as the initial version. 3395 */ 3396 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 3397 &spa->spa_ubsync.ub_version) != 0) 3398 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 3399 3400 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 3401 spa_load_failed(spa, "invalid config provided: '%s' missing", 3402 ZPOOL_CONFIG_POOL_GUID); 3403 return (SET_ERROR(EINVAL)); 3404 } 3405 3406 /* 3407 * If we are doing an import, ensure that the pool is not already 3408 * imported by checking if its pool guid already exists in the 3409 * spa namespace. 3410 * 3411 * The only case that we allow an already imported pool to be 3412 * imported again, is when the pool is checkpointed and we want to 3413 * look at its checkpointed state from userland tools like zdb. 3414 */ 3415 #ifdef _KERNEL 3416 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3417 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3418 spa_guid_exists(pool_guid, 0)) { 3419 #else 3420 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3421 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3422 spa_guid_exists(pool_guid, 0) && 3423 !spa_importing_readonly_checkpoint(spa)) { 3424 #endif 3425 spa_load_failed(spa, "a pool with guid %llu is already open", 3426 (u_longlong_t)pool_guid); 3427 return (SET_ERROR(EEXIST)); 3428 } 3429 3430 spa->spa_config_guid = pool_guid; 3431 3432 nvlist_free(spa->spa_load_info); 3433 spa->spa_load_info = fnvlist_alloc(); 3434 3435 ASSERT(spa->spa_comment == NULL); 3436 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 3437 spa->spa_comment = spa_strdup(comment); 3438 3439 ASSERT(spa->spa_compatibility == NULL); 3440 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 3441 &compatibility) == 0) 3442 spa->spa_compatibility = spa_strdup(compatibility); 3443 3444 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 3445 &spa->spa_config_txg); 3446 3447 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 3448 spa->spa_config_splitting = fnvlist_dup(nvl); 3449 3450 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 3451 spa_load_failed(spa, "invalid config provided: '%s' missing", 3452 ZPOOL_CONFIG_VDEV_TREE); 3453 return (SET_ERROR(EINVAL)); 3454 } 3455 3456 /* 3457 * Create "The Godfather" zio to hold all async IOs 3458 */ 3459 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 3460 KM_SLEEP); 3461 for (int i = 0; i < max_ncpus; i++) { 3462 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 3463 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3464 ZIO_FLAG_GODFATHER); 3465 } 3466 3467 /* 3468 * Parse the configuration into a vdev tree. We explicitly set the 3469 * value that will be returned by spa_version() since parsing the 3470 * configuration requires knowing the version number. 3471 */ 3472 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3473 parse = (type == SPA_IMPORT_EXISTING ? 3474 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 3475 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 3476 spa_config_exit(spa, SCL_ALL, FTAG); 3477 3478 if (error != 0) { 3479 spa_load_failed(spa, "unable to parse config [error=%d]", 3480 error); 3481 return (error); 3482 } 3483 3484 ASSERT(spa->spa_root_vdev == rvd); 3485 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 3486 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 3487 3488 if (type != SPA_IMPORT_ASSEMBLE) { 3489 ASSERT(spa_guid(spa) == pool_guid); 3490 } 3491 3492 return (0); 3493 } 3494 3495 /* 3496 * Recursively open all vdevs in the vdev tree. This function is called twice: 3497 * first with the untrusted config, then with the trusted config. 3498 */ 3499 static int 3500 spa_ld_open_vdevs(spa_t *spa) 3501 { 3502 int error = 0; 3503 3504 /* 3505 * spa_missing_tvds_allowed defines how many top-level vdevs can be 3506 * missing/unopenable for the root vdev to be still considered openable. 3507 */ 3508 if (spa->spa_trust_config) { 3509 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 3510 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 3511 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 3512 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 3513 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 3514 } else { 3515 spa->spa_missing_tvds_allowed = 0; 3516 } 3517 3518 spa->spa_missing_tvds_allowed = 3519 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 3520 3521 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3522 error = vdev_open(spa->spa_root_vdev); 3523 spa_config_exit(spa, SCL_ALL, FTAG); 3524 3525 if (spa->spa_missing_tvds != 0) { 3526 spa_load_note(spa, "vdev tree has %lld missing top-level " 3527 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 3528 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 3529 /* 3530 * Although theoretically we could allow users to open 3531 * incomplete pools in RW mode, we'd need to add a lot 3532 * of extra logic (e.g. adjust pool space to account 3533 * for missing vdevs). 3534 * This limitation also prevents users from accidentally 3535 * opening the pool in RW mode during data recovery and 3536 * damaging it further. 3537 */ 3538 spa_load_note(spa, "pools with missing top-level " 3539 "vdevs can only be opened in read-only mode."); 3540 error = SET_ERROR(ENXIO); 3541 } else { 3542 spa_load_note(spa, "current settings allow for maximum " 3543 "%lld missing top-level vdevs at this stage.", 3544 (u_longlong_t)spa->spa_missing_tvds_allowed); 3545 } 3546 } 3547 if (error != 0) { 3548 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 3549 error); 3550 } 3551 if (spa->spa_missing_tvds != 0 || error != 0) 3552 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 3553 3554 return (error); 3555 } 3556 3557 /* 3558 * We need to validate the vdev labels against the configuration that 3559 * we have in hand. This function is called twice: first with an untrusted 3560 * config, then with a trusted config. The validation is more strict when the 3561 * config is trusted. 3562 */ 3563 static int 3564 spa_ld_validate_vdevs(spa_t *spa) 3565 { 3566 int error = 0; 3567 vdev_t *rvd = spa->spa_root_vdev; 3568 3569 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3570 error = vdev_validate(rvd); 3571 spa_config_exit(spa, SCL_ALL, FTAG); 3572 3573 if (error != 0) { 3574 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 3575 return (error); 3576 } 3577 3578 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 3579 spa_load_failed(spa, "cannot open vdev tree after invalidating " 3580 "some vdevs"); 3581 vdev_dbgmsg_print_tree(rvd, 2); 3582 return (SET_ERROR(ENXIO)); 3583 } 3584 3585 return (0); 3586 } 3587 3588 static void 3589 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 3590 { 3591 spa->spa_state = POOL_STATE_ACTIVE; 3592 spa->spa_ubsync = spa->spa_uberblock; 3593 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 3594 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 3595 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 3596 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 3597 spa->spa_claim_max_txg = spa->spa_first_txg; 3598 spa->spa_prev_software_version = ub->ub_software_version; 3599 } 3600 3601 static int 3602 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 3603 { 3604 vdev_t *rvd = spa->spa_root_vdev; 3605 nvlist_t *label; 3606 uberblock_t *ub = &spa->spa_uberblock; 3607 boolean_t activity_check = B_FALSE; 3608 3609 /* 3610 * If we are opening the checkpointed state of the pool by 3611 * rewinding to it, at this point we will have written the 3612 * checkpointed uberblock to the vdev labels, so searching 3613 * the labels will find the right uberblock. However, if 3614 * we are opening the checkpointed state read-only, we have 3615 * not modified the labels. Therefore, we must ignore the 3616 * labels and continue using the spa_uberblock that was set 3617 * by spa_ld_checkpoint_rewind. 3618 * 3619 * Note that it would be fine to ignore the labels when 3620 * rewinding (opening writeable) as well. However, if we 3621 * crash just after writing the labels, we will end up 3622 * searching the labels. Doing so in the common case means 3623 * that this code path gets exercised normally, rather than 3624 * just in the edge case. 3625 */ 3626 if (ub->ub_checkpoint_txg != 0 && 3627 spa_importing_readonly_checkpoint(spa)) { 3628 spa_ld_select_uberblock_done(spa, ub); 3629 return (0); 3630 } 3631 3632 /* 3633 * Find the best uberblock. 3634 */ 3635 vdev_uberblock_load(rvd, ub, &label); 3636 3637 /* 3638 * If we weren't able to find a single valid uberblock, return failure. 3639 */ 3640 if (ub->ub_txg == 0) { 3641 nvlist_free(label); 3642 spa_load_failed(spa, "no valid uberblock found"); 3643 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 3644 } 3645 3646 if (spa->spa_load_max_txg != UINT64_MAX) { 3647 (void) spa_import_progress_set_max_txg(spa_guid(spa), 3648 (u_longlong_t)spa->spa_load_max_txg); 3649 } 3650 spa_load_note(spa, "using uberblock with txg=%llu", 3651 (u_longlong_t)ub->ub_txg); 3652 3653 3654 /* 3655 * For pools which have the multihost property on determine if the 3656 * pool is truly inactive and can be safely imported. Prevent 3657 * hosts which don't have a hostid set from importing the pool. 3658 */ 3659 activity_check = spa_activity_check_required(spa, ub, label, 3660 spa->spa_config); 3661 if (activity_check) { 3662 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 3663 spa_get_hostid(spa) == 0) { 3664 nvlist_free(label); 3665 fnvlist_add_uint64(spa->spa_load_info, 3666 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 3667 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 3668 } 3669 3670 int error = spa_activity_check(spa, ub, spa->spa_config); 3671 if (error) { 3672 nvlist_free(label); 3673 return (error); 3674 } 3675 3676 fnvlist_add_uint64(spa->spa_load_info, 3677 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); 3678 fnvlist_add_uint64(spa->spa_load_info, 3679 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); 3680 fnvlist_add_uint16(spa->spa_load_info, 3681 ZPOOL_CONFIG_MMP_SEQ, 3682 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)); 3683 } 3684 3685 /* 3686 * If the pool has an unsupported version we can't open it. 3687 */ 3688 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 3689 nvlist_free(label); 3690 spa_load_failed(spa, "version %llu is not supported", 3691 (u_longlong_t)ub->ub_version); 3692 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 3693 } 3694 3695 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3696 nvlist_t *features; 3697 3698 /* 3699 * If we weren't able to find what's necessary for reading the 3700 * MOS in the label, return failure. 3701 */ 3702 if (label == NULL) { 3703 spa_load_failed(spa, "label config unavailable"); 3704 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3705 ENXIO)); 3706 } 3707 3708 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 3709 &features) != 0) { 3710 nvlist_free(label); 3711 spa_load_failed(spa, "invalid label: '%s' missing", 3712 ZPOOL_CONFIG_FEATURES_FOR_READ); 3713 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3714 ENXIO)); 3715 } 3716 3717 /* 3718 * Update our in-core representation with the definitive values 3719 * from the label. 3720 */ 3721 nvlist_free(spa->spa_label_features); 3722 spa->spa_label_features = fnvlist_dup(features); 3723 } 3724 3725 nvlist_free(label); 3726 3727 /* 3728 * Look through entries in the label nvlist's features_for_read. If 3729 * there is a feature listed there which we don't understand then we 3730 * cannot open a pool. 3731 */ 3732 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3733 nvlist_t *unsup_feat; 3734 3735 unsup_feat = fnvlist_alloc(); 3736 3737 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 3738 NULL); nvp != NULL; 3739 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 3740 if (!zfeature_is_supported(nvpair_name(nvp))) { 3741 fnvlist_add_string(unsup_feat, 3742 nvpair_name(nvp), ""); 3743 } 3744 } 3745 3746 if (!nvlist_empty(unsup_feat)) { 3747 fnvlist_add_nvlist(spa->spa_load_info, 3748 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 3749 nvlist_free(unsup_feat); 3750 spa_load_failed(spa, "some features are unsupported"); 3751 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 3752 ENOTSUP)); 3753 } 3754 3755 nvlist_free(unsup_feat); 3756 } 3757 3758 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 3759 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3760 spa_try_repair(spa, spa->spa_config); 3761 spa_config_exit(spa, SCL_ALL, FTAG); 3762 nvlist_free(spa->spa_config_splitting); 3763 spa->spa_config_splitting = NULL; 3764 } 3765 3766 /* 3767 * Initialize internal SPA structures. 3768 */ 3769 spa_ld_select_uberblock_done(spa, ub); 3770 3771 return (0); 3772 } 3773 3774 static int 3775 spa_ld_open_rootbp(spa_t *spa) 3776 { 3777 int error = 0; 3778 vdev_t *rvd = spa->spa_root_vdev; 3779 3780 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 3781 if (error != 0) { 3782 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 3783 "[error=%d]", error); 3784 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3785 } 3786 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 3787 3788 return (0); 3789 } 3790 3791 static int 3792 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 3793 boolean_t reloading) 3794 { 3795 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 3796 nvlist_t *nv, *mos_config, *policy; 3797 int error = 0, copy_error; 3798 uint64_t healthy_tvds, healthy_tvds_mos; 3799 uint64_t mos_config_txg; 3800 3801 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 3802 != 0) 3803 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3804 3805 /* 3806 * If we're assembling a pool from a split, the config provided is 3807 * already trusted so there is nothing to do. 3808 */ 3809 if (type == SPA_IMPORT_ASSEMBLE) 3810 return (0); 3811 3812 healthy_tvds = spa_healthy_core_tvds(spa); 3813 3814 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 3815 != 0) { 3816 spa_load_failed(spa, "unable to retrieve MOS config"); 3817 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3818 } 3819 3820 /* 3821 * If we are doing an open, pool owner wasn't verified yet, thus do 3822 * the verification here. 3823 */ 3824 if (spa->spa_load_state == SPA_LOAD_OPEN) { 3825 error = spa_verify_host(spa, mos_config); 3826 if (error != 0) { 3827 nvlist_free(mos_config); 3828 return (error); 3829 } 3830 } 3831 3832 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 3833 3834 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3835 3836 /* 3837 * Build a new vdev tree from the trusted config 3838 */ 3839 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 3840 if (error != 0) { 3841 nvlist_free(mos_config); 3842 spa_config_exit(spa, SCL_ALL, FTAG); 3843 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 3844 error); 3845 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3846 } 3847 3848 /* 3849 * Vdev paths in the MOS may be obsolete. If the untrusted config was 3850 * obtained by scanning /dev/dsk, then it will have the right vdev 3851 * paths. We update the trusted MOS config with this information. 3852 * We first try to copy the paths with vdev_copy_path_strict, which 3853 * succeeds only when both configs have exactly the same vdev tree. 3854 * If that fails, we fall back to a more flexible method that has a 3855 * best effort policy. 3856 */ 3857 copy_error = vdev_copy_path_strict(rvd, mrvd); 3858 if (copy_error != 0 || spa_load_print_vdev_tree) { 3859 spa_load_note(spa, "provided vdev tree:"); 3860 vdev_dbgmsg_print_tree(rvd, 2); 3861 spa_load_note(spa, "MOS vdev tree:"); 3862 vdev_dbgmsg_print_tree(mrvd, 2); 3863 } 3864 if (copy_error != 0) { 3865 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 3866 "back to vdev_copy_path_relaxed"); 3867 vdev_copy_path_relaxed(rvd, mrvd); 3868 } 3869 3870 vdev_close(rvd); 3871 vdev_free(rvd); 3872 spa->spa_root_vdev = mrvd; 3873 rvd = mrvd; 3874 spa_config_exit(spa, SCL_ALL, FTAG); 3875 3876 /* 3877 * We will use spa_config if we decide to reload the spa or if spa_load 3878 * fails and we rewind. We must thus regenerate the config using the 3879 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 3880 * pass settings on how to load the pool and is not stored in the MOS. 3881 * We copy it over to our new, trusted config. 3882 */ 3883 mos_config_txg = fnvlist_lookup_uint64(mos_config, 3884 ZPOOL_CONFIG_POOL_TXG); 3885 nvlist_free(mos_config); 3886 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 3887 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 3888 &policy) == 0) 3889 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 3890 spa_config_set(spa, mos_config); 3891 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 3892 3893 /* 3894 * Now that we got the config from the MOS, we should be more strict 3895 * in checking blkptrs and can make assumptions about the consistency 3896 * of the vdev tree. spa_trust_config must be set to true before opening 3897 * vdevs in order for them to be writeable. 3898 */ 3899 spa->spa_trust_config = B_TRUE; 3900 3901 /* 3902 * Open and validate the new vdev tree 3903 */ 3904 error = spa_ld_open_vdevs(spa); 3905 if (error != 0) 3906 return (error); 3907 3908 error = spa_ld_validate_vdevs(spa); 3909 if (error != 0) 3910 return (error); 3911 3912 if (copy_error != 0 || spa_load_print_vdev_tree) { 3913 spa_load_note(spa, "final vdev tree:"); 3914 vdev_dbgmsg_print_tree(rvd, 2); 3915 } 3916 3917 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 3918 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 3919 /* 3920 * Sanity check to make sure that we are indeed loading the 3921 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 3922 * in the config provided and they happened to be the only ones 3923 * to have the latest uberblock, we could involuntarily perform 3924 * an extreme rewind. 3925 */ 3926 healthy_tvds_mos = spa_healthy_core_tvds(spa); 3927 if (healthy_tvds_mos - healthy_tvds >= 3928 SPA_SYNC_MIN_VDEVS) { 3929 spa_load_note(spa, "config provided misses too many " 3930 "top-level vdevs compared to MOS (%lld vs %lld). ", 3931 (u_longlong_t)healthy_tvds, 3932 (u_longlong_t)healthy_tvds_mos); 3933 spa_load_note(spa, "vdev tree:"); 3934 vdev_dbgmsg_print_tree(rvd, 2); 3935 if (reloading) { 3936 spa_load_failed(spa, "config was already " 3937 "provided from MOS. Aborting."); 3938 return (spa_vdev_err(rvd, 3939 VDEV_AUX_CORRUPT_DATA, EIO)); 3940 } 3941 spa_load_note(spa, "spa must be reloaded using MOS " 3942 "config"); 3943 return (SET_ERROR(EAGAIN)); 3944 } 3945 } 3946 3947 error = spa_check_for_missing_logs(spa); 3948 if (error != 0) 3949 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 3950 3951 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 3952 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 3953 "guid sum (%llu != %llu)", 3954 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 3955 (u_longlong_t)rvd->vdev_guid_sum); 3956 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 3957 ENXIO)); 3958 } 3959 3960 return (0); 3961 } 3962 3963 static int 3964 spa_ld_open_indirect_vdev_metadata(spa_t *spa) 3965 { 3966 int error = 0; 3967 vdev_t *rvd = spa->spa_root_vdev; 3968 3969 /* 3970 * Everything that we read before spa_remove_init() must be stored 3971 * on concreted vdevs. Therefore we do this as early as possible. 3972 */ 3973 error = spa_remove_init(spa); 3974 if (error != 0) { 3975 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 3976 error); 3977 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3978 } 3979 3980 /* 3981 * Retrieve information needed to condense indirect vdev mappings. 3982 */ 3983 error = spa_condense_init(spa); 3984 if (error != 0) { 3985 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 3986 error); 3987 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3988 } 3989 3990 return (0); 3991 } 3992 3993 static int 3994 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 3995 { 3996 int error = 0; 3997 vdev_t *rvd = spa->spa_root_vdev; 3998 3999 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 4000 boolean_t missing_feat_read = B_FALSE; 4001 nvlist_t *unsup_feat, *enabled_feat; 4002 4003 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 4004 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 4005 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4006 } 4007 4008 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 4009 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 4010 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4011 } 4012 4013 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 4014 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 4015 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4016 } 4017 4018 enabled_feat = fnvlist_alloc(); 4019 unsup_feat = fnvlist_alloc(); 4020 4021 if (!spa_features_check(spa, B_FALSE, 4022 unsup_feat, enabled_feat)) 4023 missing_feat_read = B_TRUE; 4024 4025 if (spa_writeable(spa) || 4026 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 4027 if (!spa_features_check(spa, B_TRUE, 4028 unsup_feat, enabled_feat)) { 4029 *missing_feat_writep = B_TRUE; 4030 } 4031 } 4032 4033 fnvlist_add_nvlist(spa->spa_load_info, 4034 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 4035 4036 if (!nvlist_empty(unsup_feat)) { 4037 fnvlist_add_nvlist(spa->spa_load_info, 4038 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 4039 } 4040 4041 fnvlist_free(enabled_feat); 4042 fnvlist_free(unsup_feat); 4043 4044 if (!missing_feat_read) { 4045 fnvlist_add_boolean(spa->spa_load_info, 4046 ZPOOL_CONFIG_CAN_RDONLY); 4047 } 4048 4049 /* 4050 * If the state is SPA_LOAD_TRYIMPORT, our objective is 4051 * twofold: to determine whether the pool is available for 4052 * import in read-write mode and (if it is not) whether the 4053 * pool is available for import in read-only mode. If the pool 4054 * is available for import in read-write mode, it is displayed 4055 * as available in userland; if it is not available for import 4056 * in read-only mode, it is displayed as unavailable in 4057 * userland. If the pool is available for import in read-only 4058 * mode but not read-write mode, it is displayed as unavailable 4059 * in userland with a special note that the pool is actually 4060 * available for open in read-only mode. 4061 * 4062 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 4063 * missing a feature for write, we must first determine whether 4064 * the pool can be opened read-only before returning to 4065 * userland in order to know whether to display the 4066 * abovementioned note. 4067 */ 4068 if (missing_feat_read || (*missing_feat_writep && 4069 spa_writeable(spa))) { 4070 spa_load_failed(spa, "pool uses unsupported features"); 4071 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 4072 ENOTSUP)); 4073 } 4074 4075 /* 4076 * Load refcounts for ZFS features from disk into an in-memory 4077 * cache during SPA initialization. 4078 */ 4079 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 4080 uint64_t refcount; 4081 4082 error = feature_get_refcount_from_disk(spa, 4083 &spa_feature_table[i], &refcount); 4084 if (error == 0) { 4085 spa->spa_feat_refcount_cache[i] = refcount; 4086 } else if (error == ENOTSUP) { 4087 spa->spa_feat_refcount_cache[i] = 4088 SPA_FEATURE_DISABLED; 4089 } else { 4090 spa_load_failed(spa, "error getting refcount " 4091 "for feature %s [error=%d]", 4092 spa_feature_table[i].fi_guid, error); 4093 return (spa_vdev_err(rvd, 4094 VDEV_AUX_CORRUPT_DATA, EIO)); 4095 } 4096 } 4097 } 4098 4099 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 4100 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 4101 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 4102 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4103 } 4104 4105 /* 4106 * Encryption was added before bookmark_v2, even though bookmark_v2 4107 * is now a dependency. If this pool has encryption enabled without 4108 * bookmark_v2, trigger an errata message. 4109 */ 4110 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 4111 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 4112 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 4113 } 4114 4115 return (0); 4116 } 4117 4118 static int 4119 spa_ld_load_special_directories(spa_t *spa) 4120 { 4121 int error = 0; 4122 vdev_t *rvd = spa->spa_root_vdev; 4123 4124 spa->spa_is_initializing = B_TRUE; 4125 error = dsl_pool_open(spa->spa_dsl_pool); 4126 spa->spa_is_initializing = B_FALSE; 4127 if (error != 0) { 4128 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 4129 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4130 } 4131 4132 return (0); 4133 } 4134 4135 static int 4136 spa_ld_get_props(spa_t *spa) 4137 { 4138 int error = 0; 4139 uint64_t obj; 4140 vdev_t *rvd = spa->spa_root_vdev; 4141 4142 /* Grab the checksum salt from the MOS. */ 4143 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4144 DMU_POOL_CHECKSUM_SALT, 1, 4145 sizeof (spa->spa_cksum_salt.zcs_bytes), 4146 spa->spa_cksum_salt.zcs_bytes); 4147 if (error == ENOENT) { 4148 /* Generate a new salt for subsequent use */ 4149 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 4150 sizeof (spa->spa_cksum_salt.zcs_bytes)); 4151 } else if (error != 0) { 4152 spa_load_failed(spa, "unable to retrieve checksum salt from " 4153 "MOS [error=%d]", error); 4154 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4155 } 4156 4157 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 4158 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4159 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 4160 if (error != 0) { 4161 spa_load_failed(spa, "error opening deferred-frees bpobj " 4162 "[error=%d]", error); 4163 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4164 } 4165 4166 /* 4167 * Load the bit that tells us to use the new accounting function 4168 * (raid-z deflation). If we have an older pool, this will not 4169 * be present. 4170 */ 4171 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 4172 if (error != 0 && error != ENOENT) 4173 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4174 4175 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 4176 &spa->spa_creation_version, B_FALSE); 4177 if (error != 0 && error != ENOENT) 4178 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4179 4180 /* 4181 * Load the persistent error log. If we have an older pool, this will 4182 * not be present. 4183 */ 4184 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 4185 B_FALSE); 4186 if (error != 0 && error != ENOENT) 4187 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4188 4189 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 4190 &spa->spa_errlog_scrub, B_FALSE); 4191 if (error != 0 && error != ENOENT) 4192 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4193 4194 /* 4195 * Load the livelist deletion field. If a livelist is queued for 4196 * deletion, indicate that in the spa 4197 */ 4198 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 4199 &spa->spa_livelists_to_delete, B_FALSE); 4200 if (error != 0 && error != ENOENT) 4201 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4202 4203 /* 4204 * Load the history object. If we have an older pool, this 4205 * will not be present. 4206 */ 4207 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 4208 if (error != 0 && error != ENOENT) 4209 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4210 4211 /* 4212 * Load the per-vdev ZAP map. If we have an older pool, this will not 4213 * be present; in this case, defer its creation to a later time to 4214 * avoid dirtying the MOS this early / out of sync context. See 4215 * spa_sync_config_object. 4216 */ 4217 4218 /* The sentinel is only available in the MOS config. */ 4219 nvlist_t *mos_config; 4220 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 4221 spa_load_failed(spa, "unable to retrieve MOS config"); 4222 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4223 } 4224 4225 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 4226 &spa->spa_all_vdev_zaps, B_FALSE); 4227 4228 if (error == ENOENT) { 4229 VERIFY(!nvlist_exists(mos_config, 4230 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 4231 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 4232 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4233 } else if (error != 0) { 4234 nvlist_free(mos_config); 4235 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4236 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 4237 /* 4238 * An older version of ZFS overwrote the sentinel value, so 4239 * we have orphaned per-vdev ZAPs in the MOS. Defer their 4240 * destruction to later; see spa_sync_config_object. 4241 */ 4242 spa->spa_avz_action = AVZ_ACTION_DESTROY; 4243 /* 4244 * We're assuming that no vdevs have had their ZAPs created 4245 * before this. Better be sure of it. 4246 */ 4247 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4248 } 4249 nvlist_free(mos_config); 4250 4251 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 4252 4253 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 4254 B_FALSE); 4255 if (error && error != ENOENT) 4256 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4257 4258 if (error == 0) { 4259 uint64_t autoreplace = 0; 4260 4261 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 4262 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 4263 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 4264 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 4265 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 4266 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 4267 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 4268 spa->spa_autoreplace = (autoreplace != 0); 4269 } 4270 4271 /* 4272 * If we are importing a pool with missing top-level vdevs, 4273 * we enforce that the pool doesn't panic or get suspended on 4274 * error since the likelihood of missing data is extremely high. 4275 */ 4276 if (spa->spa_missing_tvds > 0 && 4277 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 4278 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4279 spa_load_note(spa, "forcing failmode to 'continue' " 4280 "as some top level vdevs are missing"); 4281 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 4282 } 4283 4284 return (0); 4285 } 4286 4287 static int 4288 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 4289 { 4290 int error = 0; 4291 vdev_t *rvd = spa->spa_root_vdev; 4292 4293 /* 4294 * If we're assembling the pool from the split-off vdevs of 4295 * an existing pool, we don't want to attach the spares & cache 4296 * devices. 4297 */ 4298 4299 /* 4300 * Load any hot spares for this pool. 4301 */ 4302 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 4303 B_FALSE); 4304 if (error != 0 && error != ENOENT) 4305 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4306 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4307 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 4308 if (load_nvlist(spa, spa->spa_spares.sav_object, 4309 &spa->spa_spares.sav_config) != 0) { 4310 spa_load_failed(spa, "error loading spares nvlist"); 4311 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4312 } 4313 4314 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4315 spa_load_spares(spa); 4316 spa_config_exit(spa, SCL_ALL, FTAG); 4317 } else if (error == 0) { 4318 spa->spa_spares.sav_sync = B_TRUE; 4319 } 4320 4321 /* 4322 * Load any level 2 ARC devices for this pool. 4323 */ 4324 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 4325 &spa->spa_l2cache.sav_object, B_FALSE); 4326 if (error != 0 && error != ENOENT) 4327 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4328 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4329 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 4330 if (load_nvlist(spa, spa->spa_l2cache.sav_object, 4331 &spa->spa_l2cache.sav_config) != 0) { 4332 spa_load_failed(spa, "error loading l2cache nvlist"); 4333 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4334 } 4335 4336 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4337 spa_load_l2cache(spa); 4338 spa_config_exit(spa, SCL_ALL, FTAG); 4339 } else if (error == 0) { 4340 spa->spa_l2cache.sav_sync = B_TRUE; 4341 } 4342 4343 return (0); 4344 } 4345 4346 static int 4347 spa_ld_load_vdev_metadata(spa_t *spa) 4348 { 4349 int error = 0; 4350 vdev_t *rvd = spa->spa_root_vdev; 4351 4352 /* 4353 * If the 'multihost' property is set, then never allow a pool to 4354 * be imported when the system hostid is zero. The exception to 4355 * this rule is zdb which is always allowed to access pools. 4356 */ 4357 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 4358 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 4359 fnvlist_add_uint64(spa->spa_load_info, 4360 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4361 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4362 } 4363 4364 /* 4365 * If the 'autoreplace' property is set, then post a resource notifying 4366 * the ZFS DE that it should not issue any faults for unopenable 4367 * devices. We also iterate over the vdevs, and post a sysevent for any 4368 * unopenable vdevs so that the normal autoreplace handler can take 4369 * over. 4370 */ 4371 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4372 spa_check_removed(spa->spa_root_vdev); 4373 /* 4374 * For the import case, this is done in spa_import(), because 4375 * at this point we're using the spare definitions from 4376 * the MOS config, not necessarily from the userland config. 4377 */ 4378 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 4379 spa_aux_check_removed(&spa->spa_spares); 4380 spa_aux_check_removed(&spa->spa_l2cache); 4381 } 4382 } 4383 4384 /* 4385 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 4386 */ 4387 error = vdev_load(rvd); 4388 if (error != 0) { 4389 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 4390 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4391 } 4392 4393 error = spa_ld_log_spacemaps(spa); 4394 if (error != 0) { 4395 spa_load_failed(spa, "spa_ld_log_spacemaps failed [error=%d]", 4396 error); 4397 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4398 } 4399 4400 /* 4401 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 4402 */ 4403 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4404 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 4405 spa_config_exit(spa, SCL_ALL, FTAG); 4406 4407 return (0); 4408 } 4409 4410 static int 4411 spa_ld_load_dedup_tables(spa_t *spa) 4412 { 4413 int error = 0; 4414 vdev_t *rvd = spa->spa_root_vdev; 4415 4416 error = ddt_load(spa); 4417 if (error != 0) { 4418 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 4419 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4420 } 4421 4422 return (0); 4423 } 4424 4425 static int 4426 spa_ld_load_brt(spa_t *spa) 4427 { 4428 int error = 0; 4429 vdev_t *rvd = spa->spa_root_vdev; 4430 4431 error = brt_load(spa); 4432 if (error != 0) { 4433 spa_load_failed(spa, "brt_load failed [error=%d]", error); 4434 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4435 } 4436 4437 return (0); 4438 } 4439 4440 static int 4441 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, const char **ereport) 4442 { 4443 vdev_t *rvd = spa->spa_root_vdev; 4444 4445 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 4446 boolean_t missing = spa_check_logs(spa); 4447 if (missing) { 4448 if (spa->spa_missing_tvds != 0) { 4449 spa_load_note(spa, "spa_check_logs failed " 4450 "so dropping the logs"); 4451 } else { 4452 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 4453 spa_load_failed(spa, "spa_check_logs failed"); 4454 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 4455 ENXIO)); 4456 } 4457 } 4458 } 4459 4460 return (0); 4461 } 4462 4463 static int 4464 spa_ld_verify_pool_data(spa_t *spa) 4465 { 4466 int error = 0; 4467 vdev_t *rvd = spa->spa_root_vdev; 4468 4469 /* 4470 * We've successfully opened the pool, verify that we're ready 4471 * to start pushing transactions. 4472 */ 4473 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4474 error = spa_load_verify(spa); 4475 if (error != 0) { 4476 spa_load_failed(spa, "spa_load_verify failed " 4477 "[error=%d]", error); 4478 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4479 error)); 4480 } 4481 } 4482 4483 return (0); 4484 } 4485 4486 static void 4487 spa_ld_claim_log_blocks(spa_t *spa) 4488 { 4489 dmu_tx_t *tx; 4490 dsl_pool_t *dp = spa_get_dsl(spa); 4491 4492 /* 4493 * Claim log blocks that haven't been committed yet. 4494 * This must all happen in a single txg. 4495 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 4496 * invoked from zil_claim_log_block()'s i/o done callback. 4497 * Price of rollback is that we abandon the log. 4498 */ 4499 spa->spa_claiming = B_TRUE; 4500 4501 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 4502 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 4503 zil_claim, tx, DS_FIND_CHILDREN); 4504 dmu_tx_commit(tx); 4505 4506 spa->spa_claiming = B_FALSE; 4507 4508 spa_set_log_state(spa, SPA_LOG_GOOD); 4509 } 4510 4511 static void 4512 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 4513 boolean_t update_config_cache) 4514 { 4515 vdev_t *rvd = spa->spa_root_vdev; 4516 int need_update = B_FALSE; 4517 4518 /* 4519 * If the config cache is stale, or we have uninitialized 4520 * metaslabs (see spa_vdev_add()), then update the config. 4521 * 4522 * If this is a verbatim import, trust the current 4523 * in-core spa_config and update the disk labels. 4524 */ 4525 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 4526 spa->spa_load_state == SPA_LOAD_IMPORT || 4527 spa->spa_load_state == SPA_LOAD_RECOVER || 4528 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 4529 need_update = B_TRUE; 4530 4531 for (int c = 0; c < rvd->vdev_children; c++) 4532 if (rvd->vdev_child[c]->vdev_ms_array == 0) 4533 need_update = B_TRUE; 4534 4535 /* 4536 * Update the config cache asynchronously in case we're the 4537 * root pool, in which case the config cache isn't writable yet. 4538 */ 4539 if (need_update) 4540 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 4541 } 4542 4543 static void 4544 spa_ld_prepare_for_reload(spa_t *spa) 4545 { 4546 spa_mode_t mode = spa->spa_mode; 4547 int async_suspended = spa->spa_async_suspended; 4548 4549 spa_unload(spa); 4550 spa_deactivate(spa); 4551 spa_activate(spa, mode); 4552 4553 /* 4554 * We save the value of spa_async_suspended as it gets reset to 0 by 4555 * spa_unload(). We want to restore it back to the original value before 4556 * returning as we might be calling spa_async_resume() later. 4557 */ 4558 spa->spa_async_suspended = async_suspended; 4559 } 4560 4561 static int 4562 spa_ld_read_checkpoint_txg(spa_t *spa) 4563 { 4564 uberblock_t checkpoint; 4565 int error = 0; 4566 4567 ASSERT0(spa->spa_checkpoint_txg); 4568 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4569 4570 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4571 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4572 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4573 4574 if (error == ENOENT) 4575 return (0); 4576 4577 if (error != 0) 4578 return (error); 4579 4580 ASSERT3U(checkpoint.ub_txg, !=, 0); 4581 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 4582 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 4583 spa->spa_checkpoint_txg = checkpoint.ub_txg; 4584 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 4585 4586 return (0); 4587 } 4588 4589 static int 4590 spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 4591 { 4592 int error = 0; 4593 4594 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4595 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4596 4597 /* 4598 * Never trust the config that is provided unless we are assembling 4599 * a pool following a split. 4600 * This means don't trust blkptrs and the vdev tree in general. This 4601 * also effectively puts the spa in read-only mode since 4602 * spa_writeable() checks for spa_trust_config to be true. 4603 * We will later load a trusted config from the MOS. 4604 */ 4605 if (type != SPA_IMPORT_ASSEMBLE) 4606 spa->spa_trust_config = B_FALSE; 4607 4608 /* 4609 * Parse the config provided to create a vdev tree. 4610 */ 4611 error = spa_ld_parse_config(spa, type); 4612 if (error != 0) 4613 return (error); 4614 4615 spa_import_progress_add(spa); 4616 4617 /* 4618 * Now that we have the vdev tree, try to open each vdev. This involves 4619 * opening the underlying physical device, retrieving its geometry and 4620 * probing the vdev with a dummy I/O. The state of each vdev will be set 4621 * based on the success of those operations. After this we'll be ready 4622 * to read from the vdevs. 4623 */ 4624 error = spa_ld_open_vdevs(spa); 4625 if (error != 0) 4626 return (error); 4627 4628 /* 4629 * Read the label of each vdev and make sure that the GUIDs stored 4630 * there match the GUIDs in the config provided. 4631 * If we're assembling a new pool that's been split off from an 4632 * existing pool, the labels haven't yet been updated so we skip 4633 * validation for now. 4634 */ 4635 if (type != SPA_IMPORT_ASSEMBLE) { 4636 error = spa_ld_validate_vdevs(spa); 4637 if (error != 0) 4638 return (error); 4639 } 4640 4641 /* 4642 * Read all vdev labels to find the best uberblock (i.e. latest, 4643 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 4644 * get the list of features required to read blkptrs in the MOS from 4645 * the vdev label with the best uberblock and verify that our version 4646 * of zfs supports them all. 4647 */ 4648 error = spa_ld_select_uberblock(spa, type); 4649 if (error != 0) 4650 return (error); 4651 4652 /* 4653 * Pass that uberblock to the dsl_pool layer which will open the root 4654 * blkptr. This blkptr points to the latest version of the MOS and will 4655 * allow us to read its contents. 4656 */ 4657 error = spa_ld_open_rootbp(spa); 4658 if (error != 0) 4659 return (error); 4660 4661 return (0); 4662 } 4663 4664 static int 4665 spa_ld_checkpoint_rewind(spa_t *spa) 4666 { 4667 uberblock_t checkpoint; 4668 int error = 0; 4669 4670 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4671 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4672 4673 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4674 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4675 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4676 4677 if (error != 0) { 4678 spa_load_failed(spa, "unable to retrieve checkpointed " 4679 "uberblock from the MOS config [error=%d]", error); 4680 4681 if (error == ENOENT) 4682 error = ZFS_ERR_NO_CHECKPOINT; 4683 4684 return (error); 4685 } 4686 4687 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 4688 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 4689 4690 /* 4691 * We need to update the txg and timestamp of the checkpointed 4692 * uberblock to be higher than the latest one. This ensures that 4693 * the checkpointed uberblock is selected if we were to close and 4694 * reopen the pool right after we've written it in the vdev labels. 4695 * (also see block comment in vdev_uberblock_compare) 4696 */ 4697 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 4698 checkpoint.ub_timestamp = gethrestime_sec(); 4699 4700 /* 4701 * Set current uberblock to be the checkpointed uberblock. 4702 */ 4703 spa->spa_uberblock = checkpoint; 4704 4705 /* 4706 * If we are doing a normal rewind, then the pool is open for 4707 * writing and we sync the "updated" checkpointed uberblock to 4708 * disk. Once this is done, we've basically rewound the whole 4709 * pool and there is no way back. 4710 * 4711 * There are cases when we don't want to attempt and sync the 4712 * checkpointed uberblock to disk because we are opening a 4713 * pool as read-only. Specifically, verifying the checkpointed 4714 * state with zdb, and importing the checkpointed state to get 4715 * a "preview" of its content. 4716 */ 4717 if (spa_writeable(spa)) { 4718 vdev_t *rvd = spa->spa_root_vdev; 4719 4720 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4721 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 4722 int svdcount = 0; 4723 int children = rvd->vdev_children; 4724 int c0 = random_in_range(children); 4725 4726 for (int c = 0; c < children; c++) { 4727 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 4728 4729 /* Stop when revisiting the first vdev */ 4730 if (c > 0 && svd[0] == vd) 4731 break; 4732 4733 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 4734 !vdev_is_concrete(vd)) 4735 continue; 4736 4737 svd[svdcount++] = vd; 4738 if (svdcount == SPA_SYNC_MIN_VDEVS) 4739 break; 4740 } 4741 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg); 4742 if (error == 0) 4743 spa->spa_last_synced_guid = rvd->vdev_guid; 4744 spa_config_exit(spa, SCL_ALL, FTAG); 4745 4746 if (error != 0) { 4747 spa_load_failed(spa, "failed to write checkpointed " 4748 "uberblock to the vdev labels [error=%d]", error); 4749 return (error); 4750 } 4751 } 4752 4753 return (0); 4754 } 4755 4756 static int 4757 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 4758 boolean_t *update_config_cache) 4759 { 4760 int error; 4761 4762 /* 4763 * Parse the config for pool, open and validate vdevs, 4764 * select an uberblock, and use that uberblock to open 4765 * the MOS. 4766 */ 4767 error = spa_ld_mos_init(spa, type); 4768 if (error != 0) 4769 return (error); 4770 4771 /* 4772 * Retrieve the trusted config stored in the MOS and use it to create 4773 * a new, exact version of the vdev tree, then reopen all vdevs. 4774 */ 4775 error = spa_ld_trusted_config(spa, type, B_FALSE); 4776 if (error == EAGAIN) { 4777 if (update_config_cache != NULL) 4778 *update_config_cache = B_TRUE; 4779 4780 /* 4781 * Redo the loading process with the trusted config if it is 4782 * too different from the untrusted config. 4783 */ 4784 spa_ld_prepare_for_reload(spa); 4785 spa_load_note(spa, "RELOADING"); 4786 error = spa_ld_mos_init(spa, type); 4787 if (error != 0) 4788 return (error); 4789 4790 error = spa_ld_trusted_config(spa, type, B_TRUE); 4791 if (error != 0) 4792 return (error); 4793 4794 } else if (error != 0) { 4795 return (error); 4796 } 4797 4798 return (0); 4799 } 4800 4801 /* 4802 * Load an existing storage pool, using the config provided. This config 4803 * describes which vdevs are part of the pool and is later validated against 4804 * partial configs present in each vdev's label and an entire copy of the 4805 * config stored in the MOS. 4806 */ 4807 static int 4808 spa_load_impl(spa_t *spa, spa_import_type_t type, const char **ereport) 4809 { 4810 int error = 0; 4811 boolean_t missing_feat_write = B_FALSE; 4812 boolean_t checkpoint_rewind = 4813 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4814 boolean_t update_config_cache = B_FALSE; 4815 4816 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4817 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4818 4819 spa_load_note(spa, "LOADING"); 4820 4821 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 4822 if (error != 0) 4823 return (error); 4824 4825 /* 4826 * If we are rewinding to the checkpoint then we need to repeat 4827 * everything we've done so far in this function but this time 4828 * selecting the checkpointed uberblock and using that to open 4829 * the MOS. 4830 */ 4831 if (checkpoint_rewind) { 4832 /* 4833 * If we are rewinding to the checkpoint update config cache 4834 * anyway. 4835 */ 4836 update_config_cache = B_TRUE; 4837 4838 /* 4839 * Extract the checkpointed uberblock from the current MOS 4840 * and use this as the pool's uberblock from now on. If the 4841 * pool is imported as writeable we also write the checkpoint 4842 * uberblock to the labels, making the rewind permanent. 4843 */ 4844 error = spa_ld_checkpoint_rewind(spa); 4845 if (error != 0) 4846 return (error); 4847 4848 /* 4849 * Redo the loading process again with the 4850 * checkpointed uberblock. 4851 */ 4852 spa_ld_prepare_for_reload(spa); 4853 spa_load_note(spa, "LOADING checkpointed uberblock"); 4854 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 4855 if (error != 0) 4856 return (error); 4857 } 4858 4859 /* 4860 * Retrieve the checkpoint txg if the pool has a checkpoint. 4861 */ 4862 error = spa_ld_read_checkpoint_txg(spa); 4863 if (error != 0) 4864 return (error); 4865 4866 /* 4867 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 4868 * from the pool and their contents were re-mapped to other vdevs. Note 4869 * that everything that we read before this step must have been 4870 * rewritten on concrete vdevs after the last device removal was 4871 * initiated. Otherwise we could be reading from indirect vdevs before 4872 * we have loaded their mappings. 4873 */ 4874 error = spa_ld_open_indirect_vdev_metadata(spa); 4875 if (error != 0) 4876 return (error); 4877 4878 /* 4879 * Retrieve the full list of active features from the MOS and check if 4880 * they are all supported. 4881 */ 4882 error = spa_ld_check_features(spa, &missing_feat_write); 4883 if (error != 0) 4884 return (error); 4885 4886 /* 4887 * Load several special directories from the MOS needed by the dsl_pool 4888 * layer. 4889 */ 4890 error = spa_ld_load_special_directories(spa); 4891 if (error != 0) 4892 return (error); 4893 4894 /* 4895 * Retrieve pool properties from the MOS. 4896 */ 4897 error = spa_ld_get_props(spa); 4898 if (error != 0) 4899 return (error); 4900 4901 /* 4902 * Retrieve the list of auxiliary devices - cache devices and spares - 4903 * and open them. 4904 */ 4905 error = spa_ld_open_aux_vdevs(spa, type); 4906 if (error != 0) 4907 return (error); 4908 4909 /* 4910 * Load the metadata for all vdevs. Also check if unopenable devices 4911 * should be autoreplaced. 4912 */ 4913 error = spa_ld_load_vdev_metadata(spa); 4914 if (error != 0) 4915 return (error); 4916 4917 error = spa_ld_load_dedup_tables(spa); 4918 if (error != 0) 4919 return (error); 4920 4921 error = spa_ld_load_brt(spa); 4922 if (error != 0) 4923 return (error); 4924 4925 /* 4926 * Verify the logs now to make sure we don't have any unexpected errors 4927 * when we claim log blocks later. 4928 */ 4929 error = spa_ld_verify_logs(spa, type, ereport); 4930 if (error != 0) 4931 return (error); 4932 4933 if (missing_feat_write) { 4934 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 4935 4936 /* 4937 * At this point, we know that we can open the pool in 4938 * read-only mode but not read-write mode. We now have enough 4939 * information and can return to userland. 4940 */ 4941 return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 4942 ENOTSUP)); 4943 } 4944 4945 /* 4946 * Traverse the last txgs to make sure the pool was left off in a safe 4947 * state. When performing an extreme rewind, we verify the whole pool, 4948 * which can take a very long time. 4949 */ 4950 error = spa_ld_verify_pool_data(spa); 4951 if (error != 0) 4952 return (error); 4953 4954 /* 4955 * Calculate the deflated space for the pool. This must be done before 4956 * we write anything to the pool because we'd need to update the space 4957 * accounting using the deflated sizes. 4958 */ 4959 spa_update_dspace(spa); 4960 4961 /* 4962 * We have now retrieved all the information we needed to open the 4963 * pool. If we are importing the pool in read-write mode, a few 4964 * additional steps must be performed to finish the import. 4965 */ 4966 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 4967 spa->spa_load_max_txg == UINT64_MAX)) { 4968 uint64_t config_cache_txg = spa->spa_config_txg; 4969 4970 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 4971 4972 /* 4973 * In case of a checkpoint rewind, log the original txg 4974 * of the checkpointed uberblock. 4975 */ 4976 if (checkpoint_rewind) { 4977 spa_history_log_internal(spa, "checkpoint rewind", 4978 NULL, "rewound state to txg=%llu", 4979 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 4980 } 4981 4982 /* 4983 * Traverse the ZIL and claim all blocks. 4984 */ 4985 spa_ld_claim_log_blocks(spa); 4986 4987 /* 4988 * Kick-off the syncing thread. 4989 */ 4990 spa->spa_sync_on = B_TRUE; 4991 txg_sync_start(spa->spa_dsl_pool); 4992 mmp_thread_start(spa); 4993 4994 /* 4995 * Wait for all claims to sync. We sync up to the highest 4996 * claimed log block birth time so that claimed log blocks 4997 * don't appear to be from the future. spa_claim_max_txg 4998 * will have been set for us by ZIL traversal operations 4999 * performed above. 5000 */ 5001 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 5002 5003 /* 5004 * Check if we need to request an update of the config. On the 5005 * next sync, we would update the config stored in vdev labels 5006 * and the cachefile (by default /etc/zfs/zpool.cache). 5007 */ 5008 spa_ld_check_for_config_update(spa, config_cache_txg, 5009 update_config_cache); 5010 5011 /* 5012 * Check if a rebuild was in progress and if so resume it. 5013 * Then check all DTLs to see if anything needs resilvering. 5014 * The resilver will be deferred if a rebuild was started. 5015 */ 5016 if (vdev_rebuild_active(spa->spa_root_vdev)) { 5017 vdev_rebuild_restart(spa); 5018 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 5019 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 5020 spa_async_request(spa, SPA_ASYNC_RESILVER); 5021 } 5022 5023 /* 5024 * Log the fact that we booted up (so that we can detect if 5025 * we rebooted in the middle of an operation). 5026 */ 5027 spa_history_log_version(spa, "open", NULL); 5028 5029 spa_restart_removal(spa); 5030 spa_spawn_aux_threads(spa); 5031 5032 /* 5033 * Delete any inconsistent datasets. 5034 * 5035 * Note: 5036 * Since we may be issuing deletes for clones here, 5037 * we make sure to do so after we've spawned all the 5038 * auxiliary threads above (from which the livelist 5039 * deletion zthr is part of). 5040 */ 5041 (void) dmu_objset_find(spa_name(spa), 5042 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 5043 5044 /* 5045 * Clean up any stale temporary dataset userrefs. 5046 */ 5047 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 5048 5049 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5050 vdev_initialize_restart(spa->spa_root_vdev); 5051 vdev_trim_restart(spa->spa_root_vdev); 5052 vdev_autotrim_restart(spa); 5053 spa_config_exit(spa, SCL_CONFIG, FTAG); 5054 } 5055 5056 spa_import_progress_remove(spa_guid(spa)); 5057 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 5058 5059 spa_load_note(spa, "LOADED"); 5060 5061 return (0); 5062 } 5063 5064 static int 5065 spa_load_retry(spa_t *spa, spa_load_state_t state) 5066 { 5067 spa_mode_t mode = spa->spa_mode; 5068 5069 spa_unload(spa); 5070 spa_deactivate(spa); 5071 5072 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 5073 5074 spa_activate(spa, mode); 5075 spa_async_suspend(spa); 5076 5077 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 5078 (u_longlong_t)spa->spa_load_max_txg); 5079 5080 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 5081 } 5082 5083 /* 5084 * If spa_load() fails this function will try loading prior txg's. If 5085 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 5086 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 5087 * function will not rewind the pool and will return the same error as 5088 * spa_load(). 5089 */ 5090 static int 5091 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 5092 int rewind_flags) 5093 { 5094 nvlist_t *loadinfo = NULL; 5095 nvlist_t *config = NULL; 5096 int load_error, rewind_error; 5097 uint64_t safe_rewind_txg; 5098 uint64_t min_txg; 5099 5100 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 5101 spa->spa_load_max_txg = spa->spa_load_txg; 5102 spa_set_log_state(spa, SPA_LOG_CLEAR); 5103 } else { 5104 spa->spa_load_max_txg = max_request; 5105 if (max_request != UINT64_MAX) 5106 spa->spa_extreme_rewind = B_TRUE; 5107 } 5108 5109 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 5110 if (load_error == 0) 5111 return (0); 5112 if (load_error == ZFS_ERR_NO_CHECKPOINT) { 5113 /* 5114 * When attempting checkpoint-rewind on a pool with no 5115 * checkpoint, we should not attempt to load uberblocks 5116 * from previous txgs when spa_load fails. 5117 */ 5118 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5119 spa_import_progress_remove(spa_guid(spa)); 5120 return (load_error); 5121 } 5122 5123 if (spa->spa_root_vdev != NULL) 5124 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5125 5126 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 5127 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 5128 5129 if (rewind_flags & ZPOOL_NEVER_REWIND) { 5130 nvlist_free(config); 5131 spa_import_progress_remove(spa_guid(spa)); 5132 return (load_error); 5133 } 5134 5135 if (state == SPA_LOAD_RECOVER) { 5136 /* Price of rolling back is discarding txgs, including log */ 5137 spa_set_log_state(spa, SPA_LOG_CLEAR); 5138 } else { 5139 /* 5140 * If we aren't rolling back save the load info from our first 5141 * import attempt so that we can restore it after attempting 5142 * to rewind. 5143 */ 5144 loadinfo = spa->spa_load_info; 5145 spa->spa_load_info = fnvlist_alloc(); 5146 } 5147 5148 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 5149 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 5150 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 5151 TXG_INITIAL : safe_rewind_txg; 5152 5153 /* 5154 * Continue as long as we're finding errors, we're still within 5155 * the acceptable rewind range, and we're still finding uberblocks 5156 */ 5157 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 5158 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 5159 if (spa->spa_load_max_txg < safe_rewind_txg) 5160 spa->spa_extreme_rewind = B_TRUE; 5161 rewind_error = spa_load_retry(spa, state); 5162 } 5163 5164 spa->spa_extreme_rewind = B_FALSE; 5165 spa->spa_load_max_txg = UINT64_MAX; 5166 5167 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 5168 spa_config_set(spa, config); 5169 else 5170 nvlist_free(config); 5171 5172 if (state == SPA_LOAD_RECOVER) { 5173 ASSERT3P(loadinfo, ==, NULL); 5174 spa_import_progress_remove(spa_guid(spa)); 5175 return (rewind_error); 5176 } else { 5177 /* Store the rewind info as part of the initial load info */ 5178 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 5179 spa->spa_load_info); 5180 5181 /* Restore the initial load info */ 5182 fnvlist_free(spa->spa_load_info); 5183 spa->spa_load_info = loadinfo; 5184 5185 spa_import_progress_remove(spa_guid(spa)); 5186 return (load_error); 5187 } 5188 } 5189 5190 /* 5191 * Pool Open/Import 5192 * 5193 * The import case is identical to an open except that the configuration is sent 5194 * down from userland, instead of grabbed from the configuration cache. For the 5195 * case of an open, the pool configuration will exist in the 5196 * POOL_STATE_UNINITIALIZED state. 5197 * 5198 * The stats information (gen/count/ustats) is used to gather vdev statistics at 5199 * the same time open the pool, without having to keep around the spa_t in some 5200 * ambiguous state. 5201 */ 5202 static int 5203 spa_open_common(const char *pool, spa_t **spapp, const void *tag, 5204 nvlist_t *nvpolicy, nvlist_t **config) 5205 { 5206 spa_t *spa; 5207 spa_load_state_t state = SPA_LOAD_OPEN; 5208 int error; 5209 int locked = B_FALSE; 5210 int firstopen = B_FALSE; 5211 5212 *spapp = NULL; 5213 5214 /* 5215 * As disgusting as this is, we need to support recursive calls to this 5216 * function because dsl_dir_open() is called during spa_load(), and ends 5217 * up calling spa_open() again. The real fix is to figure out how to 5218 * avoid dsl_dir_open() calling this in the first place. 5219 */ 5220 if (MUTEX_NOT_HELD(&spa_namespace_lock)) { 5221 mutex_enter(&spa_namespace_lock); 5222 locked = B_TRUE; 5223 } 5224 5225 if ((spa = spa_lookup(pool)) == NULL) { 5226 if (locked) 5227 mutex_exit(&spa_namespace_lock); 5228 return (SET_ERROR(ENOENT)); 5229 } 5230 5231 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 5232 zpool_load_policy_t policy; 5233 5234 firstopen = B_TRUE; 5235 5236 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 5237 &policy); 5238 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 5239 state = SPA_LOAD_RECOVER; 5240 5241 spa_activate(spa, spa_mode_global); 5242 5243 if (state != SPA_LOAD_RECOVER) 5244 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 5245 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 5246 5247 zfs_dbgmsg("spa_open_common: opening %s", pool); 5248 error = spa_load_best(spa, state, policy.zlp_txg, 5249 policy.zlp_rewind); 5250 5251 if (error == EBADF) { 5252 /* 5253 * If vdev_validate() returns failure (indicated by 5254 * EBADF), it indicates that one of the vdevs indicates 5255 * that the pool has been exported or destroyed. If 5256 * this is the case, the config cache is out of sync and 5257 * we should remove the pool from the namespace. 5258 */ 5259 spa_unload(spa); 5260 spa_deactivate(spa); 5261 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 5262 spa_remove(spa); 5263 if (locked) 5264 mutex_exit(&spa_namespace_lock); 5265 return (SET_ERROR(ENOENT)); 5266 } 5267 5268 if (error) { 5269 /* 5270 * We can't open the pool, but we still have useful 5271 * information: the state of each vdev after the 5272 * attempted vdev_open(). Return this to the user. 5273 */ 5274 if (config != NULL && spa->spa_config) { 5275 *config = fnvlist_dup(spa->spa_config); 5276 fnvlist_add_nvlist(*config, 5277 ZPOOL_CONFIG_LOAD_INFO, 5278 spa->spa_load_info); 5279 } 5280 spa_unload(spa); 5281 spa_deactivate(spa); 5282 spa->spa_last_open_failed = error; 5283 if (locked) 5284 mutex_exit(&spa_namespace_lock); 5285 *spapp = NULL; 5286 return (error); 5287 } 5288 } 5289 5290 spa_open_ref(spa, tag); 5291 5292 if (config != NULL) 5293 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5294 5295 /* 5296 * If we've recovered the pool, pass back any information we 5297 * gathered while doing the load. 5298 */ 5299 if (state == SPA_LOAD_RECOVER && config != NULL) { 5300 fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 5301 spa->spa_load_info); 5302 } 5303 5304 if (locked) { 5305 spa->spa_last_open_failed = 0; 5306 spa->spa_last_ubsync_txg = 0; 5307 spa->spa_load_txg = 0; 5308 mutex_exit(&spa_namespace_lock); 5309 } 5310 5311 if (firstopen) 5312 zvol_create_minors_recursive(spa_name(spa)); 5313 5314 *spapp = spa; 5315 5316 return (0); 5317 } 5318 5319 int 5320 spa_open_rewind(const char *name, spa_t **spapp, const void *tag, 5321 nvlist_t *policy, nvlist_t **config) 5322 { 5323 return (spa_open_common(name, spapp, tag, policy, config)); 5324 } 5325 5326 int 5327 spa_open(const char *name, spa_t **spapp, const void *tag) 5328 { 5329 return (spa_open_common(name, spapp, tag, NULL, NULL)); 5330 } 5331 5332 /* 5333 * Lookup the given spa_t, incrementing the inject count in the process, 5334 * preventing it from being exported or destroyed. 5335 */ 5336 spa_t * 5337 spa_inject_addref(char *name) 5338 { 5339 spa_t *spa; 5340 5341 mutex_enter(&spa_namespace_lock); 5342 if ((spa = spa_lookup(name)) == NULL) { 5343 mutex_exit(&spa_namespace_lock); 5344 return (NULL); 5345 } 5346 spa->spa_inject_ref++; 5347 mutex_exit(&spa_namespace_lock); 5348 5349 return (spa); 5350 } 5351 5352 void 5353 spa_inject_delref(spa_t *spa) 5354 { 5355 mutex_enter(&spa_namespace_lock); 5356 spa->spa_inject_ref--; 5357 mutex_exit(&spa_namespace_lock); 5358 } 5359 5360 /* 5361 * Add spares device information to the nvlist. 5362 */ 5363 static void 5364 spa_add_spares(spa_t *spa, nvlist_t *config) 5365 { 5366 nvlist_t **spares; 5367 uint_t i, nspares; 5368 nvlist_t *nvroot; 5369 uint64_t guid; 5370 vdev_stat_t *vs; 5371 uint_t vsc; 5372 uint64_t pool; 5373 5374 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5375 5376 if (spa->spa_spares.sav_count == 0) 5377 return; 5378 5379 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 5380 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 5381 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 5382 if (nspares != 0) { 5383 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5384 (const nvlist_t * const *)spares, nspares); 5385 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5386 &spares, &nspares)); 5387 5388 /* 5389 * Go through and find any spares which have since been 5390 * repurposed as an active spare. If this is the case, update 5391 * their status appropriately. 5392 */ 5393 for (i = 0; i < nspares; i++) { 5394 guid = fnvlist_lookup_uint64(spares[i], 5395 ZPOOL_CONFIG_GUID); 5396 VERIFY0(nvlist_lookup_uint64_array(spares[i], 5397 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 5398 if (spa_spare_exists(guid, &pool, NULL) && 5399 pool != 0ULL) { 5400 vs->vs_state = VDEV_STATE_CANT_OPEN; 5401 vs->vs_aux = VDEV_AUX_SPARED; 5402 } else { 5403 vs->vs_state = 5404 spa->spa_spares.sav_vdevs[i]->vdev_state; 5405 } 5406 } 5407 } 5408 } 5409 5410 /* 5411 * Add l2cache device information to the nvlist, including vdev stats. 5412 */ 5413 static void 5414 spa_add_l2cache(spa_t *spa, nvlist_t *config) 5415 { 5416 nvlist_t **l2cache; 5417 uint_t i, j, nl2cache; 5418 nvlist_t *nvroot; 5419 uint64_t guid; 5420 vdev_t *vd; 5421 vdev_stat_t *vs; 5422 uint_t vsc; 5423 5424 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5425 5426 if (spa->spa_l2cache.sav_count == 0) 5427 return; 5428 5429 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 5430 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 5431 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 5432 if (nl2cache != 0) { 5433 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5434 (const nvlist_t * const *)l2cache, nl2cache); 5435 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5436 &l2cache, &nl2cache)); 5437 5438 /* 5439 * Update level 2 cache device stats. 5440 */ 5441 5442 for (i = 0; i < nl2cache; i++) { 5443 guid = fnvlist_lookup_uint64(l2cache[i], 5444 ZPOOL_CONFIG_GUID); 5445 5446 vd = NULL; 5447 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 5448 if (guid == 5449 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 5450 vd = spa->spa_l2cache.sav_vdevs[j]; 5451 break; 5452 } 5453 } 5454 ASSERT(vd != NULL); 5455 5456 VERIFY0(nvlist_lookup_uint64_array(l2cache[i], 5457 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 5458 vdev_get_stats(vd, vs); 5459 vdev_config_generate_stats(vd, l2cache[i]); 5460 5461 } 5462 } 5463 } 5464 5465 static void 5466 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 5467 { 5468 zap_cursor_t zc; 5469 zap_attribute_t za; 5470 5471 if (spa->spa_feat_for_read_obj != 0) { 5472 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5473 spa->spa_feat_for_read_obj); 5474 zap_cursor_retrieve(&zc, &za) == 0; 5475 zap_cursor_advance(&zc)) { 5476 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5477 za.za_num_integers == 1); 5478 VERIFY0(nvlist_add_uint64(features, za.za_name, 5479 za.za_first_integer)); 5480 } 5481 zap_cursor_fini(&zc); 5482 } 5483 5484 if (spa->spa_feat_for_write_obj != 0) { 5485 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5486 spa->spa_feat_for_write_obj); 5487 zap_cursor_retrieve(&zc, &za) == 0; 5488 zap_cursor_advance(&zc)) { 5489 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5490 za.za_num_integers == 1); 5491 VERIFY0(nvlist_add_uint64(features, za.za_name, 5492 za.za_first_integer)); 5493 } 5494 zap_cursor_fini(&zc); 5495 } 5496 } 5497 5498 static void 5499 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 5500 { 5501 int i; 5502 5503 for (i = 0; i < SPA_FEATURES; i++) { 5504 zfeature_info_t feature = spa_feature_table[i]; 5505 uint64_t refcount; 5506 5507 if (feature_get_refcount(spa, &feature, &refcount) != 0) 5508 continue; 5509 5510 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 5511 } 5512 } 5513 5514 /* 5515 * Store a list of pool features and their reference counts in the 5516 * config. 5517 * 5518 * The first time this is called on a spa, allocate a new nvlist, fetch 5519 * the pool features and reference counts from disk, then save the list 5520 * in the spa. In subsequent calls on the same spa use the saved nvlist 5521 * and refresh its values from the cached reference counts. This 5522 * ensures we don't block here on I/O on a suspended pool so 'zpool 5523 * clear' can resume the pool. 5524 */ 5525 static void 5526 spa_add_feature_stats(spa_t *spa, nvlist_t *config) 5527 { 5528 nvlist_t *features; 5529 5530 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5531 5532 mutex_enter(&spa->spa_feat_stats_lock); 5533 features = spa->spa_feat_stats; 5534 5535 if (features != NULL) { 5536 spa_feature_stats_from_cache(spa, features); 5537 } else { 5538 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 5539 spa->spa_feat_stats = features; 5540 spa_feature_stats_from_disk(spa, features); 5541 } 5542 5543 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 5544 features)); 5545 5546 mutex_exit(&spa->spa_feat_stats_lock); 5547 } 5548 5549 int 5550 spa_get_stats(const char *name, nvlist_t **config, 5551 char *altroot, size_t buflen) 5552 { 5553 int error; 5554 spa_t *spa; 5555 5556 *config = NULL; 5557 error = spa_open_common(name, &spa, FTAG, NULL, config); 5558 5559 if (spa != NULL) { 5560 /* 5561 * This still leaves a window of inconsistency where the spares 5562 * or l2cache devices could change and the config would be 5563 * self-inconsistent. 5564 */ 5565 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5566 5567 if (*config != NULL) { 5568 uint64_t loadtimes[2]; 5569 5570 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 5571 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 5572 fnvlist_add_uint64_array(*config, 5573 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2); 5574 5575 fnvlist_add_uint64(*config, 5576 ZPOOL_CONFIG_ERRCOUNT, 5577 spa_approx_errlog_size(spa)); 5578 5579 if (spa_suspended(spa)) { 5580 fnvlist_add_uint64(*config, 5581 ZPOOL_CONFIG_SUSPENDED, 5582 spa->spa_failmode); 5583 fnvlist_add_uint64(*config, 5584 ZPOOL_CONFIG_SUSPENDED_REASON, 5585 spa->spa_suspended); 5586 } 5587 5588 spa_add_spares(spa, *config); 5589 spa_add_l2cache(spa, *config); 5590 spa_add_feature_stats(spa, *config); 5591 } 5592 } 5593 5594 /* 5595 * We want to get the alternate root even for faulted pools, so we cheat 5596 * and call spa_lookup() directly. 5597 */ 5598 if (altroot) { 5599 if (spa == NULL) { 5600 mutex_enter(&spa_namespace_lock); 5601 spa = spa_lookup(name); 5602 if (spa) 5603 spa_altroot(spa, altroot, buflen); 5604 else 5605 altroot[0] = '\0'; 5606 spa = NULL; 5607 mutex_exit(&spa_namespace_lock); 5608 } else { 5609 spa_altroot(spa, altroot, buflen); 5610 } 5611 } 5612 5613 if (spa != NULL) { 5614 spa_config_exit(spa, SCL_CONFIG, FTAG); 5615 spa_close(spa, FTAG); 5616 } 5617 5618 return (error); 5619 } 5620 5621 /* 5622 * Validate that the auxiliary device array is well formed. We must have an 5623 * array of nvlists, each which describes a valid leaf vdev. If this is an 5624 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 5625 * specified, as long as they are well-formed. 5626 */ 5627 static int 5628 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 5629 spa_aux_vdev_t *sav, const char *config, uint64_t version, 5630 vdev_labeltype_t label) 5631 { 5632 nvlist_t **dev; 5633 uint_t i, ndev; 5634 vdev_t *vd; 5635 int error; 5636 5637 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5638 5639 /* 5640 * It's acceptable to have no devs specified. 5641 */ 5642 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 5643 return (0); 5644 5645 if (ndev == 0) 5646 return (SET_ERROR(EINVAL)); 5647 5648 /* 5649 * Make sure the pool is formatted with a version that supports this 5650 * device type. 5651 */ 5652 if (spa_version(spa) < version) 5653 return (SET_ERROR(ENOTSUP)); 5654 5655 /* 5656 * Set the pending device list so we correctly handle device in-use 5657 * checking. 5658 */ 5659 sav->sav_pending = dev; 5660 sav->sav_npending = ndev; 5661 5662 for (i = 0; i < ndev; i++) { 5663 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 5664 mode)) != 0) 5665 goto out; 5666 5667 if (!vd->vdev_ops->vdev_op_leaf) { 5668 vdev_free(vd); 5669 error = SET_ERROR(EINVAL); 5670 goto out; 5671 } 5672 5673 vd->vdev_top = vd; 5674 5675 if ((error = vdev_open(vd)) == 0 && 5676 (error = vdev_label_init(vd, crtxg, label)) == 0) { 5677 fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 5678 vd->vdev_guid); 5679 } 5680 5681 vdev_free(vd); 5682 5683 if (error && 5684 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 5685 goto out; 5686 else 5687 error = 0; 5688 } 5689 5690 out: 5691 sav->sav_pending = NULL; 5692 sav->sav_npending = 0; 5693 return (error); 5694 } 5695 5696 static int 5697 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 5698 { 5699 int error; 5700 5701 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5702 5703 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5704 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 5705 VDEV_LABEL_SPARE)) != 0) { 5706 return (error); 5707 } 5708 5709 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5710 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 5711 VDEV_LABEL_L2CACHE)); 5712 } 5713 5714 static void 5715 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 5716 const char *config) 5717 { 5718 int i; 5719 5720 if (sav->sav_config != NULL) { 5721 nvlist_t **olddevs; 5722 uint_t oldndevs; 5723 nvlist_t **newdevs; 5724 5725 /* 5726 * Generate new dev list by concatenating with the 5727 * current dev list. 5728 */ 5729 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config, 5730 &olddevs, &oldndevs)); 5731 5732 newdevs = kmem_alloc(sizeof (void *) * 5733 (ndevs + oldndevs), KM_SLEEP); 5734 for (i = 0; i < oldndevs; i++) 5735 newdevs[i] = fnvlist_dup(olddevs[i]); 5736 for (i = 0; i < ndevs; i++) 5737 newdevs[i + oldndevs] = fnvlist_dup(devs[i]); 5738 5739 fnvlist_remove(sav->sav_config, config); 5740 5741 fnvlist_add_nvlist_array(sav->sav_config, config, 5742 (const nvlist_t * const *)newdevs, ndevs + oldndevs); 5743 for (i = 0; i < oldndevs + ndevs; i++) 5744 nvlist_free(newdevs[i]); 5745 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 5746 } else { 5747 /* 5748 * Generate a new dev list. 5749 */ 5750 sav->sav_config = fnvlist_alloc(); 5751 fnvlist_add_nvlist_array(sav->sav_config, config, 5752 (const nvlist_t * const *)devs, ndevs); 5753 } 5754 } 5755 5756 /* 5757 * Stop and drop level 2 ARC devices 5758 */ 5759 void 5760 spa_l2cache_drop(spa_t *spa) 5761 { 5762 vdev_t *vd; 5763 int i; 5764 spa_aux_vdev_t *sav = &spa->spa_l2cache; 5765 5766 for (i = 0; i < sav->sav_count; i++) { 5767 uint64_t pool; 5768 5769 vd = sav->sav_vdevs[i]; 5770 ASSERT(vd != NULL); 5771 5772 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 5773 pool != 0ULL && l2arc_vdev_present(vd)) 5774 l2arc_remove_vdev(vd); 5775 } 5776 } 5777 5778 /* 5779 * Verify encryption parameters for spa creation. If we are encrypting, we must 5780 * have the encryption feature flag enabled. 5781 */ 5782 static int 5783 spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 5784 boolean_t has_encryption) 5785 { 5786 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 5787 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 5788 !has_encryption) 5789 return (SET_ERROR(ENOTSUP)); 5790 5791 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 5792 } 5793 5794 /* 5795 * Pool Creation 5796 */ 5797 int 5798 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 5799 nvlist_t *zplprops, dsl_crypto_params_t *dcp) 5800 { 5801 spa_t *spa; 5802 const char *altroot = NULL; 5803 vdev_t *rvd; 5804 dsl_pool_t *dp; 5805 dmu_tx_t *tx; 5806 int error = 0; 5807 uint64_t txg = TXG_INITIAL; 5808 nvlist_t **spares, **l2cache; 5809 uint_t nspares, nl2cache; 5810 uint64_t version, obj, ndraid = 0; 5811 boolean_t has_features; 5812 boolean_t has_encryption; 5813 boolean_t has_allocclass; 5814 spa_feature_t feat; 5815 const char *feat_name; 5816 const char *poolname; 5817 nvlist_t *nvl; 5818 5819 if (props == NULL || 5820 nvlist_lookup_string(props, "tname", &poolname) != 0) 5821 poolname = (char *)pool; 5822 5823 /* 5824 * If this pool already exists, return failure. 5825 */ 5826 mutex_enter(&spa_namespace_lock); 5827 if (spa_lookup(poolname) != NULL) { 5828 mutex_exit(&spa_namespace_lock); 5829 return (SET_ERROR(EEXIST)); 5830 } 5831 5832 /* 5833 * Allocate a new spa_t structure. 5834 */ 5835 nvl = fnvlist_alloc(); 5836 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 5837 (void) nvlist_lookup_string(props, 5838 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 5839 spa = spa_add(poolname, nvl, altroot); 5840 fnvlist_free(nvl); 5841 spa_activate(spa, spa_mode_global); 5842 5843 if (props && (error = spa_prop_validate(spa, props))) { 5844 spa_deactivate(spa); 5845 spa_remove(spa); 5846 mutex_exit(&spa_namespace_lock); 5847 return (error); 5848 } 5849 5850 /* 5851 * Temporary pool names should never be written to disk. 5852 */ 5853 if (poolname != pool) 5854 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 5855 5856 has_features = B_FALSE; 5857 has_encryption = B_FALSE; 5858 has_allocclass = B_FALSE; 5859 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 5860 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 5861 if (zpool_prop_feature(nvpair_name(elem))) { 5862 has_features = B_TRUE; 5863 5864 feat_name = strchr(nvpair_name(elem), '@') + 1; 5865 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 5866 if (feat == SPA_FEATURE_ENCRYPTION) 5867 has_encryption = B_TRUE; 5868 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 5869 has_allocclass = B_TRUE; 5870 } 5871 } 5872 5873 /* verify encryption params, if they were provided */ 5874 if (dcp != NULL) { 5875 error = spa_create_check_encryption_params(dcp, has_encryption); 5876 if (error != 0) { 5877 spa_deactivate(spa); 5878 spa_remove(spa); 5879 mutex_exit(&spa_namespace_lock); 5880 return (error); 5881 } 5882 } 5883 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 5884 spa_deactivate(spa); 5885 spa_remove(spa); 5886 mutex_exit(&spa_namespace_lock); 5887 return (ENOTSUP); 5888 } 5889 5890 if (has_features || nvlist_lookup_uint64(props, 5891 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 5892 version = SPA_VERSION; 5893 } 5894 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 5895 5896 spa->spa_first_txg = txg; 5897 spa->spa_uberblock.ub_txg = txg - 1; 5898 spa->spa_uberblock.ub_version = version; 5899 spa->spa_ubsync = spa->spa_uberblock; 5900 spa->spa_load_state = SPA_LOAD_CREATE; 5901 spa->spa_removing_phys.sr_state = DSS_NONE; 5902 spa->spa_removing_phys.sr_removing_vdev = -1; 5903 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 5904 spa->spa_indirect_vdevs_loaded = B_TRUE; 5905 5906 /* 5907 * Create "The Godfather" zio to hold all async IOs 5908 */ 5909 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 5910 KM_SLEEP); 5911 for (int i = 0; i < max_ncpus; i++) { 5912 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 5913 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 5914 ZIO_FLAG_GODFATHER); 5915 } 5916 5917 /* 5918 * Create the root vdev. 5919 */ 5920 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5921 5922 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 5923 5924 ASSERT(error != 0 || rvd != NULL); 5925 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 5926 5927 if (error == 0 && !zfs_allocatable_devs(nvroot)) 5928 error = SET_ERROR(EINVAL); 5929 5930 if (error == 0 && 5931 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 5932 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 && 5933 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 5934 /* 5935 * instantiate the metaslab groups (this will dirty the vdevs) 5936 * we can no longer error exit past this point 5937 */ 5938 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 5939 vdev_t *vd = rvd->vdev_child[c]; 5940 5941 vdev_metaslab_set_size(vd); 5942 vdev_expand(vd, txg); 5943 } 5944 } 5945 5946 spa_config_exit(spa, SCL_ALL, FTAG); 5947 5948 if (error != 0) { 5949 spa_unload(spa); 5950 spa_deactivate(spa); 5951 spa_remove(spa); 5952 mutex_exit(&spa_namespace_lock); 5953 return (error); 5954 } 5955 5956 /* 5957 * Get the list of spares, if specified. 5958 */ 5959 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5960 &spares, &nspares) == 0) { 5961 spa->spa_spares.sav_config = fnvlist_alloc(); 5962 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 5963 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 5964 nspares); 5965 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5966 spa_load_spares(spa); 5967 spa_config_exit(spa, SCL_ALL, FTAG); 5968 spa->spa_spares.sav_sync = B_TRUE; 5969 } 5970 5971 /* 5972 * Get the list of level 2 cache devices, if specified. 5973 */ 5974 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5975 &l2cache, &nl2cache) == 0) { 5976 VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config, 5977 NV_UNIQUE_NAME, KM_SLEEP)); 5978 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 5979 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 5980 nl2cache); 5981 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5982 spa_load_l2cache(spa); 5983 spa_config_exit(spa, SCL_ALL, FTAG); 5984 spa->spa_l2cache.sav_sync = B_TRUE; 5985 } 5986 5987 spa->spa_is_initializing = B_TRUE; 5988 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 5989 spa->spa_is_initializing = B_FALSE; 5990 5991 /* 5992 * Create DDTs (dedup tables). 5993 */ 5994 ddt_create(spa); 5995 /* 5996 * Create BRT table and BRT table object. 5997 */ 5998 brt_create(spa); 5999 6000 spa_update_dspace(spa); 6001 6002 tx = dmu_tx_create_assigned(dp, txg); 6003 6004 /* 6005 * Create the pool's history object. 6006 */ 6007 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 6008 spa_history_create_obj(spa, tx); 6009 6010 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 6011 spa_history_log_version(spa, "create", tx); 6012 6013 /* 6014 * Create the pool config object. 6015 */ 6016 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 6017 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 6018 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 6019 6020 if (zap_add(spa->spa_meta_objset, 6021 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 6022 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 6023 cmn_err(CE_PANIC, "failed to add pool config"); 6024 } 6025 6026 if (zap_add(spa->spa_meta_objset, 6027 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 6028 sizeof (uint64_t), 1, &version, tx) != 0) { 6029 cmn_err(CE_PANIC, "failed to add pool version"); 6030 } 6031 6032 /* Newly created pools with the right version are always deflated. */ 6033 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 6034 spa->spa_deflate = TRUE; 6035 if (zap_add(spa->spa_meta_objset, 6036 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 6037 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 6038 cmn_err(CE_PANIC, "failed to add deflate"); 6039 } 6040 } 6041 6042 /* 6043 * Create the deferred-free bpobj. Turn off compression 6044 * because sync-to-convergence takes longer if the blocksize 6045 * keeps changing. 6046 */ 6047 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 6048 dmu_object_set_compress(spa->spa_meta_objset, obj, 6049 ZIO_COMPRESS_OFF, tx); 6050 if (zap_add(spa->spa_meta_objset, 6051 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 6052 sizeof (uint64_t), 1, &obj, tx) != 0) { 6053 cmn_err(CE_PANIC, "failed to add bpobj"); 6054 } 6055 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 6056 spa->spa_meta_objset, obj)); 6057 6058 /* 6059 * Generate some random noise for salted checksums to operate on. 6060 */ 6061 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 6062 sizeof (spa->spa_cksum_salt.zcs_bytes)); 6063 6064 /* 6065 * Set pool properties. 6066 */ 6067 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 6068 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 6069 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 6070 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 6071 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 6072 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 6073 6074 if (props != NULL) { 6075 spa_configfile_set(spa, props, B_FALSE); 6076 spa_sync_props(props, tx); 6077 } 6078 6079 for (int i = 0; i < ndraid; i++) 6080 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6081 6082 dmu_tx_commit(tx); 6083 6084 spa->spa_sync_on = B_TRUE; 6085 txg_sync_start(dp); 6086 mmp_thread_start(spa); 6087 txg_wait_synced(dp, txg); 6088 6089 spa_spawn_aux_threads(spa); 6090 6091 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 6092 6093 /* 6094 * Don't count references from objsets that are already closed 6095 * and are making their way through the eviction process. 6096 */ 6097 spa_evicting_os_wait(spa); 6098 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 6099 spa->spa_load_state = SPA_LOAD_NONE; 6100 6101 spa_import_os(spa); 6102 6103 mutex_exit(&spa_namespace_lock); 6104 6105 return (0); 6106 } 6107 6108 /* 6109 * Import a non-root pool into the system. 6110 */ 6111 int 6112 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 6113 { 6114 spa_t *spa; 6115 const char *altroot = NULL; 6116 spa_load_state_t state = SPA_LOAD_IMPORT; 6117 zpool_load_policy_t policy; 6118 spa_mode_t mode = spa_mode_global; 6119 uint64_t readonly = B_FALSE; 6120 int error; 6121 nvlist_t *nvroot; 6122 nvlist_t **spares, **l2cache; 6123 uint_t nspares, nl2cache; 6124 6125 /* 6126 * If a pool with this name exists, return failure. 6127 */ 6128 mutex_enter(&spa_namespace_lock); 6129 if (spa_lookup(pool) != NULL) { 6130 mutex_exit(&spa_namespace_lock); 6131 return (SET_ERROR(EEXIST)); 6132 } 6133 6134 /* 6135 * Create and initialize the spa structure. 6136 */ 6137 (void) nvlist_lookup_string(props, 6138 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6139 (void) nvlist_lookup_uint64(props, 6140 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 6141 if (readonly) 6142 mode = SPA_MODE_READ; 6143 spa = spa_add(pool, config, altroot); 6144 spa->spa_import_flags = flags; 6145 6146 /* 6147 * Verbatim import - Take a pool and insert it into the namespace 6148 * as if it had been loaded at boot. 6149 */ 6150 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 6151 if (props != NULL) 6152 spa_configfile_set(spa, props, B_FALSE); 6153 6154 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE); 6155 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6156 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 6157 mutex_exit(&spa_namespace_lock); 6158 return (0); 6159 } 6160 6161 spa_activate(spa, mode); 6162 6163 /* 6164 * Don't start async tasks until we know everything is healthy. 6165 */ 6166 spa_async_suspend(spa); 6167 6168 zpool_get_load_policy(config, &policy); 6169 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6170 state = SPA_LOAD_RECOVER; 6171 6172 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 6173 6174 if (state != SPA_LOAD_RECOVER) { 6175 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6176 zfs_dbgmsg("spa_import: importing %s", pool); 6177 } else { 6178 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 6179 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 6180 } 6181 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 6182 6183 /* 6184 * Propagate anything learned while loading the pool and pass it 6185 * back to caller (i.e. rewind info, missing devices, etc). 6186 */ 6187 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info); 6188 6189 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6190 /* 6191 * Toss any existing sparelist, as it doesn't have any validity 6192 * anymore, and conflicts with spa_has_spare(). 6193 */ 6194 if (spa->spa_spares.sav_config) { 6195 nvlist_free(spa->spa_spares.sav_config); 6196 spa->spa_spares.sav_config = NULL; 6197 spa_load_spares(spa); 6198 } 6199 if (spa->spa_l2cache.sav_config) { 6200 nvlist_free(spa->spa_l2cache.sav_config); 6201 spa->spa_l2cache.sav_config = NULL; 6202 spa_load_l2cache(spa); 6203 } 6204 6205 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 6206 spa_config_exit(spa, SCL_ALL, FTAG); 6207 6208 if (props != NULL) 6209 spa_configfile_set(spa, props, B_FALSE); 6210 6211 if (error != 0 || (props && spa_writeable(spa) && 6212 (error = spa_prop_set(spa, props)))) { 6213 spa_unload(spa); 6214 spa_deactivate(spa); 6215 spa_remove(spa); 6216 mutex_exit(&spa_namespace_lock); 6217 return (error); 6218 } 6219 6220 spa_async_resume(spa); 6221 6222 /* 6223 * Override any spares and level 2 cache devices as specified by 6224 * the user, as these may have correct device names/devids, etc. 6225 */ 6226 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6227 &spares, &nspares) == 0) { 6228 if (spa->spa_spares.sav_config) 6229 fnvlist_remove(spa->spa_spares.sav_config, 6230 ZPOOL_CONFIG_SPARES); 6231 else 6232 spa->spa_spares.sav_config = fnvlist_alloc(); 6233 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 6234 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 6235 nspares); 6236 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6237 spa_load_spares(spa); 6238 spa_config_exit(spa, SCL_ALL, FTAG); 6239 spa->spa_spares.sav_sync = B_TRUE; 6240 } 6241 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6242 &l2cache, &nl2cache) == 0) { 6243 if (spa->spa_l2cache.sav_config) 6244 fnvlist_remove(spa->spa_l2cache.sav_config, 6245 ZPOOL_CONFIG_L2CACHE); 6246 else 6247 spa->spa_l2cache.sav_config = fnvlist_alloc(); 6248 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6249 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 6250 nl2cache); 6251 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6252 spa_load_l2cache(spa); 6253 spa_config_exit(spa, SCL_ALL, FTAG); 6254 spa->spa_l2cache.sav_sync = B_TRUE; 6255 } 6256 6257 /* 6258 * Check for any removed devices. 6259 */ 6260 if (spa->spa_autoreplace) { 6261 spa_aux_check_removed(&spa->spa_spares); 6262 spa_aux_check_removed(&spa->spa_l2cache); 6263 } 6264 6265 if (spa_writeable(spa)) { 6266 /* 6267 * Update the config cache to include the newly-imported pool. 6268 */ 6269 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6270 } 6271 6272 /* 6273 * It's possible that the pool was expanded while it was exported. 6274 * We kick off an async task to handle this for us. 6275 */ 6276 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 6277 6278 spa_history_log_version(spa, "import", NULL); 6279 6280 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6281 6282 mutex_exit(&spa_namespace_lock); 6283 6284 zvol_create_minors_recursive(pool); 6285 6286 spa_import_os(spa); 6287 6288 return (0); 6289 } 6290 6291 nvlist_t * 6292 spa_tryimport(nvlist_t *tryconfig) 6293 { 6294 nvlist_t *config = NULL; 6295 const char *poolname, *cachefile; 6296 spa_t *spa; 6297 uint64_t state; 6298 int error; 6299 zpool_load_policy_t policy; 6300 6301 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 6302 return (NULL); 6303 6304 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 6305 return (NULL); 6306 6307 /* 6308 * Create and initialize the spa structure. 6309 */ 6310 mutex_enter(&spa_namespace_lock); 6311 spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL); 6312 spa_activate(spa, SPA_MODE_READ); 6313 6314 /* 6315 * Rewind pool if a max txg was provided. 6316 */ 6317 zpool_get_load_policy(spa->spa_config, &policy); 6318 if (policy.zlp_txg != UINT64_MAX) { 6319 spa->spa_load_max_txg = policy.zlp_txg; 6320 spa->spa_extreme_rewind = B_TRUE; 6321 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 6322 poolname, (longlong_t)policy.zlp_txg); 6323 } else { 6324 zfs_dbgmsg("spa_tryimport: importing %s", poolname); 6325 } 6326 6327 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 6328 == 0) { 6329 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 6330 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6331 } else { 6332 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 6333 } 6334 6335 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 6336 6337 /* 6338 * If 'tryconfig' was at least parsable, return the current config. 6339 */ 6340 if (spa->spa_root_vdev != NULL) { 6341 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6342 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname); 6343 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state); 6344 fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 6345 spa->spa_uberblock.ub_timestamp); 6346 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6347 spa->spa_load_info); 6348 fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 6349 spa->spa_errata); 6350 6351 /* 6352 * If the bootfs property exists on this pool then we 6353 * copy it out so that external consumers can tell which 6354 * pools are bootable. 6355 */ 6356 if ((!error || error == EEXIST) && spa->spa_bootfs) { 6357 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6358 6359 /* 6360 * We have to play games with the name since the 6361 * pool was opened as TRYIMPORT_NAME. 6362 */ 6363 if (dsl_dsobj_to_dsname(spa_name(spa), 6364 spa->spa_bootfs, tmpname) == 0) { 6365 char *cp; 6366 char *dsname; 6367 6368 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6369 6370 cp = strchr(tmpname, '/'); 6371 if (cp == NULL) { 6372 (void) strlcpy(dsname, tmpname, 6373 MAXPATHLEN); 6374 } else { 6375 (void) snprintf(dsname, MAXPATHLEN, 6376 "%s/%s", poolname, ++cp); 6377 } 6378 fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS, 6379 dsname); 6380 kmem_free(dsname, MAXPATHLEN); 6381 } 6382 kmem_free(tmpname, MAXPATHLEN); 6383 } 6384 6385 /* 6386 * Add the list of hot spares and level 2 cache devices. 6387 */ 6388 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6389 spa_add_spares(spa, config); 6390 spa_add_l2cache(spa, config); 6391 spa_config_exit(spa, SCL_CONFIG, FTAG); 6392 } 6393 6394 spa_unload(spa); 6395 spa_deactivate(spa); 6396 spa_remove(spa); 6397 mutex_exit(&spa_namespace_lock); 6398 6399 return (config); 6400 } 6401 6402 /* 6403 * Pool export/destroy 6404 * 6405 * The act of destroying or exporting a pool is very simple. We make sure there 6406 * is no more pending I/O and any references to the pool are gone. Then, we 6407 * update the pool state and sync all the labels to disk, removing the 6408 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 6409 * we don't sync the labels or remove the configuration cache. 6410 */ 6411 static int 6412 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 6413 boolean_t force, boolean_t hardforce) 6414 { 6415 int error; 6416 spa_t *spa; 6417 6418 if (oldconfig) 6419 *oldconfig = NULL; 6420 6421 if (!(spa_mode_global & SPA_MODE_WRITE)) 6422 return (SET_ERROR(EROFS)); 6423 6424 mutex_enter(&spa_namespace_lock); 6425 if ((spa = spa_lookup(pool)) == NULL) { 6426 mutex_exit(&spa_namespace_lock); 6427 return (SET_ERROR(ENOENT)); 6428 } 6429 6430 if (spa->spa_is_exporting) { 6431 /* the pool is being exported by another thread */ 6432 mutex_exit(&spa_namespace_lock); 6433 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 6434 } 6435 spa->spa_is_exporting = B_TRUE; 6436 6437 /* 6438 * Put a hold on the pool, drop the namespace lock, stop async tasks, 6439 * reacquire the namespace lock, and see if we can export. 6440 */ 6441 spa_open_ref(spa, FTAG); 6442 mutex_exit(&spa_namespace_lock); 6443 spa_async_suspend(spa); 6444 if (spa->spa_zvol_taskq) { 6445 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 6446 taskq_wait(spa->spa_zvol_taskq); 6447 } 6448 mutex_enter(&spa_namespace_lock); 6449 spa_close(spa, FTAG); 6450 6451 if (spa->spa_state == POOL_STATE_UNINITIALIZED) 6452 goto export_spa; 6453 /* 6454 * The pool will be in core if it's openable, in which case we can 6455 * modify its state. Objsets may be open only because they're dirty, 6456 * so we have to force it to sync before checking spa_refcnt. 6457 */ 6458 if (spa->spa_sync_on) { 6459 txg_wait_synced(spa->spa_dsl_pool, 0); 6460 spa_evicting_os_wait(spa); 6461 } 6462 6463 /* 6464 * A pool cannot be exported or destroyed if there are active 6465 * references. If we are resetting a pool, allow references by 6466 * fault injection handlers. 6467 */ 6468 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 6469 error = SET_ERROR(EBUSY); 6470 goto fail; 6471 } 6472 6473 if (spa->spa_sync_on) { 6474 vdev_t *rvd = spa->spa_root_vdev; 6475 /* 6476 * A pool cannot be exported if it has an active shared spare. 6477 * This is to prevent other pools stealing the active spare 6478 * from an exported pool. At user's own will, such pool can 6479 * be forcedly exported. 6480 */ 6481 if (!force && new_state == POOL_STATE_EXPORTED && 6482 spa_has_active_shared_spare(spa)) { 6483 error = SET_ERROR(EXDEV); 6484 goto fail; 6485 } 6486 6487 /* 6488 * We're about to export or destroy this pool. Make sure 6489 * we stop all initialization and trim activity here before 6490 * we set the spa_final_txg. This will ensure that all 6491 * dirty data resulting from the initialization is 6492 * committed to disk before we unload the pool. 6493 */ 6494 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 6495 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 6496 vdev_autotrim_stop_all(spa); 6497 vdev_rebuild_stop_all(spa); 6498 6499 /* 6500 * We want this to be reflected on every label, 6501 * so mark them all dirty. spa_unload() will do the 6502 * final sync that pushes these changes out. 6503 */ 6504 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6505 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6506 spa->spa_state = new_state; 6507 vdev_config_dirty(rvd); 6508 spa_config_exit(spa, SCL_ALL, FTAG); 6509 } 6510 6511 /* 6512 * If the log space map feature is enabled and the pool is 6513 * getting exported (but not destroyed), we want to spend some 6514 * time flushing as many metaslabs as we can in an attempt to 6515 * destroy log space maps and save import time. This has to be 6516 * done before we set the spa_final_txg, otherwise 6517 * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs. 6518 * spa_should_flush_logs_on_unload() should be called after 6519 * spa_state has been set to the new_state. 6520 */ 6521 if (spa_should_flush_logs_on_unload(spa)) 6522 spa_unload_log_sm_flush_all(spa); 6523 6524 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6525 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6526 spa->spa_final_txg = spa_last_synced_txg(spa) + 6527 TXG_DEFER_SIZE + 1; 6528 spa_config_exit(spa, SCL_ALL, FTAG); 6529 } 6530 } 6531 6532 export_spa: 6533 spa_export_os(spa); 6534 6535 if (new_state == POOL_STATE_DESTROYED) 6536 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 6537 else if (new_state == POOL_STATE_EXPORTED) 6538 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 6539 6540 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 6541 spa_unload(spa); 6542 spa_deactivate(spa); 6543 } 6544 6545 if (oldconfig && spa->spa_config) 6546 *oldconfig = fnvlist_dup(spa->spa_config); 6547 6548 if (new_state != POOL_STATE_UNINITIALIZED) { 6549 if (!hardforce) 6550 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 6551 spa_remove(spa); 6552 } else { 6553 /* 6554 * If spa_remove() is not called for this spa_t and 6555 * there is any possibility that it can be reused, 6556 * we make sure to reset the exporting flag. 6557 */ 6558 spa->spa_is_exporting = B_FALSE; 6559 } 6560 6561 mutex_exit(&spa_namespace_lock); 6562 return (0); 6563 6564 fail: 6565 spa->spa_is_exporting = B_FALSE; 6566 spa_async_resume(spa); 6567 mutex_exit(&spa_namespace_lock); 6568 return (error); 6569 } 6570 6571 /* 6572 * Destroy a storage pool. 6573 */ 6574 int 6575 spa_destroy(const char *pool) 6576 { 6577 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 6578 B_FALSE, B_FALSE)); 6579 } 6580 6581 /* 6582 * Export a storage pool. 6583 */ 6584 int 6585 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 6586 boolean_t hardforce) 6587 { 6588 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 6589 force, hardforce)); 6590 } 6591 6592 /* 6593 * Similar to spa_export(), this unloads the spa_t without actually removing it 6594 * from the namespace in any way. 6595 */ 6596 int 6597 spa_reset(const char *pool) 6598 { 6599 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 6600 B_FALSE, B_FALSE)); 6601 } 6602 6603 /* 6604 * ========================================================================== 6605 * Device manipulation 6606 * ========================================================================== 6607 */ 6608 6609 /* 6610 * This is called as a synctask to increment the draid feature flag 6611 */ 6612 static void 6613 spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 6614 { 6615 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 6616 int draid = (int)(uintptr_t)arg; 6617 6618 for (int c = 0; c < draid; c++) 6619 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6620 } 6621 6622 /* 6623 * Add a device to a storage pool. 6624 */ 6625 int 6626 spa_vdev_add(spa_t *spa, nvlist_t *nvroot) 6627 { 6628 uint64_t txg, ndraid = 0; 6629 int error; 6630 vdev_t *rvd = spa->spa_root_vdev; 6631 vdev_t *vd, *tvd; 6632 nvlist_t **spares, **l2cache; 6633 uint_t nspares, nl2cache; 6634 6635 ASSERT(spa_writeable(spa)); 6636 6637 txg = spa_vdev_enter(spa); 6638 6639 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 6640 VDEV_ALLOC_ADD)) != 0) 6641 return (spa_vdev_exit(spa, NULL, txg, error)); 6642 6643 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 6644 6645 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 6646 &nspares) != 0) 6647 nspares = 0; 6648 6649 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 6650 &nl2cache) != 0) 6651 nl2cache = 0; 6652 6653 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 6654 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6655 6656 if (vd->vdev_children != 0 && 6657 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 6658 return (spa_vdev_exit(spa, vd, txg, error)); 6659 } 6660 6661 /* 6662 * The virtual dRAID spares must be added after vdev tree is created 6663 * and the vdev guids are generated. The guid of their associated 6664 * dRAID is stored in the config and used when opening the spare. 6665 */ 6666 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 6667 rvd->vdev_children)) == 0) { 6668 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 6669 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 6670 nspares = 0; 6671 } else { 6672 return (spa_vdev_exit(spa, vd, txg, error)); 6673 } 6674 6675 /* 6676 * We must validate the spares and l2cache devices after checking the 6677 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 6678 */ 6679 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 6680 return (spa_vdev_exit(spa, vd, txg, error)); 6681 6682 /* 6683 * If we are in the middle of a device removal, we can only add 6684 * devices which match the existing devices in the pool. 6685 * If we are in the middle of a removal, or have some indirect 6686 * vdevs, we can not add raidz or dRAID top levels. 6687 */ 6688 if (spa->spa_vdev_removal != NULL || 6689 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 6690 for (int c = 0; c < vd->vdev_children; c++) { 6691 tvd = vd->vdev_child[c]; 6692 if (spa->spa_vdev_removal != NULL && 6693 tvd->vdev_ashift != spa->spa_max_ashift) { 6694 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6695 } 6696 /* Fail if top level vdev is raidz or a dRAID */ 6697 if (vdev_get_nparity(tvd) != 0) 6698 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6699 6700 /* 6701 * Need the top level mirror to be 6702 * a mirror of leaf vdevs only 6703 */ 6704 if (tvd->vdev_ops == &vdev_mirror_ops) { 6705 for (uint64_t cid = 0; 6706 cid < tvd->vdev_children; cid++) { 6707 vdev_t *cvd = tvd->vdev_child[cid]; 6708 if (!cvd->vdev_ops->vdev_op_leaf) { 6709 return (spa_vdev_exit(spa, vd, 6710 txg, EINVAL)); 6711 } 6712 } 6713 } 6714 } 6715 } 6716 6717 for (int c = 0; c < vd->vdev_children; c++) { 6718 tvd = vd->vdev_child[c]; 6719 vdev_remove_child(vd, tvd); 6720 tvd->vdev_id = rvd->vdev_children; 6721 vdev_add_child(rvd, tvd); 6722 vdev_config_dirty(tvd); 6723 } 6724 6725 if (nspares != 0) { 6726 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 6727 ZPOOL_CONFIG_SPARES); 6728 spa_load_spares(spa); 6729 spa->spa_spares.sav_sync = B_TRUE; 6730 } 6731 6732 if (nl2cache != 0) { 6733 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 6734 ZPOOL_CONFIG_L2CACHE); 6735 spa_load_l2cache(spa); 6736 spa->spa_l2cache.sav_sync = B_TRUE; 6737 } 6738 6739 /* 6740 * We can't increment a feature while holding spa_vdev so we 6741 * have to do it in a synctask. 6742 */ 6743 if (ndraid != 0) { 6744 dmu_tx_t *tx; 6745 6746 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 6747 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 6748 (void *)(uintptr_t)ndraid, tx); 6749 dmu_tx_commit(tx); 6750 } 6751 6752 /* 6753 * We have to be careful when adding new vdevs to an existing pool. 6754 * If other threads start allocating from these vdevs before we 6755 * sync the config cache, and we lose power, then upon reboot we may 6756 * fail to open the pool because there are DVAs that the config cache 6757 * can't translate. Therefore, we first add the vdevs without 6758 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 6759 * and then let spa_config_update() initialize the new metaslabs. 6760 * 6761 * spa_load() checks for added-but-not-initialized vdevs, so that 6762 * if we lose power at any point in this sequence, the remaining 6763 * steps will be completed the next time we load the pool. 6764 */ 6765 (void) spa_vdev_exit(spa, vd, txg, 0); 6766 6767 mutex_enter(&spa_namespace_lock); 6768 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6769 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 6770 mutex_exit(&spa_namespace_lock); 6771 6772 return (0); 6773 } 6774 6775 /* 6776 * Attach a device to a mirror. The arguments are the path to any device 6777 * in the mirror, and the nvroot for the new device. If the path specifies 6778 * a device that is not mirrored, we automatically insert the mirror vdev. 6779 * 6780 * If 'replacing' is specified, the new device is intended to replace the 6781 * existing device; in this case the two devices are made into their own 6782 * mirror using the 'replacing' vdev, which is functionally identical to 6783 * the mirror vdev (it actually reuses all the same ops) but has a few 6784 * extra rules: you can't attach to it after it's been created, and upon 6785 * completion of resilvering, the first disk (the one being replaced) 6786 * is automatically detached. 6787 * 6788 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 6789 * should be performed instead of traditional healing reconstruction. From 6790 * an administrators perspective these are both resilver operations. 6791 */ 6792 int 6793 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 6794 int rebuild) 6795 { 6796 uint64_t txg, dtl_max_txg; 6797 vdev_t *rvd = spa->spa_root_vdev; 6798 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 6799 vdev_ops_t *pvops; 6800 char *oldvdpath, *newvdpath; 6801 int newvd_isspare; 6802 int error; 6803 6804 ASSERT(spa_writeable(spa)); 6805 6806 txg = spa_vdev_enter(spa); 6807 6808 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 6809 6810 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6811 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6812 error = (spa_has_checkpoint(spa)) ? 6813 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6814 return (spa_vdev_exit(spa, NULL, txg, error)); 6815 } 6816 6817 if (rebuild) { 6818 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 6819 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6820 6821 if (dsl_scan_resilvering(spa_get_dsl(spa))) 6822 return (spa_vdev_exit(spa, NULL, txg, 6823 ZFS_ERR_RESILVER_IN_PROGRESS)); 6824 } else { 6825 if (vdev_rebuild_active(rvd)) 6826 return (spa_vdev_exit(spa, NULL, txg, 6827 ZFS_ERR_REBUILD_IN_PROGRESS)); 6828 } 6829 6830 if (spa->spa_vdev_removal != NULL) 6831 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 6832 6833 if (oldvd == NULL) 6834 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6835 6836 if (!oldvd->vdev_ops->vdev_op_leaf) 6837 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6838 6839 pvd = oldvd->vdev_parent; 6840 6841 if (spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 6842 VDEV_ALLOC_ATTACH) != 0) 6843 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 6844 6845 if (newrootvd->vdev_children != 1) 6846 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6847 6848 newvd = newrootvd->vdev_child[0]; 6849 6850 if (!newvd->vdev_ops->vdev_op_leaf) 6851 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6852 6853 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 6854 return (spa_vdev_exit(spa, newrootvd, txg, error)); 6855 6856 /* 6857 * log, dedup and special vdevs should not be replaced by spares. 6858 */ 6859 if ((oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE || 6860 oldvd->vdev_top->vdev_islog) && newvd->vdev_isspare) { 6861 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6862 } 6863 6864 /* 6865 * A dRAID spare can only replace a child of its parent dRAID vdev. 6866 */ 6867 if (newvd->vdev_ops == &vdev_draid_spare_ops && 6868 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 6869 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6870 } 6871 6872 if (rebuild) { 6873 /* 6874 * For rebuilds, the top vdev must support reconstruction 6875 * using only space maps. This means the only allowable 6876 * vdevs types are the root vdev, a mirror, or dRAID. 6877 */ 6878 tvd = pvd; 6879 if (pvd->vdev_top != NULL) 6880 tvd = pvd->vdev_top; 6881 6882 if (tvd->vdev_ops != &vdev_mirror_ops && 6883 tvd->vdev_ops != &vdev_root_ops && 6884 tvd->vdev_ops != &vdev_draid_ops) { 6885 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6886 } 6887 } 6888 6889 if (!replacing) { 6890 /* 6891 * For attach, the only allowable parent is a mirror or the root 6892 * vdev. 6893 */ 6894 if (pvd->vdev_ops != &vdev_mirror_ops && 6895 pvd->vdev_ops != &vdev_root_ops) 6896 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6897 6898 pvops = &vdev_mirror_ops; 6899 } else { 6900 /* 6901 * Active hot spares can only be replaced by inactive hot 6902 * spares. 6903 */ 6904 if (pvd->vdev_ops == &vdev_spare_ops && 6905 oldvd->vdev_isspare && 6906 !spa_has_spare(spa, newvd->vdev_guid)) 6907 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6908 6909 /* 6910 * If the source is a hot spare, and the parent isn't already a 6911 * spare, then we want to create a new hot spare. Otherwise, we 6912 * want to create a replacing vdev. The user is not allowed to 6913 * attach to a spared vdev child unless the 'isspare' state is 6914 * the same (spare replaces spare, non-spare replaces 6915 * non-spare). 6916 */ 6917 if (pvd->vdev_ops == &vdev_replacing_ops && 6918 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 6919 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6920 } else if (pvd->vdev_ops == &vdev_spare_ops && 6921 newvd->vdev_isspare != oldvd->vdev_isspare) { 6922 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6923 } 6924 6925 if (newvd->vdev_isspare) 6926 pvops = &vdev_spare_ops; 6927 else 6928 pvops = &vdev_replacing_ops; 6929 } 6930 6931 /* 6932 * Make sure the new device is big enough. 6933 */ 6934 if (newvd->vdev_asize < vdev_get_min_asize(oldvd)) 6935 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 6936 6937 /* 6938 * The new device cannot have a higher alignment requirement 6939 * than the top-level vdev. 6940 */ 6941 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) 6942 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6943 6944 /* 6945 * If this is an in-place replacement, update oldvd's path and devid 6946 * to make it distinguishable from newvd, and unopenable from now on. 6947 */ 6948 if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { 6949 spa_strfree(oldvd->vdev_path); 6950 oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, 6951 KM_SLEEP); 6952 (void) snprintf(oldvd->vdev_path, strlen(newvd->vdev_path) + 5, 6953 "%s/%s", newvd->vdev_path, "old"); 6954 if (oldvd->vdev_devid != NULL) { 6955 spa_strfree(oldvd->vdev_devid); 6956 oldvd->vdev_devid = NULL; 6957 } 6958 } 6959 6960 /* 6961 * If the parent is not a mirror, or if we're replacing, insert the new 6962 * mirror/replacing/spare vdev above oldvd. 6963 */ 6964 if (pvd->vdev_ops != pvops) 6965 pvd = vdev_add_parent(oldvd, pvops); 6966 6967 ASSERT(pvd->vdev_top->vdev_parent == rvd); 6968 ASSERT(pvd->vdev_ops == pvops); 6969 ASSERT(oldvd->vdev_parent == pvd); 6970 6971 /* 6972 * Extract the new device from its root and add it to pvd. 6973 */ 6974 vdev_remove_child(newrootvd, newvd); 6975 newvd->vdev_id = pvd->vdev_children; 6976 newvd->vdev_crtxg = oldvd->vdev_crtxg; 6977 vdev_add_child(pvd, newvd); 6978 6979 /* 6980 * Reevaluate the parent vdev state. 6981 */ 6982 vdev_propagate_state(pvd); 6983 6984 tvd = newvd->vdev_top; 6985 ASSERT(pvd->vdev_top == tvd); 6986 ASSERT(tvd->vdev_parent == rvd); 6987 6988 vdev_config_dirty(tvd); 6989 6990 /* 6991 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 6992 * for any dmu_sync-ed blocks. It will propagate upward when 6993 * spa_vdev_exit() calls vdev_dtl_reassess(). 6994 */ 6995 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 6996 6997 vdev_dtl_dirty(newvd, DTL_MISSING, 6998 TXG_INITIAL, dtl_max_txg - TXG_INITIAL); 6999 7000 if (newvd->vdev_isspare) { 7001 spa_spare_activate(newvd); 7002 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 7003 } 7004 7005 oldvdpath = spa_strdup(oldvd->vdev_path); 7006 newvdpath = spa_strdup(newvd->vdev_path); 7007 newvd_isspare = newvd->vdev_isspare; 7008 7009 /* 7010 * Mark newvd's DTL dirty in this txg. 7011 */ 7012 vdev_dirty(tvd, VDD_DTL, newvd, txg); 7013 7014 /* 7015 * Schedule the resilver or rebuild to restart in the future. We do 7016 * this to ensure that dmu_sync-ed blocks have been stitched into the 7017 * respective datasets. 7018 */ 7019 if (rebuild) { 7020 newvd->vdev_rebuild_txg = txg; 7021 7022 vdev_rebuild(tvd); 7023 } else { 7024 newvd->vdev_resilver_txg = txg; 7025 7026 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 7027 spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) { 7028 vdev_defer_resilver(newvd); 7029 } else { 7030 dsl_scan_restart_resilver(spa->spa_dsl_pool, 7031 dtl_max_txg); 7032 } 7033 } 7034 7035 if (spa->spa_bootfs) 7036 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 7037 7038 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 7039 7040 /* 7041 * Commit the config 7042 */ 7043 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 7044 7045 spa_history_log_internal(spa, "vdev attach", NULL, 7046 "%s vdev=%s %s vdev=%s", 7047 replacing && newvd_isspare ? "spare in" : 7048 replacing ? "replace" : "attach", newvdpath, 7049 replacing ? "for" : "to", oldvdpath); 7050 7051 spa_strfree(oldvdpath); 7052 spa_strfree(newvdpath); 7053 7054 return (0); 7055 } 7056 7057 /* 7058 * Detach a device from a mirror or replacing vdev. 7059 * 7060 * If 'replace_done' is specified, only detach if the parent 7061 * is a replacing vdev. 7062 */ 7063 int 7064 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 7065 { 7066 uint64_t txg; 7067 int error; 7068 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 7069 vdev_t *vd, *pvd, *cvd, *tvd; 7070 boolean_t unspare = B_FALSE; 7071 uint64_t unspare_guid = 0; 7072 char *vdpath; 7073 7074 ASSERT(spa_writeable(spa)); 7075 7076 txg = spa_vdev_detach_enter(spa, guid); 7077 7078 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7079 7080 /* 7081 * Besides being called directly from the userland through the 7082 * ioctl interface, spa_vdev_detach() can be potentially called 7083 * at the end of spa_vdev_resilver_done(). 7084 * 7085 * In the regular case, when we have a checkpoint this shouldn't 7086 * happen as we never empty the DTLs of a vdev during the scrub 7087 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 7088 * should never get here when we have a checkpoint. 7089 * 7090 * That said, even in a case when we checkpoint the pool exactly 7091 * as spa_vdev_resilver_done() calls this function everything 7092 * should be fine as the resilver will return right away. 7093 */ 7094 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7095 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7096 error = (spa_has_checkpoint(spa)) ? 7097 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7098 return (spa_vdev_exit(spa, NULL, txg, error)); 7099 } 7100 7101 if (vd == NULL) 7102 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 7103 7104 if (!vd->vdev_ops->vdev_op_leaf) 7105 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7106 7107 pvd = vd->vdev_parent; 7108 7109 /* 7110 * If the parent/child relationship is not as expected, don't do it. 7111 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 7112 * vdev that's replacing B with C. The user's intent in replacing 7113 * is to go from M(A,B) to M(A,C). If the user decides to cancel 7114 * the replace by detaching C, the expected behavior is to end up 7115 * M(A,B). But suppose that right after deciding to detach C, 7116 * the replacement of B completes. We would have M(A,C), and then 7117 * ask to detach C, which would leave us with just A -- not what 7118 * the user wanted. To prevent this, we make sure that the 7119 * parent/child relationship hasn't changed -- in this example, 7120 * that C's parent is still the replacing vdev R. 7121 */ 7122 if (pvd->vdev_guid != pguid && pguid != 0) 7123 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7124 7125 /* 7126 * Only 'replacing' or 'spare' vdevs can be replaced. 7127 */ 7128 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 7129 pvd->vdev_ops != &vdev_spare_ops) 7130 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7131 7132 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 7133 spa_version(spa) >= SPA_VERSION_SPARES); 7134 7135 /* 7136 * Only mirror, replacing, and spare vdevs support detach. 7137 */ 7138 if (pvd->vdev_ops != &vdev_replacing_ops && 7139 pvd->vdev_ops != &vdev_mirror_ops && 7140 pvd->vdev_ops != &vdev_spare_ops) 7141 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7142 7143 /* 7144 * If this device has the only valid copy of some data, 7145 * we cannot safely detach it. 7146 */ 7147 if (vdev_dtl_required(vd)) 7148 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7149 7150 ASSERT(pvd->vdev_children >= 2); 7151 7152 /* 7153 * If we are detaching the second disk from a replacing vdev, then 7154 * check to see if we changed the original vdev's path to have "/old" 7155 * at the end in spa_vdev_attach(). If so, undo that change now. 7156 */ 7157 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 7158 vd->vdev_path != NULL) { 7159 size_t len = strlen(vd->vdev_path); 7160 7161 for (int c = 0; c < pvd->vdev_children; c++) { 7162 cvd = pvd->vdev_child[c]; 7163 7164 if (cvd == vd || cvd->vdev_path == NULL) 7165 continue; 7166 7167 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 7168 strcmp(cvd->vdev_path + len, "/old") == 0) { 7169 spa_strfree(cvd->vdev_path); 7170 cvd->vdev_path = spa_strdup(vd->vdev_path); 7171 break; 7172 } 7173 } 7174 } 7175 7176 /* 7177 * If we are detaching the original disk from a normal spare, then it 7178 * implies that the spare should become a real disk, and be removed 7179 * from the active spare list for the pool. dRAID spares on the 7180 * other hand are coupled to the pool and thus should never be removed 7181 * from the spares list. 7182 */ 7183 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 7184 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7185 7186 if (last_cvd->vdev_isspare && 7187 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 7188 unspare = B_TRUE; 7189 } 7190 } 7191 7192 /* 7193 * Erase the disk labels so the disk can be used for other things. 7194 * This must be done after all other error cases are handled, 7195 * but before we disembowel vd (so we can still do I/O to it). 7196 * But if we can't do it, don't treat the error as fatal -- 7197 * it may be that the unwritability of the disk is the reason 7198 * it's being detached! 7199 */ 7200 (void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 7201 7202 /* 7203 * Remove vd from its parent and compact the parent's children. 7204 */ 7205 vdev_remove_child(pvd, vd); 7206 vdev_compact_children(pvd); 7207 7208 /* 7209 * Remember one of the remaining children so we can get tvd below. 7210 */ 7211 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7212 7213 /* 7214 * If we need to remove the remaining child from the list of hot spares, 7215 * do it now, marking the vdev as no longer a spare in the process. 7216 * We must do this before vdev_remove_parent(), because that can 7217 * change the GUID if it creates a new toplevel GUID. For a similar 7218 * reason, we must remove the spare now, in the same txg as the detach; 7219 * otherwise someone could attach a new sibling, change the GUID, and 7220 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 7221 */ 7222 if (unspare) { 7223 ASSERT(cvd->vdev_isspare); 7224 spa_spare_remove(cvd); 7225 unspare_guid = cvd->vdev_guid; 7226 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 7227 cvd->vdev_unspare = B_TRUE; 7228 } 7229 7230 /* 7231 * If the parent mirror/replacing vdev only has one child, 7232 * the parent is no longer needed. Remove it from the tree. 7233 */ 7234 if (pvd->vdev_children == 1) { 7235 if (pvd->vdev_ops == &vdev_spare_ops) 7236 cvd->vdev_unspare = B_FALSE; 7237 vdev_remove_parent(cvd); 7238 } 7239 7240 /* 7241 * We don't set tvd until now because the parent we just removed 7242 * may have been the previous top-level vdev. 7243 */ 7244 tvd = cvd->vdev_top; 7245 ASSERT(tvd->vdev_parent == rvd); 7246 7247 /* 7248 * Reevaluate the parent vdev state. 7249 */ 7250 vdev_propagate_state(cvd); 7251 7252 /* 7253 * If the 'autoexpand' property is set on the pool then automatically 7254 * try to expand the size of the pool. For example if the device we 7255 * just detached was smaller than the others, it may be possible to 7256 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 7257 * first so that we can obtain the updated sizes of the leaf vdevs. 7258 */ 7259 if (spa->spa_autoexpand) { 7260 vdev_reopen(tvd); 7261 vdev_expand(tvd, txg); 7262 } 7263 7264 vdev_config_dirty(tvd); 7265 7266 /* 7267 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 7268 * vd->vdev_detached is set and free vd's DTL object in syncing context. 7269 * But first make sure we're not on any *other* txg's DTL list, to 7270 * prevent vd from being accessed after it's freed. 7271 */ 7272 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 7273 for (int t = 0; t < TXG_SIZE; t++) 7274 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 7275 vd->vdev_detached = B_TRUE; 7276 vdev_dirty(tvd, VDD_DTL, vd, txg); 7277 7278 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 7279 spa_notify_waiters(spa); 7280 7281 /* hang on to the spa before we release the lock */ 7282 spa_open_ref(spa, FTAG); 7283 7284 error = spa_vdev_exit(spa, vd, txg, 0); 7285 7286 spa_history_log_internal(spa, "detach", NULL, 7287 "vdev=%s", vdpath); 7288 spa_strfree(vdpath); 7289 7290 /* 7291 * If this was the removal of the original device in a hot spare vdev, 7292 * then we want to go through and remove the device from the hot spare 7293 * list of every other pool. 7294 */ 7295 if (unspare) { 7296 spa_t *altspa = NULL; 7297 7298 mutex_enter(&spa_namespace_lock); 7299 while ((altspa = spa_next(altspa)) != NULL) { 7300 if (altspa->spa_state != POOL_STATE_ACTIVE || 7301 altspa == spa) 7302 continue; 7303 7304 spa_open_ref(altspa, FTAG); 7305 mutex_exit(&spa_namespace_lock); 7306 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 7307 mutex_enter(&spa_namespace_lock); 7308 spa_close(altspa, FTAG); 7309 } 7310 mutex_exit(&spa_namespace_lock); 7311 7312 /* search the rest of the vdevs for spares to remove */ 7313 spa_vdev_resilver_done(spa); 7314 } 7315 7316 /* all done with the spa; OK to release */ 7317 mutex_enter(&spa_namespace_lock); 7318 spa_close(spa, FTAG); 7319 mutex_exit(&spa_namespace_lock); 7320 7321 return (error); 7322 } 7323 7324 static int 7325 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7326 list_t *vd_list) 7327 { 7328 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7329 7330 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7331 7332 /* Look up vdev and ensure it's a leaf. */ 7333 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7334 if (vd == NULL || vd->vdev_detached) { 7335 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7336 return (SET_ERROR(ENODEV)); 7337 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7338 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7339 return (SET_ERROR(EINVAL)); 7340 } else if (!vdev_writeable(vd)) { 7341 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7342 return (SET_ERROR(EROFS)); 7343 } 7344 mutex_enter(&vd->vdev_initialize_lock); 7345 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7346 7347 /* 7348 * When we activate an initialize action we check to see 7349 * if the vdev_initialize_thread is NULL. We do this instead 7350 * of using the vdev_initialize_state since there might be 7351 * a previous initialization process which has completed but 7352 * the thread is not exited. 7353 */ 7354 if (cmd_type == POOL_INITIALIZE_START && 7355 (vd->vdev_initialize_thread != NULL || 7356 vd->vdev_top->vdev_removing)) { 7357 mutex_exit(&vd->vdev_initialize_lock); 7358 return (SET_ERROR(EBUSY)); 7359 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 7360 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 7361 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 7362 mutex_exit(&vd->vdev_initialize_lock); 7363 return (SET_ERROR(ESRCH)); 7364 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 7365 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 7366 mutex_exit(&vd->vdev_initialize_lock); 7367 return (SET_ERROR(ESRCH)); 7368 } 7369 7370 switch (cmd_type) { 7371 case POOL_INITIALIZE_START: 7372 vdev_initialize(vd); 7373 break; 7374 case POOL_INITIALIZE_CANCEL: 7375 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 7376 break; 7377 case POOL_INITIALIZE_SUSPEND: 7378 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 7379 break; 7380 default: 7381 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7382 } 7383 mutex_exit(&vd->vdev_initialize_lock); 7384 7385 return (0); 7386 } 7387 7388 int 7389 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 7390 nvlist_t *vdev_errlist) 7391 { 7392 int total_errors = 0; 7393 list_t vd_list; 7394 7395 list_create(&vd_list, sizeof (vdev_t), 7396 offsetof(vdev_t, vdev_initialize_node)); 7397 7398 /* 7399 * We hold the namespace lock through the whole function 7400 * to prevent any changes to the pool while we're starting or 7401 * stopping initialization. The config and state locks are held so that 7402 * we can properly assess the vdev state before we commit to 7403 * the initializing operation. 7404 */ 7405 mutex_enter(&spa_namespace_lock); 7406 7407 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7408 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7409 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7410 7411 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 7412 &vd_list); 7413 if (error != 0) { 7414 char guid_as_str[MAXNAMELEN]; 7415 7416 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7417 "%llu", (unsigned long long)vdev_guid); 7418 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7419 total_errors++; 7420 } 7421 } 7422 7423 /* Wait for all initialize threads to stop. */ 7424 vdev_initialize_stop_wait(spa, &vd_list); 7425 7426 /* Sync out the initializing state */ 7427 txg_wait_synced(spa->spa_dsl_pool, 0); 7428 mutex_exit(&spa_namespace_lock); 7429 7430 list_destroy(&vd_list); 7431 7432 return (total_errors); 7433 } 7434 7435 static int 7436 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7437 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 7438 { 7439 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7440 7441 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7442 7443 /* Look up vdev and ensure it's a leaf. */ 7444 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7445 if (vd == NULL || vd->vdev_detached) { 7446 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7447 return (SET_ERROR(ENODEV)); 7448 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7449 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7450 return (SET_ERROR(EINVAL)); 7451 } else if (!vdev_writeable(vd)) { 7452 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7453 return (SET_ERROR(EROFS)); 7454 } else if (!vd->vdev_has_trim) { 7455 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7456 return (SET_ERROR(EOPNOTSUPP)); 7457 } else if (secure && !vd->vdev_has_securetrim) { 7458 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7459 return (SET_ERROR(EOPNOTSUPP)); 7460 } 7461 mutex_enter(&vd->vdev_trim_lock); 7462 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7463 7464 /* 7465 * When we activate a TRIM action we check to see if the 7466 * vdev_trim_thread is NULL. We do this instead of using the 7467 * vdev_trim_state since there might be a previous TRIM process 7468 * which has completed but the thread is not exited. 7469 */ 7470 if (cmd_type == POOL_TRIM_START && 7471 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing)) { 7472 mutex_exit(&vd->vdev_trim_lock); 7473 return (SET_ERROR(EBUSY)); 7474 } else if (cmd_type == POOL_TRIM_CANCEL && 7475 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 7476 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 7477 mutex_exit(&vd->vdev_trim_lock); 7478 return (SET_ERROR(ESRCH)); 7479 } else if (cmd_type == POOL_TRIM_SUSPEND && 7480 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 7481 mutex_exit(&vd->vdev_trim_lock); 7482 return (SET_ERROR(ESRCH)); 7483 } 7484 7485 switch (cmd_type) { 7486 case POOL_TRIM_START: 7487 vdev_trim(vd, rate, partial, secure); 7488 break; 7489 case POOL_TRIM_CANCEL: 7490 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 7491 break; 7492 case POOL_TRIM_SUSPEND: 7493 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 7494 break; 7495 default: 7496 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7497 } 7498 mutex_exit(&vd->vdev_trim_lock); 7499 7500 return (0); 7501 } 7502 7503 /* 7504 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 7505 * TRIM threads for each child vdev. These threads pass over all of the free 7506 * space in the vdev's metaslabs and issues TRIM commands for that space. 7507 */ 7508 int 7509 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 7510 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 7511 { 7512 int total_errors = 0; 7513 list_t vd_list; 7514 7515 list_create(&vd_list, sizeof (vdev_t), 7516 offsetof(vdev_t, vdev_trim_node)); 7517 7518 /* 7519 * We hold the namespace lock through the whole function 7520 * to prevent any changes to the pool while we're starting or 7521 * stopping TRIM. The config and state locks are held so that 7522 * we can properly assess the vdev state before we commit to 7523 * the TRIM operation. 7524 */ 7525 mutex_enter(&spa_namespace_lock); 7526 7527 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7528 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7529 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7530 7531 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 7532 rate, partial, secure, &vd_list); 7533 if (error != 0) { 7534 char guid_as_str[MAXNAMELEN]; 7535 7536 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7537 "%llu", (unsigned long long)vdev_guid); 7538 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7539 total_errors++; 7540 } 7541 } 7542 7543 /* Wait for all TRIM threads to stop. */ 7544 vdev_trim_stop_wait(spa, &vd_list); 7545 7546 /* Sync out the TRIM state */ 7547 txg_wait_synced(spa->spa_dsl_pool, 0); 7548 mutex_exit(&spa_namespace_lock); 7549 7550 list_destroy(&vd_list); 7551 7552 return (total_errors); 7553 } 7554 7555 /* 7556 * Split a set of devices from their mirrors, and create a new pool from them. 7557 */ 7558 int 7559 spa_vdev_split_mirror(spa_t *spa, const char *newname, nvlist_t *config, 7560 nvlist_t *props, boolean_t exp) 7561 { 7562 int error = 0; 7563 uint64_t txg, *glist; 7564 spa_t *newspa; 7565 uint_t c, children, lastlog; 7566 nvlist_t **child, *nvl, *tmp; 7567 dmu_tx_t *tx; 7568 const char *altroot = NULL; 7569 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 7570 boolean_t activate_slog; 7571 7572 ASSERT(spa_writeable(spa)); 7573 7574 txg = spa_vdev_enter(spa); 7575 7576 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7577 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7578 error = (spa_has_checkpoint(spa)) ? 7579 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7580 return (spa_vdev_exit(spa, NULL, txg, error)); 7581 } 7582 7583 /* clear the log and flush everything up to now */ 7584 activate_slog = spa_passivate_log(spa); 7585 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7586 error = spa_reset_logs(spa); 7587 txg = spa_vdev_config_enter(spa); 7588 7589 if (activate_slog) 7590 spa_activate_log(spa); 7591 7592 if (error != 0) 7593 return (spa_vdev_exit(spa, NULL, txg, error)); 7594 7595 /* check new spa name before going any further */ 7596 if (spa_lookup(newname) != NULL) 7597 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 7598 7599 /* 7600 * scan through all the children to ensure they're all mirrors 7601 */ 7602 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 7603 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 7604 &children) != 0) 7605 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7606 7607 /* first, check to ensure we've got the right child count */ 7608 rvd = spa->spa_root_vdev; 7609 lastlog = 0; 7610 for (c = 0; c < rvd->vdev_children; c++) { 7611 vdev_t *vd = rvd->vdev_child[c]; 7612 7613 /* don't count the holes & logs as children */ 7614 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 7615 !vdev_is_concrete(vd))) { 7616 if (lastlog == 0) 7617 lastlog = c; 7618 continue; 7619 } 7620 7621 lastlog = 0; 7622 } 7623 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 7624 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7625 7626 /* next, ensure no spare or cache devices are part of the split */ 7627 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 7628 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 7629 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7630 7631 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 7632 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 7633 7634 /* then, loop over each vdev and validate it */ 7635 for (c = 0; c < children; c++) { 7636 uint64_t is_hole = 0; 7637 7638 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 7639 &is_hole); 7640 7641 if (is_hole != 0) { 7642 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 7643 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 7644 continue; 7645 } else { 7646 error = SET_ERROR(EINVAL); 7647 break; 7648 } 7649 } 7650 7651 /* deal with indirect vdevs */ 7652 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 7653 &vdev_indirect_ops) 7654 continue; 7655 7656 /* which disk is going to be split? */ 7657 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 7658 &glist[c]) != 0) { 7659 error = SET_ERROR(EINVAL); 7660 break; 7661 } 7662 7663 /* look it up in the spa */ 7664 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 7665 if (vml[c] == NULL) { 7666 error = SET_ERROR(ENODEV); 7667 break; 7668 } 7669 7670 /* make sure there's nothing stopping the split */ 7671 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 7672 vml[c]->vdev_islog || 7673 !vdev_is_concrete(vml[c]) || 7674 vml[c]->vdev_isspare || 7675 vml[c]->vdev_isl2cache || 7676 !vdev_writeable(vml[c]) || 7677 vml[c]->vdev_children != 0 || 7678 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 7679 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 7680 error = SET_ERROR(EINVAL); 7681 break; 7682 } 7683 7684 if (vdev_dtl_required(vml[c]) || 7685 vdev_resilver_needed(vml[c], NULL, NULL)) { 7686 error = SET_ERROR(EBUSY); 7687 break; 7688 } 7689 7690 /* we need certain info from the top level */ 7691 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 7692 vml[c]->vdev_top->vdev_ms_array); 7693 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 7694 vml[c]->vdev_top->vdev_ms_shift); 7695 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 7696 vml[c]->vdev_top->vdev_asize); 7697 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 7698 vml[c]->vdev_top->vdev_ashift); 7699 7700 /* transfer per-vdev ZAPs */ 7701 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 7702 VERIFY0(nvlist_add_uint64(child[c], 7703 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 7704 7705 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 7706 VERIFY0(nvlist_add_uint64(child[c], 7707 ZPOOL_CONFIG_VDEV_TOP_ZAP, 7708 vml[c]->vdev_parent->vdev_top_zap)); 7709 } 7710 7711 if (error != 0) { 7712 kmem_free(vml, children * sizeof (vdev_t *)); 7713 kmem_free(glist, children * sizeof (uint64_t)); 7714 return (spa_vdev_exit(spa, NULL, txg, error)); 7715 } 7716 7717 /* stop writers from using the disks */ 7718 for (c = 0; c < children; c++) { 7719 if (vml[c] != NULL) 7720 vml[c]->vdev_offline = B_TRUE; 7721 } 7722 vdev_reopen(spa->spa_root_vdev); 7723 7724 /* 7725 * Temporarily record the splitting vdevs in the spa config. This 7726 * will disappear once the config is regenerated. 7727 */ 7728 nvl = fnvlist_alloc(); 7729 fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children); 7730 kmem_free(glist, children * sizeof (uint64_t)); 7731 7732 mutex_enter(&spa->spa_props_lock); 7733 fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl); 7734 mutex_exit(&spa->spa_props_lock); 7735 spa->spa_config_splitting = nvl; 7736 vdev_config_dirty(spa->spa_root_vdev); 7737 7738 /* configure and create the new pool */ 7739 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname); 7740 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 7741 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE); 7742 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)); 7743 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg); 7744 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 7745 spa_generate_guid(NULL)); 7746 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 7747 (void) nvlist_lookup_string(props, 7748 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 7749 7750 /* add the new pool to the namespace */ 7751 newspa = spa_add(newname, config, altroot); 7752 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 7753 newspa->spa_config_txg = spa->spa_config_txg; 7754 spa_set_log_state(newspa, SPA_LOG_CLEAR); 7755 7756 /* release the spa config lock, retaining the namespace lock */ 7757 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7758 7759 if (zio_injection_enabled) 7760 zio_handle_panic_injection(spa, FTAG, 1); 7761 7762 spa_activate(newspa, spa_mode_global); 7763 spa_async_suspend(newspa); 7764 7765 /* 7766 * Temporarily stop the initializing and TRIM activity. We set the 7767 * state to ACTIVE so that we know to resume initializing or TRIM 7768 * once the split has completed. 7769 */ 7770 list_t vd_initialize_list; 7771 list_create(&vd_initialize_list, sizeof (vdev_t), 7772 offsetof(vdev_t, vdev_initialize_node)); 7773 7774 list_t vd_trim_list; 7775 list_create(&vd_trim_list, sizeof (vdev_t), 7776 offsetof(vdev_t, vdev_trim_node)); 7777 7778 for (c = 0; c < children; c++) { 7779 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7780 mutex_enter(&vml[c]->vdev_initialize_lock); 7781 vdev_initialize_stop(vml[c], 7782 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 7783 mutex_exit(&vml[c]->vdev_initialize_lock); 7784 7785 mutex_enter(&vml[c]->vdev_trim_lock); 7786 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 7787 mutex_exit(&vml[c]->vdev_trim_lock); 7788 } 7789 } 7790 7791 vdev_initialize_stop_wait(spa, &vd_initialize_list); 7792 vdev_trim_stop_wait(spa, &vd_trim_list); 7793 7794 list_destroy(&vd_initialize_list); 7795 list_destroy(&vd_trim_list); 7796 7797 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 7798 newspa->spa_is_splitting = B_TRUE; 7799 7800 /* create the new pool from the disks of the original pool */ 7801 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 7802 if (error) 7803 goto out; 7804 7805 /* if that worked, generate a real config for the new pool */ 7806 if (newspa->spa_root_vdev != NULL) { 7807 newspa->spa_config_splitting = fnvlist_alloc(); 7808 fnvlist_add_uint64(newspa->spa_config_splitting, 7809 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)); 7810 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 7811 B_TRUE)); 7812 } 7813 7814 /* set the props */ 7815 if (props != NULL) { 7816 spa_configfile_set(newspa, props, B_FALSE); 7817 error = spa_prop_set(newspa, props); 7818 if (error) 7819 goto out; 7820 } 7821 7822 /* flush everything */ 7823 txg = spa_vdev_config_enter(newspa); 7824 vdev_config_dirty(newspa->spa_root_vdev); 7825 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 7826 7827 if (zio_injection_enabled) 7828 zio_handle_panic_injection(spa, FTAG, 2); 7829 7830 spa_async_resume(newspa); 7831 7832 /* finally, update the original pool's config */ 7833 txg = spa_vdev_config_enter(spa); 7834 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 7835 error = dmu_tx_assign(tx, TXG_WAIT); 7836 if (error != 0) 7837 dmu_tx_abort(tx); 7838 for (c = 0; c < children; c++) { 7839 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7840 vdev_t *tvd = vml[c]->vdev_top; 7841 7842 /* 7843 * Need to be sure the detachable VDEV is not 7844 * on any *other* txg's DTL list to prevent it 7845 * from being accessed after it's freed. 7846 */ 7847 for (int t = 0; t < TXG_SIZE; t++) { 7848 (void) txg_list_remove_this( 7849 &tvd->vdev_dtl_list, vml[c], t); 7850 } 7851 7852 vdev_split(vml[c]); 7853 if (error == 0) 7854 spa_history_log_internal(spa, "detach", tx, 7855 "vdev=%s", vml[c]->vdev_path); 7856 7857 vdev_free(vml[c]); 7858 } 7859 } 7860 spa->spa_avz_action = AVZ_ACTION_REBUILD; 7861 vdev_config_dirty(spa->spa_root_vdev); 7862 spa->spa_config_splitting = NULL; 7863 nvlist_free(nvl); 7864 if (error == 0) 7865 dmu_tx_commit(tx); 7866 (void) spa_vdev_exit(spa, NULL, txg, 0); 7867 7868 if (zio_injection_enabled) 7869 zio_handle_panic_injection(spa, FTAG, 3); 7870 7871 /* split is complete; log a history record */ 7872 spa_history_log_internal(newspa, "split", NULL, 7873 "from pool %s", spa_name(spa)); 7874 7875 newspa->spa_is_splitting = B_FALSE; 7876 kmem_free(vml, children * sizeof (vdev_t *)); 7877 7878 /* if we're not going to mount the filesystems in userland, export */ 7879 if (exp) 7880 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 7881 B_FALSE, B_FALSE); 7882 7883 return (error); 7884 7885 out: 7886 spa_unload(newspa); 7887 spa_deactivate(newspa); 7888 spa_remove(newspa); 7889 7890 txg = spa_vdev_config_enter(spa); 7891 7892 /* re-online all offlined disks */ 7893 for (c = 0; c < children; c++) { 7894 if (vml[c] != NULL) 7895 vml[c]->vdev_offline = B_FALSE; 7896 } 7897 7898 /* restart initializing or trimming disks as necessary */ 7899 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 7900 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 7901 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 7902 7903 vdev_reopen(spa->spa_root_vdev); 7904 7905 nvlist_free(spa->spa_config_splitting); 7906 spa->spa_config_splitting = NULL; 7907 (void) spa_vdev_exit(spa, NULL, txg, error); 7908 7909 kmem_free(vml, children * sizeof (vdev_t *)); 7910 return (error); 7911 } 7912 7913 /* 7914 * Find any device that's done replacing, or a vdev marked 'unspare' that's 7915 * currently spared, so we can detach it. 7916 */ 7917 static vdev_t * 7918 spa_vdev_resilver_done_hunt(vdev_t *vd) 7919 { 7920 vdev_t *newvd, *oldvd; 7921 7922 for (int c = 0; c < vd->vdev_children; c++) { 7923 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 7924 if (oldvd != NULL) 7925 return (oldvd); 7926 } 7927 7928 /* 7929 * Check for a completed replacement. We always consider the first 7930 * vdev in the list to be the oldest vdev, and the last one to be 7931 * the newest (see spa_vdev_attach() for how that works). In 7932 * the case where the newest vdev is faulted, we will not automatically 7933 * remove it after a resilver completes. This is OK as it will require 7934 * user intervention to determine which disk the admin wishes to keep. 7935 */ 7936 if (vd->vdev_ops == &vdev_replacing_ops) { 7937 ASSERT(vd->vdev_children > 1); 7938 7939 newvd = vd->vdev_child[vd->vdev_children - 1]; 7940 oldvd = vd->vdev_child[0]; 7941 7942 if (vdev_dtl_empty(newvd, DTL_MISSING) && 7943 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7944 !vdev_dtl_required(oldvd)) 7945 return (oldvd); 7946 } 7947 7948 /* 7949 * Check for a completed resilver with the 'unspare' flag set. 7950 * Also potentially update faulted state. 7951 */ 7952 if (vd->vdev_ops == &vdev_spare_ops) { 7953 vdev_t *first = vd->vdev_child[0]; 7954 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 7955 7956 if (last->vdev_unspare) { 7957 oldvd = first; 7958 newvd = last; 7959 } else if (first->vdev_unspare) { 7960 oldvd = last; 7961 newvd = first; 7962 } else { 7963 oldvd = NULL; 7964 } 7965 7966 if (oldvd != NULL && 7967 vdev_dtl_empty(newvd, DTL_MISSING) && 7968 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7969 !vdev_dtl_required(oldvd)) 7970 return (oldvd); 7971 7972 vdev_propagate_state(vd); 7973 7974 /* 7975 * If there are more than two spares attached to a disk, 7976 * and those spares are not required, then we want to 7977 * attempt to free them up now so that they can be used 7978 * by other pools. Once we're back down to a single 7979 * disk+spare, we stop removing them. 7980 */ 7981 if (vd->vdev_children > 2) { 7982 newvd = vd->vdev_child[1]; 7983 7984 if (newvd->vdev_isspare && last->vdev_isspare && 7985 vdev_dtl_empty(last, DTL_MISSING) && 7986 vdev_dtl_empty(last, DTL_OUTAGE) && 7987 !vdev_dtl_required(newvd)) 7988 return (newvd); 7989 } 7990 } 7991 7992 return (NULL); 7993 } 7994 7995 static void 7996 spa_vdev_resilver_done(spa_t *spa) 7997 { 7998 vdev_t *vd, *pvd, *ppvd; 7999 uint64_t guid, sguid, pguid, ppguid; 8000 8001 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 8002 8003 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 8004 pvd = vd->vdev_parent; 8005 ppvd = pvd->vdev_parent; 8006 guid = vd->vdev_guid; 8007 pguid = pvd->vdev_guid; 8008 ppguid = ppvd->vdev_guid; 8009 sguid = 0; 8010 /* 8011 * If we have just finished replacing a hot spared device, then 8012 * we need to detach the parent's first child (the original hot 8013 * spare) as well. 8014 */ 8015 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 8016 ppvd->vdev_children == 2) { 8017 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 8018 sguid = ppvd->vdev_child[1]->vdev_guid; 8019 } 8020 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 8021 8022 spa_config_exit(spa, SCL_ALL, FTAG); 8023 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 8024 return; 8025 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 8026 return; 8027 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 8028 } 8029 8030 spa_config_exit(spa, SCL_ALL, FTAG); 8031 8032 /* 8033 * If a detach was not performed above replace waiters will not have 8034 * been notified. In which case we must do so now. 8035 */ 8036 spa_notify_waiters(spa); 8037 } 8038 8039 /* 8040 * Update the stored path or FRU for this vdev. 8041 */ 8042 static int 8043 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 8044 boolean_t ispath) 8045 { 8046 vdev_t *vd; 8047 boolean_t sync = B_FALSE; 8048 8049 ASSERT(spa_writeable(spa)); 8050 8051 spa_vdev_state_enter(spa, SCL_ALL); 8052 8053 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 8054 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 8055 8056 if (!vd->vdev_ops->vdev_op_leaf) 8057 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 8058 8059 if (ispath) { 8060 if (strcmp(value, vd->vdev_path) != 0) { 8061 spa_strfree(vd->vdev_path); 8062 vd->vdev_path = spa_strdup(value); 8063 sync = B_TRUE; 8064 } 8065 } else { 8066 if (vd->vdev_fru == NULL) { 8067 vd->vdev_fru = spa_strdup(value); 8068 sync = B_TRUE; 8069 } else if (strcmp(value, vd->vdev_fru) != 0) { 8070 spa_strfree(vd->vdev_fru); 8071 vd->vdev_fru = spa_strdup(value); 8072 sync = B_TRUE; 8073 } 8074 } 8075 8076 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 8077 } 8078 8079 int 8080 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 8081 { 8082 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 8083 } 8084 8085 int 8086 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 8087 { 8088 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 8089 } 8090 8091 /* 8092 * ========================================================================== 8093 * SPA Scanning 8094 * ========================================================================== 8095 */ 8096 int 8097 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 8098 { 8099 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8100 8101 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8102 return (SET_ERROR(EBUSY)); 8103 8104 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 8105 } 8106 8107 int 8108 spa_scan_stop(spa_t *spa) 8109 { 8110 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8111 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8112 return (SET_ERROR(EBUSY)); 8113 return (dsl_scan_cancel(spa->spa_dsl_pool)); 8114 } 8115 8116 int 8117 spa_scan(spa_t *spa, pool_scan_func_t func) 8118 { 8119 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8120 8121 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 8122 return (SET_ERROR(ENOTSUP)); 8123 8124 if (func == POOL_SCAN_RESILVER && 8125 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 8126 return (SET_ERROR(ENOTSUP)); 8127 8128 /* 8129 * If a resilver was requested, but there is no DTL on a 8130 * writeable leaf device, we have nothing to do. 8131 */ 8132 if (func == POOL_SCAN_RESILVER && 8133 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 8134 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 8135 return (0); 8136 } 8137 8138 return (dsl_scan(spa->spa_dsl_pool, func)); 8139 } 8140 8141 /* 8142 * ========================================================================== 8143 * SPA async task processing 8144 * ========================================================================== 8145 */ 8146 8147 static void 8148 spa_async_remove(spa_t *spa, vdev_t *vd) 8149 { 8150 if (vd->vdev_remove_wanted) { 8151 vd->vdev_remove_wanted = B_FALSE; 8152 vd->vdev_delayed_close = B_FALSE; 8153 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 8154 8155 /* 8156 * We want to clear the stats, but we don't want to do a full 8157 * vdev_clear() as that will cause us to throw away 8158 * degraded/faulted state as well as attempt to reopen the 8159 * device, all of which is a waste. 8160 */ 8161 vd->vdev_stat.vs_read_errors = 0; 8162 vd->vdev_stat.vs_write_errors = 0; 8163 vd->vdev_stat.vs_checksum_errors = 0; 8164 8165 vdev_state_dirty(vd->vdev_top); 8166 8167 /* Tell userspace that the vdev is gone. */ 8168 zfs_post_remove(spa, vd); 8169 } 8170 8171 for (int c = 0; c < vd->vdev_children; c++) 8172 spa_async_remove(spa, vd->vdev_child[c]); 8173 } 8174 8175 static void 8176 spa_async_probe(spa_t *spa, vdev_t *vd) 8177 { 8178 if (vd->vdev_probe_wanted) { 8179 vd->vdev_probe_wanted = B_FALSE; 8180 vdev_reopen(vd); /* vdev_open() does the actual probe */ 8181 } 8182 8183 for (int c = 0; c < vd->vdev_children; c++) 8184 spa_async_probe(spa, vd->vdev_child[c]); 8185 } 8186 8187 static void 8188 spa_async_autoexpand(spa_t *spa, vdev_t *vd) 8189 { 8190 if (!spa->spa_autoexpand) 8191 return; 8192 8193 for (int c = 0; c < vd->vdev_children; c++) { 8194 vdev_t *cvd = vd->vdev_child[c]; 8195 spa_async_autoexpand(spa, cvd); 8196 } 8197 8198 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 8199 return; 8200 8201 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 8202 } 8203 8204 static __attribute__((noreturn)) void 8205 spa_async_thread(void *arg) 8206 { 8207 spa_t *spa = (spa_t *)arg; 8208 dsl_pool_t *dp = spa->spa_dsl_pool; 8209 int tasks; 8210 8211 ASSERT(spa->spa_sync_on); 8212 8213 mutex_enter(&spa->spa_async_lock); 8214 tasks = spa->spa_async_tasks; 8215 spa->spa_async_tasks = 0; 8216 mutex_exit(&spa->spa_async_lock); 8217 8218 /* 8219 * See if the config needs to be updated. 8220 */ 8221 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 8222 uint64_t old_space, new_space; 8223 8224 mutex_enter(&spa_namespace_lock); 8225 old_space = metaslab_class_get_space(spa_normal_class(spa)); 8226 old_space += metaslab_class_get_space(spa_special_class(spa)); 8227 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 8228 old_space += metaslab_class_get_space( 8229 spa_embedded_log_class(spa)); 8230 8231 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 8232 8233 new_space = metaslab_class_get_space(spa_normal_class(spa)); 8234 new_space += metaslab_class_get_space(spa_special_class(spa)); 8235 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 8236 new_space += metaslab_class_get_space( 8237 spa_embedded_log_class(spa)); 8238 mutex_exit(&spa_namespace_lock); 8239 8240 /* 8241 * If the pool grew as a result of the config update, 8242 * then log an internal history event. 8243 */ 8244 if (new_space != old_space) { 8245 spa_history_log_internal(spa, "vdev online", NULL, 8246 "pool '%s' size: %llu(+%llu)", 8247 spa_name(spa), (u_longlong_t)new_space, 8248 (u_longlong_t)(new_space - old_space)); 8249 } 8250 } 8251 8252 /* 8253 * See if any devices need to be marked REMOVED. 8254 */ 8255 if (tasks & SPA_ASYNC_REMOVE) { 8256 spa_vdev_state_enter(spa, SCL_NONE); 8257 spa_async_remove(spa, spa->spa_root_vdev); 8258 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 8259 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); 8260 for (int i = 0; i < spa->spa_spares.sav_count; i++) 8261 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); 8262 (void) spa_vdev_state_exit(spa, NULL, 0); 8263 } 8264 8265 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 8266 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8267 spa_async_autoexpand(spa, spa->spa_root_vdev); 8268 spa_config_exit(spa, SCL_CONFIG, FTAG); 8269 } 8270 8271 /* 8272 * See if any devices need to be probed. 8273 */ 8274 if (tasks & SPA_ASYNC_PROBE) { 8275 spa_vdev_state_enter(spa, SCL_NONE); 8276 spa_async_probe(spa, spa->spa_root_vdev); 8277 (void) spa_vdev_state_exit(spa, NULL, 0); 8278 } 8279 8280 /* 8281 * If any devices are done replacing, detach them. 8282 */ 8283 if (tasks & SPA_ASYNC_RESILVER_DONE || 8284 tasks & SPA_ASYNC_REBUILD_DONE) { 8285 spa_vdev_resilver_done(spa); 8286 } 8287 8288 /* 8289 * Kick off a resilver. 8290 */ 8291 if (tasks & SPA_ASYNC_RESILVER && 8292 !vdev_rebuild_active(spa->spa_root_vdev) && 8293 (!dsl_scan_resilvering(dp) || 8294 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 8295 dsl_scan_restart_resilver(dp, 0); 8296 8297 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 8298 mutex_enter(&spa_namespace_lock); 8299 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8300 vdev_initialize_restart(spa->spa_root_vdev); 8301 spa_config_exit(spa, SCL_CONFIG, FTAG); 8302 mutex_exit(&spa_namespace_lock); 8303 } 8304 8305 if (tasks & SPA_ASYNC_TRIM_RESTART) { 8306 mutex_enter(&spa_namespace_lock); 8307 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8308 vdev_trim_restart(spa->spa_root_vdev); 8309 spa_config_exit(spa, SCL_CONFIG, FTAG); 8310 mutex_exit(&spa_namespace_lock); 8311 } 8312 8313 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 8314 mutex_enter(&spa_namespace_lock); 8315 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8316 vdev_autotrim_restart(spa); 8317 spa_config_exit(spa, SCL_CONFIG, FTAG); 8318 mutex_exit(&spa_namespace_lock); 8319 } 8320 8321 /* 8322 * Kick off L2 cache whole device TRIM. 8323 */ 8324 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 8325 mutex_enter(&spa_namespace_lock); 8326 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8327 vdev_trim_l2arc(spa); 8328 spa_config_exit(spa, SCL_CONFIG, FTAG); 8329 mutex_exit(&spa_namespace_lock); 8330 } 8331 8332 /* 8333 * Kick off L2 cache rebuilding. 8334 */ 8335 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 8336 mutex_enter(&spa_namespace_lock); 8337 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 8338 l2arc_spa_rebuild_start(spa); 8339 spa_config_exit(spa, SCL_L2ARC, FTAG); 8340 mutex_exit(&spa_namespace_lock); 8341 } 8342 8343 /* 8344 * Let the world know that we're done. 8345 */ 8346 mutex_enter(&spa->spa_async_lock); 8347 spa->spa_async_thread = NULL; 8348 cv_broadcast(&spa->spa_async_cv); 8349 mutex_exit(&spa->spa_async_lock); 8350 thread_exit(); 8351 } 8352 8353 void 8354 spa_async_suspend(spa_t *spa) 8355 { 8356 mutex_enter(&spa->spa_async_lock); 8357 spa->spa_async_suspended++; 8358 while (spa->spa_async_thread != NULL) 8359 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 8360 mutex_exit(&spa->spa_async_lock); 8361 8362 spa_vdev_remove_suspend(spa); 8363 8364 zthr_t *condense_thread = spa->spa_condense_zthr; 8365 if (condense_thread != NULL) 8366 zthr_cancel(condense_thread); 8367 8368 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8369 if (discard_thread != NULL) 8370 zthr_cancel(discard_thread); 8371 8372 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8373 if (ll_delete_thread != NULL) 8374 zthr_cancel(ll_delete_thread); 8375 8376 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8377 if (ll_condense_thread != NULL) 8378 zthr_cancel(ll_condense_thread); 8379 } 8380 8381 void 8382 spa_async_resume(spa_t *spa) 8383 { 8384 mutex_enter(&spa->spa_async_lock); 8385 ASSERT(spa->spa_async_suspended != 0); 8386 spa->spa_async_suspended--; 8387 mutex_exit(&spa->spa_async_lock); 8388 spa_restart_removal(spa); 8389 8390 zthr_t *condense_thread = spa->spa_condense_zthr; 8391 if (condense_thread != NULL) 8392 zthr_resume(condense_thread); 8393 8394 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8395 if (discard_thread != NULL) 8396 zthr_resume(discard_thread); 8397 8398 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8399 if (ll_delete_thread != NULL) 8400 zthr_resume(ll_delete_thread); 8401 8402 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8403 if (ll_condense_thread != NULL) 8404 zthr_resume(ll_condense_thread); 8405 } 8406 8407 static boolean_t 8408 spa_async_tasks_pending(spa_t *spa) 8409 { 8410 uint_t non_config_tasks; 8411 uint_t config_task; 8412 boolean_t config_task_suspended; 8413 8414 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 8415 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 8416 if (spa->spa_ccw_fail_time == 0) { 8417 config_task_suspended = B_FALSE; 8418 } else { 8419 config_task_suspended = 8420 (gethrtime() - spa->spa_ccw_fail_time) < 8421 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 8422 } 8423 8424 return (non_config_tasks || (config_task && !config_task_suspended)); 8425 } 8426 8427 static void 8428 spa_async_dispatch(spa_t *spa) 8429 { 8430 mutex_enter(&spa->spa_async_lock); 8431 if (spa_async_tasks_pending(spa) && 8432 !spa->spa_async_suspended && 8433 spa->spa_async_thread == NULL) 8434 spa->spa_async_thread = thread_create(NULL, 0, 8435 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 8436 mutex_exit(&spa->spa_async_lock); 8437 } 8438 8439 void 8440 spa_async_request(spa_t *spa, int task) 8441 { 8442 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); 8443 mutex_enter(&spa->spa_async_lock); 8444 spa->spa_async_tasks |= task; 8445 mutex_exit(&spa->spa_async_lock); 8446 } 8447 8448 int 8449 spa_async_tasks(spa_t *spa) 8450 { 8451 return (spa->spa_async_tasks); 8452 } 8453 8454 /* 8455 * ========================================================================== 8456 * SPA syncing routines 8457 * ========================================================================== 8458 */ 8459 8460 8461 static int 8462 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8463 dmu_tx_t *tx) 8464 { 8465 bpobj_t *bpo = arg; 8466 bpobj_enqueue(bpo, bp, bp_freed, tx); 8467 return (0); 8468 } 8469 8470 int 8471 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8472 { 8473 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 8474 } 8475 8476 int 8477 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8478 { 8479 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 8480 } 8481 8482 static int 8483 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8484 { 8485 zio_t *pio = arg; 8486 8487 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 8488 pio->io_flags)); 8489 return (0); 8490 } 8491 8492 static int 8493 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8494 dmu_tx_t *tx) 8495 { 8496 ASSERT(!bp_freed); 8497 return (spa_free_sync_cb(arg, bp, tx)); 8498 } 8499 8500 /* 8501 * Note: this simple function is not inlined to make it easier to dtrace the 8502 * amount of time spent syncing frees. 8503 */ 8504 static void 8505 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 8506 { 8507 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8508 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 8509 VERIFY(zio_wait(zio) == 0); 8510 } 8511 8512 /* 8513 * Note: this simple function is not inlined to make it easier to dtrace the 8514 * amount of time spent syncing deferred frees. 8515 */ 8516 static void 8517 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 8518 { 8519 if (spa_sync_pass(spa) != 1) 8520 return; 8521 8522 /* 8523 * Note: 8524 * If the log space map feature is active, we stop deferring 8525 * frees to the next TXG and therefore running this function 8526 * would be considered a no-op as spa_deferred_bpobj should 8527 * not have any entries. 8528 * 8529 * That said we run this function anyway (instead of returning 8530 * immediately) for the edge-case scenario where we just 8531 * activated the log space map feature in this TXG but we have 8532 * deferred frees from the previous TXG. 8533 */ 8534 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8535 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 8536 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 8537 VERIFY0(zio_wait(zio)); 8538 } 8539 8540 static void 8541 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 8542 { 8543 char *packed = NULL; 8544 size_t bufsize; 8545 size_t nvsize = 0; 8546 dmu_buf_t *db; 8547 8548 VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); 8549 8550 /* 8551 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 8552 * information. This avoids the dmu_buf_will_dirty() path and 8553 * saves us a pre-read to get data we don't actually care about. 8554 */ 8555 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 8556 packed = vmem_alloc(bufsize, KM_SLEEP); 8557 8558 VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 8559 KM_SLEEP) == 0); 8560 memset(packed + nvsize, 0, bufsize - nvsize); 8561 8562 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); 8563 8564 vmem_free(packed, bufsize); 8565 8566 VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 8567 dmu_buf_will_dirty(db, tx); 8568 *(uint64_t *)db->db_data = nvsize; 8569 dmu_buf_rele(db, FTAG); 8570 } 8571 8572 static void 8573 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 8574 const char *config, const char *entry) 8575 { 8576 nvlist_t *nvroot; 8577 nvlist_t **list; 8578 int i; 8579 8580 if (!sav->sav_sync) 8581 return; 8582 8583 /* 8584 * Update the MOS nvlist describing the list of available devices. 8585 * spa_validate_aux() will have already made sure this nvlist is 8586 * valid and the vdevs are labeled appropriately. 8587 */ 8588 if (sav->sav_object == 0) { 8589 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 8590 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 8591 sizeof (uint64_t), tx); 8592 VERIFY(zap_update(spa->spa_meta_objset, 8593 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 8594 &sav->sav_object, tx) == 0); 8595 } 8596 8597 nvroot = fnvlist_alloc(); 8598 if (sav->sav_count == 0) { 8599 fnvlist_add_nvlist_array(nvroot, config, 8600 (const nvlist_t * const *)NULL, 0); 8601 } else { 8602 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 8603 for (i = 0; i < sav->sav_count; i++) 8604 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 8605 B_FALSE, VDEV_CONFIG_L2CACHE); 8606 fnvlist_add_nvlist_array(nvroot, config, 8607 (const nvlist_t * const *)list, sav->sav_count); 8608 for (i = 0; i < sav->sav_count; i++) 8609 nvlist_free(list[i]); 8610 kmem_free(list, sav->sav_count * sizeof (void *)); 8611 } 8612 8613 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 8614 nvlist_free(nvroot); 8615 8616 sav->sav_sync = B_FALSE; 8617 } 8618 8619 /* 8620 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 8621 * The all-vdev ZAP must be empty. 8622 */ 8623 static void 8624 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 8625 { 8626 spa_t *spa = vd->vdev_spa; 8627 8628 if (vd->vdev_top_zap != 0) { 8629 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8630 vd->vdev_top_zap, tx)); 8631 } 8632 if (vd->vdev_leaf_zap != 0) { 8633 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8634 vd->vdev_leaf_zap, tx)); 8635 } 8636 for (uint64_t i = 0; i < vd->vdev_children; i++) { 8637 spa_avz_build(vd->vdev_child[i], avz, tx); 8638 } 8639 } 8640 8641 static void 8642 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 8643 { 8644 nvlist_t *config; 8645 8646 /* 8647 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 8648 * its config may not be dirty but we still need to build per-vdev ZAPs. 8649 * Similarly, if the pool is being assembled (e.g. after a split), we 8650 * need to rebuild the AVZ although the config may not be dirty. 8651 */ 8652 if (list_is_empty(&spa->spa_config_dirty_list) && 8653 spa->spa_avz_action == AVZ_ACTION_NONE) 8654 return; 8655 8656 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 8657 8658 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 8659 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 8660 spa->spa_all_vdev_zaps != 0); 8661 8662 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 8663 /* Make and build the new AVZ */ 8664 uint64_t new_avz = zap_create(spa->spa_meta_objset, 8665 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 8666 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 8667 8668 /* Diff old AVZ with new one */ 8669 zap_cursor_t zc; 8670 zap_attribute_t za; 8671 8672 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8673 spa->spa_all_vdev_zaps); 8674 zap_cursor_retrieve(&zc, &za) == 0; 8675 zap_cursor_advance(&zc)) { 8676 uint64_t vdzap = za.za_first_integer; 8677 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 8678 vdzap) == ENOENT) { 8679 /* 8680 * ZAP is listed in old AVZ but not in new one; 8681 * destroy it 8682 */ 8683 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 8684 tx)); 8685 } 8686 } 8687 8688 zap_cursor_fini(&zc); 8689 8690 /* Destroy the old AVZ */ 8691 VERIFY0(zap_destroy(spa->spa_meta_objset, 8692 spa->spa_all_vdev_zaps, tx)); 8693 8694 /* Replace the old AVZ in the dir obj with the new one */ 8695 VERIFY0(zap_update(spa->spa_meta_objset, 8696 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 8697 sizeof (new_avz), 1, &new_avz, tx)); 8698 8699 spa->spa_all_vdev_zaps = new_avz; 8700 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 8701 zap_cursor_t zc; 8702 zap_attribute_t za; 8703 8704 /* Walk through the AVZ and destroy all listed ZAPs */ 8705 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8706 spa->spa_all_vdev_zaps); 8707 zap_cursor_retrieve(&zc, &za) == 0; 8708 zap_cursor_advance(&zc)) { 8709 uint64_t zap = za.za_first_integer; 8710 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 8711 } 8712 8713 zap_cursor_fini(&zc); 8714 8715 /* Destroy and unlink the AVZ itself */ 8716 VERIFY0(zap_destroy(spa->spa_meta_objset, 8717 spa->spa_all_vdev_zaps, tx)); 8718 VERIFY0(zap_remove(spa->spa_meta_objset, 8719 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 8720 spa->spa_all_vdev_zaps = 0; 8721 } 8722 8723 if (spa->spa_all_vdev_zaps == 0) { 8724 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 8725 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 8726 DMU_POOL_VDEV_ZAP_MAP, tx); 8727 } 8728 spa->spa_avz_action = AVZ_ACTION_NONE; 8729 8730 /* Create ZAPs for vdevs that don't have them. */ 8731 vdev_construct_zaps(spa->spa_root_vdev, tx); 8732 8733 config = spa_config_generate(spa, spa->spa_root_vdev, 8734 dmu_tx_get_txg(tx), B_FALSE); 8735 8736 /* 8737 * If we're upgrading the spa version then make sure that 8738 * the config object gets updated with the correct version. 8739 */ 8740 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 8741 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 8742 spa->spa_uberblock.ub_version); 8743 8744 spa_config_exit(spa, SCL_STATE, FTAG); 8745 8746 nvlist_free(spa->spa_config_syncing); 8747 spa->spa_config_syncing = config; 8748 8749 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 8750 } 8751 8752 static void 8753 spa_sync_version(void *arg, dmu_tx_t *tx) 8754 { 8755 uint64_t *versionp = arg; 8756 uint64_t version = *versionp; 8757 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8758 8759 /* 8760 * Setting the version is special cased when first creating the pool. 8761 */ 8762 ASSERT(tx->tx_txg != TXG_INITIAL); 8763 8764 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 8765 ASSERT(version >= spa_version(spa)); 8766 8767 spa->spa_uberblock.ub_version = version; 8768 vdev_config_dirty(spa->spa_root_vdev); 8769 spa_history_log_internal(spa, "set", tx, "version=%lld", 8770 (longlong_t)version); 8771 } 8772 8773 /* 8774 * Set zpool properties. 8775 */ 8776 static void 8777 spa_sync_props(void *arg, dmu_tx_t *tx) 8778 { 8779 nvlist_t *nvp = arg; 8780 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8781 objset_t *mos = spa->spa_meta_objset; 8782 nvpair_t *elem = NULL; 8783 8784 mutex_enter(&spa->spa_props_lock); 8785 8786 while ((elem = nvlist_next_nvpair(nvp, elem))) { 8787 uint64_t intval; 8788 const char *strval, *fname; 8789 zpool_prop_t prop; 8790 const char *propname; 8791 zprop_type_t proptype; 8792 spa_feature_t fid; 8793 8794 switch (prop = zpool_name_to_prop(nvpair_name(elem))) { 8795 case ZPOOL_PROP_INVAL: 8796 /* 8797 * We checked this earlier in spa_prop_validate(). 8798 */ 8799 ASSERT(zpool_prop_feature(nvpair_name(elem))); 8800 8801 fname = strchr(nvpair_name(elem), '@') + 1; 8802 VERIFY0(zfeature_lookup_name(fname, &fid)); 8803 8804 spa_feature_enable(spa, fid, tx); 8805 spa_history_log_internal(spa, "set", tx, 8806 "%s=enabled", nvpair_name(elem)); 8807 break; 8808 8809 case ZPOOL_PROP_VERSION: 8810 intval = fnvpair_value_uint64(elem); 8811 /* 8812 * The version is synced separately before other 8813 * properties and should be correct by now. 8814 */ 8815 ASSERT3U(spa_version(spa), >=, intval); 8816 break; 8817 8818 case ZPOOL_PROP_ALTROOT: 8819 /* 8820 * 'altroot' is a non-persistent property. It should 8821 * have been set temporarily at creation or import time. 8822 */ 8823 ASSERT(spa->spa_root != NULL); 8824 break; 8825 8826 case ZPOOL_PROP_READONLY: 8827 case ZPOOL_PROP_CACHEFILE: 8828 /* 8829 * 'readonly' and 'cachefile' are also non-persistent 8830 * properties. 8831 */ 8832 break; 8833 case ZPOOL_PROP_COMMENT: 8834 strval = fnvpair_value_string(elem); 8835 if (spa->spa_comment != NULL) 8836 spa_strfree(spa->spa_comment); 8837 spa->spa_comment = spa_strdup(strval); 8838 /* 8839 * We need to dirty the configuration on all the vdevs 8840 * so that their labels get updated. We also need to 8841 * update the cache file to keep it in sync with the 8842 * MOS version. It's unnecessary to do this for pool 8843 * creation since the vdev's configuration has already 8844 * been dirtied. 8845 */ 8846 if (tx->tx_txg != TXG_INITIAL) { 8847 vdev_config_dirty(spa->spa_root_vdev); 8848 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8849 } 8850 spa_history_log_internal(spa, "set", tx, 8851 "%s=%s", nvpair_name(elem), strval); 8852 break; 8853 case ZPOOL_PROP_COMPATIBILITY: 8854 strval = fnvpair_value_string(elem); 8855 if (spa->spa_compatibility != NULL) 8856 spa_strfree(spa->spa_compatibility); 8857 spa->spa_compatibility = spa_strdup(strval); 8858 /* 8859 * Dirty the configuration on vdevs as above. 8860 */ 8861 if (tx->tx_txg != TXG_INITIAL) { 8862 vdev_config_dirty(spa->spa_root_vdev); 8863 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8864 } 8865 8866 spa_history_log_internal(spa, "set", tx, 8867 "%s=%s", nvpair_name(elem), strval); 8868 break; 8869 8870 default: 8871 /* 8872 * Set pool property values in the poolprops mos object. 8873 */ 8874 if (spa->spa_pool_props_object == 0) { 8875 spa->spa_pool_props_object = 8876 zap_create_link(mos, DMU_OT_POOL_PROPS, 8877 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 8878 tx); 8879 } 8880 8881 /* normalize the property name */ 8882 propname = zpool_prop_to_name(prop); 8883 proptype = zpool_prop_get_type(prop); 8884 8885 if (nvpair_type(elem) == DATA_TYPE_STRING) { 8886 ASSERT(proptype == PROP_TYPE_STRING); 8887 strval = fnvpair_value_string(elem); 8888 VERIFY0(zap_update(mos, 8889 spa->spa_pool_props_object, propname, 8890 1, strlen(strval) + 1, strval, tx)); 8891 spa_history_log_internal(spa, "set", tx, 8892 "%s=%s", nvpair_name(elem), strval); 8893 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 8894 intval = fnvpair_value_uint64(elem); 8895 8896 if (proptype == PROP_TYPE_INDEX) { 8897 const char *unused; 8898 VERIFY0(zpool_prop_index_to_string( 8899 prop, intval, &unused)); 8900 } 8901 VERIFY0(zap_update(mos, 8902 spa->spa_pool_props_object, propname, 8903 8, 1, &intval, tx)); 8904 spa_history_log_internal(spa, "set", tx, 8905 "%s=%lld", nvpair_name(elem), 8906 (longlong_t)intval); 8907 8908 switch (prop) { 8909 case ZPOOL_PROP_DELEGATION: 8910 spa->spa_delegation = intval; 8911 break; 8912 case ZPOOL_PROP_BOOTFS: 8913 spa->spa_bootfs = intval; 8914 break; 8915 case ZPOOL_PROP_FAILUREMODE: 8916 spa->spa_failmode = intval; 8917 break; 8918 case ZPOOL_PROP_AUTOTRIM: 8919 spa->spa_autotrim = intval; 8920 spa_async_request(spa, 8921 SPA_ASYNC_AUTOTRIM_RESTART); 8922 break; 8923 case ZPOOL_PROP_AUTOEXPAND: 8924 spa->spa_autoexpand = intval; 8925 if (tx->tx_txg != TXG_INITIAL) 8926 spa_async_request(spa, 8927 SPA_ASYNC_AUTOEXPAND); 8928 break; 8929 case ZPOOL_PROP_MULTIHOST: 8930 spa->spa_multihost = intval; 8931 break; 8932 default: 8933 break; 8934 } 8935 } else { 8936 ASSERT(0); /* not allowed */ 8937 } 8938 } 8939 8940 } 8941 8942 mutex_exit(&spa->spa_props_lock); 8943 } 8944 8945 /* 8946 * Perform one-time upgrade on-disk changes. spa_version() does not 8947 * reflect the new version this txg, so there must be no changes this 8948 * txg to anything that the upgrade code depends on after it executes. 8949 * Therefore this must be called after dsl_pool_sync() does the sync 8950 * tasks. 8951 */ 8952 static void 8953 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 8954 { 8955 if (spa_sync_pass(spa) != 1) 8956 return; 8957 8958 dsl_pool_t *dp = spa->spa_dsl_pool; 8959 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 8960 8961 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && 8962 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { 8963 dsl_pool_create_origin(dp, tx); 8964 8965 /* Keeping the origin open increases spa_minref */ 8966 spa->spa_minref += 3; 8967 } 8968 8969 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && 8970 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { 8971 dsl_pool_upgrade_clones(dp, tx); 8972 } 8973 8974 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && 8975 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { 8976 dsl_pool_upgrade_dir_clones(dp, tx); 8977 8978 /* Keeping the freedir open increases spa_minref */ 8979 spa->spa_minref += 3; 8980 } 8981 8982 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && 8983 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8984 spa_feature_create_zap_objects(spa, tx); 8985 } 8986 8987 /* 8988 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 8989 * when possibility to use lz4 compression for metadata was added 8990 * Old pools that have this feature enabled must be upgraded to have 8991 * this feature active 8992 */ 8993 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8994 boolean_t lz4_en = spa_feature_is_enabled(spa, 8995 SPA_FEATURE_LZ4_COMPRESS); 8996 boolean_t lz4_ac = spa_feature_is_active(spa, 8997 SPA_FEATURE_LZ4_COMPRESS); 8998 8999 if (lz4_en && !lz4_ac) 9000 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 9001 } 9002 9003 /* 9004 * If we haven't written the salt, do so now. Note that the 9005 * feature may not be activated yet, but that's fine since 9006 * the presence of this ZAP entry is backwards compatible. 9007 */ 9008 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 9009 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 9010 VERIFY0(zap_add(spa->spa_meta_objset, 9011 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 9012 sizeof (spa->spa_cksum_salt.zcs_bytes), 9013 spa->spa_cksum_salt.zcs_bytes, tx)); 9014 } 9015 9016 rrw_exit(&dp->dp_config_rwlock, FTAG); 9017 } 9018 9019 static void 9020 vdev_indirect_state_sync_verify(vdev_t *vd) 9021 { 9022 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 9023 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 9024 9025 if (vd->vdev_ops == &vdev_indirect_ops) { 9026 ASSERT(vim != NULL); 9027 ASSERT(vib != NULL); 9028 } 9029 9030 uint64_t obsolete_sm_object = 0; 9031 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 9032 if (obsolete_sm_object != 0) { 9033 ASSERT(vd->vdev_obsolete_sm != NULL); 9034 ASSERT(vd->vdev_removing || 9035 vd->vdev_ops == &vdev_indirect_ops); 9036 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 9037 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 9038 ASSERT3U(obsolete_sm_object, ==, 9039 space_map_object(vd->vdev_obsolete_sm)); 9040 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 9041 space_map_allocated(vd->vdev_obsolete_sm)); 9042 } 9043 ASSERT(vd->vdev_obsolete_segments != NULL); 9044 9045 /* 9046 * Since frees / remaps to an indirect vdev can only 9047 * happen in syncing context, the obsolete segments 9048 * tree must be empty when we start syncing. 9049 */ 9050 ASSERT0(range_tree_space(vd->vdev_obsolete_segments)); 9051 } 9052 9053 /* 9054 * Set the top-level vdev's max queue depth. Evaluate each top-level's 9055 * async write queue depth in case it changed. The max queue depth will 9056 * not change in the middle of syncing out this txg. 9057 */ 9058 static void 9059 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 9060 { 9061 ASSERT(spa_writeable(spa)); 9062 9063 vdev_t *rvd = spa->spa_root_vdev; 9064 uint32_t max_queue_depth = zfs_vdev_async_write_max_active * 9065 zfs_vdev_queue_depth_pct / 100; 9066 metaslab_class_t *normal = spa_normal_class(spa); 9067 metaslab_class_t *special = spa_special_class(spa); 9068 metaslab_class_t *dedup = spa_dedup_class(spa); 9069 9070 uint64_t slots_per_allocator = 0; 9071 for (int c = 0; c < rvd->vdev_children; c++) { 9072 vdev_t *tvd = rvd->vdev_child[c]; 9073 9074 metaslab_group_t *mg = tvd->vdev_mg; 9075 if (mg == NULL || !metaslab_group_initialized(mg)) 9076 continue; 9077 9078 metaslab_class_t *mc = mg->mg_class; 9079 if (mc != normal && mc != special && mc != dedup) 9080 continue; 9081 9082 /* 9083 * It is safe to do a lock-free check here because only async 9084 * allocations look at mg_max_alloc_queue_depth, and async 9085 * allocations all happen from spa_sync(). 9086 */ 9087 for (int i = 0; i < mg->mg_allocators; i++) { 9088 ASSERT0(zfs_refcount_count( 9089 &(mg->mg_allocator[i].mga_alloc_queue_depth))); 9090 } 9091 mg->mg_max_alloc_queue_depth = max_queue_depth; 9092 9093 for (int i = 0; i < mg->mg_allocators; i++) { 9094 mg->mg_allocator[i].mga_cur_max_alloc_queue_depth = 9095 zfs_vdev_def_queue_depth; 9096 } 9097 slots_per_allocator += zfs_vdev_def_queue_depth; 9098 } 9099 9100 for (int i = 0; i < spa->spa_alloc_count; i++) { 9101 ASSERT0(zfs_refcount_count(&normal->mc_allocator[i]. 9102 mca_alloc_slots)); 9103 ASSERT0(zfs_refcount_count(&special->mc_allocator[i]. 9104 mca_alloc_slots)); 9105 ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i]. 9106 mca_alloc_slots)); 9107 normal->mc_allocator[i].mca_alloc_max_slots = 9108 slots_per_allocator; 9109 special->mc_allocator[i].mca_alloc_max_slots = 9110 slots_per_allocator; 9111 dedup->mc_allocator[i].mca_alloc_max_slots = 9112 slots_per_allocator; 9113 } 9114 normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9115 special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9116 dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9117 } 9118 9119 static void 9120 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 9121 { 9122 ASSERT(spa_writeable(spa)); 9123 9124 vdev_t *rvd = spa->spa_root_vdev; 9125 for (int c = 0; c < rvd->vdev_children; c++) { 9126 vdev_t *vd = rvd->vdev_child[c]; 9127 vdev_indirect_state_sync_verify(vd); 9128 9129 if (vdev_indirect_should_condense(vd)) { 9130 spa_condense_indirect_start_sync(vd, tx); 9131 break; 9132 } 9133 } 9134 } 9135 9136 static void 9137 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 9138 { 9139 objset_t *mos = spa->spa_meta_objset; 9140 dsl_pool_t *dp = spa->spa_dsl_pool; 9141 uint64_t txg = tx->tx_txg; 9142 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 9143 9144 do { 9145 int pass = ++spa->spa_sync_pass; 9146 9147 spa_sync_config_object(spa, tx); 9148 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 9149 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 9150 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 9151 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 9152 spa_errlog_sync(spa, txg); 9153 dsl_pool_sync(dp, txg); 9154 9155 if (pass < zfs_sync_pass_deferred_free || 9156 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 9157 /* 9158 * If the log space map feature is active we don't 9159 * care about deferred frees and the deferred bpobj 9160 * as the log space map should effectively have the 9161 * same results (i.e. appending only to one object). 9162 */ 9163 spa_sync_frees(spa, free_bpl, tx); 9164 } else { 9165 /* 9166 * We can not defer frees in pass 1, because 9167 * we sync the deferred frees later in pass 1. 9168 */ 9169 ASSERT3U(pass, >, 1); 9170 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 9171 &spa->spa_deferred_bpobj, tx); 9172 } 9173 9174 brt_sync(spa, txg); 9175 ddt_sync(spa, txg); 9176 dsl_scan_sync(dp, tx); 9177 svr_sync(spa, tx); 9178 spa_sync_upgrades(spa, tx); 9179 9180 spa_flush_metaslabs(spa, tx); 9181 9182 vdev_t *vd = NULL; 9183 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 9184 != NULL) 9185 vdev_sync(vd, txg); 9186 9187 /* 9188 * Note: We need to check if the MOS is dirty because we could 9189 * have marked the MOS dirty without updating the uberblock 9190 * (e.g. if we have sync tasks but no dirty user data). We need 9191 * to check the uberblock's rootbp because it is updated if we 9192 * have synced out dirty data (though in this case the MOS will 9193 * most likely also be dirty due to second order effects, we 9194 * don't want to rely on that here). 9195 */ 9196 if (pass == 1 && 9197 spa->spa_uberblock.ub_rootbp.blk_birth < txg && 9198 !dmu_objset_is_dirty(mos, txg)) { 9199 /* 9200 * Nothing changed on the first pass, therefore this 9201 * TXG is a no-op. Avoid syncing deferred frees, so 9202 * that we can keep this TXG as a no-op. 9203 */ 9204 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9205 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9206 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 9207 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 9208 break; 9209 } 9210 9211 spa_sync_deferred_frees(spa, tx); 9212 } while (dmu_objset_is_dirty(mos, txg)); 9213 } 9214 9215 /* 9216 * Rewrite the vdev configuration (which includes the uberblock) to 9217 * commit the transaction group. 9218 * 9219 * If there are no dirty vdevs, we sync the uberblock to a few random 9220 * top-level vdevs that are known to be visible in the config cache 9221 * (see spa_vdev_add() for a complete description). If there *are* dirty 9222 * vdevs, sync the uberblock to all vdevs. 9223 */ 9224 static void 9225 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 9226 { 9227 vdev_t *rvd = spa->spa_root_vdev; 9228 uint64_t txg = tx->tx_txg; 9229 9230 for (;;) { 9231 int error = 0; 9232 9233 /* 9234 * We hold SCL_STATE to prevent vdev open/close/etc. 9235 * while we're attempting to write the vdev labels. 9236 */ 9237 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9238 9239 if (list_is_empty(&spa->spa_config_dirty_list)) { 9240 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 9241 int svdcount = 0; 9242 int children = rvd->vdev_children; 9243 int c0 = random_in_range(children); 9244 9245 for (int c = 0; c < children; c++) { 9246 vdev_t *vd = 9247 rvd->vdev_child[(c0 + c) % children]; 9248 9249 /* Stop when revisiting the first vdev */ 9250 if (c > 0 && svd[0] == vd) 9251 break; 9252 9253 if (vd->vdev_ms_array == 0 || 9254 vd->vdev_islog || 9255 !vdev_is_concrete(vd)) 9256 continue; 9257 9258 svd[svdcount++] = vd; 9259 if (svdcount == SPA_SYNC_MIN_VDEVS) 9260 break; 9261 } 9262 error = vdev_config_sync(svd, svdcount, txg); 9263 } else { 9264 error = vdev_config_sync(rvd->vdev_child, 9265 rvd->vdev_children, txg); 9266 } 9267 9268 if (error == 0) 9269 spa->spa_last_synced_guid = rvd->vdev_guid; 9270 9271 spa_config_exit(spa, SCL_STATE, FTAG); 9272 9273 if (error == 0) 9274 break; 9275 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 9276 zio_resume_wait(spa); 9277 } 9278 } 9279 9280 /* 9281 * Sync the specified transaction group. New blocks may be dirtied as 9282 * part of the process, so we iterate until it converges. 9283 */ 9284 void 9285 spa_sync(spa_t *spa, uint64_t txg) 9286 { 9287 vdev_t *vd = NULL; 9288 9289 VERIFY(spa_writeable(spa)); 9290 9291 /* 9292 * Wait for i/os issued in open context that need to complete 9293 * before this txg syncs. 9294 */ 9295 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 9296 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 9297 ZIO_FLAG_CANFAIL); 9298 9299 /* 9300 * Now that there can be no more cloning in this transaction group, 9301 * but we are still before issuing frees, we can process pending BRT 9302 * updates. 9303 */ 9304 brt_pending_apply(spa, txg); 9305 9306 /* 9307 * Lock out configuration changes. 9308 */ 9309 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9310 9311 spa->spa_syncing_txg = txg; 9312 spa->spa_sync_pass = 0; 9313 9314 for (int i = 0; i < spa->spa_alloc_count; i++) { 9315 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9316 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9317 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9318 } 9319 9320 /* 9321 * If there are any pending vdev state changes, convert them 9322 * into config changes that go out with this transaction group. 9323 */ 9324 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9325 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9326 /* Avoid holding the write lock unless actually necessary */ 9327 if (vd->vdev_aux == NULL) { 9328 vdev_state_clean(vd); 9329 vdev_config_dirty(vd); 9330 continue; 9331 } 9332 /* 9333 * We need the write lock here because, for aux vdevs, 9334 * calling vdev_config_dirty() modifies sav_config. 9335 * This is ugly and will become unnecessary when we 9336 * eliminate the aux vdev wart by integrating all vdevs 9337 * into the root vdev tree. 9338 */ 9339 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9340 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 9341 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9342 vdev_state_clean(vd); 9343 vdev_config_dirty(vd); 9344 } 9345 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9346 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9347 } 9348 spa_config_exit(spa, SCL_STATE, FTAG); 9349 9350 dsl_pool_t *dp = spa->spa_dsl_pool; 9351 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 9352 9353 spa->spa_sync_starttime = gethrtime(); 9354 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9355 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 9356 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 9357 NSEC_TO_TICK(spa->spa_deadman_synctime)); 9358 9359 /* 9360 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 9361 * set spa_deflate if we have no raid-z vdevs. 9362 */ 9363 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 9364 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 9365 vdev_t *rvd = spa->spa_root_vdev; 9366 9367 int i; 9368 for (i = 0; i < rvd->vdev_children; i++) { 9369 vd = rvd->vdev_child[i]; 9370 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 9371 break; 9372 } 9373 if (i == rvd->vdev_children) { 9374 spa->spa_deflate = TRUE; 9375 VERIFY0(zap_add(spa->spa_meta_objset, 9376 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 9377 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 9378 } 9379 } 9380 9381 spa_sync_adjust_vdev_max_queue_depth(spa); 9382 9383 spa_sync_condense_indirect(spa, tx); 9384 9385 spa_sync_iterate_to_convergence(spa, tx); 9386 9387 #ifdef ZFS_DEBUG 9388 if (!list_is_empty(&spa->spa_config_dirty_list)) { 9389 /* 9390 * Make sure that the number of ZAPs for all the vdevs matches 9391 * the number of ZAPs in the per-vdev ZAP list. This only gets 9392 * called if the config is dirty; otherwise there may be 9393 * outstanding AVZ operations that weren't completed in 9394 * spa_sync_config_object. 9395 */ 9396 uint64_t all_vdev_zap_entry_count; 9397 ASSERT0(zap_count(spa->spa_meta_objset, 9398 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 9399 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 9400 all_vdev_zap_entry_count); 9401 } 9402 #endif 9403 9404 if (spa->spa_vdev_removal != NULL) { 9405 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 9406 } 9407 9408 spa_sync_rewrite_vdev_config(spa, tx); 9409 dmu_tx_commit(tx); 9410 9411 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9412 spa->spa_deadman_tqid = 0; 9413 9414 /* 9415 * Clear the dirty config list. 9416 */ 9417 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 9418 vdev_config_clean(vd); 9419 9420 /* 9421 * Now that the new config has synced transactionally, 9422 * let it become visible to the config cache. 9423 */ 9424 if (spa->spa_config_syncing != NULL) { 9425 spa_config_set(spa, spa->spa_config_syncing); 9426 spa->spa_config_txg = txg; 9427 spa->spa_config_syncing = NULL; 9428 } 9429 9430 dsl_pool_sync_done(dp, txg); 9431 9432 for (int i = 0; i < spa->spa_alloc_count; i++) { 9433 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9434 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9435 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9436 } 9437 9438 /* 9439 * Update usable space statistics. 9440 */ 9441 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 9442 != NULL) 9443 vdev_sync_done(vd, txg); 9444 9445 metaslab_class_evict_old(spa->spa_normal_class, txg); 9446 metaslab_class_evict_old(spa->spa_log_class, txg); 9447 9448 spa_sync_close_syncing_log_sm(spa); 9449 9450 spa_update_dspace(spa); 9451 9452 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) 9453 vdev_autotrim_kick(spa); 9454 9455 /* 9456 * It had better be the case that we didn't dirty anything 9457 * since vdev_config_sync(). 9458 */ 9459 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9460 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9461 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 9462 9463 while (zfs_pause_spa_sync) 9464 delay(1); 9465 9466 spa->spa_sync_pass = 0; 9467 9468 /* 9469 * Update the last synced uberblock here. We want to do this at 9470 * the end of spa_sync() so that consumers of spa_last_synced_txg() 9471 * will be guaranteed that all the processing associated with 9472 * that txg has been completed. 9473 */ 9474 spa->spa_ubsync = spa->spa_uberblock; 9475 spa_config_exit(spa, SCL_CONFIG, FTAG); 9476 9477 spa_handle_ignored_writes(spa); 9478 9479 /* 9480 * If any async tasks have been requested, kick them off. 9481 */ 9482 spa_async_dispatch(spa); 9483 } 9484 9485 /* 9486 * Sync all pools. We don't want to hold the namespace lock across these 9487 * operations, so we take a reference on the spa_t and drop the lock during the 9488 * sync. 9489 */ 9490 void 9491 spa_sync_allpools(void) 9492 { 9493 spa_t *spa = NULL; 9494 mutex_enter(&spa_namespace_lock); 9495 while ((spa = spa_next(spa)) != NULL) { 9496 if (spa_state(spa) != POOL_STATE_ACTIVE || 9497 !spa_writeable(spa) || spa_suspended(spa)) 9498 continue; 9499 spa_open_ref(spa, FTAG); 9500 mutex_exit(&spa_namespace_lock); 9501 txg_wait_synced(spa_get_dsl(spa), 0); 9502 mutex_enter(&spa_namespace_lock); 9503 spa_close(spa, FTAG); 9504 } 9505 mutex_exit(&spa_namespace_lock); 9506 } 9507 9508 /* 9509 * ========================================================================== 9510 * Miscellaneous routines 9511 * ========================================================================== 9512 */ 9513 9514 /* 9515 * Remove all pools in the system. 9516 */ 9517 void 9518 spa_evict_all(void) 9519 { 9520 spa_t *spa; 9521 9522 /* 9523 * Remove all cached state. All pools should be closed now, 9524 * so every spa in the AVL tree should be unreferenced. 9525 */ 9526 mutex_enter(&spa_namespace_lock); 9527 while ((spa = spa_next(NULL)) != NULL) { 9528 /* 9529 * Stop async tasks. The async thread may need to detach 9530 * a device that's been replaced, which requires grabbing 9531 * spa_namespace_lock, so we must drop it here. 9532 */ 9533 spa_open_ref(spa, FTAG); 9534 mutex_exit(&spa_namespace_lock); 9535 spa_async_suspend(spa); 9536 mutex_enter(&spa_namespace_lock); 9537 spa_close(spa, FTAG); 9538 9539 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 9540 spa_unload(spa); 9541 spa_deactivate(spa); 9542 } 9543 spa_remove(spa); 9544 } 9545 mutex_exit(&spa_namespace_lock); 9546 } 9547 9548 vdev_t * 9549 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 9550 { 9551 vdev_t *vd; 9552 int i; 9553 9554 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 9555 return (vd); 9556 9557 if (aux) { 9558 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 9559 vd = spa->spa_l2cache.sav_vdevs[i]; 9560 if (vd->vdev_guid == guid) 9561 return (vd); 9562 } 9563 9564 for (i = 0; i < spa->spa_spares.sav_count; i++) { 9565 vd = spa->spa_spares.sav_vdevs[i]; 9566 if (vd->vdev_guid == guid) 9567 return (vd); 9568 } 9569 } 9570 9571 return (NULL); 9572 } 9573 9574 void 9575 spa_upgrade(spa_t *spa, uint64_t version) 9576 { 9577 ASSERT(spa_writeable(spa)); 9578 9579 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9580 9581 /* 9582 * This should only be called for a non-faulted pool, and since a 9583 * future version would result in an unopenable pool, this shouldn't be 9584 * possible. 9585 */ 9586 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 9587 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 9588 9589 spa->spa_uberblock.ub_version = version; 9590 vdev_config_dirty(spa->spa_root_vdev); 9591 9592 spa_config_exit(spa, SCL_ALL, FTAG); 9593 9594 txg_wait_synced(spa_get_dsl(spa), 0); 9595 } 9596 9597 static boolean_t 9598 spa_has_aux_vdev(spa_t *spa, uint64_t guid, spa_aux_vdev_t *sav) 9599 { 9600 (void) spa; 9601 int i; 9602 uint64_t vdev_guid; 9603 9604 for (i = 0; i < sav->sav_count; i++) 9605 if (sav->sav_vdevs[i]->vdev_guid == guid) 9606 return (B_TRUE); 9607 9608 for (i = 0; i < sav->sav_npending; i++) { 9609 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 9610 &vdev_guid) == 0 && vdev_guid == guid) 9611 return (B_TRUE); 9612 } 9613 9614 return (B_FALSE); 9615 } 9616 9617 boolean_t 9618 spa_has_l2cache(spa_t *spa, uint64_t guid) 9619 { 9620 return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache)); 9621 } 9622 9623 boolean_t 9624 spa_has_spare(spa_t *spa, uint64_t guid) 9625 { 9626 return (spa_has_aux_vdev(spa, guid, &spa->spa_spares)); 9627 } 9628 9629 /* 9630 * Check if a pool has an active shared spare device. 9631 * Note: reference count of an active spare is 2, as a spare and as a replace 9632 */ 9633 static boolean_t 9634 spa_has_active_shared_spare(spa_t *spa) 9635 { 9636 int i, refcnt; 9637 uint64_t pool; 9638 spa_aux_vdev_t *sav = &spa->spa_spares; 9639 9640 for (i = 0; i < sav->sav_count; i++) { 9641 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 9642 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 9643 refcnt > 2) 9644 return (B_TRUE); 9645 } 9646 9647 return (B_FALSE); 9648 } 9649 9650 uint64_t 9651 spa_total_metaslabs(spa_t *spa) 9652 { 9653 vdev_t *rvd = spa->spa_root_vdev; 9654 9655 uint64_t m = 0; 9656 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 9657 vdev_t *vd = rvd->vdev_child[c]; 9658 if (!vdev_is_concrete(vd)) 9659 continue; 9660 m += vd->vdev_ms_count; 9661 } 9662 return (m); 9663 } 9664 9665 /* 9666 * Notify any waiting threads that some activity has switched from being in- 9667 * progress to not-in-progress so that the thread can wake up and determine 9668 * whether it is finished waiting. 9669 */ 9670 void 9671 spa_notify_waiters(spa_t *spa) 9672 { 9673 /* 9674 * Acquiring spa_activities_lock here prevents the cv_broadcast from 9675 * happening between the waiting thread's check and cv_wait. 9676 */ 9677 mutex_enter(&spa->spa_activities_lock); 9678 cv_broadcast(&spa->spa_activities_cv); 9679 mutex_exit(&spa->spa_activities_lock); 9680 } 9681 9682 /* 9683 * Notify any waiting threads that the pool is exporting, and then block until 9684 * they are finished using the spa_t. 9685 */ 9686 void 9687 spa_wake_waiters(spa_t *spa) 9688 { 9689 mutex_enter(&spa->spa_activities_lock); 9690 spa->spa_waiters_cancel = B_TRUE; 9691 cv_broadcast(&spa->spa_activities_cv); 9692 while (spa->spa_waiters != 0) 9693 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 9694 spa->spa_waiters_cancel = B_FALSE; 9695 mutex_exit(&spa->spa_activities_lock); 9696 } 9697 9698 /* Whether the vdev or any of its descendants are being initialized/trimmed. */ 9699 static boolean_t 9700 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 9701 { 9702 spa_t *spa = vd->vdev_spa; 9703 9704 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 9705 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9706 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 9707 activity == ZPOOL_WAIT_TRIM); 9708 9709 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 9710 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 9711 9712 mutex_exit(&spa->spa_activities_lock); 9713 mutex_enter(lock); 9714 mutex_enter(&spa->spa_activities_lock); 9715 9716 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 9717 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) : 9718 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE); 9719 mutex_exit(lock); 9720 9721 if (in_progress) 9722 return (B_TRUE); 9723 9724 for (int i = 0; i < vd->vdev_children; i++) { 9725 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 9726 activity)) 9727 return (B_TRUE); 9728 } 9729 9730 return (B_FALSE); 9731 } 9732 9733 /* 9734 * If use_guid is true, this checks whether the vdev specified by guid is 9735 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 9736 * is being initialized/trimmed. The caller must hold the config lock and 9737 * spa_activities_lock. 9738 */ 9739 static int 9740 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 9741 zpool_wait_activity_t activity, boolean_t *in_progress) 9742 { 9743 mutex_exit(&spa->spa_activities_lock); 9744 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9745 mutex_enter(&spa->spa_activities_lock); 9746 9747 vdev_t *vd; 9748 if (use_guid) { 9749 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 9750 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 9751 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9752 return (EINVAL); 9753 } 9754 } else { 9755 vd = spa->spa_root_vdev; 9756 } 9757 9758 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 9759 9760 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9761 return (0); 9762 } 9763 9764 /* 9765 * Locking for waiting threads 9766 * --------------------------- 9767 * 9768 * Waiting threads need a way to check whether a given activity is in progress, 9769 * and then, if it is, wait for it to complete. Each activity will have some 9770 * in-memory representation of the relevant on-disk state which can be used to 9771 * determine whether or not the activity is in progress. The in-memory state and 9772 * the locking used to protect it will be different for each activity, and may 9773 * not be suitable for use with a cvar (e.g., some state is protected by the 9774 * config lock). To allow waiting threads to wait without any races, another 9775 * lock, spa_activities_lock, is used. 9776 * 9777 * When the state is checked, both the activity-specific lock (if there is one) 9778 * and spa_activities_lock are held. In some cases, the activity-specific lock 9779 * is acquired explicitly (e.g. the config lock). In others, the locking is 9780 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 9781 * thread releases the activity-specific lock and, if the activity is in 9782 * progress, then cv_waits using spa_activities_lock. 9783 * 9784 * The waiting thread is woken when another thread, one completing some 9785 * activity, updates the state of the activity and then calls 9786 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 9787 * needs to hold its activity-specific lock when updating the state, and this 9788 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 9789 * 9790 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 9791 * and because it is held when the waiting thread checks the state of the 9792 * activity, it can never be the case that the completing thread both updates 9793 * the activity state and cv_broadcasts in between the waiting thread's check 9794 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 9795 * 9796 * In order to prevent deadlock, when the waiting thread does its check, in some 9797 * cases it will temporarily drop spa_activities_lock in order to acquire the 9798 * activity-specific lock. The order in which spa_activities_lock and the 9799 * activity specific lock are acquired in the waiting thread is determined by 9800 * the order in which they are acquired in the completing thread; if the 9801 * completing thread calls spa_notify_waiters with the activity-specific lock 9802 * held, then the waiting thread must also acquire the activity-specific lock 9803 * first. 9804 */ 9805 9806 static int 9807 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 9808 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 9809 { 9810 int error = 0; 9811 9812 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9813 9814 switch (activity) { 9815 case ZPOOL_WAIT_CKPT_DISCARD: 9816 *in_progress = 9817 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 9818 zap_contains(spa_meta_objset(spa), 9819 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 9820 ENOENT); 9821 break; 9822 case ZPOOL_WAIT_FREE: 9823 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 9824 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 9825 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 9826 spa_livelist_delete_check(spa)); 9827 break; 9828 case ZPOOL_WAIT_INITIALIZE: 9829 case ZPOOL_WAIT_TRIM: 9830 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 9831 activity, in_progress); 9832 break; 9833 case ZPOOL_WAIT_REPLACE: 9834 mutex_exit(&spa->spa_activities_lock); 9835 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9836 mutex_enter(&spa->spa_activities_lock); 9837 9838 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 9839 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9840 break; 9841 case ZPOOL_WAIT_REMOVE: 9842 *in_progress = (spa->spa_removing_phys.sr_state == 9843 DSS_SCANNING); 9844 break; 9845 case ZPOOL_WAIT_RESILVER: 9846 if ((*in_progress = vdev_rebuild_active(spa->spa_root_vdev))) 9847 break; 9848 zfs_fallthrough; 9849 case ZPOOL_WAIT_SCRUB: 9850 { 9851 boolean_t scanning, paused, is_scrub; 9852 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 9853 9854 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 9855 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 9856 paused = dsl_scan_is_paused_scrub(scn); 9857 *in_progress = (scanning && !paused && 9858 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 9859 break; 9860 } 9861 default: 9862 panic("unrecognized value for activity %d", activity); 9863 } 9864 9865 return (error); 9866 } 9867 9868 static int 9869 spa_wait_common(const char *pool, zpool_wait_activity_t activity, 9870 boolean_t use_tag, uint64_t tag, boolean_t *waited) 9871 { 9872 /* 9873 * The tag is used to distinguish between instances of an activity. 9874 * 'initialize' and 'trim' are the only activities that we use this for. 9875 * The other activities can only have a single instance in progress in a 9876 * pool at one time, making the tag unnecessary. 9877 * 9878 * There can be multiple devices being replaced at once, but since they 9879 * all finish once resilvering finishes, we don't bother keeping track 9880 * of them individually, we just wait for them all to finish. 9881 */ 9882 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 9883 activity != ZPOOL_WAIT_TRIM) 9884 return (EINVAL); 9885 9886 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 9887 return (EINVAL); 9888 9889 spa_t *spa; 9890 int error = spa_open(pool, &spa, FTAG); 9891 if (error != 0) 9892 return (error); 9893 9894 /* 9895 * Increment the spa's waiter count so that we can call spa_close and 9896 * still ensure that the spa_t doesn't get freed before this thread is 9897 * finished with it when the pool is exported. We want to call spa_close 9898 * before we start waiting because otherwise the additional ref would 9899 * prevent the pool from being exported or destroyed throughout the 9900 * potentially long wait. 9901 */ 9902 mutex_enter(&spa->spa_activities_lock); 9903 spa->spa_waiters++; 9904 spa_close(spa, FTAG); 9905 9906 *waited = B_FALSE; 9907 for (;;) { 9908 boolean_t in_progress; 9909 error = spa_activity_in_progress(spa, activity, use_tag, tag, 9910 &in_progress); 9911 9912 if (error || !in_progress || spa->spa_waiters_cancel) 9913 break; 9914 9915 *waited = B_TRUE; 9916 9917 if (cv_wait_sig(&spa->spa_activities_cv, 9918 &spa->spa_activities_lock) == 0) { 9919 error = EINTR; 9920 break; 9921 } 9922 } 9923 9924 spa->spa_waiters--; 9925 cv_signal(&spa->spa_waiters_cv); 9926 mutex_exit(&spa->spa_activities_lock); 9927 9928 return (error); 9929 } 9930 9931 /* 9932 * Wait for a particular instance of the specified activity to complete, where 9933 * the instance is identified by 'tag' 9934 */ 9935 int 9936 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 9937 boolean_t *waited) 9938 { 9939 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 9940 } 9941 9942 /* 9943 * Wait for all instances of the specified activity complete 9944 */ 9945 int 9946 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 9947 { 9948 9949 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 9950 } 9951 9952 sysevent_t * 9953 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9954 { 9955 sysevent_t *ev = NULL; 9956 #ifdef _KERNEL 9957 nvlist_t *resource; 9958 9959 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 9960 if (resource) { 9961 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 9962 ev->resource = resource; 9963 } 9964 #else 9965 (void) spa, (void) vd, (void) hist_nvl, (void) name; 9966 #endif 9967 return (ev); 9968 } 9969 9970 void 9971 spa_event_post(sysevent_t *ev) 9972 { 9973 #ifdef _KERNEL 9974 if (ev) { 9975 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 9976 kmem_free(ev, sizeof (*ev)); 9977 } 9978 #else 9979 (void) ev; 9980 #endif 9981 } 9982 9983 /* 9984 * Post a zevent corresponding to the given sysevent. The 'name' must be one 9985 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 9986 * filled in from the spa and (optionally) the vdev. This doesn't do anything 9987 * in the userland libzpool, as we don't want consumers to misinterpret ztest 9988 * or zdb as real changes. 9989 */ 9990 void 9991 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9992 { 9993 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 9994 } 9995 9996 /* state manipulation functions */ 9997 EXPORT_SYMBOL(spa_open); 9998 EXPORT_SYMBOL(spa_open_rewind); 9999 EXPORT_SYMBOL(spa_get_stats); 10000 EXPORT_SYMBOL(spa_create); 10001 EXPORT_SYMBOL(spa_import); 10002 EXPORT_SYMBOL(spa_tryimport); 10003 EXPORT_SYMBOL(spa_destroy); 10004 EXPORT_SYMBOL(spa_export); 10005 EXPORT_SYMBOL(spa_reset); 10006 EXPORT_SYMBOL(spa_async_request); 10007 EXPORT_SYMBOL(spa_async_suspend); 10008 EXPORT_SYMBOL(spa_async_resume); 10009 EXPORT_SYMBOL(spa_inject_addref); 10010 EXPORT_SYMBOL(spa_inject_delref); 10011 EXPORT_SYMBOL(spa_scan_stat_init); 10012 EXPORT_SYMBOL(spa_scan_get_stats); 10013 10014 /* device manipulation */ 10015 EXPORT_SYMBOL(spa_vdev_add); 10016 EXPORT_SYMBOL(spa_vdev_attach); 10017 EXPORT_SYMBOL(spa_vdev_detach); 10018 EXPORT_SYMBOL(spa_vdev_setpath); 10019 EXPORT_SYMBOL(spa_vdev_setfru); 10020 EXPORT_SYMBOL(spa_vdev_split_mirror); 10021 10022 /* spare statech is global across all pools) */ 10023 EXPORT_SYMBOL(spa_spare_add); 10024 EXPORT_SYMBOL(spa_spare_remove); 10025 EXPORT_SYMBOL(spa_spare_exists); 10026 EXPORT_SYMBOL(spa_spare_activate); 10027 10028 /* L2ARC statech is global across all pools) */ 10029 EXPORT_SYMBOL(spa_l2cache_add); 10030 EXPORT_SYMBOL(spa_l2cache_remove); 10031 EXPORT_SYMBOL(spa_l2cache_exists); 10032 EXPORT_SYMBOL(spa_l2cache_activate); 10033 EXPORT_SYMBOL(spa_l2cache_drop); 10034 10035 /* scanning */ 10036 EXPORT_SYMBOL(spa_scan); 10037 EXPORT_SYMBOL(spa_scan_stop); 10038 10039 /* spa syncing */ 10040 EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 10041 EXPORT_SYMBOL(spa_sync_allpools); 10042 10043 /* properties */ 10044 EXPORT_SYMBOL(spa_prop_set); 10045 EXPORT_SYMBOL(spa_prop_get); 10046 EXPORT_SYMBOL(spa_prop_clear_bootfs); 10047 10048 /* asynchronous event notification */ 10049 EXPORT_SYMBOL(spa_event_notify); 10050 10051 /* BEGIN CSTYLED */ 10052 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, UINT, ZMOD_RW, 10053 "log2 fraction of arc that can be used by inflight I/Os when " 10054 "verifying pool during import"); 10055 /* END CSTYLED */ 10056 10057 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 10058 "Set to traverse metadata on pool import"); 10059 10060 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 10061 "Set to traverse data on pool import"); 10062 10063 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 10064 "Print vdev tree to zfs_dbgmsg during pool import"); 10065 10066 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD, 10067 "Percentage of CPUs to run an IO worker thread"); 10068 10069 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD, 10070 "Number of threads per IO worker taskqueue"); 10071 10072 /* BEGIN CSTYLED */ 10073 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, U64, ZMOD_RW, 10074 "Allow importing pool with up to this number of missing top-level " 10075 "vdevs (in read-only mode)"); 10076 /* END CSTYLED */ 10077 10078 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, 10079 ZMOD_RW, "Set the livelist condense zthr to pause"); 10080 10081 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, 10082 ZMOD_RW, "Set the livelist condense synctask to pause"); 10083 10084 /* BEGIN CSTYLED */ 10085 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, 10086 INT, ZMOD_RW, 10087 "Whether livelist condensing was canceled in the synctask"); 10088 10089 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, 10090 INT, ZMOD_RW, 10091 "Whether livelist condensing was canceled in the zthr function"); 10092 10093 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, 10094 ZMOD_RW, 10095 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 10096 "was being condensed"); 10097 /* END CSTYLED */ 10098