1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 * Copyright 2016 Toomas Soome <tsoome@me.com> 30 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 31 * Copyright 2018 Joyent, Inc. 32 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 33 * Copyright 2017 Joyent, Inc. 34 * Copyright (c) 2017, Intel Corporation. 35 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 36 */ 37 38 /* 39 * SPA: Storage Pool Allocator 40 * 41 * This file contains all the routines used when modifying on-disk SPA state. 42 * This includes opening, importing, destroying, exporting a pool, and syncing a 43 * pool. 44 */ 45 46 #include <sys/zfs_context.h> 47 #include <sys/fm/fs/zfs.h> 48 #include <sys/spa_impl.h> 49 #include <sys/zio.h> 50 #include <sys/zio_checksum.h> 51 #include <sys/dmu.h> 52 #include <sys/dmu_tx.h> 53 #include <sys/zap.h> 54 #include <sys/zil.h> 55 #include <sys/ddt.h> 56 #include <sys/vdev_impl.h> 57 #include <sys/vdev_removal.h> 58 #include <sys/vdev_indirect_mapping.h> 59 #include <sys/vdev_indirect_births.h> 60 #include <sys/vdev_initialize.h> 61 #include <sys/vdev_rebuild.h> 62 #include <sys/vdev_trim.h> 63 #include <sys/vdev_disk.h> 64 #include <sys/vdev_draid.h> 65 #include <sys/metaslab.h> 66 #include <sys/metaslab_impl.h> 67 #include <sys/mmp.h> 68 #include <sys/uberblock_impl.h> 69 #include <sys/txg.h> 70 #include <sys/avl.h> 71 #include <sys/bpobj.h> 72 #include <sys/dmu_traverse.h> 73 #include <sys/dmu_objset.h> 74 #include <sys/unique.h> 75 #include <sys/dsl_pool.h> 76 #include <sys/dsl_dataset.h> 77 #include <sys/dsl_dir.h> 78 #include <sys/dsl_prop.h> 79 #include <sys/dsl_synctask.h> 80 #include <sys/fs/zfs.h> 81 #include <sys/arc.h> 82 #include <sys/callb.h> 83 #include <sys/systeminfo.h> 84 #include <sys/spa_boot.h> 85 #include <sys/zfs_ioctl.h> 86 #include <sys/dsl_scan.h> 87 #include <sys/zfeature.h> 88 #include <sys/dsl_destroy.h> 89 #include <sys/zvol.h> 90 91 #ifdef _KERNEL 92 #include <sys/fm/protocol.h> 93 #include <sys/fm/util.h> 94 #include <sys/callb.h> 95 #include <sys/zone.h> 96 #include <sys/vmsystm.h> 97 #endif /* _KERNEL */ 98 99 #include "zfs_prop.h" 100 #include "zfs_comutil.h" 101 102 /* 103 * The interval, in seconds, at which failed configuration cache file writes 104 * should be retried. 105 */ 106 int zfs_ccw_retry_interval = 300; 107 108 typedef enum zti_modes { 109 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 110 ZTI_MODE_BATCH, /* cpu-intensive; value is ignored */ 111 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 112 ZTI_MODE_NULL, /* don't create a taskq */ 113 ZTI_NMODES 114 } zti_modes_t; 115 116 #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 117 #define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 118 #define ZTI_BATCH { ZTI_MODE_BATCH, 0, 1 } 119 #define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 } 120 #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 121 122 #define ZTI_N(n) ZTI_P(n, 1) 123 #define ZTI_ONE ZTI_N(1) 124 125 typedef struct zio_taskq_info { 126 zti_modes_t zti_mode; 127 uint_t zti_value; 128 uint_t zti_count; 129 } zio_taskq_info_t; 130 131 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 132 "iss", "iss_h", "int", "int_h" 133 }; 134 135 /* 136 * This table defines the taskq settings for each ZFS I/O type. When 137 * initializing a pool, we use this table to create an appropriately sized 138 * taskq. Some operations are low volume and therefore have a small, static 139 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 140 * macros. Other operations process a large amount of data; the ZTI_BATCH 141 * macro causes us to create a taskq oriented for throughput. Some operations 142 * are so high frequency and short-lived that the taskq itself can become a 143 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 144 * additional degree of parallelism specified by the number of threads per- 145 * taskq and the number of taskqs; when dispatching an event in this case, the 146 * particular taskq is chosen at random. ZTI_SCALE is similar to ZTI_BATCH, 147 * but with number of taskqs also scaling with number of CPUs. 148 * 149 * The different taskq priorities are to handle the different contexts (issue 150 * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that 151 * need to be handled with minimum delay. 152 */ 153 const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 154 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 155 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 156 { ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */ 157 { ZTI_BATCH, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */ 158 { ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 159 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 160 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */ 161 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 162 }; 163 164 static void spa_sync_version(void *arg, dmu_tx_t *tx); 165 static void spa_sync_props(void *arg, dmu_tx_t *tx); 166 static boolean_t spa_has_active_shared_spare(spa_t *spa); 167 static int spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport); 168 static void spa_vdev_resilver_done(spa_t *spa); 169 170 uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 171 uint_t zio_taskq_batch_tpq; /* threads per taskq */ 172 boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 173 uint_t zio_taskq_basedc = 80; /* base duty cycle */ 174 175 boolean_t spa_create_process = B_TRUE; /* no process ==> no sysdc */ 176 177 /* 178 * Report any spa_load_verify errors found, but do not fail spa_load. 179 * This is used by zdb to analyze non-idle pools. 180 */ 181 boolean_t spa_load_verify_dryrun = B_FALSE; 182 183 /* 184 * This (illegal) pool name is used when temporarily importing a spa_t in order 185 * to get the vdev stats associated with the imported devices. 186 */ 187 #define TRYIMPORT_NAME "$import" 188 189 /* 190 * For debugging purposes: print out vdev tree during pool import. 191 */ 192 int spa_load_print_vdev_tree = B_FALSE; 193 194 /* 195 * A non-zero value for zfs_max_missing_tvds means that we allow importing 196 * pools with missing top-level vdevs. This is strictly intended for advanced 197 * pool recovery cases since missing data is almost inevitable. Pools with 198 * missing devices can only be imported read-only for safety reasons, and their 199 * fail-mode will be automatically set to "continue". 200 * 201 * With 1 missing vdev we should be able to import the pool and mount all 202 * datasets. User data that was not modified after the missing device has been 203 * added should be recoverable. This means that snapshots created prior to the 204 * addition of that device should be completely intact. 205 * 206 * With 2 missing vdevs, some datasets may fail to mount since there are 207 * dataset statistics that are stored as regular metadata. Some data might be 208 * recoverable if those vdevs were added recently. 209 * 210 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 211 * may be missing entirely. Chances of data recovery are very low. Note that 212 * there are also risks of performing an inadvertent rewind as we might be 213 * missing all the vdevs with the latest uberblocks. 214 */ 215 unsigned long zfs_max_missing_tvds = 0; 216 217 /* 218 * The parameters below are similar to zfs_max_missing_tvds but are only 219 * intended for a preliminary open of the pool with an untrusted config which 220 * might be incomplete or out-dated. 221 * 222 * We are more tolerant for pools opened from a cachefile since we could have 223 * an out-dated cachefile where a device removal was not registered. 224 * We could have set the limit arbitrarily high but in the case where devices 225 * are really missing we would want to return the proper error codes; we chose 226 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 227 * and we get a chance to retrieve the trusted config. 228 */ 229 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 230 231 /* 232 * In the case where config was assembled by scanning device paths (/dev/dsks 233 * by default) we are less tolerant since all the existing devices should have 234 * been detected and we want spa_load to return the right error codes. 235 */ 236 uint64_t zfs_max_missing_tvds_scan = 0; 237 238 /* 239 * Debugging aid that pauses spa_sync() towards the end. 240 */ 241 boolean_t zfs_pause_spa_sync = B_FALSE; 242 243 /* 244 * Variables to indicate the livelist condense zthr func should wait at certain 245 * points for the livelist to be removed - used to test condense/destroy races 246 */ 247 int zfs_livelist_condense_zthr_pause = 0; 248 int zfs_livelist_condense_sync_pause = 0; 249 250 /* 251 * Variables to track whether or not condense cancellation has been 252 * triggered in testing. 253 */ 254 int zfs_livelist_condense_sync_cancel = 0; 255 int zfs_livelist_condense_zthr_cancel = 0; 256 257 /* 258 * Variable to track whether or not extra ALLOC blkptrs were added to a 259 * livelist entry while it was being condensed (caused by the way we track 260 * remapped blkptrs in dbuf_remap_impl) 261 */ 262 int zfs_livelist_condense_new_alloc = 0; 263 264 /* 265 * ========================================================================== 266 * SPA properties routines 267 * ========================================================================== 268 */ 269 270 /* 271 * Add a (source=src, propname=propval) list to an nvlist. 272 */ 273 static void 274 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval, 275 uint64_t intval, zprop_source_t src) 276 { 277 const char *propname = zpool_prop_to_name(prop); 278 nvlist_t *propval; 279 280 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); 281 VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); 282 283 if (strval != NULL) 284 VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0); 285 else 286 VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, intval) == 0); 287 288 VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0); 289 nvlist_free(propval); 290 } 291 292 /* 293 * Get property values from the spa configuration. 294 */ 295 static void 296 spa_prop_get_config(spa_t *spa, nvlist_t **nvp) 297 { 298 vdev_t *rvd = spa->spa_root_vdev; 299 dsl_pool_t *pool = spa->spa_dsl_pool; 300 uint64_t size, alloc, cap, version; 301 const zprop_source_t src = ZPROP_SRC_NONE; 302 spa_config_dirent_t *dp; 303 metaslab_class_t *mc = spa_normal_class(spa); 304 305 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 306 307 if (rvd != NULL) { 308 alloc = metaslab_class_get_alloc(mc); 309 alloc += metaslab_class_get_alloc(spa_special_class(spa)); 310 alloc += metaslab_class_get_alloc(spa_dedup_class(spa)); 311 alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa)); 312 313 size = metaslab_class_get_space(mc); 314 size += metaslab_class_get_space(spa_special_class(spa)); 315 size += metaslab_class_get_space(spa_dedup_class(spa)); 316 size += metaslab_class_get_space(spa_embedded_log_class(spa)); 317 318 spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 319 spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src); 320 spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 321 spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL, 322 size - alloc, src); 323 spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL, 324 spa->spa_checkpoint_info.sci_dspace, src); 325 326 spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL, 327 metaslab_class_fragmentation(mc), src); 328 spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, 329 metaslab_class_expandable_space(mc), src); 330 spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL, 331 (spa_mode(spa) == SPA_MODE_READ), src); 332 333 cap = (size == 0) ? 0 : (alloc * 100 / size); 334 spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src); 335 336 spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL, 337 ddt_get_pool_dedup_ratio(spa), src); 338 339 spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL, 340 rvd->vdev_state, src); 341 342 version = spa_version(spa); 343 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 344 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 345 version, ZPROP_SRC_DEFAULT); 346 } else { 347 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 348 version, ZPROP_SRC_LOCAL); 349 } 350 spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID, 351 NULL, spa_load_guid(spa), src); 352 } 353 354 if (pool != NULL) { 355 /* 356 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 357 * when opening pools before this version freedir will be NULL. 358 */ 359 if (pool->dp_free_dir != NULL) { 360 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, 361 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 362 src); 363 } else { 364 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, 365 NULL, 0, src); 366 } 367 368 if (pool->dp_leak_dir != NULL) { 369 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, 370 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 371 src); 372 } else { 373 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, 374 NULL, 0, src); 375 } 376 } 377 378 spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 379 380 if (spa->spa_comment != NULL) { 381 spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment, 382 0, ZPROP_SRC_LOCAL); 383 } 384 385 if (spa->spa_compatibility != NULL) { 386 spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY, 387 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 388 } 389 390 if (spa->spa_root != NULL) 391 spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root, 392 0, ZPROP_SRC_LOCAL); 393 394 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 395 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 396 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 397 } else { 398 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 399 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 400 } 401 402 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 403 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 404 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 405 } else { 406 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 407 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 408 } 409 410 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 411 if (dp->scd_path == NULL) { 412 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 413 "none", 0, ZPROP_SRC_LOCAL); 414 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 415 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 416 dp->scd_path, 0, ZPROP_SRC_LOCAL); 417 } 418 } 419 } 420 421 /* 422 * Get zpool property values. 423 */ 424 int 425 spa_prop_get(spa_t *spa, nvlist_t **nvp) 426 { 427 objset_t *mos = spa->spa_meta_objset; 428 zap_cursor_t zc; 429 zap_attribute_t za; 430 dsl_pool_t *dp; 431 int err; 432 433 err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); 434 if (err) 435 return (err); 436 437 dp = spa_get_dsl(spa); 438 dsl_pool_config_enter(dp, FTAG); 439 mutex_enter(&spa->spa_props_lock); 440 441 /* 442 * Get properties from the spa config. 443 */ 444 spa_prop_get_config(spa, nvp); 445 446 /* If no pool property object, no more prop to get. */ 447 if (mos == NULL || spa->spa_pool_props_object == 0) 448 goto out; 449 450 /* 451 * Get properties from the MOS pool property object. 452 */ 453 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 454 (err = zap_cursor_retrieve(&zc, &za)) == 0; 455 zap_cursor_advance(&zc)) { 456 uint64_t intval = 0; 457 char *strval = NULL; 458 zprop_source_t src = ZPROP_SRC_DEFAULT; 459 zpool_prop_t prop; 460 461 if ((prop = zpool_name_to_prop(za.za_name)) == ZPOOL_PROP_INVAL) 462 continue; 463 464 switch (za.za_integer_length) { 465 case 8: 466 /* integer property */ 467 if (za.za_first_integer != 468 zpool_prop_default_numeric(prop)) 469 src = ZPROP_SRC_LOCAL; 470 471 if (prop == ZPOOL_PROP_BOOTFS) { 472 dsl_dataset_t *ds = NULL; 473 474 err = dsl_dataset_hold_obj(dp, 475 za.za_first_integer, FTAG, &ds); 476 if (err != 0) 477 break; 478 479 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 480 KM_SLEEP); 481 dsl_dataset_name(ds, strval); 482 dsl_dataset_rele(ds, FTAG); 483 } else { 484 strval = NULL; 485 intval = za.za_first_integer; 486 } 487 488 spa_prop_add_list(*nvp, prop, strval, intval, src); 489 490 if (strval != NULL) 491 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 492 493 break; 494 495 case 1: 496 /* string property */ 497 strval = kmem_alloc(za.za_num_integers, KM_SLEEP); 498 err = zap_lookup(mos, spa->spa_pool_props_object, 499 za.za_name, 1, za.za_num_integers, strval); 500 if (err) { 501 kmem_free(strval, za.za_num_integers); 502 break; 503 } 504 spa_prop_add_list(*nvp, prop, strval, 0, src); 505 kmem_free(strval, za.za_num_integers); 506 break; 507 508 default: 509 break; 510 } 511 } 512 zap_cursor_fini(&zc); 513 out: 514 mutex_exit(&spa->spa_props_lock); 515 dsl_pool_config_exit(dp, FTAG); 516 if (err && err != ENOENT) { 517 nvlist_free(*nvp); 518 *nvp = NULL; 519 return (err); 520 } 521 522 return (0); 523 } 524 525 /* 526 * Validate the given pool properties nvlist and modify the list 527 * for the property values to be set. 528 */ 529 static int 530 spa_prop_validate(spa_t *spa, nvlist_t *props) 531 { 532 nvpair_t *elem; 533 int error = 0, reset_bootfs = 0; 534 uint64_t objnum = 0; 535 boolean_t has_feature = B_FALSE; 536 537 elem = NULL; 538 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 539 uint64_t intval; 540 char *strval, *slash, *check, *fname; 541 const char *propname = nvpair_name(elem); 542 zpool_prop_t prop = zpool_name_to_prop(propname); 543 544 switch (prop) { 545 case ZPOOL_PROP_INVAL: 546 if (!zpool_prop_feature(propname)) { 547 error = SET_ERROR(EINVAL); 548 break; 549 } 550 551 /* 552 * Sanitize the input. 553 */ 554 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 555 error = SET_ERROR(EINVAL); 556 break; 557 } 558 559 if (nvpair_value_uint64(elem, &intval) != 0) { 560 error = SET_ERROR(EINVAL); 561 break; 562 } 563 564 if (intval != 0) { 565 error = SET_ERROR(EINVAL); 566 break; 567 } 568 569 fname = strchr(propname, '@') + 1; 570 if (zfeature_lookup_name(fname, NULL) != 0) { 571 error = SET_ERROR(EINVAL); 572 break; 573 } 574 575 has_feature = B_TRUE; 576 break; 577 578 case ZPOOL_PROP_VERSION: 579 error = nvpair_value_uint64(elem, &intval); 580 if (!error && 581 (intval < spa_version(spa) || 582 intval > SPA_VERSION_BEFORE_FEATURES || 583 has_feature)) 584 error = SET_ERROR(EINVAL); 585 break; 586 587 case ZPOOL_PROP_DELEGATION: 588 case ZPOOL_PROP_AUTOREPLACE: 589 case ZPOOL_PROP_LISTSNAPS: 590 case ZPOOL_PROP_AUTOEXPAND: 591 case ZPOOL_PROP_AUTOTRIM: 592 error = nvpair_value_uint64(elem, &intval); 593 if (!error && intval > 1) 594 error = SET_ERROR(EINVAL); 595 break; 596 597 case ZPOOL_PROP_MULTIHOST: 598 error = nvpair_value_uint64(elem, &intval); 599 if (!error && intval > 1) 600 error = SET_ERROR(EINVAL); 601 602 if (!error) { 603 uint32_t hostid = zone_get_hostid(NULL); 604 if (hostid) 605 spa->spa_hostid = hostid; 606 else 607 error = SET_ERROR(ENOTSUP); 608 } 609 610 break; 611 612 case ZPOOL_PROP_BOOTFS: 613 /* 614 * If the pool version is less than SPA_VERSION_BOOTFS, 615 * or the pool is still being created (version == 0), 616 * the bootfs property cannot be set. 617 */ 618 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 619 error = SET_ERROR(ENOTSUP); 620 break; 621 } 622 623 /* 624 * Make sure the vdev config is bootable 625 */ 626 if (!vdev_is_bootable(spa->spa_root_vdev)) { 627 error = SET_ERROR(ENOTSUP); 628 break; 629 } 630 631 reset_bootfs = 1; 632 633 error = nvpair_value_string(elem, &strval); 634 635 if (!error) { 636 objset_t *os; 637 638 if (strval == NULL || strval[0] == '\0') { 639 objnum = zpool_prop_default_numeric( 640 ZPOOL_PROP_BOOTFS); 641 break; 642 } 643 644 error = dmu_objset_hold(strval, FTAG, &os); 645 if (error != 0) 646 break; 647 648 /* Must be ZPL. */ 649 if (dmu_objset_type(os) != DMU_OST_ZFS) { 650 error = SET_ERROR(ENOTSUP); 651 } else { 652 objnum = dmu_objset_id(os); 653 } 654 dmu_objset_rele(os, FTAG); 655 } 656 break; 657 658 case ZPOOL_PROP_FAILUREMODE: 659 error = nvpair_value_uint64(elem, &intval); 660 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 661 error = SET_ERROR(EINVAL); 662 663 /* 664 * This is a special case which only occurs when 665 * the pool has completely failed. This allows 666 * the user to change the in-core failmode property 667 * without syncing it out to disk (I/Os might 668 * currently be blocked). We do this by returning 669 * EIO to the caller (spa_prop_set) to trick it 670 * into thinking we encountered a property validation 671 * error. 672 */ 673 if (!error && spa_suspended(spa)) { 674 spa->spa_failmode = intval; 675 error = SET_ERROR(EIO); 676 } 677 break; 678 679 case ZPOOL_PROP_CACHEFILE: 680 if ((error = nvpair_value_string(elem, &strval)) != 0) 681 break; 682 683 if (strval[0] == '\0') 684 break; 685 686 if (strcmp(strval, "none") == 0) 687 break; 688 689 if (strval[0] != '/') { 690 error = SET_ERROR(EINVAL); 691 break; 692 } 693 694 slash = strrchr(strval, '/'); 695 ASSERT(slash != NULL); 696 697 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 698 strcmp(slash, "/..") == 0) 699 error = SET_ERROR(EINVAL); 700 break; 701 702 case ZPOOL_PROP_COMMENT: 703 if ((error = nvpair_value_string(elem, &strval)) != 0) 704 break; 705 for (check = strval; *check != '\0'; check++) { 706 if (!isprint(*check)) { 707 error = SET_ERROR(EINVAL); 708 break; 709 } 710 } 711 if (strlen(strval) > ZPROP_MAX_COMMENT) 712 error = SET_ERROR(E2BIG); 713 break; 714 715 default: 716 break; 717 } 718 719 if (error) 720 break; 721 } 722 723 (void) nvlist_remove_all(props, 724 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 725 726 if (!error && reset_bootfs) { 727 error = nvlist_remove(props, 728 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 729 730 if (!error) { 731 error = nvlist_add_uint64(props, 732 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 733 } 734 } 735 736 return (error); 737 } 738 739 void 740 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 741 { 742 char *cachefile; 743 spa_config_dirent_t *dp; 744 745 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 746 &cachefile) != 0) 747 return; 748 749 dp = kmem_alloc(sizeof (spa_config_dirent_t), 750 KM_SLEEP); 751 752 if (cachefile[0] == '\0') 753 dp->scd_path = spa_strdup(spa_config_path); 754 else if (strcmp(cachefile, "none") == 0) 755 dp->scd_path = NULL; 756 else 757 dp->scd_path = spa_strdup(cachefile); 758 759 list_insert_head(&spa->spa_config_list, dp); 760 if (need_sync) 761 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 762 } 763 764 int 765 spa_prop_set(spa_t *spa, nvlist_t *nvp) 766 { 767 int error; 768 nvpair_t *elem = NULL; 769 boolean_t need_sync = B_FALSE; 770 771 if ((error = spa_prop_validate(spa, nvp)) != 0) 772 return (error); 773 774 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 775 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 776 777 if (prop == ZPOOL_PROP_CACHEFILE || 778 prop == ZPOOL_PROP_ALTROOT || 779 prop == ZPOOL_PROP_READONLY) 780 continue; 781 782 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 783 uint64_t ver; 784 785 if (prop == ZPOOL_PROP_VERSION) { 786 VERIFY(nvpair_value_uint64(elem, &ver) == 0); 787 } else { 788 ASSERT(zpool_prop_feature(nvpair_name(elem))); 789 ver = SPA_VERSION_FEATURES; 790 need_sync = B_TRUE; 791 } 792 793 /* Save time if the version is already set. */ 794 if (ver == spa_version(spa)) 795 continue; 796 797 /* 798 * In addition to the pool directory object, we might 799 * create the pool properties object, the features for 800 * read object, the features for write object, or the 801 * feature descriptions object. 802 */ 803 error = dsl_sync_task(spa->spa_name, NULL, 804 spa_sync_version, &ver, 805 6, ZFS_SPACE_CHECK_RESERVED); 806 if (error) 807 return (error); 808 continue; 809 } 810 811 need_sync = B_TRUE; 812 break; 813 } 814 815 if (need_sync) { 816 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 817 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 818 } 819 820 return (0); 821 } 822 823 /* 824 * If the bootfs property value is dsobj, clear it. 825 */ 826 void 827 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 828 { 829 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 830 VERIFY(zap_remove(spa->spa_meta_objset, 831 spa->spa_pool_props_object, 832 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 833 spa->spa_bootfs = 0; 834 } 835 } 836 837 /*ARGSUSED*/ 838 static int 839 spa_change_guid_check(void *arg, dmu_tx_t *tx) 840 { 841 uint64_t *newguid __maybe_unused = arg; 842 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 843 vdev_t *rvd = spa->spa_root_vdev; 844 uint64_t vdev_state; 845 846 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 847 int error = (spa_has_checkpoint(spa)) ? 848 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 849 return (SET_ERROR(error)); 850 } 851 852 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 853 vdev_state = rvd->vdev_state; 854 spa_config_exit(spa, SCL_STATE, FTAG); 855 856 if (vdev_state != VDEV_STATE_HEALTHY) 857 return (SET_ERROR(ENXIO)); 858 859 ASSERT3U(spa_guid(spa), !=, *newguid); 860 861 return (0); 862 } 863 864 static void 865 spa_change_guid_sync(void *arg, dmu_tx_t *tx) 866 { 867 uint64_t *newguid = arg; 868 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 869 uint64_t oldguid; 870 vdev_t *rvd = spa->spa_root_vdev; 871 872 oldguid = spa_guid(spa); 873 874 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 875 rvd->vdev_guid = *newguid; 876 rvd->vdev_guid_sum += (*newguid - oldguid); 877 vdev_config_dirty(rvd); 878 spa_config_exit(spa, SCL_STATE, FTAG); 879 880 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 881 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 882 } 883 884 /* 885 * Change the GUID for the pool. This is done so that we can later 886 * re-import a pool built from a clone of our own vdevs. We will modify 887 * the root vdev's guid, our own pool guid, and then mark all of our 888 * vdevs dirty. Note that we must make sure that all our vdevs are 889 * online when we do this, or else any vdevs that weren't present 890 * would be orphaned from our pool. We are also going to issue a 891 * sysevent to update any watchers. 892 */ 893 int 894 spa_change_guid(spa_t *spa) 895 { 896 int error; 897 uint64_t guid; 898 899 mutex_enter(&spa->spa_vdev_top_lock); 900 mutex_enter(&spa_namespace_lock); 901 guid = spa_generate_guid(NULL); 902 903 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 904 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 905 906 if (error == 0) { 907 spa_write_cachefile(spa, B_FALSE, B_TRUE); 908 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 909 } 910 911 mutex_exit(&spa_namespace_lock); 912 mutex_exit(&spa->spa_vdev_top_lock); 913 914 return (error); 915 } 916 917 /* 918 * ========================================================================== 919 * SPA state manipulation (open/create/destroy/import/export) 920 * ========================================================================== 921 */ 922 923 static int 924 spa_error_entry_compare(const void *a, const void *b) 925 { 926 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 927 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 928 int ret; 929 930 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 931 sizeof (zbookmark_phys_t)); 932 933 return (TREE_ISIGN(ret)); 934 } 935 936 /* 937 * Utility function which retrieves copies of the current logs and 938 * re-initializes them in the process. 939 */ 940 void 941 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 942 { 943 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 944 945 bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t)); 946 bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t)); 947 948 avl_create(&spa->spa_errlist_scrub, 949 spa_error_entry_compare, sizeof (spa_error_entry_t), 950 offsetof(spa_error_entry_t, se_avl)); 951 avl_create(&spa->spa_errlist_last, 952 spa_error_entry_compare, sizeof (spa_error_entry_t), 953 offsetof(spa_error_entry_t, se_avl)); 954 } 955 956 static void 957 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 958 { 959 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 960 enum zti_modes mode = ztip->zti_mode; 961 uint_t value = ztip->zti_value; 962 uint_t count = ztip->zti_count; 963 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 964 uint_t cpus, flags = TASKQ_DYNAMIC; 965 boolean_t batch = B_FALSE; 966 967 switch (mode) { 968 case ZTI_MODE_FIXED: 969 ASSERT3U(value, >, 0); 970 break; 971 972 case ZTI_MODE_BATCH: 973 batch = B_TRUE; 974 flags |= TASKQ_THREADS_CPU_PCT; 975 value = MIN(zio_taskq_batch_pct, 100); 976 break; 977 978 case ZTI_MODE_SCALE: 979 flags |= TASKQ_THREADS_CPU_PCT; 980 /* 981 * We want more taskqs to reduce lock contention, but we want 982 * less for better request ordering and CPU utilization. 983 */ 984 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 985 if (zio_taskq_batch_tpq > 0) { 986 count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) / 987 zio_taskq_batch_tpq); 988 } else { 989 /* 990 * Prefer 6 threads per taskq, but no more taskqs 991 * than threads in them on large systems. For 80%: 992 * 993 * taskq taskq total 994 * cpus taskqs percent threads threads 995 * ------- ------- ------- ------- ------- 996 * 1 1 80% 1 1 997 * 2 1 80% 1 1 998 * 4 1 80% 3 3 999 * 8 2 40% 3 6 1000 * 16 3 27% 4 12 1001 * 32 5 16% 5 25 1002 * 64 7 11% 7 49 1003 * 128 10 8% 10 100 1004 * 256 14 6% 15 210 1005 */ 1006 count = 1 + cpus / 6; 1007 while (count * count > cpus) 1008 count--; 1009 } 1010 /* Limit each taskq within 100% to not trigger assertion. */ 1011 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1012 value = (zio_taskq_batch_pct + count / 2) / count; 1013 break; 1014 1015 case ZTI_MODE_NULL: 1016 tqs->stqs_count = 0; 1017 tqs->stqs_taskq = NULL; 1018 return; 1019 1020 default: 1021 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1022 "spa_activate()", 1023 zio_type_name[t], zio_taskq_types[q], mode, value); 1024 break; 1025 } 1026 1027 ASSERT3U(count, >, 0); 1028 tqs->stqs_count = count; 1029 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1030 1031 for (uint_t i = 0; i < count; i++) { 1032 taskq_t *tq; 1033 char name[32]; 1034 1035 if (count > 1) 1036 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1037 zio_type_name[t], zio_taskq_types[q], i); 1038 else 1039 (void) snprintf(name, sizeof (name), "%s_%s", 1040 zio_type_name[t], zio_taskq_types[q]); 1041 1042 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1043 if (batch) 1044 flags |= TASKQ_DC_BATCH; 1045 1046 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1047 spa->spa_proc, zio_taskq_basedc, flags); 1048 } else { 1049 pri_t pri = maxclsyspri; 1050 /* 1051 * The write issue taskq can be extremely CPU 1052 * intensive. Run it at slightly less important 1053 * priority than the other taskqs. 1054 * 1055 * Under Linux and FreeBSD this means incrementing 1056 * the priority value as opposed to platforms like 1057 * illumos where it should be decremented. 1058 * 1059 * On FreeBSD, if priorities divided by four (RQ_PPQ) 1060 * are equal then a difference between them is 1061 * insignificant. 1062 */ 1063 if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) { 1064 #if defined(__linux__) 1065 pri++; 1066 #elif defined(__FreeBSD__) 1067 pri += 4; 1068 #else 1069 #error "unknown OS" 1070 #endif 1071 } 1072 tq = taskq_create_proc(name, value, pri, 50, 1073 INT_MAX, spa->spa_proc, flags); 1074 } 1075 1076 tqs->stqs_taskq[i] = tq; 1077 } 1078 } 1079 1080 static void 1081 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1082 { 1083 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1084 1085 if (tqs->stqs_taskq == NULL) { 1086 ASSERT3U(tqs->stqs_count, ==, 0); 1087 return; 1088 } 1089 1090 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1091 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1092 taskq_destroy(tqs->stqs_taskq[i]); 1093 } 1094 1095 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1096 tqs->stqs_taskq = NULL; 1097 } 1098 1099 /* 1100 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1101 * Note that a type may have multiple discrete taskqs to avoid lock contention 1102 * on the taskq itself. In that case we choose which taskq at random by using 1103 * the low bits of gethrtime(). 1104 */ 1105 void 1106 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1107 task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent) 1108 { 1109 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1110 taskq_t *tq; 1111 1112 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1113 ASSERT3U(tqs->stqs_count, !=, 0); 1114 1115 if (tqs->stqs_count == 1) { 1116 tq = tqs->stqs_taskq[0]; 1117 } else { 1118 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1119 } 1120 1121 taskq_dispatch_ent(tq, func, arg, flags, ent); 1122 } 1123 1124 /* 1125 * Same as spa_taskq_dispatch_ent() but block on the task until completion. 1126 */ 1127 void 1128 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1129 task_func_t *func, void *arg, uint_t flags) 1130 { 1131 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1132 taskq_t *tq; 1133 taskqid_t id; 1134 1135 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1136 ASSERT3U(tqs->stqs_count, !=, 0); 1137 1138 if (tqs->stqs_count == 1) { 1139 tq = tqs->stqs_taskq[0]; 1140 } else { 1141 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1142 } 1143 1144 id = taskq_dispatch(tq, func, arg, flags); 1145 if (id) 1146 taskq_wait_id(tq, id); 1147 } 1148 1149 static void 1150 spa_create_zio_taskqs(spa_t *spa) 1151 { 1152 for (int t = 0; t < ZIO_TYPES; t++) { 1153 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1154 spa_taskqs_init(spa, t, q); 1155 } 1156 } 1157 } 1158 1159 /* 1160 * Disabled until spa_thread() can be adapted for Linux. 1161 */ 1162 #undef HAVE_SPA_THREAD 1163 1164 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1165 static void 1166 spa_thread(void *arg) 1167 { 1168 psetid_t zio_taskq_psrset_bind = PS_NONE; 1169 callb_cpr_t cprinfo; 1170 1171 spa_t *spa = arg; 1172 user_t *pu = PTOU(curproc); 1173 1174 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1175 spa->spa_name); 1176 1177 ASSERT(curproc != &p0); 1178 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1179 "zpool-%s", spa->spa_name); 1180 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1181 1182 /* bind this thread to the requested psrset */ 1183 if (zio_taskq_psrset_bind != PS_NONE) { 1184 pool_lock(); 1185 mutex_enter(&cpu_lock); 1186 mutex_enter(&pidlock); 1187 mutex_enter(&curproc->p_lock); 1188 1189 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1190 0, NULL, NULL) == 0) { 1191 curthread->t_bind_pset = zio_taskq_psrset_bind; 1192 } else { 1193 cmn_err(CE_WARN, 1194 "Couldn't bind process for zfs pool \"%s\" to " 1195 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1196 } 1197 1198 mutex_exit(&curproc->p_lock); 1199 mutex_exit(&pidlock); 1200 mutex_exit(&cpu_lock); 1201 pool_unlock(); 1202 } 1203 1204 if (zio_taskq_sysdc) { 1205 sysdc_thread_enter(curthread, 100, 0); 1206 } 1207 1208 spa->spa_proc = curproc; 1209 spa->spa_did = curthread->t_did; 1210 1211 spa_create_zio_taskqs(spa); 1212 1213 mutex_enter(&spa->spa_proc_lock); 1214 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1215 1216 spa->spa_proc_state = SPA_PROC_ACTIVE; 1217 cv_broadcast(&spa->spa_proc_cv); 1218 1219 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1220 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1221 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1222 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1223 1224 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1225 spa->spa_proc_state = SPA_PROC_GONE; 1226 spa->spa_proc = &p0; 1227 cv_broadcast(&spa->spa_proc_cv); 1228 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1229 1230 mutex_enter(&curproc->p_lock); 1231 lwp_exit(); 1232 } 1233 #endif 1234 1235 /* 1236 * Activate an uninitialized pool. 1237 */ 1238 static void 1239 spa_activate(spa_t *spa, spa_mode_t mode) 1240 { 1241 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1242 1243 spa->spa_state = POOL_STATE_ACTIVE; 1244 spa->spa_mode = mode; 1245 1246 spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops); 1247 spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops); 1248 spa->spa_embedded_log_class = 1249 metaslab_class_create(spa, zfs_metaslab_ops); 1250 spa->spa_special_class = metaslab_class_create(spa, zfs_metaslab_ops); 1251 spa->spa_dedup_class = metaslab_class_create(spa, zfs_metaslab_ops); 1252 1253 /* Try to create a covering process */ 1254 mutex_enter(&spa->spa_proc_lock); 1255 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1256 ASSERT(spa->spa_proc == &p0); 1257 spa->spa_did = 0; 1258 1259 #ifdef HAVE_SPA_THREAD 1260 /* Only create a process if we're going to be around a while. */ 1261 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1262 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1263 NULL, 0) == 0) { 1264 spa->spa_proc_state = SPA_PROC_CREATED; 1265 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1266 cv_wait(&spa->spa_proc_cv, 1267 &spa->spa_proc_lock); 1268 } 1269 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1270 ASSERT(spa->spa_proc != &p0); 1271 ASSERT(spa->spa_did != 0); 1272 } else { 1273 #ifdef _KERNEL 1274 cmn_err(CE_WARN, 1275 "Couldn't create process for zfs pool \"%s\"\n", 1276 spa->spa_name); 1277 #endif 1278 } 1279 } 1280 #endif /* HAVE_SPA_THREAD */ 1281 mutex_exit(&spa->spa_proc_lock); 1282 1283 /* If we didn't create a process, we need to create our taskqs. */ 1284 if (spa->spa_proc == &p0) { 1285 spa_create_zio_taskqs(spa); 1286 } 1287 1288 for (size_t i = 0; i < TXG_SIZE; i++) { 1289 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1290 ZIO_FLAG_CANFAIL); 1291 } 1292 1293 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1294 offsetof(vdev_t, vdev_config_dirty_node)); 1295 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1296 offsetof(objset_t, os_evicting_node)); 1297 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1298 offsetof(vdev_t, vdev_state_dirty_node)); 1299 1300 txg_list_create(&spa->spa_vdev_txg_list, spa, 1301 offsetof(struct vdev, vdev_txg_node)); 1302 1303 avl_create(&spa->spa_errlist_scrub, 1304 spa_error_entry_compare, sizeof (spa_error_entry_t), 1305 offsetof(spa_error_entry_t, se_avl)); 1306 avl_create(&spa->spa_errlist_last, 1307 spa_error_entry_compare, sizeof (spa_error_entry_t), 1308 offsetof(spa_error_entry_t, se_avl)); 1309 1310 spa_keystore_init(&spa->spa_keystore); 1311 1312 /* 1313 * This taskq is used to perform zvol-minor-related tasks 1314 * asynchronously. This has several advantages, including easy 1315 * resolution of various deadlocks. 1316 * 1317 * The taskq must be single threaded to ensure tasks are always 1318 * processed in the order in which they were dispatched. 1319 * 1320 * A taskq per pool allows one to keep the pools independent. 1321 * This way if one pool is suspended, it will not impact another. 1322 * 1323 * The preferred location to dispatch a zvol minor task is a sync 1324 * task. In this context, there is easy access to the spa_t and minimal 1325 * error handling is required because the sync task must succeed. 1326 */ 1327 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1328 1, INT_MAX, 0); 1329 1330 /* 1331 * Taskq dedicated to prefetcher threads: this is used to prevent the 1332 * pool traverse code from monopolizing the global (and limited) 1333 * system_taskq by inappropriately scheduling long running tasks on it. 1334 */ 1335 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1336 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1337 1338 /* 1339 * The taskq to upgrade datasets in this pool. Currently used by 1340 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1341 */ 1342 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1343 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1344 } 1345 1346 /* 1347 * Opposite of spa_activate(). 1348 */ 1349 static void 1350 spa_deactivate(spa_t *spa) 1351 { 1352 ASSERT(spa->spa_sync_on == B_FALSE); 1353 ASSERT(spa->spa_dsl_pool == NULL); 1354 ASSERT(spa->spa_root_vdev == NULL); 1355 ASSERT(spa->spa_async_zio_root == NULL); 1356 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1357 1358 spa_evicting_os_wait(spa); 1359 1360 if (spa->spa_zvol_taskq) { 1361 taskq_destroy(spa->spa_zvol_taskq); 1362 spa->spa_zvol_taskq = NULL; 1363 } 1364 1365 if (spa->spa_prefetch_taskq) { 1366 taskq_destroy(spa->spa_prefetch_taskq); 1367 spa->spa_prefetch_taskq = NULL; 1368 } 1369 1370 if (spa->spa_upgrade_taskq) { 1371 taskq_destroy(spa->spa_upgrade_taskq); 1372 spa->spa_upgrade_taskq = NULL; 1373 } 1374 1375 txg_list_destroy(&spa->spa_vdev_txg_list); 1376 1377 list_destroy(&spa->spa_config_dirty_list); 1378 list_destroy(&spa->spa_evicting_os_list); 1379 list_destroy(&spa->spa_state_dirty_list); 1380 1381 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 1382 1383 for (int t = 0; t < ZIO_TYPES; t++) { 1384 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1385 spa_taskqs_fini(spa, t, q); 1386 } 1387 } 1388 1389 for (size_t i = 0; i < TXG_SIZE; i++) { 1390 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1391 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1392 spa->spa_txg_zio[i] = NULL; 1393 } 1394 1395 metaslab_class_destroy(spa->spa_normal_class); 1396 spa->spa_normal_class = NULL; 1397 1398 metaslab_class_destroy(spa->spa_log_class); 1399 spa->spa_log_class = NULL; 1400 1401 metaslab_class_destroy(spa->spa_embedded_log_class); 1402 spa->spa_embedded_log_class = NULL; 1403 1404 metaslab_class_destroy(spa->spa_special_class); 1405 spa->spa_special_class = NULL; 1406 1407 metaslab_class_destroy(spa->spa_dedup_class); 1408 spa->spa_dedup_class = NULL; 1409 1410 /* 1411 * If this was part of an import or the open otherwise failed, we may 1412 * still have errors left in the queues. Empty them just in case. 1413 */ 1414 spa_errlog_drain(spa); 1415 avl_destroy(&spa->spa_errlist_scrub); 1416 avl_destroy(&spa->spa_errlist_last); 1417 1418 spa_keystore_fini(&spa->spa_keystore); 1419 1420 spa->spa_state = POOL_STATE_UNINITIALIZED; 1421 1422 mutex_enter(&spa->spa_proc_lock); 1423 if (spa->spa_proc_state != SPA_PROC_NONE) { 1424 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1425 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 1426 cv_broadcast(&spa->spa_proc_cv); 1427 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 1428 ASSERT(spa->spa_proc != &p0); 1429 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1430 } 1431 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 1432 spa->spa_proc_state = SPA_PROC_NONE; 1433 } 1434 ASSERT(spa->spa_proc == &p0); 1435 mutex_exit(&spa->spa_proc_lock); 1436 1437 /* 1438 * We want to make sure spa_thread() has actually exited the ZFS 1439 * module, so that the module can't be unloaded out from underneath 1440 * it. 1441 */ 1442 if (spa->spa_did != 0) { 1443 thread_join(spa->spa_did); 1444 spa->spa_did = 0; 1445 } 1446 } 1447 1448 /* 1449 * Verify a pool configuration, and construct the vdev tree appropriately. This 1450 * will create all the necessary vdevs in the appropriate layout, with each vdev 1451 * in the CLOSED state. This will prep the pool before open/creation/import. 1452 * All vdev validation is done by the vdev_alloc() routine. 1453 */ 1454 int 1455 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 1456 uint_t id, int atype) 1457 { 1458 nvlist_t **child; 1459 uint_t children; 1460 int error; 1461 1462 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 1463 return (error); 1464 1465 if ((*vdp)->vdev_ops->vdev_op_leaf) 1466 return (0); 1467 1468 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 1469 &child, &children); 1470 1471 if (error == ENOENT) 1472 return (0); 1473 1474 if (error) { 1475 vdev_free(*vdp); 1476 *vdp = NULL; 1477 return (SET_ERROR(EINVAL)); 1478 } 1479 1480 for (int c = 0; c < children; c++) { 1481 vdev_t *vd; 1482 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 1483 atype)) != 0) { 1484 vdev_free(*vdp); 1485 *vdp = NULL; 1486 return (error); 1487 } 1488 } 1489 1490 ASSERT(*vdp != NULL); 1491 1492 return (0); 1493 } 1494 1495 static boolean_t 1496 spa_should_flush_logs_on_unload(spa_t *spa) 1497 { 1498 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 1499 return (B_FALSE); 1500 1501 if (!spa_writeable(spa)) 1502 return (B_FALSE); 1503 1504 if (!spa->spa_sync_on) 1505 return (B_FALSE); 1506 1507 if (spa_state(spa) != POOL_STATE_EXPORTED) 1508 return (B_FALSE); 1509 1510 if (zfs_keep_log_spacemaps_at_export) 1511 return (B_FALSE); 1512 1513 return (B_TRUE); 1514 } 1515 1516 /* 1517 * Opens a transaction that will set the flag that will instruct 1518 * spa_sync to attempt to flush all the metaslabs for that txg. 1519 */ 1520 static void 1521 spa_unload_log_sm_flush_all(spa_t *spa) 1522 { 1523 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 1524 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 1525 1526 ASSERT3U(spa->spa_log_flushall_txg, ==, 0); 1527 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); 1528 1529 dmu_tx_commit(tx); 1530 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 1531 } 1532 1533 static void 1534 spa_unload_log_sm_metadata(spa_t *spa) 1535 { 1536 void *cookie = NULL; 1537 spa_log_sm_t *sls; 1538 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 1539 &cookie)) != NULL) { 1540 VERIFY0(sls->sls_mscount); 1541 kmem_free(sls, sizeof (spa_log_sm_t)); 1542 } 1543 1544 for (log_summary_entry_t *e = list_head(&spa->spa_log_summary); 1545 e != NULL; e = list_head(&spa->spa_log_summary)) { 1546 VERIFY0(e->lse_mscount); 1547 list_remove(&spa->spa_log_summary, e); 1548 kmem_free(e, sizeof (log_summary_entry_t)); 1549 } 1550 1551 spa->spa_unflushed_stats.sus_nblocks = 0; 1552 spa->spa_unflushed_stats.sus_memused = 0; 1553 spa->spa_unflushed_stats.sus_blocklimit = 0; 1554 } 1555 1556 static void 1557 spa_destroy_aux_threads(spa_t *spa) 1558 { 1559 if (spa->spa_condense_zthr != NULL) { 1560 zthr_destroy(spa->spa_condense_zthr); 1561 spa->spa_condense_zthr = NULL; 1562 } 1563 if (spa->spa_checkpoint_discard_zthr != NULL) { 1564 zthr_destroy(spa->spa_checkpoint_discard_zthr); 1565 spa->spa_checkpoint_discard_zthr = NULL; 1566 } 1567 if (spa->spa_livelist_delete_zthr != NULL) { 1568 zthr_destroy(spa->spa_livelist_delete_zthr); 1569 spa->spa_livelist_delete_zthr = NULL; 1570 } 1571 if (spa->spa_livelist_condense_zthr != NULL) { 1572 zthr_destroy(spa->spa_livelist_condense_zthr); 1573 spa->spa_livelist_condense_zthr = NULL; 1574 } 1575 } 1576 1577 /* 1578 * Opposite of spa_load(). 1579 */ 1580 static void 1581 spa_unload(spa_t *spa) 1582 { 1583 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1584 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 1585 1586 spa_import_progress_remove(spa_guid(spa)); 1587 spa_load_note(spa, "UNLOADING"); 1588 1589 spa_wake_waiters(spa); 1590 1591 /* 1592 * If the log space map feature is enabled and the pool is getting 1593 * exported (but not destroyed), we want to spend some time flushing 1594 * as many metaslabs as we can in an attempt to destroy log space 1595 * maps and save import time. 1596 */ 1597 if (spa_should_flush_logs_on_unload(spa)) 1598 spa_unload_log_sm_flush_all(spa); 1599 1600 /* 1601 * Stop async tasks. 1602 */ 1603 spa_async_suspend(spa); 1604 1605 if (spa->spa_root_vdev) { 1606 vdev_t *root_vdev = spa->spa_root_vdev; 1607 vdev_initialize_stop_all(root_vdev, VDEV_INITIALIZE_ACTIVE); 1608 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 1609 vdev_autotrim_stop_all(spa); 1610 vdev_rebuild_stop_all(spa); 1611 } 1612 1613 /* 1614 * Stop syncing. 1615 */ 1616 if (spa->spa_sync_on) { 1617 txg_sync_stop(spa->spa_dsl_pool); 1618 spa->spa_sync_on = B_FALSE; 1619 } 1620 1621 /* 1622 * This ensures that there is no async metaslab prefetching 1623 * while we attempt to unload the spa. 1624 */ 1625 if (spa->spa_root_vdev != NULL) { 1626 for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++) { 1627 vdev_t *vc = spa->spa_root_vdev->vdev_child[c]; 1628 if (vc->vdev_mg != NULL) 1629 taskq_wait(vc->vdev_mg->mg_taskq); 1630 } 1631 } 1632 1633 if (spa->spa_mmp.mmp_thread) 1634 mmp_thread_stop(spa); 1635 1636 /* 1637 * Wait for any outstanding async I/O to complete. 1638 */ 1639 if (spa->spa_async_zio_root != NULL) { 1640 for (int i = 0; i < max_ncpus; i++) 1641 (void) zio_wait(spa->spa_async_zio_root[i]); 1642 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 1643 spa->spa_async_zio_root = NULL; 1644 } 1645 1646 if (spa->spa_vdev_removal != NULL) { 1647 spa_vdev_removal_destroy(spa->spa_vdev_removal); 1648 spa->spa_vdev_removal = NULL; 1649 } 1650 1651 spa_destroy_aux_threads(spa); 1652 1653 spa_condense_fini(spa); 1654 1655 bpobj_close(&spa->spa_deferred_bpobj); 1656 1657 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 1658 1659 /* 1660 * Close all vdevs. 1661 */ 1662 if (spa->spa_root_vdev) 1663 vdev_free(spa->spa_root_vdev); 1664 ASSERT(spa->spa_root_vdev == NULL); 1665 1666 /* 1667 * Close the dsl pool. 1668 */ 1669 if (spa->spa_dsl_pool) { 1670 dsl_pool_close(spa->spa_dsl_pool); 1671 spa->spa_dsl_pool = NULL; 1672 spa->spa_meta_objset = NULL; 1673 } 1674 1675 ddt_unload(spa); 1676 spa_unload_log_sm_metadata(spa); 1677 1678 /* 1679 * Drop and purge level 2 cache 1680 */ 1681 spa_l2cache_drop(spa); 1682 1683 for (int i = 0; i < spa->spa_spares.sav_count; i++) 1684 vdev_free(spa->spa_spares.sav_vdevs[i]); 1685 if (spa->spa_spares.sav_vdevs) { 1686 kmem_free(spa->spa_spares.sav_vdevs, 1687 spa->spa_spares.sav_count * sizeof (void *)); 1688 spa->spa_spares.sav_vdevs = NULL; 1689 } 1690 if (spa->spa_spares.sav_config) { 1691 nvlist_free(spa->spa_spares.sav_config); 1692 spa->spa_spares.sav_config = NULL; 1693 } 1694 spa->spa_spares.sav_count = 0; 1695 1696 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1697 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 1698 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 1699 } 1700 if (spa->spa_l2cache.sav_vdevs) { 1701 kmem_free(spa->spa_l2cache.sav_vdevs, 1702 spa->spa_l2cache.sav_count * sizeof (void *)); 1703 spa->spa_l2cache.sav_vdevs = NULL; 1704 } 1705 if (spa->spa_l2cache.sav_config) { 1706 nvlist_free(spa->spa_l2cache.sav_config); 1707 spa->spa_l2cache.sav_config = NULL; 1708 } 1709 spa->spa_l2cache.sav_count = 0; 1710 1711 spa->spa_async_suspended = 0; 1712 1713 spa->spa_indirect_vdevs_loaded = B_FALSE; 1714 1715 if (spa->spa_comment != NULL) { 1716 spa_strfree(spa->spa_comment); 1717 spa->spa_comment = NULL; 1718 } 1719 if (spa->spa_compatibility != NULL) { 1720 spa_strfree(spa->spa_compatibility); 1721 spa->spa_compatibility = NULL; 1722 } 1723 1724 spa_config_exit(spa, SCL_ALL, spa); 1725 } 1726 1727 /* 1728 * Load (or re-load) the current list of vdevs describing the active spares for 1729 * this pool. When this is called, we have some form of basic information in 1730 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 1731 * then re-generate a more complete list including status information. 1732 */ 1733 void 1734 spa_load_spares(spa_t *spa) 1735 { 1736 nvlist_t **spares; 1737 uint_t nspares; 1738 int i; 1739 vdev_t *vd, *tvd; 1740 1741 #ifndef _KERNEL 1742 /* 1743 * zdb opens both the current state of the pool and the 1744 * checkpointed state (if present), with a different spa_t. 1745 * 1746 * As spare vdevs are shared among open pools, we skip loading 1747 * them when we load the checkpointed state of the pool. 1748 */ 1749 if (!spa_writeable(spa)) 1750 return; 1751 #endif 1752 1753 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1754 1755 /* 1756 * First, close and free any existing spare vdevs. 1757 */ 1758 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1759 vd = spa->spa_spares.sav_vdevs[i]; 1760 1761 /* Undo the call to spa_activate() below */ 1762 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1763 B_FALSE)) != NULL && tvd->vdev_isspare) 1764 spa_spare_remove(tvd); 1765 vdev_close(vd); 1766 vdev_free(vd); 1767 } 1768 1769 if (spa->spa_spares.sav_vdevs) 1770 kmem_free(spa->spa_spares.sav_vdevs, 1771 spa->spa_spares.sav_count * sizeof (void *)); 1772 1773 if (spa->spa_spares.sav_config == NULL) 1774 nspares = 0; 1775 else 1776 VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 1777 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 1778 1779 spa->spa_spares.sav_count = (int)nspares; 1780 spa->spa_spares.sav_vdevs = NULL; 1781 1782 if (nspares == 0) 1783 return; 1784 1785 /* 1786 * Construct the array of vdevs, opening them to get status in the 1787 * process. For each spare, there is potentially two different vdev_t 1788 * structures associated with it: one in the list of spares (used only 1789 * for basic validation purposes) and one in the active vdev 1790 * configuration (if it's spared in). During this phase we open and 1791 * validate each vdev on the spare list. If the vdev also exists in the 1792 * active configuration, then we also mark this vdev as an active spare. 1793 */ 1794 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 1795 KM_SLEEP); 1796 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1797 VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, 1798 VDEV_ALLOC_SPARE) == 0); 1799 ASSERT(vd != NULL); 1800 1801 spa->spa_spares.sav_vdevs[i] = vd; 1802 1803 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1804 B_FALSE)) != NULL) { 1805 if (!tvd->vdev_isspare) 1806 spa_spare_add(tvd); 1807 1808 /* 1809 * We only mark the spare active if we were successfully 1810 * able to load the vdev. Otherwise, importing a pool 1811 * with a bad active spare would result in strange 1812 * behavior, because multiple pool would think the spare 1813 * is actively in use. 1814 * 1815 * There is a vulnerability here to an equally bizarre 1816 * circumstance, where a dead active spare is later 1817 * brought back to life (onlined or otherwise). Given 1818 * the rarity of this scenario, and the extra complexity 1819 * it adds, we ignore the possibility. 1820 */ 1821 if (!vdev_is_dead(tvd)) 1822 spa_spare_activate(tvd); 1823 } 1824 1825 vd->vdev_top = vd; 1826 vd->vdev_aux = &spa->spa_spares; 1827 1828 if (vdev_open(vd) != 0) 1829 continue; 1830 1831 if (vdev_validate_aux(vd) == 0) 1832 spa_spare_add(vd); 1833 } 1834 1835 /* 1836 * Recompute the stashed list of spares, with status information 1837 * this time. 1838 */ 1839 VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, 1840 DATA_TYPE_NVLIST_ARRAY) == 0); 1841 1842 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 1843 KM_SLEEP); 1844 for (i = 0; i < spa->spa_spares.sav_count; i++) 1845 spares[i] = vdev_config_generate(spa, 1846 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 1847 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 1848 ZPOOL_CONFIG_SPARES, spares, spa->spa_spares.sav_count) == 0); 1849 for (i = 0; i < spa->spa_spares.sav_count; i++) 1850 nvlist_free(spares[i]); 1851 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 1852 } 1853 1854 /* 1855 * Load (or re-load) the current list of vdevs describing the active l2cache for 1856 * this pool. When this is called, we have some form of basic information in 1857 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 1858 * then re-generate a more complete list including status information. 1859 * Devices which are already active have their details maintained, and are 1860 * not re-opened. 1861 */ 1862 void 1863 spa_load_l2cache(spa_t *spa) 1864 { 1865 nvlist_t **l2cache = NULL; 1866 uint_t nl2cache; 1867 int i, j, oldnvdevs; 1868 uint64_t guid; 1869 vdev_t *vd, **oldvdevs, **newvdevs; 1870 spa_aux_vdev_t *sav = &spa->spa_l2cache; 1871 1872 #ifndef _KERNEL 1873 /* 1874 * zdb opens both the current state of the pool and the 1875 * checkpointed state (if present), with a different spa_t. 1876 * 1877 * As L2 caches are part of the ARC which is shared among open 1878 * pools, we skip loading them when we load the checkpointed 1879 * state of the pool. 1880 */ 1881 if (!spa_writeable(spa)) 1882 return; 1883 #endif 1884 1885 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1886 1887 oldvdevs = sav->sav_vdevs; 1888 oldnvdevs = sav->sav_count; 1889 sav->sav_vdevs = NULL; 1890 sav->sav_count = 0; 1891 1892 if (sav->sav_config == NULL) { 1893 nl2cache = 0; 1894 newvdevs = NULL; 1895 goto out; 1896 } 1897 1898 VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, 1899 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 1900 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 1901 1902 /* 1903 * Process new nvlist of vdevs. 1904 */ 1905 for (i = 0; i < nl2cache; i++) { 1906 VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID, 1907 &guid) == 0); 1908 1909 newvdevs[i] = NULL; 1910 for (j = 0; j < oldnvdevs; j++) { 1911 vd = oldvdevs[j]; 1912 if (vd != NULL && guid == vd->vdev_guid) { 1913 /* 1914 * Retain previous vdev for add/remove ops. 1915 */ 1916 newvdevs[i] = vd; 1917 oldvdevs[j] = NULL; 1918 break; 1919 } 1920 } 1921 1922 if (newvdevs[i] == NULL) { 1923 /* 1924 * Create new vdev 1925 */ 1926 VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 1927 VDEV_ALLOC_L2CACHE) == 0); 1928 ASSERT(vd != NULL); 1929 newvdevs[i] = vd; 1930 1931 /* 1932 * Commit this vdev as an l2cache device, 1933 * even if it fails to open. 1934 */ 1935 spa_l2cache_add(vd); 1936 1937 vd->vdev_top = vd; 1938 vd->vdev_aux = sav; 1939 1940 spa_l2cache_activate(vd); 1941 1942 if (vdev_open(vd) != 0) 1943 continue; 1944 1945 (void) vdev_validate_aux(vd); 1946 1947 if (!vdev_is_dead(vd)) 1948 l2arc_add_vdev(spa, vd); 1949 1950 /* 1951 * Upon cache device addition to a pool or pool 1952 * creation with a cache device or if the header 1953 * of the device is invalid we issue an async 1954 * TRIM command for the whole device which will 1955 * execute if l2arc_trim_ahead > 0. 1956 */ 1957 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 1958 } 1959 } 1960 1961 sav->sav_vdevs = newvdevs; 1962 sav->sav_count = (int)nl2cache; 1963 1964 /* 1965 * Recompute the stashed list of l2cache devices, with status 1966 * information this time. 1967 */ 1968 VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 1969 DATA_TYPE_NVLIST_ARRAY) == 0); 1970 1971 if (sav->sav_count > 0) 1972 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 1973 KM_SLEEP); 1974 for (i = 0; i < sav->sav_count; i++) 1975 l2cache[i] = vdev_config_generate(spa, 1976 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 1977 VERIFY(nvlist_add_nvlist_array(sav->sav_config, 1978 ZPOOL_CONFIG_L2CACHE, l2cache, sav->sav_count) == 0); 1979 1980 out: 1981 /* 1982 * Purge vdevs that were dropped 1983 */ 1984 for (i = 0; i < oldnvdevs; i++) { 1985 uint64_t pool; 1986 1987 vd = oldvdevs[i]; 1988 if (vd != NULL) { 1989 ASSERT(vd->vdev_isl2cache); 1990 1991 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 1992 pool != 0ULL && l2arc_vdev_present(vd)) 1993 l2arc_remove_vdev(vd); 1994 vdev_clear_stats(vd); 1995 vdev_free(vd); 1996 } 1997 } 1998 1999 if (oldvdevs) 2000 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2001 2002 for (i = 0; i < sav->sav_count; i++) 2003 nvlist_free(l2cache[i]); 2004 if (sav->sav_count) 2005 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2006 } 2007 2008 static int 2009 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2010 { 2011 dmu_buf_t *db; 2012 char *packed = NULL; 2013 size_t nvsize = 0; 2014 int error; 2015 *value = NULL; 2016 2017 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2018 if (error) 2019 return (error); 2020 2021 nvsize = *(uint64_t *)db->db_data; 2022 dmu_buf_rele(db, FTAG); 2023 2024 packed = vmem_alloc(nvsize, KM_SLEEP); 2025 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2026 DMU_READ_PREFETCH); 2027 if (error == 0) 2028 error = nvlist_unpack(packed, nvsize, value, 0); 2029 vmem_free(packed, nvsize); 2030 2031 return (error); 2032 } 2033 2034 /* 2035 * Concrete top-level vdevs that are not missing and are not logs. At every 2036 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2037 */ 2038 static uint64_t 2039 spa_healthy_core_tvds(spa_t *spa) 2040 { 2041 vdev_t *rvd = spa->spa_root_vdev; 2042 uint64_t tvds = 0; 2043 2044 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2045 vdev_t *vd = rvd->vdev_child[i]; 2046 if (vd->vdev_islog) 2047 continue; 2048 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2049 tvds++; 2050 } 2051 2052 return (tvds); 2053 } 2054 2055 /* 2056 * Checks to see if the given vdev could not be opened, in which case we post a 2057 * sysevent to notify the autoreplace code that the device has been removed. 2058 */ 2059 static void 2060 spa_check_removed(vdev_t *vd) 2061 { 2062 for (uint64_t c = 0; c < vd->vdev_children; c++) 2063 spa_check_removed(vd->vdev_child[c]); 2064 2065 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2066 vdev_is_concrete(vd)) { 2067 zfs_post_autoreplace(vd->vdev_spa, vd); 2068 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2069 } 2070 } 2071 2072 static int 2073 spa_check_for_missing_logs(spa_t *spa) 2074 { 2075 vdev_t *rvd = spa->spa_root_vdev; 2076 2077 /* 2078 * If we're doing a normal import, then build up any additional 2079 * diagnostic information about missing log devices. 2080 * We'll pass this up to the user for further processing. 2081 */ 2082 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2083 nvlist_t **child, *nv; 2084 uint64_t idx = 0; 2085 2086 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2087 KM_SLEEP); 2088 VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); 2089 2090 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2091 vdev_t *tvd = rvd->vdev_child[c]; 2092 2093 /* 2094 * We consider a device as missing only if it failed 2095 * to open (i.e. offline or faulted is not considered 2096 * as missing). 2097 */ 2098 if (tvd->vdev_islog && 2099 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2100 child[idx++] = vdev_config_generate(spa, tvd, 2101 B_FALSE, VDEV_CONFIG_MISSING); 2102 } 2103 } 2104 2105 if (idx > 0) { 2106 fnvlist_add_nvlist_array(nv, 2107 ZPOOL_CONFIG_CHILDREN, child, idx); 2108 fnvlist_add_nvlist(spa->spa_load_info, 2109 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2110 2111 for (uint64_t i = 0; i < idx; i++) 2112 nvlist_free(child[i]); 2113 } 2114 nvlist_free(nv); 2115 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2116 2117 if (idx > 0) { 2118 spa_load_failed(spa, "some log devices are missing"); 2119 vdev_dbgmsg_print_tree(rvd, 2); 2120 return (SET_ERROR(ENXIO)); 2121 } 2122 } else { 2123 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2124 vdev_t *tvd = rvd->vdev_child[c]; 2125 2126 if (tvd->vdev_islog && 2127 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2128 spa_set_log_state(spa, SPA_LOG_CLEAR); 2129 spa_load_note(spa, "some log devices are " 2130 "missing, ZIL is dropped."); 2131 vdev_dbgmsg_print_tree(rvd, 2); 2132 break; 2133 } 2134 } 2135 } 2136 2137 return (0); 2138 } 2139 2140 /* 2141 * Check for missing log devices 2142 */ 2143 static boolean_t 2144 spa_check_logs(spa_t *spa) 2145 { 2146 boolean_t rv = B_FALSE; 2147 dsl_pool_t *dp = spa_get_dsl(spa); 2148 2149 switch (spa->spa_log_state) { 2150 default: 2151 break; 2152 case SPA_LOG_MISSING: 2153 /* need to recheck in case slog has been restored */ 2154 case SPA_LOG_UNKNOWN: 2155 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2156 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2157 if (rv) 2158 spa_set_log_state(spa, SPA_LOG_MISSING); 2159 break; 2160 } 2161 return (rv); 2162 } 2163 2164 /* 2165 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2166 */ 2167 static boolean_t 2168 spa_passivate_log(spa_t *spa) 2169 { 2170 vdev_t *rvd = spa->spa_root_vdev; 2171 boolean_t slog_found = B_FALSE; 2172 2173 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2174 2175 for (int c = 0; c < rvd->vdev_children; c++) { 2176 vdev_t *tvd = rvd->vdev_child[c]; 2177 2178 if (tvd->vdev_islog) { 2179 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2180 metaslab_group_passivate(tvd->vdev_mg); 2181 slog_found = B_TRUE; 2182 } 2183 } 2184 2185 return (slog_found); 2186 } 2187 2188 /* 2189 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2190 */ 2191 static void 2192 spa_activate_log(spa_t *spa) 2193 { 2194 vdev_t *rvd = spa->spa_root_vdev; 2195 2196 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2197 2198 for (int c = 0; c < rvd->vdev_children; c++) { 2199 vdev_t *tvd = rvd->vdev_child[c]; 2200 2201 if (tvd->vdev_islog) { 2202 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2203 metaslab_group_activate(tvd->vdev_mg); 2204 } 2205 } 2206 } 2207 2208 int 2209 spa_reset_logs(spa_t *spa) 2210 { 2211 int error; 2212 2213 error = dmu_objset_find(spa_name(spa), zil_reset, 2214 NULL, DS_FIND_CHILDREN); 2215 if (error == 0) { 2216 /* 2217 * We successfully offlined the log device, sync out the 2218 * current txg so that the "stubby" block can be removed 2219 * by zil_sync(). 2220 */ 2221 txg_wait_synced(spa->spa_dsl_pool, 0); 2222 } 2223 return (error); 2224 } 2225 2226 static void 2227 spa_aux_check_removed(spa_aux_vdev_t *sav) 2228 { 2229 for (int i = 0; i < sav->sav_count; i++) 2230 spa_check_removed(sav->sav_vdevs[i]); 2231 } 2232 2233 void 2234 spa_claim_notify(zio_t *zio) 2235 { 2236 spa_t *spa = zio->io_spa; 2237 2238 if (zio->io_error) 2239 return; 2240 2241 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2242 if (spa->spa_claim_max_txg < zio->io_bp->blk_birth) 2243 spa->spa_claim_max_txg = zio->io_bp->blk_birth; 2244 mutex_exit(&spa->spa_props_lock); 2245 } 2246 2247 typedef struct spa_load_error { 2248 uint64_t sle_meta_count; 2249 uint64_t sle_data_count; 2250 } spa_load_error_t; 2251 2252 static void 2253 spa_load_verify_done(zio_t *zio) 2254 { 2255 blkptr_t *bp = zio->io_bp; 2256 spa_load_error_t *sle = zio->io_private; 2257 dmu_object_type_t type = BP_GET_TYPE(bp); 2258 int error = zio->io_error; 2259 spa_t *spa = zio->io_spa; 2260 2261 abd_free(zio->io_abd); 2262 if (error) { 2263 if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && 2264 type != DMU_OT_INTENT_LOG) 2265 atomic_inc_64(&sle->sle_meta_count); 2266 else 2267 atomic_inc_64(&sle->sle_data_count); 2268 } 2269 2270 mutex_enter(&spa->spa_scrub_lock); 2271 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 2272 cv_broadcast(&spa->spa_scrub_io_cv); 2273 mutex_exit(&spa->spa_scrub_lock); 2274 } 2275 2276 /* 2277 * Maximum number of inflight bytes is the log2 fraction of the arc size. 2278 * By default, we set it to 1/16th of the arc. 2279 */ 2280 int spa_load_verify_shift = 4; 2281 int spa_load_verify_metadata = B_TRUE; 2282 int spa_load_verify_data = B_TRUE; 2283 2284 /*ARGSUSED*/ 2285 static int 2286 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 2287 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 2288 { 2289 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 2290 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 2291 return (0); 2292 /* 2293 * Note: normally this routine will not be called if 2294 * spa_load_verify_metadata is not set. However, it may be useful 2295 * to manually set the flag after the traversal has begun. 2296 */ 2297 if (!spa_load_verify_metadata) 2298 return (0); 2299 if (!BP_IS_METADATA(bp) && !spa_load_verify_data) 2300 return (0); 2301 2302 uint64_t maxinflight_bytes = 2303 arc_target_bytes() >> spa_load_verify_shift; 2304 zio_t *rio = arg; 2305 size_t size = BP_GET_PSIZE(bp); 2306 2307 mutex_enter(&spa->spa_scrub_lock); 2308 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 2309 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 2310 spa->spa_load_verify_bytes += size; 2311 mutex_exit(&spa->spa_scrub_lock); 2312 2313 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 2314 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 2315 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 2316 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 2317 return (0); 2318 } 2319 2320 /* ARGSUSED */ 2321 static int 2322 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 2323 { 2324 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 2325 return (SET_ERROR(ENAMETOOLONG)); 2326 2327 return (0); 2328 } 2329 2330 static int 2331 spa_load_verify(spa_t *spa) 2332 { 2333 zio_t *rio; 2334 spa_load_error_t sle = { 0 }; 2335 zpool_load_policy_t policy; 2336 boolean_t verify_ok = B_FALSE; 2337 int error = 0; 2338 2339 zpool_get_load_policy(spa->spa_config, &policy); 2340 2341 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND) 2342 return (0); 2343 2344 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 2345 error = dmu_objset_find_dp(spa->spa_dsl_pool, 2346 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 2347 DS_FIND_CHILDREN); 2348 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 2349 if (error != 0) 2350 return (error); 2351 2352 rio = zio_root(spa, NULL, &sle, 2353 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 2354 2355 if (spa_load_verify_metadata) { 2356 if (spa->spa_extreme_rewind) { 2357 spa_load_note(spa, "performing a complete scan of the " 2358 "pool since extreme rewind is on. This may take " 2359 "a very long time.\n (spa_load_verify_data=%u, " 2360 "spa_load_verify_metadata=%u)", 2361 spa_load_verify_data, spa_load_verify_metadata); 2362 } 2363 2364 error = traverse_pool(spa, spa->spa_verify_min_txg, 2365 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 2366 TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio); 2367 } 2368 2369 (void) zio_wait(rio); 2370 ASSERT0(spa->spa_load_verify_bytes); 2371 2372 spa->spa_load_meta_errors = sle.sle_meta_count; 2373 spa->spa_load_data_errors = sle.sle_data_count; 2374 2375 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 2376 spa_load_note(spa, "spa_load_verify found %llu metadata errors " 2377 "and %llu data errors", (u_longlong_t)sle.sle_meta_count, 2378 (u_longlong_t)sle.sle_data_count); 2379 } 2380 2381 if (spa_load_verify_dryrun || 2382 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 2383 sle.sle_data_count <= policy.zlp_maxdata)) { 2384 int64_t loss = 0; 2385 2386 verify_ok = B_TRUE; 2387 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 2388 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 2389 2390 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; 2391 VERIFY(nvlist_add_uint64(spa->spa_load_info, 2392 ZPOOL_CONFIG_LOAD_TIME, spa->spa_load_txg_ts) == 0); 2393 VERIFY(nvlist_add_int64(spa->spa_load_info, 2394 ZPOOL_CONFIG_REWIND_TIME, loss) == 0); 2395 VERIFY(nvlist_add_uint64(spa->spa_load_info, 2396 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count) == 0); 2397 } else { 2398 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 2399 } 2400 2401 if (spa_load_verify_dryrun) 2402 return (0); 2403 2404 if (error) { 2405 if (error != ENXIO && error != EIO) 2406 error = SET_ERROR(EIO); 2407 return (error); 2408 } 2409 2410 return (verify_ok ? 0 : EIO); 2411 } 2412 2413 /* 2414 * Find a value in the pool props object. 2415 */ 2416 static void 2417 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 2418 { 2419 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 2420 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 2421 } 2422 2423 /* 2424 * Find a value in the pool directory object. 2425 */ 2426 static int 2427 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 2428 { 2429 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2430 name, sizeof (uint64_t), 1, val); 2431 2432 if (error != 0 && (error != ENOENT || log_enoent)) { 2433 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 2434 "[error=%d]", name, error); 2435 } 2436 2437 return (error); 2438 } 2439 2440 static int 2441 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 2442 { 2443 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 2444 return (SET_ERROR(err)); 2445 } 2446 2447 boolean_t 2448 spa_livelist_delete_check(spa_t *spa) 2449 { 2450 return (spa->spa_livelists_to_delete != 0); 2451 } 2452 2453 /* ARGSUSED */ 2454 static boolean_t 2455 spa_livelist_delete_cb_check(void *arg, zthr_t *z) 2456 { 2457 spa_t *spa = arg; 2458 return (spa_livelist_delete_check(spa)); 2459 } 2460 2461 static int 2462 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 2463 { 2464 spa_t *spa = arg; 2465 zio_free(spa, tx->tx_txg, bp); 2466 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 2467 -bp_get_dsize_sync(spa, bp), 2468 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 2469 return (0); 2470 } 2471 2472 static int 2473 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 2474 { 2475 int err; 2476 zap_cursor_t zc; 2477 zap_attribute_t za; 2478 zap_cursor_init(&zc, os, zap_obj); 2479 err = zap_cursor_retrieve(&zc, &za); 2480 zap_cursor_fini(&zc); 2481 if (err == 0) 2482 *llp = za.za_first_integer; 2483 return (err); 2484 } 2485 2486 /* 2487 * Components of livelist deletion that must be performed in syncing 2488 * context: freeing block pointers and updating the pool-wide data 2489 * structures to indicate how much work is left to do 2490 */ 2491 typedef struct sublist_delete_arg { 2492 spa_t *spa; 2493 dsl_deadlist_t *ll; 2494 uint64_t key; 2495 bplist_t *to_free; 2496 } sublist_delete_arg_t; 2497 2498 static void 2499 sublist_delete_sync(void *arg, dmu_tx_t *tx) 2500 { 2501 sublist_delete_arg_t *sda = arg; 2502 spa_t *spa = sda->spa; 2503 dsl_deadlist_t *ll = sda->ll; 2504 uint64_t key = sda->key; 2505 bplist_t *to_free = sda->to_free; 2506 2507 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 2508 dsl_deadlist_remove_entry(ll, key, tx); 2509 } 2510 2511 typedef struct livelist_delete_arg { 2512 spa_t *spa; 2513 uint64_t ll_obj; 2514 uint64_t zap_obj; 2515 } livelist_delete_arg_t; 2516 2517 static void 2518 livelist_delete_sync(void *arg, dmu_tx_t *tx) 2519 { 2520 livelist_delete_arg_t *lda = arg; 2521 spa_t *spa = lda->spa; 2522 uint64_t ll_obj = lda->ll_obj; 2523 uint64_t zap_obj = lda->zap_obj; 2524 objset_t *mos = spa->spa_meta_objset; 2525 uint64_t count; 2526 2527 /* free the livelist and decrement the feature count */ 2528 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 2529 dsl_deadlist_free(mos, ll_obj, tx); 2530 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 2531 VERIFY0(zap_count(mos, zap_obj, &count)); 2532 if (count == 0) { 2533 /* no more livelists to delete */ 2534 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 2535 DMU_POOL_DELETED_CLONES, tx)); 2536 VERIFY0(zap_destroy(mos, zap_obj, tx)); 2537 spa->spa_livelists_to_delete = 0; 2538 spa_notify_waiters(spa); 2539 } 2540 } 2541 2542 /* 2543 * Load in the value for the livelist to be removed and open it. Then, 2544 * load its first sublist and determine which block pointers should actually 2545 * be freed. Then, call a synctask which performs the actual frees and updates 2546 * the pool-wide livelist data. 2547 */ 2548 /* ARGSUSED */ 2549 static void 2550 spa_livelist_delete_cb(void *arg, zthr_t *z) 2551 { 2552 spa_t *spa = arg; 2553 uint64_t ll_obj = 0, count; 2554 objset_t *mos = spa->spa_meta_objset; 2555 uint64_t zap_obj = spa->spa_livelists_to_delete; 2556 /* 2557 * Determine the next livelist to delete. This function should only 2558 * be called if there is at least one deleted clone. 2559 */ 2560 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 2561 VERIFY0(zap_count(mos, ll_obj, &count)); 2562 if (count > 0) { 2563 dsl_deadlist_t *ll; 2564 dsl_deadlist_entry_t *dle; 2565 bplist_t to_free; 2566 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 2567 dsl_deadlist_open(ll, mos, ll_obj); 2568 dle = dsl_deadlist_first(ll); 2569 ASSERT3P(dle, !=, NULL); 2570 bplist_create(&to_free); 2571 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 2572 z, NULL); 2573 if (err == 0) { 2574 sublist_delete_arg_t sync_arg = { 2575 .spa = spa, 2576 .ll = ll, 2577 .key = dle->dle_mintxg, 2578 .to_free = &to_free 2579 }; 2580 zfs_dbgmsg("deleting sublist (id %llu) from" 2581 " livelist %llu, %lld remaining", 2582 (u_longlong_t)dle->dle_bpobj.bpo_object, 2583 (u_longlong_t)ll_obj, (longlong_t)count - 1); 2584 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 2585 sublist_delete_sync, &sync_arg, 0, 2586 ZFS_SPACE_CHECK_DESTROY)); 2587 } else { 2588 VERIFY3U(err, ==, EINTR); 2589 } 2590 bplist_clear(&to_free); 2591 bplist_destroy(&to_free); 2592 dsl_deadlist_close(ll); 2593 kmem_free(ll, sizeof (dsl_deadlist_t)); 2594 } else { 2595 livelist_delete_arg_t sync_arg = { 2596 .spa = spa, 2597 .ll_obj = ll_obj, 2598 .zap_obj = zap_obj 2599 }; 2600 zfs_dbgmsg("deletion of livelist %llu completed", 2601 (u_longlong_t)ll_obj); 2602 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 2603 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 2604 } 2605 } 2606 2607 static void 2608 spa_start_livelist_destroy_thread(spa_t *spa) 2609 { 2610 ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL); 2611 spa->spa_livelist_delete_zthr = 2612 zthr_create("z_livelist_destroy", 2613 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa); 2614 } 2615 2616 typedef struct livelist_new_arg { 2617 bplist_t *allocs; 2618 bplist_t *frees; 2619 } livelist_new_arg_t; 2620 2621 static int 2622 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 2623 dmu_tx_t *tx) 2624 { 2625 ASSERT(tx == NULL); 2626 livelist_new_arg_t *lna = arg; 2627 if (bp_freed) { 2628 bplist_append(lna->frees, bp); 2629 } else { 2630 bplist_append(lna->allocs, bp); 2631 zfs_livelist_condense_new_alloc++; 2632 } 2633 return (0); 2634 } 2635 2636 typedef struct livelist_condense_arg { 2637 spa_t *spa; 2638 bplist_t to_keep; 2639 uint64_t first_size; 2640 uint64_t next_size; 2641 } livelist_condense_arg_t; 2642 2643 static void 2644 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 2645 { 2646 livelist_condense_arg_t *lca = arg; 2647 spa_t *spa = lca->spa; 2648 bplist_t new_frees; 2649 dsl_dataset_t *ds = spa->spa_to_condense.ds; 2650 2651 /* Have we been cancelled? */ 2652 if (spa->spa_to_condense.cancelled) { 2653 zfs_livelist_condense_sync_cancel++; 2654 goto out; 2655 } 2656 2657 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2658 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2659 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 2660 2661 /* 2662 * It's possible that the livelist was changed while the zthr was 2663 * running. Therefore, we need to check for new blkptrs in the two 2664 * entries being condensed and continue to track them in the livelist. 2665 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 2666 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 2667 * we need to sort them into two different bplists. 2668 */ 2669 uint64_t first_obj = first->dle_bpobj.bpo_object; 2670 uint64_t next_obj = next->dle_bpobj.bpo_object; 2671 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2672 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2673 2674 bplist_create(&new_frees); 2675 livelist_new_arg_t new_bps = { 2676 .allocs = &lca->to_keep, 2677 .frees = &new_frees, 2678 }; 2679 2680 if (cur_first_size > lca->first_size) { 2681 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 2682 livelist_track_new_cb, &new_bps, lca->first_size)); 2683 } 2684 if (cur_next_size > lca->next_size) { 2685 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 2686 livelist_track_new_cb, &new_bps, lca->next_size)); 2687 } 2688 2689 dsl_deadlist_clear_entry(first, ll, tx); 2690 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 2691 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 2692 2693 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 2694 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 2695 bplist_destroy(&new_frees); 2696 2697 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 2698 dsl_dataset_name(ds, dsname); 2699 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 2700 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 2701 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, 2702 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, 2703 (u_longlong_t)cur_first_size, (u_longlong_t)next_obj, 2704 (u_longlong_t)cur_next_size, 2705 (u_longlong_t)first->dle_bpobj.bpo_object, 2706 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 2707 out: 2708 dmu_buf_rele(ds->ds_dbuf, spa); 2709 spa->spa_to_condense.ds = NULL; 2710 bplist_clear(&lca->to_keep); 2711 bplist_destroy(&lca->to_keep); 2712 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2713 spa->spa_to_condense.syncing = B_FALSE; 2714 } 2715 2716 static void 2717 spa_livelist_condense_cb(void *arg, zthr_t *t) 2718 { 2719 while (zfs_livelist_condense_zthr_pause && 2720 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2721 delay(1); 2722 2723 spa_t *spa = arg; 2724 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2725 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2726 uint64_t first_size, next_size; 2727 2728 livelist_condense_arg_t *lca = 2729 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 2730 bplist_create(&lca->to_keep); 2731 2732 /* 2733 * Process the livelists (matching FREEs and ALLOCs) in open context 2734 * so we have minimal work in syncing context to condense. 2735 * 2736 * We save bpobj sizes (first_size and next_size) to use later in 2737 * syncing context to determine if entries were added to these sublists 2738 * while in open context. This is possible because the clone is still 2739 * active and open for normal writes and we want to make sure the new, 2740 * unprocessed blockpointers are inserted into the livelist normally. 2741 * 2742 * Note that dsl_process_sub_livelist() both stores the size number of 2743 * blockpointers and iterates over them while the bpobj's lock held, so 2744 * the sizes returned to us are consistent which what was actually 2745 * processed. 2746 */ 2747 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 2748 &first_size); 2749 if (err == 0) 2750 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 2751 t, &next_size); 2752 2753 if (err == 0) { 2754 while (zfs_livelist_condense_sync_pause && 2755 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2756 delay(1); 2757 2758 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2759 dmu_tx_mark_netfree(tx); 2760 dmu_tx_hold_space(tx, 1); 2761 err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE); 2762 if (err == 0) { 2763 /* 2764 * Prevent the condense zthr restarting before 2765 * the synctask completes. 2766 */ 2767 spa->spa_to_condense.syncing = B_TRUE; 2768 lca->spa = spa; 2769 lca->first_size = first_size; 2770 lca->next_size = next_size; 2771 dsl_sync_task_nowait(spa_get_dsl(spa), 2772 spa_livelist_condense_sync, lca, tx); 2773 dmu_tx_commit(tx); 2774 return; 2775 } 2776 } 2777 /* 2778 * Condensing can not continue: either it was externally stopped or 2779 * we were unable to assign to a tx because the pool has run out of 2780 * space. In the second case, we'll just end up trying to condense 2781 * again in a later txg. 2782 */ 2783 ASSERT(err != 0); 2784 bplist_clear(&lca->to_keep); 2785 bplist_destroy(&lca->to_keep); 2786 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2787 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 2788 spa->spa_to_condense.ds = NULL; 2789 if (err == EINTR) 2790 zfs_livelist_condense_zthr_cancel++; 2791 } 2792 2793 /* ARGSUSED */ 2794 /* 2795 * Check that there is something to condense but that a condense is not 2796 * already in progress and that condensing has not been cancelled. 2797 */ 2798 static boolean_t 2799 spa_livelist_condense_cb_check(void *arg, zthr_t *z) 2800 { 2801 spa_t *spa = arg; 2802 if ((spa->spa_to_condense.ds != NULL) && 2803 (spa->spa_to_condense.syncing == B_FALSE) && 2804 (spa->spa_to_condense.cancelled == B_FALSE)) { 2805 return (B_TRUE); 2806 } 2807 return (B_FALSE); 2808 } 2809 2810 static void 2811 spa_start_livelist_condensing_thread(spa_t *spa) 2812 { 2813 spa->spa_to_condense.ds = NULL; 2814 spa->spa_to_condense.first = NULL; 2815 spa->spa_to_condense.next = NULL; 2816 spa->spa_to_condense.syncing = B_FALSE; 2817 spa->spa_to_condense.cancelled = B_FALSE; 2818 2819 ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL); 2820 spa->spa_livelist_condense_zthr = 2821 zthr_create("z_livelist_condense", 2822 spa_livelist_condense_cb_check, 2823 spa_livelist_condense_cb, spa); 2824 } 2825 2826 static void 2827 spa_spawn_aux_threads(spa_t *spa) 2828 { 2829 ASSERT(spa_writeable(spa)); 2830 2831 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 2832 2833 spa_start_indirect_condensing_thread(spa); 2834 spa_start_livelist_destroy_thread(spa); 2835 spa_start_livelist_condensing_thread(spa); 2836 2837 ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL); 2838 spa->spa_checkpoint_discard_zthr = 2839 zthr_create("z_checkpoint_discard", 2840 spa_checkpoint_discard_thread_check, 2841 spa_checkpoint_discard_thread, spa); 2842 } 2843 2844 /* 2845 * Fix up config after a partly-completed split. This is done with the 2846 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 2847 * pool have that entry in their config, but only the splitting one contains 2848 * a list of all the guids of the vdevs that are being split off. 2849 * 2850 * This function determines what to do with that list: either rejoin 2851 * all the disks to the pool, or complete the splitting process. To attempt 2852 * the rejoin, each disk that is offlined is marked online again, and 2853 * we do a reopen() call. If the vdev label for every disk that was 2854 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 2855 * then we call vdev_split() on each disk, and complete the split. 2856 * 2857 * Otherwise we leave the config alone, with all the vdevs in place in 2858 * the original pool. 2859 */ 2860 static void 2861 spa_try_repair(spa_t *spa, nvlist_t *config) 2862 { 2863 uint_t extracted; 2864 uint64_t *glist; 2865 uint_t i, gcount; 2866 nvlist_t *nvl; 2867 vdev_t **vd; 2868 boolean_t attempt_reopen; 2869 2870 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 2871 return; 2872 2873 /* check that the config is complete */ 2874 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 2875 &glist, &gcount) != 0) 2876 return; 2877 2878 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 2879 2880 /* attempt to online all the vdevs & validate */ 2881 attempt_reopen = B_TRUE; 2882 for (i = 0; i < gcount; i++) { 2883 if (glist[i] == 0) /* vdev is hole */ 2884 continue; 2885 2886 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 2887 if (vd[i] == NULL) { 2888 /* 2889 * Don't bother attempting to reopen the disks; 2890 * just do the split. 2891 */ 2892 attempt_reopen = B_FALSE; 2893 } else { 2894 /* attempt to re-online it */ 2895 vd[i]->vdev_offline = B_FALSE; 2896 } 2897 } 2898 2899 if (attempt_reopen) { 2900 vdev_reopen(spa->spa_root_vdev); 2901 2902 /* check each device to see what state it's in */ 2903 for (extracted = 0, i = 0; i < gcount; i++) { 2904 if (vd[i] != NULL && 2905 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 2906 break; 2907 ++extracted; 2908 } 2909 } 2910 2911 /* 2912 * If every disk has been moved to the new pool, or if we never 2913 * even attempted to look at them, then we split them off for 2914 * good. 2915 */ 2916 if (!attempt_reopen || gcount == extracted) { 2917 for (i = 0; i < gcount; i++) 2918 if (vd[i] != NULL) 2919 vdev_split(vd[i]); 2920 vdev_reopen(spa->spa_root_vdev); 2921 } 2922 2923 kmem_free(vd, gcount * sizeof (vdev_t *)); 2924 } 2925 2926 static int 2927 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 2928 { 2929 char *ereport = FM_EREPORT_ZFS_POOL; 2930 int error; 2931 2932 spa->spa_load_state = state; 2933 (void) spa_import_progress_set_state(spa_guid(spa), 2934 spa_load_state(spa)); 2935 2936 gethrestime(&spa->spa_loaded_ts); 2937 error = spa_load_impl(spa, type, &ereport); 2938 2939 /* 2940 * Don't count references from objsets that are already closed 2941 * and are making their way through the eviction process. 2942 */ 2943 spa_evicting_os_wait(spa); 2944 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 2945 if (error) { 2946 if (error != EEXIST) { 2947 spa->spa_loaded_ts.tv_sec = 0; 2948 spa->spa_loaded_ts.tv_nsec = 0; 2949 } 2950 if (error != EBADF) { 2951 (void) zfs_ereport_post(ereport, spa, 2952 NULL, NULL, NULL, 0); 2953 } 2954 } 2955 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 2956 spa->spa_ena = 0; 2957 2958 (void) spa_import_progress_set_state(spa_guid(spa), 2959 spa_load_state(spa)); 2960 2961 return (error); 2962 } 2963 2964 #ifdef ZFS_DEBUG 2965 /* 2966 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 2967 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 2968 * spa's per-vdev ZAP list. 2969 */ 2970 static uint64_t 2971 vdev_count_verify_zaps(vdev_t *vd) 2972 { 2973 spa_t *spa = vd->vdev_spa; 2974 uint64_t total = 0; 2975 2976 if (vd->vdev_top_zap != 0) { 2977 total++; 2978 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2979 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 2980 } 2981 if (vd->vdev_leaf_zap != 0) { 2982 total++; 2983 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2984 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 2985 } 2986 2987 for (uint64_t i = 0; i < vd->vdev_children; i++) { 2988 total += vdev_count_verify_zaps(vd->vdev_child[i]); 2989 } 2990 2991 return (total); 2992 } 2993 #endif 2994 2995 /* 2996 * Determine whether the activity check is required. 2997 */ 2998 static boolean_t 2999 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label, 3000 nvlist_t *config) 3001 { 3002 uint64_t state = 0; 3003 uint64_t hostid = 0; 3004 uint64_t tryconfig_txg = 0; 3005 uint64_t tryconfig_timestamp = 0; 3006 uint16_t tryconfig_mmp_seq = 0; 3007 nvlist_t *nvinfo; 3008 3009 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3010 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 3011 (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3012 &tryconfig_txg); 3013 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3014 &tryconfig_timestamp); 3015 (void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3016 &tryconfig_mmp_seq); 3017 } 3018 3019 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); 3020 3021 /* 3022 * Disable the MMP activity check - This is used by zdb which 3023 * is intended to be used on potentially active pools. 3024 */ 3025 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) 3026 return (B_FALSE); 3027 3028 /* 3029 * Skip the activity check when the MMP feature is disabled. 3030 */ 3031 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) 3032 return (B_FALSE); 3033 3034 /* 3035 * If the tryconfig_ values are nonzero, they are the results of an 3036 * earlier tryimport. If they all match the uberblock we just found, 3037 * then the pool has not changed and we return false so we do not test 3038 * a second time. 3039 */ 3040 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && 3041 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && 3042 tryconfig_mmp_seq && tryconfig_mmp_seq == 3043 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) 3044 return (B_FALSE); 3045 3046 /* 3047 * Allow the activity check to be skipped when importing the pool 3048 * on the same host which last imported it. Since the hostid from 3049 * configuration may be stale use the one read from the label. 3050 */ 3051 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3052 hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID); 3053 3054 if (hostid == spa_get_hostid(spa)) 3055 return (B_FALSE); 3056 3057 /* 3058 * Skip the activity test when the pool was cleanly exported. 3059 */ 3060 if (state != POOL_STATE_ACTIVE) 3061 return (B_FALSE); 3062 3063 return (B_TRUE); 3064 } 3065 3066 /* 3067 * Nanoseconds the activity check must watch for changes on-disk. 3068 */ 3069 static uint64_t 3070 spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3071 { 3072 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 3073 uint64_t multihost_interval = MSEC2NSEC( 3074 MMP_INTERVAL_OK(zfs_multihost_interval)); 3075 uint64_t import_delay = MAX(NANOSEC, import_intervals * 3076 multihost_interval); 3077 3078 /* 3079 * Local tunables determine a minimum duration except for the case 3080 * where we know when the remote host will suspend the pool if MMP 3081 * writes do not land. 3082 * 3083 * See Big Theory comment at the top of mmp.c for the reasoning behind 3084 * these cases and times. 3085 */ 3086 3087 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 3088 3089 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3090 MMP_FAIL_INT(ub) > 0) { 3091 3092 /* MMP on remote host will suspend pool after failed writes */ 3093 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 3094 MMP_IMPORT_SAFETY_FACTOR / 100; 3095 3096 zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp " 3097 "mmp_fails=%llu ub_mmp mmp_interval=%llu " 3098 "import_intervals=%llu", (u_longlong_t)import_delay, 3099 (u_longlong_t)MMP_FAIL_INT(ub), 3100 (u_longlong_t)MMP_INTERVAL(ub), 3101 (u_longlong_t)import_intervals); 3102 3103 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3104 MMP_FAIL_INT(ub) == 0) { 3105 3106 /* MMP on remote host will never suspend pool */ 3107 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 3108 ub->ub_mmp_delay) * import_intervals); 3109 3110 zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp " 3111 "mmp_interval=%llu ub_mmp_delay=%llu " 3112 "import_intervals=%llu", (u_longlong_t)import_delay, 3113 (u_longlong_t)MMP_INTERVAL(ub), 3114 (u_longlong_t)ub->ub_mmp_delay, 3115 (u_longlong_t)import_intervals); 3116 3117 } else if (MMP_VALID(ub)) { 3118 /* 3119 * zfs-0.7 compatibility case 3120 */ 3121 3122 import_delay = MAX(import_delay, (multihost_interval + 3123 ub->ub_mmp_delay) * import_intervals); 3124 3125 zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu " 3126 "import_intervals=%llu leaves=%u", 3127 (u_longlong_t)import_delay, 3128 (u_longlong_t)ub->ub_mmp_delay, 3129 (u_longlong_t)import_intervals, 3130 vdev_count_leaves(spa)); 3131 } else { 3132 /* Using local tunings is the only reasonable option */ 3133 zfs_dbgmsg("pool last imported on non-MMP aware " 3134 "host using import_delay=%llu multihost_interval=%llu " 3135 "import_intervals=%llu", (u_longlong_t)import_delay, 3136 (u_longlong_t)multihost_interval, 3137 (u_longlong_t)import_intervals); 3138 } 3139 3140 return (import_delay); 3141 } 3142 3143 /* 3144 * Perform the import activity check. If the user canceled the import or 3145 * we detected activity then fail. 3146 */ 3147 static int 3148 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config) 3149 { 3150 uint64_t txg = ub->ub_txg; 3151 uint64_t timestamp = ub->ub_timestamp; 3152 uint64_t mmp_config = ub->ub_mmp_config; 3153 uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0; 3154 uint64_t import_delay; 3155 hrtime_t import_expire; 3156 nvlist_t *mmp_label = NULL; 3157 vdev_t *rvd = spa->spa_root_vdev; 3158 kcondvar_t cv; 3159 kmutex_t mtx; 3160 int error = 0; 3161 3162 cv_init(&cv, NULL, CV_DEFAULT, NULL); 3163 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 3164 mutex_enter(&mtx); 3165 3166 /* 3167 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3168 * during the earlier tryimport. If the txg recorded there is 0 then 3169 * the pool is known to be active on another host. 3170 * 3171 * Otherwise, the pool might be in use on another host. Check for 3172 * changes in the uberblocks on disk if necessary. 3173 */ 3174 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3175 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 3176 ZPOOL_CONFIG_LOAD_INFO); 3177 3178 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && 3179 fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { 3180 vdev_uberblock_load(rvd, ub, &mmp_label); 3181 error = SET_ERROR(EREMOTEIO); 3182 goto out; 3183 } 3184 } 3185 3186 import_delay = spa_activity_check_duration(spa, ub); 3187 3188 /* Add a small random factor in case of simultaneous imports (0-25%) */ 3189 import_delay += import_delay * random_in_range(250) / 1000; 3190 3191 import_expire = gethrtime() + import_delay; 3192 3193 while (gethrtime() < import_expire) { 3194 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 3195 NSEC2SEC(import_expire - gethrtime())); 3196 3197 vdev_uberblock_load(rvd, ub, &mmp_label); 3198 3199 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || 3200 mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) { 3201 zfs_dbgmsg("multihost activity detected " 3202 "txg %llu ub_txg %llu " 3203 "timestamp %llu ub_timestamp %llu " 3204 "mmp_config %#llx ub_mmp_config %#llx", 3205 (u_longlong_t)txg, (u_longlong_t)ub->ub_txg, 3206 (u_longlong_t)timestamp, 3207 (u_longlong_t)ub->ub_timestamp, 3208 (u_longlong_t)mmp_config, 3209 (u_longlong_t)ub->ub_mmp_config); 3210 3211 error = SET_ERROR(EREMOTEIO); 3212 break; 3213 } 3214 3215 if (mmp_label) { 3216 nvlist_free(mmp_label); 3217 mmp_label = NULL; 3218 } 3219 3220 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 3221 if (error != -1) { 3222 error = SET_ERROR(EINTR); 3223 break; 3224 } 3225 error = 0; 3226 } 3227 3228 out: 3229 mutex_exit(&mtx); 3230 mutex_destroy(&mtx); 3231 cv_destroy(&cv); 3232 3233 /* 3234 * If the pool is determined to be active store the status in the 3235 * spa->spa_load_info nvlist. If the remote hostname or hostid are 3236 * available from configuration read from disk store them as well. 3237 * This allows 'zpool import' to generate a more useful message. 3238 * 3239 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) 3240 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 3241 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 3242 */ 3243 if (error == EREMOTEIO) { 3244 char *hostname = "<unknown>"; 3245 uint64_t hostid = 0; 3246 3247 if (mmp_label) { 3248 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { 3249 hostname = fnvlist_lookup_string(mmp_label, 3250 ZPOOL_CONFIG_HOSTNAME); 3251 fnvlist_add_string(spa->spa_load_info, 3252 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 3253 } 3254 3255 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { 3256 hostid = fnvlist_lookup_uint64(mmp_label, 3257 ZPOOL_CONFIG_HOSTID); 3258 fnvlist_add_uint64(spa->spa_load_info, 3259 ZPOOL_CONFIG_MMP_HOSTID, hostid); 3260 } 3261 } 3262 3263 fnvlist_add_uint64(spa->spa_load_info, 3264 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); 3265 fnvlist_add_uint64(spa->spa_load_info, 3266 ZPOOL_CONFIG_MMP_TXG, 0); 3267 3268 error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); 3269 } 3270 3271 if (mmp_label) 3272 nvlist_free(mmp_label); 3273 3274 return (error); 3275 } 3276 3277 static int 3278 spa_verify_host(spa_t *spa, nvlist_t *mos_config) 3279 { 3280 uint64_t hostid; 3281 char *hostname; 3282 uint64_t myhostid = 0; 3283 3284 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 3285 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 3286 hostname = fnvlist_lookup_string(mos_config, 3287 ZPOOL_CONFIG_HOSTNAME); 3288 3289 myhostid = zone_get_hostid(NULL); 3290 3291 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 3292 cmn_err(CE_WARN, "pool '%s' could not be " 3293 "loaded as it was last accessed by " 3294 "another system (host: %s hostid: 0x%llx). " 3295 "See: https://openzfs.github.io/openzfs-docs/msg/" 3296 "ZFS-8000-EY", 3297 spa_name(spa), hostname, (u_longlong_t)hostid); 3298 spa_load_failed(spa, "hostid verification failed: pool " 3299 "last accessed by host: %s (hostid: 0x%llx)", 3300 hostname, (u_longlong_t)hostid); 3301 return (SET_ERROR(EBADF)); 3302 } 3303 } 3304 3305 return (0); 3306 } 3307 3308 static int 3309 spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 3310 { 3311 int error = 0; 3312 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 3313 int parse; 3314 vdev_t *rvd; 3315 uint64_t pool_guid; 3316 char *comment; 3317 char *compatibility; 3318 3319 /* 3320 * Versioning wasn't explicitly added to the label until later, so if 3321 * it's not present treat it as the initial version. 3322 */ 3323 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 3324 &spa->spa_ubsync.ub_version) != 0) 3325 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 3326 3327 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 3328 spa_load_failed(spa, "invalid config provided: '%s' missing", 3329 ZPOOL_CONFIG_POOL_GUID); 3330 return (SET_ERROR(EINVAL)); 3331 } 3332 3333 /* 3334 * If we are doing an import, ensure that the pool is not already 3335 * imported by checking if its pool guid already exists in the 3336 * spa namespace. 3337 * 3338 * The only case that we allow an already imported pool to be 3339 * imported again, is when the pool is checkpointed and we want to 3340 * look at its checkpointed state from userland tools like zdb. 3341 */ 3342 #ifdef _KERNEL 3343 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3344 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3345 spa_guid_exists(pool_guid, 0)) { 3346 #else 3347 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3348 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3349 spa_guid_exists(pool_guid, 0) && 3350 !spa_importing_readonly_checkpoint(spa)) { 3351 #endif 3352 spa_load_failed(spa, "a pool with guid %llu is already open", 3353 (u_longlong_t)pool_guid); 3354 return (SET_ERROR(EEXIST)); 3355 } 3356 3357 spa->spa_config_guid = pool_guid; 3358 3359 nvlist_free(spa->spa_load_info); 3360 spa->spa_load_info = fnvlist_alloc(); 3361 3362 ASSERT(spa->spa_comment == NULL); 3363 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 3364 spa->spa_comment = spa_strdup(comment); 3365 3366 ASSERT(spa->spa_compatibility == NULL); 3367 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 3368 &compatibility) == 0) 3369 spa->spa_compatibility = spa_strdup(compatibility); 3370 3371 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 3372 &spa->spa_config_txg); 3373 3374 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 3375 spa->spa_config_splitting = fnvlist_dup(nvl); 3376 3377 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 3378 spa_load_failed(spa, "invalid config provided: '%s' missing", 3379 ZPOOL_CONFIG_VDEV_TREE); 3380 return (SET_ERROR(EINVAL)); 3381 } 3382 3383 /* 3384 * Create "The Godfather" zio to hold all async IOs 3385 */ 3386 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 3387 KM_SLEEP); 3388 for (int i = 0; i < max_ncpus; i++) { 3389 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 3390 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3391 ZIO_FLAG_GODFATHER); 3392 } 3393 3394 /* 3395 * Parse the configuration into a vdev tree. We explicitly set the 3396 * value that will be returned by spa_version() since parsing the 3397 * configuration requires knowing the version number. 3398 */ 3399 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3400 parse = (type == SPA_IMPORT_EXISTING ? 3401 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 3402 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 3403 spa_config_exit(spa, SCL_ALL, FTAG); 3404 3405 if (error != 0) { 3406 spa_load_failed(spa, "unable to parse config [error=%d]", 3407 error); 3408 return (error); 3409 } 3410 3411 ASSERT(spa->spa_root_vdev == rvd); 3412 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 3413 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 3414 3415 if (type != SPA_IMPORT_ASSEMBLE) { 3416 ASSERT(spa_guid(spa) == pool_guid); 3417 } 3418 3419 return (0); 3420 } 3421 3422 /* 3423 * Recursively open all vdevs in the vdev tree. This function is called twice: 3424 * first with the untrusted config, then with the trusted config. 3425 */ 3426 static int 3427 spa_ld_open_vdevs(spa_t *spa) 3428 { 3429 int error = 0; 3430 3431 /* 3432 * spa_missing_tvds_allowed defines how many top-level vdevs can be 3433 * missing/unopenable for the root vdev to be still considered openable. 3434 */ 3435 if (spa->spa_trust_config) { 3436 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 3437 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 3438 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 3439 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 3440 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 3441 } else { 3442 spa->spa_missing_tvds_allowed = 0; 3443 } 3444 3445 spa->spa_missing_tvds_allowed = 3446 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 3447 3448 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3449 error = vdev_open(spa->spa_root_vdev); 3450 spa_config_exit(spa, SCL_ALL, FTAG); 3451 3452 if (spa->spa_missing_tvds != 0) { 3453 spa_load_note(spa, "vdev tree has %lld missing top-level " 3454 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 3455 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 3456 /* 3457 * Although theoretically we could allow users to open 3458 * incomplete pools in RW mode, we'd need to add a lot 3459 * of extra logic (e.g. adjust pool space to account 3460 * for missing vdevs). 3461 * This limitation also prevents users from accidentally 3462 * opening the pool in RW mode during data recovery and 3463 * damaging it further. 3464 */ 3465 spa_load_note(spa, "pools with missing top-level " 3466 "vdevs can only be opened in read-only mode."); 3467 error = SET_ERROR(ENXIO); 3468 } else { 3469 spa_load_note(spa, "current settings allow for maximum " 3470 "%lld missing top-level vdevs at this stage.", 3471 (u_longlong_t)spa->spa_missing_tvds_allowed); 3472 } 3473 } 3474 if (error != 0) { 3475 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 3476 error); 3477 } 3478 if (spa->spa_missing_tvds != 0 || error != 0) 3479 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 3480 3481 return (error); 3482 } 3483 3484 /* 3485 * We need to validate the vdev labels against the configuration that 3486 * we have in hand. This function is called twice: first with an untrusted 3487 * config, then with a trusted config. The validation is more strict when the 3488 * config is trusted. 3489 */ 3490 static int 3491 spa_ld_validate_vdevs(spa_t *spa) 3492 { 3493 int error = 0; 3494 vdev_t *rvd = spa->spa_root_vdev; 3495 3496 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3497 error = vdev_validate(rvd); 3498 spa_config_exit(spa, SCL_ALL, FTAG); 3499 3500 if (error != 0) { 3501 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 3502 return (error); 3503 } 3504 3505 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 3506 spa_load_failed(spa, "cannot open vdev tree after invalidating " 3507 "some vdevs"); 3508 vdev_dbgmsg_print_tree(rvd, 2); 3509 return (SET_ERROR(ENXIO)); 3510 } 3511 3512 return (0); 3513 } 3514 3515 static void 3516 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 3517 { 3518 spa->spa_state = POOL_STATE_ACTIVE; 3519 spa->spa_ubsync = spa->spa_uberblock; 3520 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 3521 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 3522 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 3523 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 3524 spa->spa_claim_max_txg = spa->spa_first_txg; 3525 spa->spa_prev_software_version = ub->ub_software_version; 3526 } 3527 3528 static int 3529 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 3530 { 3531 vdev_t *rvd = spa->spa_root_vdev; 3532 nvlist_t *label; 3533 uberblock_t *ub = &spa->spa_uberblock; 3534 boolean_t activity_check = B_FALSE; 3535 3536 /* 3537 * If we are opening the checkpointed state of the pool by 3538 * rewinding to it, at this point we will have written the 3539 * checkpointed uberblock to the vdev labels, so searching 3540 * the labels will find the right uberblock. However, if 3541 * we are opening the checkpointed state read-only, we have 3542 * not modified the labels. Therefore, we must ignore the 3543 * labels and continue using the spa_uberblock that was set 3544 * by spa_ld_checkpoint_rewind. 3545 * 3546 * Note that it would be fine to ignore the labels when 3547 * rewinding (opening writeable) as well. However, if we 3548 * crash just after writing the labels, we will end up 3549 * searching the labels. Doing so in the common case means 3550 * that this code path gets exercised normally, rather than 3551 * just in the edge case. 3552 */ 3553 if (ub->ub_checkpoint_txg != 0 && 3554 spa_importing_readonly_checkpoint(spa)) { 3555 spa_ld_select_uberblock_done(spa, ub); 3556 return (0); 3557 } 3558 3559 /* 3560 * Find the best uberblock. 3561 */ 3562 vdev_uberblock_load(rvd, ub, &label); 3563 3564 /* 3565 * If we weren't able to find a single valid uberblock, return failure. 3566 */ 3567 if (ub->ub_txg == 0) { 3568 nvlist_free(label); 3569 spa_load_failed(spa, "no valid uberblock found"); 3570 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 3571 } 3572 3573 if (spa->spa_load_max_txg != UINT64_MAX) { 3574 (void) spa_import_progress_set_max_txg(spa_guid(spa), 3575 (u_longlong_t)spa->spa_load_max_txg); 3576 } 3577 spa_load_note(spa, "using uberblock with txg=%llu", 3578 (u_longlong_t)ub->ub_txg); 3579 3580 3581 /* 3582 * For pools which have the multihost property on determine if the 3583 * pool is truly inactive and can be safely imported. Prevent 3584 * hosts which don't have a hostid set from importing the pool. 3585 */ 3586 activity_check = spa_activity_check_required(spa, ub, label, 3587 spa->spa_config); 3588 if (activity_check) { 3589 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 3590 spa_get_hostid(spa) == 0) { 3591 nvlist_free(label); 3592 fnvlist_add_uint64(spa->spa_load_info, 3593 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 3594 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 3595 } 3596 3597 int error = spa_activity_check(spa, ub, spa->spa_config); 3598 if (error) { 3599 nvlist_free(label); 3600 return (error); 3601 } 3602 3603 fnvlist_add_uint64(spa->spa_load_info, 3604 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); 3605 fnvlist_add_uint64(spa->spa_load_info, 3606 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); 3607 fnvlist_add_uint16(spa->spa_load_info, 3608 ZPOOL_CONFIG_MMP_SEQ, 3609 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)); 3610 } 3611 3612 /* 3613 * If the pool has an unsupported version we can't open it. 3614 */ 3615 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 3616 nvlist_free(label); 3617 spa_load_failed(spa, "version %llu is not supported", 3618 (u_longlong_t)ub->ub_version); 3619 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 3620 } 3621 3622 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3623 nvlist_t *features; 3624 3625 /* 3626 * If we weren't able to find what's necessary for reading the 3627 * MOS in the label, return failure. 3628 */ 3629 if (label == NULL) { 3630 spa_load_failed(spa, "label config unavailable"); 3631 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3632 ENXIO)); 3633 } 3634 3635 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 3636 &features) != 0) { 3637 nvlist_free(label); 3638 spa_load_failed(spa, "invalid label: '%s' missing", 3639 ZPOOL_CONFIG_FEATURES_FOR_READ); 3640 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3641 ENXIO)); 3642 } 3643 3644 /* 3645 * Update our in-core representation with the definitive values 3646 * from the label. 3647 */ 3648 nvlist_free(spa->spa_label_features); 3649 VERIFY(nvlist_dup(features, &spa->spa_label_features, 0) == 0); 3650 } 3651 3652 nvlist_free(label); 3653 3654 /* 3655 * Look through entries in the label nvlist's features_for_read. If 3656 * there is a feature listed there which we don't understand then we 3657 * cannot open a pool. 3658 */ 3659 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3660 nvlist_t *unsup_feat; 3661 3662 VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) == 3663 0); 3664 3665 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 3666 NULL); nvp != NULL; 3667 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 3668 if (!zfeature_is_supported(nvpair_name(nvp))) { 3669 VERIFY(nvlist_add_string(unsup_feat, 3670 nvpair_name(nvp), "") == 0); 3671 } 3672 } 3673 3674 if (!nvlist_empty(unsup_feat)) { 3675 VERIFY(nvlist_add_nvlist(spa->spa_load_info, 3676 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0); 3677 nvlist_free(unsup_feat); 3678 spa_load_failed(spa, "some features are unsupported"); 3679 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 3680 ENOTSUP)); 3681 } 3682 3683 nvlist_free(unsup_feat); 3684 } 3685 3686 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 3687 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3688 spa_try_repair(spa, spa->spa_config); 3689 spa_config_exit(spa, SCL_ALL, FTAG); 3690 nvlist_free(spa->spa_config_splitting); 3691 spa->spa_config_splitting = NULL; 3692 } 3693 3694 /* 3695 * Initialize internal SPA structures. 3696 */ 3697 spa_ld_select_uberblock_done(spa, ub); 3698 3699 return (0); 3700 } 3701 3702 static int 3703 spa_ld_open_rootbp(spa_t *spa) 3704 { 3705 int error = 0; 3706 vdev_t *rvd = spa->spa_root_vdev; 3707 3708 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 3709 if (error != 0) { 3710 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 3711 "[error=%d]", error); 3712 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3713 } 3714 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 3715 3716 return (0); 3717 } 3718 3719 static int 3720 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 3721 boolean_t reloading) 3722 { 3723 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 3724 nvlist_t *nv, *mos_config, *policy; 3725 int error = 0, copy_error; 3726 uint64_t healthy_tvds, healthy_tvds_mos; 3727 uint64_t mos_config_txg; 3728 3729 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 3730 != 0) 3731 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3732 3733 /* 3734 * If we're assembling a pool from a split, the config provided is 3735 * already trusted so there is nothing to do. 3736 */ 3737 if (type == SPA_IMPORT_ASSEMBLE) 3738 return (0); 3739 3740 healthy_tvds = spa_healthy_core_tvds(spa); 3741 3742 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 3743 != 0) { 3744 spa_load_failed(spa, "unable to retrieve MOS config"); 3745 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3746 } 3747 3748 /* 3749 * If we are doing an open, pool owner wasn't verified yet, thus do 3750 * the verification here. 3751 */ 3752 if (spa->spa_load_state == SPA_LOAD_OPEN) { 3753 error = spa_verify_host(spa, mos_config); 3754 if (error != 0) { 3755 nvlist_free(mos_config); 3756 return (error); 3757 } 3758 } 3759 3760 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 3761 3762 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3763 3764 /* 3765 * Build a new vdev tree from the trusted config 3766 */ 3767 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 3768 if (error != 0) { 3769 nvlist_free(mos_config); 3770 spa_config_exit(spa, SCL_ALL, FTAG); 3771 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 3772 error); 3773 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3774 } 3775 3776 /* 3777 * Vdev paths in the MOS may be obsolete. If the untrusted config was 3778 * obtained by scanning /dev/dsk, then it will have the right vdev 3779 * paths. We update the trusted MOS config with this information. 3780 * We first try to copy the paths with vdev_copy_path_strict, which 3781 * succeeds only when both configs have exactly the same vdev tree. 3782 * If that fails, we fall back to a more flexible method that has a 3783 * best effort policy. 3784 */ 3785 copy_error = vdev_copy_path_strict(rvd, mrvd); 3786 if (copy_error != 0 || spa_load_print_vdev_tree) { 3787 spa_load_note(spa, "provided vdev tree:"); 3788 vdev_dbgmsg_print_tree(rvd, 2); 3789 spa_load_note(spa, "MOS vdev tree:"); 3790 vdev_dbgmsg_print_tree(mrvd, 2); 3791 } 3792 if (copy_error != 0) { 3793 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 3794 "back to vdev_copy_path_relaxed"); 3795 vdev_copy_path_relaxed(rvd, mrvd); 3796 } 3797 3798 vdev_close(rvd); 3799 vdev_free(rvd); 3800 spa->spa_root_vdev = mrvd; 3801 rvd = mrvd; 3802 spa_config_exit(spa, SCL_ALL, FTAG); 3803 3804 /* 3805 * We will use spa_config if we decide to reload the spa or if spa_load 3806 * fails and we rewind. We must thus regenerate the config using the 3807 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 3808 * pass settings on how to load the pool and is not stored in the MOS. 3809 * We copy it over to our new, trusted config. 3810 */ 3811 mos_config_txg = fnvlist_lookup_uint64(mos_config, 3812 ZPOOL_CONFIG_POOL_TXG); 3813 nvlist_free(mos_config); 3814 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 3815 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 3816 &policy) == 0) 3817 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 3818 spa_config_set(spa, mos_config); 3819 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 3820 3821 /* 3822 * Now that we got the config from the MOS, we should be more strict 3823 * in checking blkptrs and can make assumptions about the consistency 3824 * of the vdev tree. spa_trust_config must be set to true before opening 3825 * vdevs in order for them to be writeable. 3826 */ 3827 spa->spa_trust_config = B_TRUE; 3828 3829 /* 3830 * Open and validate the new vdev tree 3831 */ 3832 error = spa_ld_open_vdevs(spa); 3833 if (error != 0) 3834 return (error); 3835 3836 error = spa_ld_validate_vdevs(spa); 3837 if (error != 0) 3838 return (error); 3839 3840 if (copy_error != 0 || spa_load_print_vdev_tree) { 3841 spa_load_note(spa, "final vdev tree:"); 3842 vdev_dbgmsg_print_tree(rvd, 2); 3843 } 3844 3845 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 3846 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 3847 /* 3848 * Sanity check to make sure that we are indeed loading the 3849 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 3850 * in the config provided and they happened to be the only ones 3851 * to have the latest uberblock, we could involuntarily perform 3852 * an extreme rewind. 3853 */ 3854 healthy_tvds_mos = spa_healthy_core_tvds(spa); 3855 if (healthy_tvds_mos - healthy_tvds >= 3856 SPA_SYNC_MIN_VDEVS) { 3857 spa_load_note(spa, "config provided misses too many " 3858 "top-level vdevs compared to MOS (%lld vs %lld). ", 3859 (u_longlong_t)healthy_tvds, 3860 (u_longlong_t)healthy_tvds_mos); 3861 spa_load_note(spa, "vdev tree:"); 3862 vdev_dbgmsg_print_tree(rvd, 2); 3863 if (reloading) { 3864 spa_load_failed(spa, "config was already " 3865 "provided from MOS. Aborting."); 3866 return (spa_vdev_err(rvd, 3867 VDEV_AUX_CORRUPT_DATA, EIO)); 3868 } 3869 spa_load_note(spa, "spa must be reloaded using MOS " 3870 "config"); 3871 return (SET_ERROR(EAGAIN)); 3872 } 3873 } 3874 3875 error = spa_check_for_missing_logs(spa); 3876 if (error != 0) 3877 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 3878 3879 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 3880 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 3881 "guid sum (%llu != %llu)", 3882 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 3883 (u_longlong_t)rvd->vdev_guid_sum); 3884 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 3885 ENXIO)); 3886 } 3887 3888 return (0); 3889 } 3890 3891 static int 3892 spa_ld_open_indirect_vdev_metadata(spa_t *spa) 3893 { 3894 int error = 0; 3895 vdev_t *rvd = spa->spa_root_vdev; 3896 3897 /* 3898 * Everything that we read before spa_remove_init() must be stored 3899 * on concreted vdevs. Therefore we do this as early as possible. 3900 */ 3901 error = spa_remove_init(spa); 3902 if (error != 0) { 3903 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 3904 error); 3905 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3906 } 3907 3908 /* 3909 * Retrieve information needed to condense indirect vdev mappings. 3910 */ 3911 error = spa_condense_init(spa); 3912 if (error != 0) { 3913 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 3914 error); 3915 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3916 } 3917 3918 return (0); 3919 } 3920 3921 static int 3922 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 3923 { 3924 int error = 0; 3925 vdev_t *rvd = spa->spa_root_vdev; 3926 3927 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 3928 boolean_t missing_feat_read = B_FALSE; 3929 nvlist_t *unsup_feat, *enabled_feat; 3930 3931 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 3932 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 3933 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3934 } 3935 3936 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 3937 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 3938 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3939 } 3940 3941 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 3942 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 3943 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3944 } 3945 3946 enabled_feat = fnvlist_alloc(); 3947 unsup_feat = fnvlist_alloc(); 3948 3949 if (!spa_features_check(spa, B_FALSE, 3950 unsup_feat, enabled_feat)) 3951 missing_feat_read = B_TRUE; 3952 3953 if (spa_writeable(spa) || 3954 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 3955 if (!spa_features_check(spa, B_TRUE, 3956 unsup_feat, enabled_feat)) { 3957 *missing_feat_writep = B_TRUE; 3958 } 3959 } 3960 3961 fnvlist_add_nvlist(spa->spa_load_info, 3962 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 3963 3964 if (!nvlist_empty(unsup_feat)) { 3965 fnvlist_add_nvlist(spa->spa_load_info, 3966 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 3967 } 3968 3969 fnvlist_free(enabled_feat); 3970 fnvlist_free(unsup_feat); 3971 3972 if (!missing_feat_read) { 3973 fnvlist_add_boolean(spa->spa_load_info, 3974 ZPOOL_CONFIG_CAN_RDONLY); 3975 } 3976 3977 /* 3978 * If the state is SPA_LOAD_TRYIMPORT, our objective is 3979 * twofold: to determine whether the pool is available for 3980 * import in read-write mode and (if it is not) whether the 3981 * pool is available for import in read-only mode. If the pool 3982 * is available for import in read-write mode, it is displayed 3983 * as available in userland; if it is not available for import 3984 * in read-only mode, it is displayed as unavailable in 3985 * userland. If the pool is available for import in read-only 3986 * mode but not read-write mode, it is displayed as unavailable 3987 * in userland with a special note that the pool is actually 3988 * available for open in read-only mode. 3989 * 3990 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 3991 * missing a feature for write, we must first determine whether 3992 * the pool can be opened read-only before returning to 3993 * userland in order to know whether to display the 3994 * abovementioned note. 3995 */ 3996 if (missing_feat_read || (*missing_feat_writep && 3997 spa_writeable(spa))) { 3998 spa_load_failed(spa, "pool uses unsupported features"); 3999 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 4000 ENOTSUP)); 4001 } 4002 4003 /* 4004 * Load refcounts for ZFS features from disk into an in-memory 4005 * cache during SPA initialization. 4006 */ 4007 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 4008 uint64_t refcount; 4009 4010 error = feature_get_refcount_from_disk(spa, 4011 &spa_feature_table[i], &refcount); 4012 if (error == 0) { 4013 spa->spa_feat_refcount_cache[i] = refcount; 4014 } else if (error == ENOTSUP) { 4015 spa->spa_feat_refcount_cache[i] = 4016 SPA_FEATURE_DISABLED; 4017 } else { 4018 spa_load_failed(spa, "error getting refcount " 4019 "for feature %s [error=%d]", 4020 spa_feature_table[i].fi_guid, error); 4021 return (spa_vdev_err(rvd, 4022 VDEV_AUX_CORRUPT_DATA, EIO)); 4023 } 4024 } 4025 } 4026 4027 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 4028 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 4029 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 4030 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4031 } 4032 4033 /* 4034 * Encryption was added before bookmark_v2, even though bookmark_v2 4035 * is now a dependency. If this pool has encryption enabled without 4036 * bookmark_v2, trigger an errata message. 4037 */ 4038 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 4039 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 4040 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 4041 } 4042 4043 return (0); 4044 } 4045 4046 static int 4047 spa_ld_load_special_directories(spa_t *spa) 4048 { 4049 int error = 0; 4050 vdev_t *rvd = spa->spa_root_vdev; 4051 4052 spa->spa_is_initializing = B_TRUE; 4053 error = dsl_pool_open(spa->spa_dsl_pool); 4054 spa->spa_is_initializing = B_FALSE; 4055 if (error != 0) { 4056 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 4057 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4058 } 4059 4060 return (0); 4061 } 4062 4063 static int 4064 spa_ld_get_props(spa_t *spa) 4065 { 4066 int error = 0; 4067 uint64_t obj; 4068 vdev_t *rvd = spa->spa_root_vdev; 4069 4070 /* Grab the checksum salt from the MOS. */ 4071 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4072 DMU_POOL_CHECKSUM_SALT, 1, 4073 sizeof (spa->spa_cksum_salt.zcs_bytes), 4074 spa->spa_cksum_salt.zcs_bytes); 4075 if (error == ENOENT) { 4076 /* Generate a new salt for subsequent use */ 4077 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 4078 sizeof (spa->spa_cksum_salt.zcs_bytes)); 4079 } else if (error != 0) { 4080 spa_load_failed(spa, "unable to retrieve checksum salt from " 4081 "MOS [error=%d]", error); 4082 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4083 } 4084 4085 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 4086 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4087 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 4088 if (error != 0) { 4089 spa_load_failed(spa, "error opening deferred-frees bpobj " 4090 "[error=%d]", error); 4091 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4092 } 4093 4094 /* 4095 * Load the bit that tells us to use the new accounting function 4096 * (raid-z deflation). If we have an older pool, this will not 4097 * be present. 4098 */ 4099 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 4100 if (error != 0 && error != ENOENT) 4101 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4102 4103 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 4104 &spa->spa_creation_version, B_FALSE); 4105 if (error != 0 && error != ENOENT) 4106 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4107 4108 /* 4109 * Load the persistent error log. If we have an older pool, this will 4110 * not be present. 4111 */ 4112 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 4113 B_FALSE); 4114 if (error != 0 && error != ENOENT) 4115 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4116 4117 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 4118 &spa->spa_errlog_scrub, B_FALSE); 4119 if (error != 0 && error != ENOENT) 4120 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4121 4122 /* 4123 * Load the livelist deletion field. If a livelist is queued for 4124 * deletion, indicate that in the spa 4125 */ 4126 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 4127 &spa->spa_livelists_to_delete, B_FALSE); 4128 if (error != 0 && error != ENOENT) 4129 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4130 4131 /* 4132 * Load the history object. If we have an older pool, this 4133 * will not be present. 4134 */ 4135 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 4136 if (error != 0 && error != ENOENT) 4137 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4138 4139 /* 4140 * Load the per-vdev ZAP map. If we have an older pool, this will not 4141 * be present; in this case, defer its creation to a later time to 4142 * avoid dirtying the MOS this early / out of sync context. See 4143 * spa_sync_config_object. 4144 */ 4145 4146 /* The sentinel is only available in the MOS config. */ 4147 nvlist_t *mos_config; 4148 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 4149 spa_load_failed(spa, "unable to retrieve MOS config"); 4150 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4151 } 4152 4153 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 4154 &spa->spa_all_vdev_zaps, B_FALSE); 4155 4156 if (error == ENOENT) { 4157 VERIFY(!nvlist_exists(mos_config, 4158 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 4159 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 4160 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4161 } else if (error != 0) { 4162 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4163 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 4164 /* 4165 * An older version of ZFS overwrote the sentinel value, so 4166 * we have orphaned per-vdev ZAPs in the MOS. Defer their 4167 * destruction to later; see spa_sync_config_object. 4168 */ 4169 spa->spa_avz_action = AVZ_ACTION_DESTROY; 4170 /* 4171 * We're assuming that no vdevs have had their ZAPs created 4172 * before this. Better be sure of it. 4173 */ 4174 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4175 } 4176 nvlist_free(mos_config); 4177 4178 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 4179 4180 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 4181 B_FALSE); 4182 if (error && error != ENOENT) 4183 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4184 4185 if (error == 0) { 4186 uint64_t autoreplace; 4187 4188 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 4189 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 4190 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 4191 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 4192 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 4193 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 4194 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 4195 spa->spa_autoreplace = (autoreplace != 0); 4196 } 4197 4198 /* 4199 * If we are importing a pool with missing top-level vdevs, 4200 * we enforce that the pool doesn't panic or get suspended on 4201 * error since the likelihood of missing data is extremely high. 4202 */ 4203 if (spa->spa_missing_tvds > 0 && 4204 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 4205 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4206 spa_load_note(spa, "forcing failmode to 'continue' " 4207 "as some top level vdevs are missing"); 4208 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 4209 } 4210 4211 return (0); 4212 } 4213 4214 static int 4215 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 4216 { 4217 int error = 0; 4218 vdev_t *rvd = spa->spa_root_vdev; 4219 4220 /* 4221 * If we're assembling the pool from the split-off vdevs of 4222 * an existing pool, we don't want to attach the spares & cache 4223 * devices. 4224 */ 4225 4226 /* 4227 * Load any hot spares for this pool. 4228 */ 4229 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 4230 B_FALSE); 4231 if (error != 0 && error != ENOENT) 4232 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4233 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4234 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 4235 if (load_nvlist(spa, spa->spa_spares.sav_object, 4236 &spa->spa_spares.sav_config) != 0) { 4237 spa_load_failed(spa, "error loading spares nvlist"); 4238 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4239 } 4240 4241 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4242 spa_load_spares(spa); 4243 spa_config_exit(spa, SCL_ALL, FTAG); 4244 } else if (error == 0) { 4245 spa->spa_spares.sav_sync = B_TRUE; 4246 } 4247 4248 /* 4249 * Load any level 2 ARC devices for this pool. 4250 */ 4251 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 4252 &spa->spa_l2cache.sav_object, B_FALSE); 4253 if (error != 0 && error != ENOENT) 4254 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4255 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4256 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 4257 if (load_nvlist(spa, spa->spa_l2cache.sav_object, 4258 &spa->spa_l2cache.sav_config) != 0) { 4259 spa_load_failed(spa, "error loading l2cache nvlist"); 4260 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4261 } 4262 4263 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4264 spa_load_l2cache(spa); 4265 spa_config_exit(spa, SCL_ALL, FTAG); 4266 } else if (error == 0) { 4267 spa->spa_l2cache.sav_sync = B_TRUE; 4268 } 4269 4270 return (0); 4271 } 4272 4273 static int 4274 spa_ld_load_vdev_metadata(spa_t *spa) 4275 { 4276 int error = 0; 4277 vdev_t *rvd = spa->spa_root_vdev; 4278 4279 /* 4280 * If the 'multihost' property is set, then never allow a pool to 4281 * be imported when the system hostid is zero. The exception to 4282 * this rule is zdb which is always allowed to access pools. 4283 */ 4284 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 4285 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 4286 fnvlist_add_uint64(spa->spa_load_info, 4287 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4288 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4289 } 4290 4291 /* 4292 * If the 'autoreplace' property is set, then post a resource notifying 4293 * the ZFS DE that it should not issue any faults for unopenable 4294 * devices. We also iterate over the vdevs, and post a sysevent for any 4295 * unopenable vdevs so that the normal autoreplace handler can take 4296 * over. 4297 */ 4298 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4299 spa_check_removed(spa->spa_root_vdev); 4300 /* 4301 * For the import case, this is done in spa_import(), because 4302 * at this point we're using the spare definitions from 4303 * the MOS config, not necessarily from the userland config. 4304 */ 4305 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 4306 spa_aux_check_removed(&spa->spa_spares); 4307 spa_aux_check_removed(&spa->spa_l2cache); 4308 } 4309 } 4310 4311 /* 4312 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 4313 */ 4314 error = vdev_load(rvd); 4315 if (error != 0) { 4316 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 4317 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4318 } 4319 4320 error = spa_ld_log_spacemaps(spa); 4321 if (error != 0) { 4322 spa_load_failed(spa, "spa_ld_log_sm_data failed [error=%d]", 4323 error); 4324 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4325 } 4326 4327 /* 4328 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 4329 */ 4330 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4331 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 4332 spa_config_exit(spa, SCL_ALL, FTAG); 4333 4334 return (0); 4335 } 4336 4337 static int 4338 spa_ld_load_dedup_tables(spa_t *spa) 4339 { 4340 int error = 0; 4341 vdev_t *rvd = spa->spa_root_vdev; 4342 4343 error = ddt_load(spa); 4344 if (error != 0) { 4345 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 4346 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4347 } 4348 4349 return (0); 4350 } 4351 4352 static int 4353 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, char **ereport) 4354 { 4355 vdev_t *rvd = spa->spa_root_vdev; 4356 4357 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 4358 boolean_t missing = spa_check_logs(spa); 4359 if (missing) { 4360 if (spa->spa_missing_tvds != 0) { 4361 spa_load_note(spa, "spa_check_logs failed " 4362 "so dropping the logs"); 4363 } else { 4364 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 4365 spa_load_failed(spa, "spa_check_logs failed"); 4366 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 4367 ENXIO)); 4368 } 4369 } 4370 } 4371 4372 return (0); 4373 } 4374 4375 static int 4376 spa_ld_verify_pool_data(spa_t *spa) 4377 { 4378 int error = 0; 4379 vdev_t *rvd = spa->spa_root_vdev; 4380 4381 /* 4382 * We've successfully opened the pool, verify that we're ready 4383 * to start pushing transactions. 4384 */ 4385 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4386 error = spa_load_verify(spa); 4387 if (error != 0) { 4388 spa_load_failed(spa, "spa_load_verify failed " 4389 "[error=%d]", error); 4390 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4391 error)); 4392 } 4393 } 4394 4395 return (0); 4396 } 4397 4398 static void 4399 spa_ld_claim_log_blocks(spa_t *spa) 4400 { 4401 dmu_tx_t *tx; 4402 dsl_pool_t *dp = spa_get_dsl(spa); 4403 4404 /* 4405 * Claim log blocks that haven't been committed yet. 4406 * This must all happen in a single txg. 4407 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 4408 * invoked from zil_claim_log_block()'s i/o done callback. 4409 * Price of rollback is that we abandon the log. 4410 */ 4411 spa->spa_claiming = B_TRUE; 4412 4413 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 4414 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 4415 zil_claim, tx, DS_FIND_CHILDREN); 4416 dmu_tx_commit(tx); 4417 4418 spa->spa_claiming = B_FALSE; 4419 4420 spa_set_log_state(spa, SPA_LOG_GOOD); 4421 } 4422 4423 static void 4424 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 4425 boolean_t update_config_cache) 4426 { 4427 vdev_t *rvd = spa->spa_root_vdev; 4428 int need_update = B_FALSE; 4429 4430 /* 4431 * If the config cache is stale, or we have uninitialized 4432 * metaslabs (see spa_vdev_add()), then update the config. 4433 * 4434 * If this is a verbatim import, trust the current 4435 * in-core spa_config and update the disk labels. 4436 */ 4437 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 4438 spa->spa_load_state == SPA_LOAD_IMPORT || 4439 spa->spa_load_state == SPA_LOAD_RECOVER || 4440 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 4441 need_update = B_TRUE; 4442 4443 for (int c = 0; c < rvd->vdev_children; c++) 4444 if (rvd->vdev_child[c]->vdev_ms_array == 0) 4445 need_update = B_TRUE; 4446 4447 /* 4448 * Update the config cache asynchronously in case we're the 4449 * root pool, in which case the config cache isn't writable yet. 4450 */ 4451 if (need_update) 4452 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 4453 } 4454 4455 static void 4456 spa_ld_prepare_for_reload(spa_t *spa) 4457 { 4458 spa_mode_t mode = spa->spa_mode; 4459 int async_suspended = spa->spa_async_suspended; 4460 4461 spa_unload(spa); 4462 spa_deactivate(spa); 4463 spa_activate(spa, mode); 4464 4465 /* 4466 * We save the value of spa_async_suspended as it gets reset to 0 by 4467 * spa_unload(). We want to restore it back to the original value before 4468 * returning as we might be calling spa_async_resume() later. 4469 */ 4470 spa->spa_async_suspended = async_suspended; 4471 } 4472 4473 static int 4474 spa_ld_read_checkpoint_txg(spa_t *spa) 4475 { 4476 uberblock_t checkpoint; 4477 int error = 0; 4478 4479 ASSERT0(spa->spa_checkpoint_txg); 4480 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4481 4482 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4483 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4484 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4485 4486 if (error == ENOENT) 4487 return (0); 4488 4489 if (error != 0) 4490 return (error); 4491 4492 ASSERT3U(checkpoint.ub_txg, !=, 0); 4493 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 4494 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 4495 spa->spa_checkpoint_txg = checkpoint.ub_txg; 4496 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 4497 4498 return (0); 4499 } 4500 4501 static int 4502 spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 4503 { 4504 int error = 0; 4505 4506 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4507 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4508 4509 /* 4510 * Never trust the config that is provided unless we are assembling 4511 * a pool following a split. 4512 * This means don't trust blkptrs and the vdev tree in general. This 4513 * also effectively puts the spa in read-only mode since 4514 * spa_writeable() checks for spa_trust_config to be true. 4515 * We will later load a trusted config from the MOS. 4516 */ 4517 if (type != SPA_IMPORT_ASSEMBLE) 4518 spa->spa_trust_config = B_FALSE; 4519 4520 /* 4521 * Parse the config provided to create a vdev tree. 4522 */ 4523 error = spa_ld_parse_config(spa, type); 4524 if (error != 0) 4525 return (error); 4526 4527 spa_import_progress_add(spa); 4528 4529 /* 4530 * Now that we have the vdev tree, try to open each vdev. This involves 4531 * opening the underlying physical device, retrieving its geometry and 4532 * probing the vdev with a dummy I/O. The state of each vdev will be set 4533 * based on the success of those operations. After this we'll be ready 4534 * to read from the vdevs. 4535 */ 4536 error = spa_ld_open_vdevs(spa); 4537 if (error != 0) 4538 return (error); 4539 4540 /* 4541 * Read the label of each vdev and make sure that the GUIDs stored 4542 * there match the GUIDs in the config provided. 4543 * If we're assembling a new pool that's been split off from an 4544 * existing pool, the labels haven't yet been updated so we skip 4545 * validation for now. 4546 */ 4547 if (type != SPA_IMPORT_ASSEMBLE) { 4548 error = spa_ld_validate_vdevs(spa); 4549 if (error != 0) 4550 return (error); 4551 } 4552 4553 /* 4554 * Read all vdev labels to find the best uberblock (i.e. latest, 4555 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 4556 * get the list of features required to read blkptrs in the MOS from 4557 * the vdev label with the best uberblock and verify that our version 4558 * of zfs supports them all. 4559 */ 4560 error = spa_ld_select_uberblock(spa, type); 4561 if (error != 0) 4562 return (error); 4563 4564 /* 4565 * Pass that uberblock to the dsl_pool layer which will open the root 4566 * blkptr. This blkptr points to the latest version of the MOS and will 4567 * allow us to read its contents. 4568 */ 4569 error = spa_ld_open_rootbp(spa); 4570 if (error != 0) 4571 return (error); 4572 4573 return (0); 4574 } 4575 4576 static int 4577 spa_ld_checkpoint_rewind(spa_t *spa) 4578 { 4579 uberblock_t checkpoint; 4580 int error = 0; 4581 4582 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4583 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4584 4585 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4586 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4587 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4588 4589 if (error != 0) { 4590 spa_load_failed(spa, "unable to retrieve checkpointed " 4591 "uberblock from the MOS config [error=%d]", error); 4592 4593 if (error == ENOENT) 4594 error = ZFS_ERR_NO_CHECKPOINT; 4595 4596 return (error); 4597 } 4598 4599 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 4600 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 4601 4602 /* 4603 * We need to update the txg and timestamp of the checkpointed 4604 * uberblock to be higher than the latest one. This ensures that 4605 * the checkpointed uberblock is selected if we were to close and 4606 * reopen the pool right after we've written it in the vdev labels. 4607 * (also see block comment in vdev_uberblock_compare) 4608 */ 4609 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 4610 checkpoint.ub_timestamp = gethrestime_sec(); 4611 4612 /* 4613 * Set current uberblock to be the checkpointed uberblock. 4614 */ 4615 spa->spa_uberblock = checkpoint; 4616 4617 /* 4618 * If we are doing a normal rewind, then the pool is open for 4619 * writing and we sync the "updated" checkpointed uberblock to 4620 * disk. Once this is done, we've basically rewound the whole 4621 * pool and there is no way back. 4622 * 4623 * There are cases when we don't want to attempt and sync the 4624 * checkpointed uberblock to disk because we are opening a 4625 * pool as read-only. Specifically, verifying the checkpointed 4626 * state with zdb, and importing the checkpointed state to get 4627 * a "preview" of its content. 4628 */ 4629 if (spa_writeable(spa)) { 4630 vdev_t *rvd = spa->spa_root_vdev; 4631 4632 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4633 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 4634 int svdcount = 0; 4635 int children = rvd->vdev_children; 4636 int c0 = random_in_range(children); 4637 4638 for (int c = 0; c < children; c++) { 4639 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 4640 4641 /* Stop when revisiting the first vdev */ 4642 if (c > 0 && svd[0] == vd) 4643 break; 4644 4645 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 4646 !vdev_is_concrete(vd)) 4647 continue; 4648 4649 svd[svdcount++] = vd; 4650 if (svdcount == SPA_SYNC_MIN_VDEVS) 4651 break; 4652 } 4653 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg); 4654 if (error == 0) 4655 spa->spa_last_synced_guid = rvd->vdev_guid; 4656 spa_config_exit(spa, SCL_ALL, FTAG); 4657 4658 if (error != 0) { 4659 spa_load_failed(spa, "failed to write checkpointed " 4660 "uberblock to the vdev labels [error=%d]", error); 4661 return (error); 4662 } 4663 } 4664 4665 return (0); 4666 } 4667 4668 static int 4669 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 4670 boolean_t *update_config_cache) 4671 { 4672 int error; 4673 4674 /* 4675 * Parse the config for pool, open and validate vdevs, 4676 * select an uberblock, and use that uberblock to open 4677 * the MOS. 4678 */ 4679 error = spa_ld_mos_init(spa, type); 4680 if (error != 0) 4681 return (error); 4682 4683 /* 4684 * Retrieve the trusted config stored in the MOS and use it to create 4685 * a new, exact version of the vdev tree, then reopen all vdevs. 4686 */ 4687 error = spa_ld_trusted_config(spa, type, B_FALSE); 4688 if (error == EAGAIN) { 4689 if (update_config_cache != NULL) 4690 *update_config_cache = B_TRUE; 4691 4692 /* 4693 * Redo the loading process with the trusted config if it is 4694 * too different from the untrusted config. 4695 */ 4696 spa_ld_prepare_for_reload(spa); 4697 spa_load_note(spa, "RELOADING"); 4698 error = spa_ld_mos_init(spa, type); 4699 if (error != 0) 4700 return (error); 4701 4702 error = spa_ld_trusted_config(spa, type, B_TRUE); 4703 if (error != 0) 4704 return (error); 4705 4706 } else if (error != 0) { 4707 return (error); 4708 } 4709 4710 return (0); 4711 } 4712 4713 /* 4714 * Load an existing storage pool, using the config provided. This config 4715 * describes which vdevs are part of the pool and is later validated against 4716 * partial configs present in each vdev's label and an entire copy of the 4717 * config stored in the MOS. 4718 */ 4719 static int 4720 spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport) 4721 { 4722 int error = 0; 4723 boolean_t missing_feat_write = B_FALSE; 4724 boolean_t checkpoint_rewind = 4725 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4726 boolean_t update_config_cache = B_FALSE; 4727 4728 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4729 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4730 4731 spa_load_note(spa, "LOADING"); 4732 4733 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 4734 if (error != 0) 4735 return (error); 4736 4737 /* 4738 * If we are rewinding to the checkpoint then we need to repeat 4739 * everything we've done so far in this function but this time 4740 * selecting the checkpointed uberblock and using that to open 4741 * the MOS. 4742 */ 4743 if (checkpoint_rewind) { 4744 /* 4745 * If we are rewinding to the checkpoint update config cache 4746 * anyway. 4747 */ 4748 update_config_cache = B_TRUE; 4749 4750 /* 4751 * Extract the checkpointed uberblock from the current MOS 4752 * and use this as the pool's uberblock from now on. If the 4753 * pool is imported as writeable we also write the checkpoint 4754 * uberblock to the labels, making the rewind permanent. 4755 */ 4756 error = spa_ld_checkpoint_rewind(spa); 4757 if (error != 0) 4758 return (error); 4759 4760 /* 4761 * Redo the loading process again with the 4762 * checkpointed uberblock. 4763 */ 4764 spa_ld_prepare_for_reload(spa); 4765 spa_load_note(spa, "LOADING checkpointed uberblock"); 4766 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 4767 if (error != 0) 4768 return (error); 4769 } 4770 4771 /* 4772 * Retrieve the checkpoint txg if the pool has a checkpoint. 4773 */ 4774 error = spa_ld_read_checkpoint_txg(spa); 4775 if (error != 0) 4776 return (error); 4777 4778 /* 4779 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 4780 * from the pool and their contents were re-mapped to other vdevs. Note 4781 * that everything that we read before this step must have been 4782 * rewritten on concrete vdevs after the last device removal was 4783 * initiated. Otherwise we could be reading from indirect vdevs before 4784 * we have loaded their mappings. 4785 */ 4786 error = spa_ld_open_indirect_vdev_metadata(spa); 4787 if (error != 0) 4788 return (error); 4789 4790 /* 4791 * Retrieve the full list of active features from the MOS and check if 4792 * they are all supported. 4793 */ 4794 error = spa_ld_check_features(spa, &missing_feat_write); 4795 if (error != 0) 4796 return (error); 4797 4798 /* 4799 * Load several special directories from the MOS needed by the dsl_pool 4800 * layer. 4801 */ 4802 error = spa_ld_load_special_directories(spa); 4803 if (error != 0) 4804 return (error); 4805 4806 /* 4807 * Retrieve pool properties from the MOS. 4808 */ 4809 error = spa_ld_get_props(spa); 4810 if (error != 0) 4811 return (error); 4812 4813 /* 4814 * Retrieve the list of auxiliary devices - cache devices and spares - 4815 * and open them. 4816 */ 4817 error = spa_ld_open_aux_vdevs(spa, type); 4818 if (error != 0) 4819 return (error); 4820 4821 /* 4822 * Load the metadata for all vdevs. Also check if unopenable devices 4823 * should be autoreplaced. 4824 */ 4825 error = spa_ld_load_vdev_metadata(spa); 4826 if (error != 0) 4827 return (error); 4828 4829 error = spa_ld_load_dedup_tables(spa); 4830 if (error != 0) 4831 return (error); 4832 4833 /* 4834 * Verify the logs now to make sure we don't have any unexpected errors 4835 * when we claim log blocks later. 4836 */ 4837 error = spa_ld_verify_logs(spa, type, ereport); 4838 if (error != 0) 4839 return (error); 4840 4841 if (missing_feat_write) { 4842 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 4843 4844 /* 4845 * At this point, we know that we can open the pool in 4846 * read-only mode but not read-write mode. We now have enough 4847 * information and can return to userland. 4848 */ 4849 return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 4850 ENOTSUP)); 4851 } 4852 4853 /* 4854 * Traverse the last txgs to make sure the pool was left off in a safe 4855 * state. When performing an extreme rewind, we verify the whole pool, 4856 * which can take a very long time. 4857 */ 4858 error = spa_ld_verify_pool_data(spa); 4859 if (error != 0) 4860 return (error); 4861 4862 /* 4863 * Calculate the deflated space for the pool. This must be done before 4864 * we write anything to the pool because we'd need to update the space 4865 * accounting using the deflated sizes. 4866 */ 4867 spa_update_dspace(spa); 4868 4869 /* 4870 * We have now retrieved all the information we needed to open the 4871 * pool. If we are importing the pool in read-write mode, a few 4872 * additional steps must be performed to finish the import. 4873 */ 4874 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 4875 spa->spa_load_max_txg == UINT64_MAX)) { 4876 uint64_t config_cache_txg = spa->spa_config_txg; 4877 4878 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 4879 4880 /* 4881 * In case of a checkpoint rewind, log the original txg 4882 * of the checkpointed uberblock. 4883 */ 4884 if (checkpoint_rewind) { 4885 spa_history_log_internal(spa, "checkpoint rewind", 4886 NULL, "rewound state to txg=%llu", 4887 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 4888 } 4889 4890 /* 4891 * Traverse the ZIL and claim all blocks. 4892 */ 4893 spa_ld_claim_log_blocks(spa); 4894 4895 /* 4896 * Kick-off the syncing thread. 4897 */ 4898 spa->spa_sync_on = B_TRUE; 4899 txg_sync_start(spa->spa_dsl_pool); 4900 mmp_thread_start(spa); 4901 4902 /* 4903 * Wait for all claims to sync. We sync up to the highest 4904 * claimed log block birth time so that claimed log blocks 4905 * don't appear to be from the future. spa_claim_max_txg 4906 * will have been set for us by ZIL traversal operations 4907 * performed above. 4908 */ 4909 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 4910 4911 /* 4912 * Check if we need to request an update of the config. On the 4913 * next sync, we would update the config stored in vdev labels 4914 * and the cachefile (by default /etc/zfs/zpool.cache). 4915 */ 4916 spa_ld_check_for_config_update(spa, config_cache_txg, 4917 update_config_cache); 4918 4919 /* 4920 * Check if a rebuild was in progress and if so resume it. 4921 * Then check all DTLs to see if anything needs resilvering. 4922 * The resilver will be deferred if a rebuild was started. 4923 */ 4924 if (vdev_rebuild_active(spa->spa_root_vdev)) { 4925 vdev_rebuild_restart(spa); 4926 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 4927 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 4928 spa_async_request(spa, SPA_ASYNC_RESILVER); 4929 } 4930 4931 /* 4932 * Log the fact that we booted up (so that we can detect if 4933 * we rebooted in the middle of an operation). 4934 */ 4935 spa_history_log_version(spa, "open", NULL); 4936 4937 spa_restart_removal(spa); 4938 spa_spawn_aux_threads(spa); 4939 4940 /* 4941 * Delete any inconsistent datasets. 4942 * 4943 * Note: 4944 * Since we may be issuing deletes for clones here, 4945 * we make sure to do so after we've spawned all the 4946 * auxiliary threads above (from which the livelist 4947 * deletion zthr is part of). 4948 */ 4949 (void) dmu_objset_find(spa_name(spa), 4950 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 4951 4952 /* 4953 * Clean up any stale temporary dataset userrefs. 4954 */ 4955 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 4956 4957 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 4958 vdev_initialize_restart(spa->spa_root_vdev); 4959 vdev_trim_restart(spa->spa_root_vdev); 4960 vdev_autotrim_restart(spa); 4961 spa_config_exit(spa, SCL_CONFIG, FTAG); 4962 } 4963 4964 spa_import_progress_remove(spa_guid(spa)); 4965 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 4966 4967 spa_load_note(spa, "LOADED"); 4968 4969 return (0); 4970 } 4971 4972 static int 4973 spa_load_retry(spa_t *spa, spa_load_state_t state) 4974 { 4975 spa_mode_t mode = spa->spa_mode; 4976 4977 spa_unload(spa); 4978 spa_deactivate(spa); 4979 4980 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 4981 4982 spa_activate(spa, mode); 4983 spa_async_suspend(spa); 4984 4985 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 4986 (u_longlong_t)spa->spa_load_max_txg); 4987 4988 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 4989 } 4990 4991 /* 4992 * If spa_load() fails this function will try loading prior txg's. If 4993 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 4994 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 4995 * function will not rewind the pool and will return the same error as 4996 * spa_load(). 4997 */ 4998 static int 4999 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 5000 int rewind_flags) 5001 { 5002 nvlist_t *loadinfo = NULL; 5003 nvlist_t *config = NULL; 5004 int load_error, rewind_error; 5005 uint64_t safe_rewind_txg; 5006 uint64_t min_txg; 5007 5008 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 5009 spa->spa_load_max_txg = spa->spa_load_txg; 5010 spa_set_log_state(spa, SPA_LOG_CLEAR); 5011 } else { 5012 spa->spa_load_max_txg = max_request; 5013 if (max_request != UINT64_MAX) 5014 spa->spa_extreme_rewind = B_TRUE; 5015 } 5016 5017 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 5018 if (load_error == 0) 5019 return (0); 5020 if (load_error == ZFS_ERR_NO_CHECKPOINT) { 5021 /* 5022 * When attempting checkpoint-rewind on a pool with no 5023 * checkpoint, we should not attempt to load uberblocks 5024 * from previous txgs when spa_load fails. 5025 */ 5026 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5027 spa_import_progress_remove(spa_guid(spa)); 5028 return (load_error); 5029 } 5030 5031 if (spa->spa_root_vdev != NULL) 5032 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5033 5034 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 5035 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 5036 5037 if (rewind_flags & ZPOOL_NEVER_REWIND) { 5038 nvlist_free(config); 5039 spa_import_progress_remove(spa_guid(spa)); 5040 return (load_error); 5041 } 5042 5043 if (state == SPA_LOAD_RECOVER) { 5044 /* Price of rolling back is discarding txgs, including log */ 5045 spa_set_log_state(spa, SPA_LOG_CLEAR); 5046 } else { 5047 /* 5048 * If we aren't rolling back save the load info from our first 5049 * import attempt so that we can restore it after attempting 5050 * to rewind. 5051 */ 5052 loadinfo = spa->spa_load_info; 5053 spa->spa_load_info = fnvlist_alloc(); 5054 } 5055 5056 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 5057 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 5058 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 5059 TXG_INITIAL : safe_rewind_txg; 5060 5061 /* 5062 * Continue as long as we're finding errors, we're still within 5063 * the acceptable rewind range, and we're still finding uberblocks 5064 */ 5065 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 5066 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 5067 if (spa->spa_load_max_txg < safe_rewind_txg) 5068 spa->spa_extreme_rewind = B_TRUE; 5069 rewind_error = spa_load_retry(spa, state); 5070 } 5071 5072 spa->spa_extreme_rewind = B_FALSE; 5073 spa->spa_load_max_txg = UINT64_MAX; 5074 5075 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 5076 spa_config_set(spa, config); 5077 else 5078 nvlist_free(config); 5079 5080 if (state == SPA_LOAD_RECOVER) { 5081 ASSERT3P(loadinfo, ==, NULL); 5082 spa_import_progress_remove(spa_guid(spa)); 5083 return (rewind_error); 5084 } else { 5085 /* Store the rewind info as part of the initial load info */ 5086 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 5087 spa->spa_load_info); 5088 5089 /* Restore the initial load info */ 5090 fnvlist_free(spa->spa_load_info); 5091 spa->spa_load_info = loadinfo; 5092 5093 spa_import_progress_remove(spa_guid(spa)); 5094 return (load_error); 5095 } 5096 } 5097 5098 /* 5099 * Pool Open/Import 5100 * 5101 * The import case is identical to an open except that the configuration is sent 5102 * down from userland, instead of grabbed from the configuration cache. For the 5103 * case of an open, the pool configuration will exist in the 5104 * POOL_STATE_UNINITIALIZED state. 5105 * 5106 * The stats information (gen/count/ustats) is used to gather vdev statistics at 5107 * the same time open the pool, without having to keep around the spa_t in some 5108 * ambiguous state. 5109 */ 5110 static int 5111 spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, 5112 nvlist_t **config) 5113 { 5114 spa_t *spa; 5115 spa_load_state_t state = SPA_LOAD_OPEN; 5116 int error; 5117 int locked = B_FALSE; 5118 int firstopen = B_FALSE; 5119 5120 *spapp = NULL; 5121 5122 /* 5123 * As disgusting as this is, we need to support recursive calls to this 5124 * function because dsl_dir_open() is called during spa_load(), and ends 5125 * up calling spa_open() again. The real fix is to figure out how to 5126 * avoid dsl_dir_open() calling this in the first place. 5127 */ 5128 if (MUTEX_NOT_HELD(&spa_namespace_lock)) { 5129 mutex_enter(&spa_namespace_lock); 5130 locked = B_TRUE; 5131 } 5132 5133 if ((spa = spa_lookup(pool)) == NULL) { 5134 if (locked) 5135 mutex_exit(&spa_namespace_lock); 5136 return (SET_ERROR(ENOENT)); 5137 } 5138 5139 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 5140 zpool_load_policy_t policy; 5141 5142 firstopen = B_TRUE; 5143 5144 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 5145 &policy); 5146 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 5147 state = SPA_LOAD_RECOVER; 5148 5149 spa_activate(spa, spa_mode_global); 5150 5151 if (state != SPA_LOAD_RECOVER) 5152 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 5153 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 5154 5155 zfs_dbgmsg("spa_open_common: opening %s", pool); 5156 error = spa_load_best(spa, state, policy.zlp_txg, 5157 policy.zlp_rewind); 5158 5159 if (error == EBADF) { 5160 /* 5161 * If vdev_validate() returns failure (indicated by 5162 * EBADF), it indicates that one of the vdevs indicates 5163 * that the pool has been exported or destroyed. If 5164 * this is the case, the config cache is out of sync and 5165 * we should remove the pool from the namespace. 5166 */ 5167 spa_unload(spa); 5168 spa_deactivate(spa); 5169 spa_write_cachefile(spa, B_TRUE, B_TRUE); 5170 spa_remove(spa); 5171 if (locked) 5172 mutex_exit(&spa_namespace_lock); 5173 return (SET_ERROR(ENOENT)); 5174 } 5175 5176 if (error) { 5177 /* 5178 * We can't open the pool, but we still have useful 5179 * information: the state of each vdev after the 5180 * attempted vdev_open(). Return this to the user. 5181 */ 5182 if (config != NULL && spa->spa_config) { 5183 VERIFY(nvlist_dup(spa->spa_config, config, 5184 KM_SLEEP) == 0); 5185 VERIFY(nvlist_add_nvlist(*config, 5186 ZPOOL_CONFIG_LOAD_INFO, 5187 spa->spa_load_info) == 0); 5188 } 5189 spa_unload(spa); 5190 spa_deactivate(spa); 5191 spa->spa_last_open_failed = error; 5192 if (locked) 5193 mutex_exit(&spa_namespace_lock); 5194 *spapp = NULL; 5195 return (error); 5196 } 5197 } 5198 5199 spa_open_ref(spa, tag); 5200 5201 if (config != NULL) 5202 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5203 5204 /* 5205 * If we've recovered the pool, pass back any information we 5206 * gathered while doing the load. 5207 */ 5208 if (state == SPA_LOAD_RECOVER) { 5209 VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 5210 spa->spa_load_info) == 0); 5211 } 5212 5213 if (locked) { 5214 spa->spa_last_open_failed = 0; 5215 spa->spa_last_ubsync_txg = 0; 5216 spa->spa_load_txg = 0; 5217 mutex_exit(&spa_namespace_lock); 5218 } 5219 5220 if (firstopen) 5221 zvol_create_minors_recursive(spa_name(spa)); 5222 5223 *spapp = spa; 5224 5225 return (0); 5226 } 5227 5228 int 5229 spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy, 5230 nvlist_t **config) 5231 { 5232 return (spa_open_common(name, spapp, tag, policy, config)); 5233 } 5234 5235 int 5236 spa_open(const char *name, spa_t **spapp, void *tag) 5237 { 5238 return (spa_open_common(name, spapp, tag, NULL, NULL)); 5239 } 5240 5241 /* 5242 * Lookup the given spa_t, incrementing the inject count in the process, 5243 * preventing it from being exported or destroyed. 5244 */ 5245 spa_t * 5246 spa_inject_addref(char *name) 5247 { 5248 spa_t *spa; 5249 5250 mutex_enter(&spa_namespace_lock); 5251 if ((spa = spa_lookup(name)) == NULL) { 5252 mutex_exit(&spa_namespace_lock); 5253 return (NULL); 5254 } 5255 spa->spa_inject_ref++; 5256 mutex_exit(&spa_namespace_lock); 5257 5258 return (spa); 5259 } 5260 5261 void 5262 spa_inject_delref(spa_t *spa) 5263 { 5264 mutex_enter(&spa_namespace_lock); 5265 spa->spa_inject_ref--; 5266 mutex_exit(&spa_namespace_lock); 5267 } 5268 5269 /* 5270 * Add spares device information to the nvlist. 5271 */ 5272 static void 5273 spa_add_spares(spa_t *spa, nvlist_t *config) 5274 { 5275 nvlist_t **spares; 5276 uint_t i, nspares; 5277 nvlist_t *nvroot; 5278 uint64_t guid; 5279 vdev_stat_t *vs; 5280 uint_t vsc; 5281 uint64_t pool; 5282 5283 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5284 5285 if (spa->spa_spares.sav_count == 0) 5286 return; 5287 5288 VERIFY(nvlist_lookup_nvlist(config, 5289 ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); 5290 VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 5291 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 5292 if (nspares != 0) { 5293 VERIFY(nvlist_add_nvlist_array(nvroot, 5294 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 5295 VERIFY(nvlist_lookup_nvlist_array(nvroot, 5296 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 5297 5298 /* 5299 * Go through and find any spares which have since been 5300 * repurposed as an active spare. If this is the case, update 5301 * their status appropriately. 5302 */ 5303 for (i = 0; i < nspares; i++) { 5304 VERIFY(nvlist_lookup_uint64(spares[i], 5305 ZPOOL_CONFIG_GUID, &guid) == 0); 5306 if (spa_spare_exists(guid, &pool, NULL) && 5307 pool != 0ULL) { 5308 VERIFY(nvlist_lookup_uint64_array( 5309 spares[i], ZPOOL_CONFIG_VDEV_STATS, 5310 (uint64_t **)&vs, &vsc) == 0); 5311 vs->vs_state = VDEV_STATE_CANT_OPEN; 5312 vs->vs_aux = VDEV_AUX_SPARED; 5313 } 5314 } 5315 } 5316 } 5317 5318 /* 5319 * Add l2cache device information to the nvlist, including vdev stats. 5320 */ 5321 static void 5322 spa_add_l2cache(spa_t *spa, nvlist_t *config) 5323 { 5324 nvlist_t **l2cache; 5325 uint_t i, j, nl2cache; 5326 nvlist_t *nvroot; 5327 uint64_t guid; 5328 vdev_t *vd; 5329 vdev_stat_t *vs; 5330 uint_t vsc; 5331 5332 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5333 5334 if (spa->spa_l2cache.sav_count == 0) 5335 return; 5336 5337 VERIFY(nvlist_lookup_nvlist(config, 5338 ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); 5339 VERIFY(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 5340 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 5341 if (nl2cache != 0) { 5342 VERIFY(nvlist_add_nvlist_array(nvroot, 5343 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 5344 VERIFY(nvlist_lookup_nvlist_array(nvroot, 5345 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 5346 5347 /* 5348 * Update level 2 cache device stats. 5349 */ 5350 5351 for (i = 0; i < nl2cache; i++) { 5352 VERIFY(nvlist_lookup_uint64(l2cache[i], 5353 ZPOOL_CONFIG_GUID, &guid) == 0); 5354 5355 vd = NULL; 5356 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 5357 if (guid == 5358 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 5359 vd = spa->spa_l2cache.sav_vdevs[j]; 5360 break; 5361 } 5362 } 5363 ASSERT(vd != NULL); 5364 5365 VERIFY(nvlist_lookup_uint64_array(l2cache[i], 5366 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc) 5367 == 0); 5368 vdev_get_stats(vd, vs); 5369 vdev_config_generate_stats(vd, l2cache[i]); 5370 5371 } 5372 } 5373 } 5374 5375 static void 5376 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 5377 { 5378 zap_cursor_t zc; 5379 zap_attribute_t za; 5380 5381 if (spa->spa_feat_for_read_obj != 0) { 5382 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5383 spa->spa_feat_for_read_obj); 5384 zap_cursor_retrieve(&zc, &za) == 0; 5385 zap_cursor_advance(&zc)) { 5386 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5387 za.za_num_integers == 1); 5388 VERIFY0(nvlist_add_uint64(features, za.za_name, 5389 za.za_first_integer)); 5390 } 5391 zap_cursor_fini(&zc); 5392 } 5393 5394 if (spa->spa_feat_for_write_obj != 0) { 5395 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5396 spa->spa_feat_for_write_obj); 5397 zap_cursor_retrieve(&zc, &za) == 0; 5398 zap_cursor_advance(&zc)) { 5399 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5400 za.za_num_integers == 1); 5401 VERIFY0(nvlist_add_uint64(features, za.za_name, 5402 za.za_first_integer)); 5403 } 5404 zap_cursor_fini(&zc); 5405 } 5406 } 5407 5408 static void 5409 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 5410 { 5411 int i; 5412 5413 for (i = 0; i < SPA_FEATURES; i++) { 5414 zfeature_info_t feature = spa_feature_table[i]; 5415 uint64_t refcount; 5416 5417 if (feature_get_refcount(spa, &feature, &refcount) != 0) 5418 continue; 5419 5420 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 5421 } 5422 } 5423 5424 /* 5425 * Store a list of pool features and their reference counts in the 5426 * config. 5427 * 5428 * The first time this is called on a spa, allocate a new nvlist, fetch 5429 * the pool features and reference counts from disk, then save the list 5430 * in the spa. In subsequent calls on the same spa use the saved nvlist 5431 * and refresh its values from the cached reference counts. This 5432 * ensures we don't block here on I/O on a suspended pool so 'zpool 5433 * clear' can resume the pool. 5434 */ 5435 static void 5436 spa_add_feature_stats(spa_t *spa, nvlist_t *config) 5437 { 5438 nvlist_t *features; 5439 5440 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5441 5442 mutex_enter(&spa->spa_feat_stats_lock); 5443 features = spa->spa_feat_stats; 5444 5445 if (features != NULL) { 5446 spa_feature_stats_from_cache(spa, features); 5447 } else { 5448 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 5449 spa->spa_feat_stats = features; 5450 spa_feature_stats_from_disk(spa, features); 5451 } 5452 5453 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 5454 features)); 5455 5456 mutex_exit(&spa->spa_feat_stats_lock); 5457 } 5458 5459 int 5460 spa_get_stats(const char *name, nvlist_t **config, 5461 char *altroot, size_t buflen) 5462 { 5463 int error; 5464 spa_t *spa; 5465 5466 *config = NULL; 5467 error = spa_open_common(name, &spa, FTAG, NULL, config); 5468 5469 if (spa != NULL) { 5470 /* 5471 * This still leaves a window of inconsistency where the spares 5472 * or l2cache devices could change and the config would be 5473 * self-inconsistent. 5474 */ 5475 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5476 5477 if (*config != NULL) { 5478 uint64_t loadtimes[2]; 5479 5480 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 5481 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 5482 VERIFY(nvlist_add_uint64_array(*config, 5483 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2) == 0); 5484 5485 VERIFY(nvlist_add_uint64(*config, 5486 ZPOOL_CONFIG_ERRCOUNT, 5487 spa_get_errlog_size(spa)) == 0); 5488 5489 if (spa_suspended(spa)) { 5490 VERIFY(nvlist_add_uint64(*config, 5491 ZPOOL_CONFIG_SUSPENDED, 5492 spa->spa_failmode) == 0); 5493 VERIFY(nvlist_add_uint64(*config, 5494 ZPOOL_CONFIG_SUSPENDED_REASON, 5495 spa->spa_suspended) == 0); 5496 } 5497 5498 spa_add_spares(spa, *config); 5499 spa_add_l2cache(spa, *config); 5500 spa_add_feature_stats(spa, *config); 5501 } 5502 } 5503 5504 /* 5505 * We want to get the alternate root even for faulted pools, so we cheat 5506 * and call spa_lookup() directly. 5507 */ 5508 if (altroot) { 5509 if (spa == NULL) { 5510 mutex_enter(&spa_namespace_lock); 5511 spa = spa_lookup(name); 5512 if (spa) 5513 spa_altroot(spa, altroot, buflen); 5514 else 5515 altroot[0] = '\0'; 5516 spa = NULL; 5517 mutex_exit(&spa_namespace_lock); 5518 } else { 5519 spa_altroot(spa, altroot, buflen); 5520 } 5521 } 5522 5523 if (spa != NULL) { 5524 spa_config_exit(spa, SCL_CONFIG, FTAG); 5525 spa_close(spa, FTAG); 5526 } 5527 5528 return (error); 5529 } 5530 5531 /* 5532 * Validate that the auxiliary device array is well formed. We must have an 5533 * array of nvlists, each which describes a valid leaf vdev. If this is an 5534 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 5535 * specified, as long as they are well-formed. 5536 */ 5537 static int 5538 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 5539 spa_aux_vdev_t *sav, const char *config, uint64_t version, 5540 vdev_labeltype_t label) 5541 { 5542 nvlist_t **dev; 5543 uint_t i, ndev; 5544 vdev_t *vd; 5545 int error; 5546 5547 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5548 5549 /* 5550 * It's acceptable to have no devs specified. 5551 */ 5552 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 5553 return (0); 5554 5555 if (ndev == 0) 5556 return (SET_ERROR(EINVAL)); 5557 5558 /* 5559 * Make sure the pool is formatted with a version that supports this 5560 * device type. 5561 */ 5562 if (spa_version(spa) < version) 5563 return (SET_ERROR(ENOTSUP)); 5564 5565 /* 5566 * Set the pending device list so we correctly handle device in-use 5567 * checking. 5568 */ 5569 sav->sav_pending = dev; 5570 sav->sav_npending = ndev; 5571 5572 for (i = 0; i < ndev; i++) { 5573 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 5574 mode)) != 0) 5575 goto out; 5576 5577 if (!vd->vdev_ops->vdev_op_leaf) { 5578 vdev_free(vd); 5579 error = SET_ERROR(EINVAL); 5580 goto out; 5581 } 5582 5583 vd->vdev_top = vd; 5584 5585 if ((error = vdev_open(vd)) == 0 && 5586 (error = vdev_label_init(vd, crtxg, label)) == 0) { 5587 VERIFY(nvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 5588 vd->vdev_guid) == 0); 5589 } 5590 5591 vdev_free(vd); 5592 5593 if (error && 5594 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 5595 goto out; 5596 else 5597 error = 0; 5598 } 5599 5600 out: 5601 sav->sav_pending = NULL; 5602 sav->sav_npending = 0; 5603 return (error); 5604 } 5605 5606 static int 5607 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 5608 { 5609 int error; 5610 5611 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5612 5613 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5614 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 5615 VDEV_LABEL_SPARE)) != 0) { 5616 return (error); 5617 } 5618 5619 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5620 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 5621 VDEV_LABEL_L2CACHE)); 5622 } 5623 5624 static void 5625 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 5626 const char *config) 5627 { 5628 int i; 5629 5630 if (sav->sav_config != NULL) { 5631 nvlist_t **olddevs; 5632 uint_t oldndevs; 5633 nvlist_t **newdevs; 5634 5635 /* 5636 * Generate new dev list by concatenating with the 5637 * current dev list. 5638 */ 5639 VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, config, 5640 &olddevs, &oldndevs) == 0); 5641 5642 newdevs = kmem_alloc(sizeof (void *) * 5643 (ndevs + oldndevs), KM_SLEEP); 5644 for (i = 0; i < oldndevs; i++) 5645 VERIFY(nvlist_dup(olddevs[i], &newdevs[i], 5646 KM_SLEEP) == 0); 5647 for (i = 0; i < ndevs; i++) 5648 VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs], 5649 KM_SLEEP) == 0); 5650 5651 VERIFY(nvlist_remove(sav->sav_config, config, 5652 DATA_TYPE_NVLIST_ARRAY) == 0); 5653 5654 VERIFY(nvlist_add_nvlist_array(sav->sav_config, 5655 config, newdevs, ndevs + oldndevs) == 0); 5656 for (i = 0; i < oldndevs + ndevs; i++) 5657 nvlist_free(newdevs[i]); 5658 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 5659 } else { 5660 /* 5661 * Generate a new dev list. 5662 */ 5663 VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME, 5664 KM_SLEEP) == 0); 5665 VERIFY(nvlist_add_nvlist_array(sav->sav_config, config, 5666 devs, ndevs) == 0); 5667 } 5668 } 5669 5670 /* 5671 * Stop and drop level 2 ARC devices 5672 */ 5673 void 5674 spa_l2cache_drop(spa_t *spa) 5675 { 5676 vdev_t *vd; 5677 int i; 5678 spa_aux_vdev_t *sav = &spa->spa_l2cache; 5679 5680 for (i = 0; i < sav->sav_count; i++) { 5681 uint64_t pool; 5682 5683 vd = sav->sav_vdevs[i]; 5684 ASSERT(vd != NULL); 5685 5686 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 5687 pool != 0ULL && l2arc_vdev_present(vd)) 5688 l2arc_remove_vdev(vd); 5689 } 5690 } 5691 5692 /* 5693 * Verify encryption parameters for spa creation. If we are encrypting, we must 5694 * have the encryption feature flag enabled. 5695 */ 5696 static int 5697 spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 5698 boolean_t has_encryption) 5699 { 5700 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 5701 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 5702 !has_encryption) 5703 return (SET_ERROR(ENOTSUP)); 5704 5705 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 5706 } 5707 5708 /* 5709 * Pool Creation 5710 */ 5711 int 5712 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 5713 nvlist_t *zplprops, dsl_crypto_params_t *dcp) 5714 { 5715 spa_t *spa; 5716 char *altroot = NULL; 5717 vdev_t *rvd; 5718 dsl_pool_t *dp; 5719 dmu_tx_t *tx; 5720 int error = 0; 5721 uint64_t txg = TXG_INITIAL; 5722 nvlist_t **spares, **l2cache; 5723 uint_t nspares, nl2cache; 5724 uint64_t version, obj, ndraid = 0; 5725 boolean_t has_features; 5726 boolean_t has_encryption; 5727 boolean_t has_allocclass; 5728 spa_feature_t feat; 5729 char *feat_name; 5730 char *poolname; 5731 nvlist_t *nvl; 5732 5733 if (props == NULL || 5734 nvlist_lookup_string(props, "tname", &poolname) != 0) 5735 poolname = (char *)pool; 5736 5737 /* 5738 * If this pool already exists, return failure. 5739 */ 5740 mutex_enter(&spa_namespace_lock); 5741 if (spa_lookup(poolname) != NULL) { 5742 mutex_exit(&spa_namespace_lock); 5743 return (SET_ERROR(EEXIST)); 5744 } 5745 5746 /* 5747 * Allocate a new spa_t structure. 5748 */ 5749 nvl = fnvlist_alloc(); 5750 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 5751 (void) nvlist_lookup_string(props, 5752 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 5753 spa = spa_add(poolname, nvl, altroot); 5754 fnvlist_free(nvl); 5755 spa_activate(spa, spa_mode_global); 5756 5757 if (props && (error = spa_prop_validate(spa, props))) { 5758 spa_deactivate(spa); 5759 spa_remove(spa); 5760 mutex_exit(&spa_namespace_lock); 5761 return (error); 5762 } 5763 5764 /* 5765 * Temporary pool names should never be written to disk. 5766 */ 5767 if (poolname != pool) 5768 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 5769 5770 has_features = B_FALSE; 5771 has_encryption = B_FALSE; 5772 has_allocclass = B_FALSE; 5773 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 5774 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 5775 if (zpool_prop_feature(nvpair_name(elem))) { 5776 has_features = B_TRUE; 5777 5778 feat_name = strchr(nvpair_name(elem), '@') + 1; 5779 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 5780 if (feat == SPA_FEATURE_ENCRYPTION) 5781 has_encryption = B_TRUE; 5782 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 5783 has_allocclass = B_TRUE; 5784 } 5785 } 5786 5787 /* verify encryption params, if they were provided */ 5788 if (dcp != NULL) { 5789 error = spa_create_check_encryption_params(dcp, has_encryption); 5790 if (error != 0) { 5791 spa_deactivate(spa); 5792 spa_remove(spa); 5793 mutex_exit(&spa_namespace_lock); 5794 return (error); 5795 } 5796 } 5797 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 5798 spa_deactivate(spa); 5799 spa_remove(spa); 5800 mutex_exit(&spa_namespace_lock); 5801 return (ENOTSUP); 5802 } 5803 5804 if (has_features || nvlist_lookup_uint64(props, 5805 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 5806 version = SPA_VERSION; 5807 } 5808 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 5809 5810 spa->spa_first_txg = txg; 5811 spa->spa_uberblock.ub_txg = txg - 1; 5812 spa->spa_uberblock.ub_version = version; 5813 spa->spa_ubsync = spa->spa_uberblock; 5814 spa->spa_load_state = SPA_LOAD_CREATE; 5815 spa->spa_removing_phys.sr_state = DSS_NONE; 5816 spa->spa_removing_phys.sr_removing_vdev = -1; 5817 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 5818 spa->spa_indirect_vdevs_loaded = B_TRUE; 5819 5820 /* 5821 * Create "The Godfather" zio to hold all async IOs 5822 */ 5823 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 5824 KM_SLEEP); 5825 for (int i = 0; i < max_ncpus; i++) { 5826 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 5827 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 5828 ZIO_FLAG_GODFATHER); 5829 } 5830 5831 /* 5832 * Create the root vdev. 5833 */ 5834 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5835 5836 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 5837 5838 ASSERT(error != 0 || rvd != NULL); 5839 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 5840 5841 if (error == 0 && !zfs_allocatable_devs(nvroot)) 5842 error = SET_ERROR(EINVAL); 5843 5844 if (error == 0 && 5845 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 5846 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 && 5847 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 5848 /* 5849 * instantiate the metaslab groups (this will dirty the vdevs) 5850 * we can no longer error exit past this point 5851 */ 5852 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 5853 vdev_t *vd = rvd->vdev_child[c]; 5854 5855 vdev_metaslab_set_size(vd); 5856 vdev_expand(vd, txg); 5857 } 5858 } 5859 5860 spa_config_exit(spa, SCL_ALL, FTAG); 5861 5862 if (error != 0) { 5863 spa_unload(spa); 5864 spa_deactivate(spa); 5865 spa_remove(spa); 5866 mutex_exit(&spa_namespace_lock); 5867 return (error); 5868 } 5869 5870 /* 5871 * Get the list of spares, if specified. 5872 */ 5873 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5874 &spares, &nspares) == 0) { 5875 VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME, 5876 KM_SLEEP) == 0); 5877 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 5878 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 5879 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5880 spa_load_spares(spa); 5881 spa_config_exit(spa, SCL_ALL, FTAG); 5882 spa->spa_spares.sav_sync = B_TRUE; 5883 } 5884 5885 /* 5886 * Get the list of level 2 cache devices, if specified. 5887 */ 5888 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5889 &l2cache, &nl2cache) == 0) { 5890 VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, 5891 NV_UNIQUE_NAME, KM_SLEEP) == 0); 5892 VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 5893 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 5894 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5895 spa_load_l2cache(spa); 5896 spa_config_exit(spa, SCL_ALL, FTAG); 5897 spa->spa_l2cache.sav_sync = B_TRUE; 5898 } 5899 5900 spa->spa_is_initializing = B_TRUE; 5901 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 5902 spa->spa_is_initializing = B_FALSE; 5903 5904 /* 5905 * Create DDTs (dedup tables). 5906 */ 5907 ddt_create(spa); 5908 5909 spa_update_dspace(spa); 5910 5911 tx = dmu_tx_create_assigned(dp, txg); 5912 5913 /* 5914 * Create the pool's history object. 5915 */ 5916 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 5917 spa_history_create_obj(spa, tx); 5918 5919 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 5920 spa_history_log_version(spa, "create", tx); 5921 5922 /* 5923 * Create the pool config object. 5924 */ 5925 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 5926 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 5927 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 5928 5929 if (zap_add(spa->spa_meta_objset, 5930 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 5931 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 5932 cmn_err(CE_PANIC, "failed to add pool config"); 5933 } 5934 5935 if (zap_add(spa->spa_meta_objset, 5936 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 5937 sizeof (uint64_t), 1, &version, tx) != 0) { 5938 cmn_err(CE_PANIC, "failed to add pool version"); 5939 } 5940 5941 /* Newly created pools with the right version are always deflated. */ 5942 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 5943 spa->spa_deflate = TRUE; 5944 if (zap_add(spa->spa_meta_objset, 5945 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 5946 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 5947 cmn_err(CE_PANIC, "failed to add deflate"); 5948 } 5949 } 5950 5951 /* 5952 * Create the deferred-free bpobj. Turn off compression 5953 * because sync-to-convergence takes longer if the blocksize 5954 * keeps changing. 5955 */ 5956 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 5957 dmu_object_set_compress(spa->spa_meta_objset, obj, 5958 ZIO_COMPRESS_OFF, tx); 5959 if (zap_add(spa->spa_meta_objset, 5960 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 5961 sizeof (uint64_t), 1, &obj, tx) != 0) { 5962 cmn_err(CE_PANIC, "failed to add bpobj"); 5963 } 5964 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 5965 spa->spa_meta_objset, obj)); 5966 5967 /* 5968 * Generate some random noise for salted checksums to operate on. 5969 */ 5970 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 5971 sizeof (spa->spa_cksum_salt.zcs_bytes)); 5972 5973 /* 5974 * Set pool properties. 5975 */ 5976 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 5977 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 5978 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 5979 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 5980 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 5981 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 5982 5983 if (props != NULL) { 5984 spa_configfile_set(spa, props, B_FALSE); 5985 spa_sync_props(props, tx); 5986 } 5987 5988 for (int i = 0; i < ndraid; i++) 5989 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 5990 5991 dmu_tx_commit(tx); 5992 5993 spa->spa_sync_on = B_TRUE; 5994 txg_sync_start(dp); 5995 mmp_thread_start(spa); 5996 txg_wait_synced(dp, txg); 5997 5998 spa_spawn_aux_threads(spa); 5999 6000 spa_write_cachefile(spa, B_FALSE, B_TRUE); 6001 6002 /* 6003 * Don't count references from objsets that are already closed 6004 * and are making their way through the eviction process. 6005 */ 6006 spa_evicting_os_wait(spa); 6007 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 6008 spa->spa_load_state = SPA_LOAD_NONE; 6009 6010 mutex_exit(&spa_namespace_lock); 6011 6012 return (0); 6013 } 6014 6015 /* 6016 * Import a non-root pool into the system. 6017 */ 6018 int 6019 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 6020 { 6021 spa_t *spa; 6022 char *altroot = NULL; 6023 spa_load_state_t state = SPA_LOAD_IMPORT; 6024 zpool_load_policy_t policy; 6025 spa_mode_t mode = spa_mode_global; 6026 uint64_t readonly = B_FALSE; 6027 int error; 6028 nvlist_t *nvroot; 6029 nvlist_t **spares, **l2cache; 6030 uint_t nspares, nl2cache; 6031 6032 /* 6033 * If a pool with this name exists, return failure. 6034 */ 6035 mutex_enter(&spa_namespace_lock); 6036 if (spa_lookup(pool) != NULL) { 6037 mutex_exit(&spa_namespace_lock); 6038 return (SET_ERROR(EEXIST)); 6039 } 6040 6041 /* 6042 * Create and initialize the spa structure. 6043 */ 6044 (void) nvlist_lookup_string(props, 6045 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6046 (void) nvlist_lookup_uint64(props, 6047 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 6048 if (readonly) 6049 mode = SPA_MODE_READ; 6050 spa = spa_add(pool, config, altroot); 6051 spa->spa_import_flags = flags; 6052 6053 /* 6054 * Verbatim import - Take a pool and insert it into the namespace 6055 * as if it had been loaded at boot. 6056 */ 6057 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 6058 if (props != NULL) 6059 spa_configfile_set(spa, props, B_FALSE); 6060 6061 spa_write_cachefile(spa, B_FALSE, B_TRUE); 6062 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6063 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 6064 mutex_exit(&spa_namespace_lock); 6065 return (0); 6066 } 6067 6068 spa_activate(spa, mode); 6069 6070 /* 6071 * Don't start async tasks until we know everything is healthy. 6072 */ 6073 spa_async_suspend(spa); 6074 6075 zpool_get_load_policy(config, &policy); 6076 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6077 state = SPA_LOAD_RECOVER; 6078 6079 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 6080 6081 if (state != SPA_LOAD_RECOVER) { 6082 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6083 zfs_dbgmsg("spa_import: importing %s", pool); 6084 } else { 6085 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 6086 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 6087 } 6088 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 6089 6090 /* 6091 * Propagate anything learned while loading the pool and pass it 6092 * back to caller (i.e. rewind info, missing devices, etc). 6093 */ 6094 VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6095 spa->spa_load_info) == 0); 6096 6097 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6098 /* 6099 * Toss any existing sparelist, as it doesn't have any validity 6100 * anymore, and conflicts with spa_has_spare(). 6101 */ 6102 if (spa->spa_spares.sav_config) { 6103 nvlist_free(spa->spa_spares.sav_config); 6104 spa->spa_spares.sav_config = NULL; 6105 spa_load_spares(spa); 6106 } 6107 if (spa->spa_l2cache.sav_config) { 6108 nvlist_free(spa->spa_l2cache.sav_config); 6109 spa->spa_l2cache.sav_config = NULL; 6110 spa_load_l2cache(spa); 6111 } 6112 6113 VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, 6114 &nvroot) == 0); 6115 spa_config_exit(spa, SCL_ALL, FTAG); 6116 6117 if (props != NULL) 6118 spa_configfile_set(spa, props, B_FALSE); 6119 6120 if (error != 0 || (props && spa_writeable(spa) && 6121 (error = spa_prop_set(spa, props)))) { 6122 spa_unload(spa); 6123 spa_deactivate(spa); 6124 spa_remove(spa); 6125 mutex_exit(&spa_namespace_lock); 6126 return (error); 6127 } 6128 6129 spa_async_resume(spa); 6130 6131 /* 6132 * Override any spares and level 2 cache devices as specified by 6133 * the user, as these may have correct device names/devids, etc. 6134 */ 6135 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6136 &spares, &nspares) == 0) { 6137 if (spa->spa_spares.sav_config) 6138 VERIFY(nvlist_remove(spa->spa_spares.sav_config, 6139 ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0); 6140 else 6141 VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, 6142 NV_UNIQUE_NAME, KM_SLEEP) == 0); 6143 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 6144 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 6145 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6146 spa_load_spares(spa); 6147 spa_config_exit(spa, SCL_ALL, FTAG); 6148 spa->spa_spares.sav_sync = B_TRUE; 6149 } 6150 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6151 &l2cache, &nl2cache) == 0) { 6152 if (spa->spa_l2cache.sav_config) 6153 VERIFY(nvlist_remove(spa->spa_l2cache.sav_config, 6154 ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); 6155 else 6156 VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, 6157 NV_UNIQUE_NAME, KM_SLEEP) == 0); 6158 VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6159 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 6160 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6161 spa_load_l2cache(spa); 6162 spa_config_exit(spa, SCL_ALL, FTAG); 6163 spa->spa_l2cache.sav_sync = B_TRUE; 6164 } 6165 6166 /* 6167 * Check for any removed devices. 6168 */ 6169 if (spa->spa_autoreplace) { 6170 spa_aux_check_removed(&spa->spa_spares); 6171 spa_aux_check_removed(&spa->spa_l2cache); 6172 } 6173 6174 if (spa_writeable(spa)) { 6175 /* 6176 * Update the config cache to include the newly-imported pool. 6177 */ 6178 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6179 } 6180 6181 /* 6182 * It's possible that the pool was expanded while it was exported. 6183 * We kick off an async task to handle this for us. 6184 */ 6185 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 6186 6187 spa_history_log_version(spa, "import", NULL); 6188 6189 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6190 6191 mutex_exit(&spa_namespace_lock); 6192 6193 zvol_create_minors_recursive(pool); 6194 6195 return (0); 6196 } 6197 6198 nvlist_t * 6199 spa_tryimport(nvlist_t *tryconfig) 6200 { 6201 nvlist_t *config = NULL; 6202 char *poolname, *cachefile; 6203 spa_t *spa; 6204 uint64_t state; 6205 int error; 6206 zpool_load_policy_t policy; 6207 6208 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 6209 return (NULL); 6210 6211 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 6212 return (NULL); 6213 6214 /* 6215 * Create and initialize the spa structure. 6216 */ 6217 mutex_enter(&spa_namespace_lock); 6218 spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL); 6219 spa_activate(spa, SPA_MODE_READ); 6220 6221 /* 6222 * Rewind pool if a max txg was provided. 6223 */ 6224 zpool_get_load_policy(spa->spa_config, &policy); 6225 if (policy.zlp_txg != UINT64_MAX) { 6226 spa->spa_load_max_txg = policy.zlp_txg; 6227 spa->spa_extreme_rewind = B_TRUE; 6228 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 6229 poolname, (longlong_t)policy.zlp_txg); 6230 } else { 6231 zfs_dbgmsg("spa_tryimport: importing %s", poolname); 6232 } 6233 6234 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 6235 == 0) { 6236 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 6237 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6238 } else { 6239 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 6240 } 6241 6242 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 6243 6244 /* 6245 * If 'tryconfig' was at least parsable, return the current config. 6246 */ 6247 if (spa->spa_root_vdev != NULL) { 6248 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6249 VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, 6250 poolname) == 0); 6251 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 6252 state) == 0); 6253 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 6254 spa->spa_uberblock.ub_timestamp) == 0); 6255 VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6256 spa->spa_load_info) == 0); 6257 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 6258 spa->spa_errata) == 0); 6259 6260 /* 6261 * If the bootfs property exists on this pool then we 6262 * copy it out so that external consumers can tell which 6263 * pools are bootable. 6264 */ 6265 if ((!error || error == EEXIST) && spa->spa_bootfs) { 6266 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6267 6268 /* 6269 * We have to play games with the name since the 6270 * pool was opened as TRYIMPORT_NAME. 6271 */ 6272 if (dsl_dsobj_to_dsname(spa_name(spa), 6273 spa->spa_bootfs, tmpname) == 0) { 6274 char *cp; 6275 char *dsname; 6276 6277 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6278 6279 cp = strchr(tmpname, '/'); 6280 if (cp == NULL) { 6281 (void) strlcpy(dsname, tmpname, 6282 MAXPATHLEN); 6283 } else { 6284 (void) snprintf(dsname, MAXPATHLEN, 6285 "%s/%s", poolname, ++cp); 6286 } 6287 VERIFY(nvlist_add_string(config, 6288 ZPOOL_CONFIG_BOOTFS, dsname) == 0); 6289 kmem_free(dsname, MAXPATHLEN); 6290 } 6291 kmem_free(tmpname, MAXPATHLEN); 6292 } 6293 6294 /* 6295 * Add the list of hot spares and level 2 cache devices. 6296 */ 6297 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6298 spa_add_spares(spa, config); 6299 spa_add_l2cache(spa, config); 6300 spa_config_exit(spa, SCL_CONFIG, FTAG); 6301 } 6302 6303 spa_unload(spa); 6304 spa_deactivate(spa); 6305 spa_remove(spa); 6306 mutex_exit(&spa_namespace_lock); 6307 6308 return (config); 6309 } 6310 6311 /* 6312 * Pool export/destroy 6313 * 6314 * The act of destroying or exporting a pool is very simple. We make sure there 6315 * is no more pending I/O and any references to the pool are gone. Then, we 6316 * update the pool state and sync all the labels to disk, removing the 6317 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 6318 * we don't sync the labels or remove the configuration cache. 6319 */ 6320 static int 6321 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 6322 boolean_t force, boolean_t hardforce) 6323 { 6324 int error; 6325 spa_t *spa; 6326 6327 if (oldconfig) 6328 *oldconfig = NULL; 6329 6330 if (!(spa_mode_global & SPA_MODE_WRITE)) 6331 return (SET_ERROR(EROFS)); 6332 6333 mutex_enter(&spa_namespace_lock); 6334 if ((spa = spa_lookup(pool)) == NULL) { 6335 mutex_exit(&spa_namespace_lock); 6336 return (SET_ERROR(ENOENT)); 6337 } 6338 6339 if (spa->spa_is_exporting) { 6340 /* the pool is being exported by another thread */ 6341 mutex_exit(&spa_namespace_lock); 6342 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 6343 } 6344 spa->spa_is_exporting = B_TRUE; 6345 6346 /* 6347 * Put a hold on the pool, drop the namespace lock, stop async tasks, 6348 * reacquire the namespace lock, and see if we can export. 6349 */ 6350 spa_open_ref(spa, FTAG); 6351 mutex_exit(&spa_namespace_lock); 6352 spa_async_suspend(spa); 6353 if (spa->spa_zvol_taskq) { 6354 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 6355 taskq_wait(spa->spa_zvol_taskq); 6356 } 6357 mutex_enter(&spa_namespace_lock); 6358 spa_close(spa, FTAG); 6359 6360 if (spa->spa_state == POOL_STATE_UNINITIALIZED) 6361 goto export_spa; 6362 /* 6363 * The pool will be in core if it's openable, in which case we can 6364 * modify its state. Objsets may be open only because they're dirty, 6365 * so we have to force it to sync before checking spa_refcnt. 6366 */ 6367 if (spa->spa_sync_on) { 6368 txg_wait_synced(spa->spa_dsl_pool, 0); 6369 spa_evicting_os_wait(spa); 6370 } 6371 6372 /* 6373 * A pool cannot be exported or destroyed if there are active 6374 * references. If we are resetting a pool, allow references by 6375 * fault injection handlers. 6376 */ 6377 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 6378 error = SET_ERROR(EBUSY); 6379 goto fail; 6380 } 6381 6382 if (spa->spa_sync_on) { 6383 /* 6384 * A pool cannot be exported if it has an active shared spare. 6385 * This is to prevent other pools stealing the active spare 6386 * from an exported pool. At user's own will, such pool can 6387 * be forcedly exported. 6388 */ 6389 if (!force && new_state == POOL_STATE_EXPORTED && 6390 spa_has_active_shared_spare(spa)) { 6391 error = SET_ERROR(EXDEV); 6392 goto fail; 6393 } 6394 6395 /* 6396 * We're about to export or destroy this pool. Make sure 6397 * we stop all initialization and trim activity here before 6398 * we set the spa_final_txg. This will ensure that all 6399 * dirty data resulting from the initialization is 6400 * committed to disk before we unload the pool. 6401 */ 6402 if (spa->spa_root_vdev != NULL) { 6403 vdev_t *rvd = spa->spa_root_vdev; 6404 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 6405 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 6406 vdev_autotrim_stop_all(spa); 6407 vdev_rebuild_stop_all(spa); 6408 } 6409 6410 /* 6411 * We want this to be reflected on every label, 6412 * so mark them all dirty. spa_unload() will do the 6413 * final sync that pushes these changes out. 6414 */ 6415 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6416 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6417 spa->spa_state = new_state; 6418 spa->spa_final_txg = spa_last_synced_txg(spa) + 6419 TXG_DEFER_SIZE + 1; 6420 vdev_config_dirty(spa->spa_root_vdev); 6421 spa_config_exit(spa, SCL_ALL, FTAG); 6422 } 6423 } 6424 6425 export_spa: 6426 if (new_state == POOL_STATE_DESTROYED) 6427 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 6428 else if (new_state == POOL_STATE_EXPORTED) 6429 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 6430 6431 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 6432 spa_unload(spa); 6433 spa_deactivate(spa); 6434 } 6435 6436 if (oldconfig && spa->spa_config) 6437 VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0); 6438 6439 if (new_state != POOL_STATE_UNINITIALIZED) { 6440 if (!hardforce) 6441 spa_write_cachefile(spa, B_TRUE, B_TRUE); 6442 spa_remove(spa); 6443 } else { 6444 /* 6445 * If spa_remove() is not called for this spa_t and 6446 * there is any possibility that it can be reused, 6447 * we make sure to reset the exporting flag. 6448 */ 6449 spa->spa_is_exporting = B_FALSE; 6450 } 6451 6452 mutex_exit(&spa_namespace_lock); 6453 return (0); 6454 6455 fail: 6456 spa->spa_is_exporting = B_FALSE; 6457 spa_async_resume(spa); 6458 mutex_exit(&spa_namespace_lock); 6459 return (error); 6460 } 6461 6462 /* 6463 * Destroy a storage pool. 6464 */ 6465 int 6466 spa_destroy(const char *pool) 6467 { 6468 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 6469 B_FALSE, B_FALSE)); 6470 } 6471 6472 /* 6473 * Export a storage pool. 6474 */ 6475 int 6476 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 6477 boolean_t hardforce) 6478 { 6479 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 6480 force, hardforce)); 6481 } 6482 6483 /* 6484 * Similar to spa_export(), this unloads the spa_t without actually removing it 6485 * from the namespace in any way. 6486 */ 6487 int 6488 spa_reset(const char *pool) 6489 { 6490 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 6491 B_FALSE, B_FALSE)); 6492 } 6493 6494 /* 6495 * ========================================================================== 6496 * Device manipulation 6497 * ========================================================================== 6498 */ 6499 6500 /* 6501 * This is called as a synctask to increment the draid feature flag 6502 */ 6503 static void 6504 spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 6505 { 6506 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 6507 int draid = (int)(uintptr_t)arg; 6508 6509 for (int c = 0; c < draid; c++) 6510 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6511 } 6512 6513 /* 6514 * Add a device to a storage pool. 6515 */ 6516 int 6517 spa_vdev_add(spa_t *spa, nvlist_t *nvroot) 6518 { 6519 uint64_t txg, ndraid = 0; 6520 int error; 6521 vdev_t *rvd = spa->spa_root_vdev; 6522 vdev_t *vd, *tvd; 6523 nvlist_t **spares, **l2cache; 6524 uint_t nspares, nl2cache; 6525 6526 ASSERT(spa_writeable(spa)); 6527 6528 txg = spa_vdev_enter(spa); 6529 6530 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 6531 VDEV_ALLOC_ADD)) != 0) 6532 return (spa_vdev_exit(spa, NULL, txg, error)); 6533 6534 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 6535 6536 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 6537 &nspares) != 0) 6538 nspares = 0; 6539 6540 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 6541 &nl2cache) != 0) 6542 nl2cache = 0; 6543 6544 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 6545 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6546 6547 if (vd->vdev_children != 0 && 6548 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 6549 return (spa_vdev_exit(spa, vd, txg, error)); 6550 } 6551 6552 /* 6553 * The virtual dRAID spares must be added after vdev tree is created 6554 * and the vdev guids are generated. The guid of their associated 6555 * dRAID is stored in the config and used when opening the spare. 6556 */ 6557 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 6558 rvd->vdev_children)) == 0) { 6559 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 6560 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 6561 nspares = 0; 6562 } else { 6563 return (spa_vdev_exit(spa, vd, txg, error)); 6564 } 6565 6566 /* 6567 * We must validate the spares and l2cache devices after checking the 6568 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 6569 */ 6570 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 6571 return (spa_vdev_exit(spa, vd, txg, error)); 6572 6573 /* 6574 * If we are in the middle of a device removal, we can only add 6575 * devices which match the existing devices in the pool. 6576 * If we are in the middle of a removal, or have some indirect 6577 * vdevs, we can not add raidz or dRAID top levels. 6578 */ 6579 if (spa->spa_vdev_removal != NULL || 6580 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 6581 for (int c = 0; c < vd->vdev_children; c++) { 6582 tvd = vd->vdev_child[c]; 6583 if (spa->spa_vdev_removal != NULL && 6584 tvd->vdev_ashift != spa->spa_max_ashift) { 6585 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6586 } 6587 /* Fail if top level vdev is raidz or a dRAID */ 6588 if (vdev_get_nparity(tvd) != 0) 6589 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6590 6591 /* 6592 * Need the top level mirror to be 6593 * a mirror of leaf vdevs only 6594 */ 6595 if (tvd->vdev_ops == &vdev_mirror_ops) { 6596 for (uint64_t cid = 0; 6597 cid < tvd->vdev_children; cid++) { 6598 vdev_t *cvd = tvd->vdev_child[cid]; 6599 if (!cvd->vdev_ops->vdev_op_leaf) { 6600 return (spa_vdev_exit(spa, vd, 6601 txg, EINVAL)); 6602 } 6603 } 6604 } 6605 } 6606 } 6607 6608 for (int c = 0; c < vd->vdev_children; c++) { 6609 tvd = vd->vdev_child[c]; 6610 vdev_remove_child(vd, tvd); 6611 tvd->vdev_id = rvd->vdev_children; 6612 vdev_add_child(rvd, tvd); 6613 vdev_config_dirty(tvd); 6614 } 6615 6616 if (nspares != 0) { 6617 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 6618 ZPOOL_CONFIG_SPARES); 6619 spa_load_spares(spa); 6620 spa->spa_spares.sav_sync = B_TRUE; 6621 } 6622 6623 if (nl2cache != 0) { 6624 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 6625 ZPOOL_CONFIG_L2CACHE); 6626 spa_load_l2cache(spa); 6627 spa->spa_l2cache.sav_sync = B_TRUE; 6628 } 6629 6630 /* 6631 * We can't increment a feature while holding spa_vdev so we 6632 * have to do it in a synctask. 6633 */ 6634 if (ndraid != 0) { 6635 dmu_tx_t *tx; 6636 6637 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 6638 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 6639 (void *)(uintptr_t)ndraid, tx); 6640 dmu_tx_commit(tx); 6641 } 6642 6643 /* 6644 * We have to be careful when adding new vdevs to an existing pool. 6645 * If other threads start allocating from these vdevs before we 6646 * sync the config cache, and we lose power, then upon reboot we may 6647 * fail to open the pool because there are DVAs that the config cache 6648 * can't translate. Therefore, we first add the vdevs without 6649 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 6650 * and then let spa_config_update() initialize the new metaslabs. 6651 * 6652 * spa_load() checks for added-but-not-initialized vdevs, so that 6653 * if we lose power at any point in this sequence, the remaining 6654 * steps will be completed the next time we load the pool. 6655 */ 6656 (void) spa_vdev_exit(spa, vd, txg, 0); 6657 6658 mutex_enter(&spa_namespace_lock); 6659 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6660 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 6661 mutex_exit(&spa_namespace_lock); 6662 6663 return (0); 6664 } 6665 6666 /* 6667 * Attach a device to a mirror. The arguments are the path to any device 6668 * in the mirror, and the nvroot for the new device. If the path specifies 6669 * a device that is not mirrored, we automatically insert the mirror vdev. 6670 * 6671 * If 'replacing' is specified, the new device is intended to replace the 6672 * existing device; in this case the two devices are made into their own 6673 * mirror using the 'replacing' vdev, which is functionally identical to 6674 * the mirror vdev (it actually reuses all the same ops) but has a few 6675 * extra rules: you can't attach to it after it's been created, and upon 6676 * completion of resilvering, the first disk (the one being replaced) 6677 * is automatically detached. 6678 * 6679 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 6680 * should be performed instead of traditional healing reconstruction. From 6681 * an administrators perspective these are both resilver operations. 6682 */ 6683 int 6684 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 6685 int rebuild) 6686 { 6687 uint64_t txg, dtl_max_txg; 6688 vdev_t *rvd = spa->spa_root_vdev; 6689 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 6690 vdev_ops_t *pvops; 6691 char *oldvdpath, *newvdpath; 6692 int newvd_isspare; 6693 int error; 6694 6695 ASSERT(spa_writeable(spa)); 6696 6697 txg = spa_vdev_enter(spa); 6698 6699 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 6700 6701 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6702 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6703 error = (spa_has_checkpoint(spa)) ? 6704 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6705 return (spa_vdev_exit(spa, NULL, txg, error)); 6706 } 6707 6708 if (rebuild) { 6709 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 6710 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6711 6712 if (dsl_scan_resilvering(spa_get_dsl(spa))) 6713 return (spa_vdev_exit(spa, NULL, txg, 6714 ZFS_ERR_RESILVER_IN_PROGRESS)); 6715 } else { 6716 if (vdev_rebuild_active(rvd)) 6717 return (spa_vdev_exit(spa, NULL, txg, 6718 ZFS_ERR_REBUILD_IN_PROGRESS)); 6719 } 6720 6721 if (spa->spa_vdev_removal != NULL) 6722 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 6723 6724 if (oldvd == NULL) 6725 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6726 6727 if (!oldvd->vdev_ops->vdev_op_leaf) 6728 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6729 6730 pvd = oldvd->vdev_parent; 6731 6732 if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 6733 VDEV_ALLOC_ATTACH)) != 0) 6734 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 6735 6736 if (newrootvd->vdev_children != 1) 6737 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6738 6739 newvd = newrootvd->vdev_child[0]; 6740 6741 if (!newvd->vdev_ops->vdev_op_leaf) 6742 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6743 6744 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 6745 return (spa_vdev_exit(spa, newrootvd, txg, error)); 6746 6747 /* 6748 * Spares can't replace logs 6749 */ 6750 if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare) 6751 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6752 6753 /* 6754 * A dRAID spare can only replace a child of its parent dRAID vdev. 6755 */ 6756 if (newvd->vdev_ops == &vdev_draid_spare_ops && 6757 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 6758 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6759 } 6760 6761 if (rebuild) { 6762 /* 6763 * For rebuilds, the top vdev must support reconstruction 6764 * using only space maps. This means the only allowable 6765 * vdevs types are the root vdev, a mirror, or dRAID. 6766 */ 6767 tvd = pvd; 6768 if (pvd->vdev_top != NULL) 6769 tvd = pvd->vdev_top; 6770 6771 if (tvd->vdev_ops != &vdev_mirror_ops && 6772 tvd->vdev_ops != &vdev_root_ops && 6773 tvd->vdev_ops != &vdev_draid_ops) { 6774 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6775 } 6776 } 6777 6778 if (!replacing) { 6779 /* 6780 * For attach, the only allowable parent is a mirror or the root 6781 * vdev. 6782 */ 6783 if (pvd->vdev_ops != &vdev_mirror_ops && 6784 pvd->vdev_ops != &vdev_root_ops) 6785 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6786 6787 pvops = &vdev_mirror_ops; 6788 } else { 6789 /* 6790 * Active hot spares can only be replaced by inactive hot 6791 * spares. 6792 */ 6793 if (pvd->vdev_ops == &vdev_spare_ops && 6794 oldvd->vdev_isspare && 6795 !spa_has_spare(spa, newvd->vdev_guid)) 6796 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6797 6798 /* 6799 * If the source is a hot spare, and the parent isn't already a 6800 * spare, then we want to create a new hot spare. Otherwise, we 6801 * want to create a replacing vdev. The user is not allowed to 6802 * attach to a spared vdev child unless the 'isspare' state is 6803 * the same (spare replaces spare, non-spare replaces 6804 * non-spare). 6805 */ 6806 if (pvd->vdev_ops == &vdev_replacing_ops && 6807 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 6808 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6809 } else if (pvd->vdev_ops == &vdev_spare_ops && 6810 newvd->vdev_isspare != oldvd->vdev_isspare) { 6811 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6812 } 6813 6814 if (newvd->vdev_isspare) 6815 pvops = &vdev_spare_ops; 6816 else 6817 pvops = &vdev_replacing_ops; 6818 } 6819 6820 /* 6821 * Make sure the new device is big enough. 6822 */ 6823 if (newvd->vdev_asize < vdev_get_min_asize(oldvd)) 6824 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 6825 6826 /* 6827 * The new device cannot have a higher alignment requirement 6828 * than the top-level vdev. 6829 */ 6830 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) 6831 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6832 6833 /* 6834 * If this is an in-place replacement, update oldvd's path and devid 6835 * to make it distinguishable from newvd, and unopenable from now on. 6836 */ 6837 if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { 6838 spa_strfree(oldvd->vdev_path); 6839 oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, 6840 KM_SLEEP); 6841 (void) snprintf(oldvd->vdev_path, strlen(newvd->vdev_path) + 5, 6842 "%s/%s", newvd->vdev_path, "old"); 6843 if (oldvd->vdev_devid != NULL) { 6844 spa_strfree(oldvd->vdev_devid); 6845 oldvd->vdev_devid = NULL; 6846 } 6847 } 6848 6849 /* 6850 * If the parent is not a mirror, or if we're replacing, insert the new 6851 * mirror/replacing/spare vdev above oldvd. 6852 */ 6853 if (pvd->vdev_ops != pvops) 6854 pvd = vdev_add_parent(oldvd, pvops); 6855 6856 ASSERT(pvd->vdev_top->vdev_parent == rvd); 6857 ASSERT(pvd->vdev_ops == pvops); 6858 ASSERT(oldvd->vdev_parent == pvd); 6859 6860 /* 6861 * Extract the new device from its root and add it to pvd. 6862 */ 6863 vdev_remove_child(newrootvd, newvd); 6864 newvd->vdev_id = pvd->vdev_children; 6865 newvd->vdev_crtxg = oldvd->vdev_crtxg; 6866 vdev_add_child(pvd, newvd); 6867 6868 /* 6869 * Reevaluate the parent vdev state. 6870 */ 6871 vdev_propagate_state(pvd); 6872 6873 tvd = newvd->vdev_top; 6874 ASSERT(pvd->vdev_top == tvd); 6875 ASSERT(tvd->vdev_parent == rvd); 6876 6877 vdev_config_dirty(tvd); 6878 6879 /* 6880 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 6881 * for any dmu_sync-ed blocks. It will propagate upward when 6882 * spa_vdev_exit() calls vdev_dtl_reassess(). 6883 */ 6884 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 6885 6886 vdev_dtl_dirty(newvd, DTL_MISSING, 6887 TXG_INITIAL, dtl_max_txg - TXG_INITIAL); 6888 6889 if (newvd->vdev_isspare) { 6890 spa_spare_activate(newvd); 6891 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 6892 } 6893 6894 oldvdpath = spa_strdup(oldvd->vdev_path); 6895 newvdpath = spa_strdup(newvd->vdev_path); 6896 newvd_isspare = newvd->vdev_isspare; 6897 6898 /* 6899 * Mark newvd's DTL dirty in this txg. 6900 */ 6901 vdev_dirty(tvd, VDD_DTL, newvd, txg); 6902 6903 /* 6904 * Schedule the resilver or rebuild to restart in the future. We do 6905 * this to ensure that dmu_sync-ed blocks have been stitched into the 6906 * respective datasets. 6907 */ 6908 if (rebuild) { 6909 newvd->vdev_rebuild_txg = txg; 6910 6911 vdev_rebuild(tvd); 6912 } else { 6913 newvd->vdev_resilver_txg = txg; 6914 6915 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 6916 spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) { 6917 vdev_defer_resilver(newvd); 6918 } else { 6919 dsl_scan_restart_resilver(spa->spa_dsl_pool, 6920 dtl_max_txg); 6921 } 6922 } 6923 6924 if (spa->spa_bootfs) 6925 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 6926 6927 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 6928 6929 /* 6930 * Commit the config 6931 */ 6932 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 6933 6934 spa_history_log_internal(spa, "vdev attach", NULL, 6935 "%s vdev=%s %s vdev=%s", 6936 replacing && newvd_isspare ? "spare in" : 6937 replacing ? "replace" : "attach", newvdpath, 6938 replacing ? "for" : "to", oldvdpath); 6939 6940 spa_strfree(oldvdpath); 6941 spa_strfree(newvdpath); 6942 6943 return (0); 6944 } 6945 6946 /* 6947 * Detach a device from a mirror or replacing vdev. 6948 * 6949 * If 'replace_done' is specified, only detach if the parent 6950 * is a replacing vdev. 6951 */ 6952 int 6953 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 6954 { 6955 uint64_t txg; 6956 int error; 6957 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 6958 vdev_t *vd, *pvd, *cvd, *tvd; 6959 boolean_t unspare = B_FALSE; 6960 uint64_t unspare_guid = 0; 6961 char *vdpath; 6962 6963 ASSERT(spa_writeable(spa)); 6964 6965 txg = spa_vdev_detach_enter(spa, guid); 6966 6967 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 6968 6969 /* 6970 * Besides being called directly from the userland through the 6971 * ioctl interface, spa_vdev_detach() can be potentially called 6972 * at the end of spa_vdev_resilver_done(). 6973 * 6974 * In the regular case, when we have a checkpoint this shouldn't 6975 * happen as we never empty the DTLs of a vdev during the scrub 6976 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 6977 * should never get here when we have a checkpoint. 6978 * 6979 * That said, even in a case when we checkpoint the pool exactly 6980 * as spa_vdev_resilver_done() calls this function everything 6981 * should be fine as the resilver will return right away. 6982 */ 6983 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6984 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6985 error = (spa_has_checkpoint(spa)) ? 6986 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6987 return (spa_vdev_exit(spa, NULL, txg, error)); 6988 } 6989 6990 if (vd == NULL) 6991 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6992 6993 if (!vd->vdev_ops->vdev_op_leaf) 6994 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6995 6996 pvd = vd->vdev_parent; 6997 6998 /* 6999 * If the parent/child relationship is not as expected, don't do it. 7000 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 7001 * vdev that's replacing B with C. The user's intent in replacing 7002 * is to go from M(A,B) to M(A,C). If the user decides to cancel 7003 * the replace by detaching C, the expected behavior is to end up 7004 * M(A,B). But suppose that right after deciding to detach C, 7005 * the replacement of B completes. We would have M(A,C), and then 7006 * ask to detach C, which would leave us with just A -- not what 7007 * the user wanted. To prevent this, we make sure that the 7008 * parent/child relationship hasn't changed -- in this example, 7009 * that C's parent is still the replacing vdev R. 7010 */ 7011 if (pvd->vdev_guid != pguid && pguid != 0) 7012 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7013 7014 /* 7015 * Only 'replacing' or 'spare' vdevs can be replaced. 7016 */ 7017 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 7018 pvd->vdev_ops != &vdev_spare_ops) 7019 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7020 7021 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 7022 spa_version(spa) >= SPA_VERSION_SPARES); 7023 7024 /* 7025 * Only mirror, replacing, and spare vdevs support detach. 7026 */ 7027 if (pvd->vdev_ops != &vdev_replacing_ops && 7028 pvd->vdev_ops != &vdev_mirror_ops && 7029 pvd->vdev_ops != &vdev_spare_ops) 7030 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7031 7032 /* 7033 * If this device has the only valid copy of some data, 7034 * we cannot safely detach it. 7035 */ 7036 if (vdev_dtl_required(vd)) 7037 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7038 7039 ASSERT(pvd->vdev_children >= 2); 7040 7041 /* 7042 * If we are detaching the second disk from a replacing vdev, then 7043 * check to see if we changed the original vdev's path to have "/old" 7044 * at the end in spa_vdev_attach(). If so, undo that change now. 7045 */ 7046 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 7047 vd->vdev_path != NULL) { 7048 size_t len = strlen(vd->vdev_path); 7049 7050 for (int c = 0; c < pvd->vdev_children; c++) { 7051 cvd = pvd->vdev_child[c]; 7052 7053 if (cvd == vd || cvd->vdev_path == NULL) 7054 continue; 7055 7056 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 7057 strcmp(cvd->vdev_path + len, "/old") == 0) { 7058 spa_strfree(cvd->vdev_path); 7059 cvd->vdev_path = spa_strdup(vd->vdev_path); 7060 break; 7061 } 7062 } 7063 } 7064 7065 /* 7066 * If we are detaching the original disk from a normal spare, then it 7067 * implies that the spare should become a real disk, and be removed 7068 * from the active spare list for the pool. dRAID spares on the 7069 * other hand are coupled to the pool and thus should never be removed 7070 * from the spares list. 7071 */ 7072 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 7073 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7074 7075 if (last_cvd->vdev_isspare && 7076 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 7077 unspare = B_TRUE; 7078 } 7079 } 7080 7081 /* 7082 * Erase the disk labels so the disk can be used for other things. 7083 * This must be done after all other error cases are handled, 7084 * but before we disembowel vd (so we can still do I/O to it). 7085 * But if we can't do it, don't treat the error as fatal -- 7086 * it may be that the unwritability of the disk is the reason 7087 * it's being detached! 7088 */ 7089 error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 7090 7091 /* 7092 * Remove vd from its parent and compact the parent's children. 7093 */ 7094 vdev_remove_child(pvd, vd); 7095 vdev_compact_children(pvd); 7096 7097 /* 7098 * Remember one of the remaining children so we can get tvd below. 7099 */ 7100 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7101 7102 /* 7103 * If we need to remove the remaining child from the list of hot spares, 7104 * do it now, marking the vdev as no longer a spare in the process. 7105 * We must do this before vdev_remove_parent(), because that can 7106 * change the GUID if it creates a new toplevel GUID. For a similar 7107 * reason, we must remove the spare now, in the same txg as the detach; 7108 * otherwise someone could attach a new sibling, change the GUID, and 7109 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 7110 */ 7111 if (unspare) { 7112 ASSERT(cvd->vdev_isspare); 7113 spa_spare_remove(cvd); 7114 unspare_guid = cvd->vdev_guid; 7115 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 7116 cvd->vdev_unspare = B_TRUE; 7117 } 7118 7119 /* 7120 * If the parent mirror/replacing vdev only has one child, 7121 * the parent is no longer needed. Remove it from the tree. 7122 */ 7123 if (pvd->vdev_children == 1) { 7124 if (pvd->vdev_ops == &vdev_spare_ops) 7125 cvd->vdev_unspare = B_FALSE; 7126 vdev_remove_parent(cvd); 7127 } 7128 7129 /* 7130 * We don't set tvd until now because the parent we just removed 7131 * may have been the previous top-level vdev. 7132 */ 7133 tvd = cvd->vdev_top; 7134 ASSERT(tvd->vdev_parent == rvd); 7135 7136 /* 7137 * Reevaluate the parent vdev state. 7138 */ 7139 vdev_propagate_state(cvd); 7140 7141 /* 7142 * If the 'autoexpand' property is set on the pool then automatically 7143 * try to expand the size of the pool. For example if the device we 7144 * just detached was smaller than the others, it may be possible to 7145 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 7146 * first so that we can obtain the updated sizes of the leaf vdevs. 7147 */ 7148 if (spa->spa_autoexpand) { 7149 vdev_reopen(tvd); 7150 vdev_expand(tvd, txg); 7151 } 7152 7153 vdev_config_dirty(tvd); 7154 7155 /* 7156 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 7157 * vd->vdev_detached is set and free vd's DTL object in syncing context. 7158 * But first make sure we're not on any *other* txg's DTL list, to 7159 * prevent vd from being accessed after it's freed. 7160 */ 7161 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 7162 for (int t = 0; t < TXG_SIZE; t++) 7163 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 7164 vd->vdev_detached = B_TRUE; 7165 vdev_dirty(tvd, VDD_DTL, vd, txg); 7166 7167 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 7168 spa_notify_waiters(spa); 7169 7170 /* hang on to the spa before we release the lock */ 7171 spa_open_ref(spa, FTAG); 7172 7173 error = spa_vdev_exit(spa, vd, txg, 0); 7174 7175 spa_history_log_internal(spa, "detach", NULL, 7176 "vdev=%s", vdpath); 7177 spa_strfree(vdpath); 7178 7179 /* 7180 * If this was the removal of the original device in a hot spare vdev, 7181 * then we want to go through and remove the device from the hot spare 7182 * list of every other pool. 7183 */ 7184 if (unspare) { 7185 spa_t *altspa = NULL; 7186 7187 mutex_enter(&spa_namespace_lock); 7188 while ((altspa = spa_next(altspa)) != NULL) { 7189 if (altspa->spa_state != POOL_STATE_ACTIVE || 7190 altspa == spa) 7191 continue; 7192 7193 spa_open_ref(altspa, FTAG); 7194 mutex_exit(&spa_namespace_lock); 7195 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 7196 mutex_enter(&spa_namespace_lock); 7197 spa_close(altspa, FTAG); 7198 } 7199 mutex_exit(&spa_namespace_lock); 7200 7201 /* search the rest of the vdevs for spares to remove */ 7202 spa_vdev_resilver_done(spa); 7203 } 7204 7205 /* all done with the spa; OK to release */ 7206 mutex_enter(&spa_namespace_lock); 7207 spa_close(spa, FTAG); 7208 mutex_exit(&spa_namespace_lock); 7209 7210 return (error); 7211 } 7212 7213 static int 7214 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7215 list_t *vd_list) 7216 { 7217 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7218 7219 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7220 7221 /* Look up vdev and ensure it's a leaf. */ 7222 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7223 if (vd == NULL || vd->vdev_detached) { 7224 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7225 return (SET_ERROR(ENODEV)); 7226 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7227 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7228 return (SET_ERROR(EINVAL)); 7229 } else if (!vdev_writeable(vd)) { 7230 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7231 return (SET_ERROR(EROFS)); 7232 } 7233 mutex_enter(&vd->vdev_initialize_lock); 7234 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7235 7236 /* 7237 * When we activate an initialize action we check to see 7238 * if the vdev_initialize_thread is NULL. We do this instead 7239 * of using the vdev_initialize_state since there might be 7240 * a previous initialization process which has completed but 7241 * the thread is not exited. 7242 */ 7243 if (cmd_type == POOL_INITIALIZE_START && 7244 (vd->vdev_initialize_thread != NULL || 7245 vd->vdev_top->vdev_removing)) { 7246 mutex_exit(&vd->vdev_initialize_lock); 7247 return (SET_ERROR(EBUSY)); 7248 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 7249 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 7250 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 7251 mutex_exit(&vd->vdev_initialize_lock); 7252 return (SET_ERROR(ESRCH)); 7253 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 7254 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 7255 mutex_exit(&vd->vdev_initialize_lock); 7256 return (SET_ERROR(ESRCH)); 7257 } 7258 7259 switch (cmd_type) { 7260 case POOL_INITIALIZE_START: 7261 vdev_initialize(vd); 7262 break; 7263 case POOL_INITIALIZE_CANCEL: 7264 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 7265 break; 7266 case POOL_INITIALIZE_SUSPEND: 7267 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 7268 break; 7269 default: 7270 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7271 } 7272 mutex_exit(&vd->vdev_initialize_lock); 7273 7274 return (0); 7275 } 7276 7277 int 7278 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 7279 nvlist_t *vdev_errlist) 7280 { 7281 int total_errors = 0; 7282 list_t vd_list; 7283 7284 list_create(&vd_list, sizeof (vdev_t), 7285 offsetof(vdev_t, vdev_initialize_node)); 7286 7287 /* 7288 * We hold the namespace lock through the whole function 7289 * to prevent any changes to the pool while we're starting or 7290 * stopping initialization. The config and state locks are held so that 7291 * we can properly assess the vdev state before we commit to 7292 * the initializing operation. 7293 */ 7294 mutex_enter(&spa_namespace_lock); 7295 7296 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7297 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7298 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7299 7300 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 7301 &vd_list); 7302 if (error != 0) { 7303 char guid_as_str[MAXNAMELEN]; 7304 7305 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7306 "%llu", (unsigned long long)vdev_guid); 7307 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7308 total_errors++; 7309 } 7310 } 7311 7312 /* Wait for all initialize threads to stop. */ 7313 vdev_initialize_stop_wait(spa, &vd_list); 7314 7315 /* Sync out the initializing state */ 7316 txg_wait_synced(spa->spa_dsl_pool, 0); 7317 mutex_exit(&spa_namespace_lock); 7318 7319 list_destroy(&vd_list); 7320 7321 return (total_errors); 7322 } 7323 7324 static int 7325 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7326 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 7327 { 7328 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7329 7330 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7331 7332 /* Look up vdev and ensure it's a leaf. */ 7333 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7334 if (vd == NULL || vd->vdev_detached) { 7335 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7336 return (SET_ERROR(ENODEV)); 7337 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7338 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7339 return (SET_ERROR(EINVAL)); 7340 } else if (!vdev_writeable(vd)) { 7341 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7342 return (SET_ERROR(EROFS)); 7343 } else if (!vd->vdev_has_trim) { 7344 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7345 return (SET_ERROR(EOPNOTSUPP)); 7346 } else if (secure && !vd->vdev_has_securetrim) { 7347 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7348 return (SET_ERROR(EOPNOTSUPP)); 7349 } 7350 mutex_enter(&vd->vdev_trim_lock); 7351 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7352 7353 /* 7354 * When we activate a TRIM action we check to see if the 7355 * vdev_trim_thread is NULL. We do this instead of using the 7356 * vdev_trim_state since there might be a previous TRIM process 7357 * which has completed but the thread is not exited. 7358 */ 7359 if (cmd_type == POOL_TRIM_START && 7360 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing)) { 7361 mutex_exit(&vd->vdev_trim_lock); 7362 return (SET_ERROR(EBUSY)); 7363 } else if (cmd_type == POOL_TRIM_CANCEL && 7364 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 7365 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 7366 mutex_exit(&vd->vdev_trim_lock); 7367 return (SET_ERROR(ESRCH)); 7368 } else if (cmd_type == POOL_TRIM_SUSPEND && 7369 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 7370 mutex_exit(&vd->vdev_trim_lock); 7371 return (SET_ERROR(ESRCH)); 7372 } 7373 7374 switch (cmd_type) { 7375 case POOL_TRIM_START: 7376 vdev_trim(vd, rate, partial, secure); 7377 break; 7378 case POOL_TRIM_CANCEL: 7379 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 7380 break; 7381 case POOL_TRIM_SUSPEND: 7382 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 7383 break; 7384 default: 7385 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7386 } 7387 mutex_exit(&vd->vdev_trim_lock); 7388 7389 return (0); 7390 } 7391 7392 /* 7393 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 7394 * TRIM threads for each child vdev. These threads pass over all of the free 7395 * space in the vdev's metaslabs and issues TRIM commands for that space. 7396 */ 7397 int 7398 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 7399 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 7400 { 7401 int total_errors = 0; 7402 list_t vd_list; 7403 7404 list_create(&vd_list, sizeof (vdev_t), 7405 offsetof(vdev_t, vdev_trim_node)); 7406 7407 /* 7408 * We hold the namespace lock through the whole function 7409 * to prevent any changes to the pool while we're starting or 7410 * stopping TRIM. The config and state locks are held so that 7411 * we can properly assess the vdev state before we commit to 7412 * the TRIM operation. 7413 */ 7414 mutex_enter(&spa_namespace_lock); 7415 7416 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7417 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7418 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7419 7420 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 7421 rate, partial, secure, &vd_list); 7422 if (error != 0) { 7423 char guid_as_str[MAXNAMELEN]; 7424 7425 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7426 "%llu", (unsigned long long)vdev_guid); 7427 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7428 total_errors++; 7429 } 7430 } 7431 7432 /* Wait for all TRIM threads to stop. */ 7433 vdev_trim_stop_wait(spa, &vd_list); 7434 7435 /* Sync out the TRIM state */ 7436 txg_wait_synced(spa->spa_dsl_pool, 0); 7437 mutex_exit(&spa_namespace_lock); 7438 7439 list_destroy(&vd_list); 7440 7441 return (total_errors); 7442 } 7443 7444 /* 7445 * Split a set of devices from their mirrors, and create a new pool from them. 7446 */ 7447 int 7448 spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, 7449 nvlist_t *props, boolean_t exp) 7450 { 7451 int error = 0; 7452 uint64_t txg, *glist; 7453 spa_t *newspa; 7454 uint_t c, children, lastlog; 7455 nvlist_t **child, *nvl, *tmp; 7456 dmu_tx_t *tx; 7457 char *altroot = NULL; 7458 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 7459 boolean_t activate_slog; 7460 7461 ASSERT(spa_writeable(spa)); 7462 7463 txg = spa_vdev_enter(spa); 7464 7465 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7466 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7467 error = (spa_has_checkpoint(spa)) ? 7468 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7469 return (spa_vdev_exit(spa, NULL, txg, error)); 7470 } 7471 7472 /* clear the log and flush everything up to now */ 7473 activate_slog = spa_passivate_log(spa); 7474 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7475 error = spa_reset_logs(spa); 7476 txg = spa_vdev_config_enter(spa); 7477 7478 if (activate_slog) 7479 spa_activate_log(spa); 7480 7481 if (error != 0) 7482 return (spa_vdev_exit(spa, NULL, txg, error)); 7483 7484 /* check new spa name before going any further */ 7485 if (spa_lookup(newname) != NULL) 7486 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 7487 7488 /* 7489 * scan through all the children to ensure they're all mirrors 7490 */ 7491 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 7492 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 7493 &children) != 0) 7494 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7495 7496 /* first, check to ensure we've got the right child count */ 7497 rvd = spa->spa_root_vdev; 7498 lastlog = 0; 7499 for (c = 0; c < rvd->vdev_children; c++) { 7500 vdev_t *vd = rvd->vdev_child[c]; 7501 7502 /* don't count the holes & logs as children */ 7503 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 7504 !vdev_is_concrete(vd))) { 7505 if (lastlog == 0) 7506 lastlog = c; 7507 continue; 7508 } 7509 7510 lastlog = 0; 7511 } 7512 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 7513 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7514 7515 /* next, ensure no spare or cache devices are part of the split */ 7516 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 7517 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 7518 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7519 7520 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 7521 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 7522 7523 /* then, loop over each vdev and validate it */ 7524 for (c = 0; c < children; c++) { 7525 uint64_t is_hole = 0; 7526 7527 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 7528 &is_hole); 7529 7530 if (is_hole != 0) { 7531 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 7532 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 7533 continue; 7534 } else { 7535 error = SET_ERROR(EINVAL); 7536 break; 7537 } 7538 } 7539 7540 /* deal with indirect vdevs */ 7541 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 7542 &vdev_indirect_ops) 7543 continue; 7544 7545 /* which disk is going to be split? */ 7546 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 7547 &glist[c]) != 0) { 7548 error = SET_ERROR(EINVAL); 7549 break; 7550 } 7551 7552 /* look it up in the spa */ 7553 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 7554 if (vml[c] == NULL) { 7555 error = SET_ERROR(ENODEV); 7556 break; 7557 } 7558 7559 /* make sure there's nothing stopping the split */ 7560 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 7561 vml[c]->vdev_islog || 7562 !vdev_is_concrete(vml[c]) || 7563 vml[c]->vdev_isspare || 7564 vml[c]->vdev_isl2cache || 7565 !vdev_writeable(vml[c]) || 7566 vml[c]->vdev_children != 0 || 7567 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 7568 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 7569 error = SET_ERROR(EINVAL); 7570 break; 7571 } 7572 7573 if (vdev_dtl_required(vml[c]) || 7574 vdev_resilver_needed(vml[c], NULL, NULL)) { 7575 error = SET_ERROR(EBUSY); 7576 break; 7577 } 7578 7579 /* we need certain info from the top level */ 7580 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 7581 vml[c]->vdev_top->vdev_ms_array) == 0); 7582 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 7583 vml[c]->vdev_top->vdev_ms_shift) == 0); 7584 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 7585 vml[c]->vdev_top->vdev_asize) == 0); 7586 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 7587 vml[c]->vdev_top->vdev_ashift) == 0); 7588 7589 /* transfer per-vdev ZAPs */ 7590 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 7591 VERIFY0(nvlist_add_uint64(child[c], 7592 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 7593 7594 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 7595 VERIFY0(nvlist_add_uint64(child[c], 7596 ZPOOL_CONFIG_VDEV_TOP_ZAP, 7597 vml[c]->vdev_parent->vdev_top_zap)); 7598 } 7599 7600 if (error != 0) { 7601 kmem_free(vml, children * sizeof (vdev_t *)); 7602 kmem_free(glist, children * sizeof (uint64_t)); 7603 return (spa_vdev_exit(spa, NULL, txg, error)); 7604 } 7605 7606 /* stop writers from using the disks */ 7607 for (c = 0; c < children; c++) { 7608 if (vml[c] != NULL) 7609 vml[c]->vdev_offline = B_TRUE; 7610 } 7611 vdev_reopen(spa->spa_root_vdev); 7612 7613 /* 7614 * Temporarily record the splitting vdevs in the spa config. This 7615 * will disappear once the config is regenerated. 7616 */ 7617 VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0); 7618 VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 7619 glist, children) == 0); 7620 kmem_free(glist, children * sizeof (uint64_t)); 7621 7622 mutex_enter(&spa->spa_props_lock); 7623 VERIFY(nvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, 7624 nvl) == 0); 7625 mutex_exit(&spa->spa_props_lock); 7626 spa->spa_config_splitting = nvl; 7627 vdev_config_dirty(spa->spa_root_vdev); 7628 7629 /* configure and create the new pool */ 7630 VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname) == 0); 7631 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 7632 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE) == 0); 7633 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 7634 spa_version(spa)) == 0); 7635 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, 7636 spa->spa_config_txg) == 0); 7637 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 7638 spa_generate_guid(NULL)) == 0); 7639 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 7640 (void) nvlist_lookup_string(props, 7641 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 7642 7643 /* add the new pool to the namespace */ 7644 newspa = spa_add(newname, config, altroot); 7645 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 7646 newspa->spa_config_txg = spa->spa_config_txg; 7647 spa_set_log_state(newspa, SPA_LOG_CLEAR); 7648 7649 /* release the spa config lock, retaining the namespace lock */ 7650 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7651 7652 if (zio_injection_enabled) 7653 zio_handle_panic_injection(spa, FTAG, 1); 7654 7655 spa_activate(newspa, spa_mode_global); 7656 spa_async_suspend(newspa); 7657 7658 /* 7659 * Temporarily stop the initializing and TRIM activity. We set the 7660 * state to ACTIVE so that we know to resume initializing or TRIM 7661 * once the split has completed. 7662 */ 7663 list_t vd_initialize_list; 7664 list_create(&vd_initialize_list, sizeof (vdev_t), 7665 offsetof(vdev_t, vdev_initialize_node)); 7666 7667 list_t vd_trim_list; 7668 list_create(&vd_trim_list, sizeof (vdev_t), 7669 offsetof(vdev_t, vdev_trim_node)); 7670 7671 for (c = 0; c < children; c++) { 7672 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7673 mutex_enter(&vml[c]->vdev_initialize_lock); 7674 vdev_initialize_stop(vml[c], 7675 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 7676 mutex_exit(&vml[c]->vdev_initialize_lock); 7677 7678 mutex_enter(&vml[c]->vdev_trim_lock); 7679 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 7680 mutex_exit(&vml[c]->vdev_trim_lock); 7681 } 7682 } 7683 7684 vdev_initialize_stop_wait(spa, &vd_initialize_list); 7685 vdev_trim_stop_wait(spa, &vd_trim_list); 7686 7687 list_destroy(&vd_initialize_list); 7688 list_destroy(&vd_trim_list); 7689 7690 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 7691 newspa->spa_is_splitting = B_TRUE; 7692 7693 /* create the new pool from the disks of the original pool */ 7694 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 7695 if (error) 7696 goto out; 7697 7698 /* if that worked, generate a real config for the new pool */ 7699 if (newspa->spa_root_vdev != NULL) { 7700 VERIFY(nvlist_alloc(&newspa->spa_config_splitting, 7701 NV_UNIQUE_NAME, KM_SLEEP) == 0); 7702 VERIFY(nvlist_add_uint64(newspa->spa_config_splitting, 7703 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0); 7704 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 7705 B_TRUE)); 7706 } 7707 7708 /* set the props */ 7709 if (props != NULL) { 7710 spa_configfile_set(newspa, props, B_FALSE); 7711 error = spa_prop_set(newspa, props); 7712 if (error) 7713 goto out; 7714 } 7715 7716 /* flush everything */ 7717 txg = spa_vdev_config_enter(newspa); 7718 vdev_config_dirty(newspa->spa_root_vdev); 7719 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 7720 7721 if (zio_injection_enabled) 7722 zio_handle_panic_injection(spa, FTAG, 2); 7723 7724 spa_async_resume(newspa); 7725 7726 /* finally, update the original pool's config */ 7727 txg = spa_vdev_config_enter(spa); 7728 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 7729 error = dmu_tx_assign(tx, TXG_WAIT); 7730 if (error != 0) 7731 dmu_tx_abort(tx); 7732 for (c = 0; c < children; c++) { 7733 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7734 vdev_t *tvd = vml[c]->vdev_top; 7735 7736 /* 7737 * Need to be sure the detachable VDEV is not 7738 * on any *other* txg's DTL list to prevent it 7739 * from being accessed after it's freed. 7740 */ 7741 for (int t = 0; t < TXG_SIZE; t++) { 7742 (void) txg_list_remove_this( 7743 &tvd->vdev_dtl_list, vml[c], t); 7744 } 7745 7746 vdev_split(vml[c]); 7747 if (error == 0) 7748 spa_history_log_internal(spa, "detach", tx, 7749 "vdev=%s", vml[c]->vdev_path); 7750 7751 vdev_free(vml[c]); 7752 } 7753 } 7754 spa->spa_avz_action = AVZ_ACTION_REBUILD; 7755 vdev_config_dirty(spa->spa_root_vdev); 7756 spa->spa_config_splitting = NULL; 7757 nvlist_free(nvl); 7758 if (error == 0) 7759 dmu_tx_commit(tx); 7760 (void) spa_vdev_exit(spa, NULL, txg, 0); 7761 7762 if (zio_injection_enabled) 7763 zio_handle_panic_injection(spa, FTAG, 3); 7764 7765 /* split is complete; log a history record */ 7766 spa_history_log_internal(newspa, "split", NULL, 7767 "from pool %s", spa_name(spa)); 7768 7769 newspa->spa_is_splitting = B_FALSE; 7770 kmem_free(vml, children * sizeof (vdev_t *)); 7771 7772 /* if we're not going to mount the filesystems in userland, export */ 7773 if (exp) 7774 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 7775 B_FALSE, B_FALSE); 7776 7777 return (error); 7778 7779 out: 7780 spa_unload(newspa); 7781 spa_deactivate(newspa); 7782 spa_remove(newspa); 7783 7784 txg = spa_vdev_config_enter(spa); 7785 7786 /* re-online all offlined disks */ 7787 for (c = 0; c < children; c++) { 7788 if (vml[c] != NULL) 7789 vml[c]->vdev_offline = B_FALSE; 7790 } 7791 7792 /* restart initializing or trimming disks as necessary */ 7793 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 7794 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 7795 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 7796 7797 vdev_reopen(spa->spa_root_vdev); 7798 7799 nvlist_free(spa->spa_config_splitting); 7800 spa->spa_config_splitting = NULL; 7801 (void) spa_vdev_exit(spa, NULL, txg, error); 7802 7803 kmem_free(vml, children * sizeof (vdev_t *)); 7804 return (error); 7805 } 7806 7807 /* 7808 * Find any device that's done replacing, or a vdev marked 'unspare' that's 7809 * currently spared, so we can detach it. 7810 */ 7811 static vdev_t * 7812 spa_vdev_resilver_done_hunt(vdev_t *vd) 7813 { 7814 vdev_t *newvd, *oldvd; 7815 7816 for (int c = 0; c < vd->vdev_children; c++) { 7817 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 7818 if (oldvd != NULL) 7819 return (oldvd); 7820 } 7821 7822 /* 7823 * Check for a completed replacement. We always consider the first 7824 * vdev in the list to be the oldest vdev, and the last one to be 7825 * the newest (see spa_vdev_attach() for how that works). In 7826 * the case where the newest vdev is faulted, we will not automatically 7827 * remove it after a resilver completes. This is OK as it will require 7828 * user intervention to determine which disk the admin wishes to keep. 7829 */ 7830 if (vd->vdev_ops == &vdev_replacing_ops) { 7831 ASSERT(vd->vdev_children > 1); 7832 7833 newvd = vd->vdev_child[vd->vdev_children - 1]; 7834 oldvd = vd->vdev_child[0]; 7835 7836 if (vdev_dtl_empty(newvd, DTL_MISSING) && 7837 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7838 !vdev_dtl_required(oldvd)) 7839 return (oldvd); 7840 } 7841 7842 /* 7843 * Check for a completed resilver with the 'unspare' flag set. 7844 * Also potentially update faulted state. 7845 */ 7846 if (vd->vdev_ops == &vdev_spare_ops) { 7847 vdev_t *first = vd->vdev_child[0]; 7848 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 7849 7850 if (last->vdev_unspare) { 7851 oldvd = first; 7852 newvd = last; 7853 } else if (first->vdev_unspare) { 7854 oldvd = last; 7855 newvd = first; 7856 } else { 7857 oldvd = NULL; 7858 } 7859 7860 if (oldvd != NULL && 7861 vdev_dtl_empty(newvd, DTL_MISSING) && 7862 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7863 !vdev_dtl_required(oldvd)) 7864 return (oldvd); 7865 7866 vdev_propagate_state(vd); 7867 7868 /* 7869 * If there are more than two spares attached to a disk, 7870 * and those spares are not required, then we want to 7871 * attempt to free them up now so that they can be used 7872 * by other pools. Once we're back down to a single 7873 * disk+spare, we stop removing them. 7874 */ 7875 if (vd->vdev_children > 2) { 7876 newvd = vd->vdev_child[1]; 7877 7878 if (newvd->vdev_isspare && last->vdev_isspare && 7879 vdev_dtl_empty(last, DTL_MISSING) && 7880 vdev_dtl_empty(last, DTL_OUTAGE) && 7881 !vdev_dtl_required(newvd)) 7882 return (newvd); 7883 } 7884 } 7885 7886 return (NULL); 7887 } 7888 7889 static void 7890 spa_vdev_resilver_done(spa_t *spa) 7891 { 7892 vdev_t *vd, *pvd, *ppvd; 7893 uint64_t guid, sguid, pguid, ppguid; 7894 7895 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7896 7897 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 7898 pvd = vd->vdev_parent; 7899 ppvd = pvd->vdev_parent; 7900 guid = vd->vdev_guid; 7901 pguid = pvd->vdev_guid; 7902 ppguid = ppvd->vdev_guid; 7903 sguid = 0; 7904 /* 7905 * If we have just finished replacing a hot spared device, then 7906 * we need to detach the parent's first child (the original hot 7907 * spare) as well. 7908 */ 7909 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 7910 ppvd->vdev_children == 2) { 7911 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 7912 sguid = ppvd->vdev_child[1]->vdev_guid; 7913 } 7914 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 7915 7916 spa_config_exit(spa, SCL_ALL, FTAG); 7917 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 7918 return; 7919 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 7920 return; 7921 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7922 } 7923 7924 spa_config_exit(spa, SCL_ALL, FTAG); 7925 7926 /* 7927 * If a detach was not performed above replace waiters will not have 7928 * been notified. In which case we must do so now. 7929 */ 7930 spa_notify_waiters(spa); 7931 } 7932 7933 /* 7934 * Update the stored path or FRU for this vdev. 7935 */ 7936 static int 7937 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 7938 boolean_t ispath) 7939 { 7940 vdev_t *vd; 7941 boolean_t sync = B_FALSE; 7942 7943 ASSERT(spa_writeable(spa)); 7944 7945 spa_vdev_state_enter(spa, SCL_ALL); 7946 7947 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 7948 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 7949 7950 if (!vd->vdev_ops->vdev_op_leaf) 7951 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 7952 7953 if (ispath) { 7954 if (strcmp(value, vd->vdev_path) != 0) { 7955 spa_strfree(vd->vdev_path); 7956 vd->vdev_path = spa_strdup(value); 7957 sync = B_TRUE; 7958 } 7959 } else { 7960 if (vd->vdev_fru == NULL) { 7961 vd->vdev_fru = spa_strdup(value); 7962 sync = B_TRUE; 7963 } else if (strcmp(value, vd->vdev_fru) != 0) { 7964 spa_strfree(vd->vdev_fru); 7965 vd->vdev_fru = spa_strdup(value); 7966 sync = B_TRUE; 7967 } 7968 } 7969 7970 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 7971 } 7972 7973 int 7974 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 7975 { 7976 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 7977 } 7978 7979 int 7980 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 7981 { 7982 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 7983 } 7984 7985 /* 7986 * ========================================================================== 7987 * SPA Scanning 7988 * ========================================================================== 7989 */ 7990 int 7991 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 7992 { 7993 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 7994 7995 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 7996 return (SET_ERROR(EBUSY)); 7997 7998 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 7999 } 8000 8001 int 8002 spa_scan_stop(spa_t *spa) 8003 { 8004 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8005 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8006 return (SET_ERROR(EBUSY)); 8007 return (dsl_scan_cancel(spa->spa_dsl_pool)); 8008 } 8009 8010 int 8011 spa_scan(spa_t *spa, pool_scan_func_t func) 8012 { 8013 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8014 8015 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 8016 return (SET_ERROR(ENOTSUP)); 8017 8018 if (func == POOL_SCAN_RESILVER && 8019 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 8020 return (SET_ERROR(ENOTSUP)); 8021 8022 /* 8023 * If a resilver was requested, but there is no DTL on a 8024 * writeable leaf device, we have nothing to do. 8025 */ 8026 if (func == POOL_SCAN_RESILVER && 8027 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 8028 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 8029 return (0); 8030 } 8031 8032 return (dsl_scan(spa->spa_dsl_pool, func)); 8033 } 8034 8035 /* 8036 * ========================================================================== 8037 * SPA async task processing 8038 * ========================================================================== 8039 */ 8040 8041 static void 8042 spa_async_remove(spa_t *spa, vdev_t *vd) 8043 { 8044 if (vd->vdev_remove_wanted) { 8045 vd->vdev_remove_wanted = B_FALSE; 8046 vd->vdev_delayed_close = B_FALSE; 8047 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 8048 8049 /* 8050 * We want to clear the stats, but we don't want to do a full 8051 * vdev_clear() as that will cause us to throw away 8052 * degraded/faulted state as well as attempt to reopen the 8053 * device, all of which is a waste. 8054 */ 8055 vd->vdev_stat.vs_read_errors = 0; 8056 vd->vdev_stat.vs_write_errors = 0; 8057 vd->vdev_stat.vs_checksum_errors = 0; 8058 8059 vdev_state_dirty(vd->vdev_top); 8060 8061 /* Tell userspace that the vdev is gone. */ 8062 zfs_post_remove(spa, vd); 8063 } 8064 8065 for (int c = 0; c < vd->vdev_children; c++) 8066 spa_async_remove(spa, vd->vdev_child[c]); 8067 } 8068 8069 static void 8070 spa_async_probe(spa_t *spa, vdev_t *vd) 8071 { 8072 if (vd->vdev_probe_wanted) { 8073 vd->vdev_probe_wanted = B_FALSE; 8074 vdev_reopen(vd); /* vdev_open() does the actual probe */ 8075 } 8076 8077 for (int c = 0; c < vd->vdev_children; c++) 8078 spa_async_probe(spa, vd->vdev_child[c]); 8079 } 8080 8081 static void 8082 spa_async_autoexpand(spa_t *spa, vdev_t *vd) 8083 { 8084 if (!spa->spa_autoexpand) 8085 return; 8086 8087 for (int c = 0; c < vd->vdev_children; c++) { 8088 vdev_t *cvd = vd->vdev_child[c]; 8089 spa_async_autoexpand(spa, cvd); 8090 } 8091 8092 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 8093 return; 8094 8095 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 8096 } 8097 8098 static void 8099 spa_async_thread(void *arg) 8100 { 8101 spa_t *spa = (spa_t *)arg; 8102 dsl_pool_t *dp = spa->spa_dsl_pool; 8103 int tasks; 8104 8105 ASSERT(spa->spa_sync_on); 8106 8107 mutex_enter(&spa->spa_async_lock); 8108 tasks = spa->spa_async_tasks; 8109 spa->spa_async_tasks = 0; 8110 mutex_exit(&spa->spa_async_lock); 8111 8112 /* 8113 * See if the config needs to be updated. 8114 */ 8115 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 8116 uint64_t old_space, new_space; 8117 8118 mutex_enter(&spa_namespace_lock); 8119 old_space = metaslab_class_get_space(spa_normal_class(spa)); 8120 old_space += metaslab_class_get_space(spa_special_class(spa)); 8121 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 8122 old_space += metaslab_class_get_space( 8123 spa_embedded_log_class(spa)); 8124 8125 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 8126 8127 new_space = metaslab_class_get_space(spa_normal_class(spa)); 8128 new_space += metaslab_class_get_space(spa_special_class(spa)); 8129 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 8130 new_space += metaslab_class_get_space( 8131 spa_embedded_log_class(spa)); 8132 mutex_exit(&spa_namespace_lock); 8133 8134 /* 8135 * If the pool grew as a result of the config update, 8136 * then log an internal history event. 8137 */ 8138 if (new_space != old_space) { 8139 spa_history_log_internal(spa, "vdev online", NULL, 8140 "pool '%s' size: %llu(+%llu)", 8141 spa_name(spa), (u_longlong_t)new_space, 8142 (u_longlong_t)(new_space - old_space)); 8143 } 8144 } 8145 8146 /* 8147 * See if any devices need to be marked REMOVED. 8148 */ 8149 if (tasks & SPA_ASYNC_REMOVE) { 8150 spa_vdev_state_enter(spa, SCL_NONE); 8151 spa_async_remove(spa, spa->spa_root_vdev); 8152 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 8153 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); 8154 for (int i = 0; i < spa->spa_spares.sav_count; i++) 8155 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); 8156 (void) spa_vdev_state_exit(spa, NULL, 0); 8157 } 8158 8159 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 8160 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8161 spa_async_autoexpand(spa, spa->spa_root_vdev); 8162 spa_config_exit(spa, SCL_CONFIG, FTAG); 8163 } 8164 8165 /* 8166 * See if any devices need to be probed. 8167 */ 8168 if (tasks & SPA_ASYNC_PROBE) { 8169 spa_vdev_state_enter(spa, SCL_NONE); 8170 spa_async_probe(spa, spa->spa_root_vdev); 8171 (void) spa_vdev_state_exit(spa, NULL, 0); 8172 } 8173 8174 /* 8175 * If any devices are done replacing, detach them. 8176 */ 8177 if (tasks & SPA_ASYNC_RESILVER_DONE || 8178 tasks & SPA_ASYNC_REBUILD_DONE) { 8179 spa_vdev_resilver_done(spa); 8180 } 8181 8182 /* 8183 * Kick off a resilver. 8184 */ 8185 if (tasks & SPA_ASYNC_RESILVER && 8186 !vdev_rebuild_active(spa->spa_root_vdev) && 8187 (!dsl_scan_resilvering(dp) || 8188 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 8189 dsl_scan_restart_resilver(dp, 0); 8190 8191 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 8192 mutex_enter(&spa_namespace_lock); 8193 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8194 vdev_initialize_restart(spa->spa_root_vdev); 8195 spa_config_exit(spa, SCL_CONFIG, FTAG); 8196 mutex_exit(&spa_namespace_lock); 8197 } 8198 8199 if (tasks & SPA_ASYNC_TRIM_RESTART) { 8200 mutex_enter(&spa_namespace_lock); 8201 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8202 vdev_trim_restart(spa->spa_root_vdev); 8203 spa_config_exit(spa, SCL_CONFIG, FTAG); 8204 mutex_exit(&spa_namespace_lock); 8205 } 8206 8207 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 8208 mutex_enter(&spa_namespace_lock); 8209 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8210 vdev_autotrim_restart(spa); 8211 spa_config_exit(spa, SCL_CONFIG, FTAG); 8212 mutex_exit(&spa_namespace_lock); 8213 } 8214 8215 /* 8216 * Kick off L2 cache whole device TRIM. 8217 */ 8218 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 8219 mutex_enter(&spa_namespace_lock); 8220 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8221 vdev_trim_l2arc(spa); 8222 spa_config_exit(spa, SCL_CONFIG, FTAG); 8223 mutex_exit(&spa_namespace_lock); 8224 } 8225 8226 /* 8227 * Kick off L2 cache rebuilding. 8228 */ 8229 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 8230 mutex_enter(&spa_namespace_lock); 8231 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 8232 l2arc_spa_rebuild_start(spa); 8233 spa_config_exit(spa, SCL_L2ARC, FTAG); 8234 mutex_exit(&spa_namespace_lock); 8235 } 8236 8237 /* 8238 * Let the world know that we're done. 8239 */ 8240 mutex_enter(&spa->spa_async_lock); 8241 spa->spa_async_thread = NULL; 8242 cv_broadcast(&spa->spa_async_cv); 8243 mutex_exit(&spa->spa_async_lock); 8244 thread_exit(); 8245 } 8246 8247 void 8248 spa_async_suspend(spa_t *spa) 8249 { 8250 mutex_enter(&spa->spa_async_lock); 8251 spa->spa_async_suspended++; 8252 while (spa->spa_async_thread != NULL) 8253 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 8254 mutex_exit(&spa->spa_async_lock); 8255 8256 spa_vdev_remove_suspend(spa); 8257 8258 zthr_t *condense_thread = spa->spa_condense_zthr; 8259 if (condense_thread != NULL) 8260 zthr_cancel(condense_thread); 8261 8262 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8263 if (discard_thread != NULL) 8264 zthr_cancel(discard_thread); 8265 8266 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8267 if (ll_delete_thread != NULL) 8268 zthr_cancel(ll_delete_thread); 8269 8270 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8271 if (ll_condense_thread != NULL) 8272 zthr_cancel(ll_condense_thread); 8273 } 8274 8275 void 8276 spa_async_resume(spa_t *spa) 8277 { 8278 mutex_enter(&spa->spa_async_lock); 8279 ASSERT(spa->spa_async_suspended != 0); 8280 spa->spa_async_suspended--; 8281 mutex_exit(&spa->spa_async_lock); 8282 spa_restart_removal(spa); 8283 8284 zthr_t *condense_thread = spa->spa_condense_zthr; 8285 if (condense_thread != NULL) 8286 zthr_resume(condense_thread); 8287 8288 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8289 if (discard_thread != NULL) 8290 zthr_resume(discard_thread); 8291 8292 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8293 if (ll_delete_thread != NULL) 8294 zthr_resume(ll_delete_thread); 8295 8296 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8297 if (ll_condense_thread != NULL) 8298 zthr_resume(ll_condense_thread); 8299 } 8300 8301 static boolean_t 8302 spa_async_tasks_pending(spa_t *spa) 8303 { 8304 uint_t non_config_tasks; 8305 uint_t config_task; 8306 boolean_t config_task_suspended; 8307 8308 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 8309 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 8310 if (spa->spa_ccw_fail_time == 0) { 8311 config_task_suspended = B_FALSE; 8312 } else { 8313 config_task_suspended = 8314 (gethrtime() - spa->spa_ccw_fail_time) < 8315 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 8316 } 8317 8318 return (non_config_tasks || (config_task && !config_task_suspended)); 8319 } 8320 8321 static void 8322 spa_async_dispatch(spa_t *spa) 8323 { 8324 mutex_enter(&spa->spa_async_lock); 8325 if (spa_async_tasks_pending(spa) && 8326 !spa->spa_async_suspended && 8327 spa->spa_async_thread == NULL) 8328 spa->spa_async_thread = thread_create(NULL, 0, 8329 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 8330 mutex_exit(&spa->spa_async_lock); 8331 } 8332 8333 void 8334 spa_async_request(spa_t *spa, int task) 8335 { 8336 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); 8337 mutex_enter(&spa->spa_async_lock); 8338 spa->spa_async_tasks |= task; 8339 mutex_exit(&spa->spa_async_lock); 8340 } 8341 8342 int 8343 spa_async_tasks(spa_t *spa) 8344 { 8345 return (spa->spa_async_tasks); 8346 } 8347 8348 /* 8349 * ========================================================================== 8350 * SPA syncing routines 8351 * ========================================================================== 8352 */ 8353 8354 8355 static int 8356 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8357 dmu_tx_t *tx) 8358 { 8359 bpobj_t *bpo = arg; 8360 bpobj_enqueue(bpo, bp, bp_freed, tx); 8361 return (0); 8362 } 8363 8364 int 8365 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8366 { 8367 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 8368 } 8369 8370 int 8371 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8372 { 8373 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 8374 } 8375 8376 static int 8377 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8378 { 8379 zio_t *pio = arg; 8380 8381 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 8382 pio->io_flags)); 8383 return (0); 8384 } 8385 8386 static int 8387 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8388 dmu_tx_t *tx) 8389 { 8390 ASSERT(!bp_freed); 8391 return (spa_free_sync_cb(arg, bp, tx)); 8392 } 8393 8394 /* 8395 * Note: this simple function is not inlined to make it easier to dtrace the 8396 * amount of time spent syncing frees. 8397 */ 8398 static void 8399 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 8400 { 8401 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8402 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 8403 VERIFY(zio_wait(zio) == 0); 8404 } 8405 8406 /* 8407 * Note: this simple function is not inlined to make it easier to dtrace the 8408 * amount of time spent syncing deferred frees. 8409 */ 8410 static void 8411 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 8412 { 8413 if (spa_sync_pass(spa) != 1) 8414 return; 8415 8416 /* 8417 * Note: 8418 * If the log space map feature is active, we stop deferring 8419 * frees to the next TXG and therefore running this function 8420 * would be considered a no-op as spa_deferred_bpobj should 8421 * not have any entries. 8422 * 8423 * That said we run this function anyway (instead of returning 8424 * immediately) for the edge-case scenario where we just 8425 * activated the log space map feature in this TXG but we have 8426 * deferred frees from the previous TXG. 8427 */ 8428 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8429 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 8430 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 8431 VERIFY0(zio_wait(zio)); 8432 } 8433 8434 static void 8435 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 8436 { 8437 char *packed = NULL; 8438 size_t bufsize; 8439 size_t nvsize = 0; 8440 dmu_buf_t *db; 8441 8442 VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); 8443 8444 /* 8445 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 8446 * information. This avoids the dmu_buf_will_dirty() path and 8447 * saves us a pre-read to get data we don't actually care about. 8448 */ 8449 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 8450 packed = vmem_alloc(bufsize, KM_SLEEP); 8451 8452 VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 8453 KM_SLEEP) == 0); 8454 bzero(packed + nvsize, bufsize - nvsize); 8455 8456 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); 8457 8458 vmem_free(packed, bufsize); 8459 8460 VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 8461 dmu_buf_will_dirty(db, tx); 8462 *(uint64_t *)db->db_data = nvsize; 8463 dmu_buf_rele(db, FTAG); 8464 } 8465 8466 static void 8467 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 8468 const char *config, const char *entry) 8469 { 8470 nvlist_t *nvroot; 8471 nvlist_t **list; 8472 int i; 8473 8474 if (!sav->sav_sync) 8475 return; 8476 8477 /* 8478 * Update the MOS nvlist describing the list of available devices. 8479 * spa_validate_aux() will have already made sure this nvlist is 8480 * valid and the vdevs are labeled appropriately. 8481 */ 8482 if (sav->sav_object == 0) { 8483 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 8484 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 8485 sizeof (uint64_t), tx); 8486 VERIFY(zap_update(spa->spa_meta_objset, 8487 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 8488 &sav->sav_object, tx) == 0); 8489 } 8490 8491 VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); 8492 if (sav->sav_count == 0) { 8493 VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0); 8494 } else { 8495 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 8496 for (i = 0; i < sav->sav_count; i++) 8497 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 8498 B_FALSE, VDEV_CONFIG_L2CACHE); 8499 VERIFY(nvlist_add_nvlist_array(nvroot, config, list, 8500 sav->sav_count) == 0); 8501 for (i = 0; i < sav->sav_count; i++) 8502 nvlist_free(list[i]); 8503 kmem_free(list, sav->sav_count * sizeof (void *)); 8504 } 8505 8506 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 8507 nvlist_free(nvroot); 8508 8509 sav->sav_sync = B_FALSE; 8510 } 8511 8512 /* 8513 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 8514 * The all-vdev ZAP must be empty. 8515 */ 8516 static void 8517 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 8518 { 8519 spa_t *spa = vd->vdev_spa; 8520 8521 if (vd->vdev_top_zap != 0) { 8522 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8523 vd->vdev_top_zap, tx)); 8524 } 8525 if (vd->vdev_leaf_zap != 0) { 8526 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8527 vd->vdev_leaf_zap, tx)); 8528 } 8529 for (uint64_t i = 0; i < vd->vdev_children; i++) { 8530 spa_avz_build(vd->vdev_child[i], avz, tx); 8531 } 8532 } 8533 8534 static void 8535 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 8536 { 8537 nvlist_t *config; 8538 8539 /* 8540 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 8541 * its config may not be dirty but we still need to build per-vdev ZAPs. 8542 * Similarly, if the pool is being assembled (e.g. after a split), we 8543 * need to rebuild the AVZ although the config may not be dirty. 8544 */ 8545 if (list_is_empty(&spa->spa_config_dirty_list) && 8546 spa->spa_avz_action == AVZ_ACTION_NONE) 8547 return; 8548 8549 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 8550 8551 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 8552 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 8553 spa->spa_all_vdev_zaps != 0); 8554 8555 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 8556 /* Make and build the new AVZ */ 8557 uint64_t new_avz = zap_create(spa->spa_meta_objset, 8558 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 8559 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 8560 8561 /* Diff old AVZ with new one */ 8562 zap_cursor_t zc; 8563 zap_attribute_t za; 8564 8565 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8566 spa->spa_all_vdev_zaps); 8567 zap_cursor_retrieve(&zc, &za) == 0; 8568 zap_cursor_advance(&zc)) { 8569 uint64_t vdzap = za.za_first_integer; 8570 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 8571 vdzap) == ENOENT) { 8572 /* 8573 * ZAP is listed in old AVZ but not in new one; 8574 * destroy it 8575 */ 8576 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 8577 tx)); 8578 } 8579 } 8580 8581 zap_cursor_fini(&zc); 8582 8583 /* Destroy the old AVZ */ 8584 VERIFY0(zap_destroy(spa->spa_meta_objset, 8585 spa->spa_all_vdev_zaps, tx)); 8586 8587 /* Replace the old AVZ in the dir obj with the new one */ 8588 VERIFY0(zap_update(spa->spa_meta_objset, 8589 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 8590 sizeof (new_avz), 1, &new_avz, tx)); 8591 8592 spa->spa_all_vdev_zaps = new_avz; 8593 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 8594 zap_cursor_t zc; 8595 zap_attribute_t za; 8596 8597 /* Walk through the AVZ and destroy all listed ZAPs */ 8598 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8599 spa->spa_all_vdev_zaps); 8600 zap_cursor_retrieve(&zc, &za) == 0; 8601 zap_cursor_advance(&zc)) { 8602 uint64_t zap = za.za_first_integer; 8603 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 8604 } 8605 8606 zap_cursor_fini(&zc); 8607 8608 /* Destroy and unlink the AVZ itself */ 8609 VERIFY0(zap_destroy(spa->spa_meta_objset, 8610 spa->spa_all_vdev_zaps, tx)); 8611 VERIFY0(zap_remove(spa->spa_meta_objset, 8612 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 8613 spa->spa_all_vdev_zaps = 0; 8614 } 8615 8616 if (spa->spa_all_vdev_zaps == 0) { 8617 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 8618 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 8619 DMU_POOL_VDEV_ZAP_MAP, tx); 8620 } 8621 spa->spa_avz_action = AVZ_ACTION_NONE; 8622 8623 /* Create ZAPs for vdevs that don't have them. */ 8624 vdev_construct_zaps(spa->spa_root_vdev, tx); 8625 8626 config = spa_config_generate(spa, spa->spa_root_vdev, 8627 dmu_tx_get_txg(tx), B_FALSE); 8628 8629 /* 8630 * If we're upgrading the spa version then make sure that 8631 * the config object gets updated with the correct version. 8632 */ 8633 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 8634 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 8635 spa->spa_uberblock.ub_version); 8636 8637 spa_config_exit(spa, SCL_STATE, FTAG); 8638 8639 nvlist_free(spa->spa_config_syncing); 8640 spa->spa_config_syncing = config; 8641 8642 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 8643 } 8644 8645 static void 8646 spa_sync_version(void *arg, dmu_tx_t *tx) 8647 { 8648 uint64_t *versionp = arg; 8649 uint64_t version = *versionp; 8650 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8651 8652 /* 8653 * Setting the version is special cased when first creating the pool. 8654 */ 8655 ASSERT(tx->tx_txg != TXG_INITIAL); 8656 8657 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 8658 ASSERT(version >= spa_version(spa)); 8659 8660 spa->spa_uberblock.ub_version = version; 8661 vdev_config_dirty(spa->spa_root_vdev); 8662 spa_history_log_internal(spa, "set", tx, "version=%lld", 8663 (longlong_t)version); 8664 } 8665 8666 /* 8667 * Set zpool properties. 8668 */ 8669 static void 8670 spa_sync_props(void *arg, dmu_tx_t *tx) 8671 { 8672 nvlist_t *nvp = arg; 8673 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8674 objset_t *mos = spa->spa_meta_objset; 8675 nvpair_t *elem = NULL; 8676 8677 mutex_enter(&spa->spa_props_lock); 8678 8679 while ((elem = nvlist_next_nvpair(nvp, elem))) { 8680 uint64_t intval; 8681 char *strval, *fname; 8682 zpool_prop_t prop; 8683 const char *propname; 8684 zprop_type_t proptype; 8685 spa_feature_t fid; 8686 8687 switch (prop = zpool_name_to_prop(nvpair_name(elem))) { 8688 case ZPOOL_PROP_INVAL: 8689 /* 8690 * We checked this earlier in spa_prop_validate(). 8691 */ 8692 ASSERT(zpool_prop_feature(nvpair_name(elem))); 8693 8694 fname = strchr(nvpair_name(elem), '@') + 1; 8695 VERIFY0(zfeature_lookup_name(fname, &fid)); 8696 8697 spa_feature_enable(spa, fid, tx); 8698 spa_history_log_internal(spa, "set", tx, 8699 "%s=enabled", nvpair_name(elem)); 8700 break; 8701 8702 case ZPOOL_PROP_VERSION: 8703 intval = fnvpair_value_uint64(elem); 8704 /* 8705 * The version is synced separately before other 8706 * properties and should be correct by now. 8707 */ 8708 ASSERT3U(spa_version(spa), >=, intval); 8709 break; 8710 8711 case ZPOOL_PROP_ALTROOT: 8712 /* 8713 * 'altroot' is a non-persistent property. It should 8714 * have been set temporarily at creation or import time. 8715 */ 8716 ASSERT(spa->spa_root != NULL); 8717 break; 8718 8719 case ZPOOL_PROP_READONLY: 8720 case ZPOOL_PROP_CACHEFILE: 8721 /* 8722 * 'readonly' and 'cachefile' are also non-persistent 8723 * properties. 8724 */ 8725 break; 8726 case ZPOOL_PROP_COMMENT: 8727 strval = fnvpair_value_string(elem); 8728 if (spa->spa_comment != NULL) 8729 spa_strfree(spa->spa_comment); 8730 spa->spa_comment = spa_strdup(strval); 8731 /* 8732 * We need to dirty the configuration on all the vdevs 8733 * so that their labels get updated. We also need to 8734 * update the cache file to keep it in sync with the 8735 * MOS version. It's unnecessary to do this for pool 8736 * creation since the vdev's configuration has already 8737 * been dirtied. 8738 */ 8739 if (tx->tx_txg != TXG_INITIAL) { 8740 vdev_config_dirty(spa->spa_root_vdev); 8741 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8742 } 8743 spa_history_log_internal(spa, "set", tx, 8744 "%s=%s", nvpair_name(elem), strval); 8745 break; 8746 case ZPOOL_PROP_COMPATIBILITY: 8747 strval = fnvpair_value_string(elem); 8748 if (spa->spa_compatibility != NULL) 8749 spa_strfree(spa->spa_compatibility); 8750 spa->spa_compatibility = spa_strdup(strval); 8751 /* 8752 * Dirty the configuration on vdevs as above. 8753 */ 8754 if (tx->tx_txg != TXG_INITIAL) { 8755 vdev_config_dirty(spa->spa_root_vdev); 8756 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8757 } 8758 8759 spa_history_log_internal(spa, "set", tx, 8760 "%s=%s", nvpair_name(elem), strval); 8761 break; 8762 8763 default: 8764 /* 8765 * Set pool property values in the poolprops mos object. 8766 */ 8767 if (spa->spa_pool_props_object == 0) { 8768 spa->spa_pool_props_object = 8769 zap_create_link(mos, DMU_OT_POOL_PROPS, 8770 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 8771 tx); 8772 } 8773 8774 /* normalize the property name */ 8775 propname = zpool_prop_to_name(prop); 8776 proptype = zpool_prop_get_type(prop); 8777 8778 if (nvpair_type(elem) == DATA_TYPE_STRING) { 8779 ASSERT(proptype == PROP_TYPE_STRING); 8780 strval = fnvpair_value_string(elem); 8781 VERIFY0(zap_update(mos, 8782 spa->spa_pool_props_object, propname, 8783 1, strlen(strval) + 1, strval, tx)); 8784 spa_history_log_internal(spa, "set", tx, 8785 "%s=%s", nvpair_name(elem), strval); 8786 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 8787 intval = fnvpair_value_uint64(elem); 8788 8789 if (proptype == PROP_TYPE_INDEX) { 8790 const char *unused; 8791 VERIFY0(zpool_prop_index_to_string( 8792 prop, intval, &unused)); 8793 } 8794 VERIFY0(zap_update(mos, 8795 spa->spa_pool_props_object, propname, 8796 8, 1, &intval, tx)); 8797 spa_history_log_internal(spa, "set", tx, 8798 "%s=%lld", nvpair_name(elem), 8799 (longlong_t)intval); 8800 } else { 8801 ASSERT(0); /* not allowed */ 8802 } 8803 8804 switch (prop) { 8805 case ZPOOL_PROP_DELEGATION: 8806 spa->spa_delegation = intval; 8807 break; 8808 case ZPOOL_PROP_BOOTFS: 8809 spa->spa_bootfs = intval; 8810 break; 8811 case ZPOOL_PROP_FAILUREMODE: 8812 spa->spa_failmode = intval; 8813 break; 8814 case ZPOOL_PROP_AUTOTRIM: 8815 spa->spa_autotrim = intval; 8816 spa_async_request(spa, 8817 SPA_ASYNC_AUTOTRIM_RESTART); 8818 break; 8819 case ZPOOL_PROP_AUTOEXPAND: 8820 spa->spa_autoexpand = intval; 8821 if (tx->tx_txg != TXG_INITIAL) 8822 spa_async_request(spa, 8823 SPA_ASYNC_AUTOEXPAND); 8824 break; 8825 case ZPOOL_PROP_MULTIHOST: 8826 spa->spa_multihost = intval; 8827 break; 8828 default: 8829 break; 8830 } 8831 } 8832 8833 } 8834 8835 mutex_exit(&spa->spa_props_lock); 8836 } 8837 8838 /* 8839 * Perform one-time upgrade on-disk changes. spa_version() does not 8840 * reflect the new version this txg, so there must be no changes this 8841 * txg to anything that the upgrade code depends on after it executes. 8842 * Therefore this must be called after dsl_pool_sync() does the sync 8843 * tasks. 8844 */ 8845 static void 8846 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 8847 { 8848 if (spa_sync_pass(spa) != 1) 8849 return; 8850 8851 dsl_pool_t *dp = spa->spa_dsl_pool; 8852 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 8853 8854 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && 8855 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { 8856 dsl_pool_create_origin(dp, tx); 8857 8858 /* Keeping the origin open increases spa_minref */ 8859 spa->spa_minref += 3; 8860 } 8861 8862 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && 8863 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { 8864 dsl_pool_upgrade_clones(dp, tx); 8865 } 8866 8867 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && 8868 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { 8869 dsl_pool_upgrade_dir_clones(dp, tx); 8870 8871 /* Keeping the freedir open increases spa_minref */ 8872 spa->spa_minref += 3; 8873 } 8874 8875 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && 8876 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8877 spa_feature_create_zap_objects(spa, tx); 8878 } 8879 8880 /* 8881 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 8882 * when possibility to use lz4 compression for metadata was added 8883 * Old pools that have this feature enabled must be upgraded to have 8884 * this feature active 8885 */ 8886 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8887 boolean_t lz4_en = spa_feature_is_enabled(spa, 8888 SPA_FEATURE_LZ4_COMPRESS); 8889 boolean_t lz4_ac = spa_feature_is_active(spa, 8890 SPA_FEATURE_LZ4_COMPRESS); 8891 8892 if (lz4_en && !lz4_ac) 8893 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 8894 } 8895 8896 /* 8897 * If we haven't written the salt, do so now. Note that the 8898 * feature may not be activated yet, but that's fine since 8899 * the presence of this ZAP entry is backwards compatible. 8900 */ 8901 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 8902 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 8903 VERIFY0(zap_add(spa->spa_meta_objset, 8904 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 8905 sizeof (spa->spa_cksum_salt.zcs_bytes), 8906 spa->spa_cksum_salt.zcs_bytes, tx)); 8907 } 8908 8909 rrw_exit(&dp->dp_config_rwlock, FTAG); 8910 } 8911 8912 static void 8913 vdev_indirect_state_sync_verify(vdev_t *vd) 8914 { 8915 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 8916 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 8917 8918 if (vd->vdev_ops == &vdev_indirect_ops) { 8919 ASSERT(vim != NULL); 8920 ASSERT(vib != NULL); 8921 } 8922 8923 uint64_t obsolete_sm_object = 0; 8924 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 8925 if (obsolete_sm_object != 0) { 8926 ASSERT(vd->vdev_obsolete_sm != NULL); 8927 ASSERT(vd->vdev_removing || 8928 vd->vdev_ops == &vdev_indirect_ops); 8929 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 8930 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 8931 ASSERT3U(obsolete_sm_object, ==, 8932 space_map_object(vd->vdev_obsolete_sm)); 8933 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 8934 space_map_allocated(vd->vdev_obsolete_sm)); 8935 } 8936 ASSERT(vd->vdev_obsolete_segments != NULL); 8937 8938 /* 8939 * Since frees / remaps to an indirect vdev can only 8940 * happen in syncing context, the obsolete segments 8941 * tree must be empty when we start syncing. 8942 */ 8943 ASSERT0(range_tree_space(vd->vdev_obsolete_segments)); 8944 } 8945 8946 /* 8947 * Set the top-level vdev's max queue depth. Evaluate each top-level's 8948 * async write queue depth in case it changed. The max queue depth will 8949 * not change in the middle of syncing out this txg. 8950 */ 8951 static void 8952 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 8953 { 8954 ASSERT(spa_writeable(spa)); 8955 8956 vdev_t *rvd = spa->spa_root_vdev; 8957 uint32_t max_queue_depth = zfs_vdev_async_write_max_active * 8958 zfs_vdev_queue_depth_pct / 100; 8959 metaslab_class_t *normal = spa_normal_class(spa); 8960 metaslab_class_t *special = spa_special_class(spa); 8961 metaslab_class_t *dedup = spa_dedup_class(spa); 8962 8963 uint64_t slots_per_allocator = 0; 8964 for (int c = 0; c < rvd->vdev_children; c++) { 8965 vdev_t *tvd = rvd->vdev_child[c]; 8966 8967 metaslab_group_t *mg = tvd->vdev_mg; 8968 if (mg == NULL || !metaslab_group_initialized(mg)) 8969 continue; 8970 8971 metaslab_class_t *mc = mg->mg_class; 8972 if (mc != normal && mc != special && mc != dedup) 8973 continue; 8974 8975 /* 8976 * It is safe to do a lock-free check here because only async 8977 * allocations look at mg_max_alloc_queue_depth, and async 8978 * allocations all happen from spa_sync(). 8979 */ 8980 for (int i = 0; i < mg->mg_allocators; i++) { 8981 ASSERT0(zfs_refcount_count( 8982 &(mg->mg_allocator[i].mga_alloc_queue_depth))); 8983 } 8984 mg->mg_max_alloc_queue_depth = max_queue_depth; 8985 8986 for (int i = 0; i < mg->mg_allocators; i++) { 8987 mg->mg_allocator[i].mga_cur_max_alloc_queue_depth = 8988 zfs_vdev_def_queue_depth; 8989 } 8990 slots_per_allocator += zfs_vdev_def_queue_depth; 8991 } 8992 8993 for (int i = 0; i < spa->spa_alloc_count; i++) { 8994 ASSERT0(zfs_refcount_count(&normal->mc_allocator[i]. 8995 mca_alloc_slots)); 8996 ASSERT0(zfs_refcount_count(&special->mc_allocator[i]. 8997 mca_alloc_slots)); 8998 ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i]. 8999 mca_alloc_slots)); 9000 normal->mc_allocator[i].mca_alloc_max_slots = 9001 slots_per_allocator; 9002 special->mc_allocator[i].mca_alloc_max_slots = 9003 slots_per_allocator; 9004 dedup->mc_allocator[i].mca_alloc_max_slots = 9005 slots_per_allocator; 9006 } 9007 normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9008 special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9009 dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9010 } 9011 9012 static void 9013 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 9014 { 9015 ASSERT(spa_writeable(spa)); 9016 9017 vdev_t *rvd = spa->spa_root_vdev; 9018 for (int c = 0; c < rvd->vdev_children; c++) { 9019 vdev_t *vd = rvd->vdev_child[c]; 9020 vdev_indirect_state_sync_verify(vd); 9021 9022 if (vdev_indirect_should_condense(vd)) { 9023 spa_condense_indirect_start_sync(vd, tx); 9024 break; 9025 } 9026 } 9027 } 9028 9029 static void 9030 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 9031 { 9032 objset_t *mos = spa->spa_meta_objset; 9033 dsl_pool_t *dp = spa->spa_dsl_pool; 9034 uint64_t txg = tx->tx_txg; 9035 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 9036 9037 do { 9038 int pass = ++spa->spa_sync_pass; 9039 9040 spa_sync_config_object(spa, tx); 9041 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 9042 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 9043 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 9044 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 9045 spa_errlog_sync(spa, txg); 9046 dsl_pool_sync(dp, txg); 9047 9048 if (pass < zfs_sync_pass_deferred_free || 9049 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 9050 /* 9051 * If the log space map feature is active we don't 9052 * care about deferred frees and the deferred bpobj 9053 * as the log space map should effectively have the 9054 * same results (i.e. appending only to one object). 9055 */ 9056 spa_sync_frees(spa, free_bpl, tx); 9057 } else { 9058 /* 9059 * We can not defer frees in pass 1, because 9060 * we sync the deferred frees later in pass 1. 9061 */ 9062 ASSERT3U(pass, >, 1); 9063 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 9064 &spa->spa_deferred_bpobj, tx); 9065 } 9066 9067 ddt_sync(spa, txg); 9068 dsl_scan_sync(dp, tx); 9069 svr_sync(spa, tx); 9070 spa_sync_upgrades(spa, tx); 9071 9072 spa_flush_metaslabs(spa, tx); 9073 9074 vdev_t *vd = NULL; 9075 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 9076 != NULL) 9077 vdev_sync(vd, txg); 9078 9079 /* 9080 * Note: We need to check if the MOS is dirty because we could 9081 * have marked the MOS dirty without updating the uberblock 9082 * (e.g. if we have sync tasks but no dirty user data). We need 9083 * to check the uberblock's rootbp because it is updated if we 9084 * have synced out dirty data (though in this case the MOS will 9085 * most likely also be dirty due to second order effects, we 9086 * don't want to rely on that here). 9087 */ 9088 if (pass == 1 && 9089 spa->spa_uberblock.ub_rootbp.blk_birth < txg && 9090 !dmu_objset_is_dirty(mos, txg)) { 9091 /* 9092 * Nothing changed on the first pass, therefore this 9093 * TXG is a no-op. Avoid syncing deferred frees, so 9094 * that we can keep this TXG as a no-op. 9095 */ 9096 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9097 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9098 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 9099 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 9100 break; 9101 } 9102 9103 spa_sync_deferred_frees(spa, tx); 9104 } while (dmu_objset_is_dirty(mos, txg)); 9105 } 9106 9107 /* 9108 * Rewrite the vdev configuration (which includes the uberblock) to 9109 * commit the transaction group. 9110 * 9111 * If there are no dirty vdevs, we sync the uberblock to a few random 9112 * top-level vdevs that are known to be visible in the config cache 9113 * (see spa_vdev_add() for a complete description). If there *are* dirty 9114 * vdevs, sync the uberblock to all vdevs. 9115 */ 9116 static void 9117 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 9118 { 9119 vdev_t *rvd = spa->spa_root_vdev; 9120 uint64_t txg = tx->tx_txg; 9121 9122 for (;;) { 9123 int error = 0; 9124 9125 /* 9126 * We hold SCL_STATE to prevent vdev open/close/etc. 9127 * while we're attempting to write the vdev labels. 9128 */ 9129 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9130 9131 if (list_is_empty(&spa->spa_config_dirty_list)) { 9132 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 9133 int svdcount = 0; 9134 int children = rvd->vdev_children; 9135 int c0 = random_in_range(children); 9136 9137 for (int c = 0; c < children; c++) { 9138 vdev_t *vd = 9139 rvd->vdev_child[(c0 + c) % children]; 9140 9141 /* Stop when revisiting the first vdev */ 9142 if (c > 0 && svd[0] == vd) 9143 break; 9144 9145 if (vd->vdev_ms_array == 0 || 9146 vd->vdev_islog || 9147 !vdev_is_concrete(vd)) 9148 continue; 9149 9150 svd[svdcount++] = vd; 9151 if (svdcount == SPA_SYNC_MIN_VDEVS) 9152 break; 9153 } 9154 error = vdev_config_sync(svd, svdcount, txg); 9155 } else { 9156 error = vdev_config_sync(rvd->vdev_child, 9157 rvd->vdev_children, txg); 9158 } 9159 9160 if (error == 0) 9161 spa->spa_last_synced_guid = rvd->vdev_guid; 9162 9163 spa_config_exit(spa, SCL_STATE, FTAG); 9164 9165 if (error == 0) 9166 break; 9167 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 9168 zio_resume_wait(spa); 9169 } 9170 } 9171 9172 /* 9173 * Sync the specified transaction group. New blocks may be dirtied as 9174 * part of the process, so we iterate until it converges. 9175 */ 9176 void 9177 spa_sync(spa_t *spa, uint64_t txg) 9178 { 9179 vdev_t *vd = NULL; 9180 9181 VERIFY(spa_writeable(spa)); 9182 9183 /* 9184 * Wait for i/os issued in open context that need to complete 9185 * before this txg syncs. 9186 */ 9187 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 9188 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 9189 ZIO_FLAG_CANFAIL); 9190 9191 /* 9192 * Lock out configuration changes. 9193 */ 9194 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9195 9196 spa->spa_syncing_txg = txg; 9197 spa->spa_sync_pass = 0; 9198 9199 for (int i = 0; i < spa->spa_alloc_count; i++) { 9200 mutex_enter(&spa->spa_alloc_locks[i]); 9201 VERIFY0(avl_numnodes(&spa->spa_alloc_trees[i])); 9202 mutex_exit(&spa->spa_alloc_locks[i]); 9203 } 9204 9205 /* 9206 * If there are any pending vdev state changes, convert them 9207 * into config changes that go out with this transaction group. 9208 */ 9209 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9210 while (list_head(&spa->spa_state_dirty_list) != NULL) { 9211 /* 9212 * We need the write lock here because, for aux vdevs, 9213 * calling vdev_config_dirty() modifies sav_config. 9214 * This is ugly and will become unnecessary when we 9215 * eliminate the aux vdev wart by integrating all vdevs 9216 * into the root vdev tree. 9217 */ 9218 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9219 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 9220 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9221 vdev_state_clean(vd); 9222 vdev_config_dirty(vd); 9223 } 9224 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9225 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9226 } 9227 spa_config_exit(spa, SCL_STATE, FTAG); 9228 9229 dsl_pool_t *dp = spa->spa_dsl_pool; 9230 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 9231 9232 spa->spa_sync_starttime = gethrtime(); 9233 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9234 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 9235 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 9236 NSEC_TO_TICK(spa->spa_deadman_synctime)); 9237 9238 /* 9239 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 9240 * set spa_deflate if we have no raid-z vdevs. 9241 */ 9242 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 9243 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 9244 vdev_t *rvd = spa->spa_root_vdev; 9245 9246 int i; 9247 for (i = 0; i < rvd->vdev_children; i++) { 9248 vd = rvd->vdev_child[i]; 9249 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 9250 break; 9251 } 9252 if (i == rvd->vdev_children) { 9253 spa->spa_deflate = TRUE; 9254 VERIFY0(zap_add(spa->spa_meta_objset, 9255 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 9256 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 9257 } 9258 } 9259 9260 spa_sync_adjust_vdev_max_queue_depth(spa); 9261 9262 spa_sync_condense_indirect(spa, tx); 9263 9264 spa_sync_iterate_to_convergence(spa, tx); 9265 9266 #ifdef ZFS_DEBUG 9267 if (!list_is_empty(&spa->spa_config_dirty_list)) { 9268 /* 9269 * Make sure that the number of ZAPs for all the vdevs matches 9270 * the number of ZAPs in the per-vdev ZAP list. This only gets 9271 * called if the config is dirty; otherwise there may be 9272 * outstanding AVZ operations that weren't completed in 9273 * spa_sync_config_object. 9274 */ 9275 uint64_t all_vdev_zap_entry_count; 9276 ASSERT0(zap_count(spa->spa_meta_objset, 9277 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 9278 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 9279 all_vdev_zap_entry_count); 9280 } 9281 #endif 9282 9283 if (spa->spa_vdev_removal != NULL) { 9284 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 9285 } 9286 9287 spa_sync_rewrite_vdev_config(spa, tx); 9288 dmu_tx_commit(tx); 9289 9290 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9291 spa->spa_deadman_tqid = 0; 9292 9293 /* 9294 * Clear the dirty config list. 9295 */ 9296 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 9297 vdev_config_clean(vd); 9298 9299 /* 9300 * Now that the new config has synced transactionally, 9301 * let it become visible to the config cache. 9302 */ 9303 if (spa->spa_config_syncing != NULL) { 9304 spa_config_set(spa, spa->spa_config_syncing); 9305 spa->spa_config_txg = txg; 9306 spa->spa_config_syncing = NULL; 9307 } 9308 9309 dsl_pool_sync_done(dp, txg); 9310 9311 for (int i = 0; i < spa->spa_alloc_count; i++) { 9312 mutex_enter(&spa->spa_alloc_locks[i]); 9313 VERIFY0(avl_numnodes(&spa->spa_alloc_trees[i])); 9314 mutex_exit(&spa->spa_alloc_locks[i]); 9315 } 9316 9317 /* 9318 * Update usable space statistics. 9319 */ 9320 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 9321 != NULL) 9322 vdev_sync_done(vd, txg); 9323 9324 metaslab_class_evict_old(spa->spa_normal_class, txg); 9325 metaslab_class_evict_old(spa->spa_log_class, txg); 9326 9327 spa_sync_close_syncing_log_sm(spa); 9328 9329 spa_update_dspace(spa); 9330 9331 /* 9332 * It had better be the case that we didn't dirty anything 9333 * since vdev_config_sync(). 9334 */ 9335 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9336 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9337 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 9338 9339 while (zfs_pause_spa_sync) 9340 delay(1); 9341 9342 spa->spa_sync_pass = 0; 9343 9344 /* 9345 * Update the last synced uberblock here. We want to do this at 9346 * the end of spa_sync() so that consumers of spa_last_synced_txg() 9347 * will be guaranteed that all the processing associated with 9348 * that txg has been completed. 9349 */ 9350 spa->spa_ubsync = spa->spa_uberblock; 9351 spa_config_exit(spa, SCL_CONFIG, FTAG); 9352 9353 spa_handle_ignored_writes(spa); 9354 9355 /* 9356 * If any async tasks have been requested, kick them off. 9357 */ 9358 spa_async_dispatch(spa); 9359 } 9360 9361 /* 9362 * Sync all pools. We don't want to hold the namespace lock across these 9363 * operations, so we take a reference on the spa_t and drop the lock during the 9364 * sync. 9365 */ 9366 void 9367 spa_sync_allpools(void) 9368 { 9369 spa_t *spa = NULL; 9370 mutex_enter(&spa_namespace_lock); 9371 while ((spa = spa_next(spa)) != NULL) { 9372 if (spa_state(spa) != POOL_STATE_ACTIVE || 9373 !spa_writeable(spa) || spa_suspended(spa)) 9374 continue; 9375 spa_open_ref(spa, FTAG); 9376 mutex_exit(&spa_namespace_lock); 9377 txg_wait_synced(spa_get_dsl(spa), 0); 9378 mutex_enter(&spa_namespace_lock); 9379 spa_close(spa, FTAG); 9380 } 9381 mutex_exit(&spa_namespace_lock); 9382 } 9383 9384 /* 9385 * ========================================================================== 9386 * Miscellaneous routines 9387 * ========================================================================== 9388 */ 9389 9390 /* 9391 * Remove all pools in the system. 9392 */ 9393 void 9394 spa_evict_all(void) 9395 { 9396 spa_t *spa; 9397 9398 /* 9399 * Remove all cached state. All pools should be closed now, 9400 * so every spa in the AVL tree should be unreferenced. 9401 */ 9402 mutex_enter(&spa_namespace_lock); 9403 while ((spa = spa_next(NULL)) != NULL) { 9404 /* 9405 * Stop async tasks. The async thread may need to detach 9406 * a device that's been replaced, which requires grabbing 9407 * spa_namespace_lock, so we must drop it here. 9408 */ 9409 spa_open_ref(spa, FTAG); 9410 mutex_exit(&spa_namespace_lock); 9411 spa_async_suspend(spa); 9412 mutex_enter(&spa_namespace_lock); 9413 spa_close(spa, FTAG); 9414 9415 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 9416 spa_unload(spa); 9417 spa_deactivate(spa); 9418 } 9419 spa_remove(spa); 9420 } 9421 mutex_exit(&spa_namespace_lock); 9422 } 9423 9424 vdev_t * 9425 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 9426 { 9427 vdev_t *vd; 9428 int i; 9429 9430 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 9431 return (vd); 9432 9433 if (aux) { 9434 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 9435 vd = spa->spa_l2cache.sav_vdevs[i]; 9436 if (vd->vdev_guid == guid) 9437 return (vd); 9438 } 9439 9440 for (i = 0; i < spa->spa_spares.sav_count; i++) { 9441 vd = spa->spa_spares.sav_vdevs[i]; 9442 if (vd->vdev_guid == guid) 9443 return (vd); 9444 } 9445 } 9446 9447 return (NULL); 9448 } 9449 9450 void 9451 spa_upgrade(spa_t *spa, uint64_t version) 9452 { 9453 ASSERT(spa_writeable(spa)); 9454 9455 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9456 9457 /* 9458 * This should only be called for a non-faulted pool, and since a 9459 * future version would result in an unopenable pool, this shouldn't be 9460 * possible. 9461 */ 9462 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 9463 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 9464 9465 spa->spa_uberblock.ub_version = version; 9466 vdev_config_dirty(spa->spa_root_vdev); 9467 9468 spa_config_exit(spa, SCL_ALL, FTAG); 9469 9470 txg_wait_synced(spa_get_dsl(spa), 0); 9471 } 9472 9473 boolean_t 9474 spa_has_spare(spa_t *spa, uint64_t guid) 9475 { 9476 int i; 9477 uint64_t spareguid; 9478 spa_aux_vdev_t *sav = &spa->spa_spares; 9479 9480 for (i = 0; i < sav->sav_count; i++) 9481 if (sav->sav_vdevs[i]->vdev_guid == guid) 9482 return (B_TRUE); 9483 9484 for (i = 0; i < sav->sav_npending; i++) { 9485 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 9486 &spareguid) == 0 && spareguid == guid) 9487 return (B_TRUE); 9488 } 9489 9490 return (B_FALSE); 9491 } 9492 9493 /* 9494 * Check if a pool has an active shared spare device. 9495 * Note: reference count of an active spare is 2, as a spare and as a replace 9496 */ 9497 static boolean_t 9498 spa_has_active_shared_spare(spa_t *spa) 9499 { 9500 int i, refcnt; 9501 uint64_t pool; 9502 spa_aux_vdev_t *sav = &spa->spa_spares; 9503 9504 for (i = 0; i < sav->sav_count; i++) { 9505 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 9506 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 9507 refcnt > 2) 9508 return (B_TRUE); 9509 } 9510 9511 return (B_FALSE); 9512 } 9513 9514 uint64_t 9515 spa_total_metaslabs(spa_t *spa) 9516 { 9517 vdev_t *rvd = spa->spa_root_vdev; 9518 9519 uint64_t m = 0; 9520 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 9521 vdev_t *vd = rvd->vdev_child[c]; 9522 if (!vdev_is_concrete(vd)) 9523 continue; 9524 m += vd->vdev_ms_count; 9525 } 9526 return (m); 9527 } 9528 9529 /* 9530 * Notify any waiting threads that some activity has switched from being in- 9531 * progress to not-in-progress so that the thread can wake up and determine 9532 * whether it is finished waiting. 9533 */ 9534 void 9535 spa_notify_waiters(spa_t *spa) 9536 { 9537 /* 9538 * Acquiring spa_activities_lock here prevents the cv_broadcast from 9539 * happening between the waiting thread's check and cv_wait. 9540 */ 9541 mutex_enter(&spa->spa_activities_lock); 9542 cv_broadcast(&spa->spa_activities_cv); 9543 mutex_exit(&spa->spa_activities_lock); 9544 } 9545 9546 /* 9547 * Notify any waiting threads that the pool is exporting, and then block until 9548 * they are finished using the spa_t. 9549 */ 9550 void 9551 spa_wake_waiters(spa_t *spa) 9552 { 9553 mutex_enter(&spa->spa_activities_lock); 9554 spa->spa_waiters_cancel = B_TRUE; 9555 cv_broadcast(&spa->spa_activities_cv); 9556 while (spa->spa_waiters != 0) 9557 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 9558 spa->spa_waiters_cancel = B_FALSE; 9559 mutex_exit(&spa->spa_activities_lock); 9560 } 9561 9562 /* Whether the vdev or any of its descendants are being initialized/trimmed. */ 9563 static boolean_t 9564 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 9565 { 9566 spa_t *spa = vd->vdev_spa; 9567 9568 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 9569 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9570 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 9571 activity == ZPOOL_WAIT_TRIM); 9572 9573 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 9574 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 9575 9576 mutex_exit(&spa->spa_activities_lock); 9577 mutex_enter(lock); 9578 mutex_enter(&spa->spa_activities_lock); 9579 9580 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 9581 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) : 9582 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE); 9583 mutex_exit(lock); 9584 9585 if (in_progress) 9586 return (B_TRUE); 9587 9588 for (int i = 0; i < vd->vdev_children; i++) { 9589 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 9590 activity)) 9591 return (B_TRUE); 9592 } 9593 9594 return (B_FALSE); 9595 } 9596 9597 /* 9598 * If use_guid is true, this checks whether the vdev specified by guid is 9599 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 9600 * is being initialized/trimmed. The caller must hold the config lock and 9601 * spa_activities_lock. 9602 */ 9603 static int 9604 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 9605 zpool_wait_activity_t activity, boolean_t *in_progress) 9606 { 9607 mutex_exit(&spa->spa_activities_lock); 9608 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9609 mutex_enter(&spa->spa_activities_lock); 9610 9611 vdev_t *vd; 9612 if (use_guid) { 9613 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 9614 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 9615 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9616 return (EINVAL); 9617 } 9618 } else { 9619 vd = spa->spa_root_vdev; 9620 } 9621 9622 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 9623 9624 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9625 return (0); 9626 } 9627 9628 /* 9629 * Locking for waiting threads 9630 * --------------------------- 9631 * 9632 * Waiting threads need a way to check whether a given activity is in progress, 9633 * and then, if it is, wait for it to complete. Each activity will have some 9634 * in-memory representation of the relevant on-disk state which can be used to 9635 * determine whether or not the activity is in progress. The in-memory state and 9636 * the locking used to protect it will be different for each activity, and may 9637 * not be suitable for use with a cvar (e.g., some state is protected by the 9638 * config lock). To allow waiting threads to wait without any races, another 9639 * lock, spa_activities_lock, is used. 9640 * 9641 * When the state is checked, both the activity-specific lock (if there is one) 9642 * and spa_activities_lock are held. In some cases, the activity-specific lock 9643 * is acquired explicitly (e.g. the config lock). In others, the locking is 9644 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 9645 * thread releases the activity-specific lock and, if the activity is in 9646 * progress, then cv_waits using spa_activities_lock. 9647 * 9648 * The waiting thread is woken when another thread, one completing some 9649 * activity, updates the state of the activity and then calls 9650 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 9651 * needs to hold its activity-specific lock when updating the state, and this 9652 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 9653 * 9654 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 9655 * and because it is held when the waiting thread checks the state of the 9656 * activity, it can never be the case that the completing thread both updates 9657 * the activity state and cv_broadcasts in between the waiting thread's check 9658 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 9659 * 9660 * In order to prevent deadlock, when the waiting thread does its check, in some 9661 * cases it will temporarily drop spa_activities_lock in order to acquire the 9662 * activity-specific lock. The order in which spa_activities_lock and the 9663 * activity specific lock are acquired in the waiting thread is determined by 9664 * the order in which they are acquired in the completing thread; if the 9665 * completing thread calls spa_notify_waiters with the activity-specific lock 9666 * held, then the waiting thread must also acquire the activity-specific lock 9667 * first. 9668 */ 9669 9670 static int 9671 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 9672 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 9673 { 9674 int error = 0; 9675 9676 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9677 9678 switch (activity) { 9679 case ZPOOL_WAIT_CKPT_DISCARD: 9680 *in_progress = 9681 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 9682 zap_contains(spa_meta_objset(spa), 9683 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 9684 ENOENT); 9685 break; 9686 case ZPOOL_WAIT_FREE: 9687 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 9688 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 9689 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 9690 spa_livelist_delete_check(spa)); 9691 break; 9692 case ZPOOL_WAIT_INITIALIZE: 9693 case ZPOOL_WAIT_TRIM: 9694 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 9695 activity, in_progress); 9696 break; 9697 case ZPOOL_WAIT_REPLACE: 9698 mutex_exit(&spa->spa_activities_lock); 9699 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9700 mutex_enter(&spa->spa_activities_lock); 9701 9702 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 9703 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9704 break; 9705 case ZPOOL_WAIT_REMOVE: 9706 *in_progress = (spa->spa_removing_phys.sr_state == 9707 DSS_SCANNING); 9708 break; 9709 case ZPOOL_WAIT_RESILVER: 9710 if ((*in_progress = vdev_rebuild_active(spa->spa_root_vdev))) 9711 break; 9712 /* fall through */ 9713 case ZPOOL_WAIT_SCRUB: 9714 { 9715 boolean_t scanning, paused, is_scrub; 9716 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 9717 9718 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 9719 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 9720 paused = dsl_scan_is_paused_scrub(scn); 9721 *in_progress = (scanning && !paused && 9722 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 9723 break; 9724 } 9725 default: 9726 panic("unrecognized value for activity %d", activity); 9727 } 9728 9729 return (error); 9730 } 9731 9732 static int 9733 spa_wait_common(const char *pool, zpool_wait_activity_t activity, 9734 boolean_t use_tag, uint64_t tag, boolean_t *waited) 9735 { 9736 /* 9737 * The tag is used to distinguish between instances of an activity. 9738 * 'initialize' and 'trim' are the only activities that we use this for. 9739 * The other activities can only have a single instance in progress in a 9740 * pool at one time, making the tag unnecessary. 9741 * 9742 * There can be multiple devices being replaced at once, but since they 9743 * all finish once resilvering finishes, we don't bother keeping track 9744 * of them individually, we just wait for them all to finish. 9745 */ 9746 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 9747 activity != ZPOOL_WAIT_TRIM) 9748 return (EINVAL); 9749 9750 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 9751 return (EINVAL); 9752 9753 spa_t *spa; 9754 int error = spa_open(pool, &spa, FTAG); 9755 if (error != 0) 9756 return (error); 9757 9758 /* 9759 * Increment the spa's waiter count so that we can call spa_close and 9760 * still ensure that the spa_t doesn't get freed before this thread is 9761 * finished with it when the pool is exported. We want to call spa_close 9762 * before we start waiting because otherwise the additional ref would 9763 * prevent the pool from being exported or destroyed throughout the 9764 * potentially long wait. 9765 */ 9766 mutex_enter(&spa->spa_activities_lock); 9767 spa->spa_waiters++; 9768 spa_close(spa, FTAG); 9769 9770 *waited = B_FALSE; 9771 for (;;) { 9772 boolean_t in_progress; 9773 error = spa_activity_in_progress(spa, activity, use_tag, tag, 9774 &in_progress); 9775 9776 if (error || !in_progress || spa->spa_waiters_cancel) 9777 break; 9778 9779 *waited = B_TRUE; 9780 9781 if (cv_wait_sig(&spa->spa_activities_cv, 9782 &spa->spa_activities_lock) == 0) { 9783 error = EINTR; 9784 break; 9785 } 9786 } 9787 9788 spa->spa_waiters--; 9789 cv_signal(&spa->spa_waiters_cv); 9790 mutex_exit(&spa->spa_activities_lock); 9791 9792 return (error); 9793 } 9794 9795 /* 9796 * Wait for a particular instance of the specified activity to complete, where 9797 * the instance is identified by 'tag' 9798 */ 9799 int 9800 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 9801 boolean_t *waited) 9802 { 9803 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 9804 } 9805 9806 /* 9807 * Wait for all instances of the specified activity complete 9808 */ 9809 int 9810 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 9811 { 9812 9813 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 9814 } 9815 9816 sysevent_t * 9817 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9818 { 9819 sysevent_t *ev = NULL; 9820 #ifdef _KERNEL 9821 nvlist_t *resource; 9822 9823 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 9824 if (resource) { 9825 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 9826 ev->resource = resource; 9827 } 9828 #endif 9829 return (ev); 9830 } 9831 9832 void 9833 spa_event_post(sysevent_t *ev) 9834 { 9835 #ifdef _KERNEL 9836 if (ev) { 9837 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 9838 kmem_free(ev, sizeof (*ev)); 9839 } 9840 #endif 9841 } 9842 9843 /* 9844 * Post a zevent corresponding to the given sysevent. The 'name' must be one 9845 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 9846 * filled in from the spa and (optionally) the vdev. This doesn't do anything 9847 * in the userland libzpool, as we don't want consumers to misinterpret ztest 9848 * or zdb as real changes. 9849 */ 9850 void 9851 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9852 { 9853 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 9854 } 9855 9856 /* state manipulation functions */ 9857 EXPORT_SYMBOL(spa_open); 9858 EXPORT_SYMBOL(spa_open_rewind); 9859 EXPORT_SYMBOL(spa_get_stats); 9860 EXPORT_SYMBOL(spa_create); 9861 EXPORT_SYMBOL(spa_import); 9862 EXPORT_SYMBOL(spa_tryimport); 9863 EXPORT_SYMBOL(spa_destroy); 9864 EXPORT_SYMBOL(spa_export); 9865 EXPORT_SYMBOL(spa_reset); 9866 EXPORT_SYMBOL(spa_async_request); 9867 EXPORT_SYMBOL(spa_async_suspend); 9868 EXPORT_SYMBOL(spa_async_resume); 9869 EXPORT_SYMBOL(spa_inject_addref); 9870 EXPORT_SYMBOL(spa_inject_delref); 9871 EXPORT_SYMBOL(spa_scan_stat_init); 9872 EXPORT_SYMBOL(spa_scan_get_stats); 9873 9874 /* device manipulation */ 9875 EXPORT_SYMBOL(spa_vdev_add); 9876 EXPORT_SYMBOL(spa_vdev_attach); 9877 EXPORT_SYMBOL(spa_vdev_detach); 9878 EXPORT_SYMBOL(spa_vdev_setpath); 9879 EXPORT_SYMBOL(spa_vdev_setfru); 9880 EXPORT_SYMBOL(spa_vdev_split_mirror); 9881 9882 /* spare statech is global across all pools) */ 9883 EXPORT_SYMBOL(spa_spare_add); 9884 EXPORT_SYMBOL(spa_spare_remove); 9885 EXPORT_SYMBOL(spa_spare_exists); 9886 EXPORT_SYMBOL(spa_spare_activate); 9887 9888 /* L2ARC statech is global across all pools) */ 9889 EXPORT_SYMBOL(spa_l2cache_add); 9890 EXPORT_SYMBOL(spa_l2cache_remove); 9891 EXPORT_SYMBOL(spa_l2cache_exists); 9892 EXPORT_SYMBOL(spa_l2cache_activate); 9893 EXPORT_SYMBOL(spa_l2cache_drop); 9894 9895 /* scanning */ 9896 EXPORT_SYMBOL(spa_scan); 9897 EXPORT_SYMBOL(spa_scan_stop); 9898 9899 /* spa syncing */ 9900 EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 9901 EXPORT_SYMBOL(spa_sync_allpools); 9902 9903 /* properties */ 9904 EXPORT_SYMBOL(spa_prop_set); 9905 EXPORT_SYMBOL(spa_prop_get); 9906 EXPORT_SYMBOL(spa_prop_clear_bootfs); 9907 9908 /* asynchronous event notification */ 9909 EXPORT_SYMBOL(spa_event_notify); 9910 9911 /* BEGIN CSTYLED */ 9912 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, INT, ZMOD_RW, 9913 "log2 fraction of arc that can be used by inflight I/Os when " 9914 "verifying pool during import"); 9915 9916 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 9917 "Set to traverse metadata on pool import"); 9918 9919 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 9920 "Set to traverse data on pool import"); 9921 9922 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 9923 "Print vdev tree to zfs_dbgmsg during pool import"); 9924 9925 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD, 9926 "Percentage of CPUs to run an IO worker thread"); 9927 9928 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD, 9929 "Number of threads per IO worker taskqueue"); 9930 9931 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, ULONG, ZMOD_RW, 9932 "Allow importing pool with up to this number of missing top-level " 9933 "vdevs (in read-only mode)"); 9934 9935 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, ZMOD_RW, 9936 "Set the livelist condense zthr to pause"); 9937 9938 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, ZMOD_RW, 9939 "Set the livelist condense synctask to pause"); 9940 9941 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, INT, ZMOD_RW, 9942 "Whether livelist condensing was canceled in the synctask"); 9943 9944 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, INT, ZMOD_RW, 9945 "Whether livelist condensing was canceled in the zthr function"); 9946 9947 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, ZMOD_RW, 9948 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 9949 "was being condensed"); 9950 /* END CSTYLED */ 9951