1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 * Copyright 2016 Toomas Soome <tsoome@me.com> 30 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 31 * Copyright 2018 Joyent, Inc. 32 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 33 * Copyright 2017 Joyent, Inc. 34 * Copyright (c) 2017, Intel Corporation. 35 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 36 */ 37 38 /* 39 * SPA: Storage Pool Allocator 40 * 41 * This file contains all the routines used when modifying on-disk SPA state. 42 * This includes opening, importing, destroying, exporting a pool, and syncing a 43 * pool. 44 */ 45 46 #include <sys/zfs_context.h> 47 #include <sys/fm/fs/zfs.h> 48 #include <sys/spa_impl.h> 49 #include <sys/zio.h> 50 #include <sys/zio_checksum.h> 51 #include <sys/dmu.h> 52 #include <sys/dmu_tx.h> 53 #include <sys/zap.h> 54 #include <sys/zil.h> 55 #include <sys/ddt.h> 56 #include <sys/vdev_impl.h> 57 #include <sys/vdev_removal.h> 58 #include <sys/vdev_indirect_mapping.h> 59 #include <sys/vdev_indirect_births.h> 60 #include <sys/vdev_initialize.h> 61 #include <sys/vdev_rebuild.h> 62 #include <sys/vdev_trim.h> 63 #include <sys/vdev_disk.h> 64 #include <sys/vdev_draid.h> 65 #include <sys/metaslab.h> 66 #include <sys/metaslab_impl.h> 67 #include <sys/mmp.h> 68 #include <sys/uberblock_impl.h> 69 #include <sys/txg.h> 70 #include <sys/avl.h> 71 #include <sys/bpobj.h> 72 #include <sys/dmu_traverse.h> 73 #include <sys/dmu_objset.h> 74 #include <sys/unique.h> 75 #include <sys/dsl_pool.h> 76 #include <sys/dsl_dataset.h> 77 #include <sys/dsl_dir.h> 78 #include <sys/dsl_prop.h> 79 #include <sys/dsl_synctask.h> 80 #include <sys/fs/zfs.h> 81 #include <sys/arc.h> 82 #include <sys/callb.h> 83 #include <sys/systeminfo.h> 84 #include <sys/spa_boot.h> 85 #include <sys/zfs_ioctl.h> 86 #include <sys/dsl_scan.h> 87 #include <sys/zfeature.h> 88 #include <sys/dsl_destroy.h> 89 #include <sys/zvol.h> 90 91 #ifdef _KERNEL 92 #include <sys/fm/protocol.h> 93 #include <sys/fm/util.h> 94 #include <sys/callb.h> 95 #include <sys/zone.h> 96 #include <sys/vmsystm.h> 97 #endif /* _KERNEL */ 98 99 #include "zfs_prop.h" 100 #include "zfs_comutil.h" 101 102 /* 103 * The interval, in seconds, at which failed configuration cache file writes 104 * should be retried. 105 */ 106 int zfs_ccw_retry_interval = 300; 107 108 typedef enum zti_modes { 109 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 110 ZTI_MODE_BATCH, /* cpu-intensive; value is ignored */ 111 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 112 ZTI_MODE_NULL, /* don't create a taskq */ 113 ZTI_NMODES 114 } zti_modes_t; 115 116 #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 117 #define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 118 #define ZTI_BATCH { ZTI_MODE_BATCH, 0, 1 } 119 #define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 } 120 #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 121 122 #define ZTI_N(n) ZTI_P(n, 1) 123 #define ZTI_ONE ZTI_N(1) 124 125 typedef struct zio_taskq_info { 126 zti_modes_t zti_mode; 127 uint_t zti_value; 128 uint_t zti_count; 129 } zio_taskq_info_t; 130 131 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 132 "iss", "iss_h", "int", "int_h" 133 }; 134 135 /* 136 * This table defines the taskq settings for each ZFS I/O type. When 137 * initializing a pool, we use this table to create an appropriately sized 138 * taskq. Some operations are low volume and therefore have a small, static 139 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 140 * macros. Other operations process a large amount of data; the ZTI_BATCH 141 * macro causes us to create a taskq oriented for throughput. Some operations 142 * are so high frequency and short-lived that the taskq itself can become a 143 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 144 * additional degree of parallelism specified by the number of threads per- 145 * taskq and the number of taskqs; when dispatching an event in this case, the 146 * particular taskq is chosen at random. ZTI_SCALE is similar to ZTI_BATCH, 147 * but with number of taskqs also scaling with number of CPUs. 148 * 149 * The different taskq priorities are to handle the different contexts (issue 150 * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that 151 * need to be handled with minimum delay. 152 */ 153 const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 154 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 155 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 156 { ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */ 157 { ZTI_BATCH, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */ 158 { ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 159 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 160 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */ 161 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 162 }; 163 164 static void spa_sync_version(void *arg, dmu_tx_t *tx); 165 static void spa_sync_props(void *arg, dmu_tx_t *tx); 166 static boolean_t spa_has_active_shared_spare(spa_t *spa); 167 static int spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport); 168 static void spa_vdev_resilver_done(spa_t *spa); 169 170 uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 171 uint_t zio_taskq_batch_tpq; /* threads per taskq */ 172 boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 173 uint_t zio_taskq_basedc = 80; /* base duty cycle */ 174 175 boolean_t spa_create_process = B_TRUE; /* no process ==> no sysdc */ 176 177 /* 178 * Report any spa_load_verify errors found, but do not fail spa_load. 179 * This is used by zdb to analyze non-idle pools. 180 */ 181 boolean_t spa_load_verify_dryrun = B_FALSE; 182 183 /* 184 * This (illegal) pool name is used when temporarily importing a spa_t in order 185 * to get the vdev stats associated with the imported devices. 186 */ 187 #define TRYIMPORT_NAME "$import" 188 189 /* 190 * For debugging purposes: print out vdev tree during pool import. 191 */ 192 int spa_load_print_vdev_tree = B_FALSE; 193 194 /* 195 * A non-zero value for zfs_max_missing_tvds means that we allow importing 196 * pools with missing top-level vdevs. This is strictly intended for advanced 197 * pool recovery cases since missing data is almost inevitable. Pools with 198 * missing devices can only be imported read-only for safety reasons, and their 199 * fail-mode will be automatically set to "continue". 200 * 201 * With 1 missing vdev we should be able to import the pool and mount all 202 * datasets. User data that was not modified after the missing device has been 203 * added should be recoverable. This means that snapshots created prior to the 204 * addition of that device should be completely intact. 205 * 206 * With 2 missing vdevs, some datasets may fail to mount since there are 207 * dataset statistics that are stored as regular metadata. Some data might be 208 * recoverable if those vdevs were added recently. 209 * 210 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 211 * may be missing entirely. Chances of data recovery are very low. Note that 212 * there are also risks of performing an inadvertent rewind as we might be 213 * missing all the vdevs with the latest uberblocks. 214 */ 215 unsigned long zfs_max_missing_tvds = 0; 216 217 /* 218 * The parameters below are similar to zfs_max_missing_tvds but are only 219 * intended for a preliminary open of the pool with an untrusted config which 220 * might be incomplete or out-dated. 221 * 222 * We are more tolerant for pools opened from a cachefile since we could have 223 * an out-dated cachefile where a device removal was not registered. 224 * We could have set the limit arbitrarily high but in the case where devices 225 * are really missing we would want to return the proper error codes; we chose 226 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 227 * and we get a chance to retrieve the trusted config. 228 */ 229 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 230 231 /* 232 * In the case where config was assembled by scanning device paths (/dev/dsks 233 * by default) we are less tolerant since all the existing devices should have 234 * been detected and we want spa_load to return the right error codes. 235 */ 236 uint64_t zfs_max_missing_tvds_scan = 0; 237 238 /* 239 * Debugging aid that pauses spa_sync() towards the end. 240 */ 241 boolean_t zfs_pause_spa_sync = B_FALSE; 242 243 /* 244 * Variables to indicate the livelist condense zthr func should wait at certain 245 * points for the livelist to be removed - used to test condense/destroy races 246 */ 247 int zfs_livelist_condense_zthr_pause = 0; 248 int zfs_livelist_condense_sync_pause = 0; 249 250 /* 251 * Variables to track whether or not condense cancellation has been 252 * triggered in testing. 253 */ 254 int zfs_livelist_condense_sync_cancel = 0; 255 int zfs_livelist_condense_zthr_cancel = 0; 256 257 /* 258 * Variable to track whether or not extra ALLOC blkptrs were added to a 259 * livelist entry while it was being condensed (caused by the way we track 260 * remapped blkptrs in dbuf_remap_impl) 261 */ 262 int zfs_livelist_condense_new_alloc = 0; 263 264 /* 265 * ========================================================================== 266 * SPA properties routines 267 * ========================================================================== 268 */ 269 270 /* 271 * Add a (source=src, propname=propval) list to an nvlist. 272 */ 273 static void 274 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval, 275 uint64_t intval, zprop_source_t src) 276 { 277 const char *propname = zpool_prop_to_name(prop); 278 nvlist_t *propval; 279 280 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); 281 VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); 282 283 if (strval != NULL) 284 VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0); 285 else 286 VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, intval) == 0); 287 288 VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0); 289 nvlist_free(propval); 290 } 291 292 /* 293 * Get property values from the spa configuration. 294 */ 295 static void 296 spa_prop_get_config(spa_t *spa, nvlist_t **nvp) 297 { 298 vdev_t *rvd = spa->spa_root_vdev; 299 dsl_pool_t *pool = spa->spa_dsl_pool; 300 uint64_t size, alloc, cap, version; 301 const zprop_source_t src = ZPROP_SRC_NONE; 302 spa_config_dirent_t *dp; 303 metaslab_class_t *mc = spa_normal_class(spa); 304 305 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 306 307 if (rvd != NULL) { 308 alloc = metaslab_class_get_alloc(mc); 309 alloc += metaslab_class_get_alloc(spa_special_class(spa)); 310 alloc += metaslab_class_get_alloc(spa_dedup_class(spa)); 311 alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa)); 312 313 size = metaslab_class_get_space(mc); 314 size += metaslab_class_get_space(spa_special_class(spa)); 315 size += metaslab_class_get_space(spa_dedup_class(spa)); 316 size += metaslab_class_get_space(spa_embedded_log_class(spa)); 317 318 spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 319 spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src); 320 spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 321 spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL, 322 size - alloc, src); 323 spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL, 324 spa->spa_checkpoint_info.sci_dspace, src); 325 326 spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL, 327 metaslab_class_fragmentation(mc), src); 328 spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, 329 metaslab_class_expandable_space(mc), src); 330 spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL, 331 (spa_mode(spa) == SPA_MODE_READ), src); 332 333 cap = (size == 0) ? 0 : (alloc * 100 / size); 334 spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src); 335 336 spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL, 337 ddt_get_pool_dedup_ratio(spa), src); 338 339 spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL, 340 rvd->vdev_state, src); 341 342 version = spa_version(spa); 343 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 344 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 345 version, ZPROP_SRC_DEFAULT); 346 } else { 347 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 348 version, ZPROP_SRC_LOCAL); 349 } 350 spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID, 351 NULL, spa_load_guid(spa), src); 352 } 353 354 if (pool != NULL) { 355 /* 356 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 357 * when opening pools before this version freedir will be NULL. 358 */ 359 if (pool->dp_free_dir != NULL) { 360 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, 361 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 362 src); 363 } else { 364 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, 365 NULL, 0, src); 366 } 367 368 if (pool->dp_leak_dir != NULL) { 369 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, 370 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 371 src); 372 } else { 373 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, 374 NULL, 0, src); 375 } 376 } 377 378 spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 379 380 if (spa->spa_comment != NULL) { 381 spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment, 382 0, ZPROP_SRC_LOCAL); 383 } 384 385 if (spa->spa_compatibility != NULL) { 386 spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY, 387 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 388 } 389 390 if (spa->spa_root != NULL) 391 spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root, 392 0, ZPROP_SRC_LOCAL); 393 394 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 395 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 396 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 397 } else { 398 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 399 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 400 } 401 402 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 403 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 404 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 405 } else { 406 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 407 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 408 } 409 410 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 411 if (dp->scd_path == NULL) { 412 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 413 "none", 0, ZPROP_SRC_LOCAL); 414 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 415 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 416 dp->scd_path, 0, ZPROP_SRC_LOCAL); 417 } 418 } 419 } 420 421 /* 422 * Get zpool property values. 423 */ 424 int 425 spa_prop_get(spa_t *spa, nvlist_t **nvp) 426 { 427 objset_t *mos = spa->spa_meta_objset; 428 zap_cursor_t zc; 429 zap_attribute_t za; 430 dsl_pool_t *dp; 431 int err; 432 433 err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); 434 if (err) 435 return (err); 436 437 dp = spa_get_dsl(spa); 438 dsl_pool_config_enter(dp, FTAG); 439 mutex_enter(&spa->spa_props_lock); 440 441 /* 442 * Get properties from the spa config. 443 */ 444 spa_prop_get_config(spa, nvp); 445 446 /* If no pool property object, no more prop to get. */ 447 if (mos == NULL || spa->spa_pool_props_object == 0) 448 goto out; 449 450 /* 451 * Get properties from the MOS pool property object. 452 */ 453 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 454 (err = zap_cursor_retrieve(&zc, &za)) == 0; 455 zap_cursor_advance(&zc)) { 456 uint64_t intval = 0; 457 char *strval = NULL; 458 zprop_source_t src = ZPROP_SRC_DEFAULT; 459 zpool_prop_t prop; 460 461 if ((prop = zpool_name_to_prop(za.za_name)) == ZPOOL_PROP_INVAL) 462 continue; 463 464 switch (za.za_integer_length) { 465 case 8: 466 /* integer property */ 467 if (za.za_first_integer != 468 zpool_prop_default_numeric(prop)) 469 src = ZPROP_SRC_LOCAL; 470 471 if (prop == ZPOOL_PROP_BOOTFS) { 472 dsl_dataset_t *ds = NULL; 473 474 err = dsl_dataset_hold_obj(dp, 475 za.za_first_integer, FTAG, &ds); 476 if (err != 0) 477 break; 478 479 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 480 KM_SLEEP); 481 dsl_dataset_name(ds, strval); 482 dsl_dataset_rele(ds, FTAG); 483 } else { 484 strval = NULL; 485 intval = za.za_first_integer; 486 } 487 488 spa_prop_add_list(*nvp, prop, strval, intval, src); 489 490 if (strval != NULL) 491 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 492 493 break; 494 495 case 1: 496 /* string property */ 497 strval = kmem_alloc(za.za_num_integers, KM_SLEEP); 498 err = zap_lookup(mos, spa->spa_pool_props_object, 499 za.za_name, 1, za.za_num_integers, strval); 500 if (err) { 501 kmem_free(strval, za.za_num_integers); 502 break; 503 } 504 spa_prop_add_list(*nvp, prop, strval, 0, src); 505 kmem_free(strval, za.za_num_integers); 506 break; 507 508 default: 509 break; 510 } 511 } 512 zap_cursor_fini(&zc); 513 out: 514 mutex_exit(&spa->spa_props_lock); 515 dsl_pool_config_exit(dp, FTAG); 516 if (err && err != ENOENT) { 517 nvlist_free(*nvp); 518 *nvp = NULL; 519 return (err); 520 } 521 522 return (0); 523 } 524 525 /* 526 * Validate the given pool properties nvlist and modify the list 527 * for the property values to be set. 528 */ 529 static int 530 spa_prop_validate(spa_t *spa, nvlist_t *props) 531 { 532 nvpair_t *elem; 533 int error = 0, reset_bootfs = 0; 534 uint64_t objnum = 0; 535 boolean_t has_feature = B_FALSE; 536 537 elem = NULL; 538 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 539 uint64_t intval; 540 char *strval, *slash, *check, *fname; 541 const char *propname = nvpair_name(elem); 542 zpool_prop_t prop = zpool_name_to_prop(propname); 543 544 switch (prop) { 545 case ZPOOL_PROP_INVAL: 546 if (!zpool_prop_feature(propname)) { 547 error = SET_ERROR(EINVAL); 548 break; 549 } 550 551 /* 552 * Sanitize the input. 553 */ 554 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 555 error = SET_ERROR(EINVAL); 556 break; 557 } 558 559 if (nvpair_value_uint64(elem, &intval) != 0) { 560 error = SET_ERROR(EINVAL); 561 break; 562 } 563 564 if (intval != 0) { 565 error = SET_ERROR(EINVAL); 566 break; 567 } 568 569 fname = strchr(propname, '@') + 1; 570 if (zfeature_lookup_name(fname, NULL) != 0) { 571 error = SET_ERROR(EINVAL); 572 break; 573 } 574 575 has_feature = B_TRUE; 576 break; 577 578 case ZPOOL_PROP_VERSION: 579 error = nvpair_value_uint64(elem, &intval); 580 if (!error && 581 (intval < spa_version(spa) || 582 intval > SPA_VERSION_BEFORE_FEATURES || 583 has_feature)) 584 error = SET_ERROR(EINVAL); 585 break; 586 587 case ZPOOL_PROP_DELEGATION: 588 case ZPOOL_PROP_AUTOREPLACE: 589 case ZPOOL_PROP_LISTSNAPS: 590 case ZPOOL_PROP_AUTOEXPAND: 591 case ZPOOL_PROP_AUTOTRIM: 592 error = nvpair_value_uint64(elem, &intval); 593 if (!error && intval > 1) 594 error = SET_ERROR(EINVAL); 595 break; 596 597 case ZPOOL_PROP_MULTIHOST: 598 error = nvpair_value_uint64(elem, &intval); 599 if (!error && intval > 1) 600 error = SET_ERROR(EINVAL); 601 602 if (!error) { 603 uint32_t hostid = zone_get_hostid(NULL); 604 if (hostid) 605 spa->spa_hostid = hostid; 606 else 607 error = SET_ERROR(ENOTSUP); 608 } 609 610 break; 611 612 case ZPOOL_PROP_BOOTFS: 613 /* 614 * If the pool version is less than SPA_VERSION_BOOTFS, 615 * or the pool is still being created (version == 0), 616 * the bootfs property cannot be set. 617 */ 618 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 619 error = SET_ERROR(ENOTSUP); 620 break; 621 } 622 623 /* 624 * Make sure the vdev config is bootable 625 */ 626 if (!vdev_is_bootable(spa->spa_root_vdev)) { 627 error = SET_ERROR(ENOTSUP); 628 break; 629 } 630 631 reset_bootfs = 1; 632 633 error = nvpair_value_string(elem, &strval); 634 635 if (!error) { 636 objset_t *os; 637 638 if (strval == NULL || strval[0] == '\0') { 639 objnum = zpool_prop_default_numeric( 640 ZPOOL_PROP_BOOTFS); 641 break; 642 } 643 644 error = dmu_objset_hold(strval, FTAG, &os); 645 if (error != 0) 646 break; 647 648 /* Must be ZPL. */ 649 if (dmu_objset_type(os) != DMU_OST_ZFS) { 650 error = SET_ERROR(ENOTSUP); 651 } else { 652 objnum = dmu_objset_id(os); 653 } 654 dmu_objset_rele(os, FTAG); 655 } 656 break; 657 658 case ZPOOL_PROP_FAILUREMODE: 659 error = nvpair_value_uint64(elem, &intval); 660 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 661 error = SET_ERROR(EINVAL); 662 663 /* 664 * This is a special case which only occurs when 665 * the pool has completely failed. This allows 666 * the user to change the in-core failmode property 667 * without syncing it out to disk (I/Os might 668 * currently be blocked). We do this by returning 669 * EIO to the caller (spa_prop_set) to trick it 670 * into thinking we encountered a property validation 671 * error. 672 */ 673 if (!error && spa_suspended(spa)) { 674 spa->spa_failmode = intval; 675 error = SET_ERROR(EIO); 676 } 677 break; 678 679 case ZPOOL_PROP_CACHEFILE: 680 if ((error = nvpair_value_string(elem, &strval)) != 0) 681 break; 682 683 if (strval[0] == '\0') 684 break; 685 686 if (strcmp(strval, "none") == 0) 687 break; 688 689 if (strval[0] != '/') { 690 error = SET_ERROR(EINVAL); 691 break; 692 } 693 694 slash = strrchr(strval, '/'); 695 ASSERT(slash != NULL); 696 697 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 698 strcmp(slash, "/..") == 0) 699 error = SET_ERROR(EINVAL); 700 break; 701 702 case ZPOOL_PROP_COMMENT: 703 if ((error = nvpair_value_string(elem, &strval)) != 0) 704 break; 705 for (check = strval; *check != '\0'; check++) { 706 if (!isprint(*check)) { 707 error = SET_ERROR(EINVAL); 708 break; 709 } 710 } 711 if (strlen(strval) > ZPROP_MAX_COMMENT) 712 error = SET_ERROR(E2BIG); 713 break; 714 715 default: 716 break; 717 } 718 719 if (error) 720 break; 721 } 722 723 (void) nvlist_remove_all(props, 724 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 725 726 if (!error && reset_bootfs) { 727 error = nvlist_remove(props, 728 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 729 730 if (!error) { 731 error = nvlist_add_uint64(props, 732 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 733 } 734 } 735 736 return (error); 737 } 738 739 void 740 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 741 { 742 char *cachefile; 743 spa_config_dirent_t *dp; 744 745 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 746 &cachefile) != 0) 747 return; 748 749 dp = kmem_alloc(sizeof (spa_config_dirent_t), 750 KM_SLEEP); 751 752 if (cachefile[0] == '\0') 753 dp->scd_path = spa_strdup(spa_config_path); 754 else if (strcmp(cachefile, "none") == 0) 755 dp->scd_path = NULL; 756 else 757 dp->scd_path = spa_strdup(cachefile); 758 759 list_insert_head(&spa->spa_config_list, dp); 760 if (need_sync) 761 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 762 } 763 764 int 765 spa_prop_set(spa_t *spa, nvlist_t *nvp) 766 { 767 int error; 768 nvpair_t *elem = NULL; 769 boolean_t need_sync = B_FALSE; 770 771 if ((error = spa_prop_validate(spa, nvp)) != 0) 772 return (error); 773 774 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 775 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 776 777 if (prop == ZPOOL_PROP_CACHEFILE || 778 prop == ZPOOL_PROP_ALTROOT || 779 prop == ZPOOL_PROP_READONLY) 780 continue; 781 782 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 783 uint64_t ver; 784 785 if (prop == ZPOOL_PROP_VERSION) { 786 VERIFY(nvpair_value_uint64(elem, &ver) == 0); 787 } else { 788 ASSERT(zpool_prop_feature(nvpair_name(elem))); 789 ver = SPA_VERSION_FEATURES; 790 need_sync = B_TRUE; 791 } 792 793 /* Save time if the version is already set. */ 794 if (ver == spa_version(spa)) 795 continue; 796 797 /* 798 * In addition to the pool directory object, we might 799 * create the pool properties object, the features for 800 * read object, the features for write object, or the 801 * feature descriptions object. 802 */ 803 error = dsl_sync_task(spa->spa_name, NULL, 804 spa_sync_version, &ver, 805 6, ZFS_SPACE_CHECK_RESERVED); 806 if (error) 807 return (error); 808 continue; 809 } 810 811 need_sync = B_TRUE; 812 break; 813 } 814 815 if (need_sync) { 816 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 817 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 818 } 819 820 return (0); 821 } 822 823 /* 824 * If the bootfs property value is dsobj, clear it. 825 */ 826 void 827 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 828 { 829 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 830 VERIFY(zap_remove(spa->spa_meta_objset, 831 spa->spa_pool_props_object, 832 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 833 spa->spa_bootfs = 0; 834 } 835 } 836 837 /*ARGSUSED*/ 838 static int 839 spa_change_guid_check(void *arg, dmu_tx_t *tx) 840 { 841 uint64_t *newguid __maybe_unused = arg; 842 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 843 vdev_t *rvd = spa->spa_root_vdev; 844 uint64_t vdev_state; 845 846 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 847 int error = (spa_has_checkpoint(spa)) ? 848 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 849 return (SET_ERROR(error)); 850 } 851 852 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 853 vdev_state = rvd->vdev_state; 854 spa_config_exit(spa, SCL_STATE, FTAG); 855 856 if (vdev_state != VDEV_STATE_HEALTHY) 857 return (SET_ERROR(ENXIO)); 858 859 ASSERT3U(spa_guid(spa), !=, *newguid); 860 861 return (0); 862 } 863 864 static void 865 spa_change_guid_sync(void *arg, dmu_tx_t *tx) 866 { 867 uint64_t *newguid = arg; 868 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 869 uint64_t oldguid; 870 vdev_t *rvd = spa->spa_root_vdev; 871 872 oldguid = spa_guid(spa); 873 874 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 875 rvd->vdev_guid = *newguid; 876 rvd->vdev_guid_sum += (*newguid - oldguid); 877 vdev_config_dirty(rvd); 878 spa_config_exit(spa, SCL_STATE, FTAG); 879 880 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 881 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 882 } 883 884 /* 885 * Change the GUID for the pool. This is done so that we can later 886 * re-import a pool built from a clone of our own vdevs. We will modify 887 * the root vdev's guid, our own pool guid, and then mark all of our 888 * vdevs dirty. Note that we must make sure that all our vdevs are 889 * online when we do this, or else any vdevs that weren't present 890 * would be orphaned from our pool. We are also going to issue a 891 * sysevent to update any watchers. 892 */ 893 int 894 spa_change_guid(spa_t *spa) 895 { 896 int error; 897 uint64_t guid; 898 899 mutex_enter(&spa->spa_vdev_top_lock); 900 mutex_enter(&spa_namespace_lock); 901 guid = spa_generate_guid(NULL); 902 903 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 904 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 905 906 if (error == 0) { 907 spa_write_cachefile(spa, B_FALSE, B_TRUE); 908 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 909 } 910 911 mutex_exit(&spa_namespace_lock); 912 mutex_exit(&spa->spa_vdev_top_lock); 913 914 return (error); 915 } 916 917 /* 918 * ========================================================================== 919 * SPA state manipulation (open/create/destroy/import/export) 920 * ========================================================================== 921 */ 922 923 static int 924 spa_error_entry_compare(const void *a, const void *b) 925 { 926 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 927 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 928 int ret; 929 930 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 931 sizeof (zbookmark_phys_t)); 932 933 return (TREE_ISIGN(ret)); 934 } 935 936 /* 937 * Utility function which retrieves copies of the current logs and 938 * re-initializes them in the process. 939 */ 940 void 941 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 942 { 943 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 944 945 bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t)); 946 bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t)); 947 948 avl_create(&spa->spa_errlist_scrub, 949 spa_error_entry_compare, sizeof (spa_error_entry_t), 950 offsetof(spa_error_entry_t, se_avl)); 951 avl_create(&spa->spa_errlist_last, 952 spa_error_entry_compare, sizeof (spa_error_entry_t), 953 offsetof(spa_error_entry_t, se_avl)); 954 } 955 956 static void 957 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 958 { 959 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 960 enum zti_modes mode = ztip->zti_mode; 961 uint_t value = ztip->zti_value; 962 uint_t count = ztip->zti_count; 963 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 964 uint_t cpus, flags = TASKQ_DYNAMIC; 965 boolean_t batch = B_FALSE; 966 967 switch (mode) { 968 case ZTI_MODE_FIXED: 969 ASSERT3U(value, >, 0); 970 break; 971 972 case ZTI_MODE_BATCH: 973 batch = B_TRUE; 974 flags |= TASKQ_THREADS_CPU_PCT; 975 value = MIN(zio_taskq_batch_pct, 100); 976 break; 977 978 case ZTI_MODE_SCALE: 979 flags |= TASKQ_THREADS_CPU_PCT; 980 /* 981 * We want more taskqs to reduce lock contention, but we want 982 * less for better request ordering and CPU utilization. 983 */ 984 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 985 if (zio_taskq_batch_tpq > 0) { 986 count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) / 987 zio_taskq_batch_tpq); 988 } else { 989 /* 990 * Prefer 6 threads per taskq, but no more taskqs 991 * than threads in them on large systems. For 80%: 992 * 993 * taskq taskq total 994 * cpus taskqs percent threads threads 995 * ------- ------- ------- ------- ------- 996 * 1 1 80% 1 1 997 * 2 1 80% 1 1 998 * 4 1 80% 3 3 999 * 8 2 40% 3 6 1000 * 16 3 27% 4 12 1001 * 32 5 16% 5 25 1002 * 64 7 11% 7 49 1003 * 128 10 8% 10 100 1004 * 256 14 6% 15 210 1005 */ 1006 count = 1 + cpus / 6; 1007 while (count * count > cpus) 1008 count--; 1009 } 1010 /* Limit each taskq within 100% to not trigger assertion. */ 1011 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1012 value = (zio_taskq_batch_pct + count / 2) / count; 1013 break; 1014 1015 case ZTI_MODE_NULL: 1016 tqs->stqs_count = 0; 1017 tqs->stqs_taskq = NULL; 1018 return; 1019 1020 default: 1021 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1022 "spa_activate()", 1023 zio_type_name[t], zio_taskq_types[q], mode, value); 1024 break; 1025 } 1026 1027 ASSERT3U(count, >, 0); 1028 tqs->stqs_count = count; 1029 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1030 1031 for (uint_t i = 0; i < count; i++) { 1032 taskq_t *tq; 1033 char name[32]; 1034 1035 if (count > 1) 1036 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1037 zio_type_name[t], zio_taskq_types[q], i); 1038 else 1039 (void) snprintf(name, sizeof (name), "%s_%s", 1040 zio_type_name[t], zio_taskq_types[q]); 1041 1042 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1043 if (batch) 1044 flags |= TASKQ_DC_BATCH; 1045 1046 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1047 spa->spa_proc, zio_taskq_basedc, flags); 1048 } else { 1049 pri_t pri = maxclsyspri; 1050 /* 1051 * The write issue taskq can be extremely CPU 1052 * intensive. Run it at slightly less important 1053 * priority than the other taskqs. 1054 * 1055 * Under Linux and FreeBSD this means incrementing 1056 * the priority value as opposed to platforms like 1057 * illumos where it should be decremented. 1058 * 1059 * On FreeBSD, if priorities divided by four (RQ_PPQ) 1060 * are equal then a difference between them is 1061 * insignificant. 1062 */ 1063 if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) { 1064 #if defined(__linux__) 1065 pri++; 1066 #elif defined(__FreeBSD__) 1067 pri += 4; 1068 #else 1069 #error "unknown OS" 1070 #endif 1071 } 1072 tq = taskq_create_proc(name, value, pri, 50, 1073 INT_MAX, spa->spa_proc, flags); 1074 } 1075 1076 tqs->stqs_taskq[i] = tq; 1077 } 1078 } 1079 1080 static void 1081 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1082 { 1083 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1084 1085 if (tqs->stqs_taskq == NULL) { 1086 ASSERT3U(tqs->stqs_count, ==, 0); 1087 return; 1088 } 1089 1090 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1091 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1092 taskq_destroy(tqs->stqs_taskq[i]); 1093 } 1094 1095 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1096 tqs->stqs_taskq = NULL; 1097 } 1098 1099 /* 1100 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1101 * Note that a type may have multiple discrete taskqs to avoid lock contention 1102 * on the taskq itself. In that case we choose which taskq at random by using 1103 * the low bits of gethrtime(). 1104 */ 1105 void 1106 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1107 task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent) 1108 { 1109 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1110 taskq_t *tq; 1111 1112 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1113 ASSERT3U(tqs->stqs_count, !=, 0); 1114 1115 if (tqs->stqs_count == 1) { 1116 tq = tqs->stqs_taskq[0]; 1117 } else { 1118 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1119 } 1120 1121 taskq_dispatch_ent(tq, func, arg, flags, ent); 1122 } 1123 1124 /* 1125 * Same as spa_taskq_dispatch_ent() but block on the task until completion. 1126 */ 1127 void 1128 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1129 task_func_t *func, void *arg, uint_t flags) 1130 { 1131 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1132 taskq_t *tq; 1133 taskqid_t id; 1134 1135 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1136 ASSERT3U(tqs->stqs_count, !=, 0); 1137 1138 if (tqs->stqs_count == 1) { 1139 tq = tqs->stqs_taskq[0]; 1140 } else { 1141 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1142 } 1143 1144 id = taskq_dispatch(tq, func, arg, flags); 1145 if (id) 1146 taskq_wait_id(tq, id); 1147 } 1148 1149 static void 1150 spa_create_zio_taskqs(spa_t *spa) 1151 { 1152 for (int t = 0; t < ZIO_TYPES; t++) { 1153 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1154 spa_taskqs_init(spa, t, q); 1155 } 1156 } 1157 } 1158 1159 /* 1160 * Disabled until spa_thread() can be adapted for Linux. 1161 */ 1162 #undef HAVE_SPA_THREAD 1163 1164 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1165 static void 1166 spa_thread(void *arg) 1167 { 1168 psetid_t zio_taskq_psrset_bind = PS_NONE; 1169 callb_cpr_t cprinfo; 1170 1171 spa_t *spa = arg; 1172 user_t *pu = PTOU(curproc); 1173 1174 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1175 spa->spa_name); 1176 1177 ASSERT(curproc != &p0); 1178 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1179 "zpool-%s", spa->spa_name); 1180 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1181 1182 /* bind this thread to the requested psrset */ 1183 if (zio_taskq_psrset_bind != PS_NONE) { 1184 pool_lock(); 1185 mutex_enter(&cpu_lock); 1186 mutex_enter(&pidlock); 1187 mutex_enter(&curproc->p_lock); 1188 1189 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1190 0, NULL, NULL) == 0) { 1191 curthread->t_bind_pset = zio_taskq_psrset_bind; 1192 } else { 1193 cmn_err(CE_WARN, 1194 "Couldn't bind process for zfs pool \"%s\" to " 1195 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1196 } 1197 1198 mutex_exit(&curproc->p_lock); 1199 mutex_exit(&pidlock); 1200 mutex_exit(&cpu_lock); 1201 pool_unlock(); 1202 } 1203 1204 if (zio_taskq_sysdc) { 1205 sysdc_thread_enter(curthread, 100, 0); 1206 } 1207 1208 spa->spa_proc = curproc; 1209 spa->spa_did = curthread->t_did; 1210 1211 spa_create_zio_taskqs(spa); 1212 1213 mutex_enter(&spa->spa_proc_lock); 1214 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1215 1216 spa->spa_proc_state = SPA_PROC_ACTIVE; 1217 cv_broadcast(&spa->spa_proc_cv); 1218 1219 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1220 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1221 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1222 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1223 1224 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1225 spa->spa_proc_state = SPA_PROC_GONE; 1226 spa->spa_proc = &p0; 1227 cv_broadcast(&spa->spa_proc_cv); 1228 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1229 1230 mutex_enter(&curproc->p_lock); 1231 lwp_exit(); 1232 } 1233 #endif 1234 1235 /* 1236 * Activate an uninitialized pool. 1237 */ 1238 static void 1239 spa_activate(spa_t *spa, spa_mode_t mode) 1240 { 1241 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1242 1243 spa->spa_state = POOL_STATE_ACTIVE; 1244 spa->spa_mode = mode; 1245 1246 spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops); 1247 spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops); 1248 spa->spa_embedded_log_class = 1249 metaslab_class_create(spa, zfs_metaslab_ops); 1250 spa->spa_special_class = metaslab_class_create(spa, zfs_metaslab_ops); 1251 spa->spa_dedup_class = metaslab_class_create(spa, zfs_metaslab_ops); 1252 1253 /* Try to create a covering process */ 1254 mutex_enter(&spa->spa_proc_lock); 1255 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1256 ASSERT(spa->spa_proc == &p0); 1257 spa->spa_did = 0; 1258 1259 #ifdef HAVE_SPA_THREAD 1260 /* Only create a process if we're going to be around a while. */ 1261 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1262 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1263 NULL, 0) == 0) { 1264 spa->spa_proc_state = SPA_PROC_CREATED; 1265 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1266 cv_wait(&spa->spa_proc_cv, 1267 &spa->spa_proc_lock); 1268 } 1269 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1270 ASSERT(spa->spa_proc != &p0); 1271 ASSERT(spa->spa_did != 0); 1272 } else { 1273 #ifdef _KERNEL 1274 cmn_err(CE_WARN, 1275 "Couldn't create process for zfs pool \"%s\"\n", 1276 spa->spa_name); 1277 #endif 1278 } 1279 } 1280 #endif /* HAVE_SPA_THREAD */ 1281 mutex_exit(&spa->spa_proc_lock); 1282 1283 /* If we didn't create a process, we need to create our taskqs. */ 1284 if (spa->spa_proc == &p0) { 1285 spa_create_zio_taskqs(spa); 1286 } 1287 1288 for (size_t i = 0; i < TXG_SIZE; i++) { 1289 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1290 ZIO_FLAG_CANFAIL); 1291 } 1292 1293 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1294 offsetof(vdev_t, vdev_config_dirty_node)); 1295 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1296 offsetof(objset_t, os_evicting_node)); 1297 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1298 offsetof(vdev_t, vdev_state_dirty_node)); 1299 1300 txg_list_create(&spa->spa_vdev_txg_list, spa, 1301 offsetof(struct vdev, vdev_txg_node)); 1302 1303 avl_create(&spa->spa_errlist_scrub, 1304 spa_error_entry_compare, sizeof (spa_error_entry_t), 1305 offsetof(spa_error_entry_t, se_avl)); 1306 avl_create(&spa->spa_errlist_last, 1307 spa_error_entry_compare, sizeof (spa_error_entry_t), 1308 offsetof(spa_error_entry_t, se_avl)); 1309 1310 spa_keystore_init(&spa->spa_keystore); 1311 1312 /* 1313 * This taskq is used to perform zvol-minor-related tasks 1314 * asynchronously. This has several advantages, including easy 1315 * resolution of various deadlocks. 1316 * 1317 * The taskq must be single threaded to ensure tasks are always 1318 * processed in the order in which they were dispatched. 1319 * 1320 * A taskq per pool allows one to keep the pools independent. 1321 * This way if one pool is suspended, it will not impact another. 1322 * 1323 * The preferred location to dispatch a zvol minor task is a sync 1324 * task. In this context, there is easy access to the spa_t and minimal 1325 * error handling is required because the sync task must succeed. 1326 */ 1327 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1328 1, INT_MAX, 0); 1329 1330 /* 1331 * Taskq dedicated to prefetcher threads: this is used to prevent the 1332 * pool traverse code from monopolizing the global (and limited) 1333 * system_taskq by inappropriately scheduling long running tasks on it. 1334 */ 1335 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1336 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1337 1338 /* 1339 * The taskq to upgrade datasets in this pool. Currently used by 1340 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1341 */ 1342 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1343 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1344 } 1345 1346 /* 1347 * Opposite of spa_activate(). 1348 */ 1349 static void 1350 spa_deactivate(spa_t *spa) 1351 { 1352 ASSERT(spa->spa_sync_on == B_FALSE); 1353 ASSERT(spa->spa_dsl_pool == NULL); 1354 ASSERT(spa->spa_root_vdev == NULL); 1355 ASSERT(spa->spa_async_zio_root == NULL); 1356 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1357 1358 spa_evicting_os_wait(spa); 1359 1360 if (spa->spa_zvol_taskq) { 1361 taskq_destroy(spa->spa_zvol_taskq); 1362 spa->spa_zvol_taskq = NULL; 1363 } 1364 1365 if (spa->spa_prefetch_taskq) { 1366 taskq_destroy(spa->spa_prefetch_taskq); 1367 spa->spa_prefetch_taskq = NULL; 1368 } 1369 1370 if (spa->spa_upgrade_taskq) { 1371 taskq_destroy(spa->spa_upgrade_taskq); 1372 spa->spa_upgrade_taskq = NULL; 1373 } 1374 1375 txg_list_destroy(&spa->spa_vdev_txg_list); 1376 1377 list_destroy(&spa->spa_config_dirty_list); 1378 list_destroy(&spa->spa_evicting_os_list); 1379 list_destroy(&spa->spa_state_dirty_list); 1380 1381 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 1382 1383 for (int t = 0; t < ZIO_TYPES; t++) { 1384 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1385 spa_taskqs_fini(spa, t, q); 1386 } 1387 } 1388 1389 for (size_t i = 0; i < TXG_SIZE; i++) { 1390 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1391 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1392 spa->spa_txg_zio[i] = NULL; 1393 } 1394 1395 metaslab_class_destroy(spa->spa_normal_class); 1396 spa->spa_normal_class = NULL; 1397 1398 metaslab_class_destroy(spa->spa_log_class); 1399 spa->spa_log_class = NULL; 1400 1401 metaslab_class_destroy(spa->spa_embedded_log_class); 1402 spa->spa_embedded_log_class = NULL; 1403 1404 metaslab_class_destroy(spa->spa_special_class); 1405 spa->spa_special_class = NULL; 1406 1407 metaslab_class_destroy(spa->spa_dedup_class); 1408 spa->spa_dedup_class = NULL; 1409 1410 /* 1411 * If this was part of an import or the open otherwise failed, we may 1412 * still have errors left in the queues. Empty them just in case. 1413 */ 1414 spa_errlog_drain(spa); 1415 avl_destroy(&spa->spa_errlist_scrub); 1416 avl_destroy(&spa->spa_errlist_last); 1417 1418 spa_keystore_fini(&spa->spa_keystore); 1419 1420 spa->spa_state = POOL_STATE_UNINITIALIZED; 1421 1422 mutex_enter(&spa->spa_proc_lock); 1423 if (spa->spa_proc_state != SPA_PROC_NONE) { 1424 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1425 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 1426 cv_broadcast(&spa->spa_proc_cv); 1427 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 1428 ASSERT(spa->spa_proc != &p0); 1429 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1430 } 1431 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 1432 spa->spa_proc_state = SPA_PROC_NONE; 1433 } 1434 ASSERT(spa->spa_proc == &p0); 1435 mutex_exit(&spa->spa_proc_lock); 1436 1437 /* 1438 * We want to make sure spa_thread() has actually exited the ZFS 1439 * module, so that the module can't be unloaded out from underneath 1440 * it. 1441 */ 1442 if (spa->spa_did != 0) { 1443 thread_join(spa->spa_did); 1444 spa->spa_did = 0; 1445 } 1446 } 1447 1448 /* 1449 * Verify a pool configuration, and construct the vdev tree appropriately. This 1450 * will create all the necessary vdevs in the appropriate layout, with each vdev 1451 * in the CLOSED state. This will prep the pool before open/creation/import. 1452 * All vdev validation is done by the vdev_alloc() routine. 1453 */ 1454 int 1455 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 1456 uint_t id, int atype) 1457 { 1458 nvlist_t **child; 1459 uint_t children; 1460 int error; 1461 1462 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 1463 return (error); 1464 1465 if ((*vdp)->vdev_ops->vdev_op_leaf) 1466 return (0); 1467 1468 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 1469 &child, &children); 1470 1471 if (error == ENOENT) 1472 return (0); 1473 1474 if (error) { 1475 vdev_free(*vdp); 1476 *vdp = NULL; 1477 return (SET_ERROR(EINVAL)); 1478 } 1479 1480 for (int c = 0; c < children; c++) { 1481 vdev_t *vd; 1482 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 1483 atype)) != 0) { 1484 vdev_free(*vdp); 1485 *vdp = NULL; 1486 return (error); 1487 } 1488 } 1489 1490 ASSERT(*vdp != NULL); 1491 1492 return (0); 1493 } 1494 1495 static boolean_t 1496 spa_should_flush_logs_on_unload(spa_t *spa) 1497 { 1498 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 1499 return (B_FALSE); 1500 1501 if (!spa_writeable(spa)) 1502 return (B_FALSE); 1503 1504 if (!spa->spa_sync_on) 1505 return (B_FALSE); 1506 1507 if (spa_state(spa) != POOL_STATE_EXPORTED) 1508 return (B_FALSE); 1509 1510 if (zfs_keep_log_spacemaps_at_export) 1511 return (B_FALSE); 1512 1513 return (B_TRUE); 1514 } 1515 1516 /* 1517 * Opens a transaction that will set the flag that will instruct 1518 * spa_sync to attempt to flush all the metaslabs for that txg. 1519 */ 1520 static void 1521 spa_unload_log_sm_flush_all(spa_t *spa) 1522 { 1523 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 1524 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 1525 1526 ASSERT3U(spa->spa_log_flushall_txg, ==, 0); 1527 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); 1528 1529 dmu_tx_commit(tx); 1530 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 1531 } 1532 1533 static void 1534 spa_unload_log_sm_metadata(spa_t *spa) 1535 { 1536 void *cookie = NULL; 1537 spa_log_sm_t *sls; 1538 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 1539 &cookie)) != NULL) { 1540 VERIFY0(sls->sls_mscount); 1541 kmem_free(sls, sizeof (spa_log_sm_t)); 1542 } 1543 1544 for (log_summary_entry_t *e = list_head(&spa->spa_log_summary); 1545 e != NULL; e = list_head(&spa->spa_log_summary)) { 1546 VERIFY0(e->lse_mscount); 1547 list_remove(&spa->spa_log_summary, e); 1548 kmem_free(e, sizeof (log_summary_entry_t)); 1549 } 1550 1551 spa->spa_unflushed_stats.sus_nblocks = 0; 1552 spa->spa_unflushed_stats.sus_memused = 0; 1553 spa->spa_unflushed_stats.sus_blocklimit = 0; 1554 } 1555 1556 static void 1557 spa_destroy_aux_threads(spa_t *spa) 1558 { 1559 if (spa->spa_condense_zthr != NULL) { 1560 zthr_destroy(spa->spa_condense_zthr); 1561 spa->spa_condense_zthr = NULL; 1562 } 1563 if (spa->spa_checkpoint_discard_zthr != NULL) { 1564 zthr_destroy(spa->spa_checkpoint_discard_zthr); 1565 spa->spa_checkpoint_discard_zthr = NULL; 1566 } 1567 if (spa->spa_livelist_delete_zthr != NULL) { 1568 zthr_destroy(spa->spa_livelist_delete_zthr); 1569 spa->spa_livelist_delete_zthr = NULL; 1570 } 1571 if (spa->spa_livelist_condense_zthr != NULL) { 1572 zthr_destroy(spa->spa_livelist_condense_zthr); 1573 spa->spa_livelist_condense_zthr = NULL; 1574 } 1575 } 1576 1577 /* 1578 * Opposite of spa_load(). 1579 */ 1580 static void 1581 spa_unload(spa_t *spa) 1582 { 1583 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1584 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 1585 1586 spa_import_progress_remove(spa_guid(spa)); 1587 spa_load_note(spa, "UNLOADING"); 1588 1589 spa_wake_waiters(spa); 1590 1591 /* 1592 * If the log space map feature is enabled and the pool is getting 1593 * exported (but not destroyed), we want to spend some time flushing 1594 * as many metaslabs as we can in an attempt to destroy log space 1595 * maps and save import time. 1596 */ 1597 if (spa_should_flush_logs_on_unload(spa)) 1598 spa_unload_log_sm_flush_all(spa); 1599 1600 /* 1601 * Stop async tasks. 1602 */ 1603 spa_async_suspend(spa); 1604 1605 if (spa->spa_root_vdev) { 1606 vdev_t *root_vdev = spa->spa_root_vdev; 1607 vdev_initialize_stop_all(root_vdev, VDEV_INITIALIZE_ACTIVE); 1608 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 1609 vdev_autotrim_stop_all(spa); 1610 vdev_rebuild_stop_all(spa); 1611 } 1612 1613 /* 1614 * Stop syncing. 1615 */ 1616 if (spa->spa_sync_on) { 1617 txg_sync_stop(spa->spa_dsl_pool); 1618 spa->spa_sync_on = B_FALSE; 1619 } 1620 1621 /* 1622 * This ensures that there is no async metaslab prefetching 1623 * while we attempt to unload the spa. 1624 */ 1625 if (spa->spa_root_vdev != NULL) { 1626 for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++) { 1627 vdev_t *vc = spa->spa_root_vdev->vdev_child[c]; 1628 if (vc->vdev_mg != NULL) 1629 taskq_wait(vc->vdev_mg->mg_taskq); 1630 } 1631 } 1632 1633 if (spa->spa_mmp.mmp_thread) 1634 mmp_thread_stop(spa); 1635 1636 /* 1637 * Wait for any outstanding async I/O to complete. 1638 */ 1639 if (spa->spa_async_zio_root != NULL) { 1640 for (int i = 0; i < max_ncpus; i++) 1641 (void) zio_wait(spa->spa_async_zio_root[i]); 1642 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 1643 spa->spa_async_zio_root = NULL; 1644 } 1645 1646 if (spa->spa_vdev_removal != NULL) { 1647 spa_vdev_removal_destroy(spa->spa_vdev_removal); 1648 spa->spa_vdev_removal = NULL; 1649 } 1650 1651 spa_destroy_aux_threads(spa); 1652 1653 spa_condense_fini(spa); 1654 1655 bpobj_close(&spa->spa_deferred_bpobj); 1656 1657 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 1658 1659 /* 1660 * Close all vdevs. 1661 */ 1662 if (spa->spa_root_vdev) 1663 vdev_free(spa->spa_root_vdev); 1664 ASSERT(spa->spa_root_vdev == NULL); 1665 1666 /* 1667 * Close the dsl pool. 1668 */ 1669 if (spa->spa_dsl_pool) { 1670 dsl_pool_close(spa->spa_dsl_pool); 1671 spa->spa_dsl_pool = NULL; 1672 spa->spa_meta_objset = NULL; 1673 } 1674 1675 ddt_unload(spa); 1676 spa_unload_log_sm_metadata(spa); 1677 1678 /* 1679 * Drop and purge level 2 cache 1680 */ 1681 spa_l2cache_drop(spa); 1682 1683 for (int i = 0; i < spa->spa_spares.sav_count; i++) 1684 vdev_free(spa->spa_spares.sav_vdevs[i]); 1685 if (spa->spa_spares.sav_vdevs) { 1686 kmem_free(spa->spa_spares.sav_vdevs, 1687 spa->spa_spares.sav_count * sizeof (void *)); 1688 spa->spa_spares.sav_vdevs = NULL; 1689 } 1690 if (spa->spa_spares.sav_config) { 1691 nvlist_free(spa->spa_spares.sav_config); 1692 spa->spa_spares.sav_config = NULL; 1693 } 1694 spa->spa_spares.sav_count = 0; 1695 1696 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1697 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 1698 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 1699 } 1700 if (spa->spa_l2cache.sav_vdevs) { 1701 kmem_free(spa->spa_l2cache.sav_vdevs, 1702 spa->spa_l2cache.sav_count * sizeof (void *)); 1703 spa->spa_l2cache.sav_vdevs = NULL; 1704 } 1705 if (spa->spa_l2cache.sav_config) { 1706 nvlist_free(spa->spa_l2cache.sav_config); 1707 spa->spa_l2cache.sav_config = NULL; 1708 } 1709 spa->spa_l2cache.sav_count = 0; 1710 1711 spa->spa_async_suspended = 0; 1712 1713 spa->spa_indirect_vdevs_loaded = B_FALSE; 1714 1715 if (spa->spa_comment != NULL) { 1716 spa_strfree(spa->spa_comment); 1717 spa->spa_comment = NULL; 1718 } 1719 if (spa->spa_compatibility != NULL) { 1720 spa_strfree(spa->spa_compatibility); 1721 spa->spa_compatibility = NULL; 1722 } 1723 1724 spa_config_exit(spa, SCL_ALL, spa); 1725 } 1726 1727 /* 1728 * Load (or re-load) the current list of vdevs describing the active spares for 1729 * this pool. When this is called, we have some form of basic information in 1730 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 1731 * then re-generate a more complete list including status information. 1732 */ 1733 void 1734 spa_load_spares(spa_t *spa) 1735 { 1736 nvlist_t **spares; 1737 uint_t nspares; 1738 int i; 1739 vdev_t *vd, *tvd; 1740 1741 #ifndef _KERNEL 1742 /* 1743 * zdb opens both the current state of the pool and the 1744 * checkpointed state (if present), with a different spa_t. 1745 * 1746 * As spare vdevs are shared among open pools, we skip loading 1747 * them when we load the checkpointed state of the pool. 1748 */ 1749 if (!spa_writeable(spa)) 1750 return; 1751 #endif 1752 1753 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1754 1755 /* 1756 * First, close and free any existing spare vdevs. 1757 */ 1758 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1759 vd = spa->spa_spares.sav_vdevs[i]; 1760 1761 /* Undo the call to spa_activate() below */ 1762 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1763 B_FALSE)) != NULL && tvd->vdev_isspare) 1764 spa_spare_remove(tvd); 1765 vdev_close(vd); 1766 vdev_free(vd); 1767 } 1768 1769 if (spa->spa_spares.sav_vdevs) 1770 kmem_free(spa->spa_spares.sav_vdevs, 1771 spa->spa_spares.sav_count * sizeof (void *)); 1772 1773 if (spa->spa_spares.sav_config == NULL) 1774 nspares = 0; 1775 else 1776 VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 1777 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 1778 1779 spa->spa_spares.sav_count = (int)nspares; 1780 spa->spa_spares.sav_vdevs = NULL; 1781 1782 if (nspares == 0) 1783 return; 1784 1785 /* 1786 * Construct the array of vdevs, opening them to get status in the 1787 * process. For each spare, there is potentially two different vdev_t 1788 * structures associated with it: one in the list of spares (used only 1789 * for basic validation purposes) and one in the active vdev 1790 * configuration (if it's spared in). During this phase we open and 1791 * validate each vdev on the spare list. If the vdev also exists in the 1792 * active configuration, then we also mark this vdev as an active spare. 1793 */ 1794 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 1795 KM_SLEEP); 1796 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1797 VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, 1798 VDEV_ALLOC_SPARE) == 0); 1799 ASSERT(vd != NULL); 1800 1801 spa->spa_spares.sav_vdevs[i] = vd; 1802 1803 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1804 B_FALSE)) != NULL) { 1805 if (!tvd->vdev_isspare) 1806 spa_spare_add(tvd); 1807 1808 /* 1809 * We only mark the spare active if we were successfully 1810 * able to load the vdev. Otherwise, importing a pool 1811 * with a bad active spare would result in strange 1812 * behavior, because multiple pool would think the spare 1813 * is actively in use. 1814 * 1815 * There is a vulnerability here to an equally bizarre 1816 * circumstance, where a dead active spare is later 1817 * brought back to life (onlined or otherwise). Given 1818 * the rarity of this scenario, and the extra complexity 1819 * it adds, we ignore the possibility. 1820 */ 1821 if (!vdev_is_dead(tvd)) 1822 spa_spare_activate(tvd); 1823 } 1824 1825 vd->vdev_top = vd; 1826 vd->vdev_aux = &spa->spa_spares; 1827 1828 if (vdev_open(vd) != 0) 1829 continue; 1830 1831 if (vdev_validate_aux(vd) == 0) 1832 spa_spare_add(vd); 1833 } 1834 1835 /* 1836 * Recompute the stashed list of spares, with status information 1837 * this time. 1838 */ 1839 VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, 1840 DATA_TYPE_NVLIST_ARRAY) == 0); 1841 1842 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 1843 KM_SLEEP); 1844 for (i = 0; i < spa->spa_spares.sav_count; i++) 1845 spares[i] = vdev_config_generate(spa, 1846 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 1847 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 1848 ZPOOL_CONFIG_SPARES, spares, spa->spa_spares.sav_count) == 0); 1849 for (i = 0; i < spa->spa_spares.sav_count; i++) 1850 nvlist_free(spares[i]); 1851 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 1852 } 1853 1854 /* 1855 * Load (or re-load) the current list of vdevs describing the active l2cache for 1856 * this pool. When this is called, we have some form of basic information in 1857 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 1858 * then re-generate a more complete list including status information. 1859 * Devices which are already active have their details maintained, and are 1860 * not re-opened. 1861 */ 1862 void 1863 spa_load_l2cache(spa_t *spa) 1864 { 1865 nvlist_t **l2cache = NULL; 1866 uint_t nl2cache; 1867 int i, j, oldnvdevs; 1868 uint64_t guid; 1869 vdev_t *vd, **oldvdevs, **newvdevs; 1870 spa_aux_vdev_t *sav = &spa->spa_l2cache; 1871 1872 #ifndef _KERNEL 1873 /* 1874 * zdb opens both the current state of the pool and the 1875 * checkpointed state (if present), with a different spa_t. 1876 * 1877 * As L2 caches are part of the ARC which is shared among open 1878 * pools, we skip loading them when we load the checkpointed 1879 * state of the pool. 1880 */ 1881 if (!spa_writeable(spa)) 1882 return; 1883 #endif 1884 1885 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1886 1887 oldvdevs = sav->sav_vdevs; 1888 oldnvdevs = sav->sav_count; 1889 sav->sav_vdevs = NULL; 1890 sav->sav_count = 0; 1891 1892 if (sav->sav_config == NULL) { 1893 nl2cache = 0; 1894 newvdevs = NULL; 1895 goto out; 1896 } 1897 1898 VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, 1899 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 1900 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 1901 1902 /* 1903 * Process new nvlist of vdevs. 1904 */ 1905 for (i = 0; i < nl2cache; i++) { 1906 VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID, 1907 &guid) == 0); 1908 1909 newvdevs[i] = NULL; 1910 for (j = 0; j < oldnvdevs; j++) { 1911 vd = oldvdevs[j]; 1912 if (vd != NULL && guid == vd->vdev_guid) { 1913 /* 1914 * Retain previous vdev for add/remove ops. 1915 */ 1916 newvdevs[i] = vd; 1917 oldvdevs[j] = NULL; 1918 break; 1919 } 1920 } 1921 1922 if (newvdevs[i] == NULL) { 1923 /* 1924 * Create new vdev 1925 */ 1926 VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 1927 VDEV_ALLOC_L2CACHE) == 0); 1928 ASSERT(vd != NULL); 1929 newvdevs[i] = vd; 1930 1931 /* 1932 * Commit this vdev as an l2cache device, 1933 * even if it fails to open. 1934 */ 1935 spa_l2cache_add(vd); 1936 1937 vd->vdev_top = vd; 1938 vd->vdev_aux = sav; 1939 1940 spa_l2cache_activate(vd); 1941 1942 if (vdev_open(vd) != 0) 1943 continue; 1944 1945 (void) vdev_validate_aux(vd); 1946 1947 if (!vdev_is_dead(vd)) 1948 l2arc_add_vdev(spa, vd); 1949 1950 /* 1951 * Upon cache device addition to a pool or pool 1952 * creation with a cache device or if the header 1953 * of the device is invalid we issue an async 1954 * TRIM command for the whole device which will 1955 * execute if l2arc_trim_ahead > 0. 1956 */ 1957 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 1958 } 1959 } 1960 1961 sav->sav_vdevs = newvdevs; 1962 sav->sav_count = (int)nl2cache; 1963 1964 /* 1965 * Recompute the stashed list of l2cache devices, with status 1966 * information this time. 1967 */ 1968 VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 1969 DATA_TYPE_NVLIST_ARRAY) == 0); 1970 1971 if (sav->sav_count > 0) 1972 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 1973 KM_SLEEP); 1974 for (i = 0; i < sav->sav_count; i++) 1975 l2cache[i] = vdev_config_generate(spa, 1976 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 1977 VERIFY(nvlist_add_nvlist_array(sav->sav_config, 1978 ZPOOL_CONFIG_L2CACHE, l2cache, sav->sav_count) == 0); 1979 1980 out: 1981 /* 1982 * Purge vdevs that were dropped 1983 */ 1984 for (i = 0; i < oldnvdevs; i++) { 1985 uint64_t pool; 1986 1987 vd = oldvdevs[i]; 1988 if (vd != NULL) { 1989 ASSERT(vd->vdev_isl2cache); 1990 1991 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 1992 pool != 0ULL && l2arc_vdev_present(vd)) 1993 l2arc_remove_vdev(vd); 1994 vdev_clear_stats(vd); 1995 vdev_free(vd); 1996 } 1997 } 1998 1999 if (oldvdevs) 2000 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2001 2002 for (i = 0; i < sav->sav_count; i++) 2003 nvlist_free(l2cache[i]); 2004 if (sav->sav_count) 2005 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2006 } 2007 2008 static int 2009 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2010 { 2011 dmu_buf_t *db; 2012 char *packed = NULL; 2013 size_t nvsize = 0; 2014 int error; 2015 *value = NULL; 2016 2017 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2018 if (error) 2019 return (error); 2020 2021 nvsize = *(uint64_t *)db->db_data; 2022 dmu_buf_rele(db, FTAG); 2023 2024 packed = vmem_alloc(nvsize, KM_SLEEP); 2025 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2026 DMU_READ_PREFETCH); 2027 if (error == 0) 2028 error = nvlist_unpack(packed, nvsize, value, 0); 2029 vmem_free(packed, nvsize); 2030 2031 return (error); 2032 } 2033 2034 /* 2035 * Concrete top-level vdevs that are not missing and are not logs. At every 2036 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2037 */ 2038 static uint64_t 2039 spa_healthy_core_tvds(spa_t *spa) 2040 { 2041 vdev_t *rvd = spa->spa_root_vdev; 2042 uint64_t tvds = 0; 2043 2044 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2045 vdev_t *vd = rvd->vdev_child[i]; 2046 if (vd->vdev_islog) 2047 continue; 2048 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2049 tvds++; 2050 } 2051 2052 return (tvds); 2053 } 2054 2055 /* 2056 * Checks to see if the given vdev could not be opened, in which case we post a 2057 * sysevent to notify the autoreplace code that the device has been removed. 2058 */ 2059 static void 2060 spa_check_removed(vdev_t *vd) 2061 { 2062 for (uint64_t c = 0; c < vd->vdev_children; c++) 2063 spa_check_removed(vd->vdev_child[c]); 2064 2065 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2066 vdev_is_concrete(vd)) { 2067 zfs_post_autoreplace(vd->vdev_spa, vd); 2068 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2069 } 2070 } 2071 2072 static int 2073 spa_check_for_missing_logs(spa_t *spa) 2074 { 2075 vdev_t *rvd = spa->spa_root_vdev; 2076 2077 /* 2078 * If we're doing a normal import, then build up any additional 2079 * diagnostic information about missing log devices. 2080 * We'll pass this up to the user for further processing. 2081 */ 2082 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2083 nvlist_t **child, *nv; 2084 uint64_t idx = 0; 2085 2086 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2087 KM_SLEEP); 2088 VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); 2089 2090 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2091 vdev_t *tvd = rvd->vdev_child[c]; 2092 2093 /* 2094 * We consider a device as missing only if it failed 2095 * to open (i.e. offline or faulted is not considered 2096 * as missing). 2097 */ 2098 if (tvd->vdev_islog && 2099 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2100 child[idx++] = vdev_config_generate(spa, tvd, 2101 B_FALSE, VDEV_CONFIG_MISSING); 2102 } 2103 } 2104 2105 if (idx > 0) { 2106 fnvlist_add_nvlist_array(nv, 2107 ZPOOL_CONFIG_CHILDREN, child, idx); 2108 fnvlist_add_nvlist(spa->spa_load_info, 2109 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2110 2111 for (uint64_t i = 0; i < idx; i++) 2112 nvlist_free(child[i]); 2113 } 2114 nvlist_free(nv); 2115 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2116 2117 if (idx > 0) { 2118 spa_load_failed(spa, "some log devices are missing"); 2119 vdev_dbgmsg_print_tree(rvd, 2); 2120 return (SET_ERROR(ENXIO)); 2121 } 2122 } else { 2123 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2124 vdev_t *tvd = rvd->vdev_child[c]; 2125 2126 if (tvd->vdev_islog && 2127 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2128 spa_set_log_state(spa, SPA_LOG_CLEAR); 2129 spa_load_note(spa, "some log devices are " 2130 "missing, ZIL is dropped."); 2131 vdev_dbgmsg_print_tree(rvd, 2); 2132 break; 2133 } 2134 } 2135 } 2136 2137 return (0); 2138 } 2139 2140 /* 2141 * Check for missing log devices 2142 */ 2143 static boolean_t 2144 spa_check_logs(spa_t *spa) 2145 { 2146 boolean_t rv = B_FALSE; 2147 dsl_pool_t *dp = spa_get_dsl(spa); 2148 2149 switch (spa->spa_log_state) { 2150 default: 2151 break; 2152 case SPA_LOG_MISSING: 2153 /* need to recheck in case slog has been restored */ 2154 case SPA_LOG_UNKNOWN: 2155 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2156 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2157 if (rv) 2158 spa_set_log_state(spa, SPA_LOG_MISSING); 2159 break; 2160 } 2161 return (rv); 2162 } 2163 2164 /* 2165 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2166 */ 2167 static boolean_t 2168 spa_passivate_log(spa_t *spa) 2169 { 2170 vdev_t *rvd = spa->spa_root_vdev; 2171 boolean_t slog_found = B_FALSE; 2172 2173 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2174 2175 for (int c = 0; c < rvd->vdev_children; c++) { 2176 vdev_t *tvd = rvd->vdev_child[c]; 2177 2178 if (tvd->vdev_islog) { 2179 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2180 metaslab_group_passivate(tvd->vdev_mg); 2181 slog_found = B_TRUE; 2182 } 2183 } 2184 2185 return (slog_found); 2186 } 2187 2188 /* 2189 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2190 */ 2191 static void 2192 spa_activate_log(spa_t *spa) 2193 { 2194 vdev_t *rvd = spa->spa_root_vdev; 2195 2196 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2197 2198 for (int c = 0; c < rvd->vdev_children; c++) { 2199 vdev_t *tvd = rvd->vdev_child[c]; 2200 2201 if (tvd->vdev_islog) { 2202 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2203 metaslab_group_activate(tvd->vdev_mg); 2204 } 2205 } 2206 } 2207 2208 int 2209 spa_reset_logs(spa_t *spa) 2210 { 2211 int error; 2212 2213 error = dmu_objset_find(spa_name(spa), zil_reset, 2214 NULL, DS_FIND_CHILDREN); 2215 if (error == 0) { 2216 /* 2217 * We successfully offlined the log device, sync out the 2218 * current txg so that the "stubby" block can be removed 2219 * by zil_sync(). 2220 */ 2221 txg_wait_synced(spa->spa_dsl_pool, 0); 2222 } 2223 return (error); 2224 } 2225 2226 static void 2227 spa_aux_check_removed(spa_aux_vdev_t *sav) 2228 { 2229 for (int i = 0; i < sav->sav_count; i++) 2230 spa_check_removed(sav->sav_vdevs[i]); 2231 } 2232 2233 void 2234 spa_claim_notify(zio_t *zio) 2235 { 2236 spa_t *spa = zio->io_spa; 2237 2238 if (zio->io_error) 2239 return; 2240 2241 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2242 if (spa->spa_claim_max_txg < zio->io_bp->blk_birth) 2243 spa->spa_claim_max_txg = zio->io_bp->blk_birth; 2244 mutex_exit(&spa->spa_props_lock); 2245 } 2246 2247 typedef struct spa_load_error { 2248 uint64_t sle_meta_count; 2249 uint64_t sle_data_count; 2250 } spa_load_error_t; 2251 2252 static void 2253 spa_load_verify_done(zio_t *zio) 2254 { 2255 blkptr_t *bp = zio->io_bp; 2256 spa_load_error_t *sle = zio->io_private; 2257 dmu_object_type_t type = BP_GET_TYPE(bp); 2258 int error = zio->io_error; 2259 spa_t *spa = zio->io_spa; 2260 2261 abd_free(zio->io_abd); 2262 if (error) { 2263 if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && 2264 type != DMU_OT_INTENT_LOG) 2265 atomic_inc_64(&sle->sle_meta_count); 2266 else 2267 atomic_inc_64(&sle->sle_data_count); 2268 } 2269 2270 mutex_enter(&spa->spa_scrub_lock); 2271 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 2272 cv_broadcast(&spa->spa_scrub_io_cv); 2273 mutex_exit(&spa->spa_scrub_lock); 2274 } 2275 2276 /* 2277 * Maximum number of inflight bytes is the log2 fraction of the arc size. 2278 * By default, we set it to 1/16th of the arc. 2279 */ 2280 int spa_load_verify_shift = 4; 2281 int spa_load_verify_metadata = B_TRUE; 2282 int spa_load_verify_data = B_TRUE; 2283 2284 /*ARGSUSED*/ 2285 static int 2286 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 2287 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 2288 { 2289 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 2290 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 2291 return (0); 2292 /* 2293 * Note: normally this routine will not be called if 2294 * spa_load_verify_metadata is not set. However, it may be useful 2295 * to manually set the flag after the traversal has begun. 2296 */ 2297 if (!spa_load_verify_metadata) 2298 return (0); 2299 if (!BP_IS_METADATA(bp) && !spa_load_verify_data) 2300 return (0); 2301 2302 uint64_t maxinflight_bytes = 2303 arc_target_bytes() >> spa_load_verify_shift; 2304 zio_t *rio = arg; 2305 size_t size = BP_GET_PSIZE(bp); 2306 2307 mutex_enter(&spa->spa_scrub_lock); 2308 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 2309 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 2310 spa->spa_load_verify_bytes += size; 2311 mutex_exit(&spa->spa_scrub_lock); 2312 2313 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 2314 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 2315 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 2316 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 2317 return (0); 2318 } 2319 2320 /* ARGSUSED */ 2321 static int 2322 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 2323 { 2324 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 2325 return (SET_ERROR(ENAMETOOLONG)); 2326 2327 return (0); 2328 } 2329 2330 static int 2331 spa_load_verify(spa_t *spa) 2332 { 2333 zio_t *rio; 2334 spa_load_error_t sle = { 0 }; 2335 zpool_load_policy_t policy; 2336 boolean_t verify_ok = B_FALSE; 2337 int error = 0; 2338 2339 zpool_get_load_policy(spa->spa_config, &policy); 2340 2341 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND) 2342 return (0); 2343 2344 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 2345 error = dmu_objset_find_dp(spa->spa_dsl_pool, 2346 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 2347 DS_FIND_CHILDREN); 2348 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 2349 if (error != 0) 2350 return (error); 2351 2352 rio = zio_root(spa, NULL, &sle, 2353 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 2354 2355 if (spa_load_verify_metadata) { 2356 if (spa->spa_extreme_rewind) { 2357 spa_load_note(spa, "performing a complete scan of the " 2358 "pool since extreme rewind is on. This may take " 2359 "a very long time.\n (spa_load_verify_data=%u, " 2360 "spa_load_verify_metadata=%u)", 2361 spa_load_verify_data, spa_load_verify_metadata); 2362 } 2363 2364 error = traverse_pool(spa, spa->spa_verify_min_txg, 2365 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 2366 TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio); 2367 } 2368 2369 (void) zio_wait(rio); 2370 ASSERT0(spa->spa_load_verify_bytes); 2371 2372 spa->spa_load_meta_errors = sle.sle_meta_count; 2373 spa->spa_load_data_errors = sle.sle_data_count; 2374 2375 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 2376 spa_load_note(spa, "spa_load_verify found %llu metadata errors " 2377 "and %llu data errors", (u_longlong_t)sle.sle_meta_count, 2378 (u_longlong_t)sle.sle_data_count); 2379 } 2380 2381 if (spa_load_verify_dryrun || 2382 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 2383 sle.sle_data_count <= policy.zlp_maxdata)) { 2384 int64_t loss = 0; 2385 2386 verify_ok = B_TRUE; 2387 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 2388 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 2389 2390 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; 2391 VERIFY(nvlist_add_uint64(spa->spa_load_info, 2392 ZPOOL_CONFIG_LOAD_TIME, spa->spa_load_txg_ts) == 0); 2393 VERIFY(nvlist_add_int64(spa->spa_load_info, 2394 ZPOOL_CONFIG_REWIND_TIME, loss) == 0); 2395 VERIFY(nvlist_add_uint64(spa->spa_load_info, 2396 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count) == 0); 2397 } else { 2398 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 2399 } 2400 2401 if (spa_load_verify_dryrun) 2402 return (0); 2403 2404 if (error) { 2405 if (error != ENXIO && error != EIO) 2406 error = SET_ERROR(EIO); 2407 return (error); 2408 } 2409 2410 return (verify_ok ? 0 : EIO); 2411 } 2412 2413 /* 2414 * Find a value in the pool props object. 2415 */ 2416 static void 2417 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 2418 { 2419 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 2420 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 2421 } 2422 2423 /* 2424 * Find a value in the pool directory object. 2425 */ 2426 static int 2427 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 2428 { 2429 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2430 name, sizeof (uint64_t), 1, val); 2431 2432 if (error != 0 && (error != ENOENT || log_enoent)) { 2433 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 2434 "[error=%d]", name, error); 2435 } 2436 2437 return (error); 2438 } 2439 2440 static int 2441 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 2442 { 2443 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 2444 return (SET_ERROR(err)); 2445 } 2446 2447 boolean_t 2448 spa_livelist_delete_check(spa_t *spa) 2449 { 2450 return (spa->spa_livelists_to_delete != 0); 2451 } 2452 2453 /* ARGSUSED */ 2454 static boolean_t 2455 spa_livelist_delete_cb_check(void *arg, zthr_t *z) 2456 { 2457 spa_t *spa = arg; 2458 return (spa_livelist_delete_check(spa)); 2459 } 2460 2461 static int 2462 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 2463 { 2464 spa_t *spa = arg; 2465 zio_free(spa, tx->tx_txg, bp); 2466 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 2467 -bp_get_dsize_sync(spa, bp), 2468 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 2469 return (0); 2470 } 2471 2472 static int 2473 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 2474 { 2475 int err; 2476 zap_cursor_t zc; 2477 zap_attribute_t za; 2478 zap_cursor_init(&zc, os, zap_obj); 2479 err = zap_cursor_retrieve(&zc, &za); 2480 zap_cursor_fini(&zc); 2481 if (err == 0) 2482 *llp = za.za_first_integer; 2483 return (err); 2484 } 2485 2486 /* 2487 * Components of livelist deletion that must be performed in syncing 2488 * context: freeing block pointers and updating the pool-wide data 2489 * structures to indicate how much work is left to do 2490 */ 2491 typedef struct sublist_delete_arg { 2492 spa_t *spa; 2493 dsl_deadlist_t *ll; 2494 uint64_t key; 2495 bplist_t *to_free; 2496 } sublist_delete_arg_t; 2497 2498 static void 2499 sublist_delete_sync(void *arg, dmu_tx_t *tx) 2500 { 2501 sublist_delete_arg_t *sda = arg; 2502 spa_t *spa = sda->spa; 2503 dsl_deadlist_t *ll = sda->ll; 2504 uint64_t key = sda->key; 2505 bplist_t *to_free = sda->to_free; 2506 2507 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 2508 dsl_deadlist_remove_entry(ll, key, tx); 2509 } 2510 2511 typedef struct livelist_delete_arg { 2512 spa_t *spa; 2513 uint64_t ll_obj; 2514 uint64_t zap_obj; 2515 } livelist_delete_arg_t; 2516 2517 static void 2518 livelist_delete_sync(void *arg, dmu_tx_t *tx) 2519 { 2520 livelist_delete_arg_t *lda = arg; 2521 spa_t *spa = lda->spa; 2522 uint64_t ll_obj = lda->ll_obj; 2523 uint64_t zap_obj = lda->zap_obj; 2524 objset_t *mos = spa->spa_meta_objset; 2525 uint64_t count; 2526 2527 /* free the livelist and decrement the feature count */ 2528 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 2529 dsl_deadlist_free(mos, ll_obj, tx); 2530 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 2531 VERIFY0(zap_count(mos, zap_obj, &count)); 2532 if (count == 0) { 2533 /* no more livelists to delete */ 2534 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 2535 DMU_POOL_DELETED_CLONES, tx)); 2536 VERIFY0(zap_destroy(mos, zap_obj, tx)); 2537 spa->spa_livelists_to_delete = 0; 2538 spa_notify_waiters(spa); 2539 } 2540 } 2541 2542 /* 2543 * Load in the value for the livelist to be removed and open it. Then, 2544 * load its first sublist and determine which block pointers should actually 2545 * be freed. Then, call a synctask which performs the actual frees and updates 2546 * the pool-wide livelist data. 2547 */ 2548 /* ARGSUSED */ 2549 static void 2550 spa_livelist_delete_cb(void *arg, zthr_t *z) 2551 { 2552 spa_t *spa = arg; 2553 uint64_t ll_obj = 0, count; 2554 objset_t *mos = spa->spa_meta_objset; 2555 uint64_t zap_obj = spa->spa_livelists_to_delete; 2556 /* 2557 * Determine the next livelist to delete. This function should only 2558 * be called if there is at least one deleted clone. 2559 */ 2560 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 2561 VERIFY0(zap_count(mos, ll_obj, &count)); 2562 if (count > 0) { 2563 dsl_deadlist_t *ll; 2564 dsl_deadlist_entry_t *dle; 2565 bplist_t to_free; 2566 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 2567 dsl_deadlist_open(ll, mos, ll_obj); 2568 dle = dsl_deadlist_first(ll); 2569 ASSERT3P(dle, !=, NULL); 2570 bplist_create(&to_free); 2571 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 2572 z, NULL); 2573 if (err == 0) { 2574 sublist_delete_arg_t sync_arg = { 2575 .spa = spa, 2576 .ll = ll, 2577 .key = dle->dle_mintxg, 2578 .to_free = &to_free 2579 }; 2580 zfs_dbgmsg("deleting sublist (id %llu) from" 2581 " livelist %llu, %lld remaining", 2582 (u_longlong_t)dle->dle_bpobj.bpo_object, 2583 (u_longlong_t)ll_obj, (longlong_t)count - 1); 2584 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 2585 sublist_delete_sync, &sync_arg, 0, 2586 ZFS_SPACE_CHECK_DESTROY)); 2587 } else { 2588 VERIFY3U(err, ==, EINTR); 2589 } 2590 bplist_clear(&to_free); 2591 bplist_destroy(&to_free); 2592 dsl_deadlist_close(ll); 2593 kmem_free(ll, sizeof (dsl_deadlist_t)); 2594 } else { 2595 livelist_delete_arg_t sync_arg = { 2596 .spa = spa, 2597 .ll_obj = ll_obj, 2598 .zap_obj = zap_obj 2599 }; 2600 zfs_dbgmsg("deletion of livelist %llu completed", 2601 (u_longlong_t)ll_obj); 2602 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 2603 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 2604 } 2605 } 2606 2607 static void 2608 spa_start_livelist_destroy_thread(spa_t *spa) 2609 { 2610 ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL); 2611 spa->spa_livelist_delete_zthr = 2612 zthr_create("z_livelist_destroy", 2613 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa, 2614 minclsyspri); 2615 } 2616 2617 typedef struct livelist_new_arg { 2618 bplist_t *allocs; 2619 bplist_t *frees; 2620 } livelist_new_arg_t; 2621 2622 static int 2623 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 2624 dmu_tx_t *tx) 2625 { 2626 ASSERT(tx == NULL); 2627 livelist_new_arg_t *lna = arg; 2628 if (bp_freed) { 2629 bplist_append(lna->frees, bp); 2630 } else { 2631 bplist_append(lna->allocs, bp); 2632 zfs_livelist_condense_new_alloc++; 2633 } 2634 return (0); 2635 } 2636 2637 typedef struct livelist_condense_arg { 2638 spa_t *spa; 2639 bplist_t to_keep; 2640 uint64_t first_size; 2641 uint64_t next_size; 2642 } livelist_condense_arg_t; 2643 2644 static void 2645 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 2646 { 2647 livelist_condense_arg_t *lca = arg; 2648 spa_t *spa = lca->spa; 2649 bplist_t new_frees; 2650 dsl_dataset_t *ds = spa->spa_to_condense.ds; 2651 2652 /* Have we been cancelled? */ 2653 if (spa->spa_to_condense.cancelled) { 2654 zfs_livelist_condense_sync_cancel++; 2655 goto out; 2656 } 2657 2658 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2659 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2660 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 2661 2662 /* 2663 * It's possible that the livelist was changed while the zthr was 2664 * running. Therefore, we need to check for new blkptrs in the two 2665 * entries being condensed and continue to track them in the livelist. 2666 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 2667 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 2668 * we need to sort them into two different bplists. 2669 */ 2670 uint64_t first_obj = first->dle_bpobj.bpo_object; 2671 uint64_t next_obj = next->dle_bpobj.bpo_object; 2672 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2673 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2674 2675 bplist_create(&new_frees); 2676 livelist_new_arg_t new_bps = { 2677 .allocs = &lca->to_keep, 2678 .frees = &new_frees, 2679 }; 2680 2681 if (cur_first_size > lca->first_size) { 2682 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 2683 livelist_track_new_cb, &new_bps, lca->first_size)); 2684 } 2685 if (cur_next_size > lca->next_size) { 2686 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 2687 livelist_track_new_cb, &new_bps, lca->next_size)); 2688 } 2689 2690 dsl_deadlist_clear_entry(first, ll, tx); 2691 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 2692 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 2693 2694 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 2695 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 2696 bplist_destroy(&new_frees); 2697 2698 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 2699 dsl_dataset_name(ds, dsname); 2700 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 2701 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 2702 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, 2703 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, 2704 (u_longlong_t)cur_first_size, (u_longlong_t)next_obj, 2705 (u_longlong_t)cur_next_size, 2706 (u_longlong_t)first->dle_bpobj.bpo_object, 2707 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 2708 out: 2709 dmu_buf_rele(ds->ds_dbuf, spa); 2710 spa->spa_to_condense.ds = NULL; 2711 bplist_clear(&lca->to_keep); 2712 bplist_destroy(&lca->to_keep); 2713 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2714 spa->spa_to_condense.syncing = B_FALSE; 2715 } 2716 2717 static void 2718 spa_livelist_condense_cb(void *arg, zthr_t *t) 2719 { 2720 while (zfs_livelist_condense_zthr_pause && 2721 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2722 delay(1); 2723 2724 spa_t *spa = arg; 2725 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2726 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2727 uint64_t first_size, next_size; 2728 2729 livelist_condense_arg_t *lca = 2730 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 2731 bplist_create(&lca->to_keep); 2732 2733 /* 2734 * Process the livelists (matching FREEs and ALLOCs) in open context 2735 * so we have minimal work in syncing context to condense. 2736 * 2737 * We save bpobj sizes (first_size and next_size) to use later in 2738 * syncing context to determine if entries were added to these sublists 2739 * while in open context. This is possible because the clone is still 2740 * active and open for normal writes and we want to make sure the new, 2741 * unprocessed blockpointers are inserted into the livelist normally. 2742 * 2743 * Note that dsl_process_sub_livelist() both stores the size number of 2744 * blockpointers and iterates over them while the bpobj's lock held, so 2745 * the sizes returned to us are consistent which what was actually 2746 * processed. 2747 */ 2748 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 2749 &first_size); 2750 if (err == 0) 2751 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 2752 t, &next_size); 2753 2754 if (err == 0) { 2755 while (zfs_livelist_condense_sync_pause && 2756 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2757 delay(1); 2758 2759 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2760 dmu_tx_mark_netfree(tx); 2761 dmu_tx_hold_space(tx, 1); 2762 err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE); 2763 if (err == 0) { 2764 /* 2765 * Prevent the condense zthr restarting before 2766 * the synctask completes. 2767 */ 2768 spa->spa_to_condense.syncing = B_TRUE; 2769 lca->spa = spa; 2770 lca->first_size = first_size; 2771 lca->next_size = next_size; 2772 dsl_sync_task_nowait(spa_get_dsl(spa), 2773 spa_livelist_condense_sync, lca, tx); 2774 dmu_tx_commit(tx); 2775 return; 2776 } 2777 } 2778 /* 2779 * Condensing can not continue: either it was externally stopped or 2780 * we were unable to assign to a tx because the pool has run out of 2781 * space. In the second case, we'll just end up trying to condense 2782 * again in a later txg. 2783 */ 2784 ASSERT(err != 0); 2785 bplist_clear(&lca->to_keep); 2786 bplist_destroy(&lca->to_keep); 2787 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2788 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 2789 spa->spa_to_condense.ds = NULL; 2790 if (err == EINTR) 2791 zfs_livelist_condense_zthr_cancel++; 2792 } 2793 2794 /* ARGSUSED */ 2795 /* 2796 * Check that there is something to condense but that a condense is not 2797 * already in progress and that condensing has not been cancelled. 2798 */ 2799 static boolean_t 2800 spa_livelist_condense_cb_check(void *arg, zthr_t *z) 2801 { 2802 spa_t *spa = arg; 2803 if ((spa->spa_to_condense.ds != NULL) && 2804 (spa->spa_to_condense.syncing == B_FALSE) && 2805 (spa->spa_to_condense.cancelled == B_FALSE)) { 2806 return (B_TRUE); 2807 } 2808 return (B_FALSE); 2809 } 2810 2811 static void 2812 spa_start_livelist_condensing_thread(spa_t *spa) 2813 { 2814 spa->spa_to_condense.ds = NULL; 2815 spa->spa_to_condense.first = NULL; 2816 spa->spa_to_condense.next = NULL; 2817 spa->spa_to_condense.syncing = B_FALSE; 2818 spa->spa_to_condense.cancelled = B_FALSE; 2819 2820 ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL); 2821 spa->spa_livelist_condense_zthr = 2822 zthr_create("z_livelist_condense", 2823 spa_livelist_condense_cb_check, 2824 spa_livelist_condense_cb, spa, minclsyspri); 2825 } 2826 2827 static void 2828 spa_spawn_aux_threads(spa_t *spa) 2829 { 2830 ASSERT(spa_writeable(spa)); 2831 2832 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 2833 2834 spa_start_indirect_condensing_thread(spa); 2835 spa_start_livelist_destroy_thread(spa); 2836 spa_start_livelist_condensing_thread(spa); 2837 2838 ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL); 2839 spa->spa_checkpoint_discard_zthr = 2840 zthr_create("z_checkpoint_discard", 2841 spa_checkpoint_discard_thread_check, 2842 spa_checkpoint_discard_thread, spa, minclsyspri); 2843 } 2844 2845 /* 2846 * Fix up config after a partly-completed split. This is done with the 2847 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 2848 * pool have that entry in their config, but only the splitting one contains 2849 * a list of all the guids of the vdevs that are being split off. 2850 * 2851 * This function determines what to do with that list: either rejoin 2852 * all the disks to the pool, or complete the splitting process. To attempt 2853 * the rejoin, each disk that is offlined is marked online again, and 2854 * we do a reopen() call. If the vdev label for every disk that was 2855 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 2856 * then we call vdev_split() on each disk, and complete the split. 2857 * 2858 * Otherwise we leave the config alone, with all the vdevs in place in 2859 * the original pool. 2860 */ 2861 static void 2862 spa_try_repair(spa_t *spa, nvlist_t *config) 2863 { 2864 uint_t extracted; 2865 uint64_t *glist; 2866 uint_t i, gcount; 2867 nvlist_t *nvl; 2868 vdev_t **vd; 2869 boolean_t attempt_reopen; 2870 2871 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 2872 return; 2873 2874 /* check that the config is complete */ 2875 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 2876 &glist, &gcount) != 0) 2877 return; 2878 2879 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 2880 2881 /* attempt to online all the vdevs & validate */ 2882 attempt_reopen = B_TRUE; 2883 for (i = 0; i < gcount; i++) { 2884 if (glist[i] == 0) /* vdev is hole */ 2885 continue; 2886 2887 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 2888 if (vd[i] == NULL) { 2889 /* 2890 * Don't bother attempting to reopen the disks; 2891 * just do the split. 2892 */ 2893 attempt_reopen = B_FALSE; 2894 } else { 2895 /* attempt to re-online it */ 2896 vd[i]->vdev_offline = B_FALSE; 2897 } 2898 } 2899 2900 if (attempt_reopen) { 2901 vdev_reopen(spa->spa_root_vdev); 2902 2903 /* check each device to see what state it's in */ 2904 for (extracted = 0, i = 0; i < gcount; i++) { 2905 if (vd[i] != NULL && 2906 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 2907 break; 2908 ++extracted; 2909 } 2910 } 2911 2912 /* 2913 * If every disk has been moved to the new pool, or if we never 2914 * even attempted to look at them, then we split them off for 2915 * good. 2916 */ 2917 if (!attempt_reopen || gcount == extracted) { 2918 for (i = 0; i < gcount; i++) 2919 if (vd[i] != NULL) 2920 vdev_split(vd[i]); 2921 vdev_reopen(spa->spa_root_vdev); 2922 } 2923 2924 kmem_free(vd, gcount * sizeof (vdev_t *)); 2925 } 2926 2927 static int 2928 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 2929 { 2930 char *ereport = FM_EREPORT_ZFS_POOL; 2931 int error; 2932 2933 spa->spa_load_state = state; 2934 (void) spa_import_progress_set_state(spa_guid(spa), 2935 spa_load_state(spa)); 2936 2937 gethrestime(&spa->spa_loaded_ts); 2938 error = spa_load_impl(spa, type, &ereport); 2939 2940 /* 2941 * Don't count references from objsets that are already closed 2942 * and are making their way through the eviction process. 2943 */ 2944 spa_evicting_os_wait(spa); 2945 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 2946 if (error) { 2947 if (error != EEXIST) { 2948 spa->spa_loaded_ts.tv_sec = 0; 2949 spa->spa_loaded_ts.tv_nsec = 0; 2950 } 2951 if (error != EBADF) { 2952 (void) zfs_ereport_post(ereport, spa, 2953 NULL, NULL, NULL, 0); 2954 } 2955 } 2956 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 2957 spa->spa_ena = 0; 2958 2959 (void) spa_import_progress_set_state(spa_guid(spa), 2960 spa_load_state(spa)); 2961 2962 return (error); 2963 } 2964 2965 #ifdef ZFS_DEBUG 2966 /* 2967 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 2968 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 2969 * spa's per-vdev ZAP list. 2970 */ 2971 static uint64_t 2972 vdev_count_verify_zaps(vdev_t *vd) 2973 { 2974 spa_t *spa = vd->vdev_spa; 2975 uint64_t total = 0; 2976 2977 if (vd->vdev_top_zap != 0) { 2978 total++; 2979 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2980 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 2981 } 2982 if (vd->vdev_leaf_zap != 0) { 2983 total++; 2984 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2985 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 2986 } 2987 2988 for (uint64_t i = 0; i < vd->vdev_children; i++) { 2989 total += vdev_count_verify_zaps(vd->vdev_child[i]); 2990 } 2991 2992 return (total); 2993 } 2994 #endif 2995 2996 /* 2997 * Determine whether the activity check is required. 2998 */ 2999 static boolean_t 3000 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label, 3001 nvlist_t *config) 3002 { 3003 uint64_t state = 0; 3004 uint64_t hostid = 0; 3005 uint64_t tryconfig_txg = 0; 3006 uint64_t tryconfig_timestamp = 0; 3007 uint16_t tryconfig_mmp_seq = 0; 3008 nvlist_t *nvinfo; 3009 3010 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3011 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 3012 (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3013 &tryconfig_txg); 3014 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3015 &tryconfig_timestamp); 3016 (void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3017 &tryconfig_mmp_seq); 3018 } 3019 3020 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); 3021 3022 /* 3023 * Disable the MMP activity check - This is used by zdb which 3024 * is intended to be used on potentially active pools. 3025 */ 3026 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) 3027 return (B_FALSE); 3028 3029 /* 3030 * Skip the activity check when the MMP feature is disabled. 3031 */ 3032 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) 3033 return (B_FALSE); 3034 3035 /* 3036 * If the tryconfig_ values are nonzero, they are the results of an 3037 * earlier tryimport. If they all match the uberblock we just found, 3038 * then the pool has not changed and we return false so we do not test 3039 * a second time. 3040 */ 3041 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && 3042 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && 3043 tryconfig_mmp_seq && tryconfig_mmp_seq == 3044 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) 3045 return (B_FALSE); 3046 3047 /* 3048 * Allow the activity check to be skipped when importing the pool 3049 * on the same host which last imported it. Since the hostid from 3050 * configuration may be stale use the one read from the label. 3051 */ 3052 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3053 hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID); 3054 3055 if (hostid == spa_get_hostid(spa)) 3056 return (B_FALSE); 3057 3058 /* 3059 * Skip the activity test when the pool was cleanly exported. 3060 */ 3061 if (state != POOL_STATE_ACTIVE) 3062 return (B_FALSE); 3063 3064 return (B_TRUE); 3065 } 3066 3067 /* 3068 * Nanoseconds the activity check must watch for changes on-disk. 3069 */ 3070 static uint64_t 3071 spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3072 { 3073 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 3074 uint64_t multihost_interval = MSEC2NSEC( 3075 MMP_INTERVAL_OK(zfs_multihost_interval)); 3076 uint64_t import_delay = MAX(NANOSEC, import_intervals * 3077 multihost_interval); 3078 3079 /* 3080 * Local tunables determine a minimum duration except for the case 3081 * where we know when the remote host will suspend the pool if MMP 3082 * writes do not land. 3083 * 3084 * See Big Theory comment at the top of mmp.c for the reasoning behind 3085 * these cases and times. 3086 */ 3087 3088 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 3089 3090 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3091 MMP_FAIL_INT(ub) > 0) { 3092 3093 /* MMP on remote host will suspend pool after failed writes */ 3094 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 3095 MMP_IMPORT_SAFETY_FACTOR / 100; 3096 3097 zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp " 3098 "mmp_fails=%llu ub_mmp mmp_interval=%llu " 3099 "import_intervals=%llu", (u_longlong_t)import_delay, 3100 (u_longlong_t)MMP_FAIL_INT(ub), 3101 (u_longlong_t)MMP_INTERVAL(ub), 3102 (u_longlong_t)import_intervals); 3103 3104 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3105 MMP_FAIL_INT(ub) == 0) { 3106 3107 /* MMP on remote host will never suspend pool */ 3108 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 3109 ub->ub_mmp_delay) * import_intervals); 3110 3111 zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp " 3112 "mmp_interval=%llu ub_mmp_delay=%llu " 3113 "import_intervals=%llu", (u_longlong_t)import_delay, 3114 (u_longlong_t)MMP_INTERVAL(ub), 3115 (u_longlong_t)ub->ub_mmp_delay, 3116 (u_longlong_t)import_intervals); 3117 3118 } else if (MMP_VALID(ub)) { 3119 /* 3120 * zfs-0.7 compatibility case 3121 */ 3122 3123 import_delay = MAX(import_delay, (multihost_interval + 3124 ub->ub_mmp_delay) * import_intervals); 3125 3126 zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu " 3127 "import_intervals=%llu leaves=%u", 3128 (u_longlong_t)import_delay, 3129 (u_longlong_t)ub->ub_mmp_delay, 3130 (u_longlong_t)import_intervals, 3131 vdev_count_leaves(spa)); 3132 } else { 3133 /* Using local tunings is the only reasonable option */ 3134 zfs_dbgmsg("pool last imported on non-MMP aware " 3135 "host using import_delay=%llu multihost_interval=%llu " 3136 "import_intervals=%llu", (u_longlong_t)import_delay, 3137 (u_longlong_t)multihost_interval, 3138 (u_longlong_t)import_intervals); 3139 } 3140 3141 return (import_delay); 3142 } 3143 3144 /* 3145 * Perform the import activity check. If the user canceled the import or 3146 * we detected activity then fail. 3147 */ 3148 static int 3149 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config) 3150 { 3151 uint64_t txg = ub->ub_txg; 3152 uint64_t timestamp = ub->ub_timestamp; 3153 uint64_t mmp_config = ub->ub_mmp_config; 3154 uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0; 3155 uint64_t import_delay; 3156 hrtime_t import_expire; 3157 nvlist_t *mmp_label = NULL; 3158 vdev_t *rvd = spa->spa_root_vdev; 3159 kcondvar_t cv; 3160 kmutex_t mtx; 3161 int error = 0; 3162 3163 cv_init(&cv, NULL, CV_DEFAULT, NULL); 3164 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 3165 mutex_enter(&mtx); 3166 3167 /* 3168 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3169 * during the earlier tryimport. If the txg recorded there is 0 then 3170 * the pool is known to be active on another host. 3171 * 3172 * Otherwise, the pool might be in use on another host. Check for 3173 * changes in the uberblocks on disk if necessary. 3174 */ 3175 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3176 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 3177 ZPOOL_CONFIG_LOAD_INFO); 3178 3179 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && 3180 fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { 3181 vdev_uberblock_load(rvd, ub, &mmp_label); 3182 error = SET_ERROR(EREMOTEIO); 3183 goto out; 3184 } 3185 } 3186 3187 import_delay = spa_activity_check_duration(spa, ub); 3188 3189 /* Add a small random factor in case of simultaneous imports (0-25%) */ 3190 import_delay += import_delay * random_in_range(250) / 1000; 3191 3192 import_expire = gethrtime() + import_delay; 3193 3194 while (gethrtime() < import_expire) { 3195 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 3196 NSEC2SEC(import_expire - gethrtime())); 3197 3198 vdev_uberblock_load(rvd, ub, &mmp_label); 3199 3200 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || 3201 mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) { 3202 zfs_dbgmsg("multihost activity detected " 3203 "txg %llu ub_txg %llu " 3204 "timestamp %llu ub_timestamp %llu " 3205 "mmp_config %#llx ub_mmp_config %#llx", 3206 (u_longlong_t)txg, (u_longlong_t)ub->ub_txg, 3207 (u_longlong_t)timestamp, 3208 (u_longlong_t)ub->ub_timestamp, 3209 (u_longlong_t)mmp_config, 3210 (u_longlong_t)ub->ub_mmp_config); 3211 3212 error = SET_ERROR(EREMOTEIO); 3213 break; 3214 } 3215 3216 if (mmp_label) { 3217 nvlist_free(mmp_label); 3218 mmp_label = NULL; 3219 } 3220 3221 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 3222 if (error != -1) { 3223 error = SET_ERROR(EINTR); 3224 break; 3225 } 3226 error = 0; 3227 } 3228 3229 out: 3230 mutex_exit(&mtx); 3231 mutex_destroy(&mtx); 3232 cv_destroy(&cv); 3233 3234 /* 3235 * If the pool is determined to be active store the status in the 3236 * spa->spa_load_info nvlist. If the remote hostname or hostid are 3237 * available from configuration read from disk store them as well. 3238 * This allows 'zpool import' to generate a more useful message. 3239 * 3240 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) 3241 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 3242 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 3243 */ 3244 if (error == EREMOTEIO) { 3245 char *hostname = "<unknown>"; 3246 uint64_t hostid = 0; 3247 3248 if (mmp_label) { 3249 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { 3250 hostname = fnvlist_lookup_string(mmp_label, 3251 ZPOOL_CONFIG_HOSTNAME); 3252 fnvlist_add_string(spa->spa_load_info, 3253 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 3254 } 3255 3256 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { 3257 hostid = fnvlist_lookup_uint64(mmp_label, 3258 ZPOOL_CONFIG_HOSTID); 3259 fnvlist_add_uint64(spa->spa_load_info, 3260 ZPOOL_CONFIG_MMP_HOSTID, hostid); 3261 } 3262 } 3263 3264 fnvlist_add_uint64(spa->spa_load_info, 3265 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); 3266 fnvlist_add_uint64(spa->spa_load_info, 3267 ZPOOL_CONFIG_MMP_TXG, 0); 3268 3269 error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); 3270 } 3271 3272 if (mmp_label) 3273 nvlist_free(mmp_label); 3274 3275 return (error); 3276 } 3277 3278 static int 3279 spa_verify_host(spa_t *spa, nvlist_t *mos_config) 3280 { 3281 uint64_t hostid; 3282 char *hostname; 3283 uint64_t myhostid = 0; 3284 3285 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 3286 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 3287 hostname = fnvlist_lookup_string(mos_config, 3288 ZPOOL_CONFIG_HOSTNAME); 3289 3290 myhostid = zone_get_hostid(NULL); 3291 3292 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 3293 cmn_err(CE_WARN, "pool '%s' could not be " 3294 "loaded as it was last accessed by " 3295 "another system (host: %s hostid: 0x%llx). " 3296 "See: https://openzfs.github.io/openzfs-docs/msg/" 3297 "ZFS-8000-EY", 3298 spa_name(spa), hostname, (u_longlong_t)hostid); 3299 spa_load_failed(spa, "hostid verification failed: pool " 3300 "last accessed by host: %s (hostid: 0x%llx)", 3301 hostname, (u_longlong_t)hostid); 3302 return (SET_ERROR(EBADF)); 3303 } 3304 } 3305 3306 return (0); 3307 } 3308 3309 static int 3310 spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 3311 { 3312 int error = 0; 3313 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 3314 int parse; 3315 vdev_t *rvd; 3316 uint64_t pool_guid; 3317 char *comment; 3318 char *compatibility; 3319 3320 /* 3321 * Versioning wasn't explicitly added to the label until later, so if 3322 * it's not present treat it as the initial version. 3323 */ 3324 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 3325 &spa->spa_ubsync.ub_version) != 0) 3326 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 3327 3328 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 3329 spa_load_failed(spa, "invalid config provided: '%s' missing", 3330 ZPOOL_CONFIG_POOL_GUID); 3331 return (SET_ERROR(EINVAL)); 3332 } 3333 3334 /* 3335 * If we are doing an import, ensure that the pool is not already 3336 * imported by checking if its pool guid already exists in the 3337 * spa namespace. 3338 * 3339 * The only case that we allow an already imported pool to be 3340 * imported again, is when the pool is checkpointed and we want to 3341 * look at its checkpointed state from userland tools like zdb. 3342 */ 3343 #ifdef _KERNEL 3344 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3345 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3346 spa_guid_exists(pool_guid, 0)) { 3347 #else 3348 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3349 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3350 spa_guid_exists(pool_guid, 0) && 3351 !spa_importing_readonly_checkpoint(spa)) { 3352 #endif 3353 spa_load_failed(spa, "a pool with guid %llu is already open", 3354 (u_longlong_t)pool_guid); 3355 return (SET_ERROR(EEXIST)); 3356 } 3357 3358 spa->spa_config_guid = pool_guid; 3359 3360 nvlist_free(spa->spa_load_info); 3361 spa->spa_load_info = fnvlist_alloc(); 3362 3363 ASSERT(spa->spa_comment == NULL); 3364 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 3365 spa->spa_comment = spa_strdup(comment); 3366 3367 ASSERT(spa->spa_compatibility == NULL); 3368 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 3369 &compatibility) == 0) 3370 spa->spa_compatibility = spa_strdup(compatibility); 3371 3372 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 3373 &spa->spa_config_txg); 3374 3375 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 3376 spa->spa_config_splitting = fnvlist_dup(nvl); 3377 3378 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 3379 spa_load_failed(spa, "invalid config provided: '%s' missing", 3380 ZPOOL_CONFIG_VDEV_TREE); 3381 return (SET_ERROR(EINVAL)); 3382 } 3383 3384 /* 3385 * Create "The Godfather" zio to hold all async IOs 3386 */ 3387 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 3388 KM_SLEEP); 3389 for (int i = 0; i < max_ncpus; i++) { 3390 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 3391 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3392 ZIO_FLAG_GODFATHER); 3393 } 3394 3395 /* 3396 * Parse the configuration into a vdev tree. We explicitly set the 3397 * value that will be returned by spa_version() since parsing the 3398 * configuration requires knowing the version number. 3399 */ 3400 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3401 parse = (type == SPA_IMPORT_EXISTING ? 3402 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 3403 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 3404 spa_config_exit(spa, SCL_ALL, FTAG); 3405 3406 if (error != 0) { 3407 spa_load_failed(spa, "unable to parse config [error=%d]", 3408 error); 3409 return (error); 3410 } 3411 3412 ASSERT(spa->spa_root_vdev == rvd); 3413 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 3414 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 3415 3416 if (type != SPA_IMPORT_ASSEMBLE) { 3417 ASSERT(spa_guid(spa) == pool_guid); 3418 } 3419 3420 return (0); 3421 } 3422 3423 /* 3424 * Recursively open all vdevs in the vdev tree. This function is called twice: 3425 * first with the untrusted config, then with the trusted config. 3426 */ 3427 static int 3428 spa_ld_open_vdevs(spa_t *spa) 3429 { 3430 int error = 0; 3431 3432 /* 3433 * spa_missing_tvds_allowed defines how many top-level vdevs can be 3434 * missing/unopenable for the root vdev to be still considered openable. 3435 */ 3436 if (spa->spa_trust_config) { 3437 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 3438 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 3439 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 3440 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 3441 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 3442 } else { 3443 spa->spa_missing_tvds_allowed = 0; 3444 } 3445 3446 spa->spa_missing_tvds_allowed = 3447 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 3448 3449 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3450 error = vdev_open(spa->spa_root_vdev); 3451 spa_config_exit(spa, SCL_ALL, FTAG); 3452 3453 if (spa->spa_missing_tvds != 0) { 3454 spa_load_note(spa, "vdev tree has %lld missing top-level " 3455 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 3456 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 3457 /* 3458 * Although theoretically we could allow users to open 3459 * incomplete pools in RW mode, we'd need to add a lot 3460 * of extra logic (e.g. adjust pool space to account 3461 * for missing vdevs). 3462 * This limitation also prevents users from accidentally 3463 * opening the pool in RW mode during data recovery and 3464 * damaging it further. 3465 */ 3466 spa_load_note(spa, "pools with missing top-level " 3467 "vdevs can only be opened in read-only mode."); 3468 error = SET_ERROR(ENXIO); 3469 } else { 3470 spa_load_note(spa, "current settings allow for maximum " 3471 "%lld missing top-level vdevs at this stage.", 3472 (u_longlong_t)spa->spa_missing_tvds_allowed); 3473 } 3474 } 3475 if (error != 0) { 3476 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 3477 error); 3478 } 3479 if (spa->spa_missing_tvds != 0 || error != 0) 3480 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 3481 3482 return (error); 3483 } 3484 3485 /* 3486 * We need to validate the vdev labels against the configuration that 3487 * we have in hand. This function is called twice: first with an untrusted 3488 * config, then with a trusted config. The validation is more strict when the 3489 * config is trusted. 3490 */ 3491 static int 3492 spa_ld_validate_vdevs(spa_t *spa) 3493 { 3494 int error = 0; 3495 vdev_t *rvd = spa->spa_root_vdev; 3496 3497 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3498 error = vdev_validate(rvd); 3499 spa_config_exit(spa, SCL_ALL, FTAG); 3500 3501 if (error != 0) { 3502 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 3503 return (error); 3504 } 3505 3506 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 3507 spa_load_failed(spa, "cannot open vdev tree after invalidating " 3508 "some vdevs"); 3509 vdev_dbgmsg_print_tree(rvd, 2); 3510 return (SET_ERROR(ENXIO)); 3511 } 3512 3513 return (0); 3514 } 3515 3516 static void 3517 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 3518 { 3519 spa->spa_state = POOL_STATE_ACTIVE; 3520 spa->spa_ubsync = spa->spa_uberblock; 3521 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 3522 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 3523 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 3524 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 3525 spa->spa_claim_max_txg = spa->spa_first_txg; 3526 spa->spa_prev_software_version = ub->ub_software_version; 3527 } 3528 3529 static int 3530 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 3531 { 3532 vdev_t *rvd = spa->spa_root_vdev; 3533 nvlist_t *label; 3534 uberblock_t *ub = &spa->spa_uberblock; 3535 boolean_t activity_check = B_FALSE; 3536 3537 /* 3538 * If we are opening the checkpointed state of the pool by 3539 * rewinding to it, at this point we will have written the 3540 * checkpointed uberblock to the vdev labels, so searching 3541 * the labels will find the right uberblock. However, if 3542 * we are opening the checkpointed state read-only, we have 3543 * not modified the labels. Therefore, we must ignore the 3544 * labels and continue using the spa_uberblock that was set 3545 * by spa_ld_checkpoint_rewind. 3546 * 3547 * Note that it would be fine to ignore the labels when 3548 * rewinding (opening writeable) as well. However, if we 3549 * crash just after writing the labels, we will end up 3550 * searching the labels. Doing so in the common case means 3551 * that this code path gets exercised normally, rather than 3552 * just in the edge case. 3553 */ 3554 if (ub->ub_checkpoint_txg != 0 && 3555 spa_importing_readonly_checkpoint(spa)) { 3556 spa_ld_select_uberblock_done(spa, ub); 3557 return (0); 3558 } 3559 3560 /* 3561 * Find the best uberblock. 3562 */ 3563 vdev_uberblock_load(rvd, ub, &label); 3564 3565 /* 3566 * If we weren't able to find a single valid uberblock, return failure. 3567 */ 3568 if (ub->ub_txg == 0) { 3569 nvlist_free(label); 3570 spa_load_failed(spa, "no valid uberblock found"); 3571 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 3572 } 3573 3574 if (spa->spa_load_max_txg != UINT64_MAX) { 3575 (void) spa_import_progress_set_max_txg(spa_guid(spa), 3576 (u_longlong_t)spa->spa_load_max_txg); 3577 } 3578 spa_load_note(spa, "using uberblock with txg=%llu", 3579 (u_longlong_t)ub->ub_txg); 3580 3581 3582 /* 3583 * For pools which have the multihost property on determine if the 3584 * pool is truly inactive and can be safely imported. Prevent 3585 * hosts which don't have a hostid set from importing the pool. 3586 */ 3587 activity_check = spa_activity_check_required(spa, ub, label, 3588 spa->spa_config); 3589 if (activity_check) { 3590 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 3591 spa_get_hostid(spa) == 0) { 3592 nvlist_free(label); 3593 fnvlist_add_uint64(spa->spa_load_info, 3594 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 3595 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 3596 } 3597 3598 int error = spa_activity_check(spa, ub, spa->spa_config); 3599 if (error) { 3600 nvlist_free(label); 3601 return (error); 3602 } 3603 3604 fnvlist_add_uint64(spa->spa_load_info, 3605 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); 3606 fnvlist_add_uint64(spa->spa_load_info, 3607 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); 3608 fnvlist_add_uint16(spa->spa_load_info, 3609 ZPOOL_CONFIG_MMP_SEQ, 3610 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)); 3611 } 3612 3613 /* 3614 * If the pool has an unsupported version we can't open it. 3615 */ 3616 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 3617 nvlist_free(label); 3618 spa_load_failed(spa, "version %llu is not supported", 3619 (u_longlong_t)ub->ub_version); 3620 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 3621 } 3622 3623 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3624 nvlist_t *features; 3625 3626 /* 3627 * If we weren't able to find what's necessary for reading the 3628 * MOS in the label, return failure. 3629 */ 3630 if (label == NULL) { 3631 spa_load_failed(spa, "label config unavailable"); 3632 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3633 ENXIO)); 3634 } 3635 3636 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 3637 &features) != 0) { 3638 nvlist_free(label); 3639 spa_load_failed(spa, "invalid label: '%s' missing", 3640 ZPOOL_CONFIG_FEATURES_FOR_READ); 3641 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3642 ENXIO)); 3643 } 3644 3645 /* 3646 * Update our in-core representation with the definitive values 3647 * from the label. 3648 */ 3649 nvlist_free(spa->spa_label_features); 3650 VERIFY(nvlist_dup(features, &spa->spa_label_features, 0) == 0); 3651 } 3652 3653 nvlist_free(label); 3654 3655 /* 3656 * Look through entries in the label nvlist's features_for_read. If 3657 * there is a feature listed there which we don't understand then we 3658 * cannot open a pool. 3659 */ 3660 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3661 nvlist_t *unsup_feat; 3662 3663 VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) == 3664 0); 3665 3666 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 3667 NULL); nvp != NULL; 3668 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 3669 if (!zfeature_is_supported(nvpair_name(nvp))) { 3670 VERIFY(nvlist_add_string(unsup_feat, 3671 nvpair_name(nvp), "") == 0); 3672 } 3673 } 3674 3675 if (!nvlist_empty(unsup_feat)) { 3676 VERIFY(nvlist_add_nvlist(spa->spa_load_info, 3677 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0); 3678 nvlist_free(unsup_feat); 3679 spa_load_failed(spa, "some features are unsupported"); 3680 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 3681 ENOTSUP)); 3682 } 3683 3684 nvlist_free(unsup_feat); 3685 } 3686 3687 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 3688 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3689 spa_try_repair(spa, spa->spa_config); 3690 spa_config_exit(spa, SCL_ALL, FTAG); 3691 nvlist_free(spa->spa_config_splitting); 3692 spa->spa_config_splitting = NULL; 3693 } 3694 3695 /* 3696 * Initialize internal SPA structures. 3697 */ 3698 spa_ld_select_uberblock_done(spa, ub); 3699 3700 return (0); 3701 } 3702 3703 static int 3704 spa_ld_open_rootbp(spa_t *spa) 3705 { 3706 int error = 0; 3707 vdev_t *rvd = spa->spa_root_vdev; 3708 3709 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 3710 if (error != 0) { 3711 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 3712 "[error=%d]", error); 3713 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3714 } 3715 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 3716 3717 return (0); 3718 } 3719 3720 static int 3721 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 3722 boolean_t reloading) 3723 { 3724 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 3725 nvlist_t *nv, *mos_config, *policy; 3726 int error = 0, copy_error; 3727 uint64_t healthy_tvds, healthy_tvds_mos; 3728 uint64_t mos_config_txg; 3729 3730 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 3731 != 0) 3732 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3733 3734 /* 3735 * If we're assembling a pool from a split, the config provided is 3736 * already trusted so there is nothing to do. 3737 */ 3738 if (type == SPA_IMPORT_ASSEMBLE) 3739 return (0); 3740 3741 healthy_tvds = spa_healthy_core_tvds(spa); 3742 3743 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 3744 != 0) { 3745 spa_load_failed(spa, "unable to retrieve MOS config"); 3746 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3747 } 3748 3749 /* 3750 * If we are doing an open, pool owner wasn't verified yet, thus do 3751 * the verification here. 3752 */ 3753 if (spa->spa_load_state == SPA_LOAD_OPEN) { 3754 error = spa_verify_host(spa, mos_config); 3755 if (error != 0) { 3756 nvlist_free(mos_config); 3757 return (error); 3758 } 3759 } 3760 3761 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 3762 3763 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3764 3765 /* 3766 * Build a new vdev tree from the trusted config 3767 */ 3768 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 3769 if (error != 0) { 3770 nvlist_free(mos_config); 3771 spa_config_exit(spa, SCL_ALL, FTAG); 3772 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 3773 error); 3774 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3775 } 3776 3777 /* 3778 * Vdev paths in the MOS may be obsolete. If the untrusted config was 3779 * obtained by scanning /dev/dsk, then it will have the right vdev 3780 * paths. We update the trusted MOS config with this information. 3781 * We first try to copy the paths with vdev_copy_path_strict, which 3782 * succeeds only when both configs have exactly the same vdev tree. 3783 * If that fails, we fall back to a more flexible method that has a 3784 * best effort policy. 3785 */ 3786 copy_error = vdev_copy_path_strict(rvd, mrvd); 3787 if (copy_error != 0 || spa_load_print_vdev_tree) { 3788 spa_load_note(spa, "provided vdev tree:"); 3789 vdev_dbgmsg_print_tree(rvd, 2); 3790 spa_load_note(spa, "MOS vdev tree:"); 3791 vdev_dbgmsg_print_tree(mrvd, 2); 3792 } 3793 if (copy_error != 0) { 3794 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 3795 "back to vdev_copy_path_relaxed"); 3796 vdev_copy_path_relaxed(rvd, mrvd); 3797 } 3798 3799 vdev_close(rvd); 3800 vdev_free(rvd); 3801 spa->spa_root_vdev = mrvd; 3802 rvd = mrvd; 3803 spa_config_exit(spa, SCL_ALL, FTAG); 3804 3805 /* 3806 * We will use spa_config if we decide to reload the spa or if spa_load 3807 * fails and we rewind. We must thus regenerate the config using the 3808 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 3809 * pass settings on how to load the pool and is not stored in the MOS. 3810 * We copy it over to our new, trusted config. 3811 */ 3812 mos_config_txg = fnvlist_lookup_uint64(mos_config, 3813 ZPOOL_CONFIG_POOL_TXG); 3814 nvlist_free(mos_config); 3815 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 3816 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 3817 &policy) == 0) 3818 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 3819 spa_config_set(spa, mos_config); 3820 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 3821 3822 /* 3823 * Now that we got the config from the MOS, we should be more strict 3824 * in checking blkptrs and can make assumptions about the consistency 3825 * of the vdev tree. spa_trust_config must be set to true before opening 3826 * vdevs in order for them to be writeable. 3827 */ 3828 spa->spa_trust_config = B_TRUE; 3829 3830 /* 3831 * Open and validate the new vdev tree 3832 */ 3833 error = spa_ld_open_vdevs(spa); 3834 if (error != 0) 3835 return (error); 3836 3837 error = spa_ld_validate_vdevs(spa); 3838 if (error != 0) 3839 return (error); 3840 3841 if (copy_error != 0 || spa_load_print_vdev_tree) { 3842 spa_load_note(spa, "final vdev tree:"); 3843 vdev_dbgmsg_print_tree(rvd, 2); 3844 } 3845 3846 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 3847 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 3848 /* 3849 * Sanity check to make sure that we are indeed loading the 3850 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 3851 * in the config provided and they happened to be the only ones 3852 * to have the latest uberblock, we could involuntarily perform 3853 * an extreme rewind. 3854 */ 3855 healthy_tvds_mos = spa_healthy_core_tvds(spa); 3856 if (healthy_tvds_mos - healthy_tvds >= 3857 SPA_SYNC_MIN_VDEVS) { 3858 spa_load_note(spa, "config provided misses too many " 3859 "top-level vdevs compared to MOS (%lld vs %lld). ", 3860 (u_longlong_t)healthy_tvds, 3861 (u_longlong_t)healthy_tvds_mos); 3862 spa_load_note(spa, "vdev tree:"); 3863 vdev_dbgmsg_print_tree(rvd, 2); 3864 if (reloading) { 3865 spa_load_failed(spa, "config was already " 3866 "provided from MOS. Aborting."); 3867 return (spa_vdev_err(rvd, 3868 VDEV_AUX_CORRUPT_DATA, EIO)); 3869 } 3870 spa_load_note(spa, "spa must be reloaded using MOS " 3871 "config"); 3872 return (SET_ERROR(EAGAIN)); 3873 } 3874 } 3875 3876 error = spa_check_for_missing_logs(spa); 3877 if (error != 0) 3878 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 3879 3880 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 3881 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 3882 "guid sum (%llu != %llu)", 3883 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 3884 (u_longlong_t)rvd->vdev_guid_sum); 3885 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 3886 ENXIO)); 3887 } 3888 3889 return (0); 3890 } 3891 3892 static int 3893 spa_ld_open_indirect_vdev_metadata(spa_t *spa) 3894 { 3895 int error = 0; 3896 vdev_t *rvd = spa->spa_root_vdev; 3897 3898 /* 3899 * Everything that we read before spa_remove_init() must be stored 3900 * on concreted vdevs. Therefore we do this as early as possible. 3901 */ 3902 error = spa_remove_init(spa); 3903 if (error != 0) { 3904 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 3905 error); 3906 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3907 } 3908 3909 /* 3910 * Retrieve information needed to condense indirect vdev mappings. 3911 */ 3912 error = spa_condense_init(spa); 3913 if (error != 0) { 3914 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 3915 error); 3916 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3917 } 3918 3919 return (0); 3920 } 3921 3922 static int 3923 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 3924 { 3925 int error = 0; 3926 vdev_t *rvd = spa->spa_root_vdev; 3927 3928 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 3929 boolean_t missing_feat_read = B_FALSE; 3930 nvlist_t *unsup_feat, *enabled_feat; 3931 3932 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 3933 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 3934 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3935 } 3936 3937 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 3938 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 3939 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3940 } 3941 3942 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 3943 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 3944 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3945 } 3946 3947 enabled_feat = fnvlist_alloc(); 3948 unsup_feat = fnvlist_alloc(); 3949 3950 if (!spa_features_check(spa, B_FALSE, 3951 unsup_feat, enabled_feat)) 3952 missing_feat_read = B_TRUE; 3953 3954 if (spa_writeable(spa) || 3955 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 3956 if (!spa_features_check(spa, B_TRUE, 3957 unsup_feat, enabled_feat)) { 3958 *missing_feat_writep = B_TRUE; 3959 } 3960 } 3961 3962 fnvlist_add_nvlist(spa->spa_load_info, 3963 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 3964 3965 if (!nvlist_empty(unsup_feat)) { 3966 fnvlist_add_nvlist(spa->spa_load_info, 3967 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 3968 } 3969 3970 fnvlist_free(enabled_feat); 3971 fnvlist_free(unsup_feat); 3972 3973 if (!missing_feat_read) { 3974 fnvlist_add_boolean(spa->spa_load_info, 3975 ZPOOL_CONFIG_CAN_RDONLY); 3976 } 3977 3978 /* 3979 * If the state is SPA_LOAD_TRYIMPORT, our objective is 3980 * twofold: to determine whether the pool is available for 3981 * import in read-write mode and (if it is not) whether the 3982 * pool is available for import in read-only mode. If the pool 3983 * is available for import in read-write mode, it is displayed 3984 * as available in userland; if it is not available for import 3985 * in read-only mode, it is displayed as unavailable in 3986 * userland. If the pool is available for import in read-only 3987 * mode but not read-write mode, it is displayed as unavailable 3988 * in userland with a special note that the pool is actually 3989 * available for open in read-only mode. 3990 * 3991 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 3992 * missing a feature for write, we must first determine whether 3993 * the pool can be opened read-only before returning to 3994 * userland in order to know whether to display the 3995 * abovementioned note. 3996 */ 3997 if (missing_feat_read || (*missing_feat_writep && 3998 spa_writeable(spa))) { 3999 spa_load_failed(spa, "pool uses unsupported features"); 4000 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 4001 ENOTSUP)); 4002 } 4003 4004 /* 4005 * Load refcounts for ZFS features from disk into an in-memory 4006 * cache during SPA initialization. 4007 */ 4008 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 4009 uint64_t refcount; 4010 4011 error = feature_get_refcount_from_disk(spa, 4012 &spa_feature_table[i], &refcount); 4013 if (error == 0) { 4014 spa->spa_feat_refcount_cache[i] = refcount; 4015 } else if (error == ENOTSUP) { 4016 spa->spa_feat_refcount_cache[i] = 4017 SPA_FEATURE_DISABLED; 4018 } else { 4019 spa_load_failed(spa, "error getting refcount " 4020 "for feature %s [error=%d]", 4021 spa_feature_table[i].fi_guid, error); 4022 return (spa_vdev_err(rvd, 4023 VDEV_AUX_CORRUPT_DATA, EIO)); 4024 } 4025 } 4026 } 4027 4028 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 4029 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 4030 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 4031 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4032 } 4033 4034 /* 4035 * Encryption was added before bookmark_v2, even though bookmark_v2 4036 * is now a dependency. If this pool has encryption enabled without 4037 * bookmark_v2, trigger an errata message. 4038 */ 4039 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 4040 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 4041 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 4042 } 4043 4044 return (0); 4045 } 4046 4047 static int 4048 spa_ld_load_special_directories(spa_t *spa) 4049 { 4050 int error = 0; 4051 vdev_t *rvd = spa->spa_root_vdev; 4052 4053 spa->spa_is_initializing = B_TRUE; 4054 error = dsl_pool_open(spa->spa_dsl_pool); 4055 spa->spa_is_initializing = B_FALSE; 4056 if (error != 0) { 4057 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 4058 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4059 } 4060 4061 return (0); 4062 } 4063 4064 static int 4065 spa_ld_get_props(spa_t *spa) 4066 { 4067 int error = 0; 4068 uint64_t obj; 4069 vdev_t *rvd = spa->spa_root_vdev; 4070 4071 /* Grab the checksum salt from the MOS. */ 4072 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4073 DMU_POOL_CHECKSUM_SALT, 1, 4074 sizeof (spa->spa_cksum_salt.zcs_bytes), 4075 spa->spa_cksum_salt.zcs_bytes); 4076 if (error == ENOENT) { 4077 /* Generate a new salt for subsequent use */ 4078 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 4079 sizeof (spa->spa_cksum_salt.zcs_bytes)); 4080 } else if (error != 0) { 4081 spa_load_failed(spa, "unable to retrieve checksum salt from " 4082 "MOS [error=%d]", error); 4083 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4084 } 4085 4086 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 4087 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4088 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 4089 if (error != 0) { 4090 spa_load_failed(spa, "error opening deferred-frees bpobj " 4091 "[error=%d]", error); 4092 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4093 } 4094 4095 /* 4096 * Load the bit that tells us to use the new accounting function 4097 * (raid-z deflation). If we have an older pool, this will not 4098 * be present. 4099 */ 4100 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 4101 if (error != 0 && error != ENOENT) 4102 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4103 4104 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 4105 &spa->spa_creation_version, B_FALSE); 4106 if (error != 0 && error != ENOENT) 4107 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4108 4109 /* 4110 * Load the persistent error log. If we have an older pool, this will 4111 * not be present. 4112 */ 4113 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 4114 B_FALSE); 4115 if (error != 0 && error != ENOENT) 4116 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4117 4118 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 4119 &spa->spa_errlog_scrub, B_FALSE); 4120 if (error != 0 && error != ENOENT) 4121 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4122 4123 /* 4124 * Load the livelist deletion field. If a livelist is queued for 4125 * deletion, indicate that in the spa 4126 */ 4127 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 4128 &spa->spa_livelists_to_delete, B_FALSE); 4129 if (error != 0 && error != ENOENT) 4130 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4131 4132 /* 4133 * Load the history object. If we have an older pool, this 4134 * will not be present. 4135 */ 4136 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 4137 if (error != 0 && error != ENOENT) 4138 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4139 4140 /* 4141 * Load the per-vdev ZAP map. If we have an older pool, this will not 4142 * be present; in this case, defer its creation to a later time to 4143 * avoid dirtying the MOS this early / out of sync context. See 4144 * spa_sync_config_object. 4145 */ 4146 4147 /* The sentinel is only available in the MOS config. */ 4148 nvlist_t *mos_config; 4149 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 4150 spa_load_failed(spa, "unable to retrieve MOS config"); 4151 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4152 } 4153 4154 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 4155 &spa->spa_all_vdev_zaps, B_FALSE); 4156 4157 if (error == ENOENT) { 4158 VERIFY(!nvlist_exists(mos_config, 4159 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 4160 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 4161 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4162 } else if (error != 0) { 4163 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4164 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 4165 /* 4166 * An older version of ZFS overwrote the sentinel value, so 4167 * we have orphaned per-vdev ZAPs in the MOS. Defer their 4168 * destruction to later; see spa_sync_config_object. 4169 */ 4170 spa->spa_avz_action = AVZ_ACTION_DESTROY; 4171 /* 4172 * We're assuming that no vdevs have had their ZAPs created 4173 * before this. Better be sure of it. 4174 */ 4175 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4176 } 4177 nvlist_free(mos_config); 4178 4179 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 4180 4181 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 4182 B_FALSE); 4183 if (error && error != ENOENT) 4184 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4185 4186 if (error == 0) { 4187 uint64_t autoreplace = 0; 4188 4189 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 4190 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 4191 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 4192 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 4193 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 4194 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 4195 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 4196 spa->spa_autoreplace = (autoreplace != 0); 4197 } 4198 4199 /* 4200 * If we are importing a pool with missing top-level vdevs, 4201 * we enforce that the pool doesn't panic or get suspended on 4202 * error since the likelihood of missing data is extremely high. 4203 */ 4204 if (spa->spa_missing_tvds > 0 && 4205 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 4206 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4207 spa_load_note(spa, "forcing failmode to 'continue' " 4208 "as some top level vdevs are missing"); 4209 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 4210 } 4211 4212 return (0); 4213 } 4214 4215 static int 4216 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 4217 { 4218 int error = 0; 4219 vdev_t *rvd = spa->spa_root_vdev; 4220 4221 /* 4222 * If we're assembling the pool from the split-off vdevs of 4223 * an existing pool, we don't want to attach the spares & cache 4224 * devices. 4225 */ 4226 4227 /* 4228 * Load any hot spares for this pool. 4229 */ 4230 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 4231 B_FALSE); 4232 if (error != 0 && error != ENOENT) 4233 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4234 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4235 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 4236 if (load_nvlist(spa, spa->spa_spares.sav_object, 4237 &spa->spa_spares.sav_config) != 0) { 4238 spa_load_failed(spa, "error loading spares nvlist"); 4239 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4240 } 4241 4242 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4243 spa_load_spares(spa); 4244 spa_config_exit(spa, SCL_ALL, FTAG); 4245 } else if (error == 0) { 4246 spa->spa_spares.sav_sync = B_TRUE; 4247 } 4248 4249 /* 4250 * Load any level 2 ARC devices for this pool. 4251 */ 4252 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 4253 &spa->spa_l2cache.sav_object, B_FALSE); 4254 if (error != 0 && error != ENOENT) 4255 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4256 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4257 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 4258 if (load_nvlist(spa, spa->spa_l2cache.sav_object, 4259 &spa->spa_l2cache.sav_config) != 0) { 4260 spa_load_failed(spa, "error loading l2cache nvlist"); 4261 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4262 } 4263 4264 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4265 spa_load_l2cache(spa); 4266 spa_config_exit(spa, SCL_ALL, FTAG); 4267 } else if (error == 0) { 4268 spa->spa_l2cache.sav_sync = B_TRUE; 4269 } 4270 4271 return (0); 4272 } 4273 4274 static int 4275 spa_ld_load_vdev_metadata(spa_t *spa) 4276 { 4277 int error = 0; 4278 vdev_t *rvd = spa->spa_root_vdev; 4279 4280 /* 4281 * If the 'multihost' property is set, then never allow a pool to 4282 * be imported when the system hostid is zero. The exception to 4283 * this rule is zdb which is always allowed to access pools. 4284 */ 4285 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 4286 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 4287 fnvlist_add_uint64(spa->spa_load_info, 4288 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4289 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4290 } 4291 4292 /* 4293 * If the 'autoreplace' property is set, then post a resource notifying 4294 * the ZFS DE that it should not issue any faults for unopenable 4295 * devices. We also iterate over the vdevs, and post a sysevent for any 4296 * unopenable vdevs so that the normal autoreplace handler can take 4297 * over. 4298 */ 4299 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4300 spa_check_removed(spa->spa_root_vdev); 4301 /* 4302 * For the import case, this is done in spa_import(), because 4303 * at this point we're using the spare definitions from 4304 * the MOS config, not necessarily from the userland config. 4305 */ 4306 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 4307 spa_aux_check_removed(&spa->spa_spares); 4308 spa_aux_check_removed(&spa->spa_l2cache); 4309 } 4310 } 4311 4312 /* 4313 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 4314 */ 4315 error = vdev_load(rvd); 4316 if (error != 0) { 4317 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 4318 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4319 } 4320 4321 error = spa_ld_log_spacemaps(spa); 4322 if (error != 0) { 4323 spa_load_failed(spa, "spa_ld_log_sm_data failed [error=%d]", 4324 error); 4325 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4326 } 4327 4328 /* 4329 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 4330 */ 4331 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4332 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 4333 spa_config_exit(spa, SCL_ALL, FTAG); 4334 4335 return (0); 4336 } 4337 4338 static int 4339 spa_ld_load_dedup_tables(spa_t *spa) 4340 { 4341 int error = 0; 4342 vdev_t *rvd = spa->spa_root_vdev; 4343 4344 error = ddt_load(spa); 4345 if (error != 0) { 4346 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 4347 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4348 } 4349 4350 return (0); 4351 } 4352 4353 static int 4354 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, char **ereport) 4355 { 4356 vdev_t *rvd = spa->spa_root_vdev; 4357 4358 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 4359 boolean_t missing = spa_check_logs(spa); 4360 if (missing) { 4361 if (spa->spa_missing_tvds != 0) { 4362 spa_load_note(spa, "spa_check_logs failed " 4363 "so dropping the logs"); 4364 } else { 4365 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 4366 spa_load_failed(spa, "spa_check_logs failed"); 4367 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 4368 ENXIO)); 4369 } 4370 } 4371 } 4372 4373 return (0); 4374 } 4375 4376 static int 4377 spa_ld_verify_pool_data(spa_t *spa) 4378 { 4379 int error = 0; 4380 vdev_t *rvd = spa->spa_root_vdev; 4381 4382 /* 4383 * We've successfully opened the pool, verify that we're ready 4384 * to start pushing transactions. 4385 */ 4386 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4387 error = spa_load_verify(spa); 4388 if (error != 0) { 4389 spa_load_failed(spa, "spa_load_verify failed " 4390 "[error=%d]", error); 4391 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4392 error)); 4393 } 4394 } 4395 4396 return (0); 4397 } 4398 4399 static void 4400 spa_ld_claim_log_blocks(spa_t *spa) 4401 { 4402 dmu_tx_t *tx; 4403 dsl_pool_t *dp = spa_get_dsl(spa); 4404 4405 /* 4406 * Claim log blocks that haven't been committed yet. 4407 * This must all happen in a single txg. 4408 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 4409 * invoked from zil_claim_log_block()'s i/o done callback. 4410 * Price of rollback is that we abandon the log. 4411 */ 4412 spa->spa_claiming = B_TRUE; 4413 4414 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 4415 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 4416 zil_claim, tx, DS_FIND_CHILDREN); 4417 dmu_tx_commit(tx); 4418 4419 spa->spa_claiming = B_FALSE; 4420 4421 spa_set_log_state(spa, SPA_LOG_GOOD); 4422 } 4423 4424 static void 4425 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 4426 boolean_t update_config_cache) 4427 { 4428 vdev_t *rvd = spa->spa_root_vdev; 4429 int need_update = B_FALSE; 4430 4431 /* 4432 * If the config cache is stale, or we have uninitialized 4433 * metaslabs (see spa_vdev_add()), then update the config. 4434 * 4435 * If this is a verbatim import, trust the current 4436 * in-core spa_config and update the disk labels. 4437 */ 4438 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 4439 spa->spa_load_state == SPA_LOAD_IMPORT || 4440 spa->spa_load_state == SPA_LOAD_RECOVER || 4441 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 4442 need_update = B_TRUE; 4443 4444 for (int c = 0; c < rvd->vdev_children; c++) 4445 if (rvd->vdev_child[c]->vdev_ms_array == 0) 4446 need_update = B_TRUE; 4447 4448 /* 4449 * Update the config cache asynchronously in case we're the 4450 * root pool, in which case the config cache isn't writable yet. 4451 */ 4452 if (need_update) 4453 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 4454 } 4455 4456 static void 4457 spa_ld_prepare_for_reload(spa_t *spa) 4458 { 4459 spa_mode_t mode = spa->spa_mode; 4460 int async_suspended = spa->spa_async_suspended; 4461 4462 spa_unload(spa); 4463 spa_deactivate(spa); 4464 spa_activate(spa, mode); 4465 4466 /* 4467 * We save the value of spa_async_suspended as it gets reset to 0 by 4468 * spa_unload(). We want to restore it back to the original value before 4469 * returning as we might be calling spa_async_resume() later. 4470 */ 4471 spa->spa_async_suspended = async_suspended; 4472 } 4473 4474 static int 4475 spa_ld_read_checkpoint_txg(spa_t *spa) 4476 { 4477 uberblock_t checkpoint; 4478 int error = 0; 4479 4480 ASSERT0(spa->spa_checkpoint_txg); 4481 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4482 4483 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4484 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4485 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4486 4487 if (error == ENOENT) 4488 return (0); 4489 4490 if (error != 0) 4491 return (error); 4492 4493 ASSERT3U(checkpoint.ub_txg, !=, 0); 4494 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 4495 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 4496 spa->spa_checkpoint_txg = checkpoint.ub_txg; 4497 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 4498 4499 return (0); 4500 } 4501 4502 static int 4503 spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 4504 { 4505 int error = 0; 4506 4507 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4508 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4509 4510 /* 4511 * Never trust the config that is provided unless we are assembling 4512 * a pool following a split. 4513 * This means don't trust blkptrs and the vdev tree in general. This 4514 * also effectively puts the spa in read-only mode since 4515 * spa_writeable() checks for spa_trust_config to be true. 4516 * We will later load a trusted config from the MOS. 4517 */ 4518 if (type != SPA_IMPORT_ASSEMBLE) 4519 spa->spa_trust_config = B_FALSE; 4520 4521 /* 4522 * Parse the config provided to create a vdev tree. 4523 */ 4524 error = spa_ld_parse_config(spa, type); 4525 if (error != 0) 4526 return (error); 4527 4528 spa_import_progress_add(spa); 4529 4530 /* 4531 * Now that we have the vdev tree, try to open each vdev. This involves 4532 * opening the underlying physical device, retrieving its geometry and 4533 * probing the vdev with a dummy I/O. The state of each vdev will be set 4534 * based on the success of those operations. After this we'll be ready 4535 * to read from the vdevs. 4536 */ 4537 error = spa_ld_open_vdevs(spa); 4538 if (error != 0) 4539 return (error); 4540 4541 /* 4542 * Read the label of each vdev and make sure that the GUIDs stored 4543 * there match the GUIDs in the config provided. 4544 * If we're assembling a new pool that's been split off from an 4545 * existing pool, the labels haven't yet been updated so we skip 4546 * validation for now. 4547 */ 4548 if (type != SPA_IMPORT_ASSEMBLE) { 4549 error = spa_ld_validate_vdevs(spa); 4550 if (error != 0) 4551 return (error); 4552 } 4553 4554 /* 4555 * Read all vdev labels to find the best uberblock (i.e. latest, 4556 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 4557 * get the list of features required to read blkptrs in the MOS from 4558 * the vdev label with the best uberblock and verify that our version 4559 * of zfs supports them all. 4560 */ 4561 error = spa_ld_select_uberblock(spa, type); 4562 if (error != 0) 4563 return (error); 4564 4565 /* 4566 * Pass that uberblock to the dsl_pool layer which will open the root 4567 * blkptr. This blkptr points to the latest version of the MOS and will 4568 * allow us to read its contents. 4569 */ 4570 error = spa_ld_open_rootbp(spa); 4571 if (error != 0) 4572 return (error); 4573 4574 return (0); 4575 } 4576 4577 static int 4578 spa_ld_checkpoint_rewind(spa_t *spa) 4579 { 4580 uberblock_t checkpoint; 4581 int error = 0; 4582 4583 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4584 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4585 4586 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4587 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4588 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4589 4590 if (error != 0) { 4591 spa_load_failed(spa, "unable to retrieve checkpointed " 4592 "uberblock from the MOS config [error=%d]", error); 4593 4594 if (error == ENOENT) 4595 error = ZFS_ERR_NO_CHECKPOINT; 4596 4597 return (error); 4598 } 4599 4600 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 4601 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 4602 4603 /* 4604 * We need to update the txg and timestamp of the checkpointed 4605 * uberblock to be higher than the latest one. This ensures that 4606 * the checkpointed uberblock is selected if we were to close and 4607 * reopen the pool right after we've written it in the vdev labels. 4608 * (also see block comment in vdev_uberblock_compare) 4609 */ 4610 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 4611 checkpoint.ub_timestamp = gethrestime_sec(); 4612 4613 /* 4614 * Set current uberblock to be the checkpointed uberblock. 4615 */ 4616 spa->spa_uberblock = checkpoint; 4617 4618 /* 4619 * If we are doing a normal rewind, then the pool is open for 4620 * writing and we sync the "updated" checkpointed uberblock to 4621 * disk. Once this is done, we've basically rewound the whole 4622 * pool and there is no way back. 4623 * 4624 * There are cases when we don't want to attempt and sync the 4625 * checkpointed uberblock to disk because we are opening a 4626 * pool as read-only. Specifically, verifying the checkpointed 4627 * state with zdb, and importing the checkpointed state to get 4628 * a "preview" of its content. 4629 */ 4630 if (spa_writeable(spa)) { 4631 vdev_t *rvd = spa->spa_root_vdev; 4632 4633 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4634 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 4635 int svdcount = 0; 4636 int children = rvd->vdev_children; 4637 int c0 = random_in_range(children); 4638 4639 for (int c = 0; c < children; c++) { 4640 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 4641 4642 /* Stop when revisiting the first vdev */ 4643 if (c > 0 && svd[0] == vd) 4644 break; 4645 4646 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 4647 !vdev_is_concrete(vd)) 4648 continue; 4649 4650 svd[svdcount++] = vd; 4651 if (svdcount == SPA_SYNC_MIN_VDEVS) 4652 break; 4653 } 4654 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg); 4655 if (error == 0) 4656 spa->spa_last_synced_guid = rvd->vdev_guid; 4657 spa_config_exit(spa, SCL_ALL, FTAG); 4658 4659 if (error != 0) { 4660 spa_load_failed(spa, "failed to write checkpointed " 4661 "uberblock to the vdev labels [error=%d]", error); 4662 return (error); 4663 } 4664 } 4665 4666 return (0); 4667 } 4668 4669 static int 4670 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 4671 boolean_t *update_config_cache) 4672 { 4673 int error; 4674 4675 /* 4676 * Parse the config for pool, open and validate vdevs, 4677 * select an uberblock, and use that uberblock to open 4678 * the MOS. 4679 */ 4680 error = spa_ld_mos_init(spa, type); 4681 if (error != 0) 4682 return (error); 4683 4684 /* 4685 * Retrieve the trusted config stored in the MOS and use it to create 4686 * a new, exact version of the vdev tree, then reopen all vdevs. 4687 */ 4688 error = spa_ld_trusted_config(spa, type, B_FALSE); 4689 if (error == EAGAIN) { 4690 if (update_config_cache != NULL) 4691 *update_config_cache = B_TRUE; 4692 4693 /* 4694 * Redo the loading process with the trusted config if it is 4695 * too different from the untrusted config. 4696 */ 4697 spa_ld_prepare_for_reload(spa); 4698 spa_load_note(spa, "RELOADING"); 4699 error = spa_ld_mos_init(spa, type); 4700 if (error != 0) 4701 return (error); 4702 4703 error = spa_ld_trusted_config(spa, type, B_TRUE); 4704 if (error != 0) 4705 return (error); 4706 4707 } else if (error != 0) { 4708 return (error); 4709 } 4710 4711 return (0); 4712 } 4713 4714 /* 4715 * Load an existing storage pool, using the config provided. This config 4716 * describes which vdevs are part of the pool and is later validated against 4717 * partial configs present in each vdev's label and an entire copy of the 4718 * config stored in the MOS. 4719 */ 4720 static int 4721 spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport) 4722 { 4723 int error = 0; 4724 boolean_t missing_feat_write = B_FALSE; 4725 boolean_t checkpoint_rewind = 4726 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4727 boolean_t update_config_cache = B_FALSE; 4728 4729 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4730 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4731 4732 spa_load_note(spa, "LOADING"); 4733 4734 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 4735 if (error != 0) 4736 return (error); 4737 4738 /* 4739 * If we are rewinding to the checkpoint then we need to repeat 4740 * everything we've done so far in this function but this time 4741 * selecting the checkpointed uberblock and using that to open 4742 * the MOS. 4743 */ 4744 if (checkpoint_rewind) { 4745 /* 4746 * If we are rewinding to the checkpoint update config cache 4747 * anyway. 4748 */ 4749 update_config_cache = B_TRUE; 4750 4751 /* 4752 * Extract the checkpointed uberblock from the current MOS 4753 * and use this as the pool's uberblock from now on. If the 4754 * pool is imported as writeable we also write the checkpoint 4755 * uberblock to the labels, making the rewind permanent. 4756 */ 4757 error = spa_ld_checkpoint_rewind(spa); 4758 if (error != 0) 4759 return (error); 4760 4761 /* 4762 * Redo the loading process again with the 4763 * checkpointed uberblock. 4764 */ 4765 spa_ld_prepare_for_reload(spa); 4766 spa_load_note(spa, "LOADING checkpointed uberblock"); 4767 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 4768 if (error != 0) 4769 return (error); 4770 } 4771 4772 /* 4773 * Retrieve the checkpoint txg if the pool has a checkpoint. 4774 */ 4775 error = spa_ld_read_checkpoint_txg(spa); 4776 if (error != 0) 4777 return (error); 4778 4779 /* 4780 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 4781 * from the pool and their contents were re-mapped to other vdevs. Note 4782 * that everything that we read before this step must have been 4783 * rewritten on concrete vdevs after the last device removal was 4784 * initiated. Otherwise we could be reading from indirect vdevs before 4785 * we have loaded their mappings. 4786 */ 4787 error = spa_ld_open_indirect_vdev_metadata(spa); 4788 if (error != 0) 4789 return (error); 4790 4791 /* 4792 * Retrieve the full list of active features from the MOS and check if 4793 * they are all supported. 4794 */ 4795 error = spa_ld_check_features(spa, &missing_feat_write); 4796 if (error != 0) 4797 return (error); 4798 4799 /* 4800 * Load several special directories from the MOS needed by the dsl_pool 4801 * layer. 4802 */ 4803 error = spa_ld_load_special_directories(spa); 4804 if (error != 0) 4805 return (error); 4806 4807 /* 4808 * Retrieve pool properties from the MOS. 4809 */ 4810 error = spa_ld_get_props(spa); 4811 if (error != 0) 4812 return (error); 4813 4814 /* 4815 * Retrieve the list of auxiliary devices - cache devices and spares - 4816 * and open them. 4817 */ 4818 error = spa_ld_open_aux_vdevs(spa, type); 4819 if (error != 0) 4820 return (error); 4821 4822 /* 4823 * Load the metadata for all vdevs. Also check if unopenable devices 4824 * should be autoreplaced. 4825 */ 4826 error = spa_ld_load_vdev_metadata(spa); 4827 if (error != 0) 4828 return (error); 4829 4830 error = spa_ld_load_dedup_tables(spa); 4831 if (error != 0) 4832 return (error); 4833 4834 /* 4835 * Verify the logs now to make sure we don't have any unexpected errors 4836 * when we claim log blocks later. 4837 */ 4838 error = spa_ld_verify_logs(spa, type, ereport); 4839 if (error != 0) 4840 return (error); 4841 4842 if (missing_feat_write) { 4843 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 4844 4845 /* 4846 * At this point, we know that we can open the pool in 4847 * read-only mode but not read-write mode. We now have enough 4848 * information and can return to userland. 4849 */ 4850 return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 4851 ENOTSUP)); 4852 } 4853 4854 /* 4855 * Traverse the last txgs to make sure the pool was left off in a safe 4856 * state. When performing an extreme rewind, we verify the whole pool, 4857 * which can take a very long time. 4858 */ 4859 error = spa_ld_verify_pool_data(spa); 4860 if (error != 0) 4861 return (error); 4862 4863 /* 4864 * Calculate the deflated space for the pool. This must be done before 4865 * we write anything to the pool because we'd need to update the space 4866 * accounting using the deflated sizes. 4867 */ 4868 spa_update_dspace(spa); 4869 4870 /* 4871 * We have now retrieved all the information we needed to open the 4872 * pool. If we are importing the pool in read-write mode, a few 4873 * additional steps must be performed to finish the import. 4874 */ 4875 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 4876 spa->spa_load_max_txg == UINT64_MAX)) { 4877 uint64_t config_cache_txg = spa->spa_config_txg; 4878 4879 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 4880 4881 /* 4882 * In case of a checkpoint rewind, log the original txg 4883 * of the checkpointed uberblock. 4884 */ 4885 if (checkpoint_rewind) { 4886 spa_history_log_internal(spa, "checkpoint rewind", 4887 NULL, "rewound state to txg=%llu", 4888 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 4889 } 4890 4891 /* 4892 * Traverse the ZIL and claim all blocks. 4893 */ 4894 spa_ld_claim_log_blocks(spa); 4895 4896 /* 4897 * Kick-off the syncing thread. 4898 */ 4899 spa->spa_sync_on = B_TRUE; 4900 txg_sync_start(spa->spa_dsl_pool); 4901 mmp_thread_start(spa); 4902 4903 /* 4904 * Wait for all claims to sync. We sync up to the highest 4905 * claimed log block birth time so that claimed log blocks 4906 * don't appear to be from the future. spa_claim_max_txg 4907 * will have been set for us by ZIL traversal operations 4908 * performed above. 4909 */ 4910 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 4911 4912 /* 4913 * Check if we need to request an update of the config. On the 4914 * next sync, we would update the config stored in vdev labels 4915 * and the cachefile (by default /etc/zfs/zpool.cache). 4916 */ 4917 spa_ld_check_for_config_update(spa, config_cache_txg, 4918 update_config_cache); 4919 4920 /* 4921 * Check if a rebuild was in progress and if so resume it. 4922 * Then check all DTLs to see if anything needs resilvering. 4923 * The resilver will be deferred if a rebuild was started. 4924 */ 4925 if (vdev_rebuild_active(spa->spa_root_vdev)) { 4926 vdev_rebuild_restart(spa); 4927 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 4928 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 4929 spa_async_request(spa, SPA_ASYNC_RESILVER); 4930 } 4931 4932 /* 4933 * Log the fact that we booted up (so that we can detect if 4934 * we rebooted in the middle of an operation). 4935 */ 4936 spa_history_log_version(spa, "open", NULL); 4937 4938 spa_restart_removal(spa); 4939 spa_spawn_aux_threads(spa); 4940 4941 /* 4942 * Delete any inconsistent datasets. 4943 * 4944 * Note: 4945 * Since we may be issuing deletes for clones here, 4946 * we make sure to do so after we've spawned all the 4947 * auxiliary threads above (from which the livelist 4948 * deletion zthr is part of). 4949 */ 4950 (void) dmu_objset_find(spa_name(spa), 4951 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 4952 4953 /* 4954 * Clean up any stale temporary dataset userrefs. 4955 */ 4956 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 4957 4958 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 4959 vdev_initialize_restart(spa->spa_root_vdev); 4960 vdev_trim_restart(spa->spa_root_vdev); 4961 vdev_autotrim_restart(spa); 4962 spa_config_exit(spa, SCL_CONFIG, FTAG); 4963 } 4964 4965 spa_import_progress_remove(spa_guid(spa)); 4966 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 4967 4968 spa_load_note(spa, "LOADED"); 4969 4970 return (0); 4971 } 4972 4973 static int 4974 spa_load_retry(spa_t *spa, spa_load_state_t state) 4975 { 4976 spa_mode_t mode = spa->spa_mode; 4977 4978 spa_unload(spa); 4979 spa_deactivate(spa); 4980 4981 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 4982 4983 spa_activate(spa, mode); 4984 spa_async_suspend(spa); 4985 4986 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 4987 (u_longlong_t)spa->spa_load_max_txg); 4988 4989 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 4990 } 4991 4992 /* 4993 * If spa_load() fails this function will try loading prior txg's. If 4994 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 4995 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 4996 * function will not rewind the pool and will return the same error as 4997 * spa_load(). 4998 */ 4999 static int 5000 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 5001 int rewind_flags) 5002 { 5003 nvlist_t *loadinfo = NULL; 5004 nvlist_t *config = NULL; 5005 int load_error, rewind_error; 5006 uint64_t safe_rewind_txg; 5007 uint64_t min_txg; 5008 5009 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 5010 spa->spa_load_max_txg = spa->spa_load_txg; 5011 spa_set_log_state(spa, SPA_LOG_CLEAR); 5012 } else { 5013 spa->spa_load_max_txg = max_request; 5014 if (max_request != UINT64_MAX) 5015 spa->spa_extreme_rewind = B_TRUE; 5016 } 5017 5018 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 5019 if (load_error == 0) 5020 return (0); 5021 if (load_error == ZFS_ERR_NO_CHECKPOINT) { 5022 /* 5023 * When attempting checkpoint-rewind on a pool with no 5024 * checkpoint, we should not attempt to load uberblocks 5025 * from previous txgs when spa_load fails. 5026 */ 5027 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5028 spa_import_progress_remove(spa_guid(spa)); 5029 return (load_error); 5030 } 5031 5032 if (spa->spa_root_vdev != NULL) 5033 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5034 5035 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 5036 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 5037 5038 if (rewind_flags & ZPOOL_NEVER_REWIND) { 5039 nvlist_free(config); 5040 spa_import_progress_remove(spa_guid(spa)); 5041 return (load_error); 5042 } 5043 5044 if (state == SPA_LOAD_RECOVER) { 5045 /* Price of rolling back is discarding txgs, including log */ 5046 spa_set_log_state(spa, SPA_LOG_CLEAR); 5047 } else { 5048 /* 5049 * If we aren't rolling back save the load info from our first 5050 * import attempt so that we can restore it after attempting 5051 * to rewind. 5052 */ 5053 loadinfo = spa->spa_load_info; 5054 spa->spa_load_info = fnvlist_alloc(); 5055 } 5056 5057 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 5058 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 5059 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 5060 TXG_INITIAL : safe_rewind_txg; 5061 5062 /* 5063 * Continue as long as we're finding errors, we're still within 5064 * the acceptable rewind range, and we're still finding uberblocks 5065 */ 5066 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 5067 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 5068 if (spa->spa_load_max_txg < safe_rewind_txg) 5069 spa->spa_extreme_rewind = B_TRUE; 5070 rewind_error = spa_load_retry(spa, state); 5071 } 5072 5073 spa->spa_extreme_rewind = B_FALSE; 5074 spa->spa_load_max_txg = UINT64_MAX; 5075 5076 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 5077 spa_config_set(spa, config); 5078 else 5079 nvlist_free(config); 5080 5081 if (state == SPA_LOAD_RECOVER) { 5082 ASSERT3P(loadinfo, ==, NULL); 5083 spa_import_progress_remove(spa_guid(spa)); 5084 return (rewind_error); 5085 } else { 5086 /* Store the rewind info as part of the initial load info */ 5087 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 5088 spa->spa_load_info); 5089 5090 /* Restore the initial load info */ 5091 fnvlist_free(spa->spa_load_info); 5092 spa->spa_load_info = loadinfo; 5093 5094 spa_import_progress_remove(spa_guid(spa)); 5095 return (load_error); 5096 } 5097 } 5098 5099 /* 5100 * Pool Open/Import 5101 * 5102 * The import case is identical to an open except that the configuration is sent 5103 * down from userland, instead of grabbed from the configuration cache. For the 5104 * case of an open, the pool configuration will exist in the 5105 * POOL_STATE_UNINITIALIZED state. 5106 * 5107 * The stats information (gen/count/ustats) is used to gather vdev statistics at 5108 * the same time open the pool, without having to keep around the spa_t in some 5109 * ambiguous state. 5110 */ 5111 static int 5112 spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, 5113 nvlist_t **config) 5114 { 5115 spa_t *spa; 5116 spa_load_state_t state = SPA_LOAD_OPEN; 5117 int error; 5118 int locked = B_FALSE; 5119 int firstopen = B_FALSE; 5120 5121 *spapp = NULL; 5122 5123 /* 5124 * As disgusting as this is, we need to support recursive calls to this 5125 * function because dsl_dir_open() is called during spa_load(), and ends 5126 * up calling spa_open() again. The real fix is to figure out how to 5127 * avoid dsl_dir_open() calling this in the first place. 5128 */ 5129 if (MUTEX_NOT_HELD(&spa_namespace_lock)) { 5130 mutex_enter(&spa_namespace_lock); 5131 locked = B_TRUE; 5132 } 5133 5134 if ((spa = spa_lookup(pool)) == NULL) { 5135 if (locked) 5136 mutex_exit(&spa_namespace_lock); 5137 return (SET_ERROR(ENOENT)); 5138 } 5139 5140 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 5141 zpool_load_policy_t policy; 5142 5143 firstopen = B_TRUE; 5144 5145 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 5146 &policy); 5147 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 5148 state = SPA_LOAD_RECOVER; 5149 5150 spa_activate(spa, spa_mode_global); 5151 5152 if (state != SPA_LOAD_RECOVER) 5153 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 5154 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 5155 5156 zfs_dbgmsg("spa_open_common: opening %s", pool); 5157 error = spa_load_best(spa, state, policy.zlp_txg, 5158 policy.zlp_rewind); 5159 5160 if (error == EBADF) { 5161 /* 5162 * If vdev_validate() returns failure (indicated by 5163 * EBADF), it indicates that one of the vdevs indicates 5164 * that the pool has been exported or destroyed. If 5165 * this is the case, the config cache is out of sync and 5166 * we should remove the pool from the namespace. 5167 */ 5168 spa_unload(spa); 5169 spa_deactivate(spa); 5170 spa_write_cachefile(spa, B_TRUE, B_TRUE); 5171 spa_remove(spa); 5172 if (locked) 5173 mutex_exit(&spa_namespace_lock); 5174 return (SET_ERROR(ENOENT)); 5175 } 5176 5177 if (error) { 5178 /* 5179 * We can't open the pool, but we still have useful 5180 * information: the state of each vdev after the 5181 * attempted vdev_open(). Return this to the user. 5182 */ 5183 if (config != NULL && spa->spa_config) { 5184 VERIFY(nvlist_dup(spa->spa_config, config, 5185 KM_SLEEP) == 0); 5186 VERIFY(nvlist_add_nvlist(*config, 5187 ZPOOL_CONFIG_LOAD_INFO, 5188 spa->spa_load_info) == 0); 5189 } 5190 spa_unload(spa); 5191 spa_deactivate(spa); 5192 spa->spa_last_open_failed = error; 5193 if (locked) 5194 mutex_exit(&spa_namespace_lock); 5195 *spapp = NULL; 5196 return (error); 5197 } 5198 } 5199 5200 spa_open_ref(spa, tag); 5201 5202 if (config != NULL) 5203 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5204 5205 /* 5206 * If we've recovered the pool, pass back any information we 5207 * gathered while doing the load. 5208 */ 5209 if (state == SPA_LOAD_RECOVER) { 5210 VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 5211 spa->spa_load_info) == 0); 5212 } 5213 5214 if (locked) { 5215 spa->spa_last_open_failed = 0; 5216 spa->spa_last_ubsync_txg = 0; 5217 spa->spa_load_txg = 0; 5218 mutex_exit(&spa_namespace_lock); 5219 } 5220 5221 if (firstopen) 5222 zvol_create_minors_recursive(spa_name(spa)); 5223 5224 *spapp = spa; 5225 5226 return (0); 5227 } 5228 5229 int 5230 spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy, 5231 nvlist_t **config) 5232 { 5233 return (spa_open_common(name, spapp, tag, policy, config)); 5234 } 5235 5236 int 5237 spa_open(const char *name, spa_t **spapp, void *tag) 5238 { 5239 return (spa_open_common(name, spapp, tag, NULL, NULL)); 5240 } 5241 5242 /* 5243 * Lookup the given spa_t, incrementing the inject count in the process, 5244 * preventing it from being exported or destroyed. 5245 */ 5246 spa_t * 5247 spa_inject_addref(char *name) 5248 { 5249 spa_t *spa; 5250 5251 mutex_enter(&spa_namespace_lock); 5252 if ((spa = spa_lookup(name)) == NULL) { 5253 mutex_exit(&spa_namespace_lock); 5254 return (NULL); 5255 } 5256 spa->spa_inject_ref++; 5257 mutex_exit(&spa_namespace_lock); 5258 5259 return (spa); 5260 } 5261 5262 void 5263 spa_inject_delref(spa_t *spa) 5264 { 5265 mutex_enter(&spa_namespace_lock); 5266 spa->spa_inject_ref--; 5267 mutex_exit(&spa_namespace_lock); 5268 } 5269 5270 /* 5271 * Add spares device information to the nvlist. 5272 */ 5273 static void 5274 spa_add_spares(spa_t *spa, nvlist_t *config) 5275 { 5276 nvlist_t **spares; 5277 uint_t i, nspares; 5278 nvlist_t *nvroot; 5279 uint64_t guid; 5280 vdev_stat_t *vs; 5281 uint_t vsc; 5282 uint64_t pool; 5283 5284 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5285 5286 if (spa->spa_spares.sav_count == 0) 5287 return; 5288 5289 VERIFY(nvlist_lookup_nvlist(config, 5290 ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); 5291 VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 5292 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 5293 if (nspares != 0) { 5294 VERIFY(nvlist_add_nvlist_array(nvroot, 5295 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 5296 VERIFY(nvlist_lookup_nvlist_array(nvroot, 5297 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 5298 5299 /* 5300 * Go through and find any spares which have since been 5301 * repurposed as an active spare. If this is the case, update 5302 * their status appropriately. 5303 */ 5304 for (i = 0; i < nspares; i++) { 5305 VERIFY(nvlist_lookup_uint64(spares[i], 5306 ZPOOL_CONFIG_GUID, &guid) == 0); 5307 if (spa_spare_exists(guid, &pool, NULL) && 5308 pool != 0ULL) { 5309 VERIFY(nvlist_lookup_uint64_array( 5310 spares[i], ZPOOL_CONFIG_VDEV_STATS, 5311 (uint64_t **)&vs, &vsc) == 0); 5312 vs->vs_state = VDEV_STATE_CANT_OPEN; 5313 vs->vs_aux = VDEV_AUX_SPARED; 5314 } 5315 } 5316 } 5317 } 5318 5319 /* 5320 * Add l2cache device information to the nvlist, including vdev stats. 5321 */ 5322 static void 5323 spa_add_l2cache(spa_t *spa, nvlist_t *config) 5324 { 5325 nvlist_t **l2cache; 5326 uint_t i, j, nl2cache; 5327 nvlist_t *nvroot; 5328 uint64_t guid; 5329 vdev_t *vd; 5330 vdev_stat_t *vs; 5331 uint_t vsc; 5332 5333 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5334 5335 if (spa->spa_l2cache.sav_count == 0) 5336 return; 5337 5338 VERIFY(nvlist_lookup_nvlist(config, 5339 ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); 5340 VERIFY(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 5341 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 5342 if (nl2cache != 0) { 5343 VERIFY(nvlist_add_nvlist_array(nvroot, 5344 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 5345 VERIFY(nvlist_lookup_nvlist_array(nvroot, 5346 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 5347 5348 /* 5349 * Update level 2 cache device stats. 5350 */ 5351 5352 for (i = 0; i < nl2cache; i++) { 5353 VERIFY(nvlist_lookup_uint64(l2cache[i], 5354 ZPOOL_CONFIG_GUID, &guid) == 0); 5355 5356 vd = NULL; 5357 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 5358 if (guid == 5359 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 5360 vd = spa->spa_l2cache.sav_vdevs[j]; 5361 break; 5362 } 5363 } 5364 ASSERT(vd != NULL); 5365 5366 VERIFY(nvlist_lookup_uint64_array(l2cache[i], 5367 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc) 5368 == 0); 5369 vdev_get_stats(vd, vs); 5370 vdev_config_generate_stats(vd, l2cache[i]); 5371 5372 } 5373 } 5374 } 5375 5376 static void 5377 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 5378 { 5379 zap_cursor_t zc; 5380 zap_attribute_t za; 5381 5382 if (spa->spa_feat_for_read_obj != 0) { 5383 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5384 spa->spa_feat_for_read_obj); 5385 zap_cursor_retrieve(&zc, &za) == 0; 5386 zap_cursor_advance(&zc)) { 5387 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5388 za.za_num_integers == 1); 5389 VERIFY0(nvlist_add_uint64(features, za.za_name, 5390 za.za_first_integer)); 5391 } 5392 zap_cursor_fini(&zc); 5393 } 5394 5395 if (spa->spa_feat_for_write_obj != 0) { 5396 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5397 spa->spa_feat_for_write_obj); 5398 zap_cursor_retrieve(&zc, &za) == 0; 5399 zap_cursor_advance(&zc)) { 5400 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5401 za.za_num_integers == 1); 5402 VERIFY0(nvlist_add_uint64(features, za.za_name, 5403 za.za_first_integer)); 5404 } 5405 zap_cursor_fini(&zc); 5406 } 5407 } 5408 5409 static void 5410 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 5411 { 5412 int i; 5413 5414 for (i = 0; i < SPA_FEATURES; i++) { 5415 zfeature_info_t feature = spa_feature_table[i]; 5416 uint64_t refcount; 5417 5418 if (feature_get_refcount(spa, &feature, &refcount) != 0) 5419 continue; 5420 5421 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 5422 } 5423 } 5424 5425 /* 5426 * Store a list of pool features and their reference counts in the 5427 * config. 5428 * 5429 * The first time this is called on a spa, allocate a new nvlist, fetch 5430 * the pool features and reference counts from disk, then save the list 5431 * in the spa. In subsequent calls on the same spa use the saved nvlist 5432 * and refresh its values from the cached reference counts. This 5433 * ensures we don't block here on I/O on a suspended pool so 'zpool 5434 * clear' can resume the pool. 5435 */ 5436 static void 5437 spa_add_feature_stats(spa_t *spa, nvlist_t *config) 5438 { 5439 nvlist_t *features; 5440 5441 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5442 5443 mutex_enter(&spa->spa_feat_stats_lock); 5444 features = spa->spa_feat_stats; 5445 5446 if (features != NULL) { 5447 spa_feature_stats_from_cache(spa, features); 5448 } else { 5449 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 5450 spa->spa_feat_stats = features; 5451 spa_feature_stats_from_disk(spa, features); 5452 } 5453 5454 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 5455 features)); 5456 5457 mutex_exit(&spa->spa_feat_stats_lock); 5458 } 5459 5460 int 5461 spa_get_stats(const char *name, nvlist_t **config, 5462 char *altroot, size_t buflen) 5463 { 5464 int error; 5465 spa_t *spa; 5466 5467 *config = NULL; 5468 error = spa_open_common(name, &spa, FTAG, NULL, config); 5469 5470 if (spa != NULL) { 5471 /* 5472 * This still leaves a window of inconsistency where the spares 5473 * or l2cache devices could change and the config would be 5474 * self-inconsistent. 5475 */ 5476 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5477 5478 if (*config != NULL) { 5479 uint64_t loadtimes[2]; 5480 5481 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 5482 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 5483 VERIFY(nvlist_add_uint64_array(*config, 5484 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2) == 0); 5485 5486 VERIFY(nvlist_add_uint64(*config, 5487 ZPOOL_CONFIG_ERRCOUNT, 5488 spa_get_errlog_size(spa)) == 0); 5489 5490 if (spa_suspended(spa)) { 5491 VERIFY(nvlist_add_uint64(*config, 5492 ZPOOL_CONFIG_SUSPENDED, 5493 spa->spa_failmode) == 0); 5494 VERIFY(nvlist_add_uint64(*config, 5495 ZPOOL_CONFIG_SUSPENDED_REASON, 5496 spa->spa_suspended) == 0); 5497 } 5498 5499 spa_add_spares(spa, *config); 5500 spa_add_l2cache(spa, *config); 5501 spa_add_feature_stats(spa, *config); 5502 } 5503 } 5504 5505 /* 5506 * We want to get the alternate root even for faulted pools, so we cheat 5507 * and call spa_lookup() directly. 5508 */ 5509 if (altroot) { 5510 if (spa == NULL) { 5511 mutex_enter(&spa_namespace_lock); 5512 spa = spa_lookup(name); 5513 if (spa) 5514 spa_altroot(spa, altroot, buflen); 5515 else 5516 altroot[0] = '\0'; 5517 spa = NULL; 5518 mutex_exit(&spa_namespace_lock); 5519 } else { 5520 spa_altroot(spa, altroot, buflen); 5521 } 5522 } 5523 5524 if (spa != NULL) { 5525 spa_config_exit(spa, SCL_CONFIG, FTAG); 5526 spa_close(spa, FTAG); 5527 } 5528 5529 return (error); 5530 } 5531 5532 /* 5533 * Validate that the auxiliary device array is well formed. We must have an 5534 * array of nvlists, each which describes a valid leaf vdev. If this is an 5535 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 5536 * specified, as long as they are well-formed. 5537 */ 5538 static int 5539 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 5540 spa_aux_vdev_t *sav, const char *config, uint64_t version, 5541 vdev_labeltype_t label) 5542 { 5543 nvlist_t **dev; 5544 uint_t i, ndev; 5545 vdev_t *vd; 5546 int error; 5547 5548 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5549 5550 /* 5551 * It's acceptable to have no devs specified. 5552 */ 5553 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 5554 return (0); 5555 5556 if (ndev == 0) 5557 return (SET_ERROR(EINVAL)); 5558 5559 /* 5560 * Make sure the pool is formatted with a version that supports this 5561 * device type. 5562 */ 5563 if (spa_version(spa) < version) 5564 return (SET_ERROR(ENOTSUP)); 5565 5566 /* 5567 * Set the pending device list so we correctly handle device in-use 5568 * checking. 5569 */ 5570 sav->sav_pending = dev; 5571 sav->sav_npending = ndev; 5572 5573 for (i = 0; i < ndev; i++) { 5574 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 5575 mode)) != 0) 5576 goto out; 5577 5578 if (!vd->vdev_ops->vdev_op_leaf) { 5579 vdev_free(vd); 5580 error = SET_ERROR(EINVAL); 5581 goto out; 5582 } 5583 5584 vd->vdev_top = vd; 5585 5586 if ((error = vdev_open(vd)) == 0 && 5587 (error = vdev_label_init(vd, crtxg, label)) == 0) { 5588 VERIFY(nvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 5589 vd->vdev_guid) == 0); 5590 } 5591 5592 vdev_free(vd); 5593 5594 if (error && 5595 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 5596 goto out; 5597 else 5598 error = 0; 5599 } 5600 5601 out: 5602 sav->sav_pending = NULL; 5603 sav->sav_npending = 0; 5604 return (error); 5605 } 5606 5607 static int 5608 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 5609 { 5610 int error; 5611 5612 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5613 5614 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5615 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 5616 VDEV_LABEL_SPARE)) != 0) { 5617 return (error); 5618 } 5619 5620 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5621 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 5622 VDEV_LABEL_L2CACHE)); 5623 } 5624 5625 static void 5626 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 5627 const char *config) 5628 { 5629 int i; 5630 5631 if (sav->sav_config != NULL) { 5632 nvlist_t **olddevs; 5633 uint_t oldndevs; 5634 nvlist_t **newdevs; 5635 5636 /* 5637 * Generate new dev list by concatenating with the 5638 * current dev list. 5639 */ 5640 VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, config, 5641 &olddevs, &oldndevs) == 0); 5642 5643 newdevs = kmem_alloc(sizeof (void *) * 5644 (ndevs + oldndevs), KM_SLEEP); 5645 for (i = 0; i < oldndevs; i++) 5646 VERIFY(nvlist_dup(olddevs[i], &newdevs[i], 5647 KM_SLEEP) == 0); 5648 for (i = 0; i < ndevs; i++) 5649 VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs], 5650 KM_SLEEP) == 0); 5651 5652 VERIFY(nvlist_remove(sav->sav_config, config, 5653 DATA_TYPE_NVLIST_ARRAY) == 0); 5654 5655 VERIFY(nvlist_add_nvlist_array(sav->sav_config, 5656 config, newdevs, ndevs + oldndevs) == 0); 5657 for (i = 0; i < oldndevs + ndevs; i++) 5658 nvlist_free(newdevs[i]); 5659 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 5660 } else { 5661 /* 5662 * Generate a new dev list. 5663 */ 5664 VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME, 5665 KM_SLEEP) == 0); 5666 VERIFY(nvlist_add_nvlist_array(sav->sav_config, config, 5667 devs, ndevs) == 0); 5668 } 5669 } 5670 5671 /* 5672 * Stop and drop level 2 ARC devices 5673 */ 5674 void 5675 spa_l2cache_drop(spa_t *spa) 5676 { 5677 vdev_t *vd; 5678 int i; 5679 spa_aux_vdev_t *sav = &spa->spa_l2cache; 5680 5681 for (i = 0; i < sav->sav_count; i++) { 5682 uint64_t pool; 5683 5684 vd = sav->sav_vdevs[i]; 5685 ASSERT(vd != NULL); 5686 5687 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 5688 pool != 0ULL && l2arc_vdev_present(vd)) 5689 l2arc_remove_vdev(vd); 5690 } 5691 } 5692 5693 /* 5694 * Verify encryption parameters for spa creation. If we are encrypting, we must 5695 * have the encryption feature flag enabled. 5696 */ 5697 static int 5698 spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 5699 boolean_t has_encryption) 5700 { 5701 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 5702 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 5703 !has_encryption) 5704 return (SET_ERROR(ENOTSUP)); 5705 5706 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 5707 } 5708 5709 /* 5710 * Pool Creation 5711 */ 5712 int 5713 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 5714 nvlist_t *zplprops, dsl_crypto_params_t *dcp) 5715 { 5716 spa_t *spa; 5717 char *altroot = NULL; 5718 vdev_t *rvd; 5719 dsl_pool_t *dp; 5720 dmu_tx_t *tx; 5721 int error = 0; 5722 uint64_t txg = TXG_INITIAL; 5723 nvlist_t **spares, **l2cache; 5724 uint_t nspares, nl2cache; 5725 uint64_t version, obj, ndraid = 0; 5726 boolean_t has_features; 5727 boolean_t has_encryption; 5728 boolean_t has_allocclass; 5729 spa_feature_t feat; 5730 char *feat_name; 5731 char *poolname; 5732 nvlist_t *nvl; 5733 5734 if (props == NULL || 5735 nvlist_lookup_string(props, "tname", &poolname) != 0) 5736 poolname = (char *)pool; 5737 5738 /* 5739 * If this pool already exists, return failure. 5740 */ 5741 mutex_enter(&spa_namespace_lock); 5742 if (spa_lookup(poolname) != NULL) { 5743 mutex_exit(&spa_namespace_lock); 5744 return (SET_ERROR(EEXIST)); 5745 } 5746 5747 /* 5748 * Allocate a new spa_t structure. 5749 */ 5750 nvl = fnvlist_alloc(); 5751 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 5752 (void) nvlist_lookup_string(props, 5753 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 5754 spa = spa_add(poolname, nvl, altroot); 5755 fnvlist_free(nvl); 5756 spa_activate(spa, spa_mode_global); 5757 5758 if (props && (error = spa_prop_validate(spa, props))) { 5759 spa_deactivate(spa); 5760 spa_remove(spa); 5761 mutex_exit(&spa_namespace_lock); 5762 return (error); 5763 } 5764 5765 /* 5766 * Temporary pool names should never be written to disk. 5767 */ 5768 if (poolname != pool) 5769 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 5770 5771 has_features = B_FALSE; 5772 has_encryption = B_FALSE; 5773 has_allocclass = B_FALSE; 5774 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 5775 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 5776 if (zpool_prop_feature(nvpair_name(elem))) { 5777 has_features = B_TRUE; 5778 5779 feat_name = strchr(nvpair_name(elem), '@') + 1; 5780 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 5781 if (feat == SPA_FEATURE_ENCRYPTION) 5782 has_encryption = B_TRUE; 5783 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 5784 has_allocclass = B_TRUE; 5785 } 5786 } 5787 5788 /* verify encryption params, if they were provided */ 5789 if (dcp != NULL) { 5790 error = spa_create_check_encryption_params(dcp, has_encryption); 5791 if (error != 0) { 5792 spa_deactivate(spa); 5793 spa_remove(spa); 5794 mutex_exit(&spa_namespace_lock); 5795 return (error); 5796 } 5797 } 5798 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 5799 spa_deactivate(spa); 5800 spa_remove(spa); 5801 mutex_exit(&spa_namespace_lock); 5802 return (ENOTSUP); 5803 } 5804 5805 if (has_features || nvlist_lookup_uint64(props, 5806 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 5807 version = SPA_VERSION; 5808 } 5809 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 5810 5811 spa->spa_first_txg = txg; 5812 spa->spa_uberblock.ub_txg = txg - 1; 5813 spa->spa_uberblock.ub_version = version; 5814 spa->spa_ubsync = spa->spa_uberblock; 5815 spa->spa_load_state = SPA_LOAD_CREATE; 5816 spa->spa_removing_phys.sr_state = DSS_NONE; 5817 spa->spa_removing_phys.sr_removing_vdev = -1; 5818 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 5819 spa->spa_indirect_vdevs_loaded = B_TRUE; 5820 5821 /* 5822 * Create "The Godfather" zio to hold all async IOs 5823 */ 5824 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 5825 KM_SLEEP); 5826 for (int i = 0; i < max_ncpus; i++) { 5827 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 5828 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 5829 ZIO_FLAG_GODFATHER); 5830 } 5831 5832 /* 5833 * Create the root vdev. 5834 */ 5835 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5836 5837 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 5838 5839 ASSERT(error != 0 || rvd != NULL); 5840 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 5841 5842 if (error == 0 && !zfs_allocatable_devs(nvroot)) 5843 error = SET_ERROR(EINVAL); 5844 5845 if (error == 0 && 5846 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 5847 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 && 5848 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 5849 /* 5850 * instantiate the metaslab groups (this will dirty the vdevs) 5851 * we can no longer error exit past this point 5852 */ 5853 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 5854 vdev_t *vd = rvd->vdev_child[c]; 5855 5856 vdev_metaslab_set_size(vd); 5857 vdev_expand(vd, txg); 5858 } 5859 } 5860 5861 spa_config_exit(spa, SCL_ALL, FTAG); 5862 5863 if (error != 0) { 5864 spa_unload(spa); 5865 spa_deactivate(spa); 5866 spa_remove(spa); 5867 mutex_exit(&spa_namespace_lock); 5868 return (error); 5869 } 5870 5871 /* 5872 * Get the list of spares, if specified. 5873 */ 5874 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5875 &spares, &nspares) == 0) { 5876 VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME, 5877 KM_SLEEP) == 0); 5878 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 5879 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 5880 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5881 spa_load_spares(spa); 5882 spa_config_exit(spa, SCL_ALL, FTAG); 5883 spa->spa_spares.sav_sync = B_TRUE; 5884 } 5885 5886 /* 5887 * Get the list of level 2 cache devices, if specified. 5888 */ 5889 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5890 &l2cache, &nl2cache) == 0) { 5891 VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, 5892 NV_UNIQUE_NAME, KM_SLEEP) == 0); 5893 VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 5894 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 5895 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5896 spa_load_l2cache(spa); 5897 spa_config_exit(spa, SCL_ALL, FTAG); 5898 spa->spa_l2cache.sav_sync = B_TRUE; 5899 } 5900 5901 spa->spa_is_initializing = B_TRUE; 5902 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 5903 spa->spa_is_initializing = B_FALSE; 5904 5905 /* 5906 * Create DDTs (dedup tables). 5907 */ 5908 ddt_create(spa); 5909 5910 spa_update_dspace(spa); 5911 5912 tx = dmu_tx_create_assigned(dp, txg); 5913 5914 /* 5915 * Create the pool's history object. 5916 */ 5917 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 5918 spa_history_create_obj(spa, tx); 5919 5920 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 5921 spa_history_log_version(spa, "create", tx); 5922 5923 /* 5924 * Create the pool config object. 5925 */ 5926 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 5927 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 5928 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 5929 5930 if (zap_add(spa->spa_meta_objset, 5931 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 5932 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 5933 cmn_err(CE_PANIC, "failed to add pool config"); 5934 } 5935 5936 if (zap_add(spa->spa_meta_objset, 5937 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 5938 sizeof (uint64_t), 1, &version, tx) != 0) { 5939 cmn_err(CE_PANIC, "failed to add pool version"); 5940 } 5941 5942 /* Newly created pools with the right version are always deflated. */ 5943 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 5944 spa->spa_deflate = TRUE; 5945 if (zap_add(spa->spa_meta_objset, 5946 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 5947 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 5948 cmn_err(CE_PANIC, "failed to add deflate"); 5949 } 5950 } 5951 5952 /* 5953 * Create the deferred-free bpobj. Turn off compression 5954 * because sync-to-convergence takes longer if the blocksize 5955 * keeps changing. 5956 */ 5957 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 5958 dmu_object_set_compress(spa->spa_meta_objset, obj, 5959 ZIO_COMPRESS_OFF, tx); 5960 if (zap_add(spa->spa_meta_objset, 5961 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 5962 sizeof (uint64_t), 1, &obj, tx) != 0) { 5963 cmn_err(CE_PANIC, "failed to add bpobj"); 5964 } 5965 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 5966 spa->spa_meta_objset, obj)); 5967 5968 /* 5969 * Generate some random noise for salted checksums to operate on. 5970 */ 5971 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 5972 sizeof (spa->spa_cksum_salt.zcs_bytes)); 5973 5974 /* 5975 * Set pool properties. 5976 */ 5977 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 5978 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 5979 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 5980 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 5981 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 5982 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 5983 5984 if (props != NULL) { 5985 spa_configfile_set(spa, props, B_FALSE); 5986 spa_sync_props(props, tx); 5987 } 5988 5989 for (int i = 0; i < ndraid; i++) 5990 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 5991 5992 dmu_tx_commit(tx); 5993 5994 spa->spa_sync_on = B_TRUE; 5995 txg_sync_start(dp); 5996 mmp_thread_start(spa); 5997 txg_wait_synced(dp, txg); 5998 5999 spa_spawn_aux_threads(spa); 6000 6001 spa_write_cachefile(spa, B_FALSE, B_TRUE); 6002 6003 /* 6004 * Don't count references from objsets that are already closed 6005 * and are making their way through the eviction process. 6006 */ 6007 spa_evicting_os_wait(spa); 6008 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 6009 spa->spa_load_state = SPA_LOAD_NONE; 6010 6011 mutex_exit(&spa_namespace_lock); 6012 6013 return (0); 6014 } 6015 6016 /* 6017 * Import a non-root pool into the system. 6018 */ 6019 int 6020 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 6021 { 6022 spa_t *spa; 6023 char *altroot = NULL; 6024 spa_load_state_t state = SPA_LOAD_IMPORT; 6025 zpool_load_policy_t policy; 6026 spa_mode_t mode = spa_mode_global; 6027 uint64_t readonly = B_FALSE; 6028 int error; 6029 nvlist_t *nvroot; 6030 nvlist_t **spares, **l2cache; 6031 uint_t nspares, nl2cache; 6032 6033 /* 6034 * If a pool with this name exists, return failure. 6035 */ 6036 mutex_enter(&spa_namespace_lock); 6037 if (spa_lookup(pool) != NULL) { 6038 mutex_exit(&spa_namespace_lock); 6039 return (SET_ERROR(EEXIST)); 6040 } 6041 6042 /* 6043 * Create and initialize the spa structure. 6044 */ 6045 (void) nvlist_lookup_string(props, 6046 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6047 (void) nvlist_lookup_uint64(props, 6048 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 6049 if (readonly) 6050 mode = SPA_MODE_READ; 6051 spa = spa_add(pool, config, altroot); 6052 spa->spa_import_flags = flags; 6053 6054 /* 6055 * Verbatim import - Take a pool and insert it into the namespace 6056 * as if it had been loaded at boot. 6057 */ 6058 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 6059 if (props != NULL) 6060 spa_configfile_set(spa, props, B_FALSE); 6061 6062 spa_write_cachefile(spa, B_FALSE, B_TRUE); 6063 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6064 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 6065 mutex_exit(&spa_namespace_lock); 6066 return (0); 6067 } 6068 6069 spa_activate(spa, mode); 6070 6071 /* 6072 * Don't start async tasks until we know everything is healthy. 6073 */ 6074 spa_async_suspend(spa); 6075 6076 zpool_get_load_policy(config, &policy); 6077 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6078 state = SPA_LOAD_RECOVER; 6079 6080 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 6081 6082 if (state != SPA_LOAD_RECOVER) { 6083 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6084 zfs_dbgmsg("spa_import: importing %s", pool); 6085 } else { 6086 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 6087 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 6088 } 6089 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 6090 6091 /* 6092 * Propagate anything learned while loading the pool and pass it 6093 * back to caller (i.e. rewind info, missing devices, etc). 6094 */ 6095 VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6096 spa->spa_load_info) == 0); 6097 6098 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6099 /* 6100 * Toss any existing sparelist, as it doesn't have any validity 6101 * anymore, and conflicts with spa_has_spare(). 6102 */ 6103 if (spa->spa_spares.sav_config) { 6104 nvlist_free(spa->spa_spares.sav_config); 6105 spa->spa_spares.sav_config = NULL; 6106 spa_load_spares(spa); 6107 } 6108 if (spa->spa_l2cache.sav_config) { 6109 nvlist_free(spa->spa_l2cache.sav_config); 6110 spa->spa_l2cache.sav_config = NULL; 6111 spa_load_l2cache(spa); 6112 } 6113 6114 VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, 6115 &nvroot) == 0); 6116 spa_config_exit(spa, SCL_ALL, FTAG); 6117 6118 if (props != NULL) 6119 spa_configfile_set(spa, props, B_FALSE); 6120 6121 if (error != 0 || (props && spa_writeable(spa) && 6122 (error = spa_prop_set(spa, props)))) { 6123 spa_unload(spa); 6124 spa_deactivate(spa); 6125 spa_remove(spa); 6126 mutex_exit(&spa_namespace_lock); 6127 return (error); 6128 } 6129 6130 spa_async_resume(spa); 6131 6132 /* 6133 * Override any spares and level 2 cache devices as specified by 6134 * the user, as these may have correct device names/devids, etc. 6135 */ 6136 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6137 &spares, &nspares) == 0) { 6138 if (spa->spa_spares.sav_config) 6139 VERIFY(nvlist_remove(spa->spa_spares.sav_config, 6140 ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0); 6141 else 6142 VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, 6143 NV_UNIQUE_NAME, KM_SLEEP) == 0); 6144 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 6145 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 6146 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6147 spa_load_spares(spa); 6148 spa_config_exit(spa, SCL_ALL, FTAG); 6149 spa->spa_spares.sav_sync = B_TRUE; 6150 } 6151 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6152 &l2cache, &nl2cache) == 0) { 6153 if (spa->spa_l2cache.sav_config) 6154 VERIFY(nvlist_remove(spa->spa_l2cache.sav_config, 6155 ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); 6156 else 6157 VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, 6158 NV_UNIQUE_NAME, KM_SLEEP) == 0); 6159 VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6160 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 6161 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6162 spa_load_l2cache(spa); 6163 spa_config_exit(spa, SCL_ALL, FTAG); 6164 spa->spa_l2cache.sav_sync = B_TRUE; 6165 } 6166 6167 /* 6168 * Check for any removed devices. 6169 */ 6170 if (spa->spa_autoreplace) { 6171 spa_aux_check_removed(&spa->spa_spares); 6172 spa_aux_check_removed(&spa->spa_l2cache); 6173 } 6174 6175 if (spa_writeable(spa)) { 6176 /* 6177 * Update the config cache to include the newly-imported pool. 6178 */ 6179 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6180 } 6181 6182 /* 6183 * It's possible that the pool was expanded while it was exported. 6184 * We kick off an async task to handle this for us. 6185 */ 6186 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 6187 6188 spa_history_log_version(spa, "import", NULL); 6189 6190 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6191 6192 mutex_exit(&spa_namespace_lock); 6193 6194 zvol_create_minors_recursive(pool); 6195 6196 return (0); 6197 } 6198 6199 nvlist_t * 6200 spa_tryimport(nvlist_t *tryconfig) 6201 { 6202 nvlist_t *config = NULL; 6203 char *poolname, *cachefile; 6204 spa_t *spa; 6205 uint64_t state; 6206 int error; 6207 zpool_load_policy_t policy; 6208 6209 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 6210 return (NULL); 6211 6212 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 6213 return (NULL); 6214 6215 /* 6216 * Create and initialize the spa structure. 6217 */ 6218 mutex_enter(&spa_namespace_lock); 6219 spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL); 6220 spa_activate(spa, SPA_MODE_READ); 6221 6222 /* 6223 * Rewind pool if a max txg was provided. 6224 */ 6225 zpool_get_load_policy(spa->spa_config, &policy); 6226 if (policy.zlp_txg != UINT64_MAX) { 6227 spa->spa_load_max_txg = policy.zlp_txg; 6228 spa->spa_extreme_rewind = B_TRUE; 6229 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 6230 poolname, (longlong_t)policy.zlp_txg); 6231 } else { 6232 zfs_dbgmsg("spa_tryimport: importing %s", poolname); 6233 } 6234 6235 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 6236 == 0) { 6237 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 6238 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6239 } else { 6240 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 6241 } 6242 6243 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 6244 6245 /* 6246 * If 'tryconfig' was at least parsable, return the current config. 6247 */ 6248 if (spa->spa_root_vdev != NULL) { 6249 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6250 VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, 6251 poolname) == 0); 6252 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 6253 state) == 0); 6254 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 6255 spa->spa_uberblock.ub_timestamp) == 0); 6256 VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6257 spa->spa_load_info) == 0); 6258 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 6259 spa->spa_errata) == 0); 6260 6261 /* 6262 * If the bootfs property exists on this pool then we 6263 * copy it out so that external consumers can tell which 6264 * pools are bootable. 6265 */ 6266 if ((!error || error == EEXIST) && spa->spa_bootfs) { 6267 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6268 6269 /* 6270 * We have to play games with the name since the 6271 * pool was opened as TRYIMPORT_NAME. 6272 */ 6273 if (dsl_dsobj_to_dsname(spa_name(spa), 6274 spa->spa_bootfs, tmpname) == 0) { 6275 char *cp; 6276 char *dsname; 6277 6278 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6279 6280 cp = strchr(tmpname, '/'); 6281 if (cp == NULL) { 6282 (void) strlcpy(dsname, tmpname, 6283 MAXPATHLEN); 6284 } else { 6285 (void) snprintf(dsname, MAXPATHLEN, 6286 "%s/%s", poolname, ++cp); 6287 } 6288 VERIFY(nvlist_add_string(config, 6289 ZPOOL_CONFIG_BOOTFS, dsname) == 0); 6290 kmem_free(dsname, MAXPATHLEN); 6291 } 6292 kmem_free(tmpname, MAXPATHLEN); 6293 } 6294 6295 /* 6296 * Add the list of hot spares and level 2 cache devices. 6297 */ 6298 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6299 spa_add_spares(spa, config); 6300 spa_add_l2cache(spa, config); 6301 spa_config_exit(spa, SCL_CONFIG, FTAG); 6302 } 6303 6304 spa_unload(spa); 6305 spa_deactivate(spa); 6306 spa_remove(spa); 6307 mutex_exit(&spa_namespace_lock); 6308 6309 return (config); 6310 } 6311 6312 /* 6313 * Pool export/destroy 6314 * 6315 * The act of destroying or exporting a pool is very simple. We make sure there 6316 * is no more pending I/O and any references to the pool are gone. Then, we 6317 * update the pool state and sync all the labels to disk, removing the 6318 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 6319 * we don't sync the labels or remove the configuration cache. 6320 */ 6321 static int 6322 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 6323 boolean_t force, boolean_t hardforce) 6324 { 6325 int error; 6326 spa_t *spa; 6327 6328 if (oldconfig) 6329 *oldconfig = NULL; 6330 6331 if (!(spa_mode_global & SPA_MODE_WRITE)) 6332 return (SET_ERROR(EROFS)); 6333 6334 mutex_enter(&spa_namespace_lock); 6335 if ((spa = spa_lookup(pool)) == NULL) { 6336 mutex_exit(&spa_namespace_lock); 6337 return (SET_ERROR(ENOENT)); 6338 } 6339 6340 if (spa->spa_is_exporting) { 6341 /* the pool is being exported by another thread */ 6342 mutex_exit(&spa_namespace_lock); 6343 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 6344 } 6345 spa->spa_is_exporting = B_TRUE; 6346 6347 /* 6348 * Put a hold on the pool, drop the namespace lock, stop async tasks, 6349 * reacquire the namespace lock, and see if we can export. 6350 */ 6351 spa_open_ref(spa, FTAG); 6352 mutex_exit(&spa_namespace_lock); 6353 spa_async_suspend(spa); 6354 if (spa->spa_zvol_taskq) { 6355 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 6356 taskq_wait(spa->spa_zvol_taskq); 6357 } 6358 mutex_enter(&spa_namespace_lock); 6359 spa_close(spa, FTAG); 6360 6361 if (spa->spa_state == POOL_STATE_UNINITIALIZED) 6362 goto export_spa; 6363 /* 6364 * The pool will be in core if it's openable, in which case we can 6365 * modify its state. Objsets may be open only because they're dirty, 6366 * so we have to force it to sync before checking spa_refcnt. 6367 */ 6368 if (spa->spa_sync_on) { 6369 txg_wait_synced(spa->spa_dsl_pool, 0); 6370 spa_evicting_os_wait(spa); 6371 } 6372 6373 /* 6374 * A pool cannot be exported or destroyed if there are active 6375 * references. If we are resetting a pool, allow references by 6376 * fault injection handlers. 6377 */ 6378 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 6379 error = SET_ERROR(EBUSY); 6380 goto fail; 6381 } 6382 6383 if (spa->spa_sync_on) { 6384 /* 6385 * A pool cannot be exported if it has an active shared spare. 6386 * This is to prevent other pools stealing the active spare 6387 * from an exported pool. At user's own will, such pool can 6388 * be forcedly exported. 6389 */ 6390 if (!force && new_state == POOL_STATE_EXPORTED && 6391 spa_has_active_shared_spare(spa)) { 6392 error = SET_ERROR(EXDEV); 6393 goto fail; 6394 } 6395 6396 /* 6397 * We're about to export or destroy this pool. Make sure 6398 * we stop all initialization and trim activity here before 6399 * we set the spa_final_txg. This will ensure that all 6400 * dirty data resulting from the initialization is 6401 * committed to disk before we unload the pool. 6402 */ 6403 if (spa->spa_root_vdev != NULL) { 6404 vdev_t *rvd = spa->spa_root_vdev; 6405 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 6406 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 6407 vdev_autotrim_stop_all(spa); 6408 vdev_rebuild_stop_all(spa); 6409 } 6410 6411 /* 6412 * We want this to be reflected on every label, 6413 * so mark them all dirty. spa_unload() will do the 6414 * final sync that pushes these changes out. 6415 */ 6416 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6417 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6418 spa->spa_state = new_state; 6419 spa->spa_final_txg = spa_last_synced_txg(spa) + 6420 TXG_DEFER_SIZE + 1; 6421 vdev_config_dirty(spa->spa_root_vdev); 6422 spa_config_exit(spa, SCL_ALL, FTAG); 6423 } 6424 } 6425 6426 export_spa: 6427 if (new_state == POOL_STATE_DESTROYED) 6428 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 6429 else if (new_state == POOL_STATE_EXPORTED) 6430 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 6431 6432 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 6433 spa_unload(spa); 6434 spa_deactivate(spa); 6435 } 6436 6437 if (oldconfig && spa->spa_config) 6438 VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0); 6439 6440 if (new_state != POOL_STATE_UNINITIALIZED) { 6441 if (!hardforce) 6442 spa_write_cachefile(spa, B_TRUE, B_TRUE); 6443 spa_remove(spa); 6444 } else { 6445 /* 6446 * If spa_remove() is not called for this spa_t and 6447 * there is any possibility that it can be reused, 6448 * we make sure to reset the exporting flag. 6449 */ 6450 spa->spa_is_exporting = B_FALSE; 6451 } 6452 6453 mutex_exit(&spa_namespace_lock); 6454 return (0); 6455 6456 fail: 6457 spa->spa_is_exporting = B_FALSE; 6458 spa_async_resume(spa); 6459 mutex_exit(&spa_namespace_lock); 6460 return (error); 6461 } 6462 6463 /* 6464 * Destroy a storage pool. 6465 */ 6466 int 6467 spa_destroy(const char *pool) 6468 { 6469 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 6470 B_FALSE, B_FALSE)); 6471 } 6472 6473 /* 6474 * Export a storage pool. 6475 */ 6476 int 6477 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 6478 boolean_t hardforce) 6479 { 6480 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 6481 force, hardforce)); 6482 } 6483 6484 /* 6485 * Similar to spa_export(), this unloads the spa_t without actually removing it 6486 * from the namespace in any way. 6487 */ 6488 int 6489 spa_reset(const char *pool) 6490 { 6491 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 6492 B_FALSE, B_FALSE)); 6493 } 6494 6495 /* 6496 * ========================================================================== 6497 * Device manipulation 6498 * ========================================================================== 6499 */ 6500 6501 /* 6502 * This is called as a synctask to increment the draid feature flag 6503 */ 6504 static void 6505 spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 6506 { 6507 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 6508 int draid = (int)(uintptr_t)arg; 6509 6510 for (int c = 0; c < draid; c++) 6511 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6512 } 6513 6514 /* 6515 * Add a device to a storage pool. 6516 */ 6517 int 6518 spa_vdev_add(spa_t *spa, nvlist_t *nvroot) 6519 { 6520 uint64_t txg, ndraid = 0; 6521 int error; 6522 vdev_t *rvd = spa->spa_root_vdev; 6523 vdev_t *vd, *tvd; 6524 nvlist_t **spares, **l2cache; 6525 uint_t nspares, nl2cache; 6526 6527 ASSERT(spa_writeable(spa)); 6528 6529 txg = spa_vdev_enter(spa); 6530 6531 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 6532 VDEV_ALLOC_ADD)) != 0) 6533 return (spa_vdev_exit(spa, NULL, txg, error)); 6534 6535 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 6536 6537 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 6538 &nspares) != 0) 6539 nspares = 0; 6540 6541 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 6542 &nl2cache) != 0) 6543 nl2cache = 0; 6544 6545 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 6546 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6547 6548 if (vd->vdev_children != 0 && 6549 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 6550 return (spa_vdev_exit(spa, vd, txg, error)); 6551 } 6552 6553 /* 6554 * The virtual dRAID spares must be added after vdev tree is created 6555 * and the vdev guids are generated. The guid of their associated 6556 * dRAID is stored in the config and used when opening the spare. 6557 */ 6558 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 6559 rvd->vdev_children)) == 0) { 6560 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 6561 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 6562 nspares = 0; 6563 } else { 6564 return (spa_vdev_exit(spa, vd, txg, error)); 6565 } 6566 6567 /* 6568 * We must validate the spares and l2cache devices after checking the 6569 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 6570 */ 6571 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 6572 return (spa_vdev_exit(spa, vd, txg, error)); 6573 6574 /* 6575 * If we are in the middle of a device removal, we can only add 6576 * devices which match the existing devices in the pool. 6577 * If we are in the middle of a removal, or have some indirect 6578 * vdevs, we can not add raidz or dRAID top levels. 6579 */ 6580 if (spa->spa_vdev_removal != NULL || 6581 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 6582 for (int c = 0; c < vd->vdev_children; c++) { 6583 tvd = vd->vdev_child[c]; 6584 if (spa->spa_vdev_removal != NULL && 6585 tvd->vdev_ashift != spa->spa_max_ashift) { 6586 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6587 } 6588 /* Fail if top level vdev is raidz or a dRAID */ 6589 if (vdev_get_nparity(tvd) != 0) 6590 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6591 6592 /* 6593 * Need the top level mirror to be 6594 * a mirror of leaf vdevs only 6595 */ 6596 if (tvd->vdev_ops == &vdev_mirror_ops) { 6597 for (uint64_t cid = 0; 6598 cid < tvd->vdev_children; cid++) { 6599 vdev_t *cvd = tvd->vdev_child[cid]; 6600 if (!cvd->vdev_ops->vdev_op_leaf) { 6601 return (spa_vdev_exit(spa, vd, 6602 txg, EINVAL)); 6603 } 6604 } 6605 } 6606 } 6607 } 6608 6609 for (int c = 0; c < vd->vdev_children; c++) { 6610 tvd = vd->vdev_child[c]; 6611 vdev_remove_child(vd, tvd); 6612 tvd->vdev_id = rvd->vdev_children; 6613 vdev_add_child(rvd, tvd); 6614 vdev_config_dirty(tvd); 6615 } 6616 6617 if (nspares != 0) { 6618 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 6619 ZPOOL_CONFIG_SPARES); 6620 spa_load_spares(spa); 6621 spa->spa_spares.sav_sync = B_TRUE; 6622 } 6623 6624 if (nl2cache != 0) { 6625 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 6626 ZPOOL_CONFIG_L2CACHE); 6627 spa_load_l2cache(spa); 6628 spa->spa_l2cache.sav_sync = B_TRUE; 6629 } 6630 6631 /* 6632 * We can't increment a feature while holding spa_vdev so we 6633 * have to do it in a synctask. 6634 */ 6635 if (ndraid != 0) { 6636 dmu_tx_t *tx; 6637 6638 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 6639 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 6640 (void *)(uintptr_t)ndraid, tx); 6641 dmu_tx_commit(tx); 6642 } 6643 6644 /* 6645 * We have to be careful when adding new vdevs to an existing pool. 6646 * If other threads start allocating from these vdevs before we 6647 * sync the config cache, and we lose power, then upon reboot we may 6648 * fail to open the pool because there are DVAs that the config cache 6649 * can't translate. Therefore, we first add the vdevs without 6650 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 6651 * and then let spa_config_update() initialize the new metaslabs. 6652 * 6653 * spa_load() checks for added-but-not-initialized vdevs, so that 6654 * if we lose power at any point in this sequence, the remaining 6655 * steps will be completed the next time we load the pool. 6656 */ 6657 (void) spa_vdev_exit(spa, vd, txg, 0); 6658 6659 mutex_enter(&spa_namespace_lock); 6660 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6661 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 6662 mutex_exit(&spa_namespace_lock); 6663 6664 return (0); 6665 } 6666 6667 /* 6668 * Attach a device to a mirror. The arguments are the path to any device 6669 * in the mirror, and the nvroot for the new device. If the path specifies 6670 * a device that is not mirrored, we automatically insert the mirror vdev. 6671 * 6672 * If 'replacing' is specified, the new device is intended to replace the 6673 * existing device; in this case the two devices are made into their own 6674 * mirror using the 'replacing' vdev, which is functionally identical to 6675 * the mirror vdev (it actually reuses all the same ops) but has a few 6676 * extra rules: you can't attach to it after it's been created, and upon 6677 * completion of resilvering, the first disk (the one being replaced) 6678 * is automatically detached. 6679 * 6680 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 6681 * should be performed instead of traditional healing reconstruction. From 6682 * an administrators perspective these are both resilver operations. 6683 */ 6684 int 6685 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 6686 int rebuild) 6687 { 6688 uint64_t txg, dtl_max_txg; 6689 vdev_t *rvd = spa->spa_root_vdev; 6690 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 6691 vdev_ops_t *pvops; 6692 char *oldvdpath, *newvdpath; 6693 int newvd_isspare; 6694 int error; 6695 6696 ASSERT(spa_writeable(spa)); 6697 6698 txg = spa_vdev_enter(spa); 6699 6700 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 6701 6702 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6703 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6704 error = (spa_has_checkpoint(spa)) ? 6705 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6706 return (spa_vdev_exit(spa, NULL, txg, error)); 6707 } 6708 6709 if (rebuild) { 6710 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 6711 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6712 6713 if (dsl_scan_resilvering(spa_get_dsl(spa))) 6714 return (spa_vdev_exit(spa, NULL, txg, 6715 ZFS_ERR_RESILVER_IN_PROGRESS)); 6716 } else { 6717 if (vdev_rebuild_active(rvd)) 6718 return (spa_vdev_exit(spa, NULL, txg, 6719 ZFS_ERR_REBUILD_IN_PROGRESS)); 6720 } 6721 6722 if (spa->spa_vdev_removal != NULL) 6723 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 6724 6725 if (oldvd == NULL) 6726 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6727 6728 if (!oldvd->vdev_ops->vdev_op_leaf) 6729 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6730 6731 pvd = oldvd->vdev_parent; 6732 6733 if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 6734 VDEV_ALLOC_ATTACH)) != 0) 6735 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 6736 6737 if (newrootvd->vdev_children != 1) 6738 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6739 6740 newvd = newrootvd->vdev_child[0]; 6741 6742 if (!newvd->vdev_ops->vdev_op_leaf) 6743 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6744 6745 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 6746 return (spa_vdev_exit(spa, newrootvd, txg, error)); 6747 6748 /* 6749 * Spares can't replace logs 6750 */ 6751 if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare) 6752 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6753 6754 /* 6755 * A dRAID spare can only replace a child of its parent dRAID vdev. 6756 */ 6757 if (newvd->vdev_ops == &vdev_draid_spare_ops && 6758 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 6759 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6760 } 6761 6762 if (rebuild) { 6763 /* 6764 * For rebuilds, the top vdev must support reconstruction 6765 * using only space maps. This means the only allowable 6766 * vdevs types are the root vdev, a mirror, or dRAID. 6767 */ 6768 tvd = pvd; 6769 if (pvd->vdev_top != NULL) 6770 tvd = pvd->vdev_top; 6771 6772 if (tvd->vdev_ops != &vdev_mirror_ops && 6773 tvd->vdev_ops != &vdev_root_ops && 6774 tvd->vdev_ops != &vdev_draid_ops) { 6775 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6776 } 6777 } 6778 6779 if (!replacing) { 6780 /* 6781 * For attach, the only allowable parent is a mirror or the root 6782 * vdev. 6783 */ 6784 if (pvd->vdev_ops != &vdev_mirror_ops && 6785 pvd->vdev_ops != &vdev_root_ops) 6786 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6787 6788 pvops = &vdev_mirror_ops; 6789 } else { 6790 /* 6791 * Active hot spares can only be replaced by inactive hot 6792 * spares. 6793 */ 6794 if (pvd->vdev_ops == &vdev_spare_ops && 6795 oldvd->vdev_isspare && 6796 !spa_has_spare(spa, newvd->vdev_guid)) 6797 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6798 6799 /* 6800 * If the source is a hot spare, and the parent isn't already a 6801 * spare, then we want to create a new hot spare. Otherwise, we 6802 * want to create a replacing vdev. The user is not allowed to 6803 * attach to a spared vdev child unless the 'isspare' state is 6804 * the same (spare replaces spare, non-spare replaces 6805 * non-spare). 6806 */ 6807 if (pvd->vdev_ops == &vdev_replacing_ops && 6808 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 6809 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6810 } else if (pvd->vdev_ops == &vdev_spare_ops && 6811 newvd->vdev_isspare != oldvd->vdev_isspare) { 6812 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6813 } 6814 6815 if (newvd->vdev_isspare) 6816 pvops = &vdev_spare_ops; 6817 else 6818 pvops = &vdev_replacing_ops; 6819 } 6820 6821 /* 6822 * Make sure the new device is big enough. 6823 */ 6824 if (newvd->vdev_asize < vdev_get_min_asize(oldvd)) 6825 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 6826 6827 /* 6828 * The new device cannot have a higher alignment requirement 6829 * than the top-level vdev. 6830 */ 6831 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) 6832 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6833 6834 /* 6835 * If this is an in-place replacement, update oldvd's path and devid 6836 * to make it distinguishable from newvd, and unopenable from now on. 6837 */ 6838 if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { 6839 spa_strfree(oldvd->vdev_path); 6840 oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, 6841 KM_SLEEP); 6842 (void) snprintf(oldvd->vdev_path, strlen(newvd->vdev_path) + 5, 6843 "%s/%s", newvd->vdev_path, "old"); 6844 if (oldvd->vdev_devid != NULL) { 6845 spa_strfree(oldvd->vdev_devid); 6846 oldvd->vdev_devid = NULL; 6847 } 6848 } 6849 6850 /* 6851 * If the parent is not a mirror, or if we're replacing, insert the new 6852 * mirror/replacing/spare vdev above oldvd. 6853 */ 6854 if (pvd->vdev_ops != pvops) 6855 pvd = vdev_add_parent(oldvd, pvops); 6856 6857 ASSERT(pvd->vdev_top->vdev_parent == rvd); 6858 ASSERT(pvd->vdev_ops == pvops); 6859 ASSERT(oldvd->vdev_parent == pvd); 6860 6861 /* 6862 * Extract the new device from its root and add it to pvd. 6863 */ 6864 vdev_remove_child(newrootvd, newvd); 6865 newvd->vdev_id = pvd->vdev_children; 6866 newvd->vdev_crtxg = oldvd->vdev_crtxg; 6867 vdev_add_child(pvd, newvd); 6868 6869 /* 6870 * Reevaluate the parent vdev state. 6871 */ 6872 vdev_propagate_state(pvd); 6873 6874 tvd = newvd->vdev_top; 6875 ASSERT(pvd->vdev_top == tvd); 6876 ASSERT(tvd->vdev_parent == rvd); 6877 6878 vdev_config_dirty(tvd); 6879 6880 /* 6881 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 6882 * for any dmu_sync-ed blocks. It will propagate upward when 6883 * spa_vdev_exit() calls vdev_dtl_reassess(). 6884 */ 6885 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 6886 6887 vdev_dtl_dirty(newvd, DTL_MISSING, 6888 TXG_INITIAL, dtl_max_txg - TXG_INITIAL); 6889 6890 if (newvd->vdev_isspare) { 6891 spa_spare_activate(newvd); 6892 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 6893 } 6894 6895 oldvdpath = spa_strdup(oldvd->vdev_path); 6896 newvdpath = spa_strdup(newvd->vdev_path); 6897 newvd_isspare = newvd->vdev_isspare; 6898 6899 /* 6900 * Mark newvd's DTL dirty in this txg. 6901 */ 6902 vdev_dirty(tvd, VDD_DTL, newvd, txg); 6903 6904 /* 6905 * Schedule the resilver or rebuild to restart in the future. We do 6906 * this to ensure that dmu_sync-ed blocks have been stitched into the 6907 * respective datasets. 6908 */ 6909 if (rebuild) { 6910 newvd->vdev_rebuild_txg = txg; 6911 6912 vdev_rebuild(tvd); 6913 } else { 6914 newvd->vdev_resilver_txg = txg; 6915 6916 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 6917 spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) { 6918 vdev_defer_resilver(newvd); 6919 } else { 6920 dsl_scan_restart_resilver(spa->spa_dsl_pool, 6921 dtl_max_txg); 6922 } 6923 } 6924 6925 if (spa->spa_bootfs) 6926 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 6927 6928 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 6929 6930 /* 6931 * Commit the config 6932 */ 6933 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 6934 6935 spa_history_log_internal(spa, "vdev attach", NULL, 6936 "%s vdev=%s %s vdev=%s", 6937 replacing && newvd_isspare ? "spare in" : 6938 replacing ? "replace" : "attach", newvdpath, 6939 replacing ? "for" : "to", oldvdpath); 6940 6941 spa_strfree(oldvdpath); 6942 spa_strfree(newvdpath); 6943 6944 return (0); 6945 } 6946 6947 /* 6948 * Detach a device from a mirror or replacing vdev. 6949 * 6950 * If 'replace_done' is specified, only detach if the parent 6951 * is a replacing vdev. 6952 */ 6953 int 6954 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 6955 { 6956 uint64_t txg; 6957 int error; 6958 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 6959 vdev_t *vd, *pvd, *cvd, *tvd; 6960 boolean_t unspare = B_FALSE; 6961 uint64_t unspare_guid = 0; 6962 char *vdpath; 6963 6964 ASSERT(spa_writeable(spa)); 6965 6966 txg = spa_vdev_detach_enter(spa, guid); 6967 6968 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 6969 6970 /* 6971 * Besides being called directly from the userland through the 6972 * ioctl interface, spa_vdev_detach() can be potentially called 6973 * at the end of spa_vdev_resilver_done(). 6974 * 6975 * In the regular case, when we have a checkpoint this shouldn't 6976 * happen as we never empty the DTLs of a vdev during the scrub 6977 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 6978 * should never get here when we have a checkpoint. 6979 * 6980 * That said, even in a case when we checkpoint the pool exactly 6981 * as spa_vdev_resilver_done() calls this function everything 6982 * should be fine as the resilver will return right away. 6983 */ 6984 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6985 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6986 error = (spa_has_checkpoint(spa)) ? 6987 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6988 return (spa_vdev_exit(spa, NULL, txg, error)); 6989 } 6990 6991 if (vd == NULL) 6992 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6993 6994 if (!vd->vdev_ops->vdev_op_leaf) 6995 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6996 6997 pvd = vd->vdev_parent; 6998 6999 /* 7000 * If the parent/child relationship is not as expected, don't do it. 7001 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 7002 * vdev that's replacing B with C. The user's intent in replacing 7003 * is to go from M(A,B) to M(A,C). If the user decides to cancel 7004 * the replace by detaching C, the expected behavior is to end up 7005 * M(A,B). But suppose that right after deciding to detach C, 7006 * the replacement of B completes. We would have M(A,C), and then 7007 * ask to detach C, which would leave us with just A -- not what 7008 * the user wanted. To prevent this, we make sure that the 7009 * parent/child relationship hasn't changed -- in this example, 7010 * that C's parent is still the replacing vdev R. 7011 */ 7012 if (pvd->vdev_guid != pguid && pguid != 0) 7013 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7014 7015 /* 7016 * Only 'replacing' or 'spare' vdevs can be replaced. 7017 */ 7018 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 7019 pvd->vdev_ops != &vdev_spare_ops) 7020 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7021 7022 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 7023 spa_version(spa) >= SPA_VERSION_SPARES); 7024 7025 /* 7026 * Only mirror, replacing, and spare vdevs support detach. 7027 */ 7028 if (pvd->vdev_ops != &vdev_replacing_ops && 7029 pvd->vdev_ops != &vdev_mirror_ops && 7030 pvd->vdev_ops != &vdev_spare_ops) 7031 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7032 7033 /* 7034 * If this device has the only valid copy of some data, 7035 * we cannot safely detach it. 7036 */ 7037 if (vdev_dtl_required(vd)) 7038 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7039 7040 ASSERT(pvd->vdev_children >= 2); 7041 7042 /* 7043 * If we are detaching the second disk from a replacing vdev, then 7044 * check to see if we changed the original vdev's path to have "/old" 7045 * at the end in spa_vdev_attach(). If so, undo that change now. 7046 */ 7047 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 7048 vd->vdev_path != NULL) { 7049 size_t len = strlen(vd->vdev_path); 7050 7051 for (int c = 0; c < pvd->vdev_children; c++) { 7052 cvd = pvd->vdev_child[c]; 7053 7054 if (cvd == vd || cvd->vdev_path == NULL) 7055 continue; 7056 7057 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 7058 strcmp(cvd->vdev_path + len, "/old") == 0) { 7059 spa_strfree(cvd->vdev_path); 7060 cvd->vdev_path = spa_strdup(vd->vdev_path); 7061 break; 7062 } 7063 } 7064 } 7065 7066 /* 7067 * If we are detaching the original disk from a normal spare, then it 7068 * implies that the spare should become a real disk, and be removed 7069 * from the active spare list for the pool. dRAID spares on the 7070 * other hand are coupled to the pool and thus should never be removed 7071 * from the spares list. 7072 */ 7073 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 7074 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7075 7076 if (last_cvd->vdev_isspare && 7077 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 7078 unspare = B_TRUE; 7079 } 7080 } 7081 7082 /* 7083 * Erase the disk labels so the disk can be used for other things. 7084 * This must be done after all other error cases are handled, 7085 * but before we disembowel vd (so we can still do I/O to it). 7086 * But if we can't do it, don't treat the error as fatal -- 7087 * it may be that the unwritability of the disk is the reason 7088 * it's being detached! 7089 */ 7090 error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 7091 7092 /* 7093 * Remove vd from its parent and compact the parent's children. 7094 */ 7095 vdev_remove_child(pvd, vd); 7096 vdev_compact_children(pvd); 7097 7098 /* 7099 * Remember one of the remaining children so we can get tvd below. 7100 */ 7101 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7102 7103 /* 7104 * If we need to remove the remaining child from the list of hot spares, 7105 * do it now, marking the vdev as no longer a spare in the process. 7106 * We must do this before vdev_remove_parent(), because that can 7107 * change the GUID if it creates a new toplevel GUID. For a similar 7108 * reason, we must remove the spare now, in the same txg as the detach; 7109 * otherwise someone could attach a new sibling, change the GUID, and 7110 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 7111 */ 7112 if (unspare) { 7113 ASSERT(cvd->vdev_isspare); 7114 spa_spare_remove(cvd); 7115 unspare_guid = cvd->vdev_guid; 7116 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 7117 cvd->vdev_unspare = B_TRUE; 7118 } 7119 7120 /* 7121 * If the parent mirror/replacing vdev only has one child, 7122 * the parent is no longer needed. Remove it from the tree. 7123 */ 7124 if (pvd->vdev_children == 1) { 7125 if (pvd->vdev_ops == &vdev_spare_ops) 7126 cvd->vdev_unspare = B_FALSE; 7127 vdev_remove_parent(cvd); 7128 } 7129 7130 /* 7131 * We don't set tvd until now because the parent we just removed 7132 * may have been the previous top-level vdev. 7133 */ 7134 tvd = cvd->vdev_top; 7135 ASSERT(tvd->vdev_parent == rvd); 7136 7137 /* 7138 * Reevaluate the parent vdev state. 7139 */ 7140 vdev_propagate_state(cvd); 7141 7142 /* 7143 * If the 'autoexpand' property is set on the pool then automatically 7144 * try to expand the size of the pool. For example if the device we 7145 * just detached was smaller than the others, it may be possible to 7146 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 7147 * first so that we can obtain the updated sizes of the leaf vdevs. 7148 */ 7149 if (spa->spa_autoexpand) { 7150 vdev_reopen(tvd); 7151 vdev_expand(tvd, txg); 7152 } 7153 7154 vdev_config_dirty(tvd); 7155 7156 /* 7157 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 7158 * vd->vdev_detached is set and free vd's DTL object in syncing context. 7159 * But first make sure we're not on any *other* txg's DTL list, to 7160 * prevent vd from being accessed after it's freed. 7161 */ 7162 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 7163 for (int t = 0; t < TXG_SIZE; t++) 7164 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 7165 vd->vdev_detached = B_TRUE; 7166 vdev_dirty(tvd, VDD_DTL, vd, txg); 7167 7168 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 7169 spa_notify_waiters(spa); 7170 7171 /* hang on to the spa before we release the lock */ 7172 spa_open_ref(spa, FTAG); 7173 7174 error = spa_vdev_exit(spa, vd, txg, 0); 7175 7176 spa_history_log_internal(spa, "detach", NULL, 7177 "vdev=%s", vdpath); 7178 spa_strfree(vdpath); 7179 7180 /* 7181 * If this was the removal of the original device in a hot spare vdev, 7182 * then we want to go through and remove the device from the hot spare 7183 * list of every other pool. 7184 */ 7185 if (unspare) { 7186 spa_t *altspa = NULL; 7187 7188 mutex_enter(&spa_namespace_lock); 7189 while ((altspa = spa_next(altspa)) != NULL) { 7190 if (altspa->spa_state != POOL_STATE_ACTIVE || 7191 altspa == spa) 7192 continue; 7193 7194 spa_open_ref(altspa, FTAG); 7195 mutex_exit(&spa_namespace_lock); 7196 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 7197 mutex_enter(&spa_namespace_lock); 7198 spa_close(altspa, FTAG); 7199 } 7200 mutex_exit(&spa_namespace_lock); 7201 7202 /* search the rest of the vdevs for spares to remove */ 7203 spa_vdev_resilver_done(spa); 7204 } 7205 7206 /* all done with the spa; OK to release */ 7207 mutex_enter(&spa_namespace_lock); 7208 spa_close(spa, FTAG); 7209 mutex_exit(&spa_namespace_lock); 7210 7211 return (error); 7212 } 7213 7214 static int 7215 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7216 list_t *vd_list) 7217 { 7218 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7219 7220 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7221 7222 /* Look up vdev and ensure it's a leaf. */ 7223 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7224 if (vd == NULL || vd->vdev_detached) { 7225 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7226 return (SET_ERROR(ENODEV)); 7227 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7228 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7229 return (SET_ERROR(EINVAL)); 7230 } else if (!vdev_writeable(vd)) { 7231 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7232 return (SET_ERROR(EROFS)); 7233 } 7234 mutex_enter(&vd->vdev_initialize_lock); 7235 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7236 7237 /* 7238 * When we activate an initialize action we check to see 7239 * if the vdev_initialize_thread is NULL. We do this instead 7240 * of using the vdev_initialize_state since there might be 7241 * a previous initialization process which has completed but 7242 * the thread is not exited. 7243 */ 7244 if (cmd_type == POOL_INITIALIZE_START && 7245 (vd->vdev_initialize_thread != NULL || 7246 vd->vdev_top->vdev_removing)) { 7247 mutex_exit(&vd->vdev_initialize_lock); 7248 return (SET_ERROR(EBUSY)); 7249 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 7250 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 7251 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 7252 mutex_exit(&vd->vdev_initialize_lock); 7253 return (SET_ERROR(ESRCH)); 7254 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 7255 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 7256 mutex_exit(&vd->vdev_initialize_lock); 7257 return (SET_ERROR(ESRCH)); 7258 } 7259 7260 switch (cmd_type) { 7261 case POOL_INITIALIZE_START: 7262 vdev_initialize(vd); 7263 break; 7264 case POOL_INITIALIZE_CANCEL: 7265 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 7266 break; 7267 case POOL_INITIALIZE_SUSPEND: 7268 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 7269 break; 7270 default: 7271 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7272 } 7273 mutex_exit(&vd->vdev_initialize_lock); 7274 7275 return (0); 7276 } 7277 7278 int 7279 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 7280 nvlist_t *vdev_errlist) 7281 { 7282 int total_errors = 0; 7283 list_t vd_list; 7284 7285 list_create(&vd_list, sizeof (vdev_t), 7286 offsetof(vdev_t, vdev_initialize_node)); 7287 7288 /* 7289 * We hold the namespace lock through the whole function 7290 * to prevent any changes to the pool while we're starting or 7291 * stopping initialization. The config and state locks are held so that 7292 * we can properly assess the vdev state before we commit to 7293 * the initializing operation. 7294 */ 7295 mutex_enter(&spa_namespace_lock); 7296 7297 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7298 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7299 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7300 7301 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 7302 &vd_list); 7303 if (error != 0) { 7304 char guid_as_str[MAXNAMELEN]; 7305 7306 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7307 "%llu", (unsigned long long)vdev_guid); 7308 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7309 total_errors++; 7310 } 7311 } 7312 7313 /* Wait for all initialize threads to stop. */ 7314 vdev_initialize_stop_wait(spa, &vd_list); 7315 7316 /* Sync out the initializing state */ 7317 txg_wait_synced(spa->spa_dsl_pool, 0); 7318 mutex_exit(&spa_namespace_lock); 7319 7320 list_destroy(&vd_list); 7321 7322 return (total_errors); 7323 } 7324 7325 static int 7326 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7327 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 7328 { 7329 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7330 7331 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7332 7333 /* Look up vdev and ensure it's a leaf. */ 7334 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7335 if (vd == NULL || vd->vdev_detached) { 7336 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7337 return (SET_ERROR(ENODEV)); 7338 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7339 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7340 return (SET_ERROR(EINVAL)); 7341 } else if (!vdev_writeable(vd)) { 7342 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7343 return (SET_ERROR(EROFS)); 7344 } else if (!vd->vdev_has_trim) { 7345 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7346 return (SET_ERROR(EOPNOTSUPP)); 7347 } else if (secure && !vd->vdev_has_securetrim) { 7348 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7349 return (SET_ERROR(EOPNOTSUPP)); 7350 } 7351 mutex_enter(&vd->vdev_trim_lock); 7352 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7353 7354 /* 7355 * When we activate a TRIM action we check to see if the 7356 * vdev_trim_thread is NULL. We do this instead of using the 7357 * vdev_trim_state since there might be a previous TRIM process 7358 * which has completed but the thread is not exited. 7359 */ 7360 if (cmd_type == POOL_TRIM_START && 7361 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing)) { 7362 mutex_exit(&vd->vdev_trim_lock); 7363 return (SET_ERROR(EBUSY)); 7364 } else if (cmd_type == POOL_TRIM_CANCEL && 7365 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 7366 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 7367 mutex_exit(&vd->vdev_trim_lock); 7368 return (SET_ERROR(ESRCH)); 7369 } else if (cmd_type == POOL_TRIM_SUSPEND && 7370 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 7371 mutex_exit(&vd->vdev_trim_lock); 7372 return (SET_ERROR(ESRCH)); 7373 } 7374 7375 switch (cmd_type) { 7376 case POOL_TRIM_START: 7377 vdev_trim(vd, rate, partial, secure); 7378 break; 7379 case POOL_TRIM_CANCEL: 7380 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 7381 break; 7382 case POOL_TRIM_SUSPEND: 7383 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 7384 break; 7385 default: 7386 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7387 } 7388 mutex_exit(&vd->vdev_trim_lock); 7389 7390 return (0); 7391 } 7392 7393 /* 7394 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 7395 * TRIM threads for each child vdev. These threads pass over all of the free 7396 * space in the vdev's metaslabs and issues TRIM commands for that space. 7397 */ 7398 int 7399 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 7400 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 7401 { 7402 int total_errors = 0; 7403 list_t vd_list; 7404 7405 list_create(&vd_list, sizeof (vdev_t), 7406 offsetof(vdev_t, vdev_trim_node)); 7407 7408 /* 7409 * We hold the namespace lock through the whole function 7410 * to prevent any changes to the pool while we're starting or 7411 * stopping TRIM. The config and state locks are held so that 7412 * we can properly assess the vdev state before we commit to 7413 * the TRIM operation. 7414 */ 7415 mutex_enter(&spa_namespace_lock); 7416 7417 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7418 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7419 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7420 7421 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 7422 rate, partial, secure, &vd_list); 7423 if (error != 0) { 7424 char guid_as_str[MAXNAMELEN]; 7425 7426 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7427 "%llu", (unsigned long long)vdev_guid); 7428 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7429 total_errors++; 7430 } 7431 } 7432 7433 /* Wait for all TRIM threads to stop. */ 7434 vdev_trim_stop_wait(spa, &vd_list); 7435 7436 /* Sync out the TRIM state */ 7437 txg_wait_synced(spa->spa_dsl_pool, 0); 7438 mutex_exit(&spa_namespace_lock); 7439 7440 list_destroy(&vd_list); 7441 7442 return (total_errors); 7443 } 7444 7445 /* 7446 * Split a set of devices from their mirrors, and create a new pool from them. 7447 */ 7448 int 7449 spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, 7450 nvlist_t *props, boolean_t exp) 7451 { 7452 int error = 0; 7453 uint64_t txg, *glist; 7454 spa_t *newspa; 7455 uint_t c, children, lastlog; 7456 nvlist_t **child, *nvl, *tmp; 7457 dmu_tx_t *tx; 7458 char *altroot = NULL; 7459 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 7460 boolean_t activate_slog; 7461 7462 ASSERT(spa_writeable(spa)); 7463 7464 txg = spa_vdev_enter(spa); 7465 7466 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7467 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7468 error = (spa_has_checkpoint(spa)) ? 7469 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7470 return (spa_vdev_exit(spa, NULL, txg, error)); 7471 } 7472 7473 /* clear the log and flush everything up to now */ 7474 activate_slog = spa_passivate_log(spa); 7475 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7476 error = spa_reset_logs(spa); 7477 txg = spa_vdev_config_enter(spa); 7478 7479 if (activate_slog) 7480 spa_activate_log(spa); 7481 7482 if (error != 0) 7483 return (spa_vdev_exit(spa, NULL, txg, error)); 7484 7485 /* check new spa name before going any further */ 7486 if (spa_lookup(newname) != NULL) 7487 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 7488 7489 /* 7490 * scan through all the children to ensure they're all mirrors 7491 */ 7492 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 7493 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 7494 &children) != 0) 7495 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7496 7497 /* first, check to ensure we've got the right child count */ 7498 rvd = spa->spa_root_vdev; 7499 lastlog = 0; 7500 for (c = 0; c < rvd->vdev_children; c++) { 7501 vdev_t *vd = rvd->vdev_child[c]; 7502 7503 /* don't count the holes & logs as children */ 7504 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 7505 !vdev_is_concrete(vd))) { 7506 if (lastlog == 0) 7507 lastlog = c; 7508 continue; 7509 } 7510 7511 lastlog = 0; 7512 } 7513 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 7514 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7515 7516 /* next, ensure no spare or cache devices are part of the split */ 7517 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 7518 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 7519 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7520 7521 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 7522 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 7523 7524 /* then, loop over each vdev and validate it */ 7525 for (c = 0; c < children; c++) { 7526 uint64_t is_hole = 0; 7527 7528 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 7529 &is_hole); 7530 7531 if (is_hole != 0) { 7532 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 7533 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 7534 continue; 7535 } else { 7536 error = SET_ERROR(EINVAL); 7537 break; 7538 } 7539 } 7540 7541 /* deal with indirect vdevs */ 7542 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 7543 &vdev_indirect_ops) 7544 continue; 7545 7546 /* which disk is going to be split? */ 7547 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 7548 &glist[c]) != 0) { 7549 error = SET_ERROR(EINVAL); 7550 break; 7551 } 7552 7553 /* look it up in the spa */ 7554 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 7555 if (vml[c] == NULL) { 7556 error = SET_ERROR(ENODEV); 7557 break; 7558 } 7559 7560 /* make sure there's nothing stopping the split */ 7561 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 7562 vml[c]->vdev_islog || 7563 !vdev_is_concrete(vml[c]) || 7564 vml[c]->vdev_isspare || 7565 vml[c]->vdev_isl2cache || 7566 !vdev_writeable(vml[c]) || 7567 vml[c]->vdev_children != 0 || 7568 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 7569 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 7570 error = SET_ERROR(EINVAL); 7571 break; 7572 } 7573 7574 if (vdev_dtl_required(vml[c]) || 7575 vdev_resilver_needed(vml[c], NULL, NULL)) { 7576 error = SET_ERROR(EBUSY); 7577 break; 7578 } 7579 7580 /* we need certain info from the top level */ 7581 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 7582 vml[c]->vdev_top->vdev_ms_array) == 0); 7583 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 7584 vml[c]->vdev_top->vdev_ms_shift) == 0); 7585 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 7586 vml[c]->vdev_top->vdev_asize) == 0); 7587 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 7588 vml[c]->vdev_top->vdev_ashift) == 0); 7589 7590 /* transfer per-vdev ZAPs */ 7591 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 7592 VERIFY0(nvlist_add_uint64(child[c], 7593 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 7594 7595 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 7596 VERIFY0(nvlist_add_uint64(child[c], 7597 ZPOOL_CONFIG_VDEV_TOP_ZAP, 7598 vml[c]->vdev_parent->vdev_top_zap)); 7599 } 7600 7601 if (error != 0) { 7602 kmem_free(vml, children * sizeof (vdev_t *)); 7603 kmem_free(glist, children * sizeof (uint64_t)); 7604 return (spa_vdev_exit(spa, NULL, txg, error)); 7605 } 7606 7607 /* stop writers from using the disks */ 7608 for (c = 0; c < children; c++) { 7609 if (vml[c] != NULL) 7610 vml[c]->vdev_offline = B_TRUE; 7611 } 7612 vdev_reopen(spa->spa_root_vdev); 7613 7614 /* 7615 * Temporarily record the splitting vdevs in the spa config. This 7616 * will disappear once the config is regenerated. 7617 */ 7618 VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0); 7619 VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 7620 glist, children) == 0); 7621 kmem_free(glist, children * sizeof (uint64_t)); 7622 7623 mutex_enter(&spa->spa_props_lock); 7624 VERIFY(nvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, 7625 nvl) == 0); 7626 mutex_exit(&spa->spa_props_lock); 7627 spa->spa_config_splitting = nvl; 7628 vdev_config_dirty(spa->spa_root_vdev); 7629 7630 /* configure and create the new pool */ 7631 VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname) == 0); 7632 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 7633 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE) == 0); 7634 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 7635 spa_version(spa)) == 0); 7636 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, 7637 spa->spa_config_txg) == 0); 7638 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 7639 spa_generate_guid(NULL)) == 0); 7640 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 7641 (void) nvlist_lookup_string(props, 7642 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 7643 7644 /* add the new pool to the namespace */ 7645 newspa = spa_add(newname, config, altroot); 7646 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 7647 newspa->spa_config_txg = spa->spa_config_txg; 7648 spa_set_log_state(newspa, SPA_LOG_CLEAR); 7649 7650 /* release the spa config lock, retaining the namespace lock */ 7651 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7652 7653 if (zio_injection_enabled) 7654 zio_handle_panic_injection(spa, FTAG, 1); 7655 7656 spa_activate(newspa, spa_mode_global); 7657 spa_async_suspend(newspa); 7658 7659 /* 7660 * Temporarily stop the initializing and TRIM activity. We set the 7661 * state to ACTIVE so that we know to resume initializing or TRIM 7662 * once the split has completed. 7663 */ 7664 list_t vd_initialize_list; 7665 list_create(&vd_initialize_list, sizeof (vdev_t), 7666 offsetof(vdev_t, vdev_initialize_node)); 7667 7668 list_t vd_trim_list; 7669 list_create(&vd_trim_list, sizeof (vdev_t), 7670 offsetof(vdev_t, vdev_trim_node)); 7671 7672 for (c = 0; c < children; c++) { 7673 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7674 mutex_enter(&vml[c]->vdev_initialize_lock); 7675 vdev_initialize_stop(vml[c], 7676 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 7677 mutex_exit(&vml[c]->vdev_initialize_lock); 7678 7679 mutex_enter(&vml[c]->vdev_trim_lock); 7680 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 7681 mutex_exit(&vml[c]->vdev_trim_lock); 7682 } 7683 } 7684 7685 vdev_initialize_stop_wait(spa, &vd_initialize_list); 7686 vdev_trim_stop_wait(spa, &vd_trim_list); 7687 7688 list_destroy(&vd_initialize_list); 7689 list_destroy(&vd_trim_list); 7690 7691 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 7692 newspa->spa_is_splitting = B_TRUE; 7693 7694 /* create the new pool from the disks of the original pool */ 7695 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 7696 if (error) 7697 goto out; 7698 7699 /* if that worked, generate a real config for the new pool */ 7700 if (newspa->spa_root_vdev != NULL) { 7701 VERIFY(nvlist_alloc(&newspa->spa_config_splitting, 7702 NV_UNIQUE_NAME, KM_SLEEP) == 0); 7703 VERIFY(nvlist_add_uint64(newspa->spa_config_splitting, 7704 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0); 7705 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 7706 B_TRUE)); 7707 } 7708 7709 /* set the props */ 7710 if (props != NULL) { 7711 spa_configfile_set(newspa, props, B_FALSE); 7712 error = spa_prop_set(newspa, props); 7713 if (error) 7714 goto out; 7715 } 7716 7717 /* flush everything */ 7718 txg = spa_vdev_config_enter(newspa); 7719 vdev_config_dirty(newspa->spa_root_vdev); 7720 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 7721 7722 if (zio_injection_enabled) 7723 zio_handle_panic_injection(spa, FTAG, 2); 7724 7725 spa_async_resume(newspa); 7726 7727 /* finally, update the original pool's config */ 7728 txg = spa_vdev_config_enter(spa); 7729 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 7730 error = dmu_tx_assign(tx, TXG_WAIT); 7731 if (error != 0) 7732 dmu_tx_abort(tx); 7733 for (c = 0; c < children; c++) { 7734 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7735 vdev_t *tvd = vml[c]->vdev_top; 7736 7737 /* 7738 * Need to be sure the detachable VDEV is not 7739 * on any *other* txg's DTL list to prevent it 7740 * from being accessed after it's freed. 7741 */ 7742 for (int t = 0; t < TXG_SIZE; t++) { 7743 (void) txg_list_remove_this( 7744 &tvd->vdev_dtl_list, vml[c], t); 7745 } 7746 7747 vdev_split(vml[c]); 7748 if (error == 0) 7749 spa_history_log_internal(spa, "detach", tx, 7750 "vdev=%s", vml[c]->vdev_path); 7751 7752 vdev_free(vml[c]); 7753 } 7754 } 7755 spa->spa_avz_action = AVZ_ACTION_REBUILD; 7756 vdev_config_dirty(spa->spa_root_vdev); 7757 spa->spa_config_splitting = NULL; 7758 nvlist_free(nvl); 7759 if (error == 0) 7760 dmu_tx_commit(tx); 7761 (void) spa_vdev_exit(spa, NULL, txg, 0); 7762 7763 if (zio_injection_enabled) 7764 zio_handle_panic_injection(spa, FTAG, 3); 7765 7766 /* split is complete; log a history record */ 7767 spa_history_log_internal(newspa, "split", NULL, 7768 "from pool %s", spa_name(spa)); 7769 7770 newspa->spa_is_splitting = B_FALSE; 7771 kmem_free(vml, children * sizeof (vdev_t *)); 7772 7773 /* if we're not going to mount the filesystems in userland, export */ 7774 if (exp) 7775 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 7776 B_FALSE, B_FALSE); 7777 7778 return (error); 7779 7780 out: 7781 spa_unload(newspa); 7782 spa_deactivate(newspa); 7783 spa_remove(newspa); 7784 7785 txg = spa_vdev_config_enter(spa); 7786 7787 /* re-online all offlined disks */ 7788 for (c = 0; c < children; c++) { 7789 if (vml[c] != NULL) 7790 vml[c]->vdev_offline = B_FALSE; 7791 } 7792 7793 /* restart initializing or trimming disks as necessary */ 7794 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 7795 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 7796 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 7797 7798 vdev_reopen(spa->spa_root_vdev); 7799 7800 nvlist_free(spa->spa_config_splitting); 7801 spa->spa_config_splitting = NULL; 7802 (void) spa_vdev_exit(spa, NULL, txg, error); 7803 7804 kmem_free(vml, children * sizeof (vdev_t *)); 7805 return (error); 7806 } 7807 7808 /* 7809 * Find any device that's done replacing, or a vdev marked 'unspare' that's 7810 * currently spared, so we can detach it. 7811 */ 7812 static vdev_t * 7813 spa_vdev_resilver_done_hunt(vdev_t *vd) 7814 { 7815 vdev_t *newvd, *oldvd; 7816 7817 for (int c = 0; c < vd->vdev_children; c++) { 7818 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 7819 if (oldvd != NULL) 7820 return (oldvd); 7821 } 7822 7823 /* 7824 * Check for a completed replacement. We always consider the first 7825 * vdev in the list to be the oldest vdev, and the last one to be 7826 * the newest (see spa_vdev_attach() for how that works). In 7827 * the case where the newest vdev is faulted, we will not automatically 7828 * remove it after a resilver completes. This is OK as it will require 7829 * user intervention to determine which disk the admin wishes to keep. 7830 */ 7831 if (vd->vdev_ops == &vdev_replacing_ops) { 7832 ASSERT(vd->vdev_children > 1); 7833 7834 newvd = vd->vdev_child[vd->vdev_children - 1]; 7835 oldvd = vd->vdev_child[0]; 7836 7837 if (vdev_dtl_empty(newvd, DTL_MISSING) && 7838 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7839 !vdev_dtl_required(oldvd)) 7840 return (oldvd); 7841 } 7842 7843 /* 7844 * Check for a completed resilver with the 'unspare' flag set. 7845 * Also potentially update faulted state. 7846 */ 7847 if (vd->vdev_ops == &vdev_spare_ops) { 7848 vdev_t *first = vd->vdev_child[0]; 7849 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 7850 7851 if (last->vdev_unspare) { 7852 oldvd = first; 7853 newvd = last; 7854 } else if (first->vdev_unspare) { 7855 oldvd = last; 7856 newvd = first; 7857 } else { 7858 oldvd = NULL; 7859 } 7860 7861 if (oldvd != NULL && 7862 vdev_dtl_empty(newvd, DTL_MISSING) && 7863 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7864 !vdev_dtl_required(oldvd)) 7865 return (oldvd); 7866 7867 vdev_propagate_state(vd); 7868 7869 /* 7870 * If there are more than two spares attached to a disk, 7871 * and those spares are not required, then we want to 7872 * attempt to free them up now so that they can be used 7873 * by other pools. Once we're back down to a single 7874 * disk+spare, we stop removing them. 7875 */ 7876 if (vd->vdev_children > 2) { 7877 newvd = vd->vdev_child[1]; 7878 7879 if (newvd->vdev_isspare && last->vdev_isspare && 7880 vdev_dtl_empty(last, DTL_MISSING) && 7881 vdev_dtl_empty(last, DTL_OUTAGE) && 7882 !vdev_dtl_required(newvd)) 7883 return (newvd); 7884 } 7885 } 7886 7887 return (NULL); 7888 } 7889 7890 static void 7891 spa_vdev_resilver_done(spa_t *spa) 7892 { 7893 vdev_t *vd, *pvd, *ppvd; 7894 uint64_t guid, sguid, pguid, ppguid; 7895 7896 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7897 7898 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 7899 pvd = vd->vdev_parent; 7900 ppvd = pvd->vdev_parent; 7901 guid = vd->vdev_guid; 7902 pguid = pvd->vdev_guid; 7903 ppguid = ppvd->vdev_guid; 7904 sguid = 0; 7905 /* 7906 * If we have just finished replacing a hot spared device, then 7907 * we need to detach the parent's first child (the original hot 7908 * spare) as well. 7909 */ 7910 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 7911 ppvd->vdev_children == 2) { 7912 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 7913 sguid = ppvd->vdev_child[1]->vdev_guid; 7914 } 7915 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 7916 7917 spa_config_exit(spa, SCL_ALL, FTAG); 7918 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 7919 return; 7920 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 7921 return; 7922 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7923 } 7924 7925 spa_config_exit(spa, SCL_ALL, FTAG); 7926 7927 /* 7928 * If a detach was not performed above replace waiters will not have 7929 * been notified. In which case we must do so now. 7930 */ 7931 spa_notify_waiters(spa); 7932 } 7933 7934 /* 7935 * Update the stored path or FRU for this vdev. 7936 */ 7937 static int 7938 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 7939 boolean_t ispath) 7940 { 7941 vdev_t *vd; 7942 boolean_t sync = B_FALSE; 7943 7944 ASSERT(spa_writeable(spa)); 7945 7946 spa_vdev_state_enter(spa, SCL_ALL); 7947 7948 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 7949 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 7950 7951 if (!vd->vdev_ops->vdev_op_leaf) 7952 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 7953 7954 if (ispath) { 7955 if (strcmp(value, vd->vdev_path) != 0) { 7956 spa_strfree(vd->vdev_path); 7957 vd->vdev_path = spa_strdup(value); 7958 sync = B_TRUE; 7959 } 7960 } else { 7961 if (vd->vdev_fru == NULL) { 7962 vd->vdev_fru = spa_strdup(value); 7963 sync = B_TRUE; 7964 } else if (strcmp(value, vd->vdev_fru) != 0) { 7965 spa_strfree(vd->vdev_fru); 7966 vd->vdev_fru = spa_strdup(value); 7967 sync = B_TRUE; 7968 } 7969 } 7970 7971 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 7972 } 7973 7974 int 7975 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 7976 { 7977 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 7978 } 7979 7980 int 7981 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 7982 { 7983 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 7984 } 7985 7986 /* 7987 * ========================================================================== 7988 * SPA Scanning 7989 * ========================================================================== 7990 */ 7991 int 7992 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 7993 { 7994 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 7995 7996 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 7997 return (SET_ERROR(EBUSY)); 7998 7999 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 8000 } 8001 8002 int 8003 spa_scan_stop(spa_t *spa) 8004 { 8005 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8006 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8007 return (SET_ERROR(EBUSY)); 8008 return (dsl_scan_cancel(spa->spa_dsl_pool)); 8009 } 8010 8011 int 8012 spa_scan(spa_t *spa, pool_scan_func_t func) 8013 { 8014 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8015 8016 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 8017 return (SET_ERROR(ENOTSUP)); 8018 8019 if (func == POOL_SCAN_RESILVER && 8020 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 8021 return (SET_ERROR(ENOTSUP)); 8022 8023 /* 8024 * If a resilver was requested, but there is no DTL on a 8025 * writeable leaf device, we have nothing to do. 8026 */ 8027 if (func == POOL_SCAN_RESILVER && 8028 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 8029 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 8030 return (0); 8031 } 8032 8033 return (dsl_scan(spa->spa_dsl_pool, func)); 8034 } 8035 8036 /* 8037 * ========================================================================== 8038 * SPA async task processing 8039 * ========================================================================== 8040 */ 8041 8042 static void 8043 spa_async_remove(spa_t *spa, vdev_t *vd) 8044 { 8045 if (vd->vdev_remove_wanted) { 8046 vd->vdev_remove_wanted = B_FALSE; 8047 vd->vdev_delayed_close = B_FALSE; 8048 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 8049 8050 /* 8051 * We want to clear the stats, but we don't want to do a full 8052 * vdev_clear() as that will cause us to throw away 8053 * degraded/faulted state as well as attempt to reopen the 8054 * device, all of which is a waste. 8055 */ 8056 vd->vdev_stat.vs_read_errors = 0; 8057 vd->vdev_stat.vs_write_errors = 0; 8058 vd->vdev_stat.vs_checksum_errors = 0; 8059 8060 vdev_state_dirty(vd->vdev_top); 8061 8062 /* Tell userspace that the vdev is gone. */ 8063 zfs_post_remove(spa, vd); 8064 } 8065 8066 for (int c = 0; c < vd->vdev_children; c++) 8067 spa_async_remove(spa, vd->vdev_child[c]); 8068 } 8069 8070 static void 8071 spa_async_probe(spa_t *spa, vdev_t *vd) 8072 { 8073 if (vd->vdev_probe_wanted) { 8074 vd->vdev_probe_wanted = B_FALSE; 8075 vdev_reopen(vd); /* vdev_open() does the actual probe */ 8076 } 8077 8078 for (int c = 0; c < vd->vdev_children; c++) 8079 spa_async_probe(spa, vd->vdev_child[c]); 8080 } 8081 8082 static void 8083 spa_async_autoexpand(spa_t *spa, vdev_t *vd) 8084 { 8085 if (!spa->spa_autoexpand) 8086 return; 8087 8088 for (int c = 0; c < vd->vdev_children; c++) { 8089 vdev_t *cvd = vd->vdev_child[c]; 8090 spa_async_autoexpand(spa, cvd); 8091 } 8092 8093 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 8094 return; 8095 8096 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 8097 } 8098 8099 static void 8100 spa_async_thread(void *arg) 8101 { 8102 spa_t *spa = (spa_t *)arg; 8103 dsl_pool_t *dp = spa->spa_dsl_pool; 8104 int tasks; 8105 8106 ASSERT(spa->spa_sync_on); 8107 8108 mutex_enter(&spa->spa_async_lock); 8109 tasks = spa->spa_async_tasks; 8110 spa->spa_async_tasks = 0; 8111 mutex_exit(&spa->spa_async_lock); 8112 8113 /* 8114 * See if the config needs to be updated. 8115 */ 8116 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 8117 uint64_t old_space, new_space; 8118 8119 mutex_enter(&spa_namespace_lock); 8120 old_space = metaslab_class_get_space(spa_normal_class(spa)); 8121 old_space += metaslab_class_get_space(spa_special_class(spa)); 8122 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 8123 old_space += metaslab_class_get_space( 8124 spa_embedded_log_class(spa)); 8125 8126 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 8127 8128 new_space = metaslab_class_get_space(spa_normal_class(spa)); 8129 new_space += metaslab_class_get_space(spa_special_class(spa)); 8130 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 8131 new_space += metaslab_class_get_space( 8132 spa_embedded_log_class(spa)); 8133 mutex_exit(&spa_namespace_lock); 8134 8135 /* 8136 * If the pool grew as a result of the config update, 8137 * then log an internal history event. 8138 */ 8139 if (new_space != old_space) { 8140 spa_history_log_internal(spa, "vdev online", NULL, 8141 "pool '%s' size: %llu(+%llu)", 8142 spa_name(spa), (u_longlong_t)new_space, 8143 (u_longlong_t)(new_space - old_space)); 8144 } 8145 } 8146 8147 /* 8148 * See if any devices need to be marked REMOVED. 8149 */ 8150 if (tasks & SPA_ASYNC_REMOVE) { 8151 spa_vdev_state_enter(spa, SCL_NONE); 8152 spa_async_remove(spa, spa->spa_root_vdev); 8153 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 8154 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); 8155 for (int i = 0; i < spa->spa_spares.sav_count; i++) 8156 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); 8157 (void) spa_vdev_state_exit(spa, NULL, 0); 8158 } 8159 8160 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 8161 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8162 spa_async_autoexpand(spa, spa->spa_root_vdev); 8163 spa_config_exit(spa, SCL_CONFIG, FTAG); 8164 } 8165 8166 /* 8167 * See if any devices need to be probed. 8168 */ 8169 if (tasks & SPA_ASYNC_PROBE) { 8170 spa_vdev_state_enter(spa, SCL_NONE); 8171 spa_async_probe(spa, spa->spa_root_vdev); 8172 (void) spa_vdev_state_exit(spa, NULL, 0); 8173 } 8174 8175 /* 8176 * If any devices are done replacing, detach them. 8177 */ 8178 if (tasks & SPA_ASYNC_RESILVER_DONE || 8179 tasks & SPA_ASYNC_REBUILD_DONE) { 8180 spa_vdev_resilver_done(spa); 8181 } 8182 8183 /* 8184 * Kick off a resilver. 8185 */ 8186 if (tasks & SPA_ASYNC_RESILVER && 8187 !vdev_rebuild_active(spa->spa_root_vdev) && 8188 (!dsl_scan_resilvering(dp) || 8189 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 8190 dsl_scan_restart_resilver(dp, 0); 8191 8192 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 8193 mutex_enter(&spa_namespace_lock); 8194 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8195 vdev_initialize_restart(spa->spa_root_vdev); 8196 spa_config_exit(spa, SCL_CONFIG, FTAG); 8197 mutex_exit(&spa_namespace_lock); 8198 } 8199 8200 if (tasks & SPA_ASYNC_TRIM_RESTART) { 8201 mutex_enter(&spa_namespace_lock); 8202 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8203 vdev_trim_restart(spa->spa_root_vdev); 8204 spa_config_exit(spa, SCL_CONFIG, FTAG); 8205 mutex_exit(&spa_namespace_lock); 8206 } 8207 8208 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 8209 mutex_enter(&spa_namespace_lock); 8210 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8211 vdev_autotrim_restart(spa); 8212 spa_config_exit(spa, SCL_CONFIG, FTAG); 8213 mutex_exit(&spa_namespace_lock); 8214 } 8215 8216 /* 8217 * Kick off L2 cache whole device TRIM. 8218 */ 8219 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 8220 mutex_enter(&spa_namespace_lock); 8221 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8222 vdev_trim_l2arc(spa); 8223 spa_config_exit(spa, SCL_CONFIG, FTAG); 8224 mutex_exit(&spa_namespace_lock); 8225 } 8226 8227 /* 8228 * Kick off L2 cache rebuilding. 8229 */ 8230 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 8231 mutex_enter(&spa_namespace_lock); 8232 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 8233 l2arc_spa_rebuild_start(spa); 8234 spa_config_exit(spa, SCL_L2ARC, FTAG); 8235 mutex_exit(&spa_namespace_lock); 8236 } 8237 8238 /* 8239 * Let the world know that we're done. 8240 */ 8241 mutex_enter(&spa->spa_async_lock); 8242 spa->spa_async_thread = NULL; 8243 cv_broadcast(&spa->spa_async_cv); 8244 mutex_exit(&spa->spa_async_lock); 8245 thread_exit(); 8246 } 8247 8248 void 8249 spa_async_suspend(spa_t *spa) 8250 { 8251 mutex_enter(&spa->spa_async_lock); 8252 spa->spa_async_suspended++; 8253 while (spa->spa_async_thread != NULL) 8254 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 8255 mutex_exit(&spa->spa_async_lock); 8256 8257 spa_vdev_remove_suspend(spa); 8258 8259 zthr_t *condense_thread = spa->spa_condense_zthr; 8260 if (condense_thread != NULL) 8261 zthr_cancel(condense_thread); 8262 8263 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8264 if (discard_thread != NULL) 8265 zthr_cancel(discard_thread); 8266 8267 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8268 if (ll_delete_thread != NULL) 8269 zthr_cancel(ll_delete_thread); 8270 8271 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8272 if (ll_condense_thread != NULL) 8273 zthr_cancel(ll_condense_thread); 8274 } 8275 8276 void 8277 spa_async_resume(spa_t *spa) 8278 { 8279 mutex_enter(&spa->spa_async_lock); 8280 ASSERT(spa->spa_async_suspended != 0); 8281 spa->spa_async_suspended--; 8282 mutex_exit(&spa->spa_async_lock); 8283 spa_restart_removal(spa); 8284 8285 zthr_t *condense_thread = spa->spa_condense_zthr; 8286 if (condense_thread != NULL) 8287 zthr_resume(condense_thread); 8288 8289 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8290 if (discard_thread != NULL) 8291 zthr_resume(discard_thread); 8292 8293 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8294 if (ll_delete_thread != NULL) 8295 zthr_resume(ll_delete_thread); 8296 8297 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8298 if (ll_condense_thread != NULL) 8299 zthr_resume(ll_condense_thread); 8300 } 8301 8302 static boolean_t 8303 spa_async_tasks_pending(spa_t *spa) 8304 { 8305 uint_t non_config_tasks; 8306 uint_t config_task; 8307 boolean_t config_task_suspended; 8308 8309 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 8310 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 8311 if (spa->spa_ccw_fail_time == 0) { 8312 config_task_suspended = B_FALSE; 8313 } else { 8314 config_task_suspended = 8315 (gethrtime() - spa->spa_ccw_fail_time) < 8316 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 8317 } 8318 8319 return (non_config_tasks || (config_task && !config_task_suspended)); 8320 } 8321 8322 static void 8323 spa_async_dispatch(spa_t *spa) 8324 { 8325 mutex_enter(&spa->spa_async_lock); 8326 if (spa_async_tasks_pending(spa) && 8327 !spa->spa_async_suspended && 8328 spa->spa_async_thread == NULL) 8329 spa->spa_async_thread = thread_create(NULL, 0, 8330 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 8331 mutex_exit(&spa->spa_async_lock); 8332 } 8333 8334 void 8335 spa_async_request(spa_t *spa, int task) 8336 { 8337 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); 8338 mutex_enter(&spa->spa_async_lock); 8339 spa->spa_async_tasks |= task; 8340 mutex_exit(&spa->spa_async_lock); 8341 } 8342 8343 int 8344 spa_async_tasks(spa_t *spa) 8345 { 8346 return (spa->spa_async_tasks); 8347 } 8348 8349 /* 8350 * ========================================================================== 8351 * SPA syncing routines 8352 * ========================================================================== 8353 */ 8354 8355 8356 static int 8357 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8358 dmu_tx_t *tx) 8359 { 8360 bpobj_t *bpo = arg; 8361 bpobj_enqueue(bpo, bp, bp_freed, tx); 8362 return (0); 8363 } 8364 8365 int 8366 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8367 { 8368 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 8369 } 8370 8371 int 8372 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8373 { 8374 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 8375 } 8376 8377 static int 8378 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8379 { 8380 zio_t *pio = arg; 8381 8382 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 8383 pio->io_flags)); 8384 return (0); 8385 } 8386 8387 static int 8388 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8389 dmu_tx_t *tx) 8390 { 8391 ASSERT(!bp_freed); 8392 return (spa_free_sync_cb(arg, bp, tx)); 8393 } 8394 8395 /* 8396 * Note: this simple function is not inlined to make it easier to dtrace the 8397 * amount of time spent syncing frees. 8398 */ 8399 static void 8400 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 8401 { 8402 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8403 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 8404 VERIFY(zio_wait(zio) == 0); 8405 } 8406 8407 /* 8408 * Note: this simple function is not inlined to make it easier to dtrace the 8409 * amount of time spent syncing deferred frees. 8410 */ 8411 static void 8412 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 8413 { 8414 if (spa_sync_pass(spa) != 1) 8415 return; 8416 8417 /* 8418 * Note: 8419 * If the log space map feature is active, we stop deferring 8420 * frees to the next TXG and therefore running this function 8421 * would be considered a no-op as spa_deferred_bpobj should 8422 * not have any entries. 8423 * 8424 * That said we run this function anyway (instead of returning 8425 * immediately) for the edge-case scenario where we just 8426 * activated the log space map feature in this TXG but we have 8427 * deferred frees from the previous TXG. 8428 */ 8429 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8430 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 8431 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 8432 VERIFY0(zio_wait(zio)); 8433 } 8434 8435 static void 8436 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 8437 { 8438 char *packed = NULL; 8439 size_t bufsize; 8440 size_t nvsize = 0; 8441 dmu_buf_t *db; 8442 8443 VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); 8444 8445 /* 8446 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 8447 * information. This avoids the dmu_buf_will_dirty() path and 8448 * saves us a pre-read to get data we don't actually care about. 8449 */ 8450 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 8451 packed = vmem_alloc(bufsize, KM_SLEEP); 8452 8453 VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 8454 KM_SLEEP) == 0); 8455 bzero(packed + nvsize, bufsize - nvsize); 8456 8457 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); 8458 8459 vmem_free(packed, bufsize); 8460 8461 VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 8462 dmu_buf_will_dirty(db, tx); 8463 *(uint64_t *)db->db_data = nvsize; 8464 dmu_buf_rele(db, FTAG); 8465 } 8466 8467 static void 8468 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 8469 const char *config, const char *entry) 8470 { 8471 nvlist_t *nvroot; 8472 nvlist_t **list; 8473 int i; 8474 8475 if (!sav->sav_sync) 8476 return; 8477 8478 /* 8479 * Update the MOS nvlist describing the list of available devices. 8480 * spa_validate_aux() will have already made sure this nvlist is 8481 * valid and the vdevs are labeled appropriately. 8482 */ 8483 if (sav->sav_object == 0) { 8484 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 8485 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 8486 sizeof (uint64_t), tx); 8487 VERIFY(zap_update(spa->spa_meta_objset, 8488 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 8489 &sav->sav_object, tx) == 0); 8490 } 8491 8492 VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); 8493 if (sav->sav_count == 0) { 8494 VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0); 8495 } else { 8496 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 8497 for (i = 0; i < sav->sav_count; i++) 8498 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 8499 B_FALSE, VDEV_CONFIG_L2CACHE); 8500 VERIFY(nvlist_add_nvlist_array(nvroot, config, list, 8501 sav->sav_count) == 0); 8502 for (i = 0; i < sav->sav_count; i++) 8503 nvlist_free(list[i]); 8504 kmem_free(list, sav->sav_count * sizeof (void *)); 8505 } 8506 8507 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 8508 nvlist_free(nvroot); 8509 8510 sav->sav_sync = B_FALSE; 8511 } 8512 8513 /* 8514 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 8515 * The all-vdev ZAP must be empty. 8516 */ 8517 static void 8518 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 8519 { 8520 spa_t *spa = vd->vdev_spa; 8521 8522 if (vd->vdev_top_zap != 0) { 8523 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8524 vd->vdev_top_zap, tx)); 8525 } 8526 if (vd->vdev_leaf_zap != 0) { 8527 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8528 vd->vdev_leaf_zap, tx)); 8529 } 8530 for (uint64_t i = 0; i < vd->vdev_children; i++) { 8531 spa_avz_build(vd->vdev_child[i], avz, tx); 8532 } 8533 } 8534 8535 static void 8536 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 8537 { 8538 nvlist_t *config; 8539 8540 /* 8541 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 8542 * its config may not be dirty but we still need to build per-vdev ZAPs. 8543 * Similarly, if the pool is being assembled (e.g. after a split), we 8544 * need to rebuild the AVZ although the config may not be dirty. 8545 */ 8546 if (list_is_empty(&spa->spa_config_dirty_list) && 8547 spa->spa_avz_action == AVZ_ACTION_NONE) 8548 return; 8549 8550 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 8551 8552 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 8553 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 8554 spa->spa_all_vdev_zaps != 0); 8555 8556 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 8557 /* Make and build the new AVZ */ 8558 uint64_t new_avz = zap_create(spa->spa_meta_objset, 8559 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 8560 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 8561 8562 /* Diff old AVZ with new one */ 8563 zap_cursor_t zc; 8564 zap_attribute_t za; 8565 8566 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8567 spa->spa_all_vdev_zaps); 8568 zap_cursor_retrieve(&zc, &za) == 0; 8569 zap_cursor_advance(&zc)) { 8570 uint64_t vdzap = za.za_first_integer; 8571 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 8572 vdzap) == ENOENT) { 8573 /* 8574 * ZAP is listed in old AVZ but not in new one; 8575 * destroy it 8576 */ 8577 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 8578 tx)); 8579 } 8580 } 8581 8582 zap_cursor_fini(&zc); 8583 8584 /* Destroy the old AVZ */ 8585 VERIFY0(zap_destroy(spa->spa_meta_objset, 8586 spa->spa_all_vdev_zaps, tx)); 8587 8588 /* Replace the old AVZ in the dir obj with the new one */ 8589 VERIFY0(zap_update(spa->spa_meta_objset, 8590 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 8591 sizeof (new_avz), 1, &new_avz, tx)); 8592 8593 spa->spa_all_vdev_zaps = new_avz; 8594 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 8595 zap_cursor_t zc; 8596 zap_attribute_t za; 8597 8598 /* Walk through the AVZ and destroy all listed ZAPs */ 8599 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8600 spa->spa_all_vdev_zaps); 8601 zap_cursor_retrieve(&zc, &za) == 0; 8602 zap_cursor_advance(&zc)) { 8603 uint64_t zap = za.za_first_integer; 8604 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 8605 } 8606 8607 zap_cursor_fini(&zc); 8608 8609 /* Destroy and unlink the AVZ itself */ 8610 VERIFY0(zap_destroy(spa->spa_meta_objset, 8611 spa->spa_all_vdev_zaps, tx)); 8612 VERIFY0(zap_remove(spa->spa_meta_objset, 8613 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 8614 spa->spa_all_vdev_zaps = 0; 8615 } 8616 8617 if (spa->spa_all_vdev_zaps == 0) { 8618 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 8619 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 8620 DMU_POOL_VDEV_ZAP_MAP, tx); 8621 } 8622 spa->spa_avz_action = AVZ_ACTION_NONE; 8623 8624 /* Create ZAPs for vdevs that don't have them. */ 8625 vdev_construct_zaps(spa->spa_root_vdev, tx); 8626 8627 config = spa_config_generate(spa, spa->spa_root_vdev, 8628 dmu_tx_get_txg(tx), B_FALSE); 8629 8630 /* 8631 * If we're upgrading the spa version then make sure that 8632 * the config object gets updated with the correct version. 8633 */ 8634 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 8635 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 8636 spa->spa_uberblock.ub_version); 8637 8638 spa_config_exit(spa, SCL_STATE, FTAG); 8639 8640 nvlist_free(spa->spa_config_syncing); 8641 spa->spa_config_syncing = config; 8642 8643 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 8644 } 8645 8646 static void 8647 spa_sync_version(void *arg, dmu_tx_t *tx) 8648 { 8649 uint64_t *versionp = arg; 8650 uint64_t version = *versionp; 8651 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8652 8653 /* 8654 * Setting the version is special cased when first creating the pool. 8655 */ 8656 ASSERT(tx->tx_txg != TXG_INITIAL); 8657 8658 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 8659 ASSERT(version >= spa_version(spa)); 8660 8661 spa->spa_uberblock.ub_version = version; 8662 vdev_config_dirty(spa->spa_root_vdev); 8663 spa_history_log_internal(spa, "set", tx, "version=%lld", 8664 (longlong_t)version); 8665 } 8666 8667 /* 8668 * Set zpool properties. 8669 */ 8670 static void 8671 spa_sync_props(void *arg, dmu_tx_t *tx) 8672 { 8673 nvlist_t *nvp = arg; 8674 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8675 objset_t *mos = spa->spa_meta_objset; 8676 nvpair_t *elem = NULL; 8677 8678 mutex_enter(&spa->spa_props_lock); 8679 8680 while ((elem = nvlist_next_nvpair(nvp, elem))) { 8681 uint64_t intval; 8682 char *strval, *fname; 8683 zpool_prop_t prop; 8684 const char *propname; 8685 zprop_type_t proptype; 8686 spa_feature_t fid; 8687 8688 switch (prop = zpool_name_to_prop(nvpair_name(elem))) { 8689 case ZPOOL_PROP_INVAL: 8690 /* 8691 * We checked this earlier in spa_prop_validate(). 8692 */ 8693 ASSERT(zpool_prop_feature(nvpair_name(elem))); 8694 8695 fname = strchr(nvpair_name(elem), '@') + 1; 8696 VERIFY0(zfeature_lookup_name(fname, &fid)); 8697 8698 spa_feature_enable(spa, fid, tx); 8699 spa_history_log_internal(spa, "set", tx, 8700 "%s=enabled", nvpair_name(elem)); 8701 break; 8702 8703 case ZPOOL_PROP_VERSION: 8704 intval = fnvpair_value_uint64(elem); 8705 /* 8706 * The version is synced separately before other 8707 * properties and should be correct by now. 8708 */ 8709 ASSERT3U(spa_version(spa), >=, intval); 8710 break; 8711 8712 case ZPOOL_PROP_ALTROOT: 8713 /* 8714 * 'altroot' is a non-persistent property. It should 8715 * have been set temporarily at creation or import time. 8716 */ 8717 ASSERT(spa->spa_root != NULL); 8718 break; 8719 8720 case ZPOOL_PROP_READONLY: 8721 case ZPOOL_PROP_CACHEFILE: 8722 /* 8723 * 'readonly' and 'cachefile' are also non-persistent 8724 * properties. 8725 */ 8726 break; 8727 case ZPOOL_PROP_COMMENT: 8728 strval = fnvpair_value_string(elem); 8729 if (spa->spa_comment != NULL) 8730 spa_strfree(spa->spa_comment); 8731 spa->spa_comment = spa_strdup(strval); 8732 /* 8733 * We need to dirty the configuration on all the vdevs 8734 * so that their labels get updated. We also need to 8735 * update the cache file to keep it in sync with the 8736 * MOS version. It's unnecessary to do this for pool 8737 * creation since the vdev's configuration has already 8738 * been dirtied. 8739 */ 8740 if (tx->tx_txg != TXG_INITIAL) { 8741 vdev_config_dirty(spa->spa_root_vdev); 8742 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8743 } 8744 spa_history_log_internal(spa, "set", tx, 8745 "%s=%s", nvpair_name(elem), strval); 8746 break; 8747 case ZPOOL_PROP_COMPATIBILITY: 8748 strval = fnvpair_value_string(elem); 8749 if (spa->spa_compatibility != NULL) 8750 spa_strfree(spa->spa_compatibility); 8751 spa->spa_compatibility = spa_strdup(strval); 8752 /* 8753 * Dirty the configuration on vdevs as above. 8754 */ 8755 if (tx->tx_txg != TXG_INITIAL) { 8756 vdev_config_dirty(spa->spa_root_vdev); 8757 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8758 } 8759 8760 spa_history_log_internal(spa, "set", tx, 8761 "%s=%s", nvpair_name(elem), strval); 8762 break; 8763 8764 default: 8765 /* 8766 * Set pool property values in the poolprops mos object. 8767 */ 8768 if (spa->spa_pool_props_object == 0) { 8769 spa->spa_pool_props_object = 8770 zap_create_link(mos, DMU_OT_POOL_PROPS, 8771 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 8772 tx); 8773 } 8774 8775 /* normalize the property name */ 8776 propname = zpool_prop_to_name(prop); 8777 proptype = zpool_prop_get_type(prop); 8778 8779 if (nvpair_type(elem) == DATA_TYPE_STRING) { 8780 ASSERT(proptype == PROP_TYPE_STRING); 8781 strval = fnvpair_value_string(elem); 8782 VERIFY0(zap_update(mos, 8783 spa->spa_pool_props_object, propname, 8784 1, strlen(strval) + 1, strval, tx)); 8785 spa_history_log_internal(spa, "set", tx, 8786 "%s=%s", nvpair_name(elem), strval); 8787 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 8788 intval = fnvpair_value_uint64(elem); 8789 8790 if (proptype == PROP_TYPE_INDEX) { 8791 const char *unused; 8792 VERIFY0(zpool_prop_index_to_string( 8793 prop, intval, &unused)); 8794 } 8795 VERIFY0(zap_update(mos, 8796 spa->spa_pool_props_object, propname, 8797 8, 1, &intval, tx)); 8798 spa_history_log_internal(spa, "set", tx, 8799 "%s=%lld", nvpair_name(elem), 8800 (longlong_t)intval); 8801 } else { 8802 ASSERT(0); /* not allowed */ 8803 } 8804 8805 switch (prop) { 8806 case ZPOOL_PROP_DELEGATION: 8807 spa->spa_delegation = intval; 8808 break; 8809 case ZPOOL_PROP_BOOTFS: 8810 spa->spa_bootfs = intval; 8811 break; 8812 case ZPOOL_PROP_FAILUREMODE: 8813 spa->spa_failmode = intval; 8814 break; 8815 case ZPOOL_PROP_AUTOTRIM: 8816 spa->spa_autotrim = intval; 8817 spa_async_request(spa, 8818 SPA_ASYNC_AUTOTRIM_RESTART); 8819 break; 8820 case ZPOOL_PROP_AUTOEXPAND: 8821 spa->spa_autoexpand = intval; 8822 if (tx->tx_txg != TXG_INITIAL) 8823 spa_async_request(spa, 8824 SPA_ASYNC_AUTOEXPAND); 8825 break; 8826 case ZPOOL_PROP_MULTIHOST: 8827 spa->spa_multihost = intval; 8828 break; 8829 default: 8830 break; 8831 } 8832 } 8833 8834 } 8835 8836 mutex_exit(&spa->spa_props_lock); 8837 } 8838 8839 /* 8840 * Perform one-time upgrade on-disk changes. spa_version() does not 8841 * reflect the new version this txg, so there must be no changes this 8842 * txg to anything that the upgrade code depends on after it executes. 8843 * Therefore this must be called after dsl_pool_sync() does the sync 8844 * tasks. 8845 */ 8846 static void 8847 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 8848 { 8849 if (spa_sync_pass(spa) != 1) 8850 return; 8851 8852 dsl_pool_t *dp = spa->spa_dsl_pool; 8853 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 8854 8855 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && 8856 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { 8857 dsl_pool_create_origin(dp, tx); 8858 8859 /* Keeping the origin open increases spa_minref */ 8860 spa->spa_minref += 3; 8861 } 8862 8863 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && 8864 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { 8865 dsl_pool_upgrade_clones(dp, tx); 8866 } 8867 8868 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && 8869 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { 8870 dsl_pool_upgrade_dir_clones(dp, tx); 8871 8872 /* Keeping the freedir open increases spa_minref */ 8873 spa->spa_minref += 3; 8874 } 8875 8876 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && 8877 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8878 spa_feature_create_zap_objects(spa, tx); 8879 } 8880 8881 /* 8882 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 8883 * when possibility to use lz4 compression for metadata was added 8884 * Old pools that have this feature enabled must be upgraded to have 8885 * this feature active 8886 */ 8887 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8888 boolean_t lz4_en = spa_feature_is_enabled(spa, 8889 SPA_FEATURE_LZ4_COMPRESS); 8890 boolean_t lz4_ac = spa_feature_is_active(spa, 8891 SPA_FEATURE_LZ4_COMPRESS); 8892 8893 if (lz4_en && !lz4_ac) 8894 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 8895 } 8896 8897 /* 8898 * If we haven't written the salt, do so now. Note that the 8899 * feature may not be activated yet, but that's fine since 8900 * the presence of this ZAP entry is backwards compatible. 8901 */ 8902 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 8903 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 8904 VERIFY0(zap_add(spa->spa_meta_objset, 8905 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 8906 sizeof (spa->spa_cksum_salt.zcs_bytes), 8907 spa->spa_cksum_salt.zcs_bytes, tx)); 8908 } 8909 8910 rrw_exit(&dp->dp_config_rwlock, FTAG); 8911 } 8912 8913 static void 8914 vdev_indirect_state_sync_verify(vdev_t *vd) 8915 { 8916 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 8917 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 8918 8919 if (vd->vdev_ops == &vdev_indirect_ops) { 8920 ASSERT(vim != NULL); 8921 ASSERT(vib != NULL); 8922 } 8923 8924 uint64_t obsolete_sm_object = 0; 8925 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 8926 if (obsolete_sm_object != 0) { 8927 ASSERT(vd->vdev_obsolete_sm != NULL); 8928 ASSERT(vd->vdev_removing || 8929 vd->vdev_ops == &vdev_indirect_ops); 8930 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 8931 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 8932 ASSERT3U(obsolete_sm_object, ==, 8933 space_map_object(vd->vdev_obsolete_sm)); 8934 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 8935 space_map_allocated(vd->vdev_obsolete_sm)); 8936 } 8937 ASSERT(vd->vdev_obsolete_segments != NULL); 8938 8939 /* 8940 * Since frees / remaps to an indirect vdev can only 8941 * happen in syncing context, the obsolete segments 8942 * tree must be empty when we start syncing. 8943 */ 8944 ASSERT0(range_tree_space(vd->vdev_obsolete_segments)); 8945 } 8946 8947 /* 8948 * Set the top-level vdev's max queue depth. Evaluate each top-level's 8949 * async write queue depth in case it changed. The max queue depth will 8950 * not change in the middle of syncing out this txg. 8951 */ 8952 static void 8953 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 8954 { 8955 ASSERT(spa_writeable(spa)); 8956 8957 vdev_t *rvd = spa->spa_root_vdev; 8958 uint32_t max_queue_depth = zfs_vdev_async_write_max_active * 8959 zfs_vdev_queue_depth_pct / 100; 8960 metaslab_class_t *normal = spa_normal_class(spa); 8961 metaslab_class_t *special = spa_special_class(spa); 8962 metaslab_class_t *dedup = spa_dedup_class(spa); 8963 8964 uint64_t slots_per_allocator = 0; 8965 for (int c = 0; c < rvd->vdev_children; c++) { 8966 vdev_t *tvd = rvd->vdev_child[c]; 8967 8968 metaslab_group_t *mg = tvd->vdev_mg; 8969 if (mg == NULL || !metaslab_group_initialized(mg)) 8970 continue; 8971 8972 metaslab_class_t *mc = mg->mg_class; 8973 if (mc != normal && mc != special && mc != dedup) 8974 continue; 8975 8976 /* 8977 * It is safe to do a lock-free check here because only async 8978 * allocations look at mg_max_alloc_queue_depth, and async 8979 * allocations all happen from spa_sync(). 8980 */ 8981 for (int i = 0; i < mg->mg_allocators; i++) { 8982 ASSERT0(zfs_refcount_count( 8983 &(mg->mg_allocator[i].mga_alloc_queue_depth))); 8984 } 8985 mg->mg_max_alloc_queue_depth = max_queue_depth; 8986 8987 for (int i = 0; i < mg->mg_allocators; i++) { 8988 mg->mg_allocator[i].mga_cur_max_alloc_queue_depth = 8989 zfs_vdev_def_queue_depth; 8990 } 8991 slots_per_allocator += zfs_vdev_def_queue_depth; 8992 } 8993 8994 for (int i = 0; i < spa->spa_alloc_count; i++) { 8995 ASSERT0(zfs_refcount_count(&normal->mc_allocator[i]. 8996 mca_alloc_slots)); 8997 ASSERT0(zfs_refcount_count(&special->mc_allocator[i]. 8998 mca_alloc_slots)); 8999 ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i]. 9000 mca_alloc_slots)); 9001 normal->mc_allocator[i].mca_alloc_max_slots = 9002 slots_per_allocator; 9003 special->mc_allocator[i].mca_alloc_max_slots = 9004 slots_per_allocator; 9005 dedup->mc_allocator[i].mca_alloc_max_slots = 9006 slots_per_allocator; 9007 } 9008 normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9009 special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9010 dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9011 } 9012 9013 static void 9014 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 9015 { 9016 ASSERT(spa_writeable(spa)); 9017 9018 vdev_t *rvd = spa->spa_root_vdev; 9019 for (int c = 0; c < rvd->vdev_children; c++) { 9020 vdev_t *vd = rvd->vdev_child[c]; 9021 vdev_indirect_state_sync_verify(vd); 9022 9023 if (vdev_indirect_should_condense(vd)) { 9024 spa_condense_indirect_start_sync(vd, tx); 9025 break; 9026 } 9027 } 9028 } 9029 9030 static void 9031 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 9032 { 9033 objset_t *mos = spa->spa_meta_objset; 9034 dsl_pool_t *dp = spa->spa_dsl_pool; 9035 uint64_t txg = tx->tx_txg; 9036 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 9037 9038 do { 9039 int pass = ++spa->spa_sync_pass; 9040 9041 spa_sync_config_object(spa, tx); 9042 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 9043 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 9044 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 9045 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 9046 spa_errlog_sync(spa, txg); 9047 dsl_pool_sync(dp, txg); 9048 9049 if (pass < zfs_sync_pass_deferred_free || 9050 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 9051 /* 9052 * If the log space map feature is active we don't 9053 * care about deferred frees and the deferred bpobj 9054 * as the log space map should effectively have the 9055 * same results (i.e. appending only to one object). 9056 */ 9057 spa_sync_frees(spa, free_bpl, tx); 9058 } else { 9059 /* 9060 * We can not defer frees in pass 1, because 9061 * we sync the deferred frees later in pass 1. 9062 */ 9063 ASSERT3U(pass, >, 1); 9064 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 9065 &spa->spa_deferred_bpobj, tx); 9066 } 9067 9068 ddt_sync(spa, txg); 9069 dsl_scan_sync(dp, tx); 9070 svr_sync(spa, tx); 9071 spa_sync_upgrades(spa, tx); 9072 9073 spa_flush_metaslabs(spa, tx); 9074 9075 vdev_t *vd = NULL; 9076 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 9077 != NULL) 9078 vdev_sync(vd, txg); 9079 9080 /* 9081 * Note: We need to check if the MOS is dirty because we could 9082 * have marked the MOS dirty without updating the uberblock 9083 * (e.g. if we have sync tasks but no dirty user data). We need 9084 * to check the uberblock's rootbp because it is updated if we 9085 * have synced out dirty data (though in this case the MOS will 9086 * most likely also be dirty due to second order effects, we 9087 * don't want to rely on that here). 9088 */ 9089 if (pass == 1 && 9090 spa->spa_uberblock.ub_rootbp.blk_birth < txg && 9091 !dmu_objset_is_dirty(mos, txg)) { 9092 /* 9093 * Nothing changed on the first pass, therefore this 9094 * TXG is a no-op. Avoid syncing deferred frees, so 9095 * that we can keep this TXG as a no-op. 9096 */ 9097 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9098 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9099 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 9100 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 9101 break; 9102 } 9103 9104 spa_sync_deferred_frees(spa, tx); 9105 } while (dmu_objset_is_dirty(mos, txg)); 9106 } 9107 9108 /* 9109 * Rewrite the vdev configuration (which includes the uberblock) to 9110 * commit the transaction group. 9111 * 9112 * If there are no dirty vdevs, we sync the uberblock to a few random 9113 * top-level vdevs that are known to be visible in the config cache 9114 * (see spa_vdev_add() for a complete description). If there *are* dirty 9115 * vdevs, sync the uberblock to all vdevs. 9116 */ 9117 static void 9118 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 9119 { 9120 vdev_t *rvd = spa->spa_root_vdev; 9121 uint64_t txg = tx->tx_txg; 9122 9123 for (;;) { 9124 int error = 0; 9125 9126 /* 9127 * We hold SCL_STATE to prevent vdev open/close/etc. 9128 * while we're attempting to write the vdev labels. 9129 */ 9130 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9131 9132 if (list_is_empty(&spa->spa_config_dirty_list)) { 9133 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 9134 int svdcount = 0; 9135 int children = rvd->vdev_children; 9136 int c0 = random_in_range(children); 9137 9138 for (int c = 0; c < children; c++) { 9139 vdev_t *vd = 9140 rvd->vdev_child[(c0 + c) % children]; 9141 9142 /* Stop when revisiting the first vdev */ 9143 if (c > 0 && svd[0] == vd) 9144 break; 9145 9146 if (vd->vdev_ms_array == 0 || 9147 vd->vdev_islog || 9148 !vdev_is_concrete(vd)) 9149 continue; 9150 9151 svd[svdcount++] = vd; 9152 if (svdcount == SPA_SYNC_MIN_VDEVS) 9153 break; 9154 } 9155 error = vdev_config_sync(svd, svdcount, txg); 9156 } else { 9157 error = vdev_config_sync(rvd->vdev_child, 9158 rvd->vdev_children, txg); 9159 } 9160 9161 if (error == 0) 9162 spa->spa_last_synced_guid = rvd->vdev_guid; 9163 9164 spa_config_exit(spa, SCL_STATE, FTAG); 9165 9166 if (error == 0) 9167 break; 9168 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 9169 zio_resume_wait(spa); 9170 } 9171 } 9172 9173 /* 9174 * Sync the specified transaction group. New blocks may be dirtied as 9175 * part of the process, so we iterate until it converges. 9176 */ 9177 void 9178 spa_sync(spa_t *spa, uint64_t txg) 9179 { 9180 vdev_t *vd = NULL; 9181 9182 VERIFY(spa_writeable(spa)); 9183 9184 /* 9185 * Wait for i/os issued in open context that need to complete 9186 * before this txg syncs. 9187 */ 9188 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 9189 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 9190 ZIO_FLAG_CANFAIL); 9191 9192 /* 9193 * Lock out configuration changes. 9194 */ 9195 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9196 9197 spa->spa_syncing_txg = txg; 9198 spa->spa_sync_pass = 0; 9199 9200 for (int i = 0; i < spa->spa_alloc_count; i++) { 9201 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9202 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9203 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9204 } 9205 9206 /* 9207 * If there are any pending vdev state changes, convert them 9208 * into config changes that go out with this transaction group. 9209 */ 9210 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9211 while (list_head(&spa->spa_state_dirty_list) != NULL) { 9212 /* 9213 * We need the write lock here because, for aux vdevs, 9214 * calling vdev_config_dirty() modifies sav_config. 9215 * This is ugly and will become unnecessary when we 9216 * eliminate the aux vdev wart by integrating all vdevs 9217 * into the root vdev tree. 9218 */ 9219 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9220 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 9221 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9222 vdev_state_clean(vd); 9223 vdev_config_dirty(vd); 9224 } 9225 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9226 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9227 } 9228 spa_config_exit(spa, SCL_STATE, FTAG); 9229 9230 dsl_pool_t *dp = spa->spa_dsl_pool; 9231 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 9232 9233 spa->spa_sync_starttime = gethrtime(); 9234 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9235 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 9236 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 9237 NSEC_TO_TICK(spa->spa_deadman_synctime)); 9238 9239 /* 9240 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 9241 * set spa_deflate if we have no raid-z vdevs. 9242 */ 9243 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 9244 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 9245 vdev_t *rvd = spa->spa_root_vdev; 9246 9247 int i; 9248 for (i = 0; i < rvd->vdev_children; i++) { 9249 vd = rvd->vdev_child[i]; 9250 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 9251 break; 9252 } 9253 if (i == rvd->vdev_children) { 9254 spa->spa_deflate = TRUE; 9255 VERIFY0(zap_add(spa->spa_meta_objset, 9256 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 9257 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 9258 } 9259 } 9260 9261 spa_sync_adjust_vdev_max_queue_depth(spa); 9262 9263 spa_sync_condense_indirect(spa, tx); 9264 9265 spa_sync_iterate_to_convergence(spa, tx); 9266 9267 #ifdef ZFS_DEBUG 9268 if (!list_is_empty(&spa->spa_config_dirty_list)) { 9269 /* 9270 * Make sure that the number of ZAPs for all the vdevs matches 9271 * the number of ZAPs in the per-vdev ZAP list. This only gets 9272 * called if the config is dirty; otherwise there may be 9273 * outstanding AVZ operations that weren't completed in 9274 * spa_sync_config_object. 9275 */ 9276 uint64_t all_vdev_zap_entry_count; 9277 ASSERT0(zap_count(spa->spa_meta_objset, 9278 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 9279 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 9280 all_vdev_zap_entry_count); 9281 } 9282 #endif 9283 9284 if (spa->spa_vdev_removal != NULL) { 9285 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 9286 } 9287 9288 spa_sync_rewrite_vdev_config(spa, tx); 9289 dmu_tx_commit(tx); 9290 9291 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9292 spa->spa_deadman_tqid = 0; 9293 9294 /* 9295 * Clear the dirty config list. 9296 */ 9297 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 9298 vdev_config_clean(vd); 9299 9300 /* 9301 * Now that the new config has synced transactionally, 9302 * let it become visible to the config cache. 9303 */ 9304 if (spa->spa_config_syncing != NULL) { 9305 spa_config_set(spa, spa->spa_config_syncing); 9306 spa->spa_config_txg = txg; 9307 spa->spa_config_syncing = NULL; 9308 } 9309 9310 dsl_pool_sync_done(dp, txg); 9311 9312 for (int i = 0; i < spa->spa_alloc_count; i++) { 9313 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9314 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9315 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9316 } 9317 9318 /* 9319 * Update usable space statistics. 9320 */ 9321 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 9322 != NULL) 9323 vdev_sync_done(vd, txg); 9324 9325 metaslab_class_evict_old(spa->spa_normal_class, txg); 9326 metaslab_class_evict_old(spa->spa_log_class, txg); 9327 9328 spa_sync_close_syncing_log_sm(spa); 9329 9330 spa_update_dspace(spa); 9331 9332 /* 9333 * It had better be the case that we didn't dirty anything 9334 * since vdev_config_sync(). 9335 */ 9336 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9337 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9338 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 9339 9340 while (zfs_pause_spa_sync) 9341 delay(1); 9342 9343 spa->spa_sync_pass = 0; 9344 9345 /* 9346 * Update the last synced uberblock here. We want to do this at 9347 * the end of spa_sync() so that consumers of spa_last_synced_txg() 9348 * will be guaranteed that all the processing associated with 9349 * that txg has been completed. 9350 */ 9351 spa->spa_ubsync = spa->spa_uberblock; 9352 spa_config_exit(spa, SCL_CONFIG, FTAG); 9353 9354 spa_handle_ignored_writes(spa); 9355 9356 /* 9357 * If any async tasks have been requested, kick them off. 9358 */ 9359 spa_async_dispatch(spa); 9360 } 9361 9362 /* 9363 * Sync all pools. We don't want to hold the namespace lock across these 9364 * operations, so we take a reference on the spa_t and drop the lock during the 9365 * sync. 9366 */ 9367 void 9368 spa_sync_allpools(void) 9369 { 9370 spa_t *spa = NULL; 9371 mutex_enter(&spa_namespace_lock); 9372 while ((spa = spa_next(spa)) != NULL) { 9373 if (spa_state(spa) != POOL_STATE_ACTIVE || 9374 !spa_writeable(spa) || spa_suspended(spa)) 9375 continue; 9376 spa_open_ref(spa, FTAG); 9377 mutex_exit(&spa_namespace_lock); 9378 txg_wait_synced(spa_get_dsl(spa), 0); 9379 mutex_enter(&spa_namespace_lock); 9380 spa_close(spa, FTAG); 9381 } 9382 mutex_exit(&spa_namespace_lock); 9383 } 9384 9385 /* 9386 * ========================================================================== 9387 * Miscellaneous routines 9388 * ========================================================================== 9389 */ 9390 9391 /* 9392 * Remove all pools in the system. 9393 */ 9394 void 9395 spa_evict_all(void) 9396 { 9397 spa_t *spa; 9398 9399 /* 9400 * Remove all cached state. All pools should be closed now, 9401 * so every spa in the AVL tree should be unreferenced. 9402 */ 9403 mutex_enter(&spa_namespace_lock); 9404 while ((spa = spa_next(NULL)) != NULL) { 9405 /* 9406 * Stop async tasks. The async thread may need to detach 9407 * a device that's been replaced, which requires grabbing 9408 * spa_namespace_lock, so we must drop it here. 9409 */ 9410 spa_open_ref(spa, FTAG); 9411 mutex_exit(&spa_namespace_lock); 9412 spa_async_suspend(spa); 9413 mutex_enter(&spa_namespace_lock); 9414 spa_close(spa, FTAG); 9415 9416 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 9417 spa_unload(spa); 9418 spa_deactivate(spa); 9419 } 9420 spa_remove(spa); 9421 } 9422 mutex_exit(&spa_namespace_lock); 9423 } 9424 9425 vdev_t * 9426 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 9427 { 9428 vdev_t *vd; 9429 int i; 9430 9431 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 9432 return (vd); 9433 9434 if (aux) { 9435 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 9436 vd = spa->spa_l2cache.sav_vdevs[i]; 9437 if (vd->vdev_guid == guid) 9438 return (vd); 9439 } 9440 9441 for (i = 0; i < spa->spa_spares.sav_count; i++) { 9442 vd = spa->spa_spares.sav_vdevs[i]; 9443 if (vd->vdev_guid == guid) 9444 return (vd); 9445 } 9446 } 9447 9448 return (NULL); 9449 } 9450 9451 void 9452 spa_upgrade(spa_t *spa, uint64_t version) 9453 { 9454 ASSERT(spa_writeable(spa)); 9455 9456 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9457 9458 /* 9459 * This should only be called for a non-faulted pool, and since a 9460 * future version would result in an unopenable pool, this shouldn't be 9461 * possible. 9462 */ 9463 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 9464 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 9465 9466 spa->spa_uberblock.ub_version = version; 9467 vdev_config_dirty(spa->spa_root_vdev); 9468 9469 spa_config_exit(spa, SCL_ALL, FTAG); 9470 9471 txg_wait_synced(spa_get_dsl(spa), 0); 9472 } 9473 9474 boolean_t 9475 spa_has_spare(spa_t *spa, uint64_t guid) 9476 { 9477 int i; 9478 uint64_t spareguid; 9479 spa_aux_vdev_t *sav = &spa->spa_spares; 9480 9481 for (i = 0; i < sav->sav_count; i++) 9482 if (sav->sav_vdevs[i]->vdev_guid == guid) 9483 return (B_TRUE); 9484 9485 for (i = 0; i < sav->sav_npending; i++) { 9486 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 9487 &spareguid) == 0 && spareguid == guid) 9488 return (B_TRUE); 9489 } 9490 9491 return (B_FALSE); 9492 } 9493 9494 /* 9495 * Check if a pool has an active shared spare device. 9496 * Note: reference count of an active spare is 2, as a spare and as a replace 9497 */ 9498 static boolean_t 9499 spa_has_active_shared_spare(spa_t *spa) 9500 { 9501 int i, refcnt; 9502 uint64_t pool; 9503 spa_aux_vdev_t *sav = &spa->spa_spares; 9504 9505 for (i = 0; i < sav->sav_count; i++) { 9506 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 9507 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 9508 refcnt > 2) 9509 return (B_TRUE); 9510 } 9511 9512 return (B_FALSE); 9513 } 9514 9515 uint64_t 9516 spa_total_metaslabs(spa_t *spa) 9517 { 9518 vdev_t *rvd = spa->spa_root_vdev; 9519 9520 uint64_t m = 0; 9521 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 9522 vdev_t *vd = rvd->vdev_child[c]; 9523 if (!vdev_is_concrete(vd)) 9524 continue; 9525 m += vd->vdev_ms_count; 9526 } 9527 return (m); 9528 } 9529 9530 /* 9531 * Notify any waiting threads that some activity has switched from being in- 9532 * progress to not-in-progress so that the thread can wake up and determine 9533 * whether it is finished waiting. 9534 */ 9535 void 9536 spa_notify_waiters(spa_t *spa) 9537 { 9538 /* 9539 * Acquiring spa_activities_lock here prevents the cv_broadcast from 9540 * happening between the waiting thread's check and cv_wait. 9541 */ 9542 mutex_enter(&spa->spa_activities_lock); 9543 cv_broadcast(&spa->spa_activities_cv); 9544 mutex_exit(&spa->spa_activities_lock); 9545 } 9546 9547 /* 9548 * Notify any waiting threads that the pool is exporting, and then block until 9549 * they are finished using the spa_t. 9550 */ 9551 void 9552 spa_wake_waiters(spa_t *spa) 9553 { 9554 mutex_enter(&spa->spa_activities_lock); 9555 spa->spa_waiters_cancel = B_TRUE; 9556 cv_broadcast(&spa->spa_activities_cv); 9557 while (spa->spa_waiters != 0) 9558 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 9559 spa->spa_waiters_cancel = B_FALSE; 9560 mutex_exit(&spa->spa_activities_lock); 9561 } 9562 9563 /* Whether the vdev or any of its descendants are being initialized/trimmed. */ 9564 static boolean_t 9565 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 9566 { 9567 spa_t *spa = vd->vdev_spa; 9568 9569 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 9570 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9571 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 9572 activity == ZPOOL_WAIT_TRIM); 9573 9574 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 9575 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 9576 9577 mutex_exit(&spa->spa_activities_lock); 9578 mutex_enter(lock); 9579 mutex_enter(&spa->spa_activities_lock); 9580 9581 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 9582 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) : 9583 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE); 9584 mutex_exit(lock); 9585 9586 if (in_progress) 9587 return (B_TRUE); 9588 9589 for (int i = 0; i < vd->vdev_children; i++) { 9590 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 9591 activity)) 9592 return (B_TRUE); 9593 } 9594 9595 return (B_FALSE); 9596 } 9597 9598 /* 9599 * If use_guid is true, this checks whether the vdev specified by guid is 9600 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 9601 * is being initialized/trimmed. The caller must hold the config lock and 9602 * spa_activities_lock. 9603 */ 9604 static int 9605 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 9606 zpool_wait_activity_t activity, boolean_t *in_progress) 9607 { 9608 mutex_exit(&spa->spa_activities_lock); 9609 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9610 mutex_enter(&spa->spa_activities_lock); 9611 9612 vdev_t *vd; 9613 if (use_guid) { 9614 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 9615 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 9616 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9617 return (EINVAL); 9618 } 9619 } else { 9620 vd = spa->spa_root_vdev; 9621 } 9622 9623 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 9624 9625 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9626 return (0); 9627 } 9628 9629 /* 9630 * Locking for waiting threads 9631 * --------------------------- 9632 * 9633 * Waiting threads need a way to check whether a given activity is in progress, 9634 * and then, if it is, wait for it to complete. Each activity will have some 9635 * in-memory representation of the relevant on-disk state which can be used to 9636 * determine whether or not the activity is in progress. The in-memory state and 9637 * the locking used to protect it will be different for each activity, and may 9638 * not be suitable for use with a cvar (e.g., some state is protected by the 9639 * config lock). To allow waiting threads to wait without any races, another 9640 * lock, spa_activities_lock, is used. 9641 * 9642 * When the state is checked, both the activity-specific lock (if there is one) 9643 * and spa_activities_lock are held. In some cases, the activity-specific lock 9644 * is acquired explicitly (e.g. the config lock). In others, the locking is 9645 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 9646 * thread releases the activity-specific lock and, if the activity is in 9647 * progress, then cv_waits using spa_activities_lock. 9648 * 9649 * The waiting thread is woken when another thread, one completing some 9650 * activity, updates the state of the activity and then calls 9651 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 9652 * needs to hold its activity-specific lock when updating the state, and this 9653 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 9654 * 9655 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 9656 * and because it is held when the waiting thread checks the state of the 9657 * activity, it can never be the case that the completing thread both updates 9658 * the activity state and cv_broadcasts in between the waiting thread's check 9659 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 9660 * 9661 * In order to prevent deadlock, when the waiting thread does its check, in some 9662 * cases it will temporarily drop spa_activities_lock in order to acquire the 9663 * activity-specific lock. The order in which spa_activities_lock and the 9664 * activity specific lock are acquired in the waiting thread is determined by 9665 * the order in which they are acquired in the completing thread; if the 9666 * completing thread calls spa_notify_waiters with the activity-specific lock 9667 * held, then the waiting thread must also acquire the activity-specific lock 9668 * first. 9669 */ 9670 9671 static int 9672 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 9673 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 9674 { 9675 int error = 0; 9676 9677 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9678 9679 switch (activity) { 9680 case ZPOOL_WAIT_CKPT_DISCARD: 9681 *in_progress = 9682 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 9683 zap_contains(spa_meta_objset(spa), 9684 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 9685 ENOENT); 9686 break; 9687 case ZPOOL_WAIT_FREE: 9688 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 9689 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 9690 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 9691 spa_livelist_delete_check(spa)); 9692 break; 9693 case ZPOOL_WAIT_INITIALIZE: 9694 case ZPOOL_WAIT_TRIM: 9695 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 9696 activity, in_progress); 9697 break; 9698 case ZPOOL_WAIT_REPLACE: 9699 mutex_exit(&spa->spa_activities_lock); 9700 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9701 mutex_enter(&spa->spa_activities_lock); 9702 9703 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 9704 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9705 break; 9706 case ZPOOL_WAIT_REMOVE: 9707 *in_progress = (spa->spa_removing_phys.sr_state == 9708 DSS_SCANNING); 9709 break; 9710 case ZPOOL_WAIT_RESILVER: 9711 if ((*in_progress = vdev_rebuild_active(spa->spa_root_vdev))) 9712 break; 9713 fallthrough; 9714 case ZPOOL_WAIT_SCRUB: 9715 { 9716 boolean_t scanning, paused, is_scrub; 9717 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 9718 9719 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 9720 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 9721 paused = dsl_scan_is_paused_scrub(scn); 9722 *in_progress = (scanning && !paused && 9723 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 9724 break; 9725 } 9726 default: 9727 panic("unrecognized value for activity %d", activity); 9728 } 9729 9730 return (error); 9731 } 9732 9733 static int 9734 spa_wait_common(const char *pool, zpool_wait_activity_t activity, 9735 boolean_t use_tag, uint64_t tag, boolean_t *waited) 9736 { 9737 /* 9738 * The tag is used to distinguish between instances of an activity. 9739 * 'initialize' and 'trim' are the only activities that we use this for. 9740 * The other activities can only have a single instance in progress in a 9741 * pool at one time, making the tag unnecessary. 9742 * 9743 * There can be multiple devices being replaced at once, but since they 9744 * all finish once resilvering finishes, we don't bother keeping track 9745 * of them individually, we just wait for them all to finish. 9746 */ 9747 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 9748 activity != ZPOOL_WAIT_TRIM) 9749 return (EINVAL); 9750 9751 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 9752 return (EINVAL); 9753 9754 spa_t *spa; 9755 int error = spa_open(pool, &spa, FTAG); 9756 if (error != 0) 9757 return (error); 9758 9759 /* 9760 * Increment the spa's waiter count so that we can call spa_close and 9761 * still ensure that the spa_t doesn't get freed before this thread is 9762 * finished with it when the pool is exported. We want to call spa_close 9763 * before we start waiting because otherwise the additional ref would 9764 * prevent the pool from being exported or destroyed throughout the 9765 * potentially long wait. 9766 */ 9767 mutex_enter(&spa->spa_activities_lock); 9768 spa->spa_waiters++; 9769 spa_close(spa, FTAG); 9770 9771 *waited = B_FALSE; 9772 for (;;) { 9773 boolean_t in_progress; 9774 error = spa_activity_in_progress(spa, activity, use_tag, tag, 9775 &in_progress); 9776 9777 if (error || !in_progress || spa->spa_waiters_cancel) 9778 break; 9779 9780 *waited = B_TRUE; 9781 9782 if (cv_wait_sig(&spa->spa_activities_cv, 9783 &spa->spa_activities_lock) == 0) { 9784 error = EINTR; 9785 break; 9786 } 9787 } 9788 9789 spa->spa_waiters--; 9790 cv_signal(&spa->spa_waiters_cv); 9791 mutex_exit(&spa->spa_activities_lock); 9792 9793 return (error); 9794 } 9795 9796 /* 9797 * Wait for a particular instance of the specified activity to complete, where 9798 * the instance is identified by 'tag' 9799 */ 9800 int 9801 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 9802 boolean_t *waited) 9803 { 9804 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 9805 } 9806 9807 /* 9808 * Wait for all instances of the specified activity complete 9809 */ 9810 int 9811 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 9812 { 9813 9814 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 9815 } 9816 9817 sysevent_t * 9818 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9819 { 9820 sysevent_t *ev = NULL; 9821 #ifdef _KERNEL 9822 nvlist_t *resource; 9823 9824 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 9825 if (resource) { 9826 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 9827 ev->resource = resource; 9828 } 9829 #endif 9830 return (ev); 9831 } 9832 9833 void 9834 spa_event_post(sysevent_t *ev) 9835 { 9836 #ifdef _KERNEL 9837 if (ev) { 9838 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 9839 kmem_free(ev, sizeof (*ev)); 9840 } 9841 #endif 9842 } 9843 9844 /* 9845 * Post a zevent corresponding to the given sysevent. The 'name' must be one 9846 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 9847 * filled in from the spa and (optionally) the vdev. This doesn't do anything 9848 * in the userland libzpool, as we don't want consumers to misinterpret ztest 9849 * or zdb as real changes. 9850 */ 9851 void 9852 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9853 { 9854 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 9855 } 9856 9857 /* state manipulation functions */ 9858 EXPORT_SYMBOL(spa_open); 9859 EXPORT_SYMBOL(spa_open_rewind); 9860 EXPORT_SYMBOL(spa_get_stats); 9861 EXPORT_SYMBOL(spa_create); 9862 EXPORT_SYMBOL(spa_import); 9863 EXPORT_SYMBOL(spa_tryimport); 9864 EXPORT_SYMBOL(spa_destroy); 9865 EXPORT_SYMBOL(spa_export); 9866 EXPORT_SYMBOL(spa_reset); 9867 EXPORT_SYMBOL(spa_async_request); 9868 EXPORT_SYMBOL(spa_async_suspend); 9869 EXPORT_SYMBOL(spa_async_resume); 9870 EXPORT_SYMBOL(spa_inject_addref); 9871 EXPORT_SYMBOL(spa_inject_delref); 9872 EXPORT_SYMBOL(spa_scan_stat_init); 9873 EXPORT_SYMBOL(spa_scan_get_stats); 9874 9875 /* device manipulation */ 9876 EXPORT_SYMBOL(spa_vdev_add); 9877 EXPORT_SYMBOL(spa_vdev_attach); 9878 EXPORT_SYMBOL(spa_vdev_detach); 9879 EXPORT_SYMBOL(spa_vdev_setpath); 9880 EXPORT_SYMBOL(spa_vdev_setfru); 9881 EXPORT_SYMBOL(spa_vdev_split_mirror); 9882 9883 /* spare statech is global across all pools) */ 9884 EXPORT_SYMBOL(spa_spare_add); 9885 EXPORT_SYMBOL(spa_spare_remove); 9886 EXPORT_SYMBOL(spa_spare_exists); 9887 EXPORT_SYMBOL(spa_spare_activate); 9888 9889 /* L2ARC statech is global across all pools) */ 9890 EXPORT_SYMBOL(spa_l2cache_add); 9891 EXPORT_SYMBOL(spa_l2cache_remove); 9892 EXPORT_SYMBOL(spa_l2cache_exists); 9893 EXPORT_SYMBOL(spa_l2cache_activate); 9894 EXPORT_SYMBOL(spa_l2cache_drop); 9895 9896 /* scanning */ 9897 EXPORT_SYMBOL(spa_scan); 9898 EXPORT_SYMBOL(spa_scan_stop); 9899 9900 /* spa syncing */ 9901 EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 9902 EXPORT_SYMBOL(spa_sync_allpools); 9903 9904 /* properties */ 9905 EXPORT_SYMBOL(spa_prop_set); 9906 EXPORT_SYMBOL(spa_prop_get); 9907 EXPORT_SYMBOL(spa_prop_clear_bootfs); 9908 9909 /* asynchronous event notification */ 9910 EXPORT_SYMBOL(spa_event_notify); 9911 9912 /* BEGIN CSTYLED */ 9913 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, INT, ZMOD_RW, 9914 "log2 fraction of arc that can be used by inflight I/Os when " 9915 "verifying pool during import"); 9916 9917 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 9918 "Set to traverse metadata on pool import"); 9919 9920 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 9921 "Set to traverse data on pool import"); 9922 9923 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 9924 "Print vdev tree to zfs_dbgmsg during pool import"); 9925 9926 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD, 9927 "Percentage of CPUs to run an IO worker thread"); 9928 9929 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD, 9930 "Number of threads per IO worker taskqueue"); 9931 9932 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, ULONG, ZMOD_RW, 9933 "Allow importing pool with up to this number of missing top-level " 9934 "vdevs (in read-only mode)"); 9935 9936 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, ZMOD_RW, 9937 "Set the livelist condense zthr to pause"); 9938 9939 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, ZMOD_RW, 9940 "Set the livelist condense synctask to pause"); 9941 9942 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, INT, ZMOD_RW, 9943 "Whether livelist condensing was canceled in the synctask"); 9944 9945 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, INT, ZMOD_RW, 9946 "Whether livelist condensing was canceled in the zthr function"); 9947 9948 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, ZMOD_RW, 9949 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 9950 "was being condensed"); 9951 /* END CSTYLED */ 9952