1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 * Copyright 2016 Toomas Soome <tsoome@me.com> 30 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 31 * Copyright 2018 Joyent, Inc. 32 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 33 * Copyright 2017 Joyent, Inc. 34 * Copyright (c) 2017, Intel Corporation. 35 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 36 */ 37 38 /* 39 * SPA: Storage Pool Allocator 40 * 41 * This file contains all the routines used when modifying on-disk SPA state. 42 * This includes opening, importing, destroying, exporting a pool, and syncing a 43 * pool. 44 */ 45 46 #include <sys/zfs_context.h> 47 #include <sys/fm/fs/zfs.h> 48 #include <sys/spa_impl.h> 49 #include <sys/zio.h> 50 #include <sys/zio_checksum.h> 51 #include <sys/dmu.h> 52 #include <sys/dmu_tx.h> 53 #include <sys/zap.h> 54 #include <sys/zil.h> 55 #include <sys/ddt.h> 56 #include <sys/vdev_impl.h> 57 #include <sys/vdev_removal.h> 58 #include <sys/vdev_indirect_mapping.h> 59 #include <sys/vdev_indirect_births.h> 60 #include <sys/vdev_initialize.h> 61 #include <sys/vdev_rebuild.h> 62 #include <sys/vdev_trim.h> 63 #include <sys/vdev_disk.h> 64 #include <sys/vdev_draid.h> 65 #include <sys/metaslab.h> 66 #include <sys/metaslab_impl.h> 67 #include <sys/mmp.h> 68 #include <sys/uberblock_impl.h> 69 #include <sys/txg.h> 70 #include <sys/avl.h> 71 #include <sys/bpobj.h> 72 #include <sys/dmu_traverse.h> 73 #include <sys/dmu_objset.h> 74 #include <sys/unique.h> 75 #include <sys/dsl_pool.h> 76 #include <sys/dsl_dataset.h> 77 #include <sys/dsl_dir.h> 78 #include <sys/dsl_prop.h> 79 #include <sys/dsl_synctask.h> 80 #include <sys/fs/zfs.h> 81 #include <sys/arc.h> 82 #include <sys/callb.h> 83 #include <sys/systeminfo.h> 84 #include <sys/spa_boot.h> 85 #include <sys/zfs_ioctl.h> 86 #include <sys/dsl_scan.h> 87 #include <sys/zfeature.h> 88 #include <sys/dsl_destroy.h> 89 #include <sys/zvol.h> 90 91 #ifdef _KERNEL 92 #include <sys/fm/protocol.h> 93 #include <sys/fm/util.h> 94 #include <sys/callb.h> 95 #include <sys/zone.h> 96 #include <sys/vmsystm.h> 97 #endif /* _KERNEL */ 98 99 #include "zfs_prop.h" 100 #include "zfs_comutil.h" 101 102 /* 103 * The interval, in seconds, at which failed configuration cache file writes 104 * should be retried. 105 */ 106 int zfs_ccw_retry_interval = 300; 107 108 typedef enum zti_modes { 109 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 110 ZTI_MODE_BATCH, /* cpu-intensive; value is ignored */ 111 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 112 ZTI_MODE_NULL, /* don't create a taskq */ 113 ZTI_NMODES 114 } zti_modes_t; 115 116 #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 117 #define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 118 #define ZTI_BATCH { ZTI_MODE_BATCH, 0, 1 } 119 #define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 } 120 #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 121 122 #define ZTI_N(n) ZTI_P(n, 1) 123 #define ZTI_ONE ZTI_N(1) 124 125 typedef struct zio_taskq_info { 126 zti_modes_t zti_mode; 127 uint_t zti_value; 128 uint_t zti_count; 129 } zio_taskq_info_t; 130 131 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 132 "iss", "iss_h", "int", "int_h" 133 }; 134 135 /* 136 * This table defines the taskq settings for each ZFS I/O type. When 137 * initializing a pool, we use this table to create an appropriately sized 138 * taskq. Some operations are low volume and therefore have a small, static 139 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 140 * macros. Other operations process a large amount of data; the ZTI_BATCH 141 * macro causes us to create a taskq oriented for throughput. Some operations 142 * are so high frequency and short-lived that the taskq itself can become a 143 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 144 * additional degree of parallelism specified by the number of threads per- 145 * taskq and the number of taskqs; when dispatching an event in this case, the 146 * particular taskq is chosen at random. ZTI_SCALE is similar to ZTI_BATCH, 147 * but with number of taskqs also scaling with number of CPUs. 148 * 149 * The different taskq priorities are to handle the different contexts (issue 150 * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that 151 * need to be handled with minimum delay. 152 */ 153 const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 154 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 155 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 156 { ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */ 157 { ZTI_BATCH, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */ 158 { ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 159 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 160 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */ 161 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 162 }; 163 164 static void spa_sync_version(void *arg, dmu_tx_t *tx); 165 static void spa_sync_props(void *arg, dmu_tx_t *tx); 166 static boolean_t spa_has_active_shared_spare(spa_t *spa); 167 static int spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport); 168 static void spa_vdev_resilver_done(spa_t *spa); 169 170 uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 171 uint_t zio_taskq_batch_tpq; /* threads per taskq */ 172 boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 173 uint_t zio_taskq_basedc = 80; /* base duty cycle */ 174 175 boolean_t spa_create_process = B_TRUE; /* no process ==> no sysdc */ 176 177 /* 178 * Report any spa_load_verify errors found, but do not fail spa_load. 179 * This is used by zdb to analyze non-idle pools. 180 */ 181 boolean_t spa_load_verify_dryrun = B_FALSE; 182 183 /* 184 * Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ). 185 * This is used by zdb for spacemaps verification. 186 */ 187 boolean_t spa_mode_readable_spacemaps = B_FALSE; 188 189 /* 190 * This (illegal) pool name is used when temporarily importing a spa_t in order 191 * to get the vdev stats associated with the imported devices. 192 */ 193 #define TRYIMPORT_NAME "$import" 194 195 /* 196 * For debugging purposes: print out vdev tree during pool import. 197 */ 198 int spa_load_print_vdev_tree = B_FALSE; 199 200 /* 201 * A non-zero value for zfs_max_missing_tvds means that we allow importing 202 * pools with missing top-level vdevs. This is strictly intended for advanced 203 * pool recovery cases since missing data is almost inevitable. Pools with 204 * missing devices can only be imported read-only for safety reasons, and their 205 * fail-mode will be automatically set to "continue". 206 * 207 * With 1 missing vdev we should be able to import the pool and mount all 208 * datasets. User data that was not modified after the missing device has been 209 * added should be recoverable. This means that snapshots created prior to the 210 * addition of that device should be completely intact. 211 * 212 * With 2 missing vdevs, some datasets may fail to mount since there are 213 * dataset statistics that are stored as regular metadata. Some data might be 214 * recoverable if those vdevs were added recently. 215 * 216 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 217 * may be missing entirely. Chances of data recovery are very low. Note that 218 * there are also risks of performing an inadvertent rewind as we might be 219 * missing all the vdevs with the latest uberblocks. 220 */ 221 unsigned long zfs_max_missing_tvds = 0; 222 223 /* 224 * The parameters below are similar to zfs_max_missing_tvds but are only 225 * intended for a preliminary open of the pool with an untrusted config which 226 * might be incomplete or out-dated. 227 * 228 * We are more tolerant for pools opened from a cachefile since we could have 229 * an out-dated cachefile where a device removal was not registered. 230 * We could have set the limit arbitrarily high but in the case where devices 231 * are really missing we would want to return the proper error codes; we chose 232 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 233 * and we get a chance to retrieve the trusted config. 234 */ 235 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 236 237 /* 238 * In the case where config was assembled by scanning device paths (/dev/dsks 239 * by default) we are less tolerant since all the existing devices should have 240 * been detected and we want spa_load to return the right error codes. 241 */ 242 uint64_t zfs_max_missing_tvds_scan = 0; 243 244 /* 245 * Debugging aid that pauses spa_sync() towards the end. 246 */ 247 boolean_t zfs_pause_spa_sync = B_FALSE; 248 249 /* 250 * Variables to indicate the livelist condense zthr func should wait at certain 251 * points for the livelist to be removed - used to test condense/destroy races 252 */ 253 int zfs_livelist_condense_zthr_pause = 0; 254 int zfs_livelist_condense_sync_pause = 0; 255 256 /* 257 * Variables to track whether or not condense cancellation has been 258 * triggered in testing. 259 */ 260 int zfs_livelist_condense_sync_cancel = 0; 261 int zfs_livelist_condense_zthr_cancel = 0; 262 263 /* 264 * Variable to track whether or not extra ALLOC blkptrs were added to a 265 * livelist entry while it was being condensed (caused by the way we track 266 * remapped blkptrs in dbuf_remap_impl) 267 */ 268 int zfs_livelist_condense_new_alloc = 0; 269 270 /* 271 * ========================================================================== 272 * SPA properties routines 273 * ========================================================================== 274 */ 275 276 /* 277 * Add a (source=src, propname=propval) list to an nvlist. 278 */ 279 static void 280 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval, 281 uint64_t intval, zprop_source_t src) 282 { 283 const char *propname = zpool_prop_to_name(prop); 284 nvlist_t *propval; 285 286 propval = fnvlist_alloc(); 287 fnvlist_add_uint64(propval, ZPROP_SOURCE, src); 288 289 if (strval != NULL) 290 fnvlist_add_string(propval, ZPROP_VALUE, strval); 291 else 292 fnvlist_add_uint64(propval, ZPROP_VALUE, intval); 293 294 fnvlist_add_nvlist(nvl, propname, propval); 295 nvlist_free(propval); 296 } 297 298 /* 299 * Get property values from the spa configuration. 300 */ 301 static void 302 spa_prop_get_config(spa_t *spa, nvlist_t **nvp) 303 { 304 vdev_t *rvd = spa->spa_root_vdev; 305 dsl_pool_t *pool = spa->spa_dsl_pool; 306 uint64_t size, alloc, cap, version; 307 const zprop_source_t src = ZPROP_SRC_NONE; 308 spa_config_dirent_t *dp; 309 metaslab_class_t *mc = spa_normal_class(spa); 310 311 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 312 313 if (rvd != NULL) { 314 alloc = metaslab_class_get_alloc(mc); 315 alloc += metaslab_class_get_alloc(spa_special_class(spa)); 316 alloc += metaslab_class_get_alloc(spa_dedup_class(spa)); 317 alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa)); 318 319 size = metaslab_class_get_space(mc); 320 size += metaslab_class_get_space(spa_special_class(spa)); 321 size += metaslab_class_get_space(spa_dedup_class(spa)); 322 size += metaslab_class_get_space(spa_embedded_log_class(spa)); 323 324 spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 325 spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src); 326 spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 327 spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL, 328 size - alloc, src); 329 spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL, 330 spa->spa_checkpoint_info.sci_dspace, src); 331 332 spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL, 333 metaslab_class_fragmentation(mc), src); 334 spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, 335 metaslab_class_expandable_space(mc), src); 336 spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL, 337 (spa_mode(spa) == SPA_MODE_READ), src); 338 339 cap = (size == 0) ? 0 : (alloc * 100 / size); 340 spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src); 341 342 spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL, 343 ddt_get_pool_dedup_ratio(spa), src); 344 345 spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL, 346 rvd->vdev_state, src); 347 348 version = spa_version(spa); 349 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 350 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 351 version, ZPROP_SRC_DEFAULT); 352 } else { 353 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 354 version, ZPROP_SRC_LOCAL); 355 } 356 spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID, 357 NULL, spa_load_guid(spa), src); 358 } 359 360 if (pool != NULL) { 361 /* 362 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 363 * when opening pools before this version freedir will be NULL. 364 */ 365 if (pool->dp_free_dir != NULL) { 366 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, 367 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 368 src); 369 } else { 370 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, 371 NULL, 0, src); 372 } 373 374 if (pool->dp_leak_dir != NULL) { 375 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, 376 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 377 src); 378 } else { 379 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, 380 NULL, 0, src); 381 } 382 } 383 384 spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 385 386 if (spa->spa_comment != NULL) { 387 spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment, 388 0, ZPROP_SRC_LOCAL); 389 } 390 391 if (spa->spa_compatibility != NULL) { 392 spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY, 393 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 394 } 395 396 if (spa->spa_root != NULL) 397 spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root, 398 0, ZPROP_SRC_LOCAL); 399 400 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 401 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 402 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 403 } else { 404 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 405 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 406 } 407 408 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 409 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 410 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 411 } else { 412 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 413 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 414 } 415 416 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 417 if (dp->scd_path == NULL) { 418 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 419 "none", 0, ZPROP_SRC_LOCAL); 420 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 421 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 422 dp->scd_path, 0, ZPROP_SRC_LOCAL); 423 } 424 } 425 } 426 427 /* 428 * Get zpool property values. 429 */ 430 int 431 spa_prop_get(spa_t *spa, nvlist_t **nvp) 432 { 433 objset_t *mos = spa->spa_meta_objset; 434 zap_cursor_t zc; 435 zap_attribute_t za; 436 dsl_pool_t *dp; 437 int err; 438 439 err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); 440 if (err) 441 return (err); 442 443 dp = spa_get_dsl(spa); 444 dsl_pool_config_enter(dp, FTAG); 445 mutex_enter(&spa->spa_props_lock); 446 447 /* 448 * Get properties from the spa config. 449 */ 450 spa_prop_get_config(spa, nvp); 451 452 /* If no pool property object, no more prop to get. */ 453 if (mos == NULL || spa->spa_pool_props_object == 0) 454 goto out; 455 456 /* 457 * Get properties from the MOS pool property object. 458 */ 459 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 460 (err = zap_cursor_retrieve(&zc, &za)) == 0; 461 zap_cursor_advance(&zc)) { 462 uint64_t intval = 0; 463 char *strval = NULL; 464 zprop_source_t src = ZPROP_SRC_DEFAULT; 465 zpool_prop_t prop; 466 467 if ((prop = zpool_name_to_prop(za.za_name)) == ZPOOL_PROP_INVAL) 468 continue; 469 470 switch (za.za_integer_length) { 471 case 8: 472 /* integer property */ 473 if (za.za_first_integer != 474 zpool_prop_default_numeric(prop)) 475 src = ZPROP_SRC_LOCAL; 476 477 if (prop == ZPOOL_PROP_BOOTFS) { 478 dsl_dataset_t *ds = NULL; 479 480 err = dsl_dataset_hold_obj(dp, 481 za.za_first_integer, FTAG, &ds); 482 if (err != 0) 483 break; 484 485 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 486 KM_SLEEP); 487 dsl_dataset_name(ds, strval); 488 dsl_dataset_rele(ds, FTAG); 489 } else { 490 strval = NULL; 491 intval = za.za_first_integer; 492 } 493 494 spa_prop_add_list(*nvp, prop, strval, intval, src); 495 496 if (strval != NULL) 497 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 498 499 break; 500 501 case 1: 502 /* string property */ 503 strval = kmem_alloc(za.za_num_integers, KM_SLEEP); 504 err = zap_lookup(mos, spa->spa_pool_props_object, 505 za.za_name, 1, za.za_num_integers, strval); 506 if (err) { 507 kmem_free(strval, za.za_num_integers); 508 break; 509 } 510 spa_prop_add_list(*nvp, prop, strval, 0, src); 511 kmem_free(strval, za.za_num_integers); 512 break; 513 514 default: 515 break; 516 } 517 } 518 zap_cursor_fini(&zc); 519 out: 520 mutex_exit(&spa->spa_props_lock); 521 dsl_pool_config_exit(dp, FTAG); 522 if (err && err != ENOENT) { 523 nvlist_free(*nvp); 524 *nvp = NULL; 525 return (err); 526 } 527 528 return (0); 529 } 530 531 /* 532 * Validate the given pool properties nvlist and modify the list 533 * for the property values to be set. 534 */ 535 static int 536 spa_prop_validate(spa_t *spa, nvlist_t *props) 537 { 538 nvpair_t *elem; 539 int error = 0, reset_bootfs = 0; 540 uint64_t objnum = 0; 541 boolean_t has_feature = B_FALSE; 542 543 elem = NULL; 544 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 545 uint64_t intval; 546 char *strval, *slash, *check, *fname; 547 const char *propname = nvpair_name(elem); 548 zpool_prop_t prop = zpool_name_to_prop(propname); 549 550 switch (prop) { 551 case ZPOOL_PROP_INVAL: 552 if (!zpool_prop_feature(propname)) { 553 error = SET_ERROR(EINVAL); 554 break; 555 } 556 557 /* 558 * Sanitize the input. 559 */ 560 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 561 error = SET_ERROR(EINVAL); 562 break; 563 } 564 565 if (nvpair_value_uint64(elem, &intval) != 0) { 566 error = SET_ERROR(EINVAL); 567 break; 568 } 569 570 if (intval != 0) { 571 error = SET_ERROR(EINVAL); 572 break; 573 } 574 575 fname = strchr(propname, '@') + 1; 576 if (zfeature_lookup_name(fname, NULL) != 0) { 577 error = SET_ERROR(EINVAL); 578 break; 579 } 580 581 has_feature = B_TRUE; 582 break; 583 584 case ZPOOL_PROP_VERSION: 585 error = nvpair_value_uint64(elem, &intval); 586 if (!error && 587 (intval < spa_version(spa) || 588 intval > SPA_VERSION_BEFORE_FEATURES || 589 has_feature)) 590 error = SET_ERROR(EINVAL); 591 break; 592 593 case ZPOOL_PROP_DELEGATION: 594 case ZPOOL_PROP_AUTOREPLACE: 595 case ZPOOL_PROP_LISTSNAPS: 596 case ZPOOL_PROP_AUTOEXPAND: 597 case ZPOOL_PROP_AUTOTRIM: 598 error = nvpair_value_uint64(elem, &intval); 599 if (!error && intval > 1) 600 error = SET_ERROR(EINVAL); 601 break; 602 603 case ZPOOL_PROP_MULTIHOST: 604 error = nvpair_value_uint64(elem, &intval); 605 if (!error && intval > 1) 606 error = SET_ERROR(EINVAL); 607 608 if (!error) { 609 uint32_t hostid = zone_get_hostid(NULL); 610 if (hostid) 611 spa->spa_hostid = hostid; 612 else 613 error = SET_ERROR(ENOTSUP); 614 } 615 616 break; 617 618 case ZPOOL_PROP_BOOTFS: 619 /* 620 * If the pool version is less than SPA_VERSION_BOOTFS, 621 * or the pool is still being created (version == 0), 622 * the bootfs property cannot be set. 623 */ 624 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 625 error = SET_ERROR(ENOTSUP); 626 break; 627 } 628 629 /* 630 * Make sure the vdev config is bootable 631 */ 632 if (!vdev_is_bootable(spa->spa_root_vdev)) { 633 error = SET_ERROR(ENOTSUP); 634 break; 635 } 636 637 reset_bootfs = 1; 638 639 error = nvpair_value_string(elem, &strval); 640 641 if (!error) { 642 objset_t *os; 643 644 if (strval == NULL || strval[0] == '\0') { 645 objnum = zpool_prop_default_numeric( 646 ZPOOL_PROP_BOOTFS); 647 break; 648 } 649 650 error = dmu_objset_hold(strval, FTAG, &os); 651 if (error != 0) 652 break; 653 654 /* Must be ZPL. */ 655 if (dmu_objset_type(os) != DMU_OST_ZFS) { 656 error = SET_ERROR(ENOTSUP); 657 } else { 658 objnum = dmu_objset_id(os); 659 } 660 dmu_objset_rele(os, FTAG); 661 } 662 break; 663 664 case ZPOOL_PROP_FAILUREMODE: 665 error = nvpair_value_uint64(elem, &intval); 666 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 667 error = SET_ERROR(EINVAL); 668 669 /* 670 * This is a special case which only occurs when 671 * the pool has completely failed. This allows 672 * the user to change the in-core failmode property 673 * without syncing it out to disk (I/Os might 674 * currently be blocked). We do this by returning 675 * EIO to the caller (spa_prop_set) to trick it 676 * into thinking we encountered a property validation 677 * error. 678 */ 679 if (!error && spa_suspended(spa)) { 680 spa->spa_failmode = intval; 681 error = SET_ERROR(EIO); 682 } 683 break; 684 685 case ZPOOL_PROP_CACHEFILE: 686 if ((error = nvpair_value_string(elem, &strval)) != 0) 687 break; 688 689 if (strval[0] == '\0') 690 break; 691 692 if (strcmp(strval, "none") == 0) 693 break; 694 695 if (strval[0] != '/') { 696 error = SET_ERROR(EINVAL); 697 break; 698 } 699 700 slash = strrchr(strval, '/'); 701 ASSERT(slash != NULL); 702 703 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 704 strcmp(slash, "/..") == 0) 705 error = SET_ERROR(EINVAL); 706 break; 707 708 case ZPOOL_PROP_COMMENT: 709 if ((error = nvpair_value_string(elem, &strval)) != 0) 710 break; 711 for (check = strval; *check != '\0'; check++) { 712 if (!isprint(*check)) { 713 error = SET_ERROR(EINVAL); 714 break; 715 } 716 } 717 if (strlen(strval) > ZPROP_MAX_COMMENT) 718 error = SET_ERROR(E2BIG); 719 break; 720 721 default: 722 break; 723 } 724 725 if (error) 726 break; 727 } 728 729 (void) nvlist_remove_all(props, 730 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 731 732 if (!error && reset_bootfs) { 733 error = nvlist_remove(props, 734 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 735 736 if (!error) { 737 error = nvlist_add_uint64(props, 738 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 739 } 740 } 741 742 return (error); 743 } 744 745 void 746 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 747 { 748 char *cachefile; 749 spa_config_dirent_t *dp; 750 751 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 752 &cachefile) != 0) 753 return; 754 755 dp = kmem_alloc(sizeof (spa_config_dirent_t), 756 KM_SLEEP); 757 758 if (cachefile[0] == '\0') 759 dp->scd_path = spa_strdup(spa_config_path); 760 else if (strcmp(cachefile, "none") == 0) 761 dp->scd_path = NULL; 762 else 763 dp->scd_path = spa_strdup(cachefile); 764 765 list_insert_head(&spa->spa_config_list, dp); 766 if (need_sync) 767 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 768 } 769 770 int 771 spa_prop_set(spa_t *spa, nvlist_t *nvp) 772 { 773 int error; 774 nvpair_t *elem = NULL; 775 boolean_t need_sync = B_FALSE; 776 777 if ((error = spa_prop_validate(spa, nvp)) != 0) 778 return (error); 779 780 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 781 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 782 783 if (prop == ZPOOL_PROP_CACHEFILE || 784 prop == ZPOOL_PROP_ALTROOT || 785 prop == ZPOOL_PROP_READONLY) 786 continue; 787 788 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 789 uint64_t ver = 0; 790 791 if (prop == ZPOOL_PROP_VERSION) { 792 VERIFY(nvpair_value_uint64(elem, &ver) == 0); 793 } else { 794 ASSERT(zpool_prop_feature(nvpair_name(elem))); 795 ver = SPA_VERSION_FEATURES; 796 need_sync = B_TRUE; 797 } 798 799 /* Save time if the version is already set. */ 800 if (ver == spa_version(spa)) 801 continue; 802 803 /* 804 * In addition to the pool directory object, we might 805 * create the pool properties object, the features for 806 * read object, the features for write object, or the 807 * feature descriptions object. 808 */ 809 error = dsl_sync_task(spa->spa_name, NULL, 810 spa_sync_version, &ver, 811 6, ZFS_SPACE_CHECK_RESERVED); 812 if (error) 813 return (error); 814 continue; 815 } 816 817 need_sync = B_TRUE; 818 break; 819 } 820 821 if (need_sync) { 822 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 823 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 824 } 825 826 return (0); 827 } 828 829 /* 830 * If the bootfs property value is dsobj, clear it. 831 */ 832 void 833 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 834 { 835 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 836 VERIFY(zap_remove(spa->spa_meta_objset, 837 spa->spa_pool_props_object, 838 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 839 spa->spa_bootfs = 0; 840 } 841 } 842 843 /*ARGSUSED*/ 844 static int 845 spa_change_guid_check(void *arg, dmu_tx_t *tx) 846 { 847 uint64_t *newguid __maybe_unused = arg; 848 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 849 vdev_t *rvd = spa->spa_root_vdev; 850 uint64_t vdev_state; 851 852 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 853 int error = (spa_has_checkpoint(spa)) ? 854 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 855 return (SET_ERROR(error)); 856 } 857 858 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 859 vdev_state = rvd->vdev_state; 860 spa_config_exit(spa, SCL_STATE, FTAG); 861 862 if (vdev_state != VDEV_STATE_HEALTHY) 863 return (SET_ERROR(ENXIO)); 864 865 ASSERT3U(spa_guid(spa), !=, *newguid); 866 867 return (0); 868 } 869 870 static void 871 spa_change_guid_sync(void *arg, dmu_tx_t *tx) 872 { 873 uint64_t *newguid = arg; 874 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 875 uint64_t oldguid; 876 vdev_t *rvd = spa->spa_root_vdev; 877 878 oldguid = spa_guid(spa); 879 880 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 881 rvd->vdev_guid = *newguid; 882 rvd->vdev_guid_sum += (*newguid - oldguid); 883 vdev_config_dirty(rvd); 884 spa_config_exit(spa, SCL_STATE, FTAG); 885 886 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 887 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 888 } 889 890 /* 891 * Change the GUID for the pool. This is done so that we can later 892 * re-import a pool built from a clone of our own vdevs. We will modify 893 * the root vdev's guid, our own pool guid, and then mark all of our 894 * vdevs dirty. Note that we must make sure that all our vdevs are 895 * online when we do this, or else any vdevs that weren't present 896 * would be orphaned from our pool. We are also going to issue a 897 * sysevent to update any watchers. 898 */ 899 int 900 spa_change_guid(spa_t *spa) 901 { 902 int error; 903 uint64_t guid; 904 905 mutex_enter(&spa->spa_vdev_top_lock); 906 mutex_enter(&spa_namespace_lock); 907 guid = spa_generate_guid(NULL); 908 909 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 910 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 911 912 if (error == 0) { 913 spa_write_cachefile(spa, B_FALSE, B_TRUE); 914 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 915 } 916 917 mutex_exit(&spa_namespace_lock); 918 mutex_exit(&spa->spa_vdev_top_lock); 919 920 return (error); 921 } 922 923 /* 924 * ========================================================================== 925 * SPA state manipulation (open/create/destroy/import/export) 926 * ========================================================================== 927 */ 928 929 static int 930 spa_error_entry_compare(const void *a, const void *b) 931 { 932 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 933 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 934 int ret; 935 936 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 937 sizeof (zbookmark_phys_t)); 938 939 return (TREE_ISIGN(ret)); 940 } 941 942 /* 943 * Utility function which retrieves copies of the current logs and 944 * re-initializes them in the process. 945 */ 946 void 947 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 948 { 949 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 950 951 bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t)); 952 bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t)); 953 954 avl_create(&spa->spa_errlist_scrub, 955 spa_error_entry_compare, sizeof (spa_error_entry_t), 956 offsetof(spa_error_entry_t, se_avl)); 957 avl_create(&spa->spa_errlist_last, 958 spa_error_entry_compare, sizeof (spa_error_entry_t), 959 offsetof(spa_error_entry_t, se_avl)); 960 } 961 962 static void 963 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 964 { 965 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 966 enum zti_modes mode = ztip->zti_mode; 967 uint_t value = ztip->zti_value; 968 uint_t count = ztip->zti_count; 969 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 970 uint_t cpus, flags = TASKQ_DYNAMIC; 971 boolean_t batch = B_FALSE; 972 973 switch (mode) { 974 case ZTI_MODE_FIXED: 975 ASSERT3U(value, >, 0); 976 break; 977 978 case ZTI_MODE_BATCH: 979 batch = B_TRUE; 980 flags |= TASKQ_THREADS_CPU_PCT; 981 value = MIN(zio_taskq_batch_pct, 100); 982 break; 983 984 case ZTI_MODE_SCALE: 985 flags |= TASKQ_THREADS_CPU_PCT; 986 /* 987 * We want more taskqs to reduce lock contention, but we want 988 * less for better request ordering and CPU utilization. 989 */ 990 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 991 if (zio_taskq_batch_tpq > 0) { 992 count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) / 993 zio_taskq_batch_tpq); 994 } else { 995 /* 996 * Prefer 6 threads per taskq, but no more taskqs 997 * than threads in them on large systems. For 80%: 998 * 999 * taskq taskq total 1000 * cpus taskqs percent threads threads 1001 * ------- ------- ------- ------- ------- 1002 * 1 1 80% 1 1 1003 * 2 1 80% 1 1 1004 * 4 1 80% 3 3 1005 * 8 2 40% 3 6 1006 * 16 3 27% 4 12 1007 * 32 5 16% 5 25 1008 * 64 7 11% 7 49 1009 * 128 10 8% 10 100 1010 * 256 14 6% 15 210 1011 */ 1012 count = 1 + cpus / 6; 1013 while (count * count > cpus) 1014 count--; 1015 } 1016 /* Limit each taskq within 100% to not trigger assertion. */ 1017 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1018 value = (zio_taskq_batch_pct + count / 2) / count; 1019 break; 1020 1021 case ZTI_MODE_NULL: 1022 tqs->stqs_count = 0; 1023 tqs->stqs_taskq = NULL; 1024 return; 1025 1026 default: 1027 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1028 "spa_activate()", 1029 zio_type_name[t], zio_taskq_types[q], mode, value); 1030 break; 1031 } 1032 1033 ASSERT3U(count, >, 0); 1034 tqs->stqs_count = count; 1035 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1036 1037 for (uint_t i = 0; i < count; i++) { 1038 taskq_t *tq; 1039 char name[32]; 1040 1041 if (count > 1) 1042 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1043 zio_type_name[t], zio_taskq_types[q], i); 1044 else 1045 (void) snprintf(name, sizeof (name), "%s_%s", 1046 zio_type_name[t], zio_taskq_types[q]); 1047 1048 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1049 if (batch) 1050 flags |= TASKQ_DC_BATCH; 1051 1052 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1053 spa->spa_proc, zio_taskq_basedc, flags); 1054 } else { 1055 pri_t pri = maxclsyspri; 1056 /* 1057 * The write issue taskq can be extremely CPU 1058 * intensive. Run it at slightly less important 1059 * priority than the other taskqs. 1060 * 1061 * Under Linux and FreeBSD this means incrementing 1062 * the priority value as opposed to platforms like 1063 * illumos where it should be decremented. 1064 * 1065 * On FreeBSD, if priorities divided by four (RQ_PPQ) 1066 * are equal then a difference between them is 1067 * insignificant. 1068 */ 1069 if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) { 1070 #if defined(__linux__) 1071 pri++; 1072 #elif defined(__FreeBSD__) 1073 pri += 4; 1074 #else 1075 #error "unknown OS" 1076 #endif 1077 } 1078 tq = taskq_create_proc(name, value, pri, 50, 1079 INT_MAX, spa->spa_proc, flags); 1080 } 1081 1082 tqs->stqs_taskq[i] = tq; 1083 } 1084 } 1085 1086 static void 1087 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1088 { 1089 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1090 1091 if (tqs->stqs_taskq == NULL) { 1092 ASSERT3U(tqs->stqs_count, ==, 0); 1093 return; 1094 } 1095 1096 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1097 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1098 taskq_destroy(tqs->stqs_taskq[i]); 1099 } 1100 1101 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1102 tqs->stqs_taskq = NULL; 1103 } 1104 1105 /* 1106 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1107 * Note that a type may have multiple discrete taskqs to avoid lock contention 1108 * on the taskq itself. In that case we choose which taskq at random by using 1109 * the low bits of gethrtime(). 1110 */ 1111 void 1112 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1113 task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent) 1114 { 1115 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1116 taskq_t *tq; 1117 1118 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1119 ASSERT3U(tqs->stqs_count, !=, 0); 1120 1121 if (tqs->stqs_count == 1) { 1122 tq = tqs->stqs_taskq[0]; 1123 } else { 1124 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1125 } 1126 1127 taskq_dispatch_ent(tq, func, arg, flags, ent); 1128 } 1129 1130 /* 1131 * Same as spa_taskq_dispatch_ent() but block on the task until completion. 1132 */ 1133 void 1134 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1135 task_func_t *func, void *arg, uint_t flags) 1136 { 1137 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1138 taskq_t *tq; 1139 taskqid_t id; 1140 1141 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1142 ASSERT3U(tqs->stqs_count, !=, 0); 1143 1144 if (tqs->stqs_count == 1) { 1145 tq = tqs->stqs_taskq[0]; 1146 } else { 1147 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1148 } 1149 1150 id = taskq_dispatch(tq, func, arg, flags); 1151 if (id) 1152 taskq_wait_id(tq, id); 1153 } 1154 1155 static void 1156 spa_create_zio_taskqs(spa_t *spa) 1157 { 1158 for (int t = 0; t < ZIO_TYPES; t++) { 1159 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1160 spa_taskqs_init(spa, t, q); 1161 } 1162 } 1163 } 1164 1165 /* 1166 * Disabled until spa_thread() can be adapted for Linux. 1167 */ 1168 #undef HAVE_SPA_THREAD 1169 1170 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1171 static void 1172 spa_thread(void *arg) 1173 { 1174 psetid_t zio_taskq_psrset_bind = PS_NONE; 1175 callb_cpr_t cprinfo; 1176 1177 spa_t *spa = arg; 1178 user_t *pu = PTOU(curproc); 1179 1180 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1181 spa->spa_name); 1182 1183 ASSERT(curproc != &p0); 1184 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1185 "zpool-%s", spa->spa_name); 1186 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1187 1188 /* bind this thread to the requested psrset */ 1189 if (zio_taskq_psrset_bind != PS_NONE) { 1190 pool_lock(); 1191 mutex_enter(&cpu_lock); 1192 mutex_enter(&pidlock); 1193 mutex_enter(&curproc->p_lock); 1194 1195 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1196 0, NULL, NULL) == 0) { 1197 curthread->t_bind_pset = zio_taskq_psrset_bind; 1198 } else { 1199 cmn_err(CE_WARN, 1200 "Couldn't bind process for zfs pool \"%s\" to " 1201 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1202 } 1203 1204 mutex_exit(&curproc->p_lock); 1205 mutex_exit(&pidlock); 1206 mutex_exit(&cpu_lock); 1207 pool_unlock(); 1208 } 1209 1210 if (zio_taskq_sysdc) { 1211 sysdc_thread_enter(curthread, 100, 0); 1212 } 1213 1214 spa->spa_proc = curproc; 1215 spa->spa_did = curthread->t_did; 1216 1217 spa_create_zio_taskqs(spa); 1218 1219 mutex_enter(&spa->spa_proc_lock); 1220 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1221 1222 spa->spa_proc_state = SPA_PROC_ACTIVE; 1223 cv_broadcast(&spa->spa_proc_cv); 1224 1225 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1226 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1227 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1228 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1229 1230 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1231 spa->spa_proc_state = SPA_PROC_GONE; 1232 spa->spa_proc = &p0; 1233 cv_broadcast(&spa->spa_proc_cv); 1234 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1235 1236 mutex_enter(&curproc->p_lock); 1237 lwp_exit(); 1238 } 1239 #endif 1240 1241 /* 1242 * Activate an uninitialized pool. 1243 */ 1244 static void 1245 spa_activate(spa_t *spa, spa_mode_t mode) 1246 { 1247 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1248 1249 spa->spa_state = POOL_STATE_ACTIVE; 1250 spa->spa_mode = mode; 1251 spa->spa_read_spacemaps = spa_mode_readable_spacemaps; 1252 1253 spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops); 1254 spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops); 1255 spa->spa_embedded_log_class = 1256 metaslab_class_create(spa, zfs_metaslab_ops); 1257 spa->spa_special_class = metaslab_class_create(spa, zfs_metaslab_ops); 1258 spa->spa_dedup_class = metaslab_class_create(spa, zfs_metaslab_ops); 1259 1260 /* Try to create a covering process */ 1261 mutex_enter(&spa->spa_proc_lock); 1262 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1263 ASSERT(spa->spa_proc == &p0); 1264 spa->spa_did = 0; 1265 1266 #ifdef HAVE_SPA_THREAD 1267 /* Only create a process if we're going to be around a while. */ 1268 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1269 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1270 NULL, 0) == 0) { 1271 spa->spa_proc_state = SPA_PROC_CREATED; 1272 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1273 cv_wait(&spa->spa_proc_cv, 1274 &spa->spa_proc_lock); 1275 } 1276 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1277 ASSERT(spa->spa_proc != &p0); 1278 ASSERT(spa->spa_did != 0); 1279 } else { 1280 #ifdef _KERNEL 1281 cmn_err(CE_WARN, 1282 "Couldn't create process for zfs pool \"%s\"\n", 1283 spa->spa_name); 1284 #endif 1285 } 1286 } 1287 #endif /* HAVE_SPA_THREAD */ 1288 mutex_exit(&spa->spa_proc_lock); 1289 1290 /* If we didn't create a process, we need to create our taskqs. */ 1291 if (spa->spa_proc == &p0) { 1292 spa_create_zio_taskqs(spa); 1293 } 1294 1295 for (size_t i = 0; i < TXG_SIZE; i++) { 1296 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1297 ZIO_FLAG_CANFAIL); 1298 } 1299 1300 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1301 offsetof(vdev_t, vdev_config_dirty_node)); 1302 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1303 offsetof(objset_t, os_evicting_node)); 1304 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1305 offsetof(vdev_t, vdev_state_dirty_node)); 1306 1307 txg_list_create(&spa->spa_vdev_txg_list, spa, 1308 offsetof(struct vdev, vdev_txg_node)); 1309 1310 avl_create(&spa->spa_errlist_scrub, 1311 spa_error_entry_compare, sizeof (spa_error_entry_t), 1312 offsetof(spa_error_entry_t, se_avl)); 1313 avl_create(&spa->spa_errlist_last, 1314 spa_error_entry_compare, sizeof (spa_error_entry_t), 1315 offsetof(spa_error_entry_t, se_avl)); 1316 1317 spa_keystore_init(&spa->spa_keystore); 1318 1319 /* 1320 * This taskq is used to perform zvol-minor-related tasks 1321 * asynchronously. This has several advantages, including easy 1322 * resolution of various deadlocks. 1323 * 1324 * The taskq must be single threaded to ensure tasks are always 1325 * processed in the order in which they were dispatched. 1326 * 1327 * A taskq per pool allows one to keep the pools independent. 1328 * This way if one pool is suspended, it will not impact another. 1329 * 1330 * The preferred location to dispatch a zvol minor task is a sync 1331 * task. In this context, there is easy access to the spa_t and minimal 1332 * error handling is required because the sync task must succeed. 1333 */ 1334 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1335 1, INT_MAX, 0); 1336 1337 /* 1338 * Taskq dedicated to prefetcher threads: this is used to prevent the 1339 * pool traverse code from monopolizing the global (and limited) 1340 * system_taskq by inappropriately scheduling long running tasks on it. 1341 */ 1342 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1343 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1344 1345 /* 1346 * The taskq to upgrade datasets in this pool. Currently used by 1347 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1348 */ 1349 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1350 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1351 } 1352 1353 /* 1354 * Opposite of spa_activate(). 1355 */ 1356 static void 1357 spa_deactivate(spa_t *spa) 1358 { 1359 ASSERT(spa->spa_sync_on == B_FALSE); 1360 ASSERT(spa->spa_dsl_pool == NULL); 1361 ASSERT(spa->spa_root_vdev == NULL); 1362 ASSERT(spa->spa_async_zio_root == NULL); 1363 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1364 1365 spa_evicting_os_wait(spa); 1366 1367 if (spa->spa_zvol_taskq) { 1368 taskq_destroy(spa->spa_zvol_taskq); 1369 spa->spa_zvol_taskq = NULL; 1370 } 1371 1372 if (spa->spa_prefetch_taskq) { 1373 taskq_destroy(spa->spa_prefetch_taskq); 1374 spa->spa_prefetch_taskq = NULL; 1375 } 1376 1377 if (spa->spa_upgrade_taskq) { 1378 taskq_destroy(spa->spa_upgrade_taskq); 1379 spa->spa_upgrade_taskq = NULL; 1380 } 1381 1382 txg_list_destroy(&spa->spa_vdev_txg_list); 1383 1384 list_destroy(&spa->spa_config_dirty_list); 1385 list_destroy(&spa->spa_evicting_os_list); 1386 list_destroy(&spa->spa_state_dirty_list); 1387 1388 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 1389 1390 for (int t = 0; t < ZIO_TYPES; t++) { 1391 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1392 spa_taskqs_fini(spa, t, q); 1393 } 1394 } 1395 1396 for (size_t i = 0; i < TXG_SIZE; i++) { 1397 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1398 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1399 spa->spa_txg_zio[i] = NULL; 1400 } 1401 1402 metaslab_class_destroy(spa->spa_normal_class); 1403 spa->spa_normal_class = NULL; 1404 1405 metaslab_class_destroy(spa->spa_log_class); 1406 spa->spa_log_class = NULL; 1407 1408 metaslab_class_destroy(spa->spa_embedded_log_class); 1409 spa->spa_embedded_log_class = NULL; 1410 1411 metaslab_class_destroy(spa->spa_special_class); 1412 spa->spa_special_class = NULL; 1413 1414 metaslab_class_destroy(spa->spa_dedup_class); 1415 spa->spa_dedup_class = NULL; 1416 1417 /* 1418 * If this was part of an import or the open otherwise failed, we may 1419 * still have errors left in the queues. Empty them just in case. 1420 */ 1421 spa_errlog_drain(spa); 1422 avl_destroy(&spa->spa_errlist_scrub); 1423 avl_destroy(&spa->spa_errlist_last); 1424 1425 spa_keystore_fini(&spa->spa_keystore); 1426 1427 spa->spa_state = POOL_STATE_UNINITIALIZED; 1428 1429 mutex_enter(&spa->spa_proc_lock); 1430 if (spa->spa_proc_state != SPA_PROC_NONE) { 1431 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1432 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 1433 cv_broadcast(&spa->spa_proc_cv); 1434 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 1435 ASSERT(spa->spa_proc != &p0); 1436 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1437 } 1438 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 1439 spa->spa_proc_state = SPA_PROC_NONE; 1440 } 1441 ASSERT(spa->spa_proc == &p0); 1442 mutex_exit(&spa->spa_proc_lock); 1443 1444 /* 1445 * We want to make sure spa_thread() has actually exited the ZFS 1446 * module, so that the module can't be unloaded out from underneath 1447 * it. 1448 */ 1449 if (spa->spa_did != 0) { 1450 thread_join(spa->spa_did); 1451 spa->spa_did = 0; 1452 } 1453 } 1454 1455 /* 1456 * Verify a pool configuration, and construct the vdev tree appropriately. This 1457 * will create all the necessary vdevs in the appropriate layout, with each vdev 1458 * in the CLOSED state. This will prep the pool before open/creation/import. 1459 * All vdev validation is done by the vdev_alloc() routine. 1460 */ 1461 int 1462 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 1463 uint_t id, int atype) 1464 { 1465 nvlist_t **child; 1466 uint_t children; 1467 int error; 1468 1469 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 1470 return (error); 1471 1472 if ((*vdp)->vdev_ops->vdev_op_leaf) 1473 return (0); 1474 1475 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 1476 &child, &children); 1477 1478 if (error == ENOENT) 1479 return (0); 1480 1481 if (error) { 1482 vdev_free(*vdp); 1483 *vdp = NULL; 1484 return (SET_ERROR(EINVAL)); 1485 } 1486 1487 for (int c = 0; c < children; c++) { 1488 vdev_t *vd; 1489 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 1490 atype)) != 0) { 1491 vdev_free(*vdp); 1492 *vdp = NULL; 1493 return (error); 1494 } 1495 } 1496 1497 ASSERT(*vdp != NULL); 1498 1499 return (0); 1500 } 1501 1502 static boolean_t 1503 spa_should_flush_logs_on_unload(spa_t *spa) 1504 { 1505 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 1506 return (B_FALSE); 1507 1508 if (!spa_writeable(spa)) 1509 return (B_FALSE); 1510 1511 if (!spa->spa_sync_on) 1512 return (B_FALSE); 1513 1514 if (spa_state(spa) != POOL_STATE_EXPORTED) 1515 return (B_FALSE); 1516 1517 if (zfs_keep_log_spacemaps_at_export) 1518 return (B_FALSE); 1519 1520 return (B_TRUE); 1521 } 1522 1523 /* 1524 * Opens a transaction that will set the flag that will instruct 1525 * spa_sync to attempt to flush all the metaslabs for that txg. 1526 */ 1527 static void 1528 spa_unload_log_sm_flush_all(spa_t *spa) 1529 { 1530 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 1531 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 1532 1533 ASSERT3U(spa->spa_log_flushall_txg, ==, 0); 1534 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); 1535 1536 dmu_tx_commit(tx); 1537 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 1538 } 1539 1540 static void 1541 spa_unload_log_sm_metadata(spa_t *spa) 1542 { 1543 void *cookie = NULL; 1544 spa_log_sm_t *sls; 1545 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 1546 &cookie)) != NULL) { 1547 VERIFY0(sls->sls_mscount); 1548 kmem_free(sls, sizeof (spa_log_sm_t)); 1549 } 1550 1551 for (log_summary_entry_t *e = list_head(&spa->spa_log_summary); 1552 e != NULL; e = list_head(&spa->spa_log_summary)) { 1553 VERIFY0(e->lse_mscount); 1554 list_remove(&spa->spa_log_summary, e); 1555 kmem_free(e, sizeof (log_summary_entry_t)); 1556 } 1557 1558 spa->spa_unflushed_stats.sus_nblocks = 0; 1559 spa->spa_unflushed_stats.sus_memused = 0; 1560 spa->spa_unflushed_stats.sus_blocklimit = 0; 1561 } 1562 1563 static void 1564 spa_destroy_aux_threads(spa_t *spa) 1565 { 1566 if (spa->spa_condense_zthr != NULL) { 1567 zthr_destroy(spa->spa_condense_zthr); 1568 spa->spa_condense_zthr = NULL; 1569 } 1570 if (spa->spa_checkpoint_discard_zthr != NULL) { 1571 zthr_destroy(spa->spa_checkpoint_discard_zthr); 1572 spa->spa_checkpoint_discard_zthr = NULL; 1573 } 1574 if (spa->spa_livelist_delete_zthr != NULL) { 1575 zthr_destroy(spa->spa_livelist_delete_zthr); 1576 spa->spa_livelist_delete_zthr = NULL; 1577 } 1578 if (spa->spa_livelist_condense_zthr != NULL) { 1579 zthr_destroy(spa->spa_livelist_condense_zthr); 1580 spa->spa_livelist_condense_zthr = NULL; 1581 } 1582 } 1583 1584 /* 1585 * Opposite of spa_load(). 1586 */ 1587 static void 1588 spa_unload(spa_t *spa) 1589 { 1590 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1591 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 1592 1593 spa_import_progress_remove(spa_guid(spa)); 1594 spa_load_note(spa, "UNLOADING"); 1595 1596 spa_wake_waiters(spa); 1597 1598 /* 1599 * If the log space map feature is enabled and the pool is getting 1600 * exported (but not destroyed), we want to spend some time flushing 1601 * as many metaslabs as we can in an attempt to destroy log space 1602 * maps and save import time. 1603 */ 1604 if (spa_should_flush_logs_on_unload(spa)) 1605 spa_unload_log_sm_flush_all(spa); 1606 1607 /* 1608 * Stop async tasks. 1609 */ 1610 spa_async_suspend(spa); 1611 1612 if (spa->spa_root_vdev) { 1613 vdev_t *root_vdev = spa->spa_root_vdev; 1614 vdev_initialize_stop_all(root_vdev, VDEV_INITIALIZE_ACTIVE); 1615 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 1616 vdev_autotrim_stop_all(spa); 1617 vdev_rebuild_stop_all(spa); 1618 } 1619 1620 /* 1621 * Stop syncing. 1622 */ 1623 if (spa->spa_sync_on) { 1624 txg_sync_stop(spa->spa_dsl_pool); 1625 spa->spa_sync_on = B_FALSE; 1626 } 1627 1628 /* 1629 * This ensures that there is no async metaslab prefetching 1630 * while we attempt to unload the spa. 1631 */ 1632 if (spa->spa_root_vdev != NULL) { 1633 for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++) { 1634 vdev_t *vc = spa->spa_root_vdev->vdev_child[c]; 1635 if (vc->vdev_mg != NULL) 1636 taskq_wait(vc->vdev_mg->mg_taskq); 1637 } 1638 } 1639 1640 if (spa->spa_mmp.mmp_thread) 1641 mmp_thread_stop(spa); 1642 1643 /* 1644 * Wait for any outstanding async I/O to complete. 1645 */ 1646 if (spa->spa_async_zio_root != NULL) { 1647 for (int i = 0; i < max_ncpus; i++) 1648 (void) zio_wait(spa->spa_async_zio_root[i]); 1649 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 1650 spa->spa_async_zio_root = NULL; 1651 } 1652 1653 if (spa->spa_vdev_removal != NULL) { 1654 spa_vdev_removal_destroy(spa->spa_vdev_removal); 1655 spa->spa_vdev_removal = NULL; 1656 } 1657 1658 spa_destroy_aux_threads(spa); 1659 1660 spa_condense_fini(spa); 1661 1662 bpobj_close(&spa->spa_deferred_bpobj); 1663 1664 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 1665 1666 /* 1667 * Close all vdevs. 1668 */ 1669 if (spa->spa_root_vdev) 1670 vdev_free(spa->spa_root_vdev); 1671 ASSERT(spa->spa_root_vdev == NULL); 1672 1673 /* 1674 * Close the dsl pool. 1675 */ 1676 if (spa->spa_dsl_pool) { 1677 dsl_pool_close(spa->spa_dsl_pool); 1678 spa->spa_dsl_pool = NULL; 1679 spa->spa_meta_objset = NULL; 1680 } 1681 1682 ddt_unload(spa); 1683 spa_unload_log_sm_metadata(spa); 1684 1685 /* 1686 * Drop and purge level 2 cache 1687 */ 1688 spa_l2cache_drop(spa); 1689 1690 for (int i = 0; i < spa->spa_spares.sav_count; i++) 1691 vdev_free(spa->spa_spares.sav_vdevs[i]); 1692 if (spa->spa_spares.sav_vdevs) { 1693 kmem_free(spa->spa_spares.sav_vdevs, 1694 spa->spa_spares.sav_count * sizeof (void *)); 1695 spa->spa_spares.sav_vdevs = NULL; 1696 } 1697 if (spa->spa_spares.sav_config) { 1698 nvlist_free(spa->spa_spares.sav_config); 1699 spa->spa_spares.sav_config = NULL; 1700 } 1701 spa->spa_spares.sav_count = 0; 1702 1703 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1704 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 1705 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 1706 } 1707 if (spa->spa_l2cache.sav_vdevs) { 1708 kmem_free(spa->spa_l2cache.sav_vdevs, 1709 spa->spa_l2cache.sav_count * sizeof (void *)); 1710 spa->spa_l2cache.sav_vdevs = NULL; 1711 } 1712 if (spa->spa_l2cache.sav_config) { 1713 nvlist_free(spa->spa_l2cache.sav_config); 1714 spa->spa_l2cache.sav_config = NULL; 1715 } 1716 spa->spa_l2cache.sav_count = 0; 1717 1718 spa->spa_async_suspended = 0; 1719 1720 spa->spa_indirect_vdevs_loaded = B_FALSE; 1721 1722 if (spa->spa_comment != NULL) { 1723 spa_strfree(spa->spa_comment); 1724 spa->spa_comment = NULL; 1725 } 1726 if (spa->spa_compatibility != NULL) { 1727 spa_strfree(spa->spa_compatibility); 1728 spa->spa_compatibility = NULL; 1729 } 1730 1731 spa_config_exit(spa, SCL_ALL, spa); 1732 } 1733 1734 /* 1735 * Load (or re-load) the current list of vdevs describing the active spares for 1736 * this pool. When this is called, we have some form of basic information in 1737 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 1738 * then re-generate a more complete list including status information. 1739 */ 1740 void 1741 spa_load_spares(spa_t *spa) 1742 { 1743 nvlist_t **spares; 1744 uint_t nspares; 1745 int i; 1746 vdev_t *vd, *tvd; 1747 1748 #ifndef _KERNEL 1749 /* 1750 * zdb opens both the current state of the pool and the 1751 * checkpointed state (if present), with a different spa_t. 1752 * 1753 * As spare vdevs are shared among open pools, we skip loading 1754 * them when we load the checkpointed state of the pool. 1755 */ 1756 if (!spa_writeable(spa)) 1757 return; 1758 #endif 1759 1760 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1761 1762 /* 1763 * First, close and free any existing spare vdevs. 1764 */ 1765 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1766 vd = spa->spa_spares.sav_vdevs[i]; 1767 1768 /* Undo the call to spa_activate() below */ 1769 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1770 B_FALSE)) != NULL && tvd->vdev_isspare) 1771 spa_spare_remove(tvd); 1772 vdev_close(vd); 1773 vdev_free(vd); 1774 } 1775 1776 if (spa->spa_spares.sav_vdevs) 1777 kmem_free(spa->spa_spares.sav_vdevs, 1778 spa->spa_spares.sav_count * sizeof (void *)); 1779 1780 if (spa->spa_spares.sav_config == NULL) 1781 nspares = 0; 1782 else 1783 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 1784 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 1785 1786 spa->spa_spares.sav_count = (int)nspares; 1787 spa->spa_spares.sav_vdevs = NULL; 1788 1789 if (nspares == 0) 1790 return; 1791 1792 /* 1793 * Construct the array of vdevs, opening them to get status in the 1794 * process. For each spare, there is potentially two different vdev_t 1795 * structures associated with it: one in the list of spares (used only 1796 * for basic validation purposes) and one in the active vdev 1797 * configuration (if it's spared in). During this phase we open and 1798 * validate each vdev on the spare list. If the vdev also exists in the 1799 * active configuration, then we also mark this vdev as an active spare. 1800 */ 1801 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 1802 KM_SLEEP); 1803 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1804 VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, 1805 VDEV_ALLOC_SPARE) == 0); 1806 ASSERT(vd != NULL); 1807 1808 spa->spa_spares.sav_vdevs[i] = vd; 1809 1810 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1811 B_FALSE)) != NULL) { 1812 if (!tvd->vdev_isspare) 1813 spa_spare_add(tvd); 1814 1815 /* 1816 * We only mark the spare active if we were successfully 1817 * able to load the vdev. Otherwise, importing a pool 1818 * with a bad active spare would result in strange 1819 * behavior, because multiple pool would think the spare 1820 * is actively in use. 1821 * 1822 * There is a vulnerability here to an equally bizarre 1823 * circumstance, where a dead active spare is later 1824 * brought back to life (onlined or otherwise). Given 1825 * the rarity of this scenario, and the extra complexity 1826 * it adds, we ignore the possibility. 1827 */ 1828 if (!vdev_is_dead(tvd)) 1829 spa_spare_activate(tvd); 1830 } 1831 1832 vd->vdev_top = vd; 1833 vd->vdev_aux = &spa->spa_spares; 1834 1835 if (vdev_open(vd) != 0) 1836 continue; 1837 1838 if (vdev_validate_aux(vd) == 0) 1839 spa_spare_add(vd); 1840 } 1841 1842 /* 1843 * Recompute the stashed list of spares, with status information 1844 * this time. 1845 */ 1846 fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES); 1847 1848 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 1849 KM_SLEEP); 1850 for (i = 0; i < spa->spa_spares.sav_count; i++) 1851 spares[i] = vdev_config_generate(spa, 1852 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 1853 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 1854 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 1855 spa->spa_spares.sav_count); 1856 for (i = 0; i < spa->spa_spares.sav_count; i++) 1857 nvlist_free(spares[i]); 1858 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 1859 } 1860 1861 /* 1862 * Load (or re-load) the current list of vdevs describing the active l2cache for 1863 * this pool. When this is called, we have some form of basic information in 1864 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 1865 * then re-generate a more complete list including status information. 1866 * Devices which are already active have their details maintained, and are 1867 * not re-opened. 1868 */ 1869 void 1870 spa_load_l2cache(spa_t *spa) 1871 { 1872 nvlist_t **l2cache = NULL; 1873 uint_t nl2cache; 1874 int i, j, oldnvdevs; 1875 uint64_t guid; 1876 vdev_t *vd, **oldvdevs, **newvdevs; 1877 spa_aux_vdev_t *sav = &spa->spa_l2cache; 1878 1879 #ifndef _KERNEL 1880 /* 1881 * zdb opens both the current state of the pool and the 1882 * checkpointed state (if present), with a different spa_t. 1883 * 1884 * As L2 caches are part of the ARC which is shared among open 1885 * pools, we skip loading them when we load the checkpointed 1886 * state of the pool. 1887 */ 1888 if (!spa_writeable(spa)) 1889 return; 1890 #endif 1891 1892 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1893 1894 oldvdevs = sav->sav_vdevs; 1895 oldnvdevs = sav->sav_count; 1896 sav->sav_vdevs = NULL; 1897 sav->sav_count = 0; 1898 1899 if (sav->sav_config == NULL) { 1900 nl2cache = 0; 1901 newvdevs = NULL; 1902 goto out; 1903 } 1904 1905 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, 1906 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 1907 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 1908 1909 /* 1910 * Process new nvlist of vdevs. 1911 */ 1912 for (i = 0; i < nl2cache; i++) { 1913 guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID); 1914 1915 newvdevs[i] = NULL; 1916 for (j = 0; j < oldnvdevs; j++) { 1917 vd = oldvdevs[j]; 1918 if (vd != NULL && guid == vd->vdev_guid) { 1919 /* 1920 * Retain previous vdev for add/remove ops. 1921 */ 1922 newvdevs[i] = vd; 1923 oldvdevs[j] = NULL; 1924 break; 1925 } 1926 } 1927 1928 if (newvdevs[i] == NULL) { 1929 /* 1930 * Create new vdev 1931 */ 1932 VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 1933 VDEV_ALLOC_L2CACHE) == 0); 1934 ASSERT(vd != NULL); 1935 newvdevs[i] = vd; 1936 1937 /* 1938 * Commit this vdev as an l2cache device, 1939 * even if it fails to open. 1940 */ 1941 spa_l2cache_add(vd); 1942 1943 vd->vdev_top = vd; 1944 vd->vdev_aux = sav; 1945 1946 spa_l2cache_activate(vd); 1947 1948 if (vdev_open(vd) != 0) 1949 continue; 1950 1951 (void) vdev_validate_aux(vd); 1952 1953 if (!vdev_is_dead(vd)) 1954 l2arc_add_vdev(spa, vd); 1955 1956 /* 1957 * Upon cache device addition to a pool or pool 1958 * creation with a cache device or if the header 1959 * of the device is invalid we issue an async 1960 * TRIM command for the whole device which will 1961 * execute if l2arc_trim_ahead > 0. 1962 */ 1963 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 1964 } 1965 } 1966 1967 sav->sav_vdevs = newvdevs; 1968 sav->sav_count = (int)nl2cache; 1969 1970 /* 1971 * Recompute the stashed list of l2cache devices, with status 1972 * information this time. 1973 */ 1974 fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE); 1975 1976 if (sav->sav_count > 0) 1977 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 1978 KM_SLEEP); 1979 for (i = 0; i < sav->sav_count; i++) 1980 l2cache[i] = vdev_config_generate(spa, 1981 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 1982 fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 1983 (const nvlist_t * const *)l2cache, sav->sav_count); 1984 1985 out: 1986 /* 1987 * Purge vdevs that were dropped 1988 */ 1989 for (i = 0; i < oldnvdevs; i++) { 1990 uint64_t pool; 1991 1992 vd = oldvdevs[i]; 1993 if (vd != NULL) { 1994 ASSERT(vd->vdev_isl2cache); 1995 1996 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 1997 pool != 0ULL && l2arc_vdev_present(vd)) 1998 l2arc_remove_vdev(vd); 1999 vdev_clear_stats(vd); 2000 vdev_free(vd); 2001 } 2002 } 2003 2004 if (oldvdevs) 2005 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2006 2007 for (i = 0; i < sav->sav_count; i++) 2008 nvlist_free(l2cache[i]); 2009 if (sav->sav_count) 2010 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2011 } 2012 2013 static int 2014 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2015 { 2016 dmu_buf_t *db; 2017 char *packed = NULL; 2018 size_t nvsize = 0; 2019 int error; 2020 *value = NULL; 2021 2022 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2023 if (error) 2024 return (error); 2025 2026 nvsize = *(uint64_t *)db->db_data; 2027 dmu_buf_rele(db, FTAG); 2028 2029 packed = vmem_alloc(nvsize, KM_SLEEP); 2030 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2031 DMU_READ_PREFETCH); 2032 if (error == 0) 2033 error = nvlist_unpack(packed, nvsize, value, 0); 2034 vmem_free(packed, nvsize); 2035 2036 return (error); 2037 } 2038 2039 /* 2040 * Concrete top-level vdevs that are not missing and are not logs. At every 2041 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2042 */ 2043 static uint64_t 2044 spa_healthy_core_tvds(spa_t *spa) 2045 { 2046 vdev_t *rvd = spa->spa_root_vdev; 2047 uint64_t tvds = 0; 2048 2049 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2050 vdev_t *vd = rvd->vdev_child[i]; 2051 if (vd->vdev_islog) 2052 continue; 2053 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2054 tvds++; 2055 } 2056 2057 return (tvds); 2058 } 2059 2060 /* 2061 * Checks to see if the given vdev could not be opened, in which case we post a 2062 * sysevent to notify the autoreplace code that the device has been removed. 2063 */ 2064 static void 2065 spa_check_removed(vdev_t *vd) 2066 { 2067 for (uint64_t c = 0; c < vd->vdev_children; c++) 2068 spa_check_removed(vd->vdev_child[c]); 2069 2070 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2071 vdev_is_concrete(vd)) { 2072 zfs_post_autoreplace(vd->vdev_spa, vd); 2073 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2074 } 2075 } 2076 2077 static int 2078 spa_check_for_missing_logs(spa_t *spa) 2079 { 2080 vdev_t *rvd = spa->spa_root_vdev; 2081 2082 /* 2083 * If we're doing a normal import, then build up any additional 2084 * diagnostic information about missing log devices. 2085 * We'll pass this up to the user for further processing. 2086 */ 2087 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2088 nvlist_t **child, *nv; 2089 uint64_t idx = 0; 2090 2091 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2092 KM_SLEEP); 2093 nv = fnvlist_alloc(); 2094 2095 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2096 vdev_t *tvd = rvd->vdev_child[c]; 2097 2098 /* 2099 * We consider a device as missing only if it failed 2100 * to open (i.e. offline or faulted is not considered 2101 * as missing). 2102 */ 2103 if (tvd->vdev_islog && 2104 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2105 child[idx++] = vdev_config_generate(spa, tvd, 2106 B_FALSE, VDEV_CONFIG_MISSING); 2107 } 2108 } 2109 2110 if (idx > 0) { 2111 fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 2112 (const nvlist_t * const *)child, idx); 2113 fnvlist_add_nvlist(spa->spa_load_info, 2114 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2115 2116 for (uint64_t i = 0; i < idx; i++) 2117 nvlist_free(child[i]); 2118 } 2119 nvlist_free(nv); 2120 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2121 2122 if (idx > 0) { 2123 spa_load_failed(spa, "some log devices are missing"); 2124 vdev_dbgmsg_print_tree(rvd, 2); 2125 return (SET_ERROR(ENXIO)); 2126 } 2127 } else { 2128 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2129 vdev_t *tvd = rvd->vdev_child[c]; 2130 2131 if (tvd->vdev_islog && 2132 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2133 spa_set_log_state(spa, SPA_LOG_CLEAR); 2134 spa_load_note(spa, "some log devices are " 2135 "missing, ZIL is dropped."); 2136 vdev_dbgmsg_print_tree(rvd, 2); 2137 break; 2138 } 2139 } 2140 } 2141 2142 return (0); 2143 } 2144 2145 /* 2146 * Check for missing log devices 2147 */ 2148 static boolean_t 2149 spa_check_logs(spa_t *spa) 2150 { 2151 boolean_t rv = B_FALSE; 2152 dsl_pool_t *dp = spa_get_dsl(spa); 2153 2154 switch (spa->spa_log_state) { 2155 default: 2156 break; 2157 case SPA_LOG_MISSING: 2158 /* need to recheck in case slog has been restored */ 2159 case SPA_LOG_UNKNOWN: 2160 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2161 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2162 if (rv) 2163 spa_set_log_state(spa, SPA_LOG_MISSING); 2164 break; 2165 } 2166 return (rv); 2167 } 2168 2169 /* 2170 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2171 */ 2172 static boolean_t 2173 spa_passivate_log(spa_t *spa) 2174 { 2175 vdev_t *rvd = spa->spa_root_vdev; 2176 boolean_t slog_found = B_FALSE; 2177 2178 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2179 2180 for (int c = 0; c < rvd->vdev_children; c++) { 2181 vdev_t *tvd = rvd->vdev_child[c]; 2182 2183 if (tvd->vdev_islog) { 2184 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2185 metaslab_group_passivate(tvd->vdev_mg); 2186 slog_found = B_TRUE; 2187 } 2188 } 2189 2190 return (slog_found); 2191 } 2192 2193 /* 2194 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2195 */ 2196 static void 2197 spa_activate_log(spa_t *spa) 2198 { 2199 vdev_t *rvd = spa->spa_root_vdev; 2200 2201 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2202 2203 for (int c = 0; c < rvd->vdev_children; c++) { 2204 vdev_t *tvd = rvd->vdev_child[c]; 2205 2206 if (tvd->vdev_islog) { 2207 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2208 metaslab_group_activate(tvd->vdev_mg); 2209 } 2210 } 2211 } 2212 2213 int 2214 spa_reset_logs(spa_t *spa) 2215 { 2216 int error; 2217 2218 error = dmu_objset_find(spa_name(spa), zil_reset, 2219 NULL, DS_FIND_CHILDREN); 2220 if (error == 0) { 2221 /* 2222 * We successfully offlined the log device, sync out the 2223 * current txg so that the "stubby" block can be removed 2224 * by zil_sync(). 2225 */ 2226 txg_wait_synced(spa->spa_dsl_pool, 0); 2227 } 2228 return (error); 2229 } 2230 2231 static void 2232 spa_aux_check_removed(spa_aux_vdev_t *sav) 2233 { 2234 for (int i = 0; i < sav->sav_count; i++) 2235 spa_check_removed(sav->sav_vdevs[i]); 2236 } 2237 2238 void 2239 spa_claim_notify(zio_t *zio) 2240 { 2241 spa_t *spa = zio->io_spa; 2242 2243 if (zio->io_error) 2244 return; 2245 2246 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2247 if (spa->spa_claim_max_txg < zio->io_bp->blk_birth) 2248 spa->spa_claim_max_txg = zio->io_bp->blk_birth; 2249 mutex_exit(&spa->spa_props_lock); 2250 } 2251 2252 typedef struct spa_load_error { 2253 uint64_t sle_meta_count; 2254 uint64_t sle_data_count; 2255 } spa_load_error_t; 2256 2257 static void 2258 spa_load_verify_done(zio_t *zio) 2259 { 2260 blkptr_t *bp = zio->io_bp; 2261 spa_load_error_t *sle = zio->io_private; 2262 dmu_object_type_t type = BP_GET_TYPE(bp); 2263 int error = zio->io_error; 2264 spa_t *spa = zio->io_spa; 2265 2266 abd_free(zio->io_abd); 2267 if (error) { 2268 if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && 2269 type != DMU_OT_INTENT_LOG) 2270 atomic_inc_64(&sle->sle_meta_count); 2271 else 2272 atomic_inc_64(&sle->sle_data_count); 2273 } 2274 2275 mutex_enter(&spa->spa_scrub_lock); 2276 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 2277 cv_broadcast(&spa->spa_scrub_io_cv); 2278 mutex_exit(&spa->spa_scrub_lock); 2279 } 2280 2281 /* 2282 * Maximum number of inflight bytes is the log2 fraction of the arc size. 2283 * By default, we set it to 1/16th of the arc. 2284 */ 2285 int spa_load_verify_shift = 4; 2286 int spa_load_verify_metadata = B_TRUE; 2287 int spa_load_verify_data = B_TRUE; 2288 2289 /*ARGSUSED*/ 2290 static int 2291 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 2292 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 2293 { 2294 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 2295 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 2296 return (0); 2297 /* 2298 * Note: normally this routine will not be called if 2299 * spa_load_verify_metadata is not set. However, it may be useful 2300 * to manually set the flag after the traversal has begun. 2301 */ 2302 if (!spa_load_verify_metadata) 2303 return (0); 2304 if (!BP_IS_METADATA(bp) && !spa_load_verify_data) 2305 return (0); 2306 2307 uint64_t maxinflight_bytes = 2308 arc_target_bytes() >> spa_load_verify_shift; 2309 zio_t *rio = arg; 2310 size_t size = BP_GET_PSIZE(bp); 2311 2312 mutex_enter(&spa->spa_scrub_lock); 2313 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 2314 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 2315 spa->spa_load_verify_bytes += size; 2316 mutex_exit(&spa->spa_scrub_lock); 2317 2318 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 2319 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 2320 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 2321 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 2322 return (0); 2323 } 2324 2325 /* ARGSUSED */ 2326 static int 2327 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 2328 { 2329 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 2330 return (SET_ERROR(ENAMETOOLONG)); 2331 2332 return (0); 2333 } 2334 2335 static int 2336 spa_load_verify(spa_t *spa) 2337 { 2338 zio_t *rio; 2339 spa_load_error_t sle = { 0 }; 2340 zpool_load_policy_t policy; 2341 boolean_t verify_ok = B_FALSE; 2342 int error = 0; 2343 2344 zpool_get_load_policy(spa->spa_config, &policy); 2345 2346 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND) 2347 return (0); 2348 2349 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 2350 error = dmu_objset_find_dp(spa->spa_dsl_pool, 2351 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 2352 DS_FIND_CHILDREN); 2353 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 2354 if (error != 0) 2355 return (error); 2356 2357 rio = zio_root(spa, NULL, &sle, 2358 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 2359 2360 if (spa_load_verify_metadata) { 2361 if (spa->spa_extreme_rewind) { 2362 spa_load_note(spa, "performing a complete scan of the " 2363 "pool since extreme rewind is on. This may take " 2364 "a very long time.\n (spa_load_verify_data=%u, " 2365 "spa_load_verify_metadata=%u)", 2366 spa_load_verify_data, spa_load_verify_metadata); 2367 } 2368 2369 error = traverse_pool(spa, spa->spa_verify_min_txg, 2370 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 2371 TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio); 2372 } 2373 2374 (void) zio_wait(rio); 2375 ASSERT0(spa->spa_load_verify_bytes); 2376 2377 spa->spa_load_meta_errors = sle.sle_meta_count; 2378 spa->spa_load_data_errors = sle.sle_data_count; 2379 2380 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 2381 spa_load_note(spa, "spa_load_verify found %llu metadata errors " 2382 "and %llu data errors", (u_longlong_t)sle.sle_meta_count, 2383 (u_longlong_t)sle.sle_data_count); 2384 } 2385 2386 if (spa_load_verify_dryrun || 2387 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 2388 sle.sle_data_count <= policy.zlp_maxdata)) { 2389 int64_t loss = 0; 2390 2391 verify_ok = B_TRUE; 2392 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 2393 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 2394 2395 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; 2396 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME, 2397 spa->spa_load_txg_ts); 2398 fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME, 2399 loss); 2400 fnvlist_add_uint64(spa->spa_load_info, 2401 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count); 2402 } else { 2403 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 2404 } 2405 2406 if (spa_load_verify_dryrun) 2407 return (0); 2408 2409 if (error) { 2410 if (error != ENXIO && error != EIO) 2411 error = SET_ERROR(EIO); 2412 return (error); 2413 } 2414 2415 return (verify_ok ? 0 : EIO); 2416 } 2417 2418 /* 2419 * Find a value in the pool props object. 2420 */ 2421 static void 2422 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 2423 { 2424 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 2425 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 2426 } 2427 2428 /* 2429 * Find a value in the pool directory object. 2430 */ 2431 static int 2432 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 2433 { 2434 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2435 name, sizeof (uint64_t), 1, val); 2436 2437 if (error != 0 && (error != ENOENT || log_enoent)) { 2438 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 2439 "[error=%d]", name, error); 2440 } 2441 2442 return (error); 2443 } 2444 2445 static int 2446 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 2447 { 2448 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 2449 return (SET_ERROR(err)); 2450 } 2451 2452 boolean_t 2453 spa_livelist_delete_check(spa_t *spa) 2454 { 2455 return (spa->spa_livelists_to_delete != 0); 2456 } 2457 2458 /* ARGSUSED */ 2459 static boolean_t 2460 spa_livelist_delete_cb_check(void *arg, zthr_t *z) 2461 { 2462 spa_t *spa = arg; 2463 return (spa_livelist_delete_check(spa)); 2464 } 2465 2466 static int 2467 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 2468 { 2469 spa_t *spa = arg; 2470 zio_free(spa, tx->tx_txg, bp); 2471 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 2472 -bp_get_dsize_sync(spa, bp), 2473 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 2474 return (0); 2475 } 2476 2477 static int 2478 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 2479 { 2480 int err; 2481 zap_cursor_t zc; 2482 zap_attribute_t za; 2483 zap_cursor_init(&zc, os, zap_obj); 2484 err = zap_cursor_retrieve(&zc, &za); 2485 zap_cursor_fini(&zc); 2486 if (err == 0) 2487 *llp = za.za_first_integer; 2488 return (err); 2489 } 2490 2491 /* 2492 * Components of livelist deletion that must be performed in syncing 2493 * context: freeing block pointers and updating the pool-wide data 2494 * structures to indicate how much work is left to do 2495 */ 2496 typedef struct sublist_delete_arg { 2497 spa_t *spa; 2498 dsl_deadlist_t *ll; 2499 uint64_t key; 2500 bplist_t *to_free; 2501 } sublist_delete_arg_t; 2502 2503 static void 2504 sublist_delete_sync(void *arg, dmu_tx_t *tx) 2505 { 2506 sublist_delete_arg_t *sda = arg; 2507 spa_t *spa = sda->spa; 2508 dsl_deadlist_t *ll = sda->ll; 2509 uint64_t key = sda->key; 2510 bplist_t *to_free = sda->to_free; 2511 2512 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 2513 dsl_deadlist_remove_entry(ll, key, tx); 2514 } 2515 2516 typedef struct livelist_delete_arg { 2517 spa_t *spa; 2518 uint64_t ll_obj; 2519 uint64_t zap_obj; 2520 } livelist_delete_arg_t; 2521 2522 static void 2523 livelist_delete_sync(void *arg, dmu_tx_t *tx) 2524 { 2525 livelist_delete_arg_t *lda = arg; 2526 spa_t *spa = lda->spa; 2527 uint64_t ll_obj = lda->ll_obj; 2528 uint64_t zap_obj = lda->zap_obj; 2529 objset_t *mos = spa->spa_meta_objset; 2530 uint64_t count; 2531 2532 /* free the livelist and decrement the feature count */ 2533 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 2534 dsl_deadlist_free(mos, ll_obj, tx); 2535 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 2536 VERIFY0(zap_count(mos, zap_obj, &count)); 2537 if (count == 0) { 2538 /* no more livelists to delete */ 2539 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 2540 DMU_POOL_DELETED_CLONES, tx)); 2541 VERIFY0(zap_destroy(mos, zap_obj, tx)); 2542 spa->spa_livelists_to_delete = 0; 2543 spa_notify_waiters(spa); 2544 } 2545 } 2546 2547 /* 2548 * Load in the value for the livelist to be removed and open it. Then, 2549 * load its first sublist and determine which block pointers should actually 2550 * be freed. Then, call a synctask which performs the actual frees and updates 2551 * the pool-wide livelist data. 2552 */ 2553 /* ARGSUSED */ 2554 static void 2555 spa_livelist_delete_cb(void *arg, zthr_t *z) 2556 { 2557 spa_t *spa = arg; 2558 uint64_t ll_obj = 0, count; 2559 objset_t *mos = spa->spa_meta_objset; 2560 uint64_t zap_obj = spa->spa_livelists_to_delete; 2561 /* 2562 * Determine the next livelist to delete. This function should only 2563 * be called if there is at least one deleted clone. 2564 */ 2565 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 2566 VERIFY0(zap_count(mos, ll_obj, &count)); 2567 if (count > 0) { 2568 dsl_deadlist_t *ll; 2569 dsl_deadlist_entry_t *dle; 2570 bplist_t to_free; 2571 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 2572 dsl_deadlist_open(ll, mos, ll_obj); 2573 dle = dsl_deadlist_first(ll); 2574 ASSERT3P(dle, !=, NULL); 2575 bplist_create(&to_free); 2576 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 2577 z, NULL); 2578 if (err == 0) { 2579 sublist_delete_arg_t sync_arg = { 2580 .spa = spa, 2581 .ll = ll, 2582 .key = dle->dle_mintxg, 2583 .to_free = &to_free 2584 }; 2585 zfs_dbgmsg("deleting sublist (id %llu) from" 2586 " livelist %llu, %lld remaining", 2587 (u_longlong_t)dle->dle_bpobj.bpo_object, 2588 (u_longlong_t)ll_obj, (longlong_t)count - 1); 2589 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 2590 sublist_delete_sync, &sync_arg, 0, 2591 ZFS_SPACE_CHECK_DESTROY)); 2592 } else { 2593 VERIFY3U(err, ==, EINTR); 2594 } 2595 bplist_clear(&to_free); 2596 bplist_destroy(&to_free); 2597 dsl_deadlist_close(ll); 2598 kmem_free(ll, sizeof (dsl_deadlist_t)); 2599 } else { 2600 livelist_delete_arg_t sync_arg = { 2601 .spa = spa, 2602 .ll_obj = ll_obj, 2603 .zap_obj = zap_obj 2604 }; 2605 zfs_dbgmsg("deletion of livelist %llu completed", 2606 (u_longlong_t)ll_obj); 2607 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 2608 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 2609 } 2610 } 2611 2612 static void 2613 spa_start_livelist_destroy_thread(spa_t *spa) 2614 { 2615 ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL); 2616 spa->spa_livelist_delete_zthr = 2617 zthr_create("z_livelist_destroy", 2618 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa, 2619 minclsyspri); 2620 } 2621 2622 typedef struct livelist_new_arg { 2623 bplist_t *allocs; 2624 bplist_t *frees; 2625 } livelist_new_arg_t; 2626 2627 static int 2628 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 2629 dmu_tx_t *tx) 2630 { 2631 ASSERT(tx == NULL); 2632 livelist_new_arg_t *lna = arg; 2633 if (bp_freed) { 2634 bplist_append(lna->frees, bp); 2635 } else { 2636 bplist_append(lna->allocs, bp); 2637 zfs_livelist_condense_new_alloc++; 2638 } 2639 return (0); 2640 } 2641 2642 typedef struct livelist_condense_arg { 2643 spa_t *spa; 2644 bplist_t to_keep; 2645 uint64_t first_size; 2646 uint64_t next_size; 2647 } livelist_condense_arg_t; 2648 2649 static void 2650 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 2651 { 2652 livelist_condense_arg_t *lca = arg; 2653 spa_t *spa = lca->spa; 2654 bplist_t new_frees; 2655 dsl_dataset_t *ds = spa->spa_to_condense.ds; 2656 2657 /* Have we been cancelled? */ 2658 if (spa->spa_to_condense.cancelled) { 2659 zfs_livelist_condense_sync_cancel++; 2660 goto out; 2661 } 2662 2663 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2664 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2665 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 2666 2667 /* 2668 * It's possible that the livelist was changed while the zthr was 2669 * running. Therefore, we need to check for new blkptrs in the two 2670 * entries being condensed and continue to track them in the livelist. 2671 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 2672 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 2673 * we need to sort them into two different bplists. 2674 */ 2675 uint64_t first_obj = first->dle_bpobj.bpo_object; 2676 uint64_t next_obj = next->dle_bpobj.bpo_object; 2677 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2678 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2679 2680 bplist_create(&new_frees); 2681 livelist_new_arg_t new_bps = { 2682 .allocs = &lca->to_keep, 2683 .frees = &new_frees, 2684 }; 2685 2686 if (cur_first_size > lca->first_size) { 2687 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 2688 livelist_track_new_cb, &new_bps, lca->first_size)); 2689 } 2690 if (cur_next_size > lca->next_size) { 2691 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 2692 livelist_track_new_cb, &new_bps, lca->next_size)); 2693 } 2694 2695 dsl_deadlist_clear_entry(first, ll, tx); 2696 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 2697 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 2698 2699 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 2700 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 2701 bplist_destroy(&new_frees); 2702 2703 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 2704 dsl_dataset_name(ds, dsname); 2705 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 2706 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 2707 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, 2708 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, 2709 (u_longlong_t)cur_first_size, (u_longlong_t)next_obj, 2710 (u_longlong_t)cur_next_size, 2711 (u_longlong_t)first->dle_bpobj.bpo_object, 2712 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 2713 out: 2714 dmu_buf_rele(ds->ds_dbuf, spa); 2715 spa->spa_to_condense.ds = NULL; 2716 bplist_clear(&lca->to_keep); 2717 bplist_destroy(&lca->to_keep); 2718 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2719 spa->spa_to_condense.syncing = B_FALSE; 2720 } 2721 2722 static void 2723 spa_livelist_condense_cb(void *arg, zthr_t *t) 2724 { 2725 while (zfs_livelist_condense_zthr_pause && 2726 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2727 delay(1); 2728 2729 spa_t *spa = arg; 2730 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2731 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2732 uint64_t first_size, next_size; 2733 2734 livelist_condense_arg_t *lca = 2735 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 2736 bplist_create(&lca->to_keep); 2737 2738 /* 2739 * Process the livelists (matching FREEs and ALLOCs) in open context 2740 * so we have minimal work in syncing context to condense. 2741 * 2742 * We save bpobj sizes (first_size and next_size) to use later in 2743 * syncing context to determine if entries were added to these sublists 2744 * while in open context. This is possible because the clone is still 2745 * active and open for normal writes and we want to make sure the new, 2746 * unprocessed blockpointers are inserted into the livelist normally. 2747 * 2748 * Note that dsl_process_sub_livelist() both stores the size number of 2749 * blockpointers and iterates over them while the bpobj's lock held, so 2750 * the sizes returned to us are consistent which what was actually 2751 * processed. 2752 */ 2753 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 2754 &first_size); 2755 if (err == 0) 2756 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 2757 t, &next_size); 2758 2759 if (err == 0) { 2760 while (zfs_livelist_condense_sync_pause && 2761 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2762 delay(1); 2763 2764 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2765 dmu_tx_mark_netfree(tx); 2766 dmu_tx_hold_space(tx, 1); 2767 err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE); 2768 if (err == 0) { 2769 /* 2770 * Prevent the condense zthr restarting before 2771 * the synctask completes. 2772 */ 2773 spa->spa_to_condense.syncing = B_TRUE; 2774 lca->spa = spa; 2775 lca->first_size = first_size; 2776 lca->next_size = next_size; 2777 dsl_sync_task_nowait(spa_get_dsl(spa), 2778 spa_livelist_condense_sync, lca, tx); 2779 dmu_tx_commit(tx); 2780 return; 2781 } 2782 } 2783 /* 2784 * Condensing can not continue: either it was externally stopped or 2785 * we were unable to assign to a tx because the pool has run out of 2786 * space. In the second case, we'll just end up trying to condense 2787 * again in a later txg. 2788 */ 2789 ASSERT(err != 0); 2790 bplist_clear(&lca->to_keep); 2791 bplist_destroy(&lca->to_keep); 2792 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2793 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 2794 spa->spa_to_condense.ds = NULL; 2795 if (err == EINTR) 2796 zfs_livelist_condense_zthr_cancel++; 2797 } 2798 2799 /* ARGSUSED */ 2800 /* 2801 * Check that there is something to condense but that a condense is not 2802 * already in progress and that condensing has not been cancelled. 2803 */ 2804 static boolean_t 2805 spa_livelist_condense_cb_check(void *arg, zthr_t *z) 2806 { 2807 spa_t *spa = arg; 2808 if ((spa->spa_to_condense.ds != NULL) && 2809 (spa->spa_to_condense.syncing == B_FALSE) && 2810 (spa->spa_to_condense.cancelled == B_FALSE)) { 2811 return (B_TRUE); 2812 } 2813 return (B_FALSE); 2814 } 2815 2816 static void 2817 spa_start_livelist_condensing_thread(spa_t *spa) 2818 { 2819 spa->spa_to_condense.ds = NULL; 2820 spa->spa_to_condense.first = NULL; 2821 spa->spa_to_condense.next = NULL; 2822 spa->spa_to_condense.syncing = B_FALSE; 2823 spa->spa_to_condense.cancelled = B_FALSE; 2824 2825 ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL); 2826 spa->spa_livelist_condense_zthr = 2827 zthr_create("z_livelist_condense", 2828 spa_livelist_condense_cb_check, 2829 spa_livelist_condense_cb, spa, minclsyspri); 2830 } 2831 2832 static void 2833 spa_spawn_aux_threads(spa_t *spa) 2834 { 2835 ASSERT(spa_writeable(spa)); 2836 2837 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 2838 2839 spa_start_indirect_condensing_thread(spa); 2840 spa_start_livelist_destroy_thread(spa); 2841 spa_start_livelist_condensing_thread(spa); 2842 2843 ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL); 2844 spa->spa_checkpoint_discard_zthr = 2845 zthr_create("z_checkpoint_discard", 2846 spa_checkpoint_discard_thread_check, 2847 spa_checkpoint_discard_thread, spa, minclsyspri); 2848 } 2849 2850 /* 2851 * Fix up config after a partly-completed split. This is done with the 2852 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 2853 * pool have that entry in their config, but only the splitting one contains 2854 * a list of all the guids of the vdevs that are being split off. 2855 * 2856 * This function determines what to do with that list: either rejoin 2857 * all the disks to the pool, or complete the splitting process. To attempt 2858 * the rejoin, each disk that is offlined is marked online again, and 2859 * we do a reopen() call. If the vdev label for every disk that was 2860 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 2861 * then we call vdev_split() on each disk, and complete the split. 2862 * 2863 * Otherwise we leave the config alone, with all the vdevs in place in 2864 * the original pool. 2865 */ 2866 static void 2867 spa_try_repair(spa_t *spa, nvlist_t *config) 2868 { 2869 uint_t extracted; 2870 uint64_t *glist; 2871 uint_t i, gcount; 2872 nvlist_t *nvl; 2873 vdev_t **vd; 2874 boolean_t attempt_reopen; 2875 2876 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 2877 return; 2878 2879 /* check that the config is complete */ 2880 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 2881 &glist, &gcount) != 0) 2882 return; 2883 2884 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 2885 2886 /* attempt to online all the vdevs & validate */ 2887 attempt_reopen = B_TRUE; 2888 for (i = 0; i < gcount; i++) { 2889 if (glist[i] == 0) /* vdev is hole */ 2890 continue; 2891 2892 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 2893 if (vd[i] == NULL) { 2894 /* 2895 * Don't bother attempting to reopen the disks; 2896 * just do the split. 2897 */ 2898 attempt_reopen = B_FALSE; 2899 } else { 2900 /* attempt to re-online it */ 2901 vd[i]->vdev_offline = B_FALSE; 2902 } 2903 } 2904 2905 if (attempt_reopen) { 2906 vdev_reopen(spa->spa_root_vdev); 2907 2908 /* check each device to see what state it's in */ 2909 for (extracted = 0, i = 0; i < gcount; i++) { 2910 if (vd[i] != NULL && 2911 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 2912 break; 2913 ++extracted; 2914 } 2915 } 2916 2917 /* 2918 * If every disk has been moved to the new pool, or if we never 2919 * even attempted to look at them, then we split them off for 2920 * good. 2921 */ 2922 if (!attempt_reopen || gcount == extracted) { 2923 for (i = 0; i < gcount; i++) 2924 if (vd[i] != NULL) 2925 vdev_split(vd[i]); 2926 vdev_reopen(spa->spa_root_vdev); 2927 } 2928 2929 kmem_free(vd, gcount * sizeof (vdev_t *)); 2930 } 2931 2932 static int 2933 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 2934 { 2935 char *ereport = FM_EREPORT_ZFS_POOL; 2936 int error; 2937 2938 spa->spa_load_state = state; 2939 (void) spa_import_progress_set_state(spa_guid(spa), 2940 spa_load_state(spa)); 2941 2942 gethrestime(&spa->spa_loaded_ts); 2943 error = spa_load_impl(spa, type, &ereport); 2944 2945 /* 2946 * Don't count references from objsets that are already closed 2947 * and are making their way through the eviction process. 2948 */ 2949 spa_evicting_os_wait(spa); 2950 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 2951 if (error) { 2952 if (error != EEXIST) { 2953 spa->spa_loaded_ts.tv_sec = 0; 2954 spa->spa_loaded_ts.tv_nsec = 0; 2955 } 2956 if (error != EBADF) { 2957 (void) zfs_ereport_post(ereport, spa, 2958 NULL, NULL, NULL, 0); 2959 } 2960 } 2961 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 2962 spa->spa_ena = 0; 2963 2964 (void) spa_import_progress_set_state(spa_guid(spa), 2965 spa_load_state(spa)); 2966 2967 return (error); 2968 } 2969 2970 #ifdef ZFS_DEBUG 2971 /* 2972 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 2973 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 2974 * spa's per-vdev ZAP list. 2975 */ 2976 static uint64_t 2977 vdev_count_verify_zaps(vdev_t *vd) 2978 { 2979 spa_t *spa = vd->vdev_spa; 2980 uint64_t total = 0; 2981 2982 if (vd->vdev_top_zap != 0) { 2983 total++; 2984 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2985 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 2986 } 2987 if (vd->vdev_leaf_zap != 0) { 2988 total++; 2989 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2990 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 2991 } 2992 2993 for (uint64_t i = 0; i < vd->vdev_children; i++) { 2994 total += vdev_count_verify_zaps(vd->vdev_child[i]); 2995 } 2996 2997 return (total); 2998 } 2999 #endif 3000 3001 /* 3002 * Determine whether the activity check is required. 3003 */ 3004 static boolean_t 3005 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label, 3006 nvlist_t *config) 3007 { 3008 uint64_t state = 0; 3009 uint64_t hostid = 0; 3010 uint64_t tryconfig_txg = 0; 3011 uint64_t tryconfig_timestamp = 0; 3012 uint16_t tryconfig_mmp_seq = 0; 3013 nvlist_t *nvinfo; 3014 3015 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3016 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 3017 (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3018 &tryconfig_txg); 3019 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3020 &tryconfig_timestamp); 3021 (void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3022 &tryconfig_mmp_seq); 3023 } 3024 3025 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); 3026 3027 /* 3028 * Disable the MMP activity check - This is used by zdb which 3029 * is intended to be used on potentially active pools. 3030 */ 3031 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) 3032 return (B_FALSE); 3033 3034 /* 3035 * Skip the activity check when the MMP feature is disabled. 3036 */ 3037 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) 3038 return (B_FALSE); 3039 3040 /* 3041 * If the tryconfig_ values are nonzero, they are the results of an 3042 * earlier tryimport. If they all match the uberblock we just found, 3043 * then the pool has not changed and we return false so we do not test 3044 * a second time. 3045 */ 3046 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && 3047 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && 3048 tryconfig_mmp_seq && tryconfig_mmp_seq == 3049 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) 3050 return (B_FALSE); 3051 3052 /* 3053 * Allow the activity check to be skipped when importing the pool 3054 * on the same host which last imported it. Since the hostid from 3055 * configuration may be stale use the one read from the label. 3056 */ 3057 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3058 hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID); 3059 3060 if (hostid == spa_get_hostid(spa)) 3061 return (B_FALSE); 3062 3063 /* 3064 * Skip the activity test when the pool was cleanly exported. 3065 */ 3066 if (state != POOL_STATE_ACTIVE) 3067 return (B_FALSE); 3068 3069 return (B_TRUE); 3070 } 3071 3072 /* 3073 * Nanoseconds the activity check must watch for changes on-disk. 3074 */ 3075 static uint64_t 3076 spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3077 { 3078 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 3079 uint64_t multihost_interval = MSEC2NSEC( 3080 MMP_INTERVAL_OK(zfs_multihost_interval)); 3081 uint64_t import_delay = MAX(NANOSEC, import_intervals * 3082 multihost_interval); 3083 3084 /* 3085 * Local tunables determine a minimum duration except for the case 3086 * where we know when the remote host will suspend the pool if MMP 3087 * writes do not land. 3088 * 3089 * See Big Theory comment at the top of mmp.c for the reasoning behind 3090 * these cases and times. 3091 */ 3092 3093 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 3094 3095 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3096 MMP_FAIL_INT(ub) > 0) { 3097 3098 /* MMP on remote host will suspend pool after failed writes */ 3099 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 3100 MMP_IMPORT_SAFETY_FACTOR / 100; 3101 3102 zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp " 3103 "mmp_fails=%llu ub_mmp mmp_interval=%llu " 3104 "import_intervals=%llu", (u_longlong_t)import_delay, 3105 (u_longlong_t)MMP_FAIL_INT(ub), 3106 (u_longlong_t)MMP_INTERVAL(ub), 3107 (u_longlong_t)import_intervals); 3108 3109 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3110 MMP_FAIL_INT(ub) == 0) { 3111 3112 /* MMP on remote host will never suspend pool */ 3113 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 3114 ub->ub_mmp_delay) * import_intervals); 3115 3116 zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp " 3117 "mmp_interval=%llu ub_mmp_delay=%llu " 3118 "import_intervals=%llu", (u_longlong_t)import_delay, 3119 (u_longlong_t)MMP_INTERVAL(ub), 3120 (u_longlong_t)ub->ub_mmp_delay, 3121 (u_longlong_t)import_intervals); 3122 3123 } else if (MMP_VALID(ub)) { 3124 /* 3125 * zfs-0.7 compatibility case 3126 */ 3127 3128 import_delay = MAX(import_delay, (multihost_interval + 3129 ub->ub_mmp_delay) * import_intervals); 3130 3131 zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu " 3132 "import_intervals=%llu leaves=%u", 3133 (u_longlong_t)import_delay, 3134 (u_longlong_t)ub->ub_mmp_delay, 3135 (u_longlong_t)import_intervals, 3136 vdev_count_leaves(spa)); 3137 } else { 3138 /* Using local tunings is the only reasonable option */ 3139 zfs_dbgmsg("pool last imported on non-MMP aware " 3140 "host using import_delay=%llu multihost_interval=%llu " 3141 "import_intervals=%llu", (u_longlong_t)import_delay, 3142 (u_longlong_t)multihost_interval, 3143 (u_longlong_t)import_intervals); 3144 } 3145 3146 return (import_delay); 3147 } 3148 3149 /* 3150 * Perform the import activity check. If the user canceled the import or 3151 * we detected activity then fail. 3152 */ 3153 static int 3154 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config) 3155 { 3156 uint64_t txg = ub->ub_txg; 3157 uint64_t timestamp = ub->ub_timestamp; 3158 uint64_t mmp_config = ub->ub_mmp_config; 3159 uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0; 3160 uint64_t import_delay; 3161 hrtime_t import_expire; 3162 nvlist_t *mmp_label = NULL; 3163 vdev_t *rvd = spa->spa_root_vdev; 3164 kcondvar_t cv; 3165 kmutex_t mtx; 3166 int error = 0; 3167 3168 cv_init(&cv, NULL, CV_DEFAULT, NULL); 3169 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 3170 mutex_enter(&mtx); 3171 3172 /* 3173 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3174 * during the earlier tryimport. If the txg recorded there is 0 then 3175 * the pool is known to be active on another host. 3176 * 3177 * Otherwise, the pool might be in use on another host. Check for 3178 * changes in the uberblocks on disk if necessary. 3179 */ 3180 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3181 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 3182 ZPOOL_CONFIG_LOAD_INFO); 3183 3184 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && 3185 fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { 3186 vdev_uberblock_load(rvd, ub, &mmp_label); 3187 error = SET_ERROR(EREMOTEIO); 3188 goto out; 3189 } 3190 } 3191 3192 import_delay = spa_activity_check_duration(spa, ub); 3193 3194 /* Add a small random factor in case of simultaneous imports (0-25%) */ 3195 import_delay += import_delay * random_in_range(250) / 1000; 3196 3197 import_expire = gethrtime() + import_delay; 3198 3199 while (gethrtime() < import_expire) { 3200 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 3201 NSEC2SEC(import_expire - gethrtime())); 3202 3203 vdev_uberblock_load(rvd, ub, &mmp_label); 3204 3205 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || 3206 mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) { 3207 zfs_dbgmsg("multihost activity detected " 3208 "txg %llu ub_txg %llu " 3209 "timestamp %llu ub_timestamp %llu " 3210 "mmp_config %#llx ub_mmp_config %#llx", 3211 (u_longlong_t)txg, (u_longlong_t)ub->ub_txg, 3212 (u_longlong_t)timestamp, 3213 (u_longlong_t)ub->ub_timestamp, 3214 (u_longlong_t)mmp_config, 3215 (u_longlong_t)ub->ub_mmp_config); 3216 3217 error = SET_ERROR(EREMOTEIO); 3218 break; 3219 } 3220 3221 if (mmp_label) { 3222 nvlist_free(mmp_label); 3223 mmp_label = NULL; 3224 } 3225 3226 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 3227 if (error != -1) { 3228 error = SET_ERROR(EINTR); 3229 break; 3230 } 3231 error = 0; 3232 } 3233 3234 out: 3235 mutex_exit(&mtx); 3236 mutex_destroy(&mtx); 3237 cv_destroy(&cv); 3238 3239 /* 3240 * If the pool is determined to be active store the status in the 3241 * spa->spa_load_info nvlist. If the remote hostname or hostid are 3242 * available from configuration read from disk store them as well. 3243 * This allows 'zpool import' to generate a more useful message. 3244 * 3245 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) 3246 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 3247 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 3248 */ 3249 if (error == EREMOTEIO) { 3250 char *hostname = "<unknown>"; 3251 uint64_t hostid = 0; 3252 3253 if (mmp_label) { 3254 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { 3255 hostname = fnvlist_lookup_string(mmp_label, 3256 ZPOOL_CONFIG_HOSTNAME); 3257 fnvlist_add_string(spa->spa_load_info, 3258 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 3259 } 3260 3261 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { 3262 hostid = fnvlist_lookup_uint64(mmp_label, 3263 ZPOOL_CONFIG_HOSTID); 3264 fnvlist_add_uint64(spa->spa_load_info, 3265 ZPOOL_CONFIG_MMP_HOSTID, hostid); 3266 } 3267 } 3268 3269 fnvlist_add_uint64(spa->spa_load_info, 3270 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); 3271 fnvlist_add_uint64(spa->spa_load_info, 3272 ZPOOL_CONFIG_MMP_TXG, 0); 3273 3274 error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); 3275 } 3276 3277 if (mmp_label) 3278 nvlist_free(mmp_label); 3279 3280 return (error); 3281 } 3282 3283 static int 3284 spa_verify_host(spa_t *spa, nvlist_t *mos_config) 3285 { 3286 uint64_t hostid; 3287 char *hostname; 3288 uint64_t myhostid = 0; 3289 3290 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 3291 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 3292 hostname = fnvlist_lookup_string(mos_config, 3293 ZPOOL_CONFIG_HOSTNAME); 3294 3295 myhostid = zone_get_hostid(NULL); 3296 3297 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 3298 cmn_err(CE_WARN, "pool '%s' could not be " 3299 "loaded as it was last accessed by " 3300 "another system (host: %s hostid: 0x%llx). " 3301 "See: https://openzfs.github.io/openzfs-docs/msg/" 3302 "ZFS-8000-EY", 3303 spa_name(spa), hostname, (u_longlong_t)hostid); 3304 spa_load_failed(spa, "hostid verification failed: pool " 3305 "last accessed by host: %s (hostid: 0x%llx)", 3306 hostname, (u_longlong_t)hostid); 3307 return (SET_ERROR(EBADF)); 3308 } 3309 } 3310 3311 return (0); 3312 } 3313 3314 static int 3315 spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 3316 { 3317 int error = 0; 3318 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 3319 int parse; 3320 vdev_t *rvd; 3321 uint64_t pool_guid; 3322 char *comment; 3323 char *compatibility; 3324 3325 /* 3326 * Versioning wasn't explicitly added to the label until later, so if 3327 * it's not present treat it as the initial version. 3328 */ 3329 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 3330 &spa->spa_ubsync.ub_version) != 0) 3331 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 3332 3333 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 3334 spa_load_failed(spa, "invalid config provided: '%s' missing", 3335 ZPOOL_CONFIG_POOL_GUID); 3336 return (SET_ERROR(EINVAL)); 3337 } 3338 3339 /* 3340 * If we are doing an import, ensure that the pool is not already 3341 * imported by checking if its pool guid already exists in the 3342 * spa namespace. 3343 * 3344 * The only case that we allow an already imported pool to be 3345 * imported again, is when the pool is checkpointed and we want to 3346 * look at its checkpointed state from userland tools like zdb. 3347 */ 3348 #ifdef _KERNEL 3349 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3350 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3351 spa_guid_exists(pool_guid, 0)) { 3352 #else 3353 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3354 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3355 spa_guid_exists(pool_guid, 0) && 3356 !spa_importing_readonly_checkpoint(spa)) { 3357 #endif 3358 spa_load_failed(spa, "a pool with guid %llu is already open", 3359 (u_longlong_t)pool_guid); 3360 return (SET_ERROR(EEXIST)); 3361 } 3362 3363 spa->spa_config_guid = pool_guid; 3364 3365 nvlist_free(spa->spa_load_info); 3366 spa->spa_load_info = fnvlist_alloc(); 3367 3368 ASSERT(spa->spa_comment == NULL); 3369 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 3370 spa->spa_comment = spa_strdup(comment); 3371 3372 ASSERT(spa->spa_compatibility == NULL); 3373 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 3374 &compatibility) == 0) 3375 spa->spa_compatibility = spa_strdup(compatibility); 3376 3377 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 3378 &spa->spa_config_txg); 3379 3380 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 3381 spa->spa_config_splitting = fnvlist_dup(nvl); 3382 3383 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 3384 spa_load_failed(spa, "invalid config provided: '%s' missing", 3385 ZPOOL_CONFIG_VDEV_TREE); 3386 return (SET_ERROR(EINVAL)); 3387 } 3388 3389 /* 3390 * Create "The Godfather" zio to hold all async IOs 3391 */ 3392 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 3393 KM_SLEEP); 3394 for (int i = 0; i < max_ncpus; i++) { 3395 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 3396 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3397 ZIO_FLAG_GODFATHER); 3398 } 3399 3400 /* 3401 * Parse the configuration into a vdev tree. We explicitly set the 3402 * value that will be returned by spa_version() since parsing the 3403 * configuration requires knowing the version number. 3404 */ 3405 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3406 parse = (type == SPA_IMPORT_EXISTING ? 3407 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 3408 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 3409 spa_config_exit(spa, SCL_ALL, FTAG); 3410 3411 if (error != 0) { 3412 spa_load_failed(spa, "unable to parse config [error=%d]", 3413 error); 3414 return (error); 3415 } 3416 3417 ASSERT(spa->spa_root_vdev == rvd); 3418 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 3419 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 3420 3421 if (type != SPA_IMPORT_ASSEMBLE) { 3422 ASSERT(spa_guid(spa) == pool_guid); 3423 } 3424 3425 return (0); 3426 } 3427 3428 /* 3429 * Recursively open all vdevs in the vdev tree. This function is called twice: 3430 * first with the untrusted config, then with the trusted config. 3431 */ 3432 static int 3433 spa_ld_open_vdevs(spa_t *spa) 3434 { 3435 int error = 0; 3436 3437 /* 3438 * spa_missing_tvds_allowed defines how many top-level vdevs can be 3439 * missing/unopenable for the root vdev to be still considered openable. 3440 */ 3441 if (spa->spa_trust_config) { 3442 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 3443 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 3444 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 3445 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 3446 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 3447 } else { 3448 spa->spa_missing_tvds_allowed = 0; 3449 } 3450 3451 spa->spa_missing_tvds_allowed = 3452 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 3453 3454 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3455 error = vdev_open(spa->spa_root_vdev); 3456 spa_config_exit(spa, SCL_ALL, FTAG); 3457 3458 if (spa->spa_missing_tvds != 0) { 3459 spa_load_note(spa, "vdev tree has %lld missing top-level " 3460 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 3461 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 3462 /* 3463 * Although theoretically we could allow users to open 3464 * incomplete pools in RW mode, we'd need to add a lot 3465 * of extra logic (e.g. adjust pool space to account 3466 * for missing vdevs). 3467 * This limitation also prevents users from accidentally 3468 * opening the pool in RW mode during data recovery and 3469 * damaging it further. 3470 */ 3471 spa_load_note(spa, "pools with missing top-level " 3472 "vdevs can only be opened in read-only mode."); 3473 error = SET_ERROR(ENXIO); 3474 } else { 3475 spa_load_note(spa, "current settings allow for maximum " 3476 "%lld missing top-level vdevs at this stage.", 3477 (u_longlong_t)spa->spa_missing_tvds_allowed); 3478 } 3479 } 3480 if (error != 0) { 3481 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 3482 error); 3483 } 3484 if (spa->spa_missing_tvds != 0 || error != 0) 3485 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 3486 3487 return (error); 3488 } 3489 3490 /* 3491 * We need to validate the vdev labels against the configuration that 3492 * we have in hand. This function is called twice: first with an untrusted 3493 * config, then with a trusted config. The validation is more strict when the 3494 * config is trusted. 3495 */ 3496 static int 3497 spa_ld_validate_vdevs(spa_t *spa) 3498 { 3499 int error = 0; 3500 vdev_t *rvd = spa->spa_root_vdev; 3501 3502 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3503 error = vdev_validate(rvd); 3504 spa_config_exit(spa, SCL_ALL, FTAG); 3505 3506 if (error != 0) { 3507 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 3508 return (error); 3509 } 3510 3511 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 3512 spa_load_failed(spa, "cannot open vdev tree after invalidating " 3513 "some vdevs"); 3514 vdev_dbgmsg_print_tree(rvd, 2); 3515 return (SET_ERROR(ENXIO)); 3516 } 3517 3518 return (0); 3519 } 3520 3521 static void 3522 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 3523 { 3524 spa->spa_state = POOL_STATE_ACTIVE; 3525 spa->spa_ubsync = spa->spa_uberblock; 3526 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 3527 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 3528 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 3529 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 3530 spa->spa_claim_max_txg = spa->spa_first_txg; 3531 spa->spa_prev_software_version = ub->ub_software_version; 3532 } 3533 3534 static int 3535 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 3536 { 3537 vdev_t *rvd = spa->spa_root_vdev; 3538 nvlist_t *label; 3539 uberblock_t *ub = &spa->spa_uberblock; 3540 boolean_t activity_check = B_FALSE; 3541 3542 /* 3543 * If we are opening the checkpointed state of the pool by 3544 * rewinding to it, at this point we will have written the 3545 * checkpointed uberblock to the vdev labels, so searching 3546 * the labels will find the right uberblock. However, if 3547 * we are opening the checkpointed state read-only, we have 3548 * not modified the labels. Therefore, we must ignore the 3549 * labels and continue using the spa_uberblock that was set 3550 * by spa_ld_checkpoint_rewind. 3551 * 3552 * Note that it would be fine to ignore the labels when 3553 * rewinding (opening writeable) as well. However, if we 3554 * crash just after writing the labels, we will end up 3555 * searching the labels. Doing so in the common case means 3556 * that this code path gets exercised normally, rather than 3557 * just in the edge case. 3558 */ 3559 if (ub->ub_checkpoint_txg != 0 && 3560 spa_importing_readonly_checkpoint(spa)) { 3561 spa_ld_select_uberblock_done(spa, ub); 3562 return (0); 3563 } 3564 3565 /* 3566 * Find the best uberblock. 3567 */ 3568 vdev_uberblock_load(rvd, ub, &label); 3569 3570 /* 3571 * If we weren't able to find a single valid uberblock, return failure. 3572 */ 3573 if (ub->ub_txg == 0) { 3574 nvlist_free(label); 3575 spa_load_failed(spa, "no valid uberblock found"); 3576 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 3577 } 3578 3579 if (spa->spa_load_max_txg != UINT64_MAX) { 3580 (void) spa_import_progress_set_max_txg(spa_guid(spa), 3581 (u_longlong_t)spa->spa_load_max_txg); 3582 } 3583 spa_load_note(spa, "using uberblock with txg=%llu", 3584 (u_longlong_t)ub->ub_txg); 3585 3586 3587 /* 3588 * For pools which have the multihost property on determine if the 3589 * pool is truly inactive and can be safely imported. Prevent 3590 * hosts which don't have a hostid set from importing the pool. 3591 */ 3592 activity_check = spa_activity_check_required(spa, ub, label, 3593 spa->spa_config); 3594 if (activity_check) { 3595 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 3596 spa_get_hostid(spa) == 0) { 3597 nvlist_free(label); 3598 fnvlist_add_uint64(spa->spa_load_info, 3599 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 3600 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 3601 } 3602 3603 int error = spa_activity_check(spa, ub, spa->spa_config); 3604 if (error) { 3605 nvlist_free(label); 3606 return (error); 3607 } 3608 3609 fnvlist_add_uint64(spa->spa_load_info, 3610 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); 3611 fnvlist_add_uint64(spa->spa_load_info, 3612 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); 3613 fnvlist_add_uint16(spa->spa_load_info, 3614 ZPOOL_CONFIG_MMP_SEQ, 3615 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)); 3616 } 3617 3618 /* 3619 * If the pool has an unsupported version we can't open it. 3620 */ 3621 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 3622 nvlist_free(label); 3623 spa_load_failed(spa, "version %llu is not supported", 3624 (u_longlong_t)ub->ub_version); 3625 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 3626 } 3627 3628 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3629 nvlist_t *features; 3630 3631 /* 3632 * If we weren't able to find what's necessary for reading the 3633 * MOS in the label, return failure. 3634 */ 3635 if (label == NULL) { 3636 spa_load_failed(spa, "label config unavailable"); 3637 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3638 ENXIO)); 3639 } 3640 3641 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 3642 &features) != 0) { 3643 nvlist_free(label); 3644 spa_load_failed(spa, "invalid label: '%s' missing", 3645 ZPOOL_CONFIG_FEATURES_FOR_READ); 3646 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3647 ENXIO)); 3648 } 3649 3650 /* 3651 * Update our in-core representation with the definitive values 3652 * from the label. 3653 */ 3654 nvlist_free(spa->spa_label_features); 3655 spa->spa_label_features = fnvlist_dup(features); 3656 } 3657 3658 nvlist_free(label); 3659 3660 /* 3661 * Look through entries in the label nvlist's features_for_read. If 3662 * there is a feature listed there which we don't understand then we 3663 * cannot open a pool. 3664 */ 3665 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3666 nvlist_t *unsup_feat; 3667 3668 unsup_feat = fnvlist_alloc(); 3669 3670 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 3671 NULL); nvp != NULL; 3672 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 3673 if (!zfeature_is_supported(nvpair_name(nvp))) { 3674 fnvlist_add_string(unsup_feat, 3675 nvpair_name(nvp), ""); 3676 } 3677 } 3678 3679 if (!nvlist_empty(unsup_feat)) { 3680 fnvlist_add_nvlist(spa->spa_load_info, 3681 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 3682 nvlist_free(unsup_feat); 3683 spa_load_failed(spa, "some features are unsupported"); 3684 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 3685 ENOTSUP)); 3686 } 3687 3688 nvlist_free(unsup_feat); 3689 } 3690 3691 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 3692 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3693 spa_try_repair(spa, spa->spa_config); 3694 spa_config_exit(spa, SCL_ALL, FTAG); 3695 nvlist_free(spa->spa_config_splitting); 3696 spa->spa_config_splitting = NULL; 3697 } 3698 3699 /* 3700 * Initialize internal SPA structures. 3701 */ 3702 spa_ld_select_uberblock_done(spa, ub); 3703 3704 return (0); 3705 } 3706 3707 static int 3708 spa_ld_open_rootbp(spa_t *spa) 3709 { 3710 int error = 0; 3711 vdev_t *rvd = spa->spa_root_vdev; 3712 3713 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 3714 if (error != 0) { 3715 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 3716 "[error=%d]", error); 3717 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3718 } 3719 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 3720 3721 return (0); 3722 } 3723 3724 static int 3725 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 3726 boolean_t reloading) 3727 { 3728 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 3729 nvlist_t *nv, *mos_config, *policy; 3730 int error = 0, copy_error; 3731 uint64_t healthy_tvds, healthy_tvds_mos; 3732 uint64_t mos_config_txg; 3733 3734 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 3735 != 0) 3736 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3737 3738 /* 3739 * If we're assembling a pool from a split, the config provided is 3740 * already trusted so there is nothing to do. 3741 */ 3742 if (type == SPA_IMPORT_ASSEMBLE) 3743 return (0); 3744 3745 healthy_tvds = spa_healthy_core_tvds(spa); 3746 3747 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 3748 != 0) { 3749 spa_load_failed(spa, "unable to retrieve MOS config"); 3750 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3751 } 3752 3753 /* 3754 * If we are doing an open, pool owner wasn't verified yet, thus do 3755 * the verification here. 3756 */ 3757 if (spa->spa_load_state == SPA_LOAD_OPEN) { 3758 error = spa_verify_host(spa, mos_config); 3759 if (error != 0) { 3760 nvlist_free(mos_config); 3761 return (error); 3762 } 3763 } 3764 3765 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 3766 3767 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3768 3769 /* 3770 * Build a new vdev tree from the trusted config 3771 */ 3772 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 3773 if (error != 0) { 3774 nvlist_free(mos_config); 3775 spa_config_exit(spa, SCL_ALL, FTAG); 3776 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 3777 error); 3778 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3779 } 3780 3781 /* 3782 * Vdev paths in the MOS may be obsolete. If the untrusted config was 3783 * obtained by scanning /dev/dsk, then it will have the right vdev 3784 * paths. We update the trusted MOS config with this information. 3785 * We first try to copy the paths with vdev_copy_path_strict, which 3786 * succeeds only when both configs have exactly the same vdev tree. 3787 * If that fails, we fall back to a more flexible method that has a 3788 * best effort policy. 3789 */ 3790 copy_error = vdev_copy_path_strict(rvd, mrvd); 3791 if (copy_error != 0 || spa_load_print_vdev_tree) { 3792 spa_load_note(spa, "provided vdev tree:"); 3793 vdev_dbgmsg_print_tree(rvd, 2); 3794 spa_load_note(spa, "MOS vdev tree:"); 3795 vdev_dbgmsg_print_tree(mrvd, 2); 3796 } 3797 if (copy_error != 0) { 3798 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 3799 "back to vdev_copy_path_relaxed"); 3800 vdev_copy_path_relaxed(rvd, mrvd); 3801 } 3802 3803 vdev_close(rvd); 3804 vdev_free(rvd); 3805 spa->spa_root_vdev = mrvd; 3806 rvd = mrvd; 3807 spa_config_exit(spa, SCL_ALL, FTAG); 3808 3809 /* 3810 * We will use spa_config if we decide to reload the spa or if spa_load 3811 * fails and we rewind. We must thus regenerate the config using the 3812 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 3813 * pass settings on how to load the pool and is not stored in the MOS. 3814 * We copy it over to our new, trusted config. 3815 */ 3816 mos_config_txg = fnvlist_lookup_uint64(mos_config, 3817 ZPOOL_CONFIG_POOL_TXG); 3818 nvlist_free(mos_config); 3819 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 3820 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 3821 &policy) == 0) 3822 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 3823 spa_config_set(spa, mos_config); 3824 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 3825 3826 /* 3827 * Now that we got the config from the MOS, we should be more strict 3828 * in checking blkptrs and can make assumptions about the consistency 3829 * of the vdev tree. spa_trust_config must be set to true before opening 3830 * vdevs in order for them to be writeable. 3831 */ 3832 spa->spa_trust_config = B_TRUE; 3833 3834 /* 3835 * Open and validate the new vdev tree 3836 */ 3837 error = spa_ld_open_vdevs(spa); 3838 if (error != 0) 3839 return (error); 3840 3841 error = spa_ld_validate_vdevs(spa); 3842 if (error != 0) 3843 return (error); 3844 3845 if (copy_error != 0 || spa_load_print_vdev_tree) { 3846 spa_load_note(spa, "final vdev tree:"); 3847 vdev_dbgmsg_print_tree(rvd, 2); 3848 } 3849 3850 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 3851 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 3852 /* 3853 * Sanity check to make sure that we are indeed loading the 3854 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 3855 * in the config provided and they happened to be the only ones 3856 * to have the latest uberblock, we could involuntarily perform 3857 * an extreme rewind. 3858 */ 3859 healthy_tvds_mos = spa_healthy_core_tvds(spa); 3860 if (healthy_tvds_mos - healthy_tvds >= 3861 SPA_SYNC_MIN_VDEVS) { 3862 spa_load_note(spa, "config provided misses too many " 3863 "top-level vdevs compared to MOS (%lld vs %lld). ", 3864 (u_longlong_t)healthy_tvds, 3865 (u_longlong_t)healthy_tvds_mos); 3866 spa_load_note(spa, "vdev tree:"); 3867 vdev_dbgmsg_print_tree(rvd, 2); 3868 if (reloading) { 3869 spa_load_failed(spa, "config was already " 3870 "provided from MOS. Aborting."); 3871 return (spa_vdev_err(rvd, 3872 VDEV_AUX_CORRUPT_DATA, EIO)); 3873 } 3874 spa_load_note(spa, "spa must be reloaded using MOS " 3875 "config"); 3876 return (SET_ERROR(EAGAIN)); 3877 } 3878 } 3879 3880 error = spa_check_for_missing_logs(spa); 3881 if (error != 0) 3882 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 3883 3884 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 3885 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 3886 "guid sum (%llu != %llu)", 3887 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 3888 (u_longlong_t)rvd->vdev_guid_sum); 3889 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 3890 ENXIO)); 3891 } 3892 3893 return (0); 3894 } 3895 3896 static int 3897 spa_ld_open_indirect_vdev_metadata(spa_t *spa) 3898 { 3899 int error = 0; 3900 vdev_t *rvd = spa->spa_root_vdev; 3901 3902 /* 3903 * Everything that we read before spa_remove_init() must be stored 3904 * on concreted vdevs. Therefore we do this as early as possible. 3905 */ 3906 error = spa_remove_init(spa); 3907 if (error != 0) { 3908 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 3909 error); 3910 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3911 } 3912 3913 /* 3914 * Retrieve information needed to condense indirect vdev mappings. 3915 */ 3916 error = spa_condense_init(spa); 3917 if (error != 0) { 3918 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 3919 error); 3920 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3921 } 3922 3923 return (0); 3924 } 3925 3926 static int 3927 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 3928 { 3929 int error = 0; 3930 vdev_t *rvd = spa->spa_root_vdev; 3931 3932 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 3933 boolean_t missing_feat_read = B_FALSE; 3934 nvlist_t *unsup_feat, *enabled_feat; 3935 3936 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 3937 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 3938 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3939 } 3940 3941 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 3942 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 3943 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3944 } 3945 3946 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 3947 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 3948 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3949 } 3950 3951 enabled_feat = fnvlist_alloc(); 3952 unsup_feat = fnvlist_alloc(); 3953 3954 if (!spa_features_check(spa, B_FALSE, 3955 unsup_feat, enabled_feat)) 3956 missing_feat_read = B_TRUE; 3957 3958 if (spa_writeable(spa) || 3959 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 3960 if (!spa_features_check(spa, B_TRUE, 3961 unsup_feat, enabled_feat)) { 3962 *missing_feat_writep = B_TRUE; 3963 } 3964 } 3965 3966 fnvlist_add_nvlist(spa->spa_load_info, 3967 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 3968 3969 if (!nvlist_empty(unsup_feat)) { 3970 fnvlist_add_nvlist(spa->spa_load_info, 3971 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 3972 } 3973 3974 fnvlist_free(enabled_feat); 3975 fnvlist_free(unsup_feat); 3976 3977 if (!missing_feat_read) { 3978 fnvlist_add_boolean(spa->spa_load_info, 3979 ZPOOL_CONFIG_CAN_RDONLY); 3980 } 3981 3982 /* 3983 * If the state is SPA_LOAD_TRYIMPORT, our objective is 3984 * twofold: to determine whether the pool is available for 3985 * import in read-write mode and (if it is not) whether the 3986 * pool is available for import in read-only mode. If the pool 3987 * is available for import in read-write mode, it is displayed 3988 * as available in userland; if it is not available for import 3989 * in read-only mode, it is displayed as unavailable in 3990 * userland. If the pool is available for import in read-only 3991 * mode but not read-write mode, it is displayed as unavailable 3992 * in userland with a special note that the pool is actually 3993 * available for open in read-only mode. 3994 * 3995 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 3996 * missing a feature for write, we must first determine whether 3997 * the pool can be opened read-only before returning to 3998 * userland in order to know whether to display the 3999 * abovementioned note. 4000 */ 4001 if (missing_feat_read || (*missing_feat_writep && 4002 spa_writeable(spa))) { 4003 spa_load_failed(spa, "pool uses unsupported features"); 4004 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 4005 ENOTSUP)); 4006 } 4007 4008 /* 4009 * Load refcounts for ZFS features from disk into an in-memory 4010 * cache during SPA initialization. 4011 */ 4012 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 4013 uint64_t refcount; 4014 4015 error = feature_get_refcount_from_disk(spa, 4016 &spa_feature_table[i], &refcount); 4017 if (error == 0) { 4018 spa->spa_feat_refcount_cache[i] = refcount; 4019 } else if (error == ENOTSUP) { 4020 spa->spa_feat_refcount_cache[i] = 4021 SPA_FEATURE_DISABLED; 4022 } else { 4023 spa_load_failed(spa, "error getting refcount " 4024 "for feature %s [error=%d]", 4025 spa_feature_table[i].fi_guid, error); 4026 return (spa_vdev_err(rvd, 4027 VDEV_AUX_CORRUPT_DATA, EIO)); 4028 } 4029 } 4030 } 4031 4032 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 4033 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 4034 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 4035 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4036 } 4037 4038 /* 4039 * Encryption was added before bookmark_v2, even though bookmark_v2 4040 * is now a dependency. If this pool has encryption enabled without 4041 * bookmark_v2, trigger an errata message. 4042 */ 4043 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 4044 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 4045 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 4046 } 4047 4048 return (0); 4049 } 4050 4051 static int 4052 spa_ld_load_special_directories(spa_t *spa) 4053 { 4054 int error = 0; 4055 vdev_t *rvd = spa->spa_root_vdev; 4056 4057 spa->spa_is_initializing = B_TRUE; 4058 error = dsl_pool_open(spa->spa_dsl_pool); 4059 spa->spa_is_initializing = B_FALSE; 4060 if (error != 0) { 4061 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 4062 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4063 } 4064 4065 return (0); 4066 } 4067 4068 static int 4069 spa_ld_get_props(spa_t *spa) 4070 { 4071 int error = 0; 4072 uint64_t obj; 4073 vdev_t *rvd = spa->spa_root_vdev; 4074 4075 /* Grab the checksum salt from the MOS. */ 4076 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4077 DMU_POOL_CHECKSUM_SALT, 1, 4078 sizeof (spa->spa_cksum_salt.zcs_bytes), 4079 spa->spa_cksum_salt.zcs_bytes); 4080 if (error == ENOENT) { 4081 /* Generate a new salt for subsequent use */ 4082 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 4083 sizeof (spa->spa_cksum_salt.zcs_bytes)); 4084 } else if (error != 0) { 4085 spa_load_failed(spa, "unable to retrieve checksum salt from " 4086 "MOS [error=%d]", error); 4087 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4088 } 4089 4090 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 4091 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4092 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 4093 if (error != 0) { 4094 spa_load_failed(spa, "error opening deferred-frees bpobj " 4095 "[error=%d]", error); 4096 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4097 } 4098 4099 /* 4100 * Load the bit that tells us to use the new accounting function 4101 * (raid-z deflation). If we have an older pool, this will not 4102 * be present. 4103 */ 4104 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 4105 if (error != 0 && error != ENOENT) 4106 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4107 4108 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 4109 &spa->spa_creation_version, B_FALSE); 4110 if (error != 0 && error != ENOENT) 4111 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4112 4113 /* 4114 * Load the persistent error log. If we have an older pool, this will 4115 * not be present. 4116 */ 4117 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 4118 B_FALSE); 4119 if (error != 0 && error != ENOENT) 4120 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4121 4122 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 4123 &spa->spa_errlog_scrub, B_FALSE); 4124 if (error != 0 && error != ENOENT) 4125 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4126 4127 /* 4128 * Load the livelist deletion field. If a livelist is queued for 4129 * deletion, indicate that in the spa 4130 */ 4131 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 4132 &spa->spa_livelists_to_delete, B_FALSE); 4133 if (error != 0 && error != ENOENT) 4134 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4135 4136 /* 4137 * Load the history object. If we have an older pool, this 4138 * will not be present. 4139 */ 4140 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 4141 if (error != 0 && error != ENOENT) 4142 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4143 4144 /* 4145 * Load the per-vdev ZAP map. If we have an older pool, this will not 4146 * be present; in this case, defer its creation to a later time to 4147 * avoid dirtying the MOS this early / out of sync context. See 4148 * spa_sync_config_object. 4149 */ 4150 4151 /* The sentinel is only available in the MOS config. */ 4152 nvlist_t *mos_config; 4153 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 4154 spa_load_failed(spa, "unable to retrieve MOS config"); 4155 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4156 } 4157 4158 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 4159 &spa->spa_all_vdev_zaps, B_FALSE); 4160 4161 if (error == ENOENT) { 4162 VERIFY(!nvlist_exists(mos_config, 4163 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 4164 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 4165 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4166 } else if (error != 0) { 4167 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4168 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 4169 /* 4170 * An older version of ZFS overwrote the sentinel value, so 4171 * we have orphaned per-vdev ZAPs in the MOS. Defer their 4172 * destruction to later; see spa_sync_config_object. 4173 */ 4174 spa->spa_avz_action = AVZ_ACTION_DESTROY; 4175 /* 4176 * We're assuming that no vdevs have had their ZAPs created 4177 * before this. Better be sure of it. 4178 */ 4179 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4180 } 4181 nvlist_free(mos_config); 4182 4183 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 4184 4185 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 4186 B_FALSE); 4187 if (error && error != ENOENT) 4188 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4189 4190 if (error == 0) { 4191 uint64_t autoreplace = 0; 4192 4193 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 4194 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 4195 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 4196 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 4197 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 4198 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 4199 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 4200 spa->spa_autoreplace = (autoreplace != 0); 4201 } 4202 4203 /* 4204 * If we are importing a pool with missing top-level vdevs, 4205 * we enforce that the pool doesn't panic or get suspended on 4206 * error since the likelihood of missing data is extremely high. 4207 */ 4208 if (spa->spa_missing_tvds > 0 && 4209 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 4210 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4211 spa_load_note(spa, "forcing failmode to 'continue' " 4212 "as some top level vdevs are missing"); 4213 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 4214 } 4215 4216 return (0); 4217 } 4218 4219 static int 4220 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 4221 { 4222 int error = 0; 4223 vdev_t *rvd = spa->spa_root_vdev; 4224 4225 /* 4226 * If we're assembling the pool from the split-off vdevs of 4227 * an existing pool, we don't want to attach the spares & cache 4228 * devices. 4229 */ 4230 4231 /* 4232 * Load any hot spares for this pool. 4233 */ 4234 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 4235 B_FALSE); 4236 if (error != 0 && error != ENOENT) 4237 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4238 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4239 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 4240 if (load_nvlist(spa, spa->spa_spares.sav_object, 4241 &spa->spa_spares.sav_config) != 0) { 4242 spa_load_failed(spa, "error loading spares nvlist"); 4243 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4244 } 4245 4246 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4247 spa_load_spares(spa); 4248 spa_config_exit(spa, SCL_ALL, FTAG); 4249 } else if (error == 0) { 4250 spa->spa_spares.sav_sync = B_TRUE; 4251 } 4252 4253 /* 4254 * Load any level 2 ARC devices for this pool. 4255 */ 4256 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 4257 &spa->spa_l2cache.sav_object, B_FALSE); 4258 if (error != 0 && error != ENOENT) 4259 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4260 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4261 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 4262 if (load_nvlist(spa, spa->spa_l2cache.sav_object, 4263 &spa->spa_l2cache.sav_config) != 0) { 4264 spa_load_failed(spa, "error loading l2cache nvlist"); 4265 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4266 } 4267 4268 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4269 spa_load_l2cache(spa); 4270 spa_config_exit(spa, SCL_ALL, FTAG); 4271 } else if (error == 0) { 4272 spa->spa_l2cache.sav_sync = B_TRUE; 4273 } 4274 4275 return (0); 4276 } 4277 4278 static int 4279 spa_ld_load_vdev_metadata(spa_t *spa) 4280 { 4281 int error = 0; 4282 vdev_t *rvd = spa->spa_root_vdev; 4283 4284 /* 4285 * If the 'multihost' property is set, then never allow a pool to 4286 * be imported when the system hostid is zero. The exception to 4287 * this rule is zdb which is always allowed to access pools. 4288 */ 4289 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 4290 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 4291 fnvlist_add_uint64(spa->spa_load_info, 4292 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4293 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4294 } 4295 4296 /* 4297 * If the 'autoreplace' property is set, then post a resource notifying 4298 * the ZFS DE that it should not issue any faults for unopenable 4299 * devices. We also iterate over the vdevs, and post a sysevent for any 4300 * unopenable vdevs so that the normal autoreplace handler can take 4301 * over. 4302 */ 4303 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4304 spa_check_removed(spa->spa_root_vdev); 4305 /* 4306 * For the import case, this is done in spa_import(), because 4307 * at this point we're using the spare definitions from 4308 * the MOS config, not necessarily from the userland config. 4309 */ 4310 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 4311 spa_aux_check_removed(&spa->spa_spares); 4312 spa_aux_check_removed(&spa->spa_l2cache); 4313 } 4314 } 4315 4316 /* 4317 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 4318 */ 4319 error = vdev_load(rvd); 4320 if (error != 0) { 4321 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 4322 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4323 } 4324 4325 error = spa_ld_log_spacemaps(spa); 4326 if (error != 0) { 4327 spa_load_failed(spa, "spa_ld_log_sm_data failed [error=%d]", 4328 error); 4329 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4330 } 4331 4332 /* 4333 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 4334 */ 4335 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4336 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 4337 spa_config_exit(spa, SCL_ALL, FTAG); 4338 4339 return (0); 4340 } 4341 4342 static int 4343 spa_ld_load_dedup_tables(spa_t *spa) 4344 { 4345 int error = 0; 4346 vdev_t *rvd = spa->spa_root_vdev; 4347 4348 error = ddt_load(spa); 4349 if (error != 0) { 4350 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 4351 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4352 } 4353 4354 return (0); 4355 } 4356 4357 static int 4358 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, char **ereport) 4359 { 4360 vdev_t *rvd = spa->spa_root_vdev; 4361 4362 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 4363 boolean_t missing = spa_check_logs(spa); 4364 if (missing) { 4365 if (spa->spa_missing_tvds != 0) { 4366 spa_load_note(spa, "spa_check_logs failed " 4367 "so dropping the logs"); 4368 } else { 4369 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 4370 spa_load_failed(spa, "spa_check_logs failed"); 4371 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 4372 ENXIO)); 4373 } 4374 } 4375 } 4376 4377 return (0); 4378 } 4379 4380 static int 4381 spa_ld_verify_pool_data(spa_t *spa) 4382 { 4383 int error = 0; 4384 vdev_t *rvd = spa->spa_root_vdev; 4385 4386 /* 4387 * We've successfully opened the pool, verify that we're ready 4388 * to start pushing transactions. 4389 */ 4390 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4391 error = spa_load_verify(spa); 4392 if (error != 0) { 4393 spa_load_failed(spa, "spa_load_verify failed " 4394 "[error=%d]", error); 4395 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4396 error)); 4397 } 4398 } 4399 4400 return (0); 4401 } 4402 4403 static void 4404 spa_ld_claim_log_blocks(spa_t *spa) 4405 { 4406 dmu_tx_t *tx; 4407 dsl_pool_t *dp = spa_get_dsl(spa); 4408 4409 /* 4410 * Claim log blocks that haven't been committed yet. 4411 * This must all happen in a single txg. 4412 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 4413 * invoked from zil_claim_log_block()'s i/o done callback. 4414 * Price of rollback is that we abandon the log. 4415 */ 4416 spa->spa_claiming = B_TRUE; 4417 4418 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 4419 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 4420 zil_claim, tx, DS_FIND_CHILDREN); 4421 dmu_tx_commit(tx); 4422 4423 spa->spa_claiming = B_FALSE; 4424 4425 spa_set_log_state(spa, SPA_LOG_GOOD); 4426 } 4427 4428 static void 4429 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 4430 boolean_t update_config_cache) 4431 { 4432 vdev_t *rvd = spa->spa_root_vdev; 4433 int need_update = B_FALSE; 4434 4435 /* 4436 * If the config cache is stale, or we have uninitialized 4437 * metaslabs (see spa_vdev_add()), then update the config. 4438 * 4439 * If this is a verbatim import, trust the current 4440 * in-core spa_config and update the disk labels. 4441 */ 4442 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 4443 spa->spa_load_state == SPA_LOAD_IMPORT || 4444 spa->spa_load_state == SPA_LOAD_RECOVER || 4445 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 4446 need_update = B_TRUE; 4447 4448 for (int c = 0; c < rvd->vdev_children; c++) 4449 if (rvd->vdev_child[c]->vdev_ms_array == 0) 4450 need_update = B_TRUE; 4451 4452 /* 4453 * Update the config cache asynchronously in case we're the 4454 * root pool, in which case the config cache isn't writable yet. 4455 */ 4456 if (need_update) 4457 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 4458 } 4459 4460 static void 4461 spa_ld_prepare_for_reload(spa_t *spa) 4462 { 4463 spa_mode_t mode = spa->spa_mode; 4464 int async_suspended = spa->spa_async_suspended; 4465 4466 spa_unload(spa); 4467 spa_deactivate(spa); 4468 spa_activate(spa, mode); 4469 4470 /* 4471 * We save the value of spa_async_suspended as it gets reset to 0 by 4472 * spa_unload(). We want to restore it back to the original value before 4473 * returning as we might be calling spa_async_resume() later. 4474 */ 4475 spa->spa_async_suspended = async_suspended; 4476 } 4477 4478 static int 4479 spa_ld_read_checkpoint_txg(spa_t *spa) 4480 { 4481 uberblock_t checkpoint; 4482 int error = 0; 4483 4484 ASSERT0(spa->spa_checkpoint_txg); 4485 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4486 4487 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4488 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4489 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4490 4491 if (error == ENOENT) 4492 return (0); 4493 4494 if (error != 0) 4495 return (error); 4496 4497 ASSERT3U(checkpoint.ub_txg, !=, 0); 4498 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 4499 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 4500 spa->spa_checkpoint_txg = checkpoint.ub_txg; 4501 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 4502 4503 return (0); 4504 } 4505 4506 static int 4507 spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 4508 { 4509 int error = 0; 4510 4511 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4512 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4513 4514 /* 4515 * Never trust the config that is provided unless we are assembling 4516 * a pool following a split. 4517 * This means don't trust blkptrs and the vdev tree in general. This 4518 * also effectively puts the spa in read-only mode since 4519 * spa_writeable() checks for spa_trust_config to be true. 4520 * We will later load a trusted config from the MOS. 4521 */ 4522 if (type != SPA_IMPORT_ASSEMBLE) 4523 spa->spa_trust_config = B_FALSE; 4524 4525 /* 4526 * Parse the config provided to create a vdev tree. 4527 */ 4528 error = spa_ld_parse_config(spa, type); 4529 if (error != 0) 4530 return (error); 4531 4532 spa_import_progress_add(spa); 4533 4534 /* 4535 * Now that we have the vdev tree, try to open each vdev. This involves 4536 * opening the underlying physical device, retrieving its geometry and 4537 * probing the vdev with a dummy I/O. The state of each vdev will be set 4538 * based on the success of those operations. After this we'll be ready 4539 * to read from the vdevs. 4540 */ 4541 error = spa_ld_open_vdevs(spa); 4542 if (error != 0) 4543 return (error); 4544 4545 /* 4546 * Read the label of each vdev and make sure that the GUIDs stored 4547 * there match the GUIDs in the config provided. 4548 * If we're assembling a new pool that's been split off from an 4549 * existing pool, the labels haven't yet been updated so we skip 4550 * validation for now. 4551 */ 4552 if (type != SPA_IMPORT_ASSEMBLE) { 4553 error = spa_ld_validate_vdevs(spa); 4554 if (error != 0) 4555 return (error); 4556 } 4557 4558 /* 4559 * Read all vdev labels to find the best uberblock (i.e. latest, 4560 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 4561 * get the list of features required to read blkptrs in the MOS from 4562 * the vdev label with the best uberblock and verify that our version 4563 * of zfs supports them all. 4564 */ 4565 error = spa_ld_select_uberblock(spa, type); 4566 if (error != 0) 4567 return (error); 4568 4569 /* 4570 * Pass that uberblock to the dsl_pool layer which will open the root 4571 * blkptr. This blkptr points to the latest version of the MOS and will 4572 * allow us to read its contents. 4573 */ 4574 error = spa_ld_open_rootbp(spa); 4575 if (error != 0) 4576 return (error); 4577 4578 return (0); 4579 } 4580 4581 static int 4582 spa_ld_checkpoint_rewind(spa_t *spa) 4583 { 4584 uberblock_t checkpoint; 4585 int error = 0; 4586 4587 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4588 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4589 4590 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4591 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4592 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4593 4594 if (error != 0) { 4595 spa_load_failed(spa, "unable to retrieve checkpointed " 4596 "uberblock from the MOS config [error=%d]", error); 4597 4598 if (error == ENOENT) 4599 error = ZFS_ERR_NO_CHECKPOINT; 4600 4601 return (error); 4602 } 4603 4604 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 4605 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 4606 4607 /* 4608 * We need to update the txg and timestamp of the checkpointed 4609 * uberblock to be higher than the latest one. This ensures that 4610 * the checkpointed uberblock is selected if we were to close and 4611 * reopen the pool right after we've written it in the vdev labels. 4612 * (also see block comment in vdev_uberblock_compare) 4613 */ 4614 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 4615 checkpoint.ub_timestamp = gethrestime_sec(); 4616 4617 /* 4618 * Set current uberblock to be the checkpointed uberblock. 4619 */ 4620 spa->spa_uberblock = checkpoint; 4621 4622 /* 4623 * If we are doing a normal rewind, then the pool is open for 4624 * writing and we sync the "updated" checkpointed uberblock to 4625 * disk. Once this is done, we've basically rewound the whole 4626 * pool and there is no way back. 4627 * 4628 * There are cases when we don't want to attempt and sync the 4629 * checkpointed uberblock to disk because we are opening a 4630 * pool as read-only. Specifically, verifying the checkpointed 4631 * state with zdb, and importing the checkpointed state to get 4632 * a "preview" of its content. 4633 */ 4634 if (spa_writeable(spa)) { 4635 vdev_t *rvd = spa->spa_root_vdev; 4636 4637 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4638 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 4639 int svdcount = 0; 4640 int children = rvd->vdev_children; 4641 int c0 = random_in_range(children); 4642 4643 for (int c = 0; c < children; c++) { 4644 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 4645 4646 /* Stop when revisiting the first vdev */ 4647 if (c > 0 && svd[0] == vd) 4648 break; 4649 4650 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 4651 !vdev_is_concrete(vd)) 4652 continue; 4653 4654 svd[svdcount++] = vd; 4655 if (svdcount == SPA_SYNC_MIN_VDEVS) 4656 break; 4657 } 4658 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg); 4659 if (error == 0) 4660 spa->spa_last_synced_guid = rvd->vdev_guid; 4661 spa_config_exit(spa, SCL_ALL, FTAG); 4662 4663 if (error != 0) { 4664 spa_load_failed(spa, "failed to write checkpointed " 4665 "uberblock to the vdev labels [error=%d]", error); 4666 return (error); 4667 } 4668 } 4669 4670 return (0); 4671 } 4672 4673 static int 4674 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 4675 boolean_t *update_config_cache) 4676 { 4677 int error; 4678 4679 /* 4680 * Parse the config for pool, open and validate vdevs, 4681 * select an uberblock, and use that uberblock to open 4682 * the MOS. 4683 */ 4684 error = spa_ld_mos_init(spa, type); 4685 if (error != 0) 4686 return (error); 4687 4688 /* 4689 * Retrieve the trusted config stored in the MOS and use it to create 4690 * a new, exact version of the vdev tree, then reopen all vdevs. 4691 */ 4692 error = spa_ld_trusted_config(spa, type, B_FALSE); 4693 if (error == EAGAIN) { 4694 if (update_config_cache != NULL) 4695 *update_config_cache = B_TRUE; 4696 4697 /* 4698 * Redo the loading process with the trusted config if it is 4699 * too different from the untrusted config. 4700 */ 4701 spa_ld_prepare_for_reload(spa); 4702 spa_load_note(spa, "RELOADING"); 4703 error = spa_ld_mos_init(spa, type); 4704 if (error != 0) 4705 return (error); 4706 4707 error = spa_ld_trusted_config(spa, type, B_TRUE); 4708 if (error != 0) 4709 return (error); 4710 4711 } else if (error != 0) { 4712 return (error); 4713 } 4714 4715 return (0); 4716 } 4717 4718 /* 4719 * Load an existing storage pool, using the config provided. This config 4720 * describes which vdevs are part of the pool and is later validated against 4721 * partial configs present in each vdev's label and an entire copy of the 4722 * config stored in the MOS. 4723 */ 4724 static int 4725 spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport) 4726 { 4727 int error = 0; 4728 boolean_t missing_feat_write = B_FALSE; 4729 boolean_t checkpoint_rewind = 4730 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4731 boolean_t update_config_cache = B_FALSE; 4732 4733 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4734 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4735 4736 spa_load_note(spa, "LOADING"); 4737 4738 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 4739 if (error != 0) 4740 return (error); 4741 4742 /* 4743 * If we are rewinding to the checkpoint then we need to repeat 4744 * everything we've done so far in this function but this time 4745 * selecting the checkpointed uberblock and using that to open 4746 * the MOS. 4747 */ 4748 if (checkpoint_rewind) { 4749 /* 4750 * If we are rewinding to the checkpoint update config cache 4751 * anyway. 4752 */ 4753 update_config_cache = B_TRUE; 4754 4755 /* 4756 * Extract the checkpointed uberblock from the current MOS 4757 * and use this as the pool's uberblock from now on. If the 4758 * pool is imported as writeable we also write the checkpoint 4759 * uberblock to the labels, making the rewind permanent. 4760 */ 4761 error = spa_ld_checkpoint_rewind(spa); 4762 if (error != 0) 4763 return (error); 4764 4765 /* 4766 * Redo the loading process again with the 4767 * checkpointed uberblock. 4768 */ 4769 spa_ld_prepare_for_reload(spa); 4770 spa_load_note(spa, "LOADING checkpointed uberblock"); 4771 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 4772 if (error != 0) 4773 return (error); 4774 } 4775 4776 /* 4777 * Retrieve the checkpoint txg if the pool has a checkpoint. 4778 */ 4779 error = spa_ld_read_checkpoint_txg(spa); 4780 if (error != 0) 4781 return (error); 4782 4783 /* 4784 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 4785 * from the pool and their contents were re-mapped to other vdevs. Note 4786 * that everything that we read before this step must have been 4787 * rewritten on concrete vdevs after the last device removal was 4788 * initiated. Otherwise we could be reading from indirect vdevs before 4789 * we have loaded their mappings. 4790 */ 4791 error = spa_ld_open_indirect_vdev_metadata(spa); 4792 if (error != 0) 4793 return (error); 4794 4795 /* 4796 * Retrieve the full list of active features from the MOS and check if 4797 * they are all supported. 4798 */ 4799 error = spa_ld_check_features(spa, &missing_feat_write); 4800 if (error != 0) 4801 return (error); 4802 4803 /* 4804 * Load several special directories from the MOS needed by the dsl_pool 4805 * layer. 4806 */ 4807 error = spa_ld_load_special_directories(spa); 4808 if (error != 0) 4809 return (error); 4810 4811 /* 4812 * Retrieve pool properties from the MOS. 4813 */ 4814 error = spa_ld_get_props(spa); 4815 if (error != 0) 4816 return (error); 4817 4818 /* 4819 * Retrieve the list of auxiliary devices - cache devices and spares - 4820 * and open them. 4821 */ 4822 error = spa_ld_open_aux_vdevs(spa, type); 4823 if (error != 0) 4824 return (error); 4825 4826 /* 4827 * Load the metadata for all vdevs. Also check if unopenable devices 4828 * should be autoreplaced. 4829 */ 4830 error = spa_ld_load_vdev_metadata(spa); 4831 if (error != 0) 4832 return (error); 4833 4834 error = spa_ld_load_dedup_tables(spa); 4835 if (error != 0) 4836 return (error); 4837 4838 /* 4839 * Verify the logs now to make sure we don't have any unexpected errors 4840 * when we claim log blocks later. 4841 */ 4842 error = spa_ld_verify_logs(spa, type, ereport); 4843 if (error != 0) 4844 return (error); 4845 4846 if (missing_feat_write) { 4847 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 4848 4849 /* 4850 * At this point, we know that we can open the pool in 4851 * read-only mode but not read-write mode. We now have enough 4852 * information and can return to userland. 4853 */ 4854 return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 4855 ENOTSUP)); 4856 } 4857 4858 /* 4859 * Traverse the last txgs to make sure the pool was left off in a safe 4860 * state. When performing an extreme rewind, we verify the whole pool, 4861 * which can take a very long time. 4862 */ 4863 error = spa_ld_verify_pool_data(spa); 4864 if (error != 0) 4865 return (error); 4866 4867 /* 4868 * Calculate the deflated space for the pool. This must be done before 4869 * we write anything to the pool because we'd need to update the space 4870 * accounting using the deflated sizes. 4871 */ 4872 spa_update_dspace(spa); 4873 4874 /* 4875 * We have now retrieved all the information we needed to open the 4876 * pool. If we are importing the pool in read-write mode, a few 4877 * additional steps must be performed to finish the import. 4878 */ 4879 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 4880 spa->spa_load_max_txg == UINT64_MAX)) { 4881 uint64_t config_cache_txg = spa->spa_config_txg; 4882 4883 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 4884 4885 /* 4886 * In case of a checkpoint rewind, log the original txg 4887 * of the checkpointed uberblock. 4888 */ 4889 if (checkpoint_rewind) { 4890 spa_history_log_internal(spa, "checkpoint rewind", 4891 NULL, "rewound state to txg=%llu", 4892 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 4893 } 4894 4895 /* 4896 * Traverse the ZIL and claim all blocks. 4897 */ 4898 spa_ld_claim_log_blocks(spa); 4899 4900 /* 4901 * Kick-off the syncing thread. 4902 */ 4903 spa->spa_sync_on = B_TRUE; 4904 txg_sync_start(spa->spa_dsl_pool); 4905 mmp_thread_start(spa); 4906 4907 /* 4908 * Wait for all claims to sync. We sync up to the highest 4909 * claimed log block birth time so that claimed log blocks 4910 * don't appear to be from the future. spa_claim_max_txg 4911 * will have been set for us by ZIL traversal operations 4912 * performed above. 4913 */ 4914 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 4915 4916 /* 4917 * Check if we need to request an update of the config. On the 4918 * next sync, we would update the config stored in vdev labels 4919 * and the cachefile (by default /etc/zfs/zpool.cache). 4920 */ 4921 spa_ld_check_for_config_update(spa, config_cache_txg, 4922 update_config_cache); 4923 4924 /* 4925 * Check if a rebuild was in progress and if so resume it. 4926 * Then check all DTLs to see if anything needs resilvering. 4927 * The resilver will be deferred if a rebuild was started. 4928 */ 4929 if (vdev_rebuild_active(spa->spa_root_vdev)) { 4930 vdev_rebuild_restart(spa); 4931 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 4932 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 4933 spa_async_request(spa, SPA_ASYNC_RESILVER); 4934 } 4935 4936 /* 4937 * Log the fact that we booted up (so that we can detect if 4938 * we rebooted in the middle of an operation). 4939 */ 4940 spa_history_log_version(spa, "open", NULL); 4941 4942 spa_restart_removal(spa); 4943 spa_spawn_aux_threads(spa); 4944 4945 /* 4946 * Delete any inconsistent datasets. 4947 * 4948 * Note: 4949 * Since we may be issuing deletes for clones here, 4950 * we make sure to do so after we've spawned all the 4951 * auxiliary threads above (from which the livelist 4952 * deletion zthr is part of). 4953 */ 4954 (void) dmu_objset_find(spa_name(spa), 4955 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 4956 4957 /* 4958 * Clean up any stale temporary dataset userrefs. 4959 */ 4960 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 4961 4962 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 4963 vdev_initialize_restart(spa->spa_root_vdev); 4964 vdev_trim_restart(spa->spa_root_vdev); 4965 vdev_autotrim_restart(spa); 4966 spa_config_exit(spa, SCL_CONFIG, FTAG); 4967 } 4968 4969 spa_import_progress_remove(spa_guid(spa)); 4970 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 4971 4972 spa_load_note(spa, "LOADED"); 4973 4974 return (0); 4975 } 4976 4977 static int 4978 spa_load_retry(spa_t *spa, spa_load_state_t state) 4979 { 4980 spa_mode_t mode = spa->spa_mode; 4981 4982 spa_unload(spa); 4983 spa_deactivate(spa); 4984 4985 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 4986 4987 spa_activate(spa, mode); 4988 spa_async_suspend(spa); 4989 4990 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 4991 (u_longlong_t)spa->spa_load_max_txg); 4992 4993 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 4994 } 4995 4996 /* 4997 * If spa_load() fails this function will try loading prior txg's. If 4998 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 4999 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 5000 * function will not rewind the pool and will return the same error as 5001 * spa_load(). 5002 */ 5003 static int 5004 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 5005 int rewind_flags) 5006 { 5007 nvlist_t *loadinfo = NULL; 5008 nvlist_t *config = NULL; 5009 int load_error, rewind_error; 5010 uint64_t safe_rewind_txg; 5011 uint64_t min_txg; 5012 5013 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 5014 spa->spa_load_max_txg = spa->spa_load_txg; 5015 spa_set_log_state(spa, SPA_LOG_CLEAR); 5016 } else { 5017 spa->spa_load_max_txg = max_request; 5018 if (max_request != UINT64_MAX) 5019 spa->spa_extreme_rewind = B_TRUE; 5020 } 5021 5022 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 5023 if (load_error == 0) 5024 return (0); 5025 if (load_error == ZFS_ERR_NO_CHECKPOINT) { 5026 /* 5027 * When attempting checkpoint-rewind on a pool with no 5028 * checkpoint, we should not attempt to load uberblocks 5029 * from previous txgs when spa_load fails. 5030 */ 5031 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5032 spa_import_progress_remove(spa_guid(spa)); 5033 return (load_error); 5034 } 5035 5036 if (spa->spa_root_vdev != NULL) 5037 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5038 5039 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 5040 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 5041 5042 if (rewind_flags & ZPOOL_NEVER_REWIND) { 5043 nvlist_free(config); 5044 spa_import_progress_remove(spa_guid(spa)); 5045 return (load_error); 5046 } 5047 5048 if (state == SPA_LOAD_RECOVER) { 5049 /* Price of rolling back is discarding txgs, including log */ 5050 spa_set_log_state(spa, SPA_LOG_CLEAR); 5051 } else { 5052 /* 5053 * If we aren't rolling back save the load info from our first 5054 * import attempt so that we can restore it after attempting 5055 * to rewind. 5056 */ 5057 loadinfo = spa->spa_load_info; 5058 spa->spa_load_info = fnvlist_alloc(); 5059 } 5060 5061 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 5062 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 5063 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 5064 TXG_INITIAL : safe_rewind_txg; 5065 5066 /* 5067 * Continue as long as we're finding errors, we're still within 5068 * the acceptable rewind range, and we're still finding uberblocks 5069 */ 5070 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 5071 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 5072 if (spa->spa_load_max_txg < safe_rewind_txg) 5073 spa->spa_extreme_rewind = B_TRUE; 5074 rewind_error = spa_load_retry(spa, state); 5075 } 5076 5077 spa->spa_extreme_rewind = B_FALSE; 5078 spa->spa_load_max_txg = UINT64_MAX; 5079 5080 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 5081 spa_config_set(spa, config); 5082 else 5083 nvlist_free(config); 5084 5085 if (state == SPA_LOAD_RECOVER) { 5086 ASSERT3P(loadinfo, ==, NULL); 5087 spa_import_progress_remove(spa_guid(spa)); 5088 return (rewind_error); 5089 } else { 5090 /* Store the rewind info as part of the initial load info */ 5091 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 5092 spa->spa_load_info); 5093 5094 /* Restore the initial load info */ 5095 fnvlist_free(spa->spa_load_info); 5096 spa->spa_load_info = loadinfo; 5097 5098 spa_import_progress_remove(spa_guid(spa)); 5099 return (load_error); 5100 } 5101 } 5102 5103 /* 5104 * Pool Open/Import 5105 * 5106 * The import case is identical to an open except that the configuration is sent 5107 * down from userland, instead of grabbed from the configuration cache. For the 5108 * case of an open, the pool configuration will exist in the 5109 * POOL_STATE_UNINITIALIZED state. 5110 * 5111 * The stats information (gen/count/ustats) is used to gather vdev statistics at 5112 * the same time open the pool, without having to keep around the spa_t in some 5113 * ambiguous state. 5114 */ 5115 static int 5116 spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, 5117 nvlist_t **config) 5118 { 5119 spa_t *spa; 5120 spa_load_state_t state = SPA_LOAD_OPEN; 5121 int error; 5122 int locked = B_FALSE; 5123 int firstopen = B_FALSE; 5124 5125 *spapp = NULL; 5126 5127 /* 5128 * As disgusting as this is, we need to support recursive calls to this 5129 * function because dsl_dir_open() is called during spa_load(), and ends 5130 * up calling spa_open() again. The real fix is to figure out how to 5131 * avoid dsl_dir_open() calling this in the first place. 5132 */ 5133 if (MUTEX_NOT_HELD(&spa_namespace_lock)) { 5134 mutex_enter(&spa_namespace_lock); 5135 locked = B_TRUE; 5136 } 5137 5138 if ((spa = spa_lookup(pool)) == NULL) { 5139 if (locked) 5140 mutex_exit(&spa_namespace_lock); 5141 return (SET_ERROR(ENOENT)); 5142 } 5143 5144 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 5145 zpool_load_policy_t policy; 5146 5147 firstopen = B_TRUE; 5148 5149 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 5150 &policy); 5151 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 5152 state = SPA_LOAD_RECOVER; 5153 5154 spa_activate(spa, spa_mode_global); 5155 5156 if (state != SPA_LOAD_RECOVER) 5157 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 5158 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 5159 5160 zfs_dbgmsg("spa_open_common: opening %s", pool); 5161 error = spa_load_best(spa, state, policy.zlp_txg, 5162 policy.zlp_rewind); 5163 5164 if (error == EBADF) { 5165 /* 5166 * If vdev_validate() returns failure (indicated by 5167 * EBADF), it indicates that one of the vdevs indicates 5168 * that the pool has been exported or destroyed. If 5169 * this is the case, the config cache is out of sync and 5170 * we should remove the pool from the namespace. 5171 */ 5172 spa_unload(spa); 5173 spa_deactivate(spa); 5174 spa_write_cachefile(spa, B_TRUE, B_TRUE); 5175 spa_remove(spa); 5176 if (locked) 5177 mutex_exit(&spa_namespace_lock); 5178 return (SET_ERROR(ENOENT)); 5179 } 5180 5181 if (error) { 5182 /* 5183 * We can't open the pool, but we still have useful 5184 * information: the state of each vdev after the 5185 * attempted vdev_open(). Return this to the user. 5186 */ 5187 if (config != NULL && spa->spa_config) { 5188 *config = fnvlist_dup(spa->spa_config); 5189 fnvlist_add_nvlist(*config, 5190 ZPOOL_CONFIG_LOAD_INFO, 5191 spa->spa_load_info); 5192 } 5193 spa_unload(spa); 5194 spa_deactivate(spa); 5195 spa->spa_last_open_failed = error; 5196 if (locked) 5197 mutex_exit(&spa_namespace_lock); 5198 *spapp = NULL; 5199 return (error); 5200 } 5201 } 5202 5203 spa_open_ref(spa, tag); 5204 5205 if (config != NULL) 5206 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5207 5208 /* 5209 * If we've recovered the pool, pass back any information we 5210 * gathered while doing the load. 5211 */ 5212 if (state == SPA_LOAD_RECOVER) { 5213 fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 5214 spa->spa_load_info); 5215 } 5216 5217 if (locked) { 5218 spa->spa_last_open_failed = 0; 5219 spa->spa_last_ubsync_txg = 0; 5220 spa->spa_load_txg = 0; 5221 mutex_exit(&spa_namespace_lock); 5222 } 5223 5224 if (firstopen) 5225 zvol_create_minors_recursive(spa_name(spa)); 5226 5227 *spapp = spa; 5228 5229 return (0); 5230 } 5231 5232 int 5233 spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy, 5234 nvlist_t **config) 5235 { 5236 return (spa_open_common(name, spapp, tag, policy, config)); 5237 } 5238 5239 int 5240 spa_open(const char *name, spa_t **spapp, void *tag) 5241 { 5242 return (spa_open_common(name, spapp, tag, NULL, NULL)); 5243 } 5244 5245 /* 5246 * Lookup the given spa_t, incrementing the inject count in the process, 5247 * preventing it from being exported or destroyed. 5248 */ 5249 spa_t * 5250 spa_inject_addref(char *name) 5251 { 5252 spa_t *spa; 5253 5254 mutex_enter(&spa_namespace_lock); 5255 if ((spa = spa_lookup(name)) == NULL) { 5256 mutex_exit(&spa_namespace_lock); 5257 return (NULL); 5258 } 5259 spa->spa_inject_ref++; 5260 mutex_exit(&spa_namespace_lock); 5261 5262 return (spa); 5263 } 5264 5265 void 5266 spa_inject_delref(spa_t *spa) 5267 { 5268 mutex_enter(&spa_namespace_lock); 5269 spa->spa_inject_ref--; 5270 mutex_exit(&spa_namespace_lock); 5271 } 5272 5273 /* 5274 * Add spares device information to the nvlist. 5275 */ 5276 static void 5277 spa_add_spares(spa_t *spa, nvlist_t *config) 5278 { 5279 nvlist_t **spares; 5280 uint_t i, nspares; 5281 nvlist_t *nvroot; 5282 uint64_t guid; 5283 vdev_stat_t *vs; 5284 uint_t vsc; 5285 uint64_t pool; 5286 5287 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5288 5289 if (spa->spa_spares.sav_count == 0) 5290 return; 5291 5292 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 5293 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 5294 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 5295 if (nspares != 0) { 5296 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5297 (const nvlist_t * const *)spares, nspares); 5298 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5299 &spares, &nspares)); 5300 5301 /* 5302 * Go through and find any spares which have since been 5303 * repurposed as an active spare. If this is the case, update 5304 * their status appropriately. 5305 */ 5306 for (i = 0; i < nspares; i++) { 5307 guid = fnvlist_lookup_uint64(spares[i], 5308 ZPOOL_CONFIG_GUID); 5309 if (spa_spare_exists(guid, &pool, NULL) && 5310 pool != 0ULL) { 5311 VERIFY0(nvlist_lookup_uint64_array(spares[i], 5312 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, 5313 &vsc)); 5314 vs->vs_state = VDEV_STATE_CANT_OPEN; 5315 vs->vs_aux = VDEV_AUX_SPARED; 5316 } 5317 } 5318 } 5319 } 5320 5321 /* 5322 * Add l2cache device information to the nvlist, including vdev stats. 5323 */ 5324 static void 5325 spa_add_l2cache(spa_t *spa, nvlist_t *config) 5326 { 5327 nvlist_t **l2cache; 5328 uint_t i, j, nl2cache; 5329 nvlist_t *nvroot; 5330 uint64_t guid; 5331 vdev_t *vd; 5332 vdev_stat_t *vs; 5333 uint_t vsc; 5334 5335 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5336 5337 if (spa->spa_l2cache.sav_count == 0) 5338 return; 5339 5340 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 5341 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 5342 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 5343 if (nl2cache != 0) { 5344 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5345 (const nvlist_t * const *)l2cache, nl2cache); 5346 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5347 &l2cache, &nl2cache)); 5348 5349 /* 5350 * Update level 2 cache device stats. 5351 */ 5352 5353 for (i = 0; i < nl2cache; i++) { 5354 guid = fnvlist_lookup_uint64(l2cache[i], 5355 ZPOOL_CONFIG_GUID); 5356 5357 vd = NULL; 5358 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 5359 if (guid == 5360 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 5361 vd = spa->spa_l2cache.sav_vdevs[j]; 5362 break; 5363 } 5364 } 5365 ASSERT(vd != NULL); 5366 5367 VERIFY0(nvlist_lookup_uint64_array(l2cache[i], 5368 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 5369 vdev_get_stats(vd, vs); 5370 vdev_config_generate_stats(vd, l2cache[i]); 5371 5372 } 5373 } 5374 } 5375 5376 static void 5377 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 5378 { 5379 zap_cursor_t zc; 5380 zap_attribute_t za; 5381 5382 if (spa->spa_feat_for_read_obj != 0) { 5383 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5384 spa->spa_feat_for_read_obj); 5385 zap_cursor_retrieve(&zc, &za) == 0; 5386 zap_cursor_advance(&zc)) { 5387 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5388 za.za_num_integers == 1); 5389 VERIFY0(nvlist_add_uint64(features, za.za_name, 5390 za.za_first_integer)); 5391 } 5392 zap_cursor_fini(&zc); 5393 } 5394 5395 if (spa->spa_feat_for_write_obj != 0) { 5396 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5397 spa->spa_feat_for_write_obj); 5398 zap_cursor_retrieve(&zc, &za) == 0; 5399 zap_cursor_advance(&zc)) { 5400 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5401 za.za_num_integers == 1); 5402 VERIFY0(nvlist_add_uint64(features, za.za_name, 5403 za.za_first_integer)); 5404 } 5405 zap_cursor_fini(&zc); 5406 } 5407 } 5408 5409 static void 5410 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 5411 { 5412 int i; 5413 5414 for (i = 0; i < SPA_FEATURES; i++) { 5415 zfeature_info_t feature = spa_feature_table[i]; 5416 uint64_t refcount; 5417 5418 if (feature_get_refcount(spa, &feature, &refcount) != 0) 5419 continue; 5420 5421 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 5422 } 5423 } 5424 5425 /* 5426 * Store a list of pool features and their reference counts in the 5427 * config. 5428 * 5429 * The first time this is called on a spa, allocate a new nvlist, fetch 5430 * the pool features and reference counts from disk, then save the list 5431 * in the spa. In subsequent calls on the same spa use the saved nvlist 5432 * and refresh its values from the cached reference counts. This 5433 * ensures we don't block here on I/O on a suspended pool so 'zpool 5434 * clear' can resume the pool. 5435 */ 5436 static void 5437 spa_add_feature_stats(spa_t *spa, nvlist_t *config) 5438 { 5439 nvlist_t *features; 5440 5441 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5442 5443 mutex_enter(&spa->spa_feat_stats_lock); 5444 features = spa->spa_feat_stats; 5445 5446 if (features != NULL) { 5447 spa_feature_stats_from_cache(spa, features); 5448 } else { 5449 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 5450 spa->spa_feat_stats = features; 5451 spa_feature_stats_from_disk(spa, features); 5452 } 5453 5454 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 5455 features)); 5456 5457 mutex_exit(&spa->spa_feat_stats_lock); 5458 } 5459 5460 int 5461 spa_get_stats(const char *name, nvlist_t **config, 5462 char *altroot, size_t buflen) 5463 { 5464 int error; 5465 spa_t *spa; 5466 5467 *config = NULL; 5468 error = spa_open_common(name, &spa, FTAG, NULL, config); 5469 5470 if (spa != NULL) { 5471 /* 5472 * This still leaves a window of inconsistency where the spares 5473 * or l2cache devices could change and the config would be 5474 * self-inconsistent. 5475 */ 5476 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5477 5478 if (*config != NULL) { 5479 uint64_t loadtimes[2]; 5480 5481 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 5482 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 5483 fnvlist_add_uint64_array(*config, 5484 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2); 5485 5486 fnvlist_add_uint64(*config, 5487 ZPOOL_CONFIG_ERRCOUNT, 5488 spa_get_errlog_size(spa)); 5489 5490 if (spa_suspended(spa)) { 5491 fnvlist_add_uint64(*config, 5492 ZPOOL_CONFIG_SUSPENDED, 5493 spa->spa_failmode); 5494 fnvlist_add_uint64(*config, 5495 ZPOOL_CONFIG_SUSPENDED_REASON, 5496 spa->spa_suspended); 5497 } 5498 5499 spa_add_spares(spa, *config); 5500 spa_add_l2cache(spa, *config); 5501 spa_add_feature_stats(spa, *config); 5502 } 5503 } 5504 5505 /* 5506 * We want to get the alternate root even for faulted pools, so we cheat 5507 * and call spa_lookup() directly. 5508 */ 5509 if (altroot) { 5510 if (spa == NULL) { 5511 mutex_enter(&spa_namespace_lock); 5512 spa = spa_lookup(name); 5513 if (spa) 5514 spa_altroot(spa, altroot, buflen); 5515 else 5516 altroot[0] = '\0'; 5517 spa = NULL; 5518 mutex_exit(&spa_namespace_lock); 5519 } else { 5520 spa_altroot(spa, altroot, buflen); 5521 } 5522 } 5523 5524 if (spa != NULL) { 5525 spa_config_exit(spa, SCL_CONFIG, FTAG); 5526 spa_close(spa, FTAG); 5527 } 5528 5529 return (error); 5530 } 5531 5532 /* 5533 * Validate that the auxiliary device array is well formed. We must have an 5534 * array of nvlists, each which describes a valid leaf vdev. If this is an 5535 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 5536 * specified, as long as they are well-formed. 5537 */ 5538 static int 5539 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 5540 spa_aux_vdev_t *sav, const char *config, uint64_t version, 5541 vdev_labeltype_t label) 5542 { 5543 nvlist_t **dev; 5544 uint_t i, ndev; 5545 vdev_t *vd; 5546 int error; 5547 5548 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5549 5550 /* 5551 * It's acceptable to have no devs specified. 5552 */ 5553 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 5554 return (0); 5555 5556 if (ndev == 0) 5557 return (SET_ERROR(EINVAL)); 5558 5559 /* 5560 * Make sure the pool is formatted with a version that supports this 5561 * device type. 5562 */ 5563 if (spa_version(spa) < version) 5564 return (SET_ERROR(ENOTSUP)); 5565 5566 /* 5567 * Set the pending device list so we correctly handle device in-use 5568 * checking. 5569 */ 5570 sav->sav_pending = dev; 5571 sav->sav_npending = ndev; 5572 5573 for (i = 0; i < ndev; i++) { 5574 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 5575 mode)) != 0) 5576 goto out; 5577 5578 if (!vd->vdev_ops->vdev_op_leaf) { 5579 vdev_free(vd); 5580 error = SET_ERROR(EINVAL); 5581 goto out; 5582 } 5583 5584 vd->vdev_top = vd; 5585 5586 if ((error = vdev_open(vd)) == 0 && 5587 (error = vdev_label_init(vd, crtxg, label)) == 0) { 5588 fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 5589 vd->vdev_guid); 5590 } 5591 5592 vdev_free(vd); 5593 5594 if (error && 5595 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 5596 goto out; 5597 else 5598 error = 0; 5599 } 5600 5601 out: 5602 sav->sav_pending = NULL; 5603 sav->sav_npending = 0; 5604 return (error); 5605 } 5606 5607 static int 5608 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 5609 { 5610 int error; 5611 5612 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5613 5614 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5615 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 5616 VDEV_LABEL_SPARE)) != 0) { 5617 return (error); 5618 } 5619 5620 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5621 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 5622 VDEV_LABEL_L2CACHE)); 5623 } 5624 5625 static void 5626 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 5627 const char *config) 5628 { 5629 int i; 5630 5631 if (sav->sav_config != NULL) { 5632 nvlist_t **olddevs; 5633 uint_t oldndevs; 5634 nvlist_t **newdevs; 5635 5636 /* 5637 * Generate new dev list by concatenating with the 5638 * current dev list. 5639 */ 5640 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config, 5641 &olddevs, &oldndevs)); 5642 5643 newdevs = kmem_alloc(sizeof (void *) * 5644 (ndevs + oldndevs), KM_SLEEP); 5645 for (i = 0; i < oldndevs; i++) 5646 newdevs[i] = fnvlist_dup(olddevs[i]); 5647 for (i = 0; i < ndevs; i++) 5648 newdevs[i + oldndevs] = fnvlist_dup(devs[i]); 5649 5650 fnvlist_remove(sav->sav_config, config); 5651 5652 fnvlist_add_nvlist_array(sav->sav_config, config, 5653 (const nvlist_t * const *)newdevs, ndevs + oldndevs); 5654 for (i = 0; i < oldndevs + ndevs; i++) 5655 nvlist_free(newdevs[i]); 5656 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 5657 } else { 5658 /* 5659 * Generate a new dev list. 5660 */ 5661 sav->sav_config = fnvlist_alloc(); 5662 fnvlist_add_nvlist_array(sav->sav_config, config, 5663 (const nvlist_t * const *)devs, ndevs); 5664 } 5665 } 5666 5667 /* 5668 * Stop and drop level 2 ARC devices 5669 */ 5670 void 5671 spa_l2cache_drop(spa_t *spa) 5672 { 5673 vdev_t *vd; 5674 int i; 5675 spa_aux_vdev_t *sav = &spa->spa_l2cache; 5676 5677 for (i = 0; i < sav->sav_count; i++) { 5678 uint64_t pool; 5679 5680 vd = sav->sav_vdevs[i]; 5681 ASSERT(vd != NULL); 5682 5683 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 5684 pool != 0ULL && l2arc_vdev_present(vd)) 5685 l2arc_remove_vdev(vd); 5686 } 5687 } 5688 5689 /* 5690 * Verify encryption parameters for spa creation. If we are encrypting, we must 5691 * have the encryption feature flag enabled. 5692 */ 5693 static int 5694 spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 5695 boolean_t has_encryption) 5696 { 5697 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 5698 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 5699 !has_encryption) 5700 return (SET_ERROR(ENOTSUP)); 5701 5702 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 5703 } 5704 5705 /* 5706 * Pool Creation 5707 */ 5708 int 5709 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 5710 nvlist_t *zplprops, dsl_crypto_params_t *dcp) 5711 { 5712 spa_t *spa; 5713 char *altroot = NULL; 5714 vdev_t *rvd; 5715 dsl_pool_t *dp; 5716 dmu_tx_t *tx; 5717 int error = 0; 5718 uint64_t txg = TXG_INITIAL; 5719 nvlist_t **spares, **l2cache; 5720 uint_t nspares, nl2cache; 5721 uint64_t version, obj, ndraid = 0; 5722 boolean_t has_features; 5723 boolean_t has_encryption; 5724 boolean_t has_allocclass; 5725 spa_feature_t feat; 5726 char *feat_name; 5727 char *poolname; 5728 nvlist_t *nvl; 5729 5730 if (props == NULL || 5731 nvlist_lookup_string(props, "tname", &poolname) != 0) 5732 poolname = (char *)pool; 5733 5734 /* 5735 * If this pool already exists, return failure. 5736 */ 5737 mutex_enter(&spa_namespace_lock); 5738 if (spa_lookup(poolname) != NULL) { 5739 mutex_exit(&spa_namespace_lock); 5740 return (SET_ERROR(EEXIST)); 5741 } 5742 5743 /* 5744 * Allocate a new spa_t structure. 5745 */ 5746 nvl = fnvlist_alloc(); 5747 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 5748 (void) nvlist_lookup_string(props, 5749 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 5750 spa = spa_add(poolname, nvl, altroot); 5751 fnvlist_free(nvl); 5752 spa_activate(spa, spa_mode_global); 5753 5754 if (props && (error = spa_prop_validate(spa, props))) { 5755 spa_deactivate(spa); 5756 spa_remove(spa); 5757 mutex_exit(&spa_namespace_lock); 5758 return (error); 5759 } 5760 5761 /* 5762 * Temporary pool names should never be written to disk. 5763 */ 5764 if (poolname != pool) 5765 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 5766 5767 has_features = B_FALSE; 5768 has_encryption = B_FALSE; 5769 has_allocclass = B_FALSE; 5770 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 5771 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 5772 if (zpool_prop_feature(nvpair_name(elem))) { 5773 has_features = B_TRUE; 5774 5775 feat_name = strchr(nvpair_name(elem), '@') + 1; 5776 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 5777 if (feat == SPA_FEATURE_ENCRYPTION) 5778 has_encryption = B_TRUE; 5779 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 5780 has_allocclass = B_TRUE; 5781 } 5782 } 5783 5784 /* verify encryption params, if they were provided */ 5785 if (dcp != NULL) { 5786 error = spa_create_check_encryption_params(dcp, has_encryption); 5787 if (error != 0) { 5788 spa_deactivate(spa); 5789 spa_remove(spa); 5790 mutex_exit(&spa_namespace_lock); 5791 return (error); 5792 } 5793 } 5794 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 5795 spa_deactivate(spa); 5796 spa_remove(spa); 5797 mutex_exit(&spa_namespace_lock); 5798 return (ENOTSUP); 5799 } 5800 5801 if (has_features || nvlist_lookup_uint64(props, 5802 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 5803 version = SPA_VERSION; 5804 } 5805 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 5806 5807 spa->spa_first_txg = txg; 5808 spa->spa_uberblock.ub_txg = txg - 1; 5809 spa->spa_uberblock.ub_version = version; 5810 spa->spa_ubsync = spa->spa_uberblock; 5811 spa->spa_load_state = SPA_LOAD_CREATE; 5812 spa->spa_removing_phys.sr_state = DSS_NONE; 5813 spa->spa_removing_phys.sr_removing_vdev = -1; 5814 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 5815 spa->spa_indirect_vdevs_loaded = B_TRUE; 5816 5817 /* 5818 * Create "The Godfather" zio to hold all async IOs 5819 */ 5820 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 5821 KM_SLEEP); 5822 for (int i = 0; i < max_ncpus; i++) { 5823 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 5824 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 5825 ZIO_FLAG_GODFATHER); 5826 } 5827 5828 /* 5829 * Create the root vdev. 5830 */ 5831 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5832 5833 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 5834 5835 ASSERT(error != 0 || rvd != NULL); 5836 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 5837 5838 if (error == 0 && !zfs_allocatable_devs(nvroot)) 5839 error = SET_ERROR(EINVAL); 5840 5841 if (error == 0 && 5842 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 5843 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 && 5844 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 5845 /* 5846 * instantiate the metaslab groups (this will dirty the vdevs) 5847 * we can no longer error exit past this point 5848 */ 5849 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 5850 vdev_t *vd = rvd->vdev_child[c]; 5851 5852 vdev_metaslab_set_size(vd); 5853 vdev_expand(vd, txg); 5854 } 5855 } 5856 5857 spa_config_exit(spa, SCL_ALL, FTAG); 5858 5859 if (error != 0) { 5860 spa_unload(spa); 5861 spa_deactivate(spa); 5862 spa_remove(spa); 5863 mutex_exit(&spa_namespace_lock); 5864 return (error); 5865 } 5866 5867 /* 5868 * Get the list of spares, if specified. 5869 */ 5870 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5871 &spares, &nspares) == 0) { 5872 spa->spa_spares.sav_config = fnvlist_alloc(); 5873 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 5874 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 5875 nspares); 5876 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5877 spa_load_spares(spa); 5878 spa_config_exit(spa, SCL_ALL, FTAG); 5879 spa->spa_spares.sav_sync = B_TRUE; 5880 } 5881 5882 /* 5883 * Get the list of level 2 cache devices, if specified. 5884 */ 5885 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5886 &l2cache, &nl2cache) == 0) { 5887 VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config, 5888 NV_UNIQUE_NAME, KM_SLEEP)); 5889 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 5890 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 5891 nl2cache); 5892 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5893 spa_load_l2cache(spa); 5894 spa_config_exit(spa, SCL_ALL, FTAG); 5895 spa->spa_l2cache.sav_sync = B_TRUE; 5896 } 5897 5898 spa->spa_is_initializing = B_TRUE; 5899 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 5900 spa->spa_is_initializing = B_FALSE; 5901 5902 /* 5903 * Create DDTs (dedup tables). 5904 */ 5905 ddt_create(spa); 5906 5907 spa_update_dspace(spa); 5908 5909 tx = dmu_tx_create_assigned(dp, txg); 5910 5911 /* 5912 * Create the pool's history object. 5913 */ 5914 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 5915 spa_history_create_obj(spa, tx); 5916 5917 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 5918 spa_history_log_version(spa, "create", tx); 5919 5920 /* 5921 * Create the pool config object. 5922 */ 5923 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 5924 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 5925 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 5926 5927 if (zap_add(spa->spa_meta_objset, 5928 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 5929 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 5930 cmn_err(CE_PANIC, "failed to add pool config"); 5931 } 5932 5933 if (zap_add(spa->spa_meta_objset, 5934 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 5935 sizeof (uint64_t), 1, &version, tx) != 0) { 5936 cmn_err(CE_PANIC, "failed to add pool version"); 5937 } 5938 5939 /* Newly created pools with the right version are always deflated. */ 5940 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 5941 spa->spa_deflate = TRUE; 5942 if (zap_add(spa->spa_meta_objset, 5943 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 5944 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 5945 cmn_err(CE_PANIC, "failed to add deflate"); 5946 } 5947 } 5948 5949 /* 5950 * Create the deferred-free bpobj. Turn off compression 5951 * because sync-to-convergence takes longer if the blocksize 5952 * keeps changing. 5953 */ 5954 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 5955 dmu_object_set_compress(spa->spa_meta_objset, obj, 5956 ZIO_COMPRESS_OFF, tx); 5957 if (zap_add(spa->spa_meta_objset, 5958 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 5959 sizeof (uint64_t), 1, &obj, tx) != 0) { 5960 cmn_err(CE_PANIC, "failed to add bpobj"); 5961 } 5962 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 5963 spa->spa_meta_objset, obj)); 5964 5965 /* 5966 * Generate some random noise for salted checksums to operate on. 5967 */ 5968 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 5969 sizeof (spa->spa_cksum_salt.zcs_bytes)); 5970 5971 /* 5972 * Set pool properties. 5973 */ 5974 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 5975 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 5976 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 5977 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 5978 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 5979 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 5980 5981 if (props != NULL) { 5982 spa_configfile_set(spa, props, B_FALSE); 5983 spa_sync_props(props, tx); 5984 } 5985 5986 for (int i = 0; i < ndraid; i++) 5987 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 5988 5989 dmu_tx_commit(tx); 5990 5991 spa->spa_sync_on = B_TRUE; 5992 txg_sync_start(dp); 5993 mmp_thread_start(spa); 5994 txg_wait_synced(dp, txg); 5995 5996 spa_spawn_aux_threads(spa); 5997 5998 spa_write_cachefile(spa, B_FALSE, B_TRUE); 5999 6000 /* 6001 * Don't count references from objsets that are already closed 6002 * and are making their way through the eviction process. 6003 */ 6004 spa_evicting_os_wait(spa); 6005 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 6006 spa->spa_load_state = SPA_LOAD_NONE; 6007 6008 mutex_exit(&spa_namespace_lock); 6009 6010 return (0); 6011 } 6012 6013 /* 6014 * Import a non-root pool into the system. 6015 */ 6016 int 6017 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 6018 { 6019 spa_t *spa; 6020 char *altroot = NULL; 6021 spa_load_state_t state = SPA_LOAD_IMPORT; 6022 zpool_load_policy_t policy; 6023 spa_mode_t mode = spa_mode_global; 6024 uint64_t readonly = B_FALSE; 6025 int error; 6026 nvlist_t *nvroot; 6027 nvlist_t **spares, **l2cache; 6028 uint_t nspares, nl2cache; 6029 6030 /* 6031 * If a pool with this name exists, return failure. 6032 */ 6033 mutex_enter(&spa_namespace_lock); 6034 if (spa_lookup(pool) != NULL) { 6035 mutex_exit(&spa_namespace_lock); 6036 return (SET_ERROR(EEXIST)); 6037 } 6038 6039 /* 6040 * Create and initialize the spa structure. 6041 */ 6042 (void) nvlist_lookup_string(props, 6043 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6044 (void) nvlist_lookup_uint64(props, 6045 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 6046 if (readonly) 6047 mode = SPA_MODE_READ; 6048 spa = spa_add(pool, config, altroot); 6049 spa->spa_import_flags = flags; 6050 6051 /* 6052 * Verbatim import - Take a pool and insert it into the namespace 6053 * as if it had been loaded at boot. 6054 */ 6055 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 6056 if (props != NULL) 6057 spa_configfile_set(spa, props, B_FALSE); 6058 6059 spa_write_cachefile(spa, B_FALSE, B_TRUE); 6060 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6061 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 6062 mutex_exit(&spa_namespace_lock); 6063 return (0); 6064 } 6065 6066 spa_activate(spa, mode); 6067 6068 /* 6069 * Don't start async tasks until we know everything is healthy. 6070 */ 6071 spa_async_suspend(spa); 6072 6073 zpool_get_load_policy(config, &policy); 6074 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6075 state = SPA_LOAD_RECOVER; 6076 6077 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 6078 6079 if (state != SPA_LOAD_RECOVER) { 6080 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6081 zfs_dbgmsg("spa_import: importing %s", pool); 6082 } else { 6083 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 6084 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 6085 } 6086 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 6087 6088 /* 6089 * Propagate anything learned while loading the pool and pass it 6090 * back to caller (i.e. rewind info, missing devices, etc). 6091 */ 6092 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info); 6093 6094 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6095 /* 6096 * Toss any existing sparelist, as it doesn't have any validity 6097 * anymore, and conflicts with spa_has_spare(). 6098 */ 6099 if (spa->spa_spares.sav_config) { 6100 nvlist_free(spa->spa_spares.sav_config); 6101 spa->spa_spares.sav_config = NULL; 6102 spa_load_spares(spa); 6103 } 6104 if (spa->spa_l2cache.sav_config) { 6105 nvlist_free(spa->spa_l2cache.sav_config); 6106 spa->spa_l2cache.sav_config = NULL; 6107 spa_load_l2cache(spa); 6108 } 6109 6110 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 6111 spa_config_exit(spa, SCL_ALL, FTAG); 6112 6113 if (props != NULL) 6114 spa_configfile_set(spa, props, B_FALSE); 6115 6116 if (error != 0 || (props && spa_writeable(spa) && 6117 (error = spa_prop_set(spa, props)))) { 6118 spa_unload(spa); 6119 spa_deactivate(spa); 6120 spa_remove(spa); 6121 mutex_exit(&spa_namespace_lock); 6122 return (error); 6123 } 6124 6125 spa_async_resume(spa); 6126 6127 /* 6128 * Override any spares and level 2 cache devices as specified by 6129 * the user, as these may have correct device names/devids, etc. 6130 */ 6131 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6132 &spares, &nspares) == 0) { 6133 if (spa->spa_spares.sav_config) 6134 fnvlist_remove(spa->spa_spares.sav_config, 6135 ZPOOL_CONFIG_SPARES); 6136 else 6137 spa->spa_spares.sav_config = fnvlist_alloc(); 6138 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 6139 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 6140 nspares); 6141 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6142 spa_load_spares(spa); 6143 spa_config_exit(spa, SCL_ALL, FTAG); 6144 spa->spa_spares.sav_sync = B_TRUE; 6145 } 6146 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6147 &l2cache, &nl2cache) == 0) { 6148 if (spa->spa_l2cache.sav_config) 6149 fnvlist_remove(spa->spa_l2cache.sav_config, 6150 ZPOOL_CONFIG_L2CACHE); 6151 else 6152 spa->spa_l2cache.sav_config = fnvlist_alloc(); 6153 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6154 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 6155 nl2cache); 6156 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6157 spa_load_l2cache(spa); 6158 spa_config_exit(spa, SCL_ALL, FTAG); 6159 spa->spa_l2cache.sav_sync = B_TRUE; 6160 } 6161 6162 /* 6163 * Check for any removed devices. 6164 */ 6165 if (spa->spa_autoreplace) { 6166 spa_aux_check_removed(&spa->spa_spares); 6167 spa_aux_check_removed(&spa->spa_l2cache); 6168 } 6169 6170 if (spa_writeable(spa)) { 6171 /* 6172 * Update the config cache to include the newly-imported pool. 6173 */ 6174 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6175 } 6176 6177 /* 6178 * It's possible that the pool was expanded while it was exported. 6179 * We kick off an async task to handle this for us. 6180 */ 6181 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 6182 6183 spa_history_log_version(spa, "import", NULL); 6184 6185 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6186 6187 mutex_exit(&spa_namespace_lock); 6188 6189 zvol_create_minors_recursive(pool); 6190 6191 return (0); 6192 } 6193 6194 nvlist_t * 6195 spa_tryimport(nvlist_t *tryconfig) 6196 { 6197 nvlist_t *config = NULL; 6198 char *poolname, *cachefile; 6199 spa_t *spa; 6200 uint64_t state; 6201 int error; 6202 zpool_load_policy_t policy; 6203 6204 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 6205 return (NULL); 6206 6207 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 6208 return (NULL); 6209 6210 /* 6211 * Create and initialize the spa structure. 6212 */ 6213 mutex_enter(&spa_namespace_lock); 6214 spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL); 6215 spa_activate(spa, SPA_MODE_READ); 6216 6217 /* 6218 * Rewind pool if a max txg was provided. 6219 */ 6220 zpool_get_load_policy(spa->spa_config, &policy); 6221 if (policy.zlp_txg != UINT64_MAX) { 6222 spa->spa_load_max_txg = policy.zlp_txg; 6223 spa->spa_extreme_rewind = B_TRUE; 6224 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 6225 poolname, (longlong_t)policy.zlp_txg); 6226 } else { 6227 zfs_dbgmsg("spa_tryimport: importing %s", poolname); 6228 } 6229 6230 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 6231 == 0) { 6232 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 6233 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6234 } else { 6235 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 6236 } 6237 6238 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 6239 6240 /* 6241 * If 'tryconfig' was at least parsable, return the current config. 6242 */ 6243 if (spa->spa_root_vdev != NULL) { 6244 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6245 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname); 6246 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state); 6247 fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 6248 spa->spa_uberblock.ub_timestamp); 6249 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6250 spa->spa_load_info); 6251 fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 6252 spa->spa_errata); 6253 6254 /* 6255 * If the bootfs property exists on this pool then we 6256 * copy it out so that external consumers can tell which 6257 * pools are bootable. 6258 */ 6259 if ((!error || error == EEXIST) && spa->spa_bootfs) { 6260 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6261 6262 /* 6263 * We have to play games with the name since the 6264 * pool was opened as TRYIMPORT_NAME. 6265 */ 6266 if (dsl_dsobj_to_dsname(spa_name(spa), 6267 spa->spa_bootfs, tmpname) == 0) { 6268 char *cp; 6269 char *dsname; 6270 6271 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6272 6273 cp = strchr(tmpname, '/'); 6274 if (cp == NULL) { 6275 (void) strlcpy(dsname, tmpname, 6276 MAXPATHLEN); 6277 } else { 6278 (void) snprintf(dsname, MAXPATHLEN, 6279 "%s/%s", poolname, ++cp); 6280 } 6281 fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS, 6282 dsname); 6283 kmem_free(dsname, MAXPATHLEN); 6284 } 6285 kmem_free(tmpname, MAXPATHLEN); 6286 } 6287 6288 /* 6289 * Add the list of hot spares and level 2 cache devices. 6290 */ 6291 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6292 spa_add_spares(spa, config); 6293 spa_add_l2cache(spa, config); 6294 spa_config_exit(spa, SCL_CONFIG, FTAG); 6295 } 6296 6297 spa_unload(spa); 6298 spa_deactivate(spa); 6299 spa_remove(spa); 6300 mutex_exit(&spa_namespace_lock); 6301 6302 return (config); 6303 } 6304 6305 /* 6306 * Pool export/destroy 6307 * 6308 * The act of destroying or exporting a pool is very simple. We make sure there 6309 * is no more pending I/O and any references to the pool are gone. Then, we 6310 * update the pool state and sync all the labels to disk, removing the 6311 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 6312 * we don't sync the labels or remove the configuration cache. 6313 */ 6314 static int 6315 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 6316 boolean_t force, boolean_t hardforce) 6317 { 6318 int error; 6319 spa_t *spa; 6320 6321 if (oldconfig) 6322 *oldconfig = NULL; 6323 6324 if (!(spa_mode_global & SPA_MODE_WRITE)) 6325 return (SET_ERROR(EROFS)); 6326 6327 mutex_enter(&spa_namespace_lock); 6328 if ((spa = spa_lookup(pool)) == NULL) { 6329 mutex_exit(&spa_namespace_lock); 6330 return (SET_ERROR(ENOENT)); 6331 } 6332 6333 if (spa->spa_is_exporting) { 6334 /* the pool is being exported by another thread */ 6335 mutex_exit(&spa_namespace_lock); 6336 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 6337 } 6338 spa->spa_is_exporting = B_TRUE; 6339 6340 /* 6341 * Put a hold on the pool, drop the namespace lock, stop async tasks, 6342 * reacquire the namespace lock, and see if we can export. 6343 */ 6344 spa_open_ref(spa, FTAG); 6345 mutex_exit(&spa_namespace_lock); 6346 spa_async_suspend(spa); 6347 if (spa->spa_zvol_taskq) { 6348 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 6349 taskq_wait(spa->spa_zvol_taskq); 6350 } 6351 mutex_enter(&spa_namespace_lock); 6352 spa_close(spa, FTAG); 6353 6354 if (spa->spa_state == POOL_STATE_UNINITIALIZED) 6355 goto export_spa; 6356 /* 6357 * The pool will be in core if it's openable, in which case we can 6358 * modify its state. Objsets may be open only because they're dirty, 6359 * so we have to force it to sync before checking spa_refcnt. 6360 */ 6361 if (spa->spa_sync_on) { 6362 txg_wait_synced(spa->spa_dsl_pool, 0); 6363 spa_evicting_os_wait(spa); 6364 } 6365 6366 /* 6367 * A pool cannot be exported or destroyed if there are active 6368 * references. If we are resetting a pool, allow references by 6369 * fault injection handlers. 6370 */ 6371 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 6372 error = SET_ERROR(EBUSY); 6373 goto fail; 6374 } 6375 6376 if (spa->spa_sync_on) { 6377 /* 6378 * A pool cannot be exported if it has an active shared spare. 6379 * This is to prevent other pools stealing the active spare 6380 * from an exported pool. At user's own will, such pool can 6381 * be forcedly exported. 6382 */ 6383 if (!force && new_state == POOL_STATE_EXPORTED && 6384 spa_has_active_shared_spare(spa)) { 6385 error = SET_ERROR(EXDEV); 6386 goto fail; 6387 } 6388 6389 /* 6390 * We're about to export or destroy this pool. Make sure 6391 * we stop all initialization and trim activity here before 6392 * we set the spa_final_txg. This will ensure that all 6393 * dirty data resulting from the initialization is 6394 * committed to disk before we unload the pool. 6395 */ 6396 if (spa->spa_root_vdev != NULL) { 6397 vdev_t *rvd = spa->spa_root_vdev; 6398 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 6399 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 6400 vdev_autotrim_stop_all(spa); 6401 vdev_rebuild_stop_all(spa); 6402 } 6403 6404 /* 6405 * We want this to be reflected on every label, 6406 * so mark them all dirty. spa_unload() will do the 6407 * final sync that pushes these changes out. 6408 */ 6409 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6410 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6411 spa->spa_state = new_state; 6412 spa->spa_final_txg = spa_last_synced_txg(spa) + 6413 TXG_DEFER_SIZE + 1; 6414 vdev_config_dirty(spa->spa_root_vdev); 6415 spa_config_exit(spa, SCL_ALL, FTAG); 6416 } 6417 } 6418 6419 export_spa: 6420 if (new_state == POOL_STATE_DESTROYED) 6421 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 6422 else if (new_state == POOL_STATE_EXPORTED) 6423 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 6424 6425 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 6426 spa_unload(spa); 6427 spa_deactivate(spa); 6428 } 6429 6430 if (oldconfig && spa->spa_config) 6431 *oldconfig = fnvlist_dup(spa->spa_config); 6432 6433 if (new_state != POOL_STATE_UNINITIALIZED) { 6434 if (!hardforce) 6435 spa_write_cachefile(spa, B_TRUE, B_TRUE); 6436 spa_remove(spa); 6437 } else { 6438 /* 6439 * If spa_remove() is not called for this spa_t and 6440 * there is any possibility that it can be reused, 6441 * we make sure to reset the exporting flag. 6442 */ 6443 spa->spa_is_exporting = B_FALSE; 6444 } 6445 6446 mutex_exit(&spa_namespace_lock); 6447 return (0); 6448 6449 fail: 6450 spa->spa_is_exporting = B_FALSE; 6451 spa_async_resume(spa); 6452 mutex_exit(&spa_namespace_lock); 6453 return (error); 6454 } 6455 6456 /* 6457 * Destroy a storage pool. 6458 */ 6459 int 6460 spa_destroy(const char *pool) 6461 { 6462 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 6463 B_FALSE, B_FALSE)); 6464 } 6465 6466 /* 6467 * Export a storage pool. 6468 */ 6469 int 6470 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 6471 boolean_t hardforce) 6472 { 6473 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 6474 force, hardforce)); 6475 } 6476 6477 /* 6478 * Similar to spa_export(), this unloads the spa_t without actually removing it 6479 * from the namespace in any way. 6480 */ 6481 int 6482 spa_reset(const char *pool) 6483 { 6484 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 6485 B_FALSE, B_FALSE)); 6486 } 6487 6488 /* 6489 * ========================================================================== 6490 * Device manipulation 6491 * ========================================================================== 6492 */ 6493 6494 /* 6495 * This is called as a synctask to increment the draid feature flag 6496 */ 6497 static void 6498 spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 6499 { 6500 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 6501 int draid = (int)(uintptr_t)arg; 6502 6503 for (int c = 0; c < draid; c++) 6504 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6505 } 6506 6507 /* 6508 * Add a device to a storage pool. 6509 */ 6510 int 6511 spa_vdev_add(spa_t *spa, nvlist_t *nvroot) 6512 { 6513 uint64_t txg, ndraid = 0; 6514 int error; 6515 vdev_t *rvd = spa->spa_root_vdev; 6516 vdev_t *vd, *tvd; 6517 nvlist_t **spares, **l2cache; 6518 uint_t nspares, nl2cache; 6519 6520 ASSERT(spa_writeable(spa)); 6521 6522 txg = spa_vdev_enter(spa); 6523 6524 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 6525 VDEV_ALLOC_ADD)) != 0) 6526 return (spa_vdev_exit(spa, NULL, txg, error)); 6527 6528 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 6529 6530 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 6531 &nspares) != 0) 6532 nspares = 0; 6533 6534 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 6535 &nl2cache) != 0) 6536 nl2cache = 0; 6537 6538 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 6539 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6540 6541 if (vd->vdev_children != 0 && 6542 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 6543 return (spa_vdev_exit(spa, vd, txg, error)); 6544 } 6545 6546 /* 6547 * The virtual dRAID spares must be added after vdev tree is created 6548 * and the vdev guids are generated. The guid of their associated 6549 * dRAID is stored in the config and used when opening the spare. 6550 */ 6551 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 6552 rvd->vdev_children)) == 0) { 6553 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 6554 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 6555 nspares = 0; 6556 } else { 6557 return (spa_vdev_exit(spa, vd, txg, error)); 6558 } 6559 6560 /* 6561 * We must validate the spares and l2cache devices after checking the 6562 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 6563 */ 6564 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 6565 return (spa_vdev_exit(spa, vd, txg, error)); 6566 6567 /* 6568 * If we are in the middle of a device removal, we can only add 6569 * devices which match the existing devices in the pool. 6570 * If we are in the middle of a removal, or have some indirect 6571 * vdevs, we can not add raidz or dRAID top levels. 6572 */ 6573 if (spa->spa_vdev_removal != NULL || 6574 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 6575 for (int c = 0; c < vd->vdev_children; c++) { 6576 tvd = vd->vdev_child[c]; 6577 if (spa->spa_vdev_removal != NULL && 6578 tvd->vdev_ashift != spa->spa_max_ashift) { 6579 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6580 } 6581 /* Fail if top level vdev is raidz or a dRAID */ 6582 if (vdev_get_nparity(tvd) != 0) 6583 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6584 6585 /* 6586 * Need the top level mirror to be 6587 * a mirror of leaf vdevs only 6588 */ 6589 if (tvd->vdev_ops == &vdev_mirror_ops) { 6590 for (uint64_t cid = 0; 6591 cid < tvd->vdev_children; cid++) { 6592 vdev_t *cvd = tvd->vdev_child[cid]; 6593 if (!cvd->vdev_ops->vdev_op_leaf) { 6594 return (spa_vdev_exit(spa, vd, 6595 txg, EINVAL)); 6596 } 6597 } 6598 } 6599 } 6600 } 6601 6602 for (int c = 0; c < vd->vdev_children; c++) { 6603 tvd = vd->vdev_child[c]; 6604 vdev_remove_child(vd, tvd); 6605 tvd->vdev_id = rvd->vdev_children; 6606 vdev_add_child(rvd, tvd); 6607 vdev_config_dirty(tvd); 6608 } 6609 6610 if (nspares != 0) { 6611 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 6612 ZPOOL_CONFIG_SPARES); 6613 spa_load_spares(spa); 6614 spa->spa_spares.sav_sync = B_TRUE; 6615 } 6616 6617 if (nl2cache != 0) { 6618 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 6619 ZPOOL_CONFIG_L2CACHE); 6620 spa_load_l2cache(spa); 6621 spa->spa_l2cache.sav_sync = B_TRUE; 6622 } 6623 6624 /* 6625 * We can't increment a feature while holding spa_vdev so we 6626 * have to do it in a synctask. 6627 */ 6628 if (ndraid != 0) { 6629 dmu_tx_t *tx; 6630 6631 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 6632 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 6633 (void *)(uintptr_t)ndraid, tx); 6634 dmu_tx_commit(tx); 6635 } 6636 6637 /* 6638 * We have to be careful when adding new vdevs to an existing pool. 6639 * If other threads start allocating from these vdevs before we 6640 * sync the config cache, and we lose power, then upon reboot we may 6641 * fail to open the pool because there are DVAs that the config cache 6642 * can't translate. Therefore, we first add the vdevs without 6643 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 6644 * and then let spa_config_update() initialize the new metaslabs. 6645 * 6646 * spa_load() checks for added-but-not-initialized vdevs, so that 6647 * if we lose power at any point in this sequence, the remaining 6648 * steps will be completed the next time we load the pool. 6649 */ 6650 (void) spa_vdev_exit(spa, vd, txg, 0); 6651 6652 mutex_enter(&spa_namespace_lock); 6653 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6654 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 6655 mutex_exit(&spa_namespace_lock); 6656 6657 return (0); 6658 } 6659 6660 /* 6661 * Attach a device to a mirror. The arguments are the path to any device 6662 * in the mirror, and the nvroot for the new device. If the path specifies 6663 * a device that is not mirrored, we automatically insert the mirror vdev. 6664 * 6665 * If 'replacing' is specified, the new device is intended to replace the 6666 * existing device; in this case the two devices are made into their own 6667 * mirror using the 'replacing' vdev, which is functionally identical to 6668 * the mirror vdev (it actually reuses all the same ops) but has a few 6669 * extra rules: you can't attach to it after it's been created, and upon 6670 * completion of resilvering, the first disk (the one being replaced) 6671 * is automatically detached. 6672 * 6673 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 6674 * should be performed instead of traditional healing reconstruction. From 6675 * an administrators perspective these are both resilver operations. 6676 */ 6677 int 6678 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 6679 int rebuild) 6680 { 6681 uint64_t txg, dtl_max_txg; 6682 vdev_t *rvd = spa->spa_root_vdev; 6683 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 6684 vdev_ops_t *pvops; 6685 char *oldvdpath, *newvdpath; 6686 int newvd_isspare; 6687 int error; 6688 6689 ASSERT(spa_writeable(spa)); 6690 6691 txg = spa_vdev_enter(spa); 6692 6693 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 6694 6695 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6696 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6697 error = (spa_has_checkpoint(spa)) ? 6698 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6699 return (spa_vdev_exit(spa, NULL, txg, error)); 6700 } 6701 6702 if (rebuild) { 6703 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 6704 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6705 6706 if (dsl_scan_resilvering(spa_get_dsl(spa))) 6707 return (spa_vdev_exit(spa, NULL, txg, 6708 ZFS_ERR_RESILVER_IN_PROGRESS)); 6709 } else { 6710 if (vdev_rebuild_active(rvd)) 6711 return (spa_vdev_exit(spa, NULL, txg, 6712 ZFS_ERR_REBUILD_IN_PROGRESS)); 6713 } 6714 6715 if (spa->spa_vdev_removal != NULL) 6716 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 6717 6718 if (oldvd == NULL) 6719 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6720 6721 if (!oldvd->vdev_ops->vdev_op_leaf) 6722 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6723 6724 pvd = oldvd->vdev_parent; 6725 6726 if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 6727 VDEV_ALLOC_ATTACH)) != 0) 6728 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 6729 6730 if (newrootvd->vdev_children != 1) 6731 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6732 6733 newvd = newrootvd->vdev_child[0]; 6734 6735 if (!newvd->vdev_ops->vdev_op_leaf) 6736 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6737 6738 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 6739 return (spa_vdev_exit(spa, newrootvd, txg, error)); 6740 6741 /* 6742 * Spares can't replace logs 6743 */ 6744 if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare) 6745 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6746 6747 /* 6748 * A dRAID spare can only replace a child of its parent dRAID vdev. 6749 */ 6750 if (newvd->vdev_ops == &vdev_draid_spare_ops && 6751 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 6752 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6753 } 6754 6755 if (rebuild) { 6756 /* 6757 * For rebuilds, the top vdev must support reconstruction 6758 * using only space maps. This means the only allowable 6759 * vdevs types are the root vdev, a mirror, or dRAID. 6760 */ 6761 tvd = pvd; 6762 if (pvd->vdev_top != NULL) 6763 tvd = pvd->vdev_top; 6764 6765 if (tvd->vdev_ops != &vdev_mirror_ops && 6766 tvd->vdev_ops != &vdev_root_ops && 6767 tvd->vdev_ops != &vdev_draid_ops) { 6768 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6769 } 6770 } 6771 6772 if (!replacing) { 6773 /* 6774 * For attach, the only allowable parent is a mirror or the root 6775 * vdev. 6776 */ 6777 if (pvd->vdev_ops != &vdev_mirror_ops && 6778 pvd->vdev_ops != &vdev_root_ops) 6779 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6780 6781 pvops = &vdev_mirror_ops; 6782 } else { 6783 /* 6784 * Active hot spares can only be replaced by inactive hot 6785 * spares. 6786 */ 6787 if (pvd->vdev_ops == &vdev_spare_ops && 6788 oldvd->vdev_isspare && 6789 !spa_has_spare(spa, newvd->vdev_guid)) 6790 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6791 6792 /* 6793 * If the source is a hot spare, and the parent isn't already a 6794 * spare, then we want to create a new hot spare. Otherwise, we 6795 * want to create a replacing vdev. The user is not allowed to 6796 * attach to a spared vdev child unless the 'isspare' state is 6797 * the same (spare replaces spare, non-spare replaces 6798 * non-spare). 6799 */ 6800 if (pvd->vdev_ops == &vdev_replacing_ops && 6801 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 6802 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6803 } else if (pvd->vdev_ops == &vdev_spare_ops && 6804 newvd->vdev_isspare != oldvd->vdev_isspare) { 6805 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6806 } 6807 6808 if (newvd->vdev_isspare) 6809 pvops = &vdev_spare_ops; 6810 else 6811 pvops = &vdev_replacing_ops; 6812 } 6813 6814 /* 6815 * Make sure the new device is big enough. 6816 */ 6817 if (newvd->vdev_asize < vdev_get_min_asize(oldvd)) 6818 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 6819 6820 /* 6821 * The new device cannot have a higher alignment requirement 6822 * than the top-level vdev. 6823 */ 6824 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) 6825 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6826 6827 /* 6828 * If this is an in-place replacement, update oldvd's path and devid 6829 * to make it distinguishable from newvd, and unopenable from now on. 6830 */ 6831 if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { 6832 spa_strfree(oldvd->vdev_path); 6833 oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, 6834 KM_SLEEP); 6835 (void) snprintf(oldvd->vdev_path, strlen(newvd->vdev_path) + 5, 6836 "%s/%s", newvd->vdev_path, "old"); 6837 if (oldvd->vdev_devid != NULL) { 6838 spa_strfree(oldvd->vdev_devid); 6839 oldvd->vdev_devid = NULL; 6840 } 6841 } 6842 6843 /* 6844 * If the parent is not a mirror, or if we're replacing, insert the new 6845 * mirror/replacing/spare vdev above oldvd. 6846 */ 6847 if (pvd->vdev_ops != pvops) 6848 pvd = vdev_add_parent(oldvd, pvops); 6849 6850 ASSERT(pvd->vdev_top->vdev_parent == rvd); 6851 ASSERT(pvd->vdev_ops == pvops); 6852 ASSERT(oldvd->vdev_parent == pvd); 6853 6854 /* 6855 * Extract the new device from its root and add it to pvd. 6856 */ 6857 vdev_remove_child(newrootvd, newvd); 6858 newvd->vdev_id = pvd->vdev_children; 6859 newvd->vdev_crtxg = oldvd->vdev_crtxg; 6860 vdev_add_child(pvd, newvd); 6861 6862 /* 6863 * Reevaluate the parent vdev state. 6864 */ 6865 vdev_propagate_state(pvd); 6866 6867 tvd = newvd->vdev_top; 6868 ASSERT(pvd->vdev_top == tvd); 6869 ASSERT(tvd->vdev_parent == rvd); 6870 6871 vdev_config_dirty(tvd); 6872 6873 /* 6874 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 6875 * for any dmu_sync-ed blocks. It will propagate upward when 6876 * spa_vdev_exit() calls vdev_dtl_reassess(). 6877 */ 6878 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 6879 6880 vdev_dtl_dirty(newvd, DTL_MISSING, 6881 TXG_INITIAL, dtl_max_txg - TXG_INITIAL); 6882 6883 if (newvd->vdev_isspare) { 6884 spa_spare_activate(newvd); 6885 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 6886 } 6887 6888 oldvdpath = spa_strdup(oldvd->vdev_path); 6889 newvdpath = spa_strdup(newvd->vdev_path); 6890 newvd_isspare = newvd->vdev_isspare; 6891 6892 /* 6893 * Mark newvd's DTL dirty in this txg. 6894 */ 6895 vdev_dirty(tvd, VDD_DTL, newvd, txg); 6896 6897 /* 6898 * Schedule the resilver or rebuild to restart in the future. We do 6899 * this to ensure that dmu_sync-ed blocks have been stitched into the 6900 * respective datasets. 6901 */ 6902 if (rebuild) { 6903 newvd->vdev_rebuild_txg = txg; 6904 6905 vdev_rebuild(tvd); 6906 } else { 6907 newvd->vdev_resilver_txg = txg; 6908 6909 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 6910 spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) { 6911 vdev_defer_resilver(newvd); 6912 } else { 6913 dsl_scan_restart_resilver(spa->spa_dsl_pool, 6914 dtl_max_txg); 6915 } 6916 } 6917 6918 if (spa->spa_bootfs) 6919 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 6920 6921 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 6922 6923 /* 6924 * Commit the config 6925 */ 6926 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 6927 6928 spa_history_log_internal(spa, "vdev attach", NULL, 6929 "%s vdev=%s %s vdev=%s", 6930 replacing && newvd_isspare ? "spare in" : 6931 replacing ? "replace" : "attach", newvdpath, 6932 replacing ? "for" : "to", oldvdpath); 6933 6934 spa_strfree(oldvdpath); 6935 spa_strfree(newvdpath); 6936 6937 return (0); 6938 } 6939 6940 /* 6941 * Detach a device from a mirror or replacing vdev. 6942 * 6943 * If 'replace_done' is specified, only detach if the parent 6944 * is a replacing vdev. 6945 */ 6946 int 6947 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 6948 { 6949 uint64_t txg; 6950 int error; 6951 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 6952 vdev_t *vd, *pvd, *cvd, *tvd; 6953 boolean_t unspare = B_FALSE; 6954 uint64_t unspare_guid = 0; 6955 char *vdpath; 6956 6957 ASSERT(spa_writeable(spa)); 6958 6959 txg = spa_vdev_detach_enter(spa, guid); 6960 6961 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 6962 6963 /* 6964 * Besides being called directly from the userland through the 6965 * ioctl interface, spa_vdev_detach() can be potentially called 6966 * at the end of spa_vdev_resilver_done(). 6967 * 6968 * In the regular case, when we have a checkpoint this shouldn't 6969 * happen as we never empty the DTLs of a vdev during the scrub 6970 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 6971 * should never get here when we have a checkpoint. 6972 * 6973 * That said, even in a case when we checkpoint the pool exactly 6974 * as spa_vdev_resilver_done() calls this function everything 6975 * should be fine as the resilver will return right away. 6976 */ 6977 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6978 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6979 error = (spa_has_checkpoint(spa)) ? 6980 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6981 return (spa_vdev_exit(spa, NULL, txg, error)); 6982 } 6983 6984 if (vd == NULL) 6985 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6986 6987 if (!vd->vdev_ops->vdev_op_leaf) 6988 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6989 6990 pvd = vd->vdev_parent; 6991 6992 /* 6993 * If the parent/child relationship is not as expected, don't do it. 6994 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 6995 * vdev that's replacing B with C. The user's intent in replacing 6996 * is to go from M(A,B) to M(A,C). If the user decides to cancel 6997 * the replace by detaching C, the expected behavior is to end up 6998 * M(A,B). But suppose that right after deciding to detach C, 6999 * the replacement of B completes. We would have M(A,C), and then 7000 * ask to detach C, which would leave us with just A -- not what 7001 * the user wanted. To prevent this, we make sure that the 7002 * parent/child relationship hasn't changed -- in this example, 7003 * that C's parent is still the replacing vdev R. 7004 */ 7005 if (pvd->vdev_guid != pguid && pguid != 0) 7006 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7007 7008 /* 7009 * Only 'replacing' or 'spare' vdevs can be replaced. 7010 */ 7011 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 7012 pvd->vdev_ops != &vdev_spare_ops) 7013 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7014 7015 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 7016 spa_version(spa) >= SPA_VERSION_SPARES); 7017 7018 /* 7019 * Only mirror, replacing, and spare vdevs support detach. 7020 */ 7021 if (pvd->vdev_ops != &vdev_replacing_ops && 7022 pvd->vdev_ops != &vdev_mirror_ops && 7023 pvd->vdev_ops != &vdev_spare_ops) 7024 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7025 7026 /* 7027 * If this device has the only valid copy of some data, 7028 * we cannot safely detach it. 7029 */ 7030 if (vdev_dtl_required(vd)) 7031 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7032 7033 ASSERT(pvd->vdev_children >= 2); 7034 7035 /* 7036 * If we are detaching the second disk from a replacing vdev, then 7037 * check to see if we changed the original vdev's path to have "/old" 7038 * at the end in spa_vdev_attach(). If so, undo that change now. 7039 */ 7040 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 7041 vd->vdev_path != NULL) { 7042 size_t len = strlen(vd->vdev_path); 7043 7044 for (int c = 0; c < pvd->vdev_children; c++) { 7045 cvd = pvd->vdev_child[c]; 7046 7047 if (cvd == vd || cvd->vdev_path == NULL) 7048 continue; 7049 7050 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 7051 strcmp(cvd->vdev_path + len, "/old") == 0) { 7052 spa_strfree(cvd->vdev_path); 7053 cvd->vdev_path = spa_strdup(vd->vdev_path); 7054 break; 7055 } 7056 } 7057 } 7058 7059 /* 7060 * If we are detaching the original disk from a normal spare, then it 7061 * implies that the spare should become a real disk, and be removed 7062 * from the active spare list for the pool. dRAID spares on the 7063 * other hand are coupled to the pool and thus should never be removed 7064 * from the spares list. 7065 */ 7066 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 7067 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7068 7069 if (last_cvd->vdev_isspare && 7070 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 7071 unspare = B_TRUE; 7072 } 7073 } 7074 7075 /* 7076 * Erase the disk labels so the disk can be used for other things. 7077 * This must be done after all other error cases are handled, 7078 * but before we disembowel vd (so we can still do I/O to it). 7079 * But if we can't do it, don't treat the error as fatal -- 7080 * it may be that the unwritability of the disk is the reason 7081 * it's being detached! 7082 */ 7083 error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 7084 7085 /* 7086 * Remove vd from its parent and compact the parent's children. 7087 */ 7088 vdev_remove_child(pvd, vd); 7089 vdev_compact_children(pvd); 7090 7091 /* 7092 * Remember one of the remaining children so we can get tvd below. 7093 */ 7094 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7095 7096 /* 7097 * If we need to remove the remaining child from the list of hot spares, 7098 * do it now, marking the vdev as no longer a spare in the process. 7099 * We must do this before vdev_remove_parent(), because that can 7100 * change the GUID if it creates a new toplevel GUID. For a similar 7101 * reason, we must remove the spare now, in the same txg as the detach; 7102 * otherwise someone could attach a new sibling, change the GUID, and 7103 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 7104 */ 7105 if (unspare) { 7106 ASSERT(cvd->vdev_isspare); 7107 spa_spare_remove(cvd); 7108 unspare_guid = cvd->vdev_guid; 7109 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 7110 cvd->vdev_unspare = B_TRUE; 7111 } 7112 7113 /* 7114 * If the parent mirror/replacing vdev only has one child, 7115 * the parent is no longer needed. Remove it from the tree. 7116 */ 7117 if (pvd->vdev_children == 1) { 7118 if (pvd->vdev_ops == &vdev_spare_ops) 7119 cvd->vdev_unspare = B_FALSE; 7120 vdev_remove_parent(cvd); 7121 } 7122 7123 /* 7124 * We don't set tvd until now because the parent we just removed 7125 * may have been the previous top-level vdev. 7126 */ 7127 tvd = cvd->vdev_top; 7128 ASSERT(tvd->vdev_parent == rvd); 7129 7130 /* 7131 * Reevaluate the parent vdev state. 7132 */ 7133 vdev_propagate_state(cvd); 7134 7135 /* 7136 * If the 'autoexpand' property is set on the pool then automatically 7137 * try to expand the size of the pool. For example if the device we 7138 * just detached was smaller than the others, it may be possible to 7139 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 7140 * first so that we can obtain the updated sizes of the leaf vdevs. 7141 */ 7142 if (spa->spa_autoexpand) { 7143 vdev_reopen(tvd); 7144 vdev_expand(tvd, txg); 7145 } 7146 7147 vdev_config_dirty(tvd); 7148 7149 /* 7150 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 7151 * vd->vdev_detached is set and free vd's DTL object in syncing context. 7152 * But first make sure we're not on any *other* txg's DTL list, to 7153 * prevent vd from being accessed after it's freed. 7154 */ 7155 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 7156 for (int t = 0; t < TXG_SIZE; t++) 7157 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 7158 vd->vdev_detached = B_TRUE; 7159 vdev_dirty(tvd, VDD_DTL, vd, txg); 7160 7161 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 7162 spa_notify_waiters(spa); 7163 7164 /* hang on to the spa before we release the lock */ 7165 spa_open_ref(spa, FTAG); 7166 7167 error = spa_vdev_exit(spa, vd, txg, 0); 7168 7169 spa_history_log_internal(spa, "detach", NULL, 7170 "vdev=%s", vdpath); 7171 spa_strfree(vdpath); 7172 7173 /* 7174 * If this was the removal of the original device in a hot spare vdev, 7175 * then we want to go through and remove the device from the hot spare 7176 * list of every other pool. 7177 */ 7178 if (unspare) { 7179 spa_t *altspa = NULL; 7180 7181 mutex_enter(&spa_namespace_lock); 7182 while ((altspa = spa_next(altspa)) != NULL) { 7183 if (altspa->spa_state != POOL_STATE_ACTIVE || 7184 altspa == spa) 7185 continue; 7186 7187 spa_open_ref(altspa, FTAG); 7188 mutex_exit(&spa_namespace_lock); 7189 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 7190 mutex_enter(&spa_namespace_lock); 7191 spa_close(altspa, FTAG); 7192 } 7193 mutex_exit(&spa_namespace_lock); 7194 7195 /* search the rest of the vdevs for spares to remove */ 7196 spa_vdev_resilver_done(spa); 7197 } 7198 7199 /* all done with the spa; OK to release */ 7200 mutex_enter(&spa_namespace_lock); 7201 spa_close(spa, FTAG); 7202 mutex_exit(&spa_namespace_lock); 7203 7204 return (error); 7205 } 7206 7207 static int 7208 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7209 list_t *vd_list) 7210 { 7211 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7212 7213 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7214 7215 /* Look up vdev and ensure it's a leaf. */ 7216 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7217 if (vd == NULL || vd->vdev_detached) { 7218 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7219 return (SET_ERROR(ENODEV)); 7220 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7221 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7222 return (SET_ERROR(EINVAL)); 7223 } else if (!vdev_writeable(vd)) { 7224 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7225 return (SET_ERROR(EROFS)); 7226 } 7227 mutex_enter(&vd->vdev_initialize_lock); 7228 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7229 7230 /* 7231 * When we activate an initialize action we check to see 7232 * if the vdev_initialize_thread is NULL. We do this instead 7233 * of using the vdev_initialize_state since there might be 7234 * a previous initialization process which has completed but 7235 * the thread is not exited. 7236 */ 7237 if (cmd_type == POOL_INITIALIZE_START && 7238 (vd->vdev_initialize_thread != NULL || 7239 vd->vdev_top->vdev_removing)) { 7240 mutex_exit(&vd->vdev_initialize_lock); 7241 return (SET_ERROR(EBUSY)); 7242 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 7243 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 7244 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 7245 mutex_exit(&vd->vdev_initialize_lock); 7246 return (SET_ERROR(ESRCH)); 7247 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 7248 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 7249 mutex_exit(&vd->vdev_initialize_lock); 7250 return (SET_ERROR(ESRCH)); 7251 } 7252 7253 switch (cmd_type) { 7254 case POOL_INITIALIZE_START: 7255 vdev_initialize(vd); 7256 break; 7257 case POOL_INITIALIZE_CANCEL: 7258 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 7259 break; 7260 case POOL_INITIALIZE_SUSPEND: 7261 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 7262 break; 7263 default: 7264 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7265 } 7266 mutex_exit(&vd->vdev_initialize_lock); 7267 7268 return (0); 7269 } 7270 7271 int 7272 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 7273 nvlist_t *vdev_errlist) 7274 { 7275 int total_errors = 0; 7276 list_t vd_list; 7277 7278 list_create(&vd_list, sizeof (vdev_t), 7279 offsetof(vdev_t, vdev_initialize_node)); 7280 7281 /* 7282 * We hold the namespace lock through the whole function 7283 * to prevent any changes to the pool while we're starting or 7284 * stopping initialization. The config and state locks are held so that 7285 * we can properly assess the vdev state before we commit to 7286 * the initializing operation. 7287 */ 7288 mutex_enter(&spa_namespace_lock); 7289 7290 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7291 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7292 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7293 7294 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 7295 &vd_list); 7296 if (error != 0) { 7297 char guid_as_str[MAXNAMELEN]; 7298 7299 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7300 "%llu", (unsigned long long)vdev_guid); 7301 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7302 total_errors++; 7303 } 7304 } 7305 7306 /* Wait for all initialize threads to stop. */ 7307 vdev_initialize_stop_wait(spa, &vd_list); 7308 7309 /* Sync out the initializing state */ 7310 txg_wait_synced(spa->spa_dsl_pool, 0); 7311 mutex_exit(&spa_namespace_lock); 7312 7313 list_destroy(&vd_list); 7314 7315 return (total_errors); 7316 } 7317 7318 static int 7319 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7320 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 7321 { 7322 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7323 7324 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7325 7326 /* Look up vdev and ensure it's a leaf. */ 7327 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7328 if (vd == NULL || vd->vdev_detached) { 7329 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7330 return (SET_ERROR(ENODEV)); 7331 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7332 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7333 return (SET_ERROR(EINVAL)); 7334 } else if (!vdev_writeable(vd)) { 7335 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7336 return (SET_ERROR(EROFS)); 7337 } else if (!vd->vdev_has_trim) { 7338 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7339 return (SET_ERROR(EOPNOTSUPP)); 7340 } else if (secure && !vd->vdev_has_securetrim) { 7341 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7342 return (SET_ERROR(EOPNOTSUPP)); 7343 } 7344 mutex_enter(&vd->vdev_trim_lock); 7345 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7346 7347 /* 7348 * When we activate a TRIM action we check to see if the 7349 * vdev_trim_thread is NULL. We do this instead of using the 7350 * vdev_trim_state since there might be a previous TRIM process 7351 * which has completed but the thread is not exited. 7352 */ 7353 if (cmd_type == POOL_TRIM_START && 7354 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing)) { 7355 mutex_exit(&vd->vdev_trim_lock); 7356 return (SET_ERROR(EBUSY)); 7357 } else if (cmd_type == POOL_TRIM_CANCEL && 7358 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 7359 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 7360 mutex_exit(&vd->vdev_trim_lock); 7361 return (SET_ERROR(ESRCH)); 7362 } else if (cmd_type == POOL_TRIM_SUSPEND && 7363 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 7364 mutex_exit(&vd->vdev_trim_lock); 7365 return (SET_ERROR(ESRCH)); 7366 } 7367 7368 switch (cmd_type) { 7369 case POOL_TRIM_START: 7370 vdev_trim(vd, rate, partial, secure); 7371 break; 7372 case POOL_TRIM_CANCEL: 7373 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 7374 break; 7375 case POOL_TRIM_SUSPEND: 7376 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 7377 break; 7378 default: 7379 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7380 } 7381 mutex_exit(&vd->vdev_trim_lock); 7382 7383 return (0); 7384 } 7385 7386 /* 7387 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 7388 * TRIM threads for each child vdev. These threads pass over all of the free 7389 * space in the vdev's metaslabs and issues TRIM commands for that space. 7390 */ 7391 int 7392 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 7393 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 7394 { 7395 int total_errors = 0; 7396 list_t vd_list; 7397 7398 list_create(&vd_list, sizeof (vdev_t), 7399 offsetof(vdev_t, vdev_trim_node)); 7400 7401 /* 7402 * We hold the namespace lock through the whole function 7403 * to prevent any changes to the pool while we're starting or 7404 * stopping TRIM. The config and state locks are held so that 7405 * we can properly assess the vdev state before we commit to 7406 * the TRIM operation. 7407 */ 7408 mutex_enter(&spa_namespace_lock); 7409 7410 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7411 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7412 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7413 7414 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 7415 rate, partial, secure, &vd_list); 7416 if (error != 0) { 7417 char guid_as_str[MAXNAMELEN]; 7418 7419 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7420 "%llu", (unsigned long long)vdev_guid); 7421 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7422 total_errors++; 7423 } 7424 } 7425 7426 /* Wait for all TRIM threads to stop. */ 7427 vdev_trim_stop_wait(spa, &vd_list); 7428 7429 /* Sync out the TRIM state */ 7430 txg_wait_synced(spa->spa_dsl_pool, 0); 7431 mutex_exit(&spa_namespace_lock); 7432 7433 list_destroy(&vd_list); 7434 7435 return (total_errors); 7436 } 7437 7438 /* 7439 * Split a set of devices from their mirrors, and create a new pool from them. 7440 */ 7441 int 7442 spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, 7443 nvlist_t *props, boolean_t exp) 7444 { 7445 int error = 0; 7446 uint64_t txg, *glist; 7447 spa_t *newspa; 7448 uint_t c, children, lastlog; 7449 nvlist_t **child, *nvl, *tmp; 7450 dmu_tx_t *tx; 7451 char *altroot = NULL; 7452 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 7453 boolean_t activate_slog; 7454 7455 ASSERT(spa_writeable(spa)); 7456 7457 txg = spa_vdev_enter(spa); 7458 7459 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7460 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7461 error = (spa_has_checkpoint(spa)) ? 7462 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7463 return (spa_vdev_exit(spa, NULL, txg, error)); 7464 } 7465 7466 /* clear the log and flush everything up to now */ 7467 activate_slog = spa_passivate_log(spa); 7468 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7469 error = spa_reset_logs(spa); 7470 txg = spa_vdev_config_enter(spa); 7471 7472 if (activate_slog) 7473 spa_activate_log(spa); 7474 7475 if (error != 0) 7476 return (spa_vdev_exit(spa, NULL, txg, error)); 7477 7478 /* check new spa name before going any further */ 7479 if (spa_lookup(newname) != NULL) 7480 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 7481 7482 /* 7483 * scan through all the children to ensure they're all mirrors 7484 */ 7485 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 7486 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 7487 &children) != 0) 7488 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7489 7490 /* first, check to ensure we've got the right child count */ 7491 rvd = spa->spa_root_vdev; 7492 lastlog = 0; 7493 for (c = 0; c < rvd->vdev_children; c++) { 7494 vdev_t *vd = rvd->vdev_child[c]; 7495 7496 /* don't count the holes & logs as children */ 7497 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 7498 !vdev_is_concrete(vd))) { 7499 if (lastlog == 0) 7500 lastlog = c; 7501 continue; 7502 } 7503 7504 lastlog = 0; 7505 } 7506 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 7507 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7508 7509 /* next, ensure no spare or cache devices are part of the split */ 7510 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 7511 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 7512 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7513 7514 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 7515 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 7516 7517 /* then, loop over each vdev and validate it */ 7518 for (c = 0; c < children; c++) { 7519 uint64_t is_hole = 0; 7520 7521 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 7522 &is_hole); 7523 7524 if (is_hole != 0) { 7525 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 7526 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 7527 continue; 7528 } else { 7529 error = SET_ERROR(EINVAL); 7530 break; 7531 } 7532 } 7533 7534 /* deal with indirect vdevs */ 7535 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 7536 &vdev_indirect_ops) 7537 continue; 7538 7539 /* which disk is going to be split? */ 7540 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 7541 &glist[c]) != 0) { 7542 error = SET_ERROR(EINVAL); 7543 break; 7544 } 7545 7546 /* look it up in the spa */ 7547 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 7548 if (vml[c] == NULL) { 7549 error = SET_ERROR(ENODEV); 7550 break; 7551 } 7552 7553 /* make sure there's nothing stopping the split */ 7554 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 7555 vml[c]->vdev_islog || 7556 !vdev_is_concrete(vml[c]) || 7557 vml[c]->vdev_isspare || 7558 vml[c]->vdev_isl2cache || 7559 !vdev_writeable(vml[c]) || 7560 vml[c]->vdev_children != 0 || 7561 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 7562 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 7563 error = SET_ERROR(EINVAL); 7564 break; 7565 } 7566 7567 if (vdev_dtl_required(vml[c]) || 7568 vdev_resilver_needed(vml[c], NULL, NULL)) { 7569 error = SET_ERROR(EBUSY); 7570 break; 7571 } 7572 7573 /* we need certain info from the top level */ 7574 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 7575 vml[c]->vdev_top->vdev_ms_array); 7576 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 7577 vml[c]->vdev_top->vdev_ms_shift); 7578 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 7579 vml[c]->vdev_top->vdev_asize); 7580 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 7581 vml[c]->vdev_top->vdev_ashift); 7582 7583 /* transfer per-vdev ZAPs */ 7584 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 7585 VERIFY0(nvlist_add_uint64(child[c], 7586 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 7587 7588 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 7589 VERIFY0(nvlist_add_uint64(child[c], 7590 ZPOOL_CONFIG_VDEV_TOP_ZAP, 7591 vml[c]->vdev_parent->vdev_top_zap)); 7592 } 7593 7594 if (error != 0) { 7595 kmem_free(vml, children * sizeof (vdev_t *)); 7596 kmem_free(glist, children * sizeof (uint64_t)); 7597 return (spa_vdev_exit(spa, NULL, txg, error)); 7598 } 7599 7600 /* stop writers from using the disks */ 7601 for (c = 0; c < children; c++) { 7602 if (vml[c] != NULL) 7603 vml[c]->vdev_offline = B_TRUE; 7604 } 7605 vdev_reopen(spa->spa_root_vdev); 7606 7607 /* 7608 * Temporarily record the splitting vdevs in the spa config. This 7609 * will disappear once the config is regenerated. 7610 */ 7611 nvl = fnvlist_alloc(); 7612 fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children); 7613 kmem_free(glist, children * sizeof (uint64_t)); 7614 7615 mutex_enter(&spa->spa_props_lock); 7616 fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl); 7617 mutex_exit(&spa->spa_props_lock); 7618 spa->spa_config_splitting = nvl; 7619 vdev_config_dirty(spa->spa_root_vdev); 7620 7621 /* configure and create the new pool */ 7622 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname); 7623 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 7624 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE); 7625 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)); 7626 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg); 7627 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 7628 spa_generate_guid(NULL)); 7629 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 7630 (void) nvlist_lookup_string(props, 7631 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 7632 7633 /* add the new pool to the namespace */ 7634 newspa = spa_add(newname, config, altroot); 7635 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 7636 newspa->spa_config_txg = spa->spa_config_txg; 7637 spa_set_log_state(newspa, SPA_LOG_CLEAR); 7638 7639 /* release the spa config lock, retaining the namespace lock */ 7640 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7641 7642 if (zio_injection_enabled) 7643 zio_handle_panic_injection(spa, FTAG, 1); 7644 7645 spa_activate(newspa, spa_mode_global); 7646 spa_async_suspend(newspa); 7647 7648 /* 7649 * Temporarily stop the initializing and TRIM activity. We set the 7650 * state to ACTIVE so that we know to resume initializing or TRIM 7651 * once the split has completed. 7652 */ 7653 list_t vd_initialize_list; 7654 list_create(&vd_initialize_list, sizeof (vdev_t), 7655 offsetof(vdev_t, vdev_initialize_node)); 7656 7657 list_t vd_trim_list; 7658 list_create(&vd_trim_list, sizeof (vdev_t), 7659 offsetof(vdev_t, vdev_trim_node)); 7660 7661 for (c = 0; c < children; c++) { 7662 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7663 mutex_enter(&vml[c]->vdev_initialize_lock); 7664 vdev_initialize_stop(vml[c], 7665 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 7666 mutex_exit(&vml[c]->vdev_initialize_lock); 7667 7668 mutex_enter(&vml[c]->vdev_trim_lock); 7669 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 7670 mutex_exit(&vml[c]->vdev_trim_lock); 7671 } 7672 } 7673 7674 vdev_initialize_stop_wait(spa, &vd_initialize_list); 7675 vdev_trim_stop_wait(spa, &vd_trim_list); 7676 7677 list_destroy(&vd_initialize_list); 7678 list_destroy(&vd_trim_list); 7679 7680 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 7681 newspa->spa_is_splitting = B_TRUE; 7682 7683 /* create the new pool from the disks of the original pool */ 7684 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 7685 if (error) 7686 goto out; 7687 7688 /* if that worked, generate a real config for the new pool */ 7689 if (newspa->spa_root_vdev != NULL) { 7690 newspa->spa_config_splitting = fnvlist_alloc(); 7691 fnvlist_add_uint64(newspa->spa_config_splitting, 7692 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)); 7693 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 7694 B_TRUE)); 7695 } 7696 7697 /* set the props */ 7698 if (props != NULL) { 7699 spa_configfile_set(newspa, props, B_FALSE); 7700 error = spa_prop_set(newspa, props); 7701 if (error) 7702 goto out; 7703 } 7704 7705 /* flush everything */ 7706 txg = spa_vdev_config_enter(newspa); 7707 vdev_config_dirty(newspa->spa_root_vdev); 7708 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 7709 7710 if (zio_injection_enabled) 7711 zio_handle_panic_injection(spa, FTAG, 2); 7712 7713 spa_async_resume(newspa); 7714 7715 /* finally, update the original pool's config */ 7716 txg = spa_vdev_config_enter(spa); 7717 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 7718 error = dmu_tx_assign(tx, TXG_WAIT); 7719 if (error != 0) 7720 dmu_tx_abort(tx); 7721 for (c = 0; c < children; c++) { 7722 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7723 vdev_t *tvd = vml[c]->vdev_top; 7724 7725 /* 7726 * Need to be sure the detachable VDEV is not 7727 * on any *other* txg's DTL list to prevent it 7728 * from being accessed after it's freed. 7729 */ 7730 for (int t = 0; t < TXG_SIZE; t++) { 7731 (void) txg_list_remove_this( 7732 &tvd->vdev_dtl_list, vml[c], t); 7733 } 7734 7735 vdev_split(vml[c]); 7736 if (error == 0) 7737 spa_history_log_internal(spa, "detach", tx, 7738 "vdev=%s", vml[c]->vdev_path); 7739 7740 vdev_free(vml[c]); 7741 } 7742 } 7743 spa->spa_avz_action = AVZ_ACTION_REBUILD; 7744 vdev_config_dirty(spa->spa_root_vdev); 7745 spa->spa_config_splitting = NULL; 7746 nvlist_free(nvl); 7747 if (error == 0) 7748 dmu_tx_commit(tx); 7749 (void) spa_vdev_exit(spa, NULL, txg, 0); 7750 7751 if (zio_injection_enabled) 7752 zio_handle_panic_injection(spa, FTAG, 3); 7753 7754 /* split is complete; log a history record */ 7755 spa_history_log_internal(newspa, "split", NULL, 7756 "from pool %s", spa_name(spa)); 7757 7758 newspa->spa_is_splitting = B_FALSE; 7759 kmem_free(vml, children * sizeof (vdev_t *)); 7760 7761 /* if we're not going to mount the filesystems in userland, export */ 7762 if (exp) 7763 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 7764 B_FALSE, B_FALSE); 7765 7766 return (error); 7767 7768 out: 7769 spa_unload(newspa); 7770 spa_deactivate(newspa); 7771 spa_remove(newspa); 7772 7773 txg = spa_vdev_config_enter(spa); 7774 7775 /* re-online all offlined disks */ 7776 for (c = 0; c < children; c++) { 7777 if (vml[c] != NULL) 7778 vml[c]->vdev_offline = B_FALSE; 7779 } 7780 7781 /* restart initializing or trimming disks as necessary */ 7782 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 7783 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 7784 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 7785 7786 vdev_reopen(spa->spa_root_vdev); 7787 7788 nvlist_free(spa->spa_config_splitting); 7789 spa->spa_config_splitting = NULL; 7790 (void) spa_vdev_exit(spa, NULL, txg, error); 7791 7792 kmem_free(vml, children * sizeof (vdev_t *)); 7793 return (error); 7794 } 7795 7796 /* 7797 * Find any device that's done replacing, or a vdev marked 'unspare' that's 7798 * currently spared, so we can detach it. 7799 */ 7800 static vdev_t * 7801 spa_vdev_resilver_done_hunt(vdev_t *vd) 7802 { 7803 vdev_t *newvd, *oldvd; 7804 7805 for (int c = 0; c < vd->vdev_children; c++) { 7806 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 7807 if (oldvd != NULL) 7808 return (oldvd); 7809 } 7810 7811 /* 7812 * Check for a completed replacement. We always consider the first 7813 * vdev in the list to be the oldest vdev, and the last one to be 7814 * the newest (see spa_vdev_attach() for how that works). In 7815 * the case where the newest vdev is faulted, we will not automatically 7816 * remove it after a resilver completes. This is OK as it will require 7817 * user intervention to determine which disk the admin wishes to keep. 7818 */ 7819 if (vd->vdev_ops == &vdev_replacing_ops) { 7820 ASSERT(vd->vdev_children > 1); 7821 7822 newvd = vd->vdev_child[vd->vdev_children - 1]; 7823 oldvd = vd->vdev_child[0]; 7824 7825 if (vdev_dtl_empty(newvd, DTL_MISSING) && 7826 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7827 !vdev_dtl_required(oldvd)) 7828 return (oldvd); 7829 } 7830 7831 /* 7832 * Check for a completed resilver with the 'unspare' flag set. 7833 * Also potentially update faulted state. 7834 */ 7835 if (vd->vdev_ops == &vdev_spare_ops) { 7836 vdev_t *first = vd->vdev_child[0]; 7837 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 7838 7839 if (last->vdev_unspare) { 7840 oldvd = first; 7841 newvd = last; 7842 } else if (first->vdev_unspare) { 7843 oldvd = last; 7844 newvd = first; 7845 } else { 7846 oldvd = NULL; 7847 } 7848 7849 if (oldvd != NULL && 7850 vdev_dtl_empty(newvd, DTL_MISSING) && 7851 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7852 !vdev_dtl_required(oldvd)) 7853 return (oldvd); 7854 7855 vdev_propagate_state(vd); 7856 7857 /* 7858 * If there are more than two spares attached to a disk, 7859 * and those spares are not required, then we want to 7860 * attempt to free them up now so that they can be used 7861 * by other pools. Once we're back down to a single 7862 * disk+spare, we stop removing them. 7863 */ 7864 if (vd->vdev_children > 2) { 7865 newvd = vd->vdev_child[1]; 7866 7867 if (newvd->vdev_isspare && last->vdev_isspare && 7868 vdev_dtl_empty(last, DTL_MISSING) && 7869 vdev_dtl_empty(last, DTL_OUTAGE) && 7870 !vdev_dtl_required(newvd)) 7871 return (newvd); 7872 } 7873 } 7874 7875 return (NULL); 7876 } 7877 7878 static void 7879 spa_vdev_resilver_done(spa_t *spa) 7880 { 7881 vdev_t *vd, *pvd, *ppvd; 7882 uint64_t guid, sguid, pguid, ppguid; 7883 7884 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7885 7886 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 7887 pvd = vd->vdev_parent; 7888 ppvd = pvd->vdev_parent; 7889 guid = vd->vdev_guid; 7890 pguid = pvd->vdev_guid; 7891 ppguid = ppvd->vdev_guid; 7892 sguid = 0; 7893 /* 7894 * If we have just finished replacing a hot spared device, then 7895 * we need to detach the parent's first child (the original hot 7896 * spare) as well. 7897 */ 7898 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 7899 ppvd->vdev_children == 2) { 7900 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 7901 sguid = ppvd->vdev_child[1]->vdev_guid; 7902 } 7903 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 7904 7905 spa_config_exit(spa, SCL_ALL, FTAG); 7906 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 7907 return; 7908 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 7909 return; 7910 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7911 } 7912 7913 spa_config_exit(spa, SCL_ALL, FTAG); 7914 7915 /* 7916 * If a detach was not performed above replace waiters will not have 7917 * been notified. In which case we must do so now. 7918 */ 7919 spa_notify_waiters(spa); 7920 } 7921 7922 /* 7923 * Update the stored path or FRU for this vdev. 7924 */ 7925 static int 7926 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 7927 boolean_t ispath) 7928 { 7929 vdev_t *vd; 7930 boolean_t sync = B_FALSE; 7931 7932 ASSERT(spa_writeable(spa)); 7933 7934 spa_vdev_state_enter(spa, SCL_ALL); 7935 7936 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 7937 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 7938 7939 if (!vd->vdev_ops->vdev_op_leaf) 7940 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 7941 7942 if (ispath) { 7943 if (strcmp(value, vd->vdev_path) != 0) { 7944 spa_strfree(vd->vdev_path); 7945 vd->vdev_path = spa_strdup(value); 7946 sync = B_TRUE; 7947 } 7948 } else { 7949 if (vd->vdev_fru == NULL) { 7950 vd->vdev_fru = spa_strdup(value); 7951 sync = B_TRUE; 7952 } else if (strcmp(value, vd->vdev_fru) != 0) { 7953 spa_strfree(vd->vdev_fru); 7954 vd->vdev_fru = spa_strdup(value); 7955 sync = B_TRUE; 7956 } 7957 } 7958 7959 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 7960 } 7961 7962 int 7963 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 7964 { 7965 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 7966 } 7967 7968 int 7969 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 7970 { 7971 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 7972 } 7973 7974 /* 7975 * ========================================================================== 7976 * SPA Scanning 7977 * ========================================================================== 7978 */ 7979 int 7980 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 7981 { 7982 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 7983 7984 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 7985 return (SET_ERROR(EBUSY)); 7986 7987 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 7988 } 7989 7990 int 7991 spa_scan_stop(spa_t *spa) 7992 { 7993 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 7994 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 7995 return (SET_ERROR(EBUSY)); 7996 return (dsl_scan_cancel(spa->spa_dsl_pool)); 7997 } 7998 7999 int 8000 spa_scan(spa_t *spa, pool_scan_func_t func) 8001 { 8002 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8003 8004 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 8005 return (SET_ERROR(ENOTSUP)); 8006 8007 if (func == POOL_SCAN_RESILVER && 8008 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 8009 return (SET_ERROR(ENOTSUP)); 8010 8011 /* 8012 * If a resilver was requested, but there is no DTL on a 8013 * writeable leaf device, we have nothing to do. 8014 */ 8015 if (func == POOL_SCAN_RESILVER && 8016 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 8017 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 8018 return (0); 8019 } 8020 8021 return (dsl_scan(spa->spa_dsl_pool, func)); 8022 } 8023 8024 /* 8025 * ========================================================================== 8026 * SPA async task processing 8027 * ========================================================================== 8028 */ 8029 8030 static void 8031 spa_async_remove(spa_t *spa, vdev_t *vd) 8032 { 8033 if (vd->vdev_remove_wanted) { 8034 vd->vdev_remove_wanted = B_FALSE; 8035 vd->vdev_delayed_close = B_FALSE; 8036 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 8037 8038 /* 8039 * We want to clear the stats, but we don't want to do a full 8040 * vdev_clear() as that will cause us to throw away 8041 * degraded/faulted state as well as attempt to reopen the 8042 * device, all of which is a waste. 8043 */ 8044 vd->vdev_stat.vs_read_errors = 0; 8045 vd->vdev_stat.vs_write_errors = 0; 8046 vd->vdev_stat.vs_checksum_errors = 0; 8047 8048 vdev_state_dirty(vd->vdev_top); 8049 8050 /* Tell userspace that the vdev is gone. */ 8051 zfs_post_remove(spa, vd); 8052 } 8053 8054 for (int c = 0; c < vd->vdev_children; c++) 8055 spa_async_remove(spa, vd->vdev_child[c]); 8056 } 8057 8058 static void 8059 spa_async_probe(spa_t *spa, vdev_t *vd) 8060 { 8061 if (vd->vdev_probe_wanted) { 8062 vd->vdev_probe_wanted = B_FALSE; 8063 vdev_reopen(vd); /* vdev_open() does the actual probe */ 8064 } 8065 8066 for (int c = 0; c < vd->vdev_children; c++) 8067 spa_async_probe(spa, vd->vdev_child[c]); 8068 } 8069 8070 static void 8071 spa_async_autoexpand(spa_t *spa, vdev_t *vd) 8072 { 8073 if (!spa->spa_autoexpand) 8074 return; 8075 8076 for (int c = 0; c < vd->vdev_children; c++) { 8077 vdev_t *cvd = vd->vdev_child[c]; 8078 spa_async_autoexpand(spa, cvd); 8079 } 8080 8081 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 8082 return; 8083 8084 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 8085 } 8086 8087 static void 8088 spa_async_thread(void *arg) 8089 { 8090 spa_t *spa = (spa_t *)arg; 8091 dsl_pool_t *dp = spa->spa_dsl_pool; 8092 int tasks; 8093 8094 ASSERT(spa->spa_sync_on); 8095 8096 mutex_enter(&spa->spa_async_lock); 8097 tasks = spa->spa_async_tasks; 8098 spa->spa_async_tasks = 0; 8099 mutex_exit(&spa->spa_async_lock); 8100 8101 /* 8102 * See if the config needs to be updated. 8103 */ 8104 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 8105 uint64_t old_space, new_space; 8106 8107 mutex_enter(&spa_namespace_lock); 8108 old_space = metaslab_class_get_space(spa_normal_class(spa)); 8109 old_space += metaslab_class_get_space(spa_special_class(spa)); 8110 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 8111 old_space += metaslab_class_get_space( 8112 spa_embedded_log_class(spa)); 8113 8114 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 8115 8116 new_space = metaslab_class_get_space(spa_normal_class(spa)); 8117 new_space += metaslab_class_get_space(spa_special_class(spa)); 8118 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 8119 new_space += metaslab_class_get_space( 8120 spa_embedded_log_class(spa)); 8121 mutex_exit(&spa_namespace_lock); 8122 8123 /* 8124 * If the pool grew as a result of the config update, 8125 * then log an internal history event. 8126 */ 8127 if (new_space != old_space) { 8128 spa_history_log_internal(spa, "vdev online", NULL, 8129 "pool '%s' size: %llu(+%llu)", 8130 spa_name(spa), (u_longlong_t)new_space, 8131 (u_longlong_t)(new_space - old_space)); 8132 } 8133 } 8134 8135 /* 8136 * See if any devices need to be marked REMOVED. 8137 */ 8138 if (tasks & SPA_ASYNC_REMOVE) { 8139 spa_vdev_state_enter(spa, SCL_NONE); 8140 spa_async_remove(spa, spa->spa_root_vdev); 8141 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 8142 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); 8143 for (int i = 0; i < spa->spa_spares.sav_count; i++) 8144 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); 8145 (void) spa_vdev_state_exit(spa, NULL, 0); 8146 } 8147 8148 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 8149 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8150 spa_async_autoexpand(spa, spa->spa_root_vdev); 8151 spa_config_exit(spa, SCL_CONFIG, FTAG); 8152 } 8153 8154 /* 8155 * See if any devices need to be probed. 8156 */ 8157 if (tasks & SPA_ASYNC_PROBE) { 8158 spa_vdev_state_enter(spa, SCL_NONE); 8159 spa_async_probe(spa, spa->spa_root_vdev); 8160 (void) spa_vdev_state_exit(spa, NULL, 0); 8161 } 8162 8163 /* 8164 * If any devices are done replacing, detach them. 8165 */ 8166 if (tasks & SPA_ASYNC_RESILVER_DONE || 8167 tasks & SPA_ASYNC_REBUILD_DONE) { 8168 spa_vdev_resilver_done(spa); 8169 } 8170 8171 /* 8172 * Kick off a resilver. 8173 */ 8174 if (tasks & SPA_ASYNC_RESILVER && 8175 !vdev_rebuild_active(spa->spa_root_vdev) && 8176 (!dsl_scan_resilvering(dp) || 8177 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 8178 dsl_scan_restart_resilver(dp, 0); 8179 8180 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 8181 mutex_enter(&spa_namespace_lock); 8182 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8183 vdev_initialize_restart(spa->spa_root_vdev); 8184 spa_config_exit(spa, SCL_CONFIG, FTAG); 8185 mutex_exit(&spa_namespace_lock); 8186 } 8187 8188 if (tasks & SPA_ASYNC_TRIM_RESTART) { 8189 mutex_enter(&spa_namespace_lock); 8190 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8191 vdev_trim_restart(spa->spa_root_vdev); 8192 spa_config_exit(spa, SCL_CONFIG, FTAG); 8193 mutex_exit(&spa_namespace_lock); 8194 } 8195 8196 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 8197 mutex_enter(&spa_namespace_lock); 8198 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8199 vdev_autotrim_restart(spa); 8200 spa_config_exit(spa, SCL_CONFIG, FTAG); 8201 mutex_exit(&spa_namespace_lock); 8202 } 8203 8204 /* 8205 * Kick off L2 cache whole device TRIM. 8206 */ 8207 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 8208 mutex_enter(&spa_namespace_lock); 8209 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8210 vdev_trim_l2arc(spa); 8211 spa_config_exit(spa, SCL_CONFIG, FTAG); 8212 mutex_exit(&spa_namespace_lock); 8213 } 8214 8215 /* 8216 * Kick off L2 cache rebuilding. 8217 */ 8218 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 8219 mutex_enter(&spa_namespace_lock); 8220 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 8221 l2arc_spa_rebuild_start(spa); 8222 spa_config_exit(spa, SCL_L2ARC, FTAG); 8223 mutex_exit(&spa_namespace_lock); 8224 } 8225 8226 /* 8227 * Let the world know that we're done. 8228 */ 8229 mutex_enter(&spa->spa_async_lock); 8230 spa->spa_async_thread = NULL; 8231 cv_broadcast(&spa->spa_async_cv); 8232 mutex_exit(&spa->spa_async_lock); 8233 thread_exit(); 8234 } 8235 8236 void 8237 spa_async_suspend(spa_t *spa) 8238 { 8239 mutex_enter(&spa->spa_async_lock); 8240 spa->spa_async_suspended++; 8241 while (spa->spa_async_thread != NULL) 8242 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 8243 mutex_exit(&spa->spa_async_lock); 8244 8245 spa_vdev_remove_suspend(spa); 8246 8247 zthr_t *condense_thread = spa->spa_condense_zthr; 8248 if (condense_thread != NULL) 8249 zthr_cancel(condense_thread); 8250 8251 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8252 if (discard_thread != NULL) 8253 zthr_cancel(discard_thread); 8254 8255 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8256 if (ll_delete_thread != NULL) 8257 zthr_cancel(ll_delete_thread); 8258 8259 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8260 if (ll_condense_thread != NULL) 8261 zthr_cancel(ll_condense_thread); 8262 } 8263 8264 void 8265 spa_async_resume(spa_t *spa) 8266 { 8267 mutex_enter(&spa->spa_async_lock); 8268 ASSERT(spa->spa_async_suspended != 0); 8269 spa->spa_async_suspended--; 8270 mutex_exit(&spa->spa_async_lock); 8271 spa_restart_removal(spa); 8272 8273 zthr_t *condense_thread = spa->spa_condense_zthr; 8274 if (condense_thread != NULL) 8275 zthr_resume(condense_thread); 8276 8277 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8278 if (discard_thread != NULL) 8279 zthr_resume(discard_thread); 8280 8281 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8282 if (ll_delete_thread != NULL) 8283 zthr_resume(ll_delete_thread); 8284 8285 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8286 if (ll_condense_thread != NULL) 8287 zthr_resume(ll_condense_thread); 8288 } 8289 8290 static boolean_t 8291 spa_async_tasks_pending(spa_t *spa) 8292 { 8293 uint_t non_config_tasks; 8294 uint_t config_task; 8295 boolean_t config_task_suspended; 8296 8297 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 8298 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 8299 if (spa->spa_ccw_fail_time == 0) { 8300 config_task_suspended = B_FALSE; 8301 } else { 8302 config_task_suspended = 8303 (gethrtime() - spa->spa_ccw_fail_time) < 8304 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 8305 } 8306 8307 return (non_config_tasks || (config_task && !config_task_suspended)); 8308 } 8309 8310 static void 8311 spa_async_dispatch(spa_t *spa) 8312 { 8313 mutex_enter(&spa->spa_async_lock); 8314 if (spa_async_tasks_pending(spa) && 8315 !spa->spa_async_suspended && 8316 spa->spa_async_thread == NULL) 8317 spa->spa_async_thread = thread_create(NULL, 0, 8318 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 8319 mutex_exit(&spa->spa_async_lock); 8320 } 8321 8322 void 8323 spa_async_request(spa_t *spa, int task) 8324 { 8325 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); 8326 mutex_enter(&spa->spa_async_lock); 8327 spa->spa_async_tasks |= task; 8328 mutex_exit(&spa->spa_async_lock); 8329 } 8330 8331 int 8332 spa_async_tasks(spa_t *spa) 8333 { 8334 return (spa->spa_async_tasks); 8335 } 8336 8337 /* 8338 * ========================================================================== 8339 * SPA syncing routines 8340 * ========================================================================== 8341 */ 8342 8343 8344 static int 8345 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8346 dmu_tx_t *tx) 8347 { 8348 bpobj_t *bpo = arg; 8349 bpobj_enqueue(bpo, bp, bp_freed, tx); 8350 return (0); 8351 } 8352 8353 int 8354 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8355 { 8356 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 8357 } 8358 8359 int 8360 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8361 { 8362 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 8363 } 8364 8365 static int 8366 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8367 { 8368 zio_t *pio = arg; 8369 8370 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 8371 pio->io_flags)); 8372 return (0); 8373 } 8374 8375 static int 8376 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8377 dmu_tx_t *tx) 8378 { 8379 ASSERT(!bp_freed); 8380 return (spa_free_sync_cb(arg, bp, tx)); 8381 } 8382 8383 /* 8384 * Note: this simple function is not inlined to make it easier to dtrace the 8385 * amount of time spent syncing frees. 8386 */ 8387 static void 8388 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 8389 { 8390 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8391 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 8392 VERIFY(zio_wait(zio) == 0); 8393 } 8394 8395 /* 8396 * Note: this simple function is not inlined to make it easier to dtrace the 8397 * amount of time spent syncing deferred frees. 8398 */ 8399 static void 8400 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 8401 { 8402 if (spa_sync_pass(spa) != 1) 8403 return; 8404 8405 /* 8406 * Note: 8407 * If the log space map feature is active, we stop deferring 8408 * frees to the next TXG and therefore running this function 8409 * would be considered a no-op as spa_deferred_bpobj should 8410 * not have any entries. 8411 * 8412 * That said we run this function anyway (instead of returning 8413 * immediately) for the edge-case scenario where we just 8414 * activated the log space map feature in this TXG but we have 8415 * deferred frees from the previous TXG. 8416 */ 8417 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8418 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 8419 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 8420 VERIFY0(zio_wait(zio)); 8421 } 8422 8423 static void 8424 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 8425 { 8426 char *packed = NULL; 8427 size_t bufsize; 8428 size_t nvsize = 0; 8429 dmu_buf_t *db; 8430 8431 VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); 8432 8433 /* 8434 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 8435 * information. This avoids the dmu_buf_will_dirty() path and 8436 * saves us a pre-read to get data we don't actually care about. 8437 */ 8438 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 8439 packed = vmem_alloc(bufsize, KM_SLEEP); 8440 8441 VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 8442 KM_SLEEP) == 0); 8443 bzero(packed + nvsize, bufsize - nvsize); 8444 8445 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); 8446 8447 vmem_free(packed, bufsize); 8448 8449 VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 8450 dmu_buf_will_dirty(db, tx); 8451 *(uint64_t *)db->db_data = nvsize; 8452 dmu_buf_rele(db, FTAG); 8453 } 8454 8455 static void 8456 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 8457 const char *config, const char *entry) 8458 { 8459 nvlist_t *nvroot; 8460 nvlist_t **list; 8461 int i; 8462 8463 if (!sav->sav_sync) 8464 return; 8465 8466 /* 8467 * Update the MOS nvlist describing the list of available devices. 8468 * spa_validate_aux() will have already made sure this nvlist is 8469 * valid and the vdevs are labeled appropriately. 8470 */ 8471 if (sav->sav_object == 0) { 8472 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 8473 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 8474 sizeof (uint64_t), tx); 8475 VERIFY(zap_update(spa->spa_meta_objset, 8476 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 8477 &sav->sav_object, tx) == 0); 8478 } 8479 8480 nvroot = fnvlist_alloc(); 8481 if (sav->sav_count == 0) { 8482 fnvlist_add_nvlist_array(nvroot, config, 8483 (const nvlist_t * const *)NULL, 0); 8484 } else { 8485 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 8486 for (i = 0; i < sav->sav_count; i++) 8487 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 8488 B_FALSE, VDEV_CONFIG_L2CACHE); 8489 fnvlist_add_nvlist_array(nvroot, config, 8490 (const nvlist_t * const *)list, sav->sav_count); 8491 for (i = 0; i < sav->sav_count; i++) 8492 nvlist_free(list[i]); 8493 kmem_free(list, sav->sav_count * sizeof (void *)); 8494 } 8495 8496 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 8497 nvlist_free(nvroot); 8498 8499 sav->sav_sync = B_FALSE; 8500 } 8501 8502 /* 8503 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 8504 * The all-vdev ZAP must be empty. 8505 */ 8506 static void 8507 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 8508 { 8509 spa_t *spa = vd->vdev_spa; 8510 8511 if (vd->vdev_top_zap != 0) { 8512 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8513 vd->vdev_top_zap, tx)); 8514 } 8515 if (vd->vdev_leaf_zap != 0) { 8516 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8517 vd->vdev_leaf_zap, tx)); 8518 } 8519 for (uint64_t i = 0; i < vd->vdev_children; i++) { 8520 spa_avz_build(vd->vdev_child[i], avz, tx); 8521 } 8522 } 8523 8524 static void 8525 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 8526 { 8527 nvlist_t *config; 8528 8529 /* 8530 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 8531 * its config may not be dirty but we still need to build per-vdev ZAPs. 8532 * Similarly, if the pool is being assembled (e.g. after a split), we 8533 * need to rebuild the AVZ although the config may not be dirty. 8534 */ 8535 if (list_is_empty(&spa->spa_config_dirty_list) && 8536 spa->spa_avz_action == AVZ_ACTION_NONE) 8537 return; 8538 8539 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 8540 8541 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 8542 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 8543 spa->spa_all_vdev_zaps != 0); 8544 8545 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 8546 /* Make and build the new AVZ */ 8547 uint64_t new_avz = zap_create(spa->spa_meta_objset, 8548 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 8549 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 8550 8551 /* Diff old AVZ with new one */ 8552 zap_cursor_t zc; 8553 zap_attribute_t za; 8554 8555 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8556 spa->spa_all_vdev_zaps); 8557 zap_cursor_retrieve(&zc, &za) == 0; 8558 zap_cursor_advance(&zc)) { 8559 uint64_t vdzap = za.za_first_integer; 8560 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 8561 vdzap) == ENOENT) { 8562 /* 8563 * ZAP is listed in old AVZ but not in new one; 8564 * destroy it 8565 */ 8566 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 8567 tx)); 8568 } 8569 } 8570 8571 zap_cursor_fini(&zc); 8572 8573 /* Destroy the old AVZ */ 8574 VERIFY0(zap_destroy(spa->spa_meta_objset, 8575 spa->spa_all_vdev_zaps, tx)); 8576 8577 /* Replace the old AVZ in the dir obj with the new one */ 8578 VERIFY0(zap_update(spa->spa_meta_objset, 8579 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 8580 sizeof (new_avz), 1, &new_avz, tx)); 8581 8582 spa->spa_all_vdev_zaps = new_avz; 8583 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 8584 zap_cursor_t zc; 8585 zap_attribute_t za; 8586 8587 /* Walk through the AVZ and destroy all listed ZAPs */ 8588 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8589 spa->spa_all_vdev_zaps); 8590 zap_cursor_retrieve(&zc, &za) == 0; 8591 zap_cursor_advance(&zc)) { 8592 uint64_t zap = za.za_first_integer; 8593 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 8594 } 8595 8596 zap_cursor_fini(&zc); 8597 8598 /* Destroy and unlink the AVZ itself */ 8599 VERIFY0(zap_destroy(spa->spa_meta_objset, 8600 spa->spa_all_vdev_zaps, tx)); 8601 VERIFY0(zap_remove(spa->spa_meta_objset, 8602 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 8603 spa->spa_all_vdev_zaps = 0; 8604 } 8605 8606 if (spa->spa_all_vdev_zaps == 0) { 8607 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 8608 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 8609 DMU_POOL_VDEV_ZAP_MAP, tx); 8610 } 8611 spa->spa_avz_action = AVZ_ACTION_NONE; 8612 8613 /* Create ZAPs for vdevs that don't have them. */ 8614 vdev_construct_zaps(spa->spa_root_vdev, tx); 8615 8616 config = spa_config_generate(spa, spa->spa_root_vdev, 8617 dmu_tx_get_txg(tx), B_FALSE); 8618 8619 /* 8620 * If we're upgrading the spa version then make sure that 8621 * the config object gets updated with the correct version. 8622 */ 8623 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 8624 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 8625 spa->spa_uberblock.ub_version); 8626 8627 spa_config_exit(spa, SCL_STATE, FTAG); 8628 8629 nvlist_free(spa->spa_config_syncing); 8630 spa->spa_config_syncing = config; 8631 8632 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 8633 } 8634 8635 static void 8636 spa_sync_version(void *arg, dmu_tx_t *tx) 8637 { 8638 uint64_t *versionp = arg; 8639 uint64_t version = *versionp; 8640 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8641 8642 /* 8643 * Setting the version is special cased when first creating the pool. 8644 */ 8645 ASSERT(tx->tx_txg != TXG_INITIAL); 8646 8647 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 8648 ASSERT(version >= spa_version(spa)); 8649 8650 spa->spa_uberblock.ub_version = version; 8651 vdev_config_dirty(spa->spa_root_vdev); 8652 spa_history_log_internal(spa, "set", tx, "version=%lld", 8653 (longlong_t)version); 8654 } 8655 8656 /* 8657 * Set zpool properties. 8658 */ 8659 static void 8660 spa_sync_props(void *arg, dmu_tx_t *tx) 8661 { 8662 nvlist_t *nvp = arg; 8663 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8664 objset_t *mos = spa->spa_meta_objset; 8665 nvpair_t *elem = NULL; 8666 8667 mutex_enter(&spa->spa_props_lock); 8668 8669 while ((elem = nvlist_next_nvpair(nvp, elem))) { 8670 uint64_t intval; 8671 char *strval, *fname; 8672 zpool_prop_t prop; 8673 const char *propname; 8674 zprop_type_t proptype; 8675 spa_feature_t fid; 8676 8677 switch (prop = zpool_name_to_prop(nvpair_name(elem))) { 8678 case ZPOOL_PROP_INVAL: 8679 /* 8680 * We checked this earlier in spa_prop_validate(). 8681 */ 8682 ASSERT(zpool_prop_feature(nvpair_name(elem))); 8683 8684 fname = strchr(nvpair_name(elem), '@') + 1; 8685 VERIFY0(zfeature_lookup_name(fname, &fid)); 8686 8687 spa_feature_enable(spa, fid, tx); 8688 spa_history_log_internal(spa, "set", tx, 8689 "%s=enabled", nvpair_name(elem)); 8690 break; 8691 8692 case ZPOOL_PROP_VERSION: 8693 intval = fnvpair_value_uint64(elem); 8694 /* 8695 * The version is synced separately before other 8696 * properties and should be correct by now. 8697 */ 8698 ASSERT3U(spa_version(spa), >=, intval); 8699 break; 8700 8701 case ZPOOL_PROP_ALTROOT: 8702 /* 8703 * 'altroot' is a non-persistent property. It should 8704 * have been set temporarily at creation or import time. 8705 */ 8706 ASSERT(spa->spa_root != NULL); 8707 break; 8708 8709 case ZPOOL_PROP_READONLY: 8710 case ZPOOL_PROP_CACHEFILE: 8711 /* 8712 * 'readonly' and 'cachefile' are also non-persistent 8713 * properties. 8714 */ 8715 break; 8716 case ZPOOL_PROP_COMMENT: 8717 strval = fnvpair_value_string(elem); 8718 if (spa->spa_comment != NULL) 8719 spa_strfree(spa->spa_comment); 8720 spa->spa_comment = spa_strdup(strval); 8721 /* 8722 * We need to dirty the configuration on all the vdevs 8723 * so that their labels get updated. We also need to 8724 * update the cache file to keep it in sync with the 8725 * MOS version. It's unnecessary to do this for pool 8726 * creation since the vdev's configuration has already 8727 * been dirtied. 8728 */ 8729 if (tx->tx_txg != TXG_INITIAL) { 8730 vdev_config_dirty(spa->spa_root_vdev); 8731 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8732 } 8733 spa_history_log_internal(spa, "set", tx, 8734 "%s=%s", nvpair_name(elem), strval); 8735 break; 8736 case ZPOOL_PROP_COMPATIBILITY: 8737 strval = fnvpair_value_string(elem); 8738 if (spa->spa_compatibility != NULL) 8739 spa_strfree(spa->spa_compatibility); 8740 spa->spa_compatibility = spa_strdup(strval); 8741 /* 8742 * Dirty the configuration on vdevs as above. 8743 */ 8744 if (tx->tx_txg != TXG_INITIAL) { 8745 vdev_config_dirty(spa->spa_root_vdev); 8746 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 8747 } 8748 8749 spa_history_log_internal(spa, "set", tx, 8750 "%s=%s", nvpair_name(elem), strval); 8751 break; 8752 8753 default: 8754 /* 8755 * Set pool property values in the poolprops mos object. 8756 */ 8757 if (spa->spa_pool_props_object == 0) { 8758 spa->spa_pool_props_object = 8759 zap_create_link(mos, DMU_OT_POOL_PROPS, 8760 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 8761 tx); 8762 } 8763 8764 /* normalize the property name */ 8765 propname = zpool_prop_to_name(prop); 8766 proptype = zpool_prop_get_type(prop); 8767 8768 if (nvpair_type(elem) == DATA_TYPE_STRING) { 8769 ASSERT(proptype == PROP_TYPE_STRING); 8770 strval = fnvpair_value_string(elem); 8771 VERIFY0(zap_update(mos, 8772 spa->spa_pool_props_object, propname, 8773 1, strlen(strval) + 1, strval, tx)); 8774 spa_history_log_internal(spa, "set", tx, 8775 "%s=%s", nvpair_name(elem), strval); 8776 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 8777 intval = fnvpair_value_uint64(elem); 8778 8779 if (proptype == PROP_TYPE_INDEX) { 8780 const char *unused; 8781 VERIFY0(zpool_prop_index_to_string( 8782 prop, intval, &unused)); 8783 } 8784 VERIFY0(zap_update(mos, 8785 spa->spa_pool_props_object, propname, 8786 8, 1, &intval, tx)); 8787 spa_history_log_internal(spa, "set", tx, 8788 "%s=%lld", nvpair_name(elem), 8789 (longlong_t)intval); 8790 } else { 8791 ASSERT(0); /* not allowed */ 8792 } 8793 8794 switch (prop) { 8795 case ZPOOL_PROP_DELEGATION: 8796 spa->spa_delegation = intval; 8797 break; 8798 case ZPOOL_PROP_BOOTFS: 8799 spa->spa_bootfs = intval; 8800 break; 8801 case ZPOOL_PROP_FAILUREMODE: 8802 spa->spa_failmode = intval; 8803 break; 8804 case ZPOOL_PROP_AUTOTRIM: 8805 spa->spa_autotrim = intval; 8806 spa_async_request(spa, 8807 SPA_ASYNC_AUTOTRIM_RESTART); 8808 break; 8809 case ZPOOL_PROP_AUTOEXPAND: 8810 spa->spa_autoexpand = intval; 8811 if (tx->tx_txg != TXG_INITIAL) 8812 spa_async_request(spa, 8813 SPA_ASYNC_AUTOEXPAND); 8814 break; 8815 case ZPOOL_PROP_MULTIHOST: 8816 spa->spa_multihost = intval; 8817 break; 8818 default: 8819 break; 8820 } 8821 } 8822 8823 } 8824 8825 mutex_exit(&spa->spa_props_lock); 8826 } 8827 8828 /* 8829 * Perform one-time upgrade on-disk changes. spa_version() does not 8830 * reflect the new version this txg, so there must be no changes this 8831 * txg to anything that the upgrade code depends on after it executes. 8832 * Therefore this must be called after dsl_pool_sync() does the sync 8833 * tasks. 8834 */ 8835 static void 8836 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 8837 { 8838 if (spa_sync_pass(spa) != 1) 8839 return; 8840 8841 dsl_pool_t *dp = spa->spa_dsl_pool; 8842 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 8843 8844 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && 8845 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { 8846 dsl_pool_create_origin(dp, tx); 8847 8848 /* Keeping the origin open increases spa_minref */ 8849 spa->spa_minref += 3; 8850 } 8851 8852 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && 8853 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { 8854 dsl_pool_upgrade_clones(dp, tx); 8855 } 8856 8857 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && 8858 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { 8859 dsl_pool_upgrade_dir_clones(dp, tx); 8860 8861 /* Keeping the freedir open increases spa_minref */ 8862 spa->spa_minref += 3; 8863 } 8864 8865 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && 8866 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8867 spa_feature_create_zap_objects(spa, tx); 8868 } 8869 8870 /* 8871 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 8872 * when possibility to use lz4 compression for metadata was added 8873 * Old pools that have this feature enabled must be upgraded to have 8874 * this feature active 8875 */ 8876 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8877 boolean_t lz4_en = spa_feature_is_enabled(spa, 8878 SPA_FEATURE_LZ4_COMPRESS); 8879 boolean_t lz4_ac = spa_feature_is_active(spa, 8880 SPA_FEATURE_LZ4_COMPRESS); 8881 8882 if (lz4_en && !lz4_ac) 8883 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 8884 } 8885 8886 /* 8887 * If we haven't written the salt, do so now. Note that the 8888 * feature may not be activated yet, but that's fine since 8889 * the presence of this ZAP entry is backwards compatible. 8890 */ 8891 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 8892 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 8893 VERIFY0(zap_add(spa->spa_meta_objset, 8894 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 8895 sizeof (spa->spa_cksum_salt.zcs_bytes), 8896 spa->spa_cksum_salt.zcs_bytes, tx)); 8897 } 8898 8899 rrw_exit(&dp->dp_config_rwlock, FTAG); 8900 } 8901 8902 static void 8903 vdev_indirect_state_sync_verify(vdev_t *vd) 8904 { 8905 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 8906 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 8907 8908 if (vd->vdev_ops == &vdev_indirect_ops) { 8909 ASSERT(vim != NULL); 8910 ASSERT(vib != NULL); 8911 } 8912 8913 uint64_t obsolete_sm_object = 0; 8914 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 8915 if (obsolete_sm_object != 0) { 8916 ASSERT(vd->vdev_obsolete_sm != NULL); 8917 ASSERT(vd->vdev_removing || 8918 vd->vdev_ops == &vdev_indirect_ops); 8919 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 8920 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 8921 ASSERT3U(obsolete_sm_object, ==, 8922 space_map_object(vd->vdev_obsolete_sm)); 8923 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 8924 space_map_allocated(vd->vdev_obsolete_sm)); 8925 } 8926 ASSERT(vd->vdev_obsolete_segments != NULL); 8927 8928 /* 8929 * Since frees / remaps to an indirect vdev can only 8930 * happen in syncing context, the obsolete segments 8931 * tree must be empty when we start syncing. 8932 */ 8933 ASSERT0(range_tree_space(vd->vdev_obsolete_segments)); 8934 } 8935 8936 /* 8937 * Set the top-level vdev's max queue depth. Evaluate each top-level's 8938 * async write queue depth in case it changed. The max queue depth will 8939 * not change in the middle of syncing out this txg. 8940 */ 8941 static void 8942 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 8943 { 8944 ASSERT(spa_writeable(spa)); 8945 8946 vdev_t *rvd = spa->spa_root_vdev; 8947 uint32_t max_queue_depth = zfs_vdev_async_write_max_active * 8948 zfs_vdev_queue_depth_pct / 100; 8949 metaslab_class_t *normal = spa_normal_class(spa); 8950 metaslab_class_t *special = spa_special_class(spa); 8951 metaslab_class_t *dedup = spa_dedup_class(spa); 8952 8953 uint64_t slots_per_allocator = 0; 8954 for (int c = 0; c < rvd->vdev_children; c++) { 8955 vdev_t *tvd = rvd->vdev_child[c]; 8956 8957 metaslab_group_t *mg = tvd->vdev_mg; 8958 if (mg == NULL || !metaslab_group_initialized(mg)) 8959 continue; 8960 8961 metaslab_class_t *mc = mg->mg_class; 8962 if (mc != normal && mc != special && mc != dedup) 8963 continue; 8964 8965 /* 8966 * It is safe to do a lock-free check here because only async 8967 * allocations look at mg_max_alloc_queue_depth, and async 8968 * allocations all happen from spa_sync(). 8969 */ 8970 for (int i = 0; i < mg->mg_allocators; i++) { 8971 ASSERT0(zfs_refcount_count( 8972 &(mg->mg_allocator[i].mga_alloc_queue_depth))); 8973 } 8974 mg->mg_max_alloc_queue_depth = max_queue_depth; 8975 8976 for (int i = 0; i < mg->mg_allocators; i++) { 8977 mg->mg_allocator[i].mga_cur_max_alloc_queue_depth = 8978 zfs_vdev_def_queue_depth; 8979 } 8980 slots_per_allocator += zfs_vdev_def_queue_depth; 8981 } 8982 8983 for (int i = 0; i < spa->spa_alloc_count; i++) { 8984 ASSERT0(zfs_refcount_count(&normal->mc_allocator[i]. 8985 mca_alloc_slots)); 8986 ASSERT0(zfs_refcount_count(&special->mc_allocator[i]. 8987 mca_alloc_slots)); 8988 ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i]. 8989 mca_alloc_slots)); 8990 normal->mc_allocator[i].mca_alloc_max_slots = 8991 slots_per_allocator; 8992 special->mc_allocator[i].mca_alloc_max_slots = 8993 slots_per_allocator; 8994 dedup->mc_allocator[i].mca_alloc_max_slots = 8995 slots_per_allocator; 8996 } 8997 normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 8998 special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 8999 dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9000 } 9001 9002 static void 9003 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 9004 { 9005 ASSERT(spa_writeable(spa)); 9006 9007 vdev_t *rvd = spa->spa_root_vdev; 9008 for (int c = 0; c < rvd->vdev_children; c++) { 9009 vdev_t *vd = rvd->vdev_child[c]; 9010 vdev_indirect_state_sync_verify(vd); 9011 9012 if (vdev_indirect_should_condense(vd)) { 9013 spa_condense_indirect_start_sync(vd, tx); 9014 break; 9015 } 9016 } 9017 } 9018 9019 static void 9020 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 9021 { 9022 objset_t *mos = spa->spa_meta_objset; 9023 dsl_pool_t *dp = spa->spa_dsl_pool; 9024 uint64_t txg = tx->tx_txg; 9025 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 9026 9027 do { 9028 int pass = ++spa->spa_sync_pass; 9029 9030 spa_sync_config_object(spa, tx); 9031 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 9032 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 9033 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 9034 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 9035 spa_errlog_sync(spa, txg); 9036 dsl_pool_sync(dp, txg); 9037 9038 if (pass < zfs_sync_pass_deferred_free || 9039 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 9040 /* 9041 * If the log space map feature is active we don't 9042 * care about deferred frees and the deferred bpobj 9043 * as the log space map should effectively have the 9044 * same results (i.e. appending only to one object). 9045 */ 9046 spa_sync_frees(spa, free_bpl, tx); 9047 } else { 9048 /* 9049 * We can not defer frees in pass 1, because 9050 * we sync the deferred frees later in pass 1. 9051 */ 9052 ASSERT3U(pass, >, 1); 9053 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 9054 &spa->spa_deferred_bpobj, tx); 9055 } 9056 9057 ddt_sync(spa, txg); 9058 dsl_scan_sync(dp, tx); 9059 svr_sync(spa, tx); 9060 spa_sync_upgrades(spa, tx); 9061 9062 spa_flush_metaslabs(spa, tx); 9063 9064 vdev_t *vd = NULL; 9065 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 9066 != NULL) 9067 vdev_sync(vd, txg); 9068 9069 /* 9070 * Note: We need to check if the MOS is dirty because we could 9071 * have marked the MOS dirty without updating the uberblock 9072 * (e.g. if we have sync tasks but no dirty user data). We need 9073 * to check the uberblock's rootbp because it is updated if we 9074 * have synced out dirty data (though in this case the MOS will 9075 * most likely also be dirty due to second order effects, we 9076 * don't want to rely on that here). 9077 */ 9078 if (pass == 1 && 9079 spa->spa_uberblock.ub_rootbp.blk_birth < txg && 9080 !dmu_objset_is_dirty(mos, txg)) { 9081 /* 9082 * Nothing changed on the first pass, therefore this 9083 * TXG is a no-op. Avoid syncing deferred frees, so 9084 * that we can keep this TXG as a no-op. 9085 */ 9086 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9087 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9088 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 9089 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 9090 break; 9091 } 9092 9093 spa_sync_deferred_frees(spa, tx); 9094 } while (dmu_objset_is_dirty(mos, txg)); 9095 } 9096 9097 /* 9098 * Rewrite the vdev configuration (which includes the uberblock) to 9099 * commit the transaction group. 9100 * 9101 * If there are no dirty vdevs, we sync the uberblock to a few random 9102 * top-level vdevs that are known to be visible in the config cache 9103 * (see spa_vdev_add() for a complete description). If there *are* dirty 9104 * vdevs, sync the uberblock to all vdevs. 9105 */ 9106 static void 9107 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 9108 { 9109 vdev_t *rvd = spa->spa_root_vdev; 9110 uint64_t txg = tx->tx_txg; 9111 9112 for (;;) { 9113 int error = 0; 9114 9115 /* 9116 * We hold SCL_STATE to prevent vdev open/close/etc. 9117 * while we're attempting to write the vdev labels. 9118 */ 9119 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9120 9121 if (list_is_empty(&spa->spa_config_dirty_list)) { 9122 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 9123 int svdcount = 0; 9124 int children = rvd->vdev_children; 9125 int c0 = random_in_range(children); 9126 9127 for (int c = 0; c < children; c++) { 9128 vdev_t *vd = 9129 rvd->vdev_child[(c0 + c) % children]; 9130 9131 /* Stop when revisiting the first vdev */ 9132 if (c > 0 && svd[0] == vd) 9133 break; 9134 9135 if (vd->vdev_ms_array == 0 || 9136 vd->vdev_islog || 9137 !vdev_is_concrete(vd)) 9138 continue; 9139 9140 svd[svdcount++] = vd; 9141 if (svdcount == SPA_SYNC_MIN_VDEVS) 9142 break; 9143 } 9144 error = vdev_config_sync(svd, svdcount, txg); 9145 } else { 9146 error = vdev_config_sync(rvd->vdev_child, 9147 rvd->vdev_children, txg); 9148 } 9149 9150 if (error == 0) 9151 spa->spa_last_synced_guid = rvd->vdev_guid; 9152 9153 spa_config_exit(spa, SCL_STATE, FTAG); 9154 9155 if (error == 0) 9156 break; 9157 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 9158 zio_resume_wait(spa); 9159 } 9160 } 9161 9162 /* 9163 * Sync the specified transaction group. New blocks may be dirtied as 9164 * part of the process, so we iterate until it converges. 9165 */ 9166 void 9167 spa_sync(spa_t *spa, uint64_t txg) 9168 { 9169 vdev_t *vd = NULL; 9170 9171 VERIFY(spa_writeable(spa)); 9172 9173 /* 9174 * Wait for i/os issued in open context that need to complete 9175 * before this txg syncs. 9176 */ 9177 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 9178 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 9179 ZIO_FLAG_CANFAIL); 9180 9181 /* 9182 * Lock out configuration changes. 9183 */ 9184 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9185 9186 spa->spa_syncing_txg = txg; 9187 spa->spa_sync_pass = 0; 9188 9189 for (int i = 0; i < spa->spa_alloc_count; i++) { 9190 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9191 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9192 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9193 } 9194 9195 /* 9196 * If there are any pending vdev state changes, convert them 9197 * into config changes that go out with this transaction group. 9198 */ 9199 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9200 while (list_head(&spa->spa_state_dirty_list) != NULL) { 9201 /* 9202 * We need the write lock here because, for aux vdevs, 9203 * calling vdev_config_dirty() modifies sav_config. 9204 * This is ugly and will become unnecessary when we 9205 * eliminate the aux vdev wart by integrating all vdevs 9206 * into the root vdev tree. 9207 */ 9208 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9209 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 9210 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9211 vdev_state_clean(vd); 9212 vdev_config_dirty(vd); 9213 } 9214 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9215 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9216 } 9217 spa_config_exit(spa, SCL_STATE, FTAG); 9218 9219 dsl_pool_t *dp = spa->spa_dsl_pool; 9220 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 9221 9222 spa->spa_sync_starttime = gethrtime(); 9223 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9224 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 9225 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 9226 NSEC_TO_TICK(spa->spa_deadman_synctime)); 9227 9228 /* 9229 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 9230 * set spa_deflate if we have no raid-z vdevs. 9231 */ 9232 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 9233 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 9234 vdev_t *rvd = spa->spa_root_vdev; 9235 9236 int i; 9237 for (i = 0; i < rvd->vdev_children; i++) { 9238 vd = rvd->vdev_child[i]; 9239 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 9240 break; 9241 } 9242 if (i == rvd->vdev_children) { 9243 spa->spa_deflate = TRUE; 9244 VERIFY0(zap_add(spa->spa_meta_objset, 9245 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 9246 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 9247 } 9248 } 9249 9250 spa_sync_adjust_vdev_max_queue_depth(spa); 9251 9252 spa_sync_condense_indirect(spa, tx); 9253 9254 spa_sync_iterate_to_convergence(spa, tx); 9255 9256 #ifdef ZFS_DEBUG 9257 if (!list_is_empty(&spa->spa_config_dirty_list)) { 9258 /* 9259 * Make sure that the number of ZAPs for all the vdevs matches 9260 * the number of ZAPs in the per-vdev ZAP list. This only gets 9261 * called if the config is dirty; otherwise there may be 9262 * outstanding AVZ operations that weren't completed in 9263 * spa_sync_config_object. 9264 */ 9265 uint64_t all_vdev_zap_entry_count; 9266 ASSERT0(zap_count(spa->spa_meta_objset, 9267 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 9268 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 9269 all_vdev_zap_entry_count); 9270 } 9271 #endif 9272 9273 if (spa->spa_vdev_removal != NULL) { 9274 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 9275 } 9276 9277 spa_sync_rewrite_vdev_config(spa, tx); 9278 dmu_tx_commit(tx); 9279 9280 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9281 spa->spa_deadman_tqid = 0; 9282 9283 /* 9284 * Clear the dirty config list. 9285 */ 9286 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 9287 vdev_config_clean(vd); 9288 9289 /* 9290 * Now that the new config has synced transactionally, 9291 * let it become visible to the config cache. 9292 */ 9293 if (spa->spa_config_syncing != NULL) { 9294 spa_config_set(spa, spa->spa_config_syncing); 9295 spa->spa_config_txg = txg; 9296 spa->spa_config_syncing = NULL; 9297 } 9298 9299 dsl_pool_sync_done(dp, txg); 9300 9301 for (int i = 0; i < spa->spa_alloc_count; i++) { 9302 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9303 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9304 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9305 } 9306 9307 /* 9308 * Update usable space statistics. 9309 */ 9310 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 9311 != NULL) 9312 vdev_sync_done(vd, txg); 9313 9314 metaslab_class_evict_old(spa->spa_normal_class, txg); 9315 metaslab_class_evict_old(spa->spa_log_class, txg); 9316 9317 spa_sync_close_syncing_log_sm(spa); 9318 9319 spa_update_dspace(spa); 9320 9321 /* 9322 * It had better be the case that we didn't dirty anything 9323 * since vdev_config_sync(). 9324 */ 9325 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9326 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9327 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 9328 9329 while (zfs_pause_spa_sync) 9330 delay(1); 9331 9332 spa->spa_sync_pass = 0; 9333 9334 /* 9335 * Update the last synced uberblock here. We want to do this at 9336 * the end of spa_sync() so that consumers of spa_last_synced_txg() 9337 * will be guaranteed that all the processing associated with 9338 * that txg has been completed. 9339 */ 9340 spa->spa_ubsync = spa->spa_uberblock; 9341 spa_config_exit(spa, SCL_CONFIG, FTAG); 9342 9343 spa_handle_ignored_writes(spa); 9344 9345 /* 9346 * If any async tasks have been requested, kick them off. 9347 */ 9348 spa_async_dispatch(spa); 9349 } 9350 9351 /* 9352 * Sync all pools. We don't want to hold the namespace lock across these 9353 * operations, so we take a reference on the spa_t and drop the lock during the 9354 * sync. 9355 */ 9356 void 9357 spa_sync_allpools(void) 9358 { 9359 spa_t *spa = NULL; 9360 mutex_enter(&spa_namespace_lock); 9361 while ((spa = spa_next(spa)) != NULL) { 9362 if (spa_state(spa) != POOL_STATE_ACTIVE || 9363 !spa_writeable(spa) || spa_suspended(spa)) 9364 continue; 9365 spa_open_ref(spa, FTAG); 9366 mutex_exit(&spa_namespace_lock); 9367 txg_wait_synced(spa_get_dsl(spa), 0); 9368 mutex_enter(&spa_namespace_lock); 9369 spa_close(spa, FTAG); 9370 } 9371 mutex_exit(&spa_namespace_lock); 9372 } 9373 9374 /* 9375 * ========================================================================== 9376 * Miscellaneous routines 9377 * ========================================================================== 9378 */ 9379 9380 /* 9381 * Remove all pools in the system. 9382 */ 9383 void 9384 spa_evict_all(void) 9385 { 9386 spa_t *spa; 9387 9388 /* 9389 * Remove all cached state. All pools should be closed now, 9390 * so every spa in the AVL tree should be unreferenced. 9391 */ 9392 mutex_enter(&spa_namespace_lock); 9393 while ((spa = spa_next(NULL)) != NULL) { 9394 /* 9395 * Stop async tasks. The async thread may need to detach 9396 * a device that's been replaced, which requires grabbing 9397 * spa_namespace_lock, so we must drop it here. 9398 */ 9399 spa_open_ref(spa, FTAG); 9400 mutex_exit(&spa_namespace_lock); 9401 spa_async_suspend(spa); 9402 mutex_enter(&spa_namespace_lock); 9403 spa_close(spa, FTAG); 9404 9405 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 9406 spa_unload(spa); 9407 spa_deactivate(spa); 9408 } 9409 spa_remove(spa); 9410 } 9411 mutex_exit(&spa_namespace_lock); 9412 } 9413 9414 vdev_t * 9415 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 9416 { 9417 vdev_t *vd; 9418 int i; 9419 9420 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 9421 return (vd); 9422 9423 if (aux) { 9424 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 9425 vd = spa->spa_l2cache.sav_vdevs[i]; 9426 if (vd->vdev_guid == guid) 9427 return (vd); 9428 } 9429 9430 for (i = 0; i < spa->spa_spares.sav_count; i++) { 9431 vd = spa->spa_spares.sav_vdevs[i]; 9432 if (vd->vdev_guid == guid) 9433 return (vd); 9434 } 9435 } 9436 9437 return (NULL); 9438 } 9439 9440 void 9441 spa_upgrade(spa_t *spa, uint64_t version) 9442 { 9443 ASSERT(spa_writeable(spa)); 9444 9445 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9446 9447 /* 9448 * This should only be called for a non-faulted pool, and since a 9449 * future version would result in an unopenable pool, this shouldn't be 9450 * possible. 9451 */ 9452 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 9453 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 9454 9455 spa->spa_uberblock.ub_version = version; 9456 vdev_config_dirty(spa->spa_root_vdev); 9457 9458 spa_config_exit(spa, SCL_ALL, FTAG); 9459 9460 txg_wait_synced(spa_get_dsl(spa), 0); 9461 } 9462 9463 static boolean_t 9464 spa_has_aux_vdev(spa_t *spa, uint64_t guid, spa_aux_vdev_t *sav) 9465 { 9466 int i; 9467 uint64_t vdev_guid; 9468 9469 for (i = 0; i < sav->sav_count; i++) 9470 if (sav->sav_vdevs[i]->vdev_guid == guid) 9471 return (B_TRUE); 9472 9473 for (i = 0; i < sav->sav_npending; i++) { 9474 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 9475 &vdev_guid) == 0 && vdev_guid == guid) 9476 return (B_TRUE); 9477 } 9478 9479 return (B_FALSE); 9480 } 9481 9482 boolean_t 9483 spa_has_l2cache(spa_t *spa, uint64_t guid) 9484 { 9485 return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache)); 9486 } 9487 9488 boolean_t 9489 spa_has_spare(spa_t *spa, uint64_t guid) 9490 { 9491 return (spa_has_aux_vdev(spa, guid, &spa->spa_spares)); 9492 } 9493 9494 /* 9495 * Check if a pool has an active shared spare device. 9496 * Note: reference count of an active spare is 2, as a spare and as a replace 9497 */ 9498 static boolean_t 9499 spa_has_active_shared_spare(spa_t *spa) 9500 { 9501 int i, refcnt; 9502 uint64_t pool; 9503 spa_aux_vdev_t *sav = &spa->spa_spares; 9504 9505 for (i = 0; i < sav->sav_count; i++) { 9506 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 9507 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 9508 refcnt > 2) 9509 return (B_TRUE); 9510 } 9511 9512 return (B_FALSE); 9513 } 9514 9515 uint64_t 9516 spa_total_metaslabs(spa_t *spa) 9517 { 9518 vdev_t *rvd = spa->spa_root_vdev; 9519 9520 uint64_t m = 0; 9521 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 9522 vdev_t *vd = rvd->vdev_child[c]; 9523 if (!vdev_is_concrete(vd)) 9524 continue; 9525 m += vd->vdev_ms_count; 9526 } 9527 return (m); 9528 } 9529 9530 /* 9531 * Notify any waiting threads that some activity has switched from being in- 9532 * progress to not-in-progress so that the thread can wake up and determine 9533 * whether it is finished waiting. 9534 */ 9535 void 9536 spa_notify_waiters(spa_t *spa) 9537 { 9538 /* 9539 * Acquiring spa_activities_lock here prevents the cv_broadcast from 9540 * happening between the waiting thread's check and cv_wait. 9541 */ 9542 mutex_enter(&spa->spa_activities_lock); 9543 cv_broadcast(&spa->spa_activities_cv); 9544 mutex_exit(&spa->spa_activities_lock); 9545 } 9546 9547 /* 9548 * Notify any waiting threads that the pool is exporting, and then block until 9549 * they are finished using the spa_t. 9550 */ 9551 void 9552 spa_wake_waiters(spa_t *spa) 9553 { 9554 mutex_enter(&spa->spa_activities_lock); 9555 spa->spa_waiters_cancel = B_TRUE; 9556 cv_broadcast(&spa->spa_activities_cv); 9557 while (spa->spa_waiters != 0) 9558 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 9559 spa->spa_waiters_cancel = B_FALSE; 9560 mutex_exit(&spa->spa_activities_lock); 9561 } 9562 9563 /* Whether the vdev or any of its descendants are being initialized/trimmed. */ 9564 static boolean_t 9565 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 9566 { 9567 spa_t *spa = vd->vdev_spa; 9568 9569 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 9570 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9571 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 9572 activity == ZPOOL_WAIT_TRIM); 9573 9574 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 9575 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 9576 9577 mutex_exit(&spa->spa_activities_lock); 9578 mutex_enter(lock); 9579 mutex_enter(&spa->spa_activities_lock); 9580 9581 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 9582 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) : 9583 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE); 9584 mutex_exit(lock); 9585 9586 if (in_progress) 9587 return (B_TRUE); 9588 9589 for (int i = 0; i < vd->vdev_children; i++) { 9590 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 9591 activity)) 9592 return (B_TRUE); 9593 } 9594 9595 return (B_FALSE); 9596 } 9597 9598 /* 9599 * If use_guid is true, this checks whether the vdev specified by guid is 9600 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 9601 * is being initialized/trimmed. The caller must hold the config lock and 9602 * spa_activities_lock. 9603 */ 9604 static int 9605 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 9606 zpool_wait_activity_t activity, boolean_t *in_progress) 9607 { 9608 mutex_exit(&spa->spa_activities_lock); 9609 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9610 mutex_enter(&spa->spa_activities_lock); 9611 9612 vdev_t *vd; 9613 if (use_guid) { 9614 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 9615 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 9616 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9617 return (EINVAL); 9618 } 9619 } else { 9620 vd = spa->spa_root_vdev; 9621 } 9622 9623 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 9624 9625 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9626 return (0); 9627 } 9628 9629 /* 9630 * Locking for waiting threads 9631 * --------------------------- 9632 * 9633 * Waiting threads need a way to check whether a given activity is in progress, 9634 * and then, if it is, wait for it to complete. Each activity will have some 9635 * in-memory representation of the relevant on-disk state which can be used to 9636 * determine whether or not the activity is in progress. The in-memory state and 9637 * the locking used to protect it will be different for each activity, and may 9638 * not be suitable for use with a cvar (e.g., some state is protected by the 9639 * config lock). To allow waiting threads to wait without any races, another 9640 * lock, spa_activities_lock, is used. 9641 * 9642 * When the state is checked, both the activity-specific lock (if there is one) 9643 * and spa_activities_lock are held. In some cases, the activity-specific lock 9644 * is acquired explicitly (e.g. the config lock). In others, the locking is 9645 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 9646 * thread releases the activity-specific lock and, if the activity is in 9647 * progress, then cv_waits using spa_activities_lock. 9648 * 9649 * The waiting thread is woken when another thread, one completing some 9650 * activity, updates the state of the activity and then calls 9651 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 9652 * needs to hold its activity-specific lock when updating the state, and this 9653 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 9654 * 9655 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 9656 * and because it is held when the waiting thread checks the state of the 9657 * activity, it can never be the case that the completing thread both updates 9658 * the activity state and cv_broadcasts in between the waiting thread's check 9659 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 9660 * 9661 * In order to prevent deadlock, when the waiting thread does its check, in some 9662 * cases it will temporarily drop spa_activities_lock in order to acquire the 9663 * activity-specific lock. The order in which spa_activities_lock and the 9664 * activity specific lock are acquired in the waiting thread is determined by 9665 * the order in which they are acquired in the completing thread; if the 9666 * completing thread calls spa_notify_waiters with the activity-specific lock 9667 * held, then the waiting thread must also acquire the activity-specific lock 9668 * first. 9669 */ 9670 9671 static int 9672 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 9673 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 9674 { 9675 int error = 0; 9676 9677 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9678 9679 switch (activity) { 9680 case ZPOOL_WAIT_CKPT_DISCARD: 9681 *in_progress = 9682 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 9683 zap_contains(spa_meta_objset(spa), 9684 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 9685 ENOENT); 9686 break; 9687 case ZPOOL_WAIT_FREE: 9688 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 9689 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 9690 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 9691 spa_livelist_delete_check(spa)); 9692 break; 9693 case ZPOOL_WAIT_INITIALIZE: 9694 case ZPOOL_WAIT_TRIM: 9695 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 9696 activity, in_progress); 9697 break; 9698 case ZPOOL_WAIT_REPLACE: 9699 mutex_exit(&spa->spa_activities_lock); 9700 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9701 mutex_enter(&spa->spa_activities_lock); 9702 9703 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 9704 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9705 break; 9706 case ZPOOL_WAIT_REMOVE: 9707 *in_progress = (spa->spa_removing_phys.sr_state == 9708 DSS_SCANNING); 9709 break; 9710 case ZPOOL_WAIT_RESILVER: 9711 if ((*in_progress = vdev_rebuild_active(spa->spa_root_vdev))) 9712 break; 9713 fallthrough; 9714 case ZPOOL_WAIT_SCRUB: 9715 { 9716 boolean_t scanning, paused, is_scrub; 9717 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 9718 9719 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 9720 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 9721 paused = dsl_scan_is_paused_scrub(scn); 9722 *in_progress = (scanning && !paused && 9723 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 9724 break; 9725 } 9726 default: 9727 panic("unrecognized value for activity %d", activity); 9728 } 9729 9730 return (error); 9731 } 9732 9733 static int 9734 spa_wait_common(const char *pool, zpool_wait_activity_t activity, 9735 boolean_t use_tag, uint64_t tag, boolean_t *waited) 9736 { 9737 /* 9738 * The tag is used to distinguish between instances of an activity. 9739 * 'initialize' and 'trim' are the only activities that we use this for. 9740 * The other activities can only have a single instance in progress in a 9741 * pool at one time, making the tag unnecessary. 9742 * 9743 * There can be multiple devices being replaced at once, but since they 9744 * all finish once resilvering finishes, we don't bother keeping track 9745 * of them individually, we just wait for them all to finish. 9746 */ 9747 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 9748 activity != ZPOOL_WAIT_TRIM) 9749 return (EINVAL); 9750 9751 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 9752 return (EINVAL); 9753 9754 spa_t *spa; 9755 int error = spa_open(pool, &spa, FTAG); 9756 if (error != 0) 9757 return (error); 9758 9759 /* 9760 * Increment the spa's waiter count so that we can call spa_close and 9761 * still ensure that the spa_t doesn't get freed before this thread is 9762 * finished with it when the pool is exported. We want to call spa_close 9763 * before we start waiting because otherwise the additional ref would 9764 * prevent the pool from being exported or destroyed throughout the 9765 * potentially long wait. 9766 */ 9767 mutex_enter(&spa->spa_activities_lock); 9768 spa->spa_waiters++; 9769 spa_close(spa, FTAG); 9770 9771 *waited = B_FALSE; 9772 for (;;) { 9773 boolean_t in_progress; 9774 error = spa_activity_in_progress(spa, activity, use_tag, tag, 9775 &in_progress); 9776 9777 if (error || !in_progress || spa->spa_waiters_cancel) 9778 break; 9779 9780 *waited = B_TRUE; 9781 9782 if (cv_wait_sig(&spa->spa_activities_cv, 9783 &spa->spa_activities_lock) == 0) { 9784 error = EINTR; 9785 break; 9786 } 9787 } 9788 9789 spa->spa_waiters--; 9790 cv_signal(&spa->spa_waiters_cv); 9791 mutex_exit(&spa->spa_activities_lock); 9792 9793 return (error); 9794 } 9795 9796 /* 9797 * Wait for a particular instance of the specified activity to complete, where 9798 * the instance is identified by 'tag' 9799 */ 9800 int 9801 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 9802 boolean_t *waited) 9803 { 9804 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 9805 } 9806 9807 /* 9808 * Wait for all instances of the specified activity complete 9809 */ 9810 int 9811 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 9812 { 9813 9814 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 9815 } 9816 9817 sysevent_t * 9818 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9819 { 9820 sysevent_t *ev = NULL; 9821 #ifdef _KERNEL 9822 nvlist_t *resource; 9823 9824 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 9825 if (resource) { 9826 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 9827 ev->resource = resource; 9828 } 9829 #endif 9830 return (ev); 9831 } 9832 9833 void 9834 spa_event_post(sysevent_t *ev) 9835 { 9836 #ifdef _KERNEL 9837 if (ev) { 9838 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 9839 kmem_free(ev, sizeof (*ev)); 9840 } 9841 #endif 9842 } 9843 9844 /* 9845 * Post a zevent corresponding to the given sysevent. The 'name' must be one 9846 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 9847 * filled in from the spa and (optionally) the vdev. This doesn't do anything 9848 * in the userland libzpool, as we don't want consumers to misinterpret ztest 9849 * or zdb as real changes. 9850 */ 9851 void 9852 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9853 { 9854 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 9855 } 9856 9857 /* state manipulation functions */ 9858 EXPORT_SYMBOL(spa_open); 9859 EXPORT_SYMBOL(spa_open_rewind); 9860 EXPORT_SYMBOL(spa_get_stats); 9861 EXPORT_SYMBOL(spa_create); 9862 EXPORT_SYMBOL(spa_import); 9863 EXPORT_SYMBOL(spa_tryimport); 9864 EXPORT_SYMBOL(spa_destroy); 9865 EXPORT_SYMBOL(spa_export); 9866 EXPORT_SYMBOL(spa_reset); 9867 EXPORT_SYMBOL(spa_async_request); 9868 EXPORT_SYMBOL(spa_async_suspend); 9869 EXPORT_SYMBOL(spa_async_resume); 9870 EXPORT_SYMBOL(spa_inject_addref); 9871 EXPORT_SYMBOL(spa_inject_delref); 9872 EXPORT_SYMBOL(spa_scan_stat_init); 9873 EXPORT_SYMBOL(spa_scan_get_stats); 9874 9875 /* device manipulation */ 9876 EXPORT_SYMBOL(spa_vdev_add); 9877 EXPORT_SYMBOL(spa_vdev_attach); 9878 EXPORT_SYMBOL(spa_vdev_detach); 9879 EXPORT_SYMBOL(spa_vdev_setpath); 9880 EXPORT_SYMBOL(spa_vdev_setfru); 9881 EXPORT_SYMBOL(spa_vdev_split_mirror); 9882 9883 /* spare statech is global across all pools) */ 9884 EXPORT_SYMBOL(spa_spare_add); 9885 EXPORT_SYMBOL(spa_spare_remove); 9886 EXPORT_SYMBOL(spa_spare_exists); 9887 EXPORT_SYMBOL(spa_spare_activate); 9888 9889 /* L2ARC statech is global across all pools) */ 9890 EXPORT_SYMBOL(spa_l2cache_add); 9891 EXPORT_SYMBOL(spa_l2cache_remove); 9892 EXPORT_SYMBOL(spa_l2cache_exists); 9893 EXPORT_SYMBOL(spa_l2cache_activate); 9894 EXPORT_SYMBOL(spa_l2cache_drop); 9895 9896 /* scanning */ 9897 EXPORT_SYMBOL(spa_scan); 9898 EXPORT_SYMBOL(spa_scan_stop); 9899 9900 /* spa syncing */ 9901 EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 9902 EXPORT_SYMBOL(spa_sync_allpools); 9903 9904 /* properties */ 9905 EXPORT_SYMBOL(spa_prop_set); 9906 EXPORT_SYMBOL(spa_prop_get); 9907 EXPORT_SYMBOL(spa_prop_clear_bootfs); 9908 9909 /* asynchronous event notification */ 9910 EXPORT_SYMBOL(spa_event_notify); 9911 9912 /* BEGIN CSTYLED */ 9913 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, INT, ZMOD_RW, 9914 "log2 fraction of arc that can be used by inflight I/Os when " 9915 "verifying pool during import"); 9916 9917 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 9918 "Set to traverse metadata on pool import"); 9919 9920 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 9921 "Set to traverse data on pool import"); 9922 9923 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 9924 "Print vdev tree to zfs_dbgmsg during pool import"); 9925 9926 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD, 9927 "Percentage of CPUs to run an IO worker thread"); 9928 9929 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD, 9930 "Number of threads per IO worker taskqueue"); 9931 9932 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, ULONG, ZMOD_RW, 9933 "Allow importing pool with up to this number of missing top-level " 9934 "vdevs (in read-only mode)"); 9935 9936 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, ZMOD_RW, 9937 "Set the livelist condense zthr to pause"); 9938 9939 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, ZMOD_RW, 9940 "Set the livelist condense synctask to pause"); 9941 9942 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, INT, ZMOD_RW, 9943 "Whether livelist condensing was canceled in the synctask"); 9944 9945 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, INT, ZMOD_RW, 9946 "Whether livelist condensing was canceled in the zthr function"); 9947 9948 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, ZMOD_RW, 9949 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 9950 "was being condensed"); 9951 /* END CSTYLED */ 9952