1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 * Copyright 2016 Toomas Soome <tsoome@me.com> 30 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 31 * Copyright 2018 Joyent, Inc. 32 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 33 * Copyright 2017 Joyent, Inc. 34 * Copyright (c) 2017, Intel Corporation. 35 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 36 * Copyright (c) 2023 Hewlett Packard Enterprise Development LP. 37 */ 38 39 /* 40 * SPA: Storage Pool Allocator 41 * 42 * This file contains all the routines used when modifying on-disk SPA state. 43 * This includes opening, importing, destroying, exporting a pool, and syncing a 44 * pool. 45 */ 46 47 #include <sys/zfs_context.h> 48 #include <sys/fm/fs/zfs.h> 49 #include <sys/spa_impl.h> 50 #include <sys/zio.h> 51 #include <sys/zio_checksum.h> 52 #include <sys/dmu.h> 53 #include <sys/dmu_tx.h> 54 #include <sys/zap.h> 55 #include <sys/zil.h> 56 #include <sys/brt.h> 57 #include <sys/ddt.h> 58 #include <sys/vdev_impl.h> 59 #include <sys/vdev_removal.h> 60 #include <sys/vdev_indirect_mapping.h> 61 #include <sys/vdev_indirect_births.h> 62 #include <sys/vdev_initialize.h> 63 #include <sys/vdev_rebuild.h> 64 #include <sys/vdev_trim.h> 65 #include <sys/vdev_disk.h> 66 #include <sys/vdev_raidz.h> 67 #include <sys/vdev_draid.h> 68 #include <sys/metaslab.h> 69 #include <sys/metaslab_impl.h> 70 #include <sys/mmp.h> 71 #include <sys/uberblock_impl.h> 72 #include <sys/txg.h> 73 #include <sys/avl.h> 74 #include <sys/bpobj.h> 75 #include <sys/dmu_traverse.h> 76 #include <sys/dmu_objset.h> 77 #include <sys/unique.h> 78 #include <sys/dsl_pool.h> 79 #include <sys/dsl_dataset.h> 80 #include <sys/dsl_dir.h> 81 #include <sys/dsl_prop.h> 82 #include <sys/dsl_synctask.h> 83 #include <sys/fs/zfs.h> 84 #include <sys/arc.h> 85 #include <sys/callb.h> 86 #include <sys/systeminfo.h> 87 #include <sys/zfs_ioctl.h> 88 #include <sys/dsl_scan.h> 89 #include <sys/zfeature.h> 90 #include <sys/dsl_destroy.h> 91 #include <sys/zvol.h> 92 93 #ifdef _KERNEL 94 #include <sys/fm/protocol.h> 95 #include <sys/fm/util.h> 96 #include <sys/callb.h> 97 #include <sys/zone.h> 98 #include <sys/vmsystm.h> 99 #endif /* _KERNEL */ 100 101 #include "zfs_prop.h" 102 #include "zfs_comutil.h" 103 #include <cityhash.h> 104 105 /* 106 * spa_thread() existed on Illumos as a parent thread for the various worker 107 * threads that actually run the pool, as a way to both reference the entire 108 * pool work as a single object, and to share properties like scheduling 109 * options. It has not yet been adapted to Linux or FreeBSD. This define is 110 * used to mark related parts of the code to make things easier for the reader, 111 * and to compile this code out. It can be removed when someone implements it, 112 * moves it to some Illumos-specific place, or removes it entirely. 113 */ 114 #undef HAVE_SPA_THREAD 115 116 /* 117 * The "System Duty Cycle" scheduling class is an Illumos feature to help 118 * prevent CPU-intensive kernel threads from affecting latency on interactive 119 * threads. It doesn't exist on Linux or FreeBSD, so the supporting code is 120 * gated behind a define. On Illumos SDC depends on spa_thread(), but 121 * spa_thread() also has other uses, so this is a separate define. 122 */ 123 #undef HAVE_SYSDC 124 125 /* 126 * The interval, in seconds, at which failed configuration cache file writes 127 * should be retried. 128 */ 129 int zfs_ccw_retry_interval = 300; 130 131 typedef enum zti_modes { 132 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 133 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 134 ZTI_MODE_SYNC, /* sync thread assigned */ 135 ZTI_MODE_NULL, /* don't create a taskq */ 136 ZTI_NMODES 137 } zti_modes_t; 138 139 #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 140 #define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 141 #define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 } 142 #define ZTI_SYNC { ZTI_MODE_SYNC, 0, 1 } 143 #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 144 145 #define ZTI_N(n) ZTI_P(n, 1) 146 #define ZTI_ONE ZTI_N(1) 147 148 typedef struct zio_taskq_info { 149 zti_modes_t zti_mode; 150 uint_t zti_value; 151 uint_t zti_count; 152 } zio_taskq_info_t; 153 154 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 155 "iss", "iss_h", "int", "int_h" 156 }; 157 158 /* 159 * This table defines the taskq settings for each ZFS I/O type. When 160 * initializing a pool, we use this table to create an appropriately sized 161 * taskq. Some operations are low volume and therefore have a small, static 162 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 163 * macros. Other operations process a large amount of data; the ZTI_SCALE 164 * macro causes us to create a taskq oriented for throughput. Some operations 165 * are so high frequency and short-lived that the taskq itself can become a 166 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 167 * additional degree of parallelism specified by the number of threads per- 168 * taskq and the number of taskqs; when dispatching an event in this case, the 169 * particular taskq is chosen at random. ZTI_SCALE uses a number of taskqs 170 * that scales with the number of CPUs. 171 * 172 * The different taskq priorities are to handle the different contexts (issue 173 * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that 174 * need to be handled with minimum delay. 175 */ 176 static const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 177 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 178 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 179 { ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */ 180 { ZTI_SYNC, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */ 181 { ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 182 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 183 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */ 184 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 185 }; 186 187 static void spa_sync_version(void *arg, dmu_tx_t *tx); 188 static void spa_sync_props(void *arg, dmu_tx_t *tx); 189 static boolean_t spa_has_active_shared_spare(spa_t *spa); 190 static int spa_load_impl(spa_t *spa, spa_import_type_t type, 191 const char **ereport); 192 static void spa_vdev_resilver_done(spa_t *spa); 193 194 /* 195 * Percentage of all CPUs that can be used by the metaslab preload taskq. 196 */ 197 static uint_t metaslab_preload_pct = 50; 198 199 static uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 200 static uint_t zio_taskq_batch_tpq; /* threads per taskq */ 201 202 #ifdef HAVE_SYSDC 203 static const boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 204 static const uint_t zio_taskq_basedc = 80; /* base duty cycle */ 205 #endif 206 207 #ifdef HAVE_SPA_THREAD 208 static const boolean_t spa_create_process = B_TRUE; /* no process => no sysdc */ 209 #endif 210 211 static uint_t zio_taskq_wr_iss_ncpus = 0; 212 213 /* 214 * Report any spa_load_verify errors found, but do not fail spa_load. 215 * This is used by zdb to analyze non-idle pools. 216 */ 217 boolean_t spa_load_verify_dryrun = B_FALSE; 218 219 /* 220 * Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ). 221 * This is used by zdb for spacemaps verification. 222 */ 223 boolean_t spa_mode_readable_spacemaps = B_FALSE; 224 225 /* 226 * This (illegal) pool name is used when temporarily importing a spa_t in order 227 * to get the vdev stats associated with the imported devices. 228 */ 229 #define TRYIMPORT_NAME "$import" 230 231 /* 232 * For debugging purposes: print out vdev tree during pool import. 233 */ 234 static int spa_load_print_vdev_tree = B_FALSE; 235 236 /* 237 * A non-zero value for zfs_max_missing_tvds means that we allow importing 238 * pools with missing top-level vdevs. This is strictly intended for advanced 239 * pool recovery cases since missing data is almost inevitable. Pools with 240 * missing devices can only be imported read-only for safety reasons, and their 241 * fail-mode will be automatically set to "continue". 242 * 243 * With 1 missing vdev we should be able to import the pool and mount all 244 * datasets. User data that was not modified after the missing device has been 245 * added should be recoverable. This means that snapshots created prior to the 246 * addition of that device should be completely intact. 247 * 248 * With 2 missing vdevs, some datasets may fail to mount since there are 249 * dataset statistics that are stored as regular metadata. Some data might be 250 * recoverable if those vdevs were added recently. 251 * 252 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 253 * may be missing entirely. Chances of data recovery are very low. Note that 254 * there are also risks of performing an inadvertent rewind as we might be 255 * missing all the vdevs with the latest uberblocks. 256 */ 257 uint64_t zfs_max_missing_tvds = 0; 258 259 /* 260 * The parameters below are similar to zfs_max_missing_tvds but are only 261 * intended for a preliminary open of the pool with an untrusted config which 262 * might be incomplete or out-dated. 263 * 264 * We are more tolerant for pools opened from a cachefile since we could have 265 * an out-dated cachefile where a device removal was not registered. 266 * We could have set the limit arbitrarily high but in the case where devices 267 * are really missing we would want to return the proper error codes; we chose 268 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 269 * and we get a chance to retrieve the trusted config. 270 */ 271 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 272 273 /* 274 * In the case where config was assembled by scanning device paths (/dev/dsks 275 * by default) we are less tolerant since all the existing devices should have 276 * been detected and we want spa_load to return the right error codes. 277 */ 278 uint64_t zfs_max_missing_tvds_scan = 0; 279 280 /* 281 * Debugging aid that pauses spa_sync() towards the end. 282 */ 283 static const boolean_t zfs_pause_spa_sync = B_FALSE; 284 285 /* 286 * Variables to indicate the livelist condense zthr func should wait at certain 287 * points for the livelist to be removed - used to test condense/destroy races 288 */ 289 static int zfs_livelist_condense_zthr_pause = 0; 290 static int zfs_livelist_condense_sync_pause = 0; 291 292 /* 293 * Variables to track whether or not condense cancellation has been 294 * triggered in testing. 295 */ 296 static int zfs_livelist_condense_sync_cancel = 0; 297 static int zfs_livelist_condense_zthr_cancel = 0; 298 299 /* 300 * Variable to track whether or not extra ALLOC blkptrs were added to a 301 * livelist entry while it was being condensed (caused by the way we track 302 * remapped blkptrs in dbuf_remap_impl) 303 */ 304 static int zfs_livelist_condense_new_alloc = 0; 305 306 /* 307 * ========================================================================== 308 * SPA properties routines 309 * ========================================================================== 310 */ 311 312 /* 313 * Add a (source=src, propname=propval) list to an nvlist. 314 */ 315 static void 316 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, const char *strval, 317 uint64_t intval, zprop_source_t src) 318 { 319 const char *propname = zpool_prop_to_name(prop); 320 nvlist_t *propval; 321 322 propval = fnvlist_alloc(); 323 fnvlist_add_uint64(propval, ZPROP_SOURCE, src); 324 325 if (strval != NULL) 326 fnvlist_add_string(propval, ZPROP_VALUE, strval); 327 else 328 fnvlist_add_uint64(propval, ZPROP_VALUE, intval); 329 330 fnvlist_add_nvlist(nvl, propname, propval); 331 nvlist_free(propval); 332 } 333 334 /* 335 * Add a user property (source=src, propname=propval) to an nvlist. 336 */ 337 static void 338 spa_prop_add_user(nvlist_t *nvl, const char *propname, char *strval, 339 zprop_source_t src) 340 { 341 nvlist_t *propval; 342 343 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); 344 VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); 345 VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0); 346 VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0); 347 nvlist_free(propval); 348 } 349 350 /* 351 * Get property values from the spa configuration. 352 */ 353 static void 354 spa_prop_get_config(spa_t *spa, nvlist_t **nvp) 355 { 356 vdev_t *rvd = spa->spa_root_vdev; 357 dsl_pool_t *pool = spa->spa_dsl_pool; 358 uint64_t size, alloc, cap, version; 359 const zprop_source_t src = ZPROP_SRC_NONE; 360 spa_config_dirent_t *dp; 361 metaslab_class_t *mc = spa_normal_class(spa); 362 363 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 364 365 if (rvd != NULL) { 366 alloc = metaslab_class_get_alloc(mc); 367 alloc += metaslab_class_get_alloc(spa_special_class(spa)); 368 alloc += metaslab_class_get_alloc(spa_dedup_class(spa)); 369 alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa)); 370 371 size = metaslab_class_get_space(mc); 372 size += metaslab_class_get_space(spa_special_class(spa)); 373 size += metaslab_class_get_space(spa_dedup_class(spa)); 374 size += metaslab_class_get_space(spa_embedded_log_class(spa)); 375 376 spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 377 spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src); 378 spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 379 spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL, 380 size - alloc, src); 381 spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL, 382 spa->spa_checkpoint_info.sci_dspace, src); 383 384 spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL, 385 metaslab_class_fragmentation(mc), src); 386 spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, 387 metaslab_class_expandable_space(mc), src); 388 spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL, 389 (spa_mode(spa) == SPA_MODE_READ), src); 390 391 cap = (size == 0) ? 0 : (alloc * 100 / size); 392 spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src); 393 394 spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL, 395 ddt_get_pool_dedup_ratio(spa), src); 396 spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONEUSED, NULL, 397 brt_get_used(spa), src); 398 spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONESAVED, NULL, 399 brt_get_saved(spa), src); 400 spa_prop_add_list(*nvp, ZPOOL_PROP_BCLONERATIO, NULL, 401 brt_get_ratio(spa), src); 402 403 spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL, 404 rvd->vdev_state, src); 405 406 version = spa_version(spa); 407 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 408 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 409 version, ZPROP_SRC_DEFAULT); 410 } else { 411 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 412 version, ZPROP_SRC_LOCAL); 413 } 414 spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID, 415 NULL, spa_load_guid(spa), src); 416 } 417 418 if (pool != NULL) { 419 /* 420 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 421 * when opening pools before this version freedir will be NULL. 422 */ 423 if (pool->dp_free_dir != NULL) { 424 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, 425 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 426 src); 427 } else { 428 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, 429 NULL, 0, src); 430 } 431 432 if (pool->dp_leak_dir != NULL) { 433 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, 434 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 435 src); 436 } else { 437 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, 438 NULL, 0, src); 439 } 440 } 441 442 spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 443 444 if (spa->spa_comment != NULL) { 445 spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment, 446 0, ZPROP_SRC_LOCAL); 447 } 448 449 if (spa->spa_compatibility != NULL) { 450 spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY, 451 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 452 } 453 454 if (spa->spa_root != NULL) 455 spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root, 456 0, ZPROP_SRC_LOCAL); 457 458 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 459 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 460 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 461 } else { 462 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 463 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 464 } 465 466 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 467 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 468 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 469 } else { 470 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 471 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 472 } 473 474 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 475 if (dp->scd_path == NULL) { 476 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 477 "none", 0, ZPROP_SRC_LOCAL); 478 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 479 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 480 dp->scd_path, 0, ZPROP_SRC_LOCAL); 481 } 482 } 483 } 484 485 /* 486 * Get zpool property values. 487 */ 488 int 489 spa_prop_get(spa_t *spa, nvlist_t **nvp) 490 { 491 objset_t *mos = spa->spa_meta_objset; 492 zap_cursor_t zc; 493 zap_attribute_t za; 494 dsl_pool_t *dp; 495 int err; 496 497 err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); 498 if (err) 499 return (err); 500 501 dp = spa_get_dsl(spa); 502 dsl_pool_config_enter(dp, FTAG); 503 mutex_enter(&spa->spa_props_lock); 504 505 /* 506 * Get properties from the spa config. 507 */ 508 spa_prop_get_config(spa, nvp); 509 510 /* If no pool property object, no more prop to get. */ 511 if (mos == NULL || spa->spa_pool_props_object == 0) 512 goto out; 513 514 /* 515 * Get properties from the MOS pool property object. 516 */ 517 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 518 (err = zap_cursor_retrieve(&zc, &za)) == 0; 519 zap_cursor_advance(&zc)) { 520 uint64_t intval = 0; 521 char *strval = NULL; 522 zprop_source_t src = ZPROP_SRC_DEFAULT; 523 zpool_prop_t prop; 524 525 if ((prop = zpool_name_to_prop(za.za_name)) == 526 ZPOOL_PROP_INVAL && !zfs_prop_user(za.za_name)) 527 continue; 528 529 switch (za.za_integer_length) { 530 case 8: 531 /* integer property */ 532 if (za.za_first_integer != 533 zpool_prop_default_numeric(prop)) 534 src = ZPROP_SRC_LOCAL; 535 536 if (prop == ZPOOL_PROP_BOOTFS) { 537 dsl_dataset_t *ds = NULL; 538 539 err = dsl_dataset_hold_obj(dp, 540 za.za_first_integer, FTAG, &ds); 541 if (err != 0) 542 break; 543 544 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 545 KM_SLEEP); 546 dsl_dataset_name(ds, strval); 547 dsl_dataset_rele(ds, FTAG); 548 } else { 549 strval = NULL; 550 intval = za.za_first_integer; 551 } 552 553 spa_prop_add_list(*nvp, prop, strval, intval, src); 554 555 if (strval != NULL) 556 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 557 558 break; 559 560 case 1: 561 /* string property */ 562 strval = kmem_alloc(za.za_num_integers, KM_SLEEP); 563 err = zap_lookup(mos, spa->spa_pool_props_object, 564 za.za_name, 1, za.za_num_integers, strval); 565 if (err) { 566 kmem_free(strval, za.za_num_integers); 567 break; 568 } 569 if (prop != ZPOOL_PROP_INVAL) { 570 spa_prop_add_list(*nvp, prop, strval, 0, src); 571 } else { 572 src = ZPROP_SRC_LOCAL; 573 spa_prop_add_user(*nvp, za.za_name, strval, 574 src); 575 } 576 kmem_free(strval, za.za_num_integers); 577 break; 578 579 default: 580 break; 581 } 582 } 583 zap_cursor_fini(&zc); 584 out: 585 mutex_exit(&spa->spa_props_lock); 586 dsl_pool_config_exit(dp, FTAG); 587 if (err && err != ENOENT) { 588 nvlist_free(*nvp); 589 *nvp = NULL; 590 return (err); 591 } 592 593 return (0); 594 } 595 596 /* 597 * Validate the given pool properties nvlist and modify the list 598 * for the property values to be set. 599 */ 600 static int 601 spa_prop_validate(spa_t *spa, nvlist_t *props) 602 { 603 nvpair_t *elem; 604 int error = 0, reset_bootfs = 0; 605 uint64_t objnum = 0; 606 boolean_t has_feature = B_FALSE; 607 608 elem = NULL; 609 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 610 uint64_t intval; 611 const char *strval, *slash, *check, *fname; 612 const char *propname = nvpair_name(elem); 613 zpool_prop_t prop = zpool_name_to_prop(propname); 614 615 switch (prop) { 616 case ZPOOL_PROP_INVAL: 617 /* 618 * Sanitize the input. 619 */ 620 if (zfs_prop_user(propname)) { 621 if (strlen(propname) >= ZAP_MAXNAMELEN) { 622 error = SET_ERROR(ENAMETOOLONG); 623 break; 624 } 625 626 if (strlen(fnvpair_value_string(elem)) >= 627 ZAP_MAXVALUELEN) { 628 error = SET_ERROR(E2BIG); 629 break; 630 } 631 } else if (zpool_prop_feature(propname)) { 632 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 633 error = SET_ERROR(EINVAL); 634 break; 635 } 636 637 if (nvpair_value_uint64(elem, &intval) != 0) { 638 error = SET_ERROR(EINVAL); 639 break; 640 } 641 642 if (intval != 0) { 643 error = SET_ERROR(EINVAL); 644 break; 645 } 646 647 fname = strchr(propname, '@') + 1; 648 if (zfeature_lookup_name(fname, NULL) != 0) { 649 error = SET_ERROR(EINVAL); 650 break; 651 } 652 653 has_feature = B_TRUE; 654 } else { 655 error = SET_ERROR(EINVAL); 656 break; 657 } 658 break; 659 660 case ZPOOL_PROP_VERSION: 661 error = nvpair_value_uint64(elem, &intval); 662 if (!error && 663 (intval < spa_version(spa) || 664 intval > SPA_VERSION_BEFORE_FEATURES || 665 has_feature)) 666 error = SET_ERROR(EINVAL); 667 break; 668 669 case ZPOOL_PROP_DELEGATION: 670 case ZPOOL_PROP_AUTOREPLACE: 671 case ZPOOL_PROP_LISTSNAPS: 672 case ZPOOL_PROP_AUTOEXPAND: 673 case ZPOOL_PROP_AUTOTRIM: 674 error = nvpair_value_uint64(elem, &intval); 675 if (!error && intval > 1) 676 error = SET_ERROR(EINVAL); 677 break; 678 679 case ZPOOL_PROP_MULTIHOST: 680 error = nvpair_value_uint64(elem, &intval); 681 if (!error && intval > 1) 682 error = SET_ERROR(EINVAL); 683 684 if (!error) { 685 uint32_t hostid = zone_get_hostid(NULL); 686 if (hostid) 687 spa->spa_hostid = hostid; 688 else 689 error = SET_ERROR(ENOTSUP); 690 } 691 692 break; 693 694 case ZPOOL_PROP_BOOTFS: 695 /* 696 * If the pool version is less than SPA_VERSION_BOOTFS, 697 * or the pool is still being created (version == 0), 698 * the bootfs property cannot be set. 699 */ 700 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 701 error = SET_ERROR(ENOTSUP); 702 break; 703 } 704 705 /* 706 * Make sure the vdev config is bootable 707 */ 708 if (!vdev_is_bootable(spa->spa_root_vdev)) { 709 error = SET_ERROR(ENOTSUP); 710 break; 711 } 712 713 reset_bootfs = 1; 714 715 error = nvpair_value_string(elem, &strval); 716 717 if (!error) { 718 objset_t *os; 719 720 if (strval == NULL || strval[0] == '\0') { 721 objnum = zpool_prop_default_numeric( 722 ZPOOL_PROP_BOOTFS); 723 break; 724 } 725 726 error = dmu_objset_hold(strval, FTAG, &os); 727 if (error != 0) 728 break; 729 730 /* Must be ZPL. */ 731 if (dmu_objset_type(os) != DMU_OST_ZFS) { 732 error = SET_ERROR(ENOTSUP); 733 } else { 734 objnum = dmu_objset_id(os); 735 } 736 dmu_objset_rele(os, FTAG); 737 } 738 break; 739 740 case ZPOOL_PROP_FAILUREMODE: 741 error = nvpair_value_uint64(elem, &intval); 742 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 743 error = SET_ERROR(EINVAL); 744 745 /* 746 * This is a special case which only occurs when 747 * the pool has completely failed. This allows 748 * the user to change the in-core failmode property 749 * without syncing it out to disk (I/Os might 750 * currently be blocked). We do this by returning 751 * EIO to the caller (spa_prop_set) to trick it 752 * into thinking we encountered a property validation 753 * error. 754 */ 755 if (!error && spa_suspended(spa)) { 756 spa->spa_failmode = intval; 757 error = SET_ERROR(EIO); 758 } 759 break; 760 761 case ZPOOL_PROP_CACHEFILE: 762 if ((error = nvpair_value_string(elem, &strval)) != 0) 763 break; 764 765 if (strval[0] == '\0') 766 break; 767 768 if (strcmp(strval, "none") == 0) 769 break; 770 771 if (strval[0] != '/') { 772 error = SET_ERROR(EINVAL); 773 break; 774 } 775 776 slash = strrchr(strval, '/'); 777 ASSERT(slash != NULL); 778 779 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 780 strcmp(slash, "/..") == 0) 781 error = SET_ERROR(EINVAL); 782 break; 783 784 case ZPOOL_PROP_COMMENT: 785 if ((error = nvpair_value_string(elem, &strval)) != 0) 786 break; 787 for (check = strval; *check != '\0'; check++) { 788 if (!isprint(*check)) { 789 error = SET_ERROR(EINVAL); 790 break; 791 } 792 } 793 if (strlen(strval) > ZPROP_MAX_COMMENT) 794 error = SET_ERROR(E2BIG); 795 break; 796 797 default: 798 break; 799 } 800 801 if (error) 802 break; 803 } 804 805 (void) nvlist_remove_all(props, 806 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 807 808 if (!error && reset_bootfs) { 809 error = nvlist_remove(props, 810 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 811 812 if (!error) { 813 error = nvlist_add_uint64(props, 814 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 815 } 816 } 817 818 return (error); 819 } 820 821 void 822 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 823 { 824 const char *cachefile; 825 spa_config_dirent_t *dp; 826 827 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 828 &cachefile) != 0) 829 return; 830 831 dp = kmem_alloc(sizeof (spa_config_dirent_t), 832 KM_SLEEP); 833 834 if (cachefile[0] == '\0') 835 dp->scd_path = spa_strdup(spa_config_path); 836 else if (strcmp(cachefile, "none") == 0) 837 dp->scd_path = NULL; 838 else 839 dp->scd_path = spa_strdup(cachefile); 840 841 list_insert_head(&spa->spa_config_list, dp); 842 if (need_sync) 843 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 844 } 845 846 int 847 spa_prop_set(spa_t *spa, nvlist_t *nvp) 848 { 849 int error; 850 nvpair_t *elem = NULL; 851 boolean_t need_sync = B_FALSE; 852 853 if ((error = spa_prop_validate(spa, nvp)) != 0) 854 return (error); 855 856 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 857 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 858 859 if (prop == ZPOOL_PROP_CACHEFILE || 860 prop == ZPOOL_PROP_ALTROOT || 861 prop == ZPOOL_PROP_READONLY) 862 continue; 863 864 if (prop == ZPOOL_PROP_INVAL && 865 zfs_prop_user(nvpair_name(elem))) { 866 need_sync = B_TRUE; 867 break; 868 } 869 870 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 871 uint64_t ver = 0; 872 873 if (prop == ZPOOL_PROP_VERSION) { 874 VERIFY(nvpair_value_uint64(elem, &ver) == 0); 875 } else { 876 ASSERT(zpool_prop_feature(nvpair_name(elem))); 877 ver = SPA_VERSION_FEATURES; 878 need_sync = B_TRUE; 879 } 880 881 /* Save time if the version is already set. */ 882 if (ver == spa_version(spa)) 883 continue; 884 885 /* 886 * In addition to the pool directory object, we might 887 * create the pool properties object, the features for 888 * read object, the features for write object, or the 889 * feature descriptions object. 890 */ 891 error = dsl_sync_task(spa->spa_name, NULL, 892 spa_sync_version, &ver, 893 6, ZFS_SPACE_CHECK_RESERVED); 894 if (error) 895 return (error); 896 continue; 897 } 898 899 need_sync = B_TRUE; 900 break; 901 } 902 903 if (need_sync) { 904 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 905 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 906 } 907 908 return (0); 909 } 910 911 /* 912 * If the bootfs property value is dsobj, clear it. 913 */ 914 void 915 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 916 { 917 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 918 VERIFY(zap_remove(spa->spa_meta_objset, 919 spa->spa_pool_props_object, 920 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 921 spa->spa_bootfs = 0; 922 } 923 } 924 925 static int 926 spa_change_guid_check(void *arg, dmu_tx_t *tx) 927 { 928 uint64_t *newguid __maybe_unused = arg; 929 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 930 vdev_t *rvd = spa->spa_root_vdev; 931 uint64_t vdev_state; 932 933 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 934 int error = (spa_has_checkpoint(spa)) ? 935 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 936 return (SET_ERROR(error)); 937 } 938 939 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 940 vdev_state = rvd->vdev_state; 941 spa_config_exit(spa, SCL_STATE, FTAG); 942 943 if (vdev_state != VDEV_STATE_HEALTHY) 944 return (SET_ERROR(ENXIO)); 945 946 ASSERT3U(spa_guid(spa), !=, *newguid); 947 948 return (0); 949 } 950 951 static void 952 spa_change_guid_sync(void *arg, dmu_tx_t *tx) 953 { 954 uint64_t *newguid = arg; 955 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 956 uint64_t oldguid; 957 vdev_t *rvd = spa->spa_root_vdev; 958 959 oldguid = spa_guid(spa); 960 961 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 962 rvd->vdev_guid = *newguid; 963 rvd->vdev_guid_sum += (*newguid - oldguid); 964 vdev_config_dirty(rvd); 965 spa_config_exit(spa, SCL_STATE, FTAG); 966 967 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 968 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 969 } 970 971 /* 972 * Change the GUID for the pool. This is done so that we can later 973 * re-import a pool built from a clone of our own vdevs. We will modify 974 * the root vdev's guid, our own pool guid, and then mark all of our 975 * vdevs dirty. Note that we must make sure that all our vdevs are 976 * online when we do this, or else any vdevs that weren't present 977 * would be orphaned from our pool. We are also going to issue a 978 * sysevent to update any watchers. 979 */ 980 int 981 spa_change_guid(spa_t *spa) 982 { 983 int error; 984 uint64_t guid; 985 986 mutex_enter(&spa->spa_vdev_top_lock); 987 mutex_enter(&spa_namespace_lock); 988 guid = spa_generate_guid(NULL); 989 990 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 991 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 992 993 if (error == 0) { 994 /* 995 * Clear the kobj flag from all the vdevs to allow 996 * vdev_cache_process_kobj_evt() to post events to all the 997 * vdevs since GUID is updated. 998 */ 999 vdev_clear_kobj_evt(spa->spa_root_vdev); 1000 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 1001 vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]); 1002 1003 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 1004 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 1005 } 1006 1007 mutex_exit(&spa_namespace_lock); 1008 mutex_exit(&spa->spa_vdev_top_lock); 1009 1010 return (error); 1011 } 1012 1013 /* 1014 * ========================================================================== 1015 * SPA state manipulation (open/create/destroy/import/export) 1016 * ========================================================================== 1017 */ 1018 1019 static int 1020 spa_error_entry_compare(const void *a, const void *b) 1021 { 1022 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 1023 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 1024 int ret; 1025 1026 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 1027 sizeof (zbookmark_phys_t)); 1028 1029 return (TREE_ISIGN(ret)); 1030 } 1031 1032 /* 1033 * Utility function which retrieves copies of the current logs and 1034 * re-initializes them in the process. 1035 */ 1036 void 1037 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 1038 { 1039 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 1040 1041 memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t)); 1042 memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t)); 1043 1044 avl_create(&spa->spa_errlist_scrub, 1045 spa_error_entry_compare, sizeof (spa_error_entry_t), 1046 offsetof(spa_error_entry_t, se_avl)); 1047 avl_create(&spa->spa_errlist_last, 1048 spa_error_entry_compare, sizeof (spa_error_entry_t), 1049 offsetof(spa_error_entry_t, se_avl)); 1050 } 1051 1052 static void 1053 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1054 { 1055 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 1056 enum zti_modes mode = ztip->zti_mode; 1057 uint_t value = ztip->zti_value; 1058 uint_t count = ztip->zti_count; 1059 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1060 uint_t cpus, flags = TASKQ_DYNAMIC; 1061 1062 switch (mode) { 1063 case ZTI_MODE_FIXED: 1064 ASSERT3U(value, >, 0); 1065 break; 1066 1067 case ZTI_MODE_SYNC: 1068 1069 /* 1070 * Create one wr_iss taskq for every 'zio_taskq_wr_iss_ncpus', 1071 * not to exceed the number of spa allocators. 1072 */ 1073 if (zio_taskq_wr_iss_ncpus == 0) { 1074 count = MAX(boot_ncpus / spa->spa_alloc_count, 1); 1075 } else { 1076 count = MAX(1, 1077 boot_ncpus / MAX(1, zio_taskq_wr_iss_ncpus)); 1078 } 1079 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1080 count = MIN(count, spa->spa_alloc_count); 1081 1082 /* 1083 * zio_taskq_batch_pct is unbounded and may exceed 100%, but no 1084 * single taskq may have more threads than 100% of online cpus. 1085 */ 1086 value = (zio_taskq_batch_pct + count / 2) / count; 1087 value = MIN(value, 100); 1088 flags |= TASKQ_THREADS_CPU_PCT; 1089 break; 1090 1091 case ZTI_MODE_SCALE: 1092 flags |= TASKQ_THREADS_CPU_PCT; 1093 /* 1094 * We want more taskqs to reduce lock contention, but we want 1095 * less for better request ordering and CPU utilization. 1096 */ 1097 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 1098 if (zio_taskq_batch_tpq > 0) { 1099 count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) / 1100 zio_taskq_batch_tpq); 1101 } else { 1102 /* 1103 * Prefer 6 threads per taskq, but no more taskqs 1104 * than threads in them on large systems. For 80%: 1105 * 1106 * taskq taskq total 1107 * cpus taskqs percent threads threads 1108 * ------- ------- ------- ------- ------- 1109 * 1 1 80% 1 1 1110 * 2 1 80% 1 1 1111 * 4 1 80% 3 3 1112 * 8 2 40% 3 6 1113 * 16 3 27% 4 12 1114 * 32 5 16% 5 25 1115 * 64 7 11% 7 49 1116 * 128 10 8% 10 100 1117 * 256 14 6% 15 210 1118 */ 1119 count = 1 + cpus / 6; 1120 while (count * count > cpus) 1121 count--; 1122 } 1123 /* Limit each taskq within 100% to not trigger assertion. */ 1124 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1125 value = (zio_taskq_batch_pct + count / 2) / count; 1126 break; 1127 1128 case ZTI_MODE_NULL: 1129 tqs->stqs_count = 0; 1130 tqs->stqs_taskq = NULL; 1131 return; 1132 1133 default: 1134 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1135 "spa_taskqs_init()", 1136 zio_type_name[t], zio_taskq_types[q], mode, value); 1137 break; 1138 } 1139 1140 ASSERT3U(count, >, 0); 1141 tqs->stqs_count = count; 1142 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1143 1144 for (uint_t i = 0; i < count; i++) { 1145 taskq_t *tq; 1146 char name[32]; 1147 1148 if (count > 1) 1149 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1150 zio_type_name[t], zio_taskq_types[q], i); 1151 else 1152 (void) snprintf(name, sizeof (name), "%s_%s", 1153 zio_type_name[t], zio_taskq_types[q]); 1154 1155 #ifdef HAVE_SYSDC 1156 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1157 (void) zio_taskq_basedc; 1158 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1159 spa->spa_proc, zio_taskq_basedc, flags); 1160 } else { 1161 #endif 1162 pri_t pri = maxclsyspri; 1163 /* 1164 * The write issue taskq can be extremely CPU 1165 * intensive. Run it at slightly less important 1166 * priority than the other taskqs. 1167 * 1168 * Under Linux and FreeBSD this means incrementing 1169 * the priority value as opposed to platforms like 1170 * illumos where it should be decremented. 1171 * 1172 * On FreeBSD, if priorities divided by four (RQ_PPQ) 1173 * are equal then a difference between them is 1174 * insignificant. 1175 */ 1176 if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) { 1177 #if defined(__linux__) 1178 pri++; 1179 #elif defined(__FreeBSD__) 1180 pri += 4; 1181 #else 1182 #error "unknown OS" 1183 #endif 1184 } 1185 tq = taskq_create_proc(name, value, pri, 50, 1186 INT_MAX, spa->spa_proc, flags); 1187 #ifdef HAVE_SYSDC 1188 } 1189 #endif 1190 1191 tqs->stqs_taskq[i] = tq; 1192 } 1193 } 1194 1195 static void 1196 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1197 { 1198 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1199 1200 if (tqs->stqs_taskq == NULL) { 1201 ASSERT3U(tqs->stqs_count, ==, 0); 1202 return; 1203 } 1204 1205 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1206 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1207 taskq_destroy(tqs->stqs_taskq[i]); 1208 } 1209 1210 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1211 tqs->stqs_taskq = NULL; 1212 } 1213 1214 /* 1215 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1216 * Note that a type may have multiple discrete taskqs to avoid lock contention 1217 * on the taskq itself. 1218 */ 1219 static taskq_t * 1220 spa_taskq_dispatch_select(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1221 zio_t *zio) 1222 { 1223 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1224 taskq_t *tq; 1225 1226 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1227 ASSERT3U(tqs->stqs_count, !=, 0); 1228 1229 if ((t == ZIO_TYPE_WRITE) && (q == ZIO_TASKQ_ISSUE) && 1230 (zio != NULL) && (zio->io_wr_iss_tq != NULL)) { 1231 /* dispatch to assigned write issue taskq */ 1232 tq = zio->io_wr_iss_tq; 1233 return (tq); 1234 } 1235 1236 if (tqs->stqs_count == 1) { 1237 tq = tqs->stqs_taskq[0]; 1238 } else { 1239 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1240 } 1241 return (tq); 1242 } 1243 1244 void 1245 spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1246 task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent, 1247 zio_t *zio) 1248 { 1249 taskq_t *tq = spa_taskq_dispatch_select(spa, t, q, zio); 1250 taskq_dispatch_ent(tq, func, arg, flags, ent); 1251 } 1252 1253 /* 1254 * Same as spa_taskq_dispatch_ent() but block on the task until completion. 1255 */ 1256 void 1257 spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1258 task_func_t *func, void *arg, uint_t flags) 1259 { 1260 taskq_t *tq = spa_taskq_dispatch_select(spa, t, q, NULL); 1261 taskqid_t id = taskq_dispatch(tq, func, arg, flags); 1262 if (id) 1263 taskq_wait_id(tq, id); 1264 } 1265 1266 static void 1267 spa_create_zio_taskqs(spa_t *spa) 1268 { 1269 for (int t = 0; t < ZIO_TYPES; t++) { 1270 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1271 spa_taskqs_init(spa, t, q); 1272 } 1273 } 1274 } 1275 1276 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1277 static void 1278 spa_thread(void *arg) 1279 { 1280 psetid_t zio_taskq_psrset_bind = PS_NONE; 1281 callb_cpr_t cprinfo; 1282 1283 spa_t *spa = arg; 1284 user_t *pu = PTOU(curproc); 1285 1286 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1287 spa->spa_name); 1288 1289 ASSERT(curproc != &p0); 1290 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1291 "zpool-%s", spa->spa_name); 1292 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1293 1294 /* bind this thread to the requested psrset */ 1295 if (zio_taskq_psrset_bind != PS_NONE) { 1296 pool_lock(); 1297 mutex_enter(&cpu_lock); 1298 mutex_enter(&pidlock); 1299 mutex_enter(&curproc->p_lock); 1300 1301 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1302 0, NULL, NULL) == 0) { 1303 curthread->t_bind_pset = zio_taskq_psrset_bind; 1304 } else { 1305 cmn_err(CE_WARN, 1306 "Couldn't bind process for zfs pool \"%s\" to " 1307 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1308 } 1309 1310 mutex_exit(&curproc->p_lock); 1311 mutex_exit(&pidlock); 1312 mutex_exit(&cpu_lock); 1313 pool_unlock(); 1314 } 1315 1316 #ifdef HAVE_SYSDC 1317 if (zio_taskq_sysdc) { 1318 sysdc_thread_enter(curthread, 100, 0); 1319 } 1320 #endif 1321 1322 spa->spa_proc = curproc; 1323 spa->spa_did = curthread->t_did; 1324 1325 spa_create_zio_taskqs(spa); 1326 1327 mutex_enter(&spa->spa_proc_lock); 1328 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1329 1330 spa->spa_proc_state = SPA_PROC_ACTIVE; 1331 cv_broadcast(&spa->spa_proc_cv); 1332 1333 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1334 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1335 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1336 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1337 1338 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1339 spa->spa_proc_state = SPA_PROC_GONE; 1340 spa->spa_proc = &p0; 1341 cv_broadcast(&spa->spa_proc_cv); 1342 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1343 1344 mutex_enter(&curproc->p_lock); 1345 lwp_exit(); 1346 } 1347 #endif 1348 1349 extern metaslab_ops_t *metaslab_allocator(spa_t *spa); 1350 1351 /* 1352 * Activate an uninitialized pool. 1353 */ 1354 static void 1355 spa_activate(spa_t *spa, spa_mode_t mode) 1356 { 1357 metaslab_ops_t *msp = metaslab_allocator(spa); 1358 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1359 1360 spa->spa_state = POOL_STATE_ACTIVE; 1361 spa->spa_mode = mode; 1362 spa->spa_read_spacemaps = spa_mode_readable_spacemaps; 1363 1364 spa->spa_normal_class = metaslab_class_create(spa, msp); 1365 spa->spa_log_class = metaslab_class_create(spa, msp); 1366 spa->spa_embedded_log_class = metaslab_class_create(spa, msp); 1367 spa->spa_special_class = metaslab_class_create(spa, msp); 1368 spa->spa_dedup_class = metaslab_class_create(spa, msp); 1369 1370 /* Try to create a covering process */ 1371 mutex_enter(&spa->spa_proc_lock); 1372 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1373 ASSERT(spa->spa_proc == &p0); 1374 spa->spa_did = 0; 1375 1376 #ifdef HAVE_SPA_THREAD 1377 /* Only create a process if we're going to be around a while. */ 1378 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1379 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1380 NULL, 0) == 0) { 1381 spa->spa_proc_state = SPA_PROC_CREATED; 1382 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1383 cv_wait(&spa->spa_proc_cv, 1384 &spa->spa_proc_lock); 1385 } 1386 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1387 ASSERT(spa->spa_proc != &p0); 1388 ASSERT(spa->spa_did != 0); 1389 } else { 1390 #ifdef _KERNEL 1391 cmn_err(CE_WARN, 1392 "Couldn't create process for zfs pool \"%s\"\n", 1393 spa->spa_name); 1394 #endif 1395 } 1396 } 1397 #endif /* HAVE_SPA_THREAD */ 1398 mutex_exit(&spa->spa_proc_lock); 1399 1400 /* If we didn't create a process, we need to create our taskqs. */ 1401 if (spa->spa_proc == &p0) { 1402 spa_create_zio_taskqs(spa); 1403 } 1404 1405 for (size_t i = 0; i < TXG_SIZE; i++) { 1406 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1407 ZIO_FLAG_CANFAIL); 1408 } 1409 1410 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1411 offsetof(vdev_t, vdev_config_dirty_node)); 1412 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1413 offsetof(objset_t, os_evicting_node)); 1414 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1415 offsetof(vdev_t, vdev_state_dirty_node)); 1416 1417 txg_list_create(&spa->spa_vdev_txg_list, spa, 1418 offsetof(struct vdev, vdev_txg_node)); 1419 1420 avl_create(&spa->spa_errlist_scrub, 1421 spa_error_entry_compare, sizeof (spa_error_entry_t), 1422 offsetof(spa_error_entry_t, se_avl)); 1423 avl_create(&spa->spa_errlist_last, 1424 spa_error_entry_compare, sizeof (spa_error_entry_t), 1425 offsetof(spa_error_entry_t, se_avl)); 1426 avl_create(&spa->spa_errlist_healed, 1427 spa_error_entry_compare, sizeof (spa_error_entry_t), 1428 offsetof(spa_error_entry_t, se_avl)); 1429 1430 spa_activate_os(spa); 1431 1432 spa_keystore_init(&spa->spa_keystore); 1433 1434 /* 1435 * This taskq is used to perform zvol-minor-related tasks 1436 * asynchronously. This has several advantages, including easy 1437 * resolution of various deadlocks. 1438 * 1439 * The taskq must be single threaded to ensure tasks are always 1440 * processed in the order in which they were dispatched. 1441 * 1442 * A taskq per pool allows one to keep the pools independent. 1443 * This way if one pool is suspended, it will not impact another. 1444 * 1445 * The preferred location to dispatch a zvol minor task is a sync 1446 * task. In this context, there is easy access to the spa_t and minimal 1447 * error handling is required because the sync task must succeed. 1448 */ 1449 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1450 1, INT_MAX, 0); 1451 1452 /* 1453 * The taskq to preload metaslabs. 1454 */ 1455 spa->spa_metaslab_taskq = taskq_create("z_metaslab", 1456 metaslab_preload_pct, maxclsyspri, 1, INT_MAX, 1457 TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1458 1459 /* 1460 * Taskq dedicated to prefetcher threads: this is used to prevent the 1461 * pool traverse code from monopolizing the global (and limited) 1462 * system_taskq by inappropriately scheduling long running tasks on it. 1463 */ 1464 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1465 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1466 1467 /* 1468 * The taskq to upgrade datasets in this pool. Currently used by 1469 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1470 */ 1471 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1472 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1473 } 1474 1475 /* 1476 * Opposite of spa_activate(). 1477 */ 1478 static void 1479 spa_deactivate(spa_t *spa) 1480 { 1481 ASSERT(spa->spa_sync_on == B_FALSE); 1482 ASSERT(spa->spa_dsl_pool == NULL); 1483 ASSERT(spa->spa_root_vdev == NULL); 1484 ASSERT(spa->spa_async_zio_root == NULL); 1485 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1486 1487 spa_evicting_os_wait(spa); 1488 1489 if (spa->spa_zvol_taskq) { 1490 taskq_destroy(spa->spa_zvol_taskq); 1491 spa->spa_zvol_taskq = NULL; 1492 } 1493 1494 if (spa->spa_metaslab_taskq) { 1495 taskq_destroy(spa->spa_metaslab_taskq); 1496 spa->spa_metaslab_taskq = NULL; 1497 } 1498 1499 if (spa->spa_prefetch_taskq) { 1500 taskq_destroy(spa->spa_prefetch_taskq); 1501 spa->spa_prefetch_taskq = NULL; 1502 } 1503 1504 if (spa->spa_upgrade_taskq) { 1505 taskq_destroy(spa->spa_upgrade_taskq); 1506 spa->spa_upgrade_taskq = NULL; 1507 } 1508 1509 txg_list_destroy(&spa->spa_vdev_txg_list); 1510 1511 list_destroy(&spa->spa_config_dirty_list); 1512 list_destroy(&spa->spa_evicting_os_list); 1513 list_destroy(&spa->spa_state_dirty_list); 1514 1515 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 1516 1517 for (int t = 0; t < ZIO_TYPES; t++) { 1518 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1519 spa_taskqs_fini(spa, t, q); 1520 } 1521 } 1522 1523 for (size_t i = 0; i < TXG_SIZE; i++) { 1524 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1525 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1526 spa->spa_txg_zio[i] = NULL; 1527 } 1528 1529 metaslab_class_destroy(spa->spa_normal_class); 1530 spa->spa_normal_class = NULL; 1531 1532 metaslab_class_destroy(spa->spa_log_class); 1533 spa->spa_log_class = NULL; 1534 1535 metaslab_class_destroy(spa->spa_embedded_log_class); 1536 spa->spa_embedded_log_class = NULL; 1537 1538 metaslab_class_destroy(spa->spa_special_class); 1539 spa->spa_special_class = NULL; 1540 1541 metaslab_class_destroy(spa->spa_dedup_class); 1542 spa->spa_dedup_class = NULL; 1543 1544 /* 1545 * If this was part of an import or the open otherwise failed, we may 1546 * still have errors left in the queues. Empty them just in case. 1547 */ 1548 spa_errlog_drain(spa); 1549 avl_destroy(&spa->spa_errlist_scrub); 1550 avl_destroy(&spa->spa_errlist_last); 1551 avl_destroy(&spa->spa_errlist_healed); 1552 1553 spa_keystore_fini(&spa->spa_keystore); 1554 1555 spa->spa_state = POOL_STATE_UNINITIALIZED; 1556 1557 mutex_enter(&spa->spa_proc_lock); 1558 if (spa->spa_proc_state != SPA_PROC_NONE) { 1559 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1560 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 1561 cv_broadcast(&spa->spa_proc_cv); 1562 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 1563 ASSERT(spa->spa_proc != &p0); 1564 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1565 } 1566 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 1567 spa->spa_proc_state = SPA_PROC_NONE; 1568 } 1569 ASSERT(spa->spa_proc == &p0); 1570 mutex_exit(&spa->spa_proc_lock); 1571 1572 /* 1573 * We want to make sure spa_thread() has actually exited the ZFS 1574 * module, so that the module can't be unloaded out from underneath 1575 * it. 1576 */ 1577 if (spa->spa_did != 0) { 1578 thread_join(spa->spa_did); 1579 spa->spa_did = 0; 1580 } 1581 1582 spa_deactivate_os(spa); 1583 1584 } 1585 1586 /* 1587 * Verify a pool configuration, and construct the vdev tree appropriately. This 1588 * will create all the necessary vdevs in the appropriate layout, with each vdev 1589 * in the CLOSED state. This will prep the pool before open/creation/import. 1590 * All vdev validation is done by the vdev_alloc() routine. 1591 */ 1592 int 1593 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 1594 uint_t id, int atype) 1595 { 1596 nvlist_t **child; 1597 uint_t children; 1598 int error; 1599 1600 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 1601 return (error); 1602 1603 if ((*vdp)->vdev_ops->vdev_op_leaf) 1604 return (0); 1605 1606 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 1607 &child, &children); 1608 1609 if (error == ENOENT) 1610 return (0); 1611 1612 if (error) { 1613 vdev_free(*vdp); 1614 *vdp = NULL; 1615 return (SET_ERROR(EINVAL)); 1616 } 1617 1618 for (int c = 0; c < children; c++) { 1619 vdev_t *vd; 1620 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 1621 atype)) != 0) { 1622 vdev_free(*vdp); 1623 *vdp = NULL; 1624 return (error); 1625 } 1626 } 1627 1628 ASSERT(*vdp != NULL); 1629 1630 return (0); 1631 } 1632 1633 static boolean_t 1634 spa_should_flush_logs_on_unload(spa_t *spa) 1635 { 1636 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 1637 return (B_FALSE); 1638 1639 if (!spa_writeable(spa)) 1640 return (B_FALSE); 1641 1642 if (!spa->spa_sync_on) 1643 return (B_FALSE); 1644 1645 if (spa_state(spa) != POOL_STATE_EXPORTED) 1646 return (B_FALSE); 1647 1648 if (zfs_keep_log_spacemaps_at_export) 1649 return (B_FALSE); 1650 1651 return (B_TRUE); 1652 } 1653 1654 /* 1655 * Opens a transaction that will set the flag that will instruct 1656 * spa_sync to attempt to flush all the metaslabs for that txg. 1657 */ 1658 static void 1659 spa_unload_log_sm_flush_all(spa_t *spa) 1660 { 1661 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 1662 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 1663 1664 ASSERT3U(spa->spa_log_flushall_txg, ==, 0); 1665 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); 1666 1667 dmu_tx_commit(tx); 1668 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 1669 } 1670 1671 static void 1672 spa_unload_log_sm_metadata(spa_t *spa) 1673 { 1674 void *cookie = NULL; 1675 spa_log_sm_t *sls; 1676 log_summary_entry_t *e; 1677 1678 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 1679 &cookie)) != NULL) { 1680 VERIFY0(sls->sls_mscount); 1681 kmem_free(sls, sizeof (spa_log_sm_t)); 1682 } 1683 1684 while ((e = list_remove_head(&spa->spa_log_summary)) != NULL) { 1685 VERIFY0(e->lse_mscount); 1686 kmem_free(e, sizeof (log_summary_entry_t)); 1687 } 1688 1689 spa->spa_unflushed_stats.sus_nblocks = 0; 1690 spa->spa_unflushed_stats.sus_memused = 0; 1691 spa->spa_unflushed_stats.sus_blocklimit = 0; 1692 } 1693 1694 static void 1695 spa_destroy_aux_threads(spa_t *spa) 1696 { 1697 if (spa->spa_condense_zthr != NULL) { 1698 zthr_destroy(spa->spa_condense_zthr); 1699 spa->spa_condense_zthr = NULL; 1700 } 1701 if (spa->spa_checkpoint_discard_zthr != NULL) { 1702 zthr_destroy(spa->spa_checkpoint_discard_zthr); 1703 spa->spa_checkpoint_discard_zthr = NULL; 1704 } 1705 if (spa->spa_livelist_delete_zthr != NULL) { 1706 zthr_destroy(spa->spa_livelist_delete_zthr); 1707 spa->spa_livelist_delete_zthr = NULL; 1708 } 1709 if (spa->spa_livelist_condense_zthr != NULL) { 1710 zthr_destroy(spa->spa_livelist_condense_zthr); 1711 spa->spa_livelist_condense_zthr = NULL; 1712 } 1713 if (spa->spa_raidz_expand_zthr != NULL) { 1714 zthr_destroy(spa->spa_raidz_expand_zthr); 1715 spa->spa_raidz_expand_zthr = NULL; 1716 } 1717 } 1718 1719 /* 1720 * Opposite of spa_load(). 1721 */ 1722 static void 1723 spa_unload(spa_t *spa) 1724 { 1725 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1726 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 1727 1728 spa_import_progress_remove(spa_guid(spa)); 1729 spa_load_note(spa, "UNLOADING"); 1730 1731 spa_wake_waiters(spa); 1732 1733 /* 1734 * If we have set the spa_final_txg, we have already performed the 1735 * tasks below in spa_export_common(). We should not redo it here since 1736 * we delay the final TXGs beyond what spa_final_txg is set at. 1737 */ 1738 if (spa->spa_final_txg == UINT64_MAX) { 1739 /* 1740 * If the log space map feature is enabled and the pool is 1741 * getting exported (but not destroyed), we want to spend some 1742 * time flushing as many metaslabs as we can in an attempt to 1743 * destroy log space maps and save import time. 1744 */ 1745 if (spa_should_flush_logs_on_unload(spa)) 1746 spa_unload_log_sm_flush_all(spa); 1747 1748 /* 1749 * Stop async tasks. 1750 */ 1751 spa_async_suspend(spa); 1752 1753 if (spa->spa_root_vdev) { 1754 vdev_t *root_vdev = spa->spa_root_vdev; 1755 vdev_initialize_stop_all(root_vdev, 1756 VDEV_INITIALIZE_ACTIVE); 1757 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 1758 vdev_autotrim_stop_all(spa); 1759 vdev_rebuild_stop_all(spa); 1760 } 1761 } 1762 1763 /* 1764 * Stop syncing. 1765 */ 1766 if (spa->spa_sync_on) { 1767 txg_sync_stop(spa->spa_dsl_pool); 1768 spa->spa_sync_on = B_FALSE; 1769 } 1770 1771 /* 1772 * This ensures that there is no async metaslab prefetching 1773 * while we attempt to unload the spa. 1774 */ 1775 taskq_wait(spa->spa_metaslab_taskq); 1776 1777 if (spa->spa_mmp.mmp_thread) 1778 mmp_thread_stop(spa); 1779 1780 /* 1781 * Wait for any outstanding async I/O to complete. 1782 */ 1783 if (spa->spa_async_zio_root != NULL) { 1784 for (int i = 0; i < max_ncpus; i++) 1785 (void) zio_wait(spa->spa_async_zio_root[i]); 1786 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 1787 spa->spa_async_zio_root = NULL; 1788 } 1789 1790 if (spa->spa_vdev_removal != NULL) { 1791 spa_vdev_removal_destroy(spa->spa_vdev_removal); 1792 spa->spa_vdev_removal = NULL; 1793 } 1794 1795 spa_destroy_aux_threads(spa); 1796 1797 spa_condense_fini(spa); 1798 1799 bpobj_close(&spa->spa_deferred_bpobj); 1800 1801 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 1802 1803 /* 1804 * Close all vdevs. 1805 */ 1806 if (spa->spa_root_vdev) 1807 vdev_free(spa->spa_root_vdev); 1808 ASSERT(spa->spa_root_vdev == NULL); 1809 1810 /* 1811 * Close the dsl pool. 1812 */ 1813 if (spa->spa_dsl_pool) { 1814 dsl_pool_close(spa->spa_dsl_pool); 1815 spa->spa_dsl_pool = NULL; 1816 spa->spa_meta_objset = NULL; 1817 } 1818 1819 ddt_unload(spa); 1820 brt_unload(spa); 1821 spa_unload_log_sm_metadata(spa); 1822 1823 /* 1824 * Drop and purge level 2 cache 1825 */ 1826 spa_l2cache_drop(spa); 1827 1828 if (spa->spa_spares.sav_vdevs) { 1829 for (int i = 0; i < spa->spa_spares.sav_count; i++) 1830 vdev_free(spa->spa_spares.sav_vdevs[i]); 1831 kmem_free(spa->spa_spares.sav_vdevs, 1832 spa->spa_spares.sav_count * sizeof (void *)); 1833 spa->spa_spares.sav_vdevs = NULL; 1834 } 1835 if (spa->spa_spares.sav_config) { 1836 nvlist_free(spa->spa_spares.sav_config); 1837 spa->spa_spares.sav_config = NULL; 1838 } 1839 spa->spa_spares.sav_count = 0; 1840 1841 if (spa->spa_l2cache.sav_vdevs) { 1842 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1843 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 1844 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 1845 } 1846 kmem_free(spa->spa_l2cache.sav_vdevs, 1847 spa->spa_l2cache.sav_count * sizeof (void *)); 1848 spa->spa_l2cache.sav_vdevs = NULL; 1849 } 1850 if (spa->spa_l2cache.sav_config) { 1851 nvlist_free(spa->spa_l2cache.sav_config); 1852 spa->spa_l2cache.sav_config = NULL; 1853 } 1854 spa->spa_l2cache.sav_count = 0; 1855 1856 spa->spa_async_suspended = 0; 1857 1858 spa->spa_indirect_vdevs_loaded = B_FALSE; 1859 1860 if (spa->spa_comment != NULL) { 1861 spa_strfree(spa->spa_comment); 1862 spa->spa_comment = NULL; 1863 } 1864 if (spa->spa_compatibility != NULL) { 1865 spa_strfree(spa->spa_compatibility); 1866 spa->spa_compatibility = NULL; 1867 } 1868 1869 spa->spa_raidz_expand = NULL; 1870 1871 spa_config_exit(spa, SCL_ALL, spa); 1872 } 1873 1874 /* 1875 * Load (or re-load) the current list of vdevs describing the active spares for 1876 * this pool. When this is called, we have some form of basic information in 1877 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 1878 * then re-generate a more complete list including status information. 1879 */ 1880 void 1881 spa_load_spares(spa_t *spa) 1882 { 1883 nvlist_t **spares; 1884 uint_t nspares; 1885 int i; 1886 vdev_t *vd, *tvd; 1887 1888 #ifndef _KERNEL 1889 /* 1890 * zdb opens both the current state of the pool and the 1891 * checkpointed state (if present), with a different spa_t. 1892 * 1893 * As spare vdevs are shared among open pools, we skip loading 1894 * them when we load the checkpointed state of the pool. 1895 */ 1896 if (!spa_writeable(spa)) 1897 return; 1898 #endif 1899 1900 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1901 1902 /* 1903 * First, close and free any existing spare vdevs. 1904 */ 1905 if (spa->spa_spares.sav_vdevs) { 1906 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1907 vd = spa->spa_spares.sav_vdevs[i]; 1908 1909 /* Undo the call to spa_activate() below */ 1910 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1911 B_FALSE)) != NULL && tvd->vdev_isspare) 1912 spa_spare_remove(tvd); 1913 vdev_close(vd); 1914 vdev_free(vd); 1915 } 1916 1917 kmem_free(spa->spa_spares.sav_vdevs, 1918 spa->spa_spares.sav_count * sizeof (void *)); 1919 } 1920 1921 if (spa->spa_spares.sav_config == NULL) 1922 nspares = 0; 1923 else 1924 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 1925 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 1926 1927 spa->spa_spares.sav_count = (int)nspares; 1928 spa->spa_spares.sav_vdevs = NULL; 1929 1930 if (nspares == 0) 1931 return; 1932 1933 /* 1934 * Construct the array of vdevs, opening them to get status in the 1935 * process. For each spare, there is potentially two different vdev_t 1936 * structures associated with it: one in the list of spares (used only 1937 * for basic validation purposes) and one in the active vdev 1938 * configuration (if it's spared in). During this phase we open and 1939 * validate each vdev on the spare list. If the vdev also exists in the 1940 * active configuration, then we also mark this vdev as an active spare. 1941 */ 1942 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 1943 KM_SLEEP); 1944 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1945 VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, 1946 VDEV_ALLOC_SPARE) == 0); 1947 ASSERT(vd != NULL); 1948 1949 spa->spa_spares.sav_vdevs[i] = vd; 1950 1951 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1952 B_FALSE)) != NULL) { 1953 if (!tvd->vdev_isspare) 1954 spa_spare_add(tvd); 1955 1956 /* 1957 * We only mark the spare active if we were successfully 1958 * able to load the vdev. Otherwise, importing a pool 1959 * with a bad active spare would result in strange 1960 * behavior, because multiple pool would think the spare 1961 * is actively in use. 1962 * 1963 * There is a vulnerability here to an equally bizarre 1964 * circumstance, where a dead active spare is later 1965 * brought back to life (onlined or otherwise). Given 1966 * the rarity of this scenario, and the extra complexity 1967 * it adds, we ignore the possibility. 1968 */ 1969 if (!vdev_is_dead(tvd)) 1970 spa_spare_activate(tvd); 1971 } 1972 1973 vd->vdev_top = vd; 1974 vd->vdev_aux = &spa->spa_spares; 1975 1976 if (vdev_open(vd) != 0) 1977 continue; 1978 1979 if (vdev_validate_aux(vd) == 0) 1980 spa_spare_add(vd); 1981 } 1982 1983 /* 1984 * Recompute the stashed list of spares, with status information 1985 * this time. 1986 */ 1987 fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES); 1988 1989 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 1990 KM_SLEEP); 1991 for (i = 0; i < spa->spa_spares.sav_count; i++) 1992 spares[i] = vdev_config_generate(spa, 1993 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 1994 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 1995 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 1996 spa->spa_spares.sav_count); 1997 for (i = 0; i < spa->spa_spares.sav_count; i++) 1998 nvlist_free(spares[i]); 1999 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 2000 } 2001 2002 /* 2003 * Load (or re-load) the current list of vdevs describing the active l2cache for 2004 * this pool. When this is called, we have some form of basic information in 2005 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 2006 * then re-generate a more complete list including status information. 2007 * Devices which are already active have their details maintained, and are 2008 * not re-opened. 2009 */ 2010 void 2011 spa_load_l2cache(spa_t *spa) 2012 { 2013 nvlist_t **l2cache = NULL; 2014 uint_t nl2cache; 2015 int i, j, oldnvdevs; 2016 uint64_t guid; 2017 vdev_t *vd, **oldvdevs, **newvdevs; 2018 spa_aux_vdev_t *sav = &spa->spa_l2cache; 2019 2020 #ifndef _KERNEL 2021 /* 2022 * zdb opens both the current state of the pool and the 2023 * checkpointed state (if present), with a different spa_t. 2024 * 2025 * As L2 caches are part of the ARC which is shared among open 2026 * pools, we skip loading them when we load the checkpointed 2027 * state of the pool. 2028 */ 2029 if (!spa_writeable(spa)) 2030 return; 2031 #endif 2032 2033 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 2034 2035 oldvdevs = sav->sav_vdevs; 2036 oldnvdevs = sav->sav_count; 2037 sav->sav_vdevs = NULL; 2038 sav->sav_count = 0; 2039 2040 if (sav->sav_config == NULL) { 2041 nl2cache = 0; 2042 newvdevs = NULL; 2043 goto out; 2044 } 2045 2046 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, 2047 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 2048 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 2049 2050 /* 2051 * Process new nvlist of vdevs. 2052 */ 2053 for (i = 0; i < nl2cache; i++) { 2054 guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID); 2055 2056 newvdevs[i] = NULL; 2057 for (j = 0; j < oldnvdevs; j++) { 2058 vd = oldvdevs[j]; 2059 if (vd != NULL && guid == vd->vdev_guid) { 2060 /* 2061 * Retain previous vdev for add/remove ops. 2062 */ 2063 newvdevs[i] = vd; 2064 oldvdevs[j] = NULL; 2065 break; 2066 } 2067 } 2068 2069 if (newvdevs[i] == NULL) { 2070 /* 2071 * Create new vdev 2072 */ 2073 VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 2074 VDEV_ALLOC_L2CACHE) == 0); 2075 ASSERT(vd != NULL); 2076 newvdevs[i] = vd; 2077 2078 /* 2079 * Commit this vdev as an l2cache device, 2080 * even if it fails to open. 2081 */ 2082 spa_l2cache_add(vd); 2083 2084 vd->vdev_top = vd; 2085 vd->vdev_aux = sav; 2086 2087 spa_l2cache_activate(vd); 2088 2089 if (vdev_open(vd) != 0) 2090 continue; 2091 2092 (void) vdev_validate_aux(vd); 2093 2094 if (!vdev_is_dead(vd)) 2095 l2arc_add_vdev(spa, vd); 2096 2097 /* 2098 * Upon cache device addition to a pool or pool 2099 * creation with a cache device or if the header 2100 * of the device is invalid we issue an async 2101 * TRIM command for the whole device which will 2102 * execute if l2arc_trim_ahead > 0. 2103 */ 2104 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 2105 } 2106 } 2107 2108 sav->sav_vdevs = newvdevs; 2109 sav->sav_count = (int)nl2cache; 2110 2111 /* 2112 * Recompute the stashed list of l2cache devices, with status 2113 * information this time. 2114 */ 2115 fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE); 2116 2117 if (sav->sav_count > 0) 2118 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 2119 KM_SLEEP); 2120 for (i = 0; i < sav->sav_count; i++) 2121 l2cache[i] = vdev_config_generate(spa, 2122 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 2123 fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 2124 (const nvlist_t * const *)l2cache, sav->sav_count); 2125 2126 out: 2127 /* 2128 * Purge vdevs that were dropped 2129 */ 2130 if (oldvdevs) { 2131 for (i = 0; i < oldnvdevs; i++) { 2132 uint64_t pool; 2133 2134 vd = oldvdevs[i]; 2135 if (vd != NULL) { 2136 ASSERT(vd->vdev_isl2cache); 2137 2138 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 2139 pool != 0ULL && l2arc_vdev_present(vd)) 2140 l2arc_remove_vdev(vd); 2141 vdev_clear_stats(vd); 2142 vdev_free(vd); 2143 } 2144 } 2145 2146 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2147 } 2148 2149 for (i = 0; i < sav->sav_count; i++) 2150 nvlist_free(l2cache[i]); 2151 if (sav->sav_count) 2152 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2153 } 2154 2155 static int 2156 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2157 { 2158 dmu_buf_t *db; 2159 char *packed = NULL; 2160 size_t nvsize = 0; 2161 int error; 2162 *value = NULL; 2163 2164 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2165 if (error) 2166 return (error); 2167 2168 nvsize = *(uint64_t *)db->db_data; 2169 dmu_buf_rele(db, FTAG); 2170 2171 packed = vmem_alloc(nvsize, KM_SLEEP); 2172 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2173 DMU_READ_PREFETCH); 2174 if (error == 0) 2175 error = nvlist_unpack(packed, nvsize, value, 0); 2176 vmem_free(packed, nvsize); 2177 2178 return (error); 2179 } 2180 2181 /* 2182 * Concrete top-level vdevs that are not missing and are not logs. At every 2183 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2184 */ 2185 static uint64_t 2186 spa_healthy_core_tvds(spa_t *spa) 2187 { 2188 vdev_t *rvd = spa->spa_root_vdev; 2189 uint64_t tvds = 0; 2190 2191 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2192 vdev_t *vd = rvd->vdev_child[i]; 2193 if (vd->vdev_islog) 2194 continue; 2195 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2196 tvds++; 2197 } 2198 2199 return (tvds); 2200 } 2201 2202 /* 2203 * Checks to see if the given vdev could not be opened, in which case we post a 2204 * sysevent to notify the autoreplace code that the device has been removed. 2205 */ 2206 static void 2207 spa_check_removed(vdev_t *vd) 2208 { 2209 for (uint64_t c = 0; c < vd->vdev_children; c++) 2210 spa_check_removed(vd->vdev_child[c]); 2211 2212 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2213 vdev_is_concrete(vd)) { 2214 zfs_post_autoreplace(vd->vdev_spa, vd); 2215 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2216 } 2217 } 2218 2219 static int 2220 spa_check_for_missing_logs(spa_t *spa) 2221 { 2222 vdev_t *rvd = spa->spa_root_vdev; 2223 2224 /* 2225 * If we're doing a normal import, then build up any additional 2226 * diagnostic information about missing log devices. 2227 * We'll pass this up to the user for further processing. 2228 */ 2229 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2230 nvlist_t **child, *nv; 2231 uint64_t idx = 0; 2232 2233 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2234 KM_SLEEP); 2235 nv = fnvlist_alloc(); 2236 2237 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2238 vdev_t *tvd = rvd->vdev_child[c]; 2239 2240 /* 2241 * We consider a device as missing only if it failed 2242 * to open (i.e. offline or faulted is not considered 2243 * as missing). 2244 */ 2245 if (tvd->vdev_islog && 2246 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2247 child[idx++] = vdev_config_generate(spa, tvd, 2248 B_FALSE, VDEV_CONFIG_MISSING); 2249 } 2250 } 2251 2252 if (idx > 0) { 2253 fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 2254 (const nvlist_t * const *)child, idx); 2255 fnvlist_add_nvlist(spa->spa_load_info, 2256 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2257 2258 for (uint64_t i = 0; i < idx; i++) 2259 nvlist_free(child[i]); 2260 } 2261 nvlist_free(nv); 2262 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2263 2264 if (idx > 0) { 2265 spa_load_failed(spa, "some log devices are missing"); 2266 vdev_dbgmsg_print_tree(rvd, 2); 2267 return (SET_ERROR(ENXIO)); 2268 } 2269 } else { 2270 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2271 vdev_t *tvd = rvd->vdev_child[c]; 2272 2273 if (tvd->vdev_islog && 2274 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2275 spa_set_log_state(spa, SPA_LOG_CLEAR); 2276 spa_load_note(spa, "some log devices are " 2277 "missing, ZIL is dropped."); 2278 vdev_dbgmsg_print_tree(rvd, 2); 2279 break; 2280 } 2281 } 2282 } 2283 2284 return (0); 2285 } 2286 2287 /* 2288 * Check for missing log devices 2289 */ 2290 static boolean_t 2291 spa_check_logs(spa_t *spa) 2292 { 2293 boolean_t rv = B_FALSE; 2294 dsl_pool_t *dp = spa_get_dsl(spa); 2295 2296 switch (spa->spa_log_state) { 2297 default: 2298 break; 2299 case SPA_LOG_MISSING: 2300 /* need to recheck in case slog has been restored */ 2301 case SPA_LOG_UNKNOWN: 2302 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2303 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2304 if (rv) 2305 spa_set_log_state(spa, SPA_LOG_MISSING); 2306 break; 2307 } 2308 return (rv); 2309 } 2310 2311 /* 2312 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2313 */ 2314 static boolean_t 2315 spa_passivate_log(spa_t *spa) 2316 { 2317 vdev_t *rvd = spa->spa_root_vdev; 2318 boolean_t slog_found = B_FALSE; 2319 2320 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2321 2322 for (int c = 0; c < rvd->vdev_children; c++) { 2323 vdev_t *tvd = rvd->vdev_child[c]; 2324 2325 if (tvd->vdev_islog) { 2326 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2327 metaslab_group_passivate(tvd->vdev_mg); 2328 slog_found = B_TRUE; 2329 } 2330 } 2331 2332 return (slog_found); 2333 } 2334 2335 /* 2336 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2337 */ 2338 static void 2339 spa_activate_log(spa_t *spa) 2340 { 2341 vdev_t *rvd = spa->spa_root_vdev; 2342 2343 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2344 2345 for (int c = 0; c < rvd->vdev_children; c++) { 2346 vdev_t *tvd = rvd->vdev_child[c]; 2347 2348 if (tvd->vdev_islog) { 2349 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2350 metaslab_group_activate(tvd->vdev_mg); 2351 } 2352 } 2353 } 2354 2355 int 2356 spa_reset_logs(spa_t *spa) 2357 { 2358 int error; 2359 2360 error = dmu_objset_find(spa_name(spa), zil_reset, 2361 NULL, DS_FIND_CHILDREN); 2362 if (error == 0) { 2363 /* 2364 * We successfully offlined the log device, sync out the 2365 * current txg so that the "stubby" block can be removed 2366 * by zil_sync(). 2367 */ 2368 txg_wait_synced(spa->spa_dsl_pool, 0); 2369 } 2370 return (error); 2371 } 2372 2373 static void 2374 spa_aux_check_removed(spa_aux_vdev_t *sav) 2375 { 2376 for (int i = 0; i < sav->sav_count; i++) 2377 spa_check_removed(sav->sav_vdevs[i]); 2378 } 2379 2380 void 2381 spa_claim_notify(zio_t *zio) 2382 { 2383 spa_t *spa = zio->io_spa; 2384 2385 if (zio->io_error) 2386 return; 2387 2388 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2389 if (spa->spa_claim_max_txg < zio->io_bp->blk_birth) 2390 spa->spa_claim_max_txg = zio->io_bp->blk_birth; 2391 mutex_exit(&spa->spa_props_lock); 2392 } 2393 2394 typedef struct spa_load_error { 2395 boolean_t sle_verify_data; 2396 uint64_t sle_meta_count; 2397 uint64_t sle_data_count; 2398 } spa_load_error_t; 2399 2400 static void 2401 spa_load_verify_done(zio_t *zio) 2402 { 2403 blkptr_t *bp = zio->io_bp; 2404 spa_load_error_t *sle = zio->io_private; 2405 dmu_object_type_t type = BP_GET_TYPE(bp); 2406 int error = zio->io_error; 2407 spa_t *spa = zio->io_spa; 2408 2409 abd_free(zio->io_abd); 2410 if (error) { 2411 if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && 2412 type != DMU_OT_INTENT_LOG) 2413 atomic_inc_64(&sle->sle_meta_count); 2414 else 2415 atomic_inc_64(&sle->sle_data_count); 2416 } 2417 2418 mutex_enter(&spa->spa_scrub_lock); 2419 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 2420 cv_broadcast(&spa->spa_scrub_io_cv); 2421 mutex_exit(&spa->spa_scrub_lock); 2422 } 2423 2424 /* 2425 * Maximum number of inflight bytes is the log2 fraction of the arc size. 2426 * By default, we set it to 1/16th of the arc. 2427 */ 2428 static uint_t spa_load_verify_shift = 4; 2429 static int spa_load_verify_metadata = B_TRUE; 2430 static int spa_load_verify_data = B_TRUE; 2431 2432 static int 2433 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 2434 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 2435 { 2436 zio_t *rio = arg; 2437 spa_load_error_t *sle = rio->io_private; 2438 2439 (void) zilog, (void) dnp; 2440 2441 /* 2442 * Note: normally this routine will not be called if 2443 * spa_load_verify_metadata is not set. However, it may be useful 2444 * to manually set the flag after the traversal has begun. 2445 */ 2446 if (!spa_load_verify_metadata) 2447 return (0); 2448 2449 /* 2450 * Sanity check the block pointer in order to detect obvious damage 2451 * before using the contents in subsequent checks or in zio_read(). 2452 * When damaged consider it to be a metadata error since we cannot 2453 * trust the BP_GET_TYPE and BP_GET_LEVEL values. 2454 */ 2455 if (!zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) { 2456 atomic_inc_64(&sle->sle_meta_count); 2457 return (0); 2458 } 2459 2460 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 2461 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 2462 return (0); 2463 2464 if (!BP_IS_METADATA(bp) && 2465 (!spa_load_verify_data || !sle->sle_verify_data)) 2466 return (0); 2467 2468 uint64_t maxinflight_bytes = 2469 arc_target_bytes() >> spa_load_verify_shift; 2470 size_t size = BP_GET_PSIZE(bp); 2471 2472 mutex_enter(&spa->spa_scrub_lock); 2473 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 2474 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 2475 spa->spa_load_verify_bytes += size; 2476 mutex_exit(&spa->spa_scrub_lock); 2477 2478 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 2479 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 2480 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 2481 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 2482 return (0); 2483 } 2484 2485 static int 2486 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 2487 { 2488 (void) dp, (void) arg; 2489 2490 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 2491 return (SET_ERROR(ENAMETOOLONG)); 2492 2493 return (0); 2494 } 2495 2496 static int 2497 spa_load_verify(spa_t *spa) 2498 { 2499 zio_t *rio; 2500 spa_load_error_t sle = { 0 }; 2501 zpool_load_policy_t policy; 2502 boolean_t verify_ok = B_FALSE; 2503 int error = 0; 2504 2505 zpool_get_load_policy(spa->spa_config, &policy); 2506 2507 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND || 2508 policy.zlp_maxmeta == UINT64_MAX) 2509 return (0); 2510 2511 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 2512 error = dmu_objset_find_dp(spa->spa_dsl_pool, 2513 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 2514 DS_FIND_CHILDREN); 2515 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 2516 if (error != 0) 2517 return (error); 2518 2519 /* 2520 * Verify data only if we are rewinding or error limit was set. 2521 * Otherwise nothing except dbgmsg care about it to waste time. 2522 */ 2523 sle.sle_verify_data = (policy.zlp_rewind & ZPOOL_REWIND_MASK) || 2524 (policy.zlp_maxdata < UINT64_MAX); 2525 2526 rio = zio_root(spa, NULL, &sle, 2527 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 2528 2529 if (spa_load_verify_metadata) { 2530 if (spa->spa_extreme_rewind) { 2531 spa_load_note(spa, "performing a complete scan of the " 2532 "pool since extreme rewind is on. This may take " 2533 "a very long time.\n (spa_load_verify_data=%u, " 2534 "spa_load_verify_metadata=%u)", 2535 spa_load_verify_data, spa_load_verify_metadata); 2536 } 2537 2538 error = traverse_pool(spa, spa->spa_verify_min_txg, 2539 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 2540 TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio); 2541 } 2542 2543 (void) zio_wait(rio); 2544 ASSERT0(spa->spa_load_verify_bytes); 2545 2546 spa->spa_load_meta_errors = sle.sle_meta_count; 2547 spa->spa_load_data_errors = sle.sle_data_count; 2548 2549 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 2550 spa_load_note(spa, "spa_load_verify found %llu metadata errors " 2551 "and %llu data errors", (u_longlong_t)sle.sle_meta_count, 2552 (u_longlong_t)sle.sle_data_count); 2553 } 2554 2555 if (spa_load_verify_dryrun || 2556 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 2557 sle.sle_data_count <= policy.zlp_maxdata)) { 2558 int64_t loss = 0; 2559 2560 verify_ok = B_TRUE; 2561 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 2562 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 2563 2564 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; 2565 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME, 2566 spa->spa_load_txg_ts); 2567 fnvlist_add_int64(spa->spa_load_info, ZPOOL_CONFIG_REWIND_TIME, 2568 loss); 2569 fnvlist_add_uint64(spa->spa_load_info, 2570 ZPOOL_CONFIG_LOAD_META_ERRORS, sle.sle_meta_count); 2571 fnvlist_add_uint64(spa->spa_load_info, 2572 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count); 2573 } else { 2574 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 2575 } 2576 2577 if (spa_load_verify_dryrun) 2578 return (0); 2579 2580 if (error) { 2581 if (error != ENXIO && error != EIO) 2582 error = SET_ERROR(EIO); 2583 return (error); 2584 } 2585 2586 return (verify_ok ? 0 : EIO); 2587 } 2588 2589 /* 2590 * Find a value in the pool props object. 2591 */ 2592 static void 2593 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 2594 { 2595 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 2596 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 2597 } 2598 2599 /* 2600 * Find a value in the pool directory object. 2601 */ 2602 static int 2603 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 2604 { 2605 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2606 name, sizeof (uint64_t), 1, val); 2607 2608 if (error != 0 && (error != ENOENT || log_enoent)) { 2609 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 2610 "[error=%d]", name, error); 2611 } 2612 2613 return (error); 2614 } 2615 2616 static int 2617 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 2618 { 2619 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 2620 return (SET_ERROR(err)); 2621 } 2622 2623 boolean_t 2624 spa_livelist_delete_check(spa_t *spa) 2625 { 2626 return (spa->spa_livelists_to_delete != 0); 2627 } 2628 2629 static boolean_t 2630 spa_livelist_delete_cb_check(void *arg, zthr_t *z) 2631 { 2632 (void) z; 2633 spa_t *spa = arg; 2634 return (spa_livelist_delete_check(spa)); 2635 } 2636 2637 static int 2638 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 2639 { 2640 spa_t *spa = arg; 2641 zio_free(spa, tx->tx_txg, bp); 2642 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 2643 -bp_get_dsize_sync(spa, bp), 2644 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 2645 return (0); 2646 } 2647 2648 static int 2649 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 2650 { 2651 int err; 2652 zap_cursor_t zc; 2653 zap_attribute_t za; 2654 zap_cursor_init(&zc, os, zap_obj); 2655 err = zap_cursor_retrieve(&zc, &za); 2656 zap_cursor_fini(&zc); 2657 if (err == 0) 2658 *llp = za.za_first_integer; 2659 return (err); 2660 } 2661 2662 /* 2663 * Components of livelist deletion that must be performed in syncing 2664 * context: freeing block pointers and updating the pool-wide data 2665 * structures to indicate how much work is left to do 2666 */ 2667 typedef struct sublist_delete_arg { 2668 spa_t *spa; 2669 dsl_deadlist_t *ll; 2670 uint64_t key; 2671 bplist_t *to_free; 2672 } sublist_delete_arg_t; 2673 2674 static void 2675 sublist_delete_sync(void *arg, dmu_tx_t *tx) 2676 { 2677 sublist_delete_arg_t *sda = arg; 2678 spa_t *spa = sda->spa; 2679 dsl_deadlist_t *ll = sda->ll; 2680 uint64_t key = sda->key; 2681 bplist_t *to_free = sda->to_free; 2682 2683 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 2684 dsl_deadlist_remove_entry(ll, key, tx); 2685 } 2686 2687 typedef struct livelist_delete_arg { 2688 spa_t *spa; 2689 uint64_t ll_obj; 2690 uint64_t zap_obj; 2691 } livelist_delete_arg_t; 2692 2693 static void 2694 livelist_delete_sync(void *arg, dmu_tx_t *tx) 2695 { 2696 livelist_delete_arg_t *lda = arg; 2697 spa_t *spa = lda->spa; 2698 uint64_t ll_obj = lda->ll_obj; 2699 uint64_t zap_obj = lda->zap_obj; 2700 objset_t *mos = spa->spa_meta_objset; 2701 uint64_t count; 2702 2703 /* free the livelist and decrement the feature count */ 2704 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 2705 dsl_deadlist_free(mos, ll_obj, tx); 2706 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 2707 VERIFY0(zap_count(mos, zap_obj, &count)); 2708 if (count == 0) { 2709 /* no more livelists to delete */ 2710 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 2711 DMU_POOL_DELETED_CLONES, tx)); 2712 VERIFY0(zap_destroy(mos, zap_obj, tx)); 2713 spa->spa_livelists_to_delete = 0; 2714 spa_notify_waiters(spa); 2715 } 2716 } 2717 2718 /* 2719 * Load in the value for the livelist to be removed and open it. Then, 2720 * load its first sublist and determine which block pointers should actually 2721 * be freed. Then, call a synctask which performs the actual frees and updates 2722 * the pool-wide livelist data. 2723 */ 2724 static void 2725 spa_livelist_delete_cb(void *arg, zthr_t *z) 2726 { 2727 spa_t *spa = arg; 2728 uint64_t ll_obj = 0, count; 2729 objset_t *mos = spa->spa_meta_objset; 2730 uint64_t zap_obj = spa->spa_livelists_to_delete; 2731 /* 2732 * Determine the next livelist to delete. This function should only 2733 * be called if there is at least one deleted clone. 2734 */ 2735 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 2736 VERIFY0(zap_count(mos, ll_obj, &count)); 2737 if (count > 0) { 2738 dsl_deadlist_t *ll; 2739 dsl_deadlist_entry_t *dle; 2740 bplist_t to_free; 2741 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 2742 dsl_deadlist_open(ll, mos, ll_obj); 2743 dle = dsl_deadlist_first(ll); 2744 ASSERT3P(dle, !=, NULL); 2745 bplist_create(&to_free); 2746 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 2747 z, NULL); 2748 if (err == 0) { 2749 sublist_delete_arg_t sync_arg = { 2750 .spa = spa, 2751 .ll = ll, 2752 .key = dle->dle_mintxg, 2753 .to_free = &to_free 2754 }; 2755 zfs_dbgmsg("deleting sublist (id %llu) from" 2756 " livelist %llu, %lld remaining", 2757 (u_longlong_t)dle->dle_bpobj.bpo_object, 2758 (u_longlong_t)ll_obj, (longlong_t)count - 1); 2759 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 2760 sublist_delete_sync, &sync_arg, 0, 2761 ZFS_SPACE_CHECK_DESTROY)); 2762 } else { 2763 VERIFY3U(err, ==, EINTR); 2764 } 2765 bplist_clear(&to_free); 2766 bplist_destroy(&to_free); 2767 dsl_deadlist_close(ll); 2768 kmem_free(ll, sizeof (dsl_deadlist_t)); 2769 } else { 2770 livelist_delete_arg_t sync_arg = { 2771 .spa = spa, 2772 .ll_obj = ll_obj, 2773 .zap_obj = zap_obj 2774 }; 2775 zfs_dbgmsg("deletion of livelist %llu completed", 2776 (u_longlong_t)ll_obj); 2777 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 2778 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 2779 } 2780 } 2781 2782 static void 2783 spa_start_livelist_destroy_thread(spa_t *spa) 2784 { 2785 ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL); 2786 spa->spa_livelist_delete_zthr = 2787 zthr_create("z_livelist_destroy", 2788 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa, 2789 minclsyspri); 2790 } 2791 2792 typedef struct livelist_new_arg { 2793 bplist_t *allocs; 2794 bplist_t *frees; 2795 } livelist_new_arg_t; 2796 2797 static int 2798 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 2799 dmu_tx_t *tx) 2800 { 2801 ASSERT(tx == NULL); 2802 livelist_new_arg_t *lna = arg; 2803 if (bp_freed) { 2804 bplist_append(lna->frees, bp); 2805 } else { 2806 bplist_append(lna->allocs, bp); 2807 zfs_livelist_condense_new_alloc++; 2808 } 2809 return (0); 2810 } 2811 2812 typedef struct livelist_condense_arg { 2813 spa_t *spa; 2814 bplist_t to_keep; 2815 uint64_t first_size; 2816 uint64_t next_size; 2817 } livelist_condense_arg_t; 2818 2819 static void 2820 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 2821 { 2822 livelist_condense_arg_t *lca = arg; 2823 spa_t *spa = lca->spa; 2824 bplist_t new_frees; 2825 dsl_dataset_t *ds = spa->spa_to_condense.ds; 2826 2827 /* Have we been cancelled? */ 2828 if (spa->spa_to_condense.cancelled) { 2829 zfs_livelist_condense_sync_cancel++; 2830 goto out; 2831 } 2832 2833 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2834 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2835 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 2836 2837 /* 2838 * It's possible that the livelist was changed while the zthr was 2839 * running. Therefore, we need to check for new blkptrs in the two 2840 * entries being condensed and continue to track them in the livelist. 2841 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 2842 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 2843 * we need to sort them into two different bplists. 2844 */ 2845 uint64_t first_obj = first->dle_bpobj.bpo_object; 2846 uint64_t next_obj = next->dle_bpobj.bpo_object; 2847 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2848 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2849 2850 bplist_create(&new_frees); 2851 livelist_new_arg_t new_bps = { 2852 .allocs = &lca->to_keep, 2853 .frees = &new_frees, 2854 }; 2855 2856 if (cur_first_size > lca->first_size) { 2857 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 2858 livelist_track_new_cb, &new_bps, lca->first_size)); 2859 } 2860 if (cur_next_size > lca->next_size) { 2861 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 2862 livelist_track_new_cb, &new_bps, lca->next_size)); 2863 } 2864 2865 dsl_deadlist_clear_entry(first, ll, tx); 2866 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 2867 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 2868 2869 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 2870 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 2871 bplist_destroy(&new_frees); 2872 2873 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 2874 dsl_dataset_name(ds, dsname); 2875 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 2876 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 2877 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, 2878 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, 2879 (u_longlong_t)cur_first_size, (u_longlong_t)next_obj, 2880 (u_longlong_t)cur_next_size, 2881 (u_longlong_t)first->dle_bpobj.bpo_object, 2882 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 2883 out: 2884 dmu_buf_rele(ds->ds_dbuf, spa); 2885 spa->spa_to_condense.ds = NULL; 2886 bplist_clear(&lca->to_keep); 2887 bplist_destroy(&lca->to_keep); 2888 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2889 spa->spa_to_condense.syncing = B_FALSE; 2890 } 2891 2892 static void 2893 spa_livelist_condense_cb(void *arg, zthr_t *t) 2894 { 2895 while (zfs_livelist_condense_zthr_pause && 2896 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2897 delay(1); 2898 2899 spa_t *spa = arg; 2900 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2901 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2902 uint64_t first_size, next_size; 2903 2904 livelist_condense_arg_t *lca = 2905 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 2906 bplist_create(&lca->to_keep); 2907 2908 /* 2909 * Process the livelists (matching FREEs and ALLOCs) in open context 2910 * so we have minimal work in syncing context to condense. 2911 * 2912 * We save bpobj sizes (first_size and next_size) to use later in 2913 * syncing context to determine if entries were added to these sublists 2914 * while in open context. This is possible because the clone is still 2915 * active and open for normal writes and we want to make sure the new, 2916 * unprocessed blockpointers are inserted into the livelist normally. 2917 * 2918 * Note that dsl_process_sub_livelist() both stores the size number of 2919 * blockpointers and iterates over them while the bpobj's lock held, so 2920 * the sizes returned to us are consistent which what was actually 2921 * processed. 2922 */ 2923 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 2924 &first_size); 2925 if (err == 0) 2926 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 2927 t, &next_size); 2928 2929 if (err == 0) { 2930 while (zfs_livelist_condense_sync_pause && 2931 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2932 delay(1); 2933 2934 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2935 dmu_tx_mark_netfree(tx); 2936 dmu_tx_hold_space(tx, 1); 2937 err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE); 2938 if (err == 0) { 2939 /* 2940 * Prevent the condense zthr restarting before 2941 * the synctask completes. 2942 */ 2943 spa->spa_to_condense.syncing = B_TRUE; 2944 lca->spa = spa; 2945 lca->first_size = first_size; 2946 lca->next_size = next_size; 2947 dsl_sync_task_nowait(spa_get_dsl(spa), 2948 spa_livelist_condense_sync, lca, tx); 2949 dmu_tx_commit(tx); 2950 return; 2951 } 2952 } 2953 /* 2954 * Condensing can not continue: either it was externally stopped or 2955 * we were unable to assign to a tx because the pool has run out of 2956 * space. In the second case, we'll just end up trying to condense 2957 * again in a later txg. 2958 */ 2959 ASSERT(err != 0); 2960 bplist_clear(&lca->to_keep); 2961 bplist_destroy(&lca->to_keep); 2962 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2963 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 2964 spa->spa_to_condense.ds = NULL; 2965 if (err == EINTR) 2966 zfs_livelist_condense_zthr_cancel++; 2967 } 2968 2969 /* 2970 * Check that there is something to condense but that a condense is not 2971 * already in progress and that condensing has not been cancelled. 2972 */ 2973 static boolean_t 2974 spa_livelist_condense_cb_check(void *arg, zthr_t *z) 2975 { 2976 (void) z; 2977 spa_t *spa = arg; 2978 if ((spa->spa_to_condense.ds != NULL) && 2979 (spa->spa_to_condense.syncing == B_FALSE) && 2980 (spa->spa_to_condense.cancelled == B_FALSE)) { 2981 return (B_TRUE); 2982 } 2983 return (B_FALSE); 2984 } 2985 2986 static void 2987 spa_start_livelist_condensing_thread(spa_t *spa) 2988 { 2989 spa->spa_to_condense.ds = NULL; 2990 spa->spa_to_condense.first = NULL; 2991 spa->spa_to_condense.next = NULL; 2992 spa->spa_to_condense.syncing = B_FALSE; 2993 spa->spa_to_condense.cancelled = B_FALSE; 2994 2995 ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL); 2996 spa->spa_livelist_condense_zthr = 2997 zthr_create("z_livelist_condense", 2998 spa_livelist_condense_cb_check, 2999 spa_livelist_condense_cb, spa, minclsyspri); 3000 } 3001 3002 static void 3003 spa_spawn_aux_threads(spa_t *spa) 3004 { 3005 ASSERT(spa_writeable(spa)); 3006 3007 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 3008 3009 spa_start_raidz_expansion_thread(spa); 3010 spa_start_indirect_condensing_thread(spa); 3011 spa_start_livelist_destroy_thread(spa); 3012 spa_start_livelist_condensing_thread(spa); 3013 3014 ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL); 3015 spa->spa_checkpoint_discard_zthr = 3016 zthr_create("z_checkpoint_discard", 3017 spa_checkpoint_discard_thread_check, 3018 spa_checkpoint_discard_thread, spa, minclsyspri); 3019 } 3020 3021 /* 3022 * Fix up config after a partly-completed split. This is done with the 3023 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 3024 * pool have that entry in their config, but only the splitting one contains 3025 * a list of all the guids of the vdevs that are being split off. 3026 * 3027 * This function determines what to do with that list: either rejoin 3028 * all the disks to the pool, or complete the splitting process. To attempt 3029 * the rejoin, each disk that is offlined is marked online again, and 3030 * we do a reopen() call. If the vdev label for every disk that was 3031 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 3032 * then we call vdev_split() on each disk, and complete the split. 3033 * 3034 * Otherwise we leave the config alone, with all the vdevs in place in 3035 * the original pool. 3036 */ 3037 static void 3038 spa_try_repair(spa_t *spa, nvlist_t *config) 3039 { 3040 uint_t extracted; 3041 uint64_t *glist; 3042 uint_t i, gcount; 3043 nvlist_t *nvl; 3044 vdev_t **vd; 3045 boolean_t attempt_reopen; 3046 3047 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 3048 return; 3049 3050 /* check that the config is complete */ 3051 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 3052 &glist, &gcount) != 0) 3053 return; 3054 3055 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 3056 3057 /* attempt to online all the vdevs & validate */ 3058 attempt_reopen = B_TRUE; 3059 for (i = 0; i < gcount; i++) { 3060 if (glist[i] == 0) /* vdev is hole */ 3061 continue; 3062 3063 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 3064 if (vd[i] == NULL) { 3065 /* 3066 * Don't bother attempting to reopen the disks; 3067 * just do the split. 3068 */ 3069 attempt_reopen = B_FALSE; 3070 } else { 3071 /* attempt to re-online it */ 3072 vd[i]->vdev_offline = B_FALSE; 3073 } 3074 } 3075 3076 if (attempt_reopen) { 3077 vdev_reopen(spa->spa_root_vdev); 3078 3079 /* check each device to see what state it's in */ 3080 for (extracted = 0, i = 0; i < gcount; i++) { 3081 if (vd[i] != NULL && 3082 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 3083 break; 3084 ++extracted; 3085 } 3086 } 3087 3088 /* 3089 * If every disk has been moved to the new pool, or if we never 3090 * even attempted to look at them, then we split them off for 3091 * good. 3092 */ 3093 if (!attempt_reopen || gcount == extracted) { 3094 for (i = 0; i < gcount; i++) 3095 if (vd[i] != NULL) 3096 vdev_split(vd[i]); 3097 vdev_reopen(spa->spa_root_vdev); 3098 } 3099 3100 kmem_free(vd, gcount * sizeof (vdev_t *)); 3101 } 3102 3103 static int 3104 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 3105 { 3106 const char *ereport = FM_EREPORT_ZFS_POOL; 3107 int error; 3108 3109 spa->spa_load_state = state; 3110 (void) spa_import_progress_set_state(spa_guid(spa), 3111 spa_load_state(spa)); 3112 3113 gethrestime(&spa->spa_loaded_ts); 3114 error = spa_load_impl(spa, type, &ereport); 3115 3116 /* 3117 * Don't count references from objsets that are already closed 3118 * and are making their way through the eviction process. 3119 */ 3120 spa_evicting_os_wait(spa); 3121 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 3122 if (error) { 3123 if (error != EEXIST) { 3124 spa->spa_loaded_ts.tv_sec = 0; 3125 spa->spa_loaded_ts.tv_nsec = 0; 3126 } 3127 if (error != EBADF) { 3128 (void) zfs_ereport_post(ereport, spa, 3129 NULL, NULL, NULL, 0); 3130 } 3131 } 3132 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 3133 spa->spa_ena = 0; 3134 3135 (void) spa_import_progress_set_state(spa_guid(spa), 3136 spa_load_state(spa)); 3137 3138 return (error); 3139 } 3140 3141 #ifdef ZFS_DEBUG 3142 /* 3143 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 3144 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 3145 * spa's per-vdev ZAP list. 3146 */ 3147 static uint64_t 3148 vdev_count_verify_zaps(vdev_t *vd) 3149 { 3150 spa_t *spa = vd->vdev_spa; 3151 uint64_t total = 0; 3152 3153 if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) && 3154 vd->vdev_root_zap != 0) { 3155 total++; 3156 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3157 spa->spa_all_vdev_zaps, vd->vdev_root_zap)); 3158 } 3159 if (vd->vdev_top_zap != 0) { 3160 total++; 3161 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3162 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 3163 } 3164 if (vd->vdev_leaf_zap != 0) { 3165 total++; 3166 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3167 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 3168 } 3169 3170 for (uint64_t i = 0; i < vd->vdev_children; i++) { 3171 total += vdev_count_verify_zaps(vd->vdev_child[i]); 3172 } 3173 3174 return (total); 3175 } 3176 #else 3177 #define vdev_count_verify_zaps(vd) ((void) sizeof (vd), 0) 3178 #endif 3179 3180 /* 3181 * Determine whether the activity check is required. 3182 */ 3183 static boolean_t 3184 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label, 3185 nvlist_t *config) 3186 { 3187 uint64_t state = 0; 3188 uint64_t hostid = 0; 3189 uint64_t tryconfig_txg = 0; 3190 uint64_t tryconfig_timestamp = 0; 3191 uint16_t tryconfig_mmp_seq = 0; 3192 nvlist_t *nvinfo; 3193 3194 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3195 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 3196 (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3197 &tryconfig_txg); 3198 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3199 &tryconfig_timestamp); 3200 (void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3201 &tryconfig_mmp_seq); 3202 } 3203 3204 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); 3205 3206 /* 3207 * Disable the MMP activity check - This is used by zdb which 3208 * is intended to be used on potentially active pools. 3209 */ 3210 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) 3211 return (B_FALSE); 3212 3213 /* 3214 * Skip the activity check when the MMP feature is disabled. 3215 */ 3216 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) 3217 return (B_FALSE); 3218 3219 /* 3220 * If the tryconfig_ values are nonzero, they are the results of an 3221 * earlier tryimport. If they all match the uberblock we just found, 3222 * then the pool has not changed and we return false so we do not test 3223 * a second time. 3224 */ 3225 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && 3226 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && 3227 tryconfig_mmp_seq && tryconfig_mmp_seq == 3228 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) 3229 return (B_FALSE); 3230 3231 /* 3232 * Allow the activity check to be skipped when importing the pool 3233 * on the same host which last imported it. Since the hostid from 3234 * configuration may be stale use the one read from the label. 3235 */ 3236 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3237 hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID); 3238 3239 if (hostid == spa_get_hostid(spa)) 3240 return (B_FALSE); 3241 3242 /* 3243 * Skip the activity test when the pool was cleanly exported. 3244 */ 3245 if (state != POOL_STATE_ACTIVE) 3246 return (B_FALSE); 3247 3248 return (B_TRUE); 3249 } 3250 3251 /* 3252 * Nanoseconds the activity check must watch for changes on-disk. 3253 */ 3254 static uint64_t 3255 spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3256 { 3257 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 3258 uint64_t multihost_interval = MSEC2NSEC( 3259 MMP_INTERVAL_OK(zfs_multihost_interval)); 3260 uint64_t import_delay = MAX(NANOSEC, import_intervals * 3261 multihost_interval); 3262 3263 /* 3264 * Local tunables determine a minimum duration except for the case 3265 * where we know when the remote host will suspend the pool if MMP 3266 * writes do not land. 3267 * 3268 * See Big Theory comment at the top of mmp.c for the reasoning behind 3269 * these cases and times. 3270 */ 3271 3272 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 3273 3274 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3275 MMP_FAIL_INT(ub) > 0) { 3276 3277 /* MMP on remote host will suspend pool after failed writes */ 3278 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 3279 MMP_IMPORT_SAFETY_FACTOR / 100; 3280 3281 zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp " 3282 "mmp_fails=%llu ub_mmp mmp_interval=%llu " 3283 "import_intervals=%llu", (u_longlong_t)import_delay, 3284 (u_longlong_t)MMP_FAIL_INT(ub), 3285 (u_longlong_t)MMP_INTERVAL(ub), 3286 (u_longlong_t)import_intervals); 3287 3288 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3289 MMP_FAIL_INT(ub) == 0) { 3290 3291 /* MMP on remote host will never suspend pool */ 3292 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 3293 ub->ub_mmp_delay) * import_intervals); 3294 3295 zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp " 3296 "mmp_interval=%llu ub_mmp_delay=%llu " 3297 "import_intervals=%llu", (u_longlong_t)import_delay, 3298 (u_longlong_t)MMP_INTERVAL(ub), 3299 (u_longlong_t)ub->ub_mmp_delay, 3300 (u_longlong_t)import_intervals); 3301 3302 } else if (MMP_VALID(ub)) { 3303 /* 3304 * zfs-0.7 compatibility case 3305 */ 3306 3307 import_delay = MAX(import_delay, (multihost_interval + 3308 ub->ub_mmp_delay) * import_intervals); 3309 3310 zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu " 3311 "import_intervals=%llu leaves=%u", 3312 (u_longlong_t)import_delay, 3313 (u_longlong_t)ub->ub_mmp_delay, 3314 (u_longlong_t)import_intervals, 3315 vdev_count_leaves(spa)); 3316 } else { 3317 /* Using local tunings is the only reasonable option */ 3318 zfs_dbgmsg("pool last imported on non-MMP aware " 3319 "host using import_delay=%llu multihost_interval=%llu " 3320 "import_intervals=%llu", (u_longlong_t)import_delay, 3321 (u_longlong_t)multihost_interval, 3322 (u_longlong_t)import_intervals); 3323 } 3324 3325 return (import_delay); 3326 } 3327 3328 /* 3329 * Perform the import activity check. If the user canceled the import or 3330 * we detected activity then fail. 3331 */ 3332 static int 3333 spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config) 3334 { 3335 uint64_t txg = ub->ub_txg; 3336 uint64_t timestamp = ub->ub_timestamp; 3337 uint64_t mmp_config = ub->ub_mmp_config; 3338 uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0; 3339 uint64_t import_delay; 3340 hrtime_t import_expire; 3341 nvlist_t *mmp_label = NULL; 3342 vdev_t *rvd = spa->spa_root_vdev; 3343 kcondvar_t cv; 3344 kmutex_t mtx; 3345 int error = 0; 3346 3347 cv_init(&cv, NULL, CV_DEFAULT, NULL); 3348 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 3349 mutex_enter(&mtx); 3350 3351 /* 3352 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3353 * during the earlier tryimport. If the txg recorded there is 0 then 3354 * the pool is known to be active on another host. 3355 * 3356 * Otherwise, the pool might be in use on another host. Check for 3357 * changes in the uberblocks on disk if necessary. 3358 */ 3359 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3360 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 3361 ZPOOL_CONFIG_LOAD_INFO); 3362 3363 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && 3364 fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { 3365 vdev_uberblock_load(rvd, ub, &mmp_label); 3366 error = SET_ERROR(EREMOTEIO); 3367 goto out; 3368 } 3369 } 3370 3371 import_delay = spa_activity_check_duration(spa, ub); 3372 3373 /* Add a small random factor in case of simultaneous imports (0-25%) */ 3374 import_delay += import_delay * random_in_range(250) / 1000; 3375 3376 import_expire = gethrtime() + import_delay; 3377 3378 while (gethrtime() < import_expire) { 3379 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 3380 NSEC2SEC(import_expire - gethrtime())); 3381 3382 vdev_uberblock_load(rvd, ub, &mmp_label); 3383 3384 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || 3385 mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) { 3386 zfs_dbgmsg("multihost activity detected " 3387 "txg %llu ub_txg %llu " 3388 "timestamp %llu ub_timestamp %llu " 3389 "mmp_config %#llx ub_mmp_config %#llx", 3390 (u_longlong_t)txg, (u_longlong_t)ub->ub_txg, 3391 (u_longlong_t)timestamp, 3392 (u_longlong_t)ub->ub_timestamp, 3393 (u_longlong_t)mmp_config, 3394 (u_longlong_t)ub->ub_mmp_config); 3395 3396 error = SET_ERROR(EREMOTEIO); 3397 break; 3398 } 3399 3400 if (mmp_label) { 3401 nvlist_free(mmp_label); 3402 mmp_label = NULL; 3403 } 3404 3405 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 3406 if (error != -1) { 3407 error = SET_ERROR(EINTR); 3408 break; 3409 } 3410 error = 0; 3411 } 3412 3413 out: 3414 mutex_exit(&mtx); 3415 mutex_destroy(&mtx); 3416 cv_destroy(&cv); 3417 3418 /* 3419 * If the pool is determined to be active store the status in the 3420 * spa->spa_load_info nvlist. If the remote hostname or hostid are 3421 * available from configuration read from disk store them as well. 3422 * This allows 'zpool import' to generate a more useful message. 3423 * 3424 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) 3425 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 3426 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 3427 */ 3428 if (error == EREMOTEIO) { 3429 const char *hostname = "<unknown>"; 3430 uint64_t hostid = 0; 3431 3432 if (mmp_label) { 3433 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { 3434 hostname = fnvlist_lookup_string(mmp_label, 3435 ZPOOL_CONFIG_HOSTNAME); 3436 fnvlist_add_string(spa->spa_load_info, 3437 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 3438 } 3439 3440 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { 3441 hostid = fnvlist_lookup_uint64(mmp_label, 3442 ZPOOL_CONFIG_HOSTID); 3443 fnvlist_add_uint64(spa->spa_load_info, 3444 ZPOOL_CONFIG_MMP_HOSTID, hostid); 3445 } 3446 } 3447 3448 fnvlist_add_uint64(spa->spa_load_info, 3449 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); 3450 fnvlist_add_uint64(spa->spa_load_info, 3451 ZPOOL_CONFIG_MMP_TXG, 0); 3452 3453 error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); 3454 } 3455 3456 if (mmp_label) 3457 nvlist_free(mmp_label); 3458 3459 return (error); 3460 } 3461 3462 static int 3463 spa_verify_host(spa_t *spa, nvlist_t *mos_config) 3464 { 3465 uint64_t hostid; 3466 const char *hostname; 3467 uint64_t myhostid = 0; 3468 3469 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 3470 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 3471 hostname = fnvlist_lookup_string(mos_config, 3472 ZPOOL_CONFIG_HOSTNAME); 3473 3474 myhostid = zone_get_hostid(NULL); 3475 3476 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 3477 cmn_err(CE_WARN, "pool '%s' could not be " 3478 "loaded as it was last accessed by " 3479 "another system (host: %s hostid: 0x%llx). " 3480 "See: https://openzfs.github.io/openzfs-docs/msg/" 3481 "ZFS-8000-EY", 3482 spa_name(spa), hostname, (u_longlong_t)hostid); 3483 spa_load_failed(spa, "hostid verification failed: pool " 3484 "last accessed by host: %s (hostid: 0x%llx)", 3485 hostname, (u_longlong_t)hostid); 3486 return (SET_ERROR(EBADF)); 3487 } 3488 } 3489 3490 return (0); 3491 } 3492 3493 static int 3494 spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 3495 { 3496 int error = 0; 3497 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 3498 int parse; 3499 vdev_t *rvd; 3500 uint64_t pool_guid; 3501 const char *comment; 3502 const char *compatibility; 3503 3504 /* 3505 * Versioning wasn't explicitly added to the label until later, so if 3506 * it's not present treat it as the initial version. 3507 */ 3508 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 3509 &spa->spa_ubsync.ub_version) != 0) 3510 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 3511 3512 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 3513 spa_load_failed(spa, "invalid config provided: '%s' missing", 3514 ZPOOL_CONFIG_POOL_GUID); 3515 return (SET_ERROR(EINVAL)); 3516 } 3517 3518 /* 3519 * If we are doing an import, ensure that the pool is not already 3520 * imported by checking if its pool guid already exists in the 3521 * spa namespace. 3522 * 3523 * The only case that we allow an already imported pool to be 3524 * imported again, is when the pool is checkpointed and we want to 3525 * look at its checkpointed state from userland tools like zdb. 3526 */ 3527 #ifdef _KERNEL 3528 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3529 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3530 spa_guid_exists(pool_guid, 0)) { 3531 #else 3532 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3533 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3534 spa_guid_exists(pool_guid, 0) && 3535 !spa_importing_readonly_checkpoint(spa)) { 3536 #endif 3537 spa_load_failed(spa, "a pool with guid %llu is already open", 3538 (u_longlong_t)pool_guid); 3539 return (SET_ERROR(EEXIST)); 3540 } 3541 3542 spa->spa_config_guid = pool_guid; 3543 3544 nvlist_free(spa->spa_load_info); 3545 spa->spa_load_info = fnvlist_alloc(); 3546 3547 ASSERT(spa->spa_comment == NULL); 3548 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 3549 spa->spa_comment = spa_strdup(comment); 3550 3551 ASSERT(spa->spa_compatibility == NULL); 3552 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 3553 &compatibility) == 0) 3554 spa->spa_compatibility = spa_strdup(compatibility); 3555 3556 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 3557 &spa->spa_config_txg); 3558 3559 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 3560 spa->spa_config_splitting = fnvlist_dup(nvl); 3561 3562 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 3563 spa_load_failed(spa, "invalid config provided: '%s' missing", 3564 ZPOOL_CONFIG_VDEV_TREE); 3565 return (SET_ERROR(EINVAL)); 3566 } 3567 3568 /* 3569 * Create "The Godfather" zio to hold all async IOs 3570 */ 3571 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 3572 KM_SLEEP); 3573 for (int i = 0; i < max_ncpus; i++) { 3574 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 3575 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3576 ZIO_FLAG_GODFATHER); 3577 } 3578 3579 /* 3580 * Parse the configuration into a vdev tree. We explicitly set the 3581 * value that will be returned by spa_version() since parsing the 3582 * configuration requires knowing the version number. 3583 */ 3584 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3585 parse = (type == SPA_IMPORT_EXISTING ? 3586 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 3587 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 3588 spa_config_exit(spa, SCL_ALL, FTAG); 3589 3590 if (error != 0) { 3591 spa_load_failed(spa, "unable to parse config [error=%d]", 3592 error); 3593 return (error); 3594 } 3595 3596 ASSERT(spa->spa_root_vdev == rvd); 3597 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 3598 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 3599 3600 if (type != SPA_IMPORT_ASSEMBLE) { 3601 ASSERT(spa_guid(spa) == pool_guid); 3602 } 3603 3604 return (0); 3605 } 3606 3607 /* 3608 * Recursively open all vdevs in the vdev tree. This function is called twice: 3609 * first with the untrusted config, then with the trusted config. 3610 */ 3611 static int 3612 spa_ld_open_vdevs(spa_t *spa) 3613 { 3614 int error = 0; 3615 3616 /* 3617 * spa_missing_tvds_allowed defines how many top-level vdevs can be 3618 * missing/unopenable for the root vdev to be still considered openable. 3619 */ 3620 if (spa->spa_trust_config) { 3621 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 3622 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 3623 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 3624 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 3625 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 3626 } else { 3627 spa->spa_missing_tvds_allowed = 0; 3628 } 3629 3630 spa->spa_missing_tvds_allowed = 3631 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 3632 3633 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3634 error = vdev_open(spa->spa_root_vdev); 3635 spa_config_exit(spa, SCL_ALL, FTAG); 3636 3637 if (spa->spa_missing_tvds != 0) { 3638 spa_load_note(spa, "vdev tree has %lld missing top-level " 3639 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 3640 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 3641 /* 3642 * Although theoretically we could allow users to open 3643 * incomplete pools in RW mode, we'd need to add a lot 3644 * of extra logic (e.g. adjust pool space to account 3645 * for missing vdevs). 3646 * This limitation also prevents users from accidentally 3647 * opening the pool in RW mode during data recovery and 3648 * damaging it further. 3649 */ 3650 spa_load_note(spa, "pools with missing top-level " 3651 "vdevs can only be opened in read-only mode."); 3652 error = SET_ERROR(ENXIO); 3653 } else { 3654 spa_load_note(spa, "current settings allow for maximum " 3655 "%lld missing top-level vdevs at this stage.", 3656 (u_longlong_t)spa->spa_missing_tvds_allowed); 3657 } 3658 } 3659 if (error != 0) { 3660 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 3661 error); 3662 } 3663 if (spa->spa_missing_tvds != 0 || error != 0) 3664 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 3665 3666 return (error); 3667 } 3668 3669 /* 3670 * We need to validate the vdev labels against the configuration that 3671 * we have in hand. This function is called twice: first with an untrusted 3672 * config, then with a trusted config. The validation is more strict when the 3673 * config is trusted. 3674 */ 3675 static int 3676 spa_ld_validate_vdevs(spa_t *spa) 3677 { 3678 int error = 0; 3679 vdev_t *rvd = spa->spa_root_vdev; 3680 3681 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3682 error = vdev_validate(rvd); 3683 spa_config_exit(spa, SCL_ALL, FTAG); 3684 3685 if (error != 0) { 3686 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 3687 return (error); 3688 } 3689 3690 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 3691 spa_load_failed(spa, "cannot open vdev tree after invalidating " 3692 "some vdevs"); 3693 vdev_dbgmsg_print_tree(rvd, 2); 3694 return (SET_ERROR(ENXIO)); 3695 } 3696 3697 return (0); 3698 } 3699 3700 static void 3701 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 3702 { 3703 spa->spa_state = POOL_STATE_ACTIVE; 3704 spa->spa_ubsync = spa->spa_uberblock; 3705 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 3706 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 3707 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 3708 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 3709 spa->spa_claim_max_txg = spa->spa_first_txg; 3710 spa->spa_prev_software_version = ub->ub_software_version; 3711 } 3712 3713 static int 3714 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 3715 { 3716 vdev_t *rvd = spa->spa_root_vdev; 3717 nvlist_t *label; 3718 uberblock_t *ub = &spa->spa_uberblock; 3719 boolean_t activity_check = B_FALSE; 3720 3721 /* 3722 * If we are opening the checkpointed state of the pool by 3723 * rewinding to it, at this point we will have written the 3724 * checkpointed uberblock to the vdev labels, so searching 3725 * the labels will find the right uberblock. However, if 3726 * we are opening the checkpointed state read-only, we have 3727 * not modified the labels. Therefore, we must ignore the 3728 * labels and continue using the spa_uberblock that was set 3729 * by spa_ld_checkpoint_rewind. 3730 * 3731 * Note that it would be fine to ignore the labels when 3732 * rewinding (opening writeable) as well. However, if we 3733 * crash just after writing the labels, we will end up 3734 * searching the labels. Doing so in the common case means 3735 * that this code path gets exercised normally, rather than 3736 * just in the edge case. 3737 */ 3738 if (ub->ub_checkpoint_txg != 0 && 3739 spa_importing_readonly_checkpoint(spa)) { 3740 spa_ld_select_uberblock_done(spa, ub); 3741 return (0); 3742 } 3743 3744 /* 3745 * Find the best uberblock. 3746 */ 3747 vdev_uberblock_load(rvd, ub, &label); 3748 3749 /* 3750 * If we weren't able to find a single valid uberblock, return failure. 3751 */ 3752 if (ub->ub_txg == 0) { 3753 nvlist_free(label); 3754 spa_load_failed(spa, "no valid uberblock found"); 3755 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 3756 } 3757 3758 if (spa->spa_load_max_txg != UINT64_MAX) { 3759 (void) spa_import_progress_set_max_txg(spa_guid(spa), 3760 (u_longlong_t)spa->spa_load_max_txg); 3761 } 3762 spa_load_note(spa, "using uberblock with txg=%llu", 3763 (u_longlong_t)ub->ub_txg); 3764 if (ub->ub_raidz_reflow_info != 0) { 3765 spa_load_note(spa, "uberblock raidz_reflow_info: " 3766 "state=%u offset=%llu", 3767 (int)RRSS_GET_STATE(ub), 3768 (u_longlong_t)RRSS_GET_OFFSET(ub)); 3769 } 3770 3771 3772 /* 3773 * For pools which have the multihost property on determine if the 3774 * pool is truly inactive and can be safely imported. Prevent 3775 * hosts which don't have a hostid set from importing the pool. 3776 */ 3777 activity_check = spa_activity_check_required(spa, ub, label, 3778 spa->spa_config); 3779 if (activity_check) { 3780 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 3781 spa_get_hostid(spa) == 0) { 3782 nvlist_free(label); 3783 fnvlist_add_uint64(spa->spa_load_info, 3784 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 3785 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 3786 } 3787 3788 int error = spa_activity_check(spa, ub, spa->spa_config); 3789 if (error) { 3790 nvlist_free(label); 3791 return (error); 3792 } 3793 3794 fnvlist_add_uint64(spa->spa_load_info, 3795 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); 3796 fnvlist_add_uint64(spa->spa_load_info, 3797 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); 3798 fnvlist_add_uint16(spa->spa_load_info, 3799 ZPOOL_CONFIG_MMP_SEQ, 3800 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)); 3801 } 3802 3803 /* 3804 * If the pool has an unsupported version we can't open it. 3805 */ 3806 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 3807 nvlist_free(label); 3808 spa_load_failed(spa, "version %llu is not supported", 3809 (u_longlong_t)ub->ub_version); 3810 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 3811 } 3812 3813 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3814 nvlist_t *features; 3815 3816 /* 3817 * If we weren't able to find what's necessary for reading the 3818 * MOS in the label, return failure. 3819 */ 3820 if (label == NULL) { 3821 spa_load_failed(spa, "label config unavailable"); 3822 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3823 ENXIO)); 3824 } 3825 3826 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 3827 &features) != 0) { 3828 nvlist_free(label); 3829 spa_load_failed(spa, "invalid label: '%s' missing", 3830 ZPOOL_CONFIG_FEATURES_FOR_READ); 3831 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3832 ENXIO)); 3833 } 3834 3835 /* 3836 * Update our in-core representation with the definitive values 3837 * from the label. 3838 */ 3839 nvlist_free(spa->spa_label_features); 3840 spa->spa_label_features = fnvlist_dup(features); 3841 } 3842 3843 nvlist_free(label); 3844 3845 /* 3846 * Look through entries in the label nvlist's features_for_read. If 3847 * there is a feature listed there which we don't understand then we 3848 * cannot open a pool. 3849 */ 3850 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3851 nvlist_t *unsup_feat; 3852 3853 unsup_feat = fnvlist_alloc(); 3854 3855 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 3856 NULL); nvp != NULL; 3857 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 3858 if (!zfeature_is_supported(nvpair_name(nvp))) { 3859 fnvlist_add_string(unsup_feat, 3860 nvpair_name(nvp), ""); 3861 } 3862 } 3863 3864 if (!nvlist_empty(unsup_feat)) { 3865 fnvlist_add_nvlist(spa->spa_load_info, 3866 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 3867 nvlist_free(unsup_feat); 3868 spa_load_failed(spa, "some features are unsupported"); 3869 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 3870 ENOTSUP)); 3871 } 3872 3873 nvlist_free(unsup_feat); 3874 } 3875 3876 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 3877 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3878 spa_try_repair(spa, spa->spa_config); 3879 spa_config_exit(spa, SCL_ALL, FTAG); 3880 nvlist_free(spa->spa_config_splitting); 3881 spa->spa_config_splitting = NULL; 3882 } 3883 3884 /* 3885 * Initialize internal SPA structures. 3886 */ 3887 spa_ld_select_uberblock_done(spa, ub); 3888 3889 return (0); 3890 } 3891 3892 static int 3893 spa_ld_open_rootbp(spa_t *spa) 3894 { 3895 int error = 0; 3896 vdev_t *rvd = spa->spa_root_vdev; 3897 3898 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 3899 if (error != 0) { 3900 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 3901 "[error=%d]", error); 3902 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3903 } 3904 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 3905 3906 return (0); 3907 } 3908 3909 static int 3910 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 3911 boolean_t reloading) 3912 { 3913 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 3914 nvlist_t *nv, *mos_config, *policy; 3915 int error = 0, copy_error; 3916 uint64_t healthy_tvds, healthy_tvds_mos; 3917 uint64_t mos_config_txg; 3918 3919 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 3920 != 0) 3921 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3922 3923 /* 3924 * If we're assembling a pool from a split, the config provided is 3925 * already trusted so there is nothing to do. 3926 */ 3927 if (type == SPA_IMPORT_ASSEMBLE) 3928 return (0); 3929 3930 healthy_tvds = spa_healthy_core_tvds(spa); 3931 3932 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 3933 != 0) { 3934 spa_load_failed(spa, "unable to retrieve MOS config"); 3935 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3936 } 3937 3938 /* 3939 * If we are doing an open, pool owner wasn't verified yet, thus do 3940 * the verification here. 3941 */ 3942 if (spa->spa_load_state == SPA_LOAD_OPEN) { 3943 error = spa_verify_host(spa, mos_config); 3944 if (error != 0) { 3945 nvlist_free(mos_config); 3946 return (error); 3947 } 3948 } 3949 3950 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 3951 3952 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3953 3954 /* 3955 * Build a new vdev tree from the trusted config 3956 */ 3957 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 3958 if (error != 0) { 3959 nvlist_free(mos_config); 3960 spa_config_exit(spa, SCL_ALL, FTAG); 3961 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 3962 error); 3963 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3964 } 3965 3966 /* 3967 * Vdev paths in the MOS may be obsolete. If the untrusted config was 3968 * obtained by scanning /dev/dsk, then it will have the right vdev 3969 * paths. We update the trusted MOS config with this information. 3970 * We first try to copy the paths with vdev_copy_path_strict, which 3971 * succeeds only when both configs have exactly the same vdev tree. 3972 * If that fails, we fall back to a more flexible method that has a 3973 * best effort policy. 3974 */ 3975 copy_error = vdev_copy_path_strict(rvd, mrvd); 3976 if (copy_error != 0 || spa_load_print_vdev_tree) { 3977 spa_load_note(spa, "provided vdev tree:"); 3978 vdev_dbgmsg_print_tree(rvd, 2); 3979 spa_load_note(spa, "MOS vdev tree:"); 3980 vdev_dbgmsg_print_tree(mrvd, 2); 3981 } 3982 if (copy_error != 0) { 3983 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 3984 "back to vdev_copy_path_relaxed"); 3985 vdev_copy_path_relaxed(rvd, mrvd); 3986 } 3987 3988 vdev_close(rvd); 3989 vdev_free(rvd); 3990 spa->spa_root_vdev = mrvd; 3991 rvd = mrvd; 3992 spa_config_exit(spa, SCL_ALL, FTAG); 3993 3994 /* 3995 * If 'zpool import' used a cached config, then the on-disk hostid and 3996 * hostname may be different to the cached config in ways that should 3997 * prevent import. Userspace can't discover this without a scan, but 3998 * we know, so we add these values to LOAD_INFO so the caller can know 3999 * the difference. 4000 * 4001 * Note that we have to do this before the config is regenerated, 4002 * because the new config will have the hostid and hostname for this 4003 * host, in readiness for import. 4004 */ 4005 if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTID)) 4006 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_HOSTID, 4007 fnvlist_lookup_uint64(mos_config, ZPOOL_CONFIG_HOSTID)); 4008 if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTNAME)) 4009 fnvlist_add_string(spa->spa_load_info, ZPOOL_CONFIG_HOSTNAME, 4010 fnvlist_lookup_string(mos_config, ZPOOL_CONFIG_HOSTNAME)); 4011 4012 /* 4013 * We will use spa_config if we decide to reload the spa or if spa_load 4014 * fails and we rewind. We must thus regenerate the config using the 4015 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 4016 * pass settings on how to load the pool and is not stored in the MOS. 4017 * We copy it over to our new, trusted config. 4018 */ 4019 mos_config_txg = fnvlist_lookup_uint64(mos_config, 4020 ZPOOL_CONFIG_POOL_TXG); 4021 nvlist_free(mos_config); 4022 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 4023 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 4024 &policy) == 0) 4025 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 4026 spa_config_set(spa, mos_config); 4027 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 4028 4029 /* 4030 * Now that we got the config from the MOS, we should be more strict 4031 * in checking blkptrs and can make assumptions about the consistency 4032 * of the vdev tree. spa_trust_config must be set to true before opening 4033 * vdevs in order for them to be writeable. 4034 */ 4035 spa->spa_trust_config = B_TRUE; 4036 4037 /* 4038 * Open and validate the new vdev tree 4039 */ 4040 error = spa_ld_open_vdevs(spa); 4041 if (error != 0) 4042 return (error); 4043 4044 error = spa_ld_validate_vdevs(spa); 4045 if (error != 0) 4046 return (error); 4047 4048 if (copy_error != 0 || spa_load_print_vdev_tree) { 4049 spa_load_note(spa, "final vdev tree:"); 4050 vdev_dbgmsg_print_tree(rvd, 2); 4051 } 4052 4053 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 4054 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 4055 /* 4056 * Sanity check to make sure that we are indeed loading the 4057 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 4058 * in the config provided and they happened to be the only ones 4059 * to have the latest uberblock, we could involuntarily perform 4060 * an extreme rewind. 4061 */ 4062 healthy_tvds_mos = spa_healthy_core_tvds(spa); 4063 if (healthy_tvds_mos - healthy_tvds >= 4064 SPA_SYNC_MIN_VDEVS) { 4065 spa_load_note(spa, "config provided misses too many " 4066 "top-level vdevs compared to MOS (%lld vs %lld). ", 4067 (u_longlong_t)healthy_tvds, 4068 (u_longlong_t)healthy_tvds_mos); 4069 spa_load_note(spa, "vdev tree:"); 4070 vdev_dbgmsg_print_tree(rvd, 2); 4071 if (reloading) { 4072 spa_load_failed(spa, "config was already " 4073 "provided from MOS. Aborting."); 4074 return (spa_vdev_err(rvd, 4075 VDEV_AUX_CORRUPT_DATA, EIO)); 4076 } 4077 spa_load_note(spa, "spa must be reloaded using MOS " 4078 "config"); 4079 return (SET_ERROR(EAGAIN)); 4080 } 4081 } 4082 4083 error = spa_check_for_missing_logs(spa); 4084 if (error != 0) 4085 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 4086 4087 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 4088 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 4089 "guid sum (%llu != %llu)", 4090 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 4091 (u_longlong_t)rvd->vdev_guid_sum); 4092 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 4093 ENXIO)); 4094 } 4095 4096 return (0); 4097 } 4098 4099 static int 4100 spa_ld_open_indirect_vdev_metadata(spa_t *spa) 4101 { 4102 int error = 0; 4103 vdev_t *rvd = spa->spa_root_vdev; 4104 4105 /* 4106 * Everything that we read before spa_remove_init() must be stored 4107 * on concreted vdevs. Therefore we do this as early as possible. 4108 */ 4109 error = spa_remove_init(spa); 4110 if (error != 0) { 4111 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 4112 error); 4113 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4114 } 4115 4116 /* 4117 * Retrieve information needed to condense indirect vdev mappings. 4118 */ 4119 error = spa_condense_init(spa); 4120 if (error != 0) { 4121 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 4122 error); 4123 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4124 } 4125 4126 return (0); 4127 } 4128 4129 static int 4130 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 4131 { 4132 int error = 0; 4133 vdev_t *rvd = spa->spa_root_vdev; 4134 4135 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 4136 boolean_t missing_feat_read = B_FALSE; 4137 nvlist_t *unsup_feat, *enabled_feat; 4138 4139 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 4140 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 4141 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4142 } 4143 4144 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 4145 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 4146 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4147 } 4148 4149 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 4150 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 4151 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4152 } 4153 4154 enabled_feat = fnvlist_alloc(); 4155 unsup_feat = fnvlist_alloc(); 4156 4157 if (!spa_features_check(spa, B_FALSE, 4158 unsup_feat, enabled_feat)) 4159 missing_feat_read = B_TRUE; 4160 4161 if (spa_writeable(spa) || 4162 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 4163 if (!spa_features_check(spa, B_TRUE, 4164 unsup_feat, enabled_feat)) { 4165 *missing_feat_writep = B_TRUE; 4166 } 4167 } 4168 4169 fnvlist_add_nvlist(spa->spa_load_info, 4170 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 4171 4172 if (!nvlist_empty(unsup_feat)) { 4173 fnvlist_add_nvlist(spa->spa_load_info, 4174 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 4175 } 4176 4177 fnvlist_free(enabled_feat); 4178 fnvlist_free(unsup_feat); 4179 4180 if (!missing_feat_read) { 4181 fnvlist_add_boolean(spa->spa_load_info, 4182 ZPOOL_CONFIG_CAN_RDONLY); 4183 } 4184 4185 /* 4186 * If the state is SPA_LOAD_TRYIMPORT, our objective is 4187 * twofold: to determine whether the pool is available for 4188 * import in read-write mode and (if it is not) whether the 4189 * pool is available for import in read-only mode. If the pool 4190 * is available for import in read-write mode, it is displayed 4191 * as available in userland; if it is not available for import 4192 * in read-only mode, it is displayed as unavailable in 4193 * userland. If the pool is available for import in read-only 4194 * mode but not read-write mode, it is displayed as unavailable 4195 * in userland with a special note that the pool is actually 4196 * available for open in read-only mode. 4197 * 4198 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 4199 * missing a feature for write, we must first determine whether 4200 * the pool can be opened read-only before returning to 4201 * userland in order to know whether to display the 4202 * abovementioned note. 4203 */ 4204 if (missing_feat_read || (*missing_feat_writep && 4205 spa_writeable(spa))) { 4206 spa_load_failed(spa, "pool uses unsupported features"); 4207 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 4208 ENOTSUP)); 4209 } 4210 4211 /* 4212 * Load refcounts for ZFS features from disk into an in-memory 4213 * cache during SPA initialization. 4214 */ 4215 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 4216 uint64_t refcount; 4217 4218 error = feature_get_refcount_from_disk(spa, 4219 &spa_feature_table[i], &refcount); 4220 if (error == 0) { 4221 spa->spa_feat_refcount_cache[i] = refcount; 4222 } else if (error == ENOTSUP) { 4223 spa->spa_feat_refcount_cache[i] = 4224 SPA_FEATURE_DISABLED; 4225 } else { 4226 spa_load_failed(spa, "error getting refcount " 4227 "for feature %s [error=%d]", 4228 spa_feature_table[i].fi_guid, error); 4229 return (spa_vdev_err(rvd, 4230 VDEV_AUX_CORRUPT_DATA, EIO)); 4231 } 4232 } 4233 } 4234 4235 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 4236 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 4237 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 4238 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4239 } 4240 4241 /* 4242 * Encryption was added before bookmark_v2, even though bookmark_v2 4243 * is now a dependency. If this pool has encryption enabled without 4244 * bookmark_v2, trigger an errata message. 4245 */ 4246 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 4247 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 4248 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 4249 } 4250 4251 return (0); 4252 } 4253 4254 static int 4255 spa_ld_load_special_directories(spa_t *spa) 4256 { 4257 int error = 0; 4258 vdev_t *rvd = spa->spa_root_vdev; 4259 4260 spa->spa_is_initializing = B_TRUE; 4261 error = dsl_pool_open(spa->spa_dsl_pool); 4262 spa->spa_is_initializing = B_FALSE; 4263 if (error != 0) { 4264 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 4265 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4266 } 4267 4268 return (0); 4269 } 4270 4271 static int 4272 spa_ld_get_props(spa_t *spa) 4273 { 4274 int error = 0; 4275 uint64_t obj; 4276 vdev_t *rvd = spa->spa_root_vdev; 4277 4278 /* Grab the checksum salt from the MOS. */ 4279 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4280 DMU_POOL_CHECKSUM_SALT, 1, 4281 sizeof (spa->spa_cksum_salt.zcs_bytes), 4282 spa->spa_cksum_salt.zcs_bytes); 4283 if (error == ENOENT) { 4284 /* Generate a new salt for subsequent use */ 4285 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 4286 sizeof (spa->spa_cksum_salt.zcs_bytes)); 4287 } else if (error != 0) { 4288 spa_load_failed(spa, "unable to retrieve checksum salt from " 4289 "MOS [error=%d]", error); 4290 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4291 } 4292 4293 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 4294 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4295 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 4296 if (error != 0) { 4297 spa_load_failed(spa, "error opening deferred-frees bpobj " 4298 "[error=%d]", error); 4299 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4300 } 4301 4302 /* 4303 * Load the bit that tells us to use the new accounting function 4304 * (raid-z deflation). If we have an older pool, this will not 4305 * be present. 4306 */ 4307 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 4308 if (error != 0 && error != ENOENT) 4309 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4310 4311 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 4312 &spa->spa_creation_version, B_FALSE); 4313 if (error != 0 && error != ENOENT) 4314 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4315 4316 /* 4317 * Load the persistent error log. If we have an older pool, this will 4318 * not be present. 4319 */ 4320 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 4321 B_FALSE); 4322 if (error != 0 && error != ENOENT) 4323 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4324 4325 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 4326 &spa->spa_errlog_scrub, B_FALSE); 4327 if (error != 0 && error != ENOENT) 4328 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4329 4330 /* 4331 * Load the livelist deletion field. If a livelist is queued for 4332 * deletion, indicate that in the spa 4333 */ 4334 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 4335 &spa->spa_livelists_to_delete, B_FALSE); 4336 if (error != 0 && error != ENOENT) 4337 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4338 4339 /* 4340 * Load the history object. If we have an older pool, this 4341 * will not be present. 4342 */ 4343 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 4344 if (error != 0 && error != ENOENT) 4345 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4346 4347 /* 4348 * Load the per-vdev ZAP map. If we have an older pool, this will not 4349 * be present; in this case, defer its creation to a later time to 4350 * avoid dirtying the MOS this early / out of sync context. See 4351 * spa_sync_config_object. 4352 */ 4353 4354 /* The sentinel is only available in the MOS config. */ 4355 nvlist_t *mos_config; 4356 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 4357 spa_load_failed(spa, "unable to retrieve MOS config"); 4358 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4359 } 4360 4361 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 4362 &spa->spa_all_vdev_zaps, B_FALSE); 4363 4364 if (error == ENOENT) { 4365 VERIFY(!nvlist_exists(mos_config, 4366 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 4367 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 4368 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4369 } else if (error != 0) { 4370 nvlist_free(mos_config); 4371 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4372 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 4373 /* 4374 * An older version of ZFS overwrote the sentinel value, so 4375 * we have orphaned per-vdev ZAPs in the MOS. Defer their 4376 * destruction to later; see spa_sync_config_object. 4377 */ 4378 spa->spa_avz_action = AVZ_ACTION_DESTROY; 4379 /* 4380 * We're assuming that no vdevs have had their ZAPs created 4381 * before this. Better be sure of it. 4382 */ 4383 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4384 } 4385 nvlist_free(mos_config); 4386 4387 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 4388 4389 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 4390 B_FALSE); 4391 if (error && error != ENOENT) 4392 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4393 4394 if (error == 0) { 4395 uint64_t autoreplace = 0; 4396 4397 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 4398 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 4399 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 4400 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 4401 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 4402 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 4403 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 4404 spa->spa_autoreplace = (autoreplace != 0); 4405 } 4406 4407 /* 4408 * If we are importing a pool with missing top-level vdevs, 4409 * we enforce that the pool doesn't panic or get suspended on 4410 * error since the likelihood of missing data is extremely high. 4411 */ 4412 if (spa->spa_missing_tvds > 0 && 4413 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 4414 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4415 spa_load_note(spa, "forcing failmode to 'continue' " 4416 "as some top level vdevs are missing"); 4417 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 4418 } 4419 4420 return (0); 4421 } 4422 4423 static int 4424 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 4425 { 4426 int error = 0; 4427 vdev_t *rvd = spa->spa_root_vdev; 4428 4429 /* 4430 * If we're assembling the pool from the split-off vdevs of 4431 * an existing pool, we don't want to attach the spares & cache 4432 * devices. 4433 */ 4434 4435 /* 4436 * Load any hot spares for this pool. 4437 */ 4438 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 4439 B_FALSE); 4440 if (error != 0 && error != ENOENT) 4441 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4442 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4443 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 4444 if (load_nvlist(spa, spa->spa_spares.sav_object, 4445 &spa->spa_spares.sav_config) != 0) { 4446 spa_load_failed(spa, "error loading spares nvlist"); 4447 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4448 } 4449 4450 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4451 spa_load_spares(spa); 4452 spa_config_exit(spa, SCL_ALL, FTAG); 4453 } else if (error == 0) { 4454 spa->spa_spares.sav_sync = B_TRUE; 4455 } 4456 4457 /* 4458 * Load any level 2 ARC devices for this pool. 4459 */ 4460 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 4461 &spa->spa_l2cache.sav_object, B_FALSE); 4462 if (error != 0 && error != ENOENT) 4463 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4464 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4465 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 4466 if (load_nvlist(spa, spa->spa_l2cache.sav_object, 4467 &spa->spa_l2cache.sav_config) != 0) { 4468 spa_load_failed(spa, "error loading l2cache nvlist"); 4469 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4470 } 4471 4472 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4473 spa_load_l2cache(spa); 4474 spa_config_exit(spa, SCL_ALL, FTAG); 4475 } else if (error == 0) { 4476 spa->spa_l2cache.sav_sync = B_TRUE; 4477 } 4478 4479 return (0); 4480 } 4481 4482 static int 4483 spa_ld_load_vdev_metadata(spa_t *spa) 4484 { 4485 int error = 0; 4486 vdev_t *rvd = spa->spa_root_vdev; 4487 4488 /* 4489 * If the 'multihost' property is set, then never allow a pool to 4490 * be imported when the system hostid is zero. The exception to 4491 * this rule is zdb which is always allowed to access pools. 4492 */ 4493 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 4494 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 4495 fnvlist_add_uint64(spa->spa_load_info, 4496 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4497 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4498 } 4499 4500 /* 4501 * If the 'autoreplace' property is set, then post a resource notifying 4502 * the ZFS DE that it should not issue any faults for unopenable 4503 * devices. We also iterate over the vdevs, and post a sysevent for any 4504 * unopenable vdevs so that the normal autoreplace handler can take 4505 * over. 4506 */ 4507 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4508 spa_check_removed(spa->spa_root_vdev); 4509 /* 4510 * For the import case, this is done in spa_import(), because 4511 * at this point we're using the spare definitions from 4512 * the MOS config, not necessarily from the userland config. 4513 */ 4514 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 4515 spa_aux_check_removed(&spa->spa_spares); 4516 spa_aux_check_removed(&spa->spa_l2cache); 4517 } 4518 } 4519 4520 /* 4521 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 4522 */ 4523 error = vdev_load(rvd); 4524 if (error != 0) { 4525 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 4526 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4527 } 4528 4529 error = spa_ld_log_spacemaps(spa); 4530 if (error != 0) { 4531 spa_load_failed(spa, "spa_ld_log_spacemaps failed [error=%d]", 4532 error); 4533 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4534 } 4535 4536 /* 4537 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 4538 */ 4539 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4540 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 4541 spa_config_exit(spa, SCL_ALL, FTAG); 4542 4543 return (0); 4544 } 4545 4546 static int 4547 spa_ld_load_dedup_tables(spa_t *spa) 4548 { 4549 int error = 0; 4550 vdev_t *rvd = spa->spa_root_vdev; 4551 4552 error = ddt_load(spa); 4553 if (error != 0) { 4554 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 4555 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4556 } 4557 4558 return (0); 4559 } 4560 4561 static int 4562 spa_ld_load_brt(spa_t *spa) 4563 { 4564 int error = 0; 4565 vdev_t *rvd = spa->spa_root_vdev; 4566 4567 error = brt_load(spa); 4568 if (error != 0) { 4569 spa_load_failed(spa, "brt_load failed [error=%d]", error); 4570 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4571 } 4572 4573 return (0); 4574 } 4575 4576 static int 4577 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, const char **ereport) 4578 { 4579 vdev_t *rvd = spa->spa_root_vdev; 4580 4581 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 4582 boolean_t missing = spa_check_logs(spa); 4583 if (missing) { 4584 if (spa->spa_missing_tvds != 0) { 4585 spa_load_note(spa, "spa_check_logs failed " 4586 "so dropping the logs"); 4587 } else { 4588 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 4589 spa_load_failed(spa, "spa_check_logs failed"); 4590 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 4591 ENXIO)); 4592 } 4593 } 4594 } 4595 4596 return (0); 4597 } 4598 4599 static int 4600 spa_ld_verify_pool_data(spa_t *spa) 4601 { 4602 int error = 0; 4603 vdev_t *rvd = spa->spa_root_vdev; 4604 4605 /* 4606 * We've successfully opened the pool, verify that we're ready 4607 * to start pushing transactions. 4608 */ 4609 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4610 error = spa_load_verify(spa); 4611 if (error != 0) { 4612 spa_load_failed(spa, "spa_load_verify failed " 4613 "[error=%d]", error); 4614 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4615 error)); 4616 } 4617 } 4618 4619 return (0); 4620 } 4621 4622 static void 4623 spa_ld_claim_log_blocks(spa_t *spa) 4624 { 4625 dmu_tx_t *tx; 4626 dsl_pool_t *dp = spa_get_dsl(spa); 4627 4628 /* 4629 * Claim log blocks that haven't been committed yet. 4630 * This must all happen in a single txg. 4631 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 4632 * invoked from zil_claim_log_block()'s i/o done callback. 4633 * Price of rollback is that we abandon the log. 4634 */ 4635 spa->spa_claiming = B_TRUE; 4636 4637 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 4638 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 4639 zil_claim, tx, DS_FIND_CHILDREN); 4640 dmu_tx_commit(tx); 4641 4642 spa->spa_claiming = B_FALSE; 4643 4644 spa_set_log_state(spa, SPA_LOG_GOOD); 4645 } 4646 4647 static void 4648 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 4649 boolean_t update_config_cache) 4650 { 4651 vdev_t *rvd = spa->spa_root_vdev; 4652 int need_update = B_FALSE; 4653 4654 /* 4655 * If the config cache is stale, or we have uninitialized 4656 * metaslabs (see spa_vdev_add()), then update the config. 4657 * 4658 * If this is a verbatim import, trust the current 4659 * in-core spa_config and update the disk labels. 4660 */ 4661 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 4662 spa->spa_load_state == SPA_LOAD_IMPORT || 4663 spa->spa_load_state == SPA_LOAD_RECOVER || 4664 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 4665 need_update = B_TRUE; 4666 4667 for (int c = 0; c < rvd->vdev_children; c++) 4668 if (rvd->vdev_child[c]->vdev_ms_array == 0) 4669 need_update = B_TRUE; 4670 4671 /* 4672 * Update the config cache asynchronously in case we're the 4673 * root pool, in which case the config cache isn't writable yet. 4674 */ 4675 if (need_update) 4676 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 4677 } 4678 4679 static void 4680 spa_ld_prepare_for_reload(spa_t *spa) 4681 { 4682 spa_mode_t mode = spa->spa_mode; 4683 int async_suspended = spa->spa_async_suspended; 4684 4685 spa_unload(spa); 4686 spa_deactivate(spa); 4687 spa_activate(spa, mode); 4688 4689 /* 4690 * We save the value of spa_async_suspended as it gets reset to 0 by 4691 * spa_unload(). We want to restore it back to the original value before 4692 * returning as we might be calling spa_async_resume() later. 4693 */ 4694 spa->spa_async_suspended = async_suspended; 4695 } 4696 4697 static int 4698 spa_ld_read_checkpoint_txg(spa_t *spa) 4699 { 4700 uberblock_t checkpoint; 4701 int error = 0; 4702 4703 ASSERT0(spa->spa_checkpoint_txg); 4704 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4705 4706 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4707 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4708 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4709 4710 if (error == ENOENT) 4711 return (0); 4712 4713 if (error != 0) 4714 return (error); 4715 4716 ASSERT3U(checkpoint.ub_txg, !=, 0); 4717 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 4718 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 4719 spa->spa_checkpoint_txg = checkpoint.ub_txg; 4720 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 4721 4722 return (0); 4723 } 4724 4725 static int 4726 spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 4727 { 4728 int error = 0; 4729 4730 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4731 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4732 4733 /* 4734 * Never trust the config that is provided unless we are assembling 4735 * a pool following a split. 4736 * This means don't trust blkptrs and the vdev tree in general. This 4737 * also effectively puts the spa in read-only mode since 4738 * spa_writeable() checks for spa_trust_config to be true. 4739 * We will later load a trusted config from the MOS. 4740 */ 4741 if (type != SPA_IMPORT_ASSEMBLE) 4742 spa->spa_trust_config = B_FALSE; 4743 4744 /* 4745 * Parse the config provided to create a vdev tree. 4746 */ 4747 error = spa_ld_parse_config(spa, type); 4748 if (error != 0) 4749 return (error); 4750 4751 spa_import_progress_add(spa); 4752 4753 /* 4754 * Now that we have the vdev tree, try to open each vdev. This involves 4755 * opening the underlying physical device, retrieving its geometry and 4756 * probing the vdev with a dummy I/O. The state of each vdev will be set 4757 * based on the success of those operations. After this we'll be ready 4758 * to read from the vdevs. 4759 */ 4760 error = spa_ld_open_vdevs(spa); 4761 if (error != 0) 4762 return (error); 4763 4764 /* 4765 * Read the label of each vdev and make sure that the GUIDs stored 4766 * there match the GUIDs in the config provided. 4767 * If we're assembling a new pool that's been split off from an 4768 * existing pool, the labels haven't yet been updated so we skip 4769 * validation for now. 4770 */ 4771 if (type != SPA_IMPORT_ASSEMBLE) { 4772 error = spa_ld_validate_vdevs(spa); 4773 if (error != 0) 4774 return (error); 4775 } 4776 4777 /* 4778 * Read all vdev labels to find the best uberblock (i.e. latest, 4779 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 4780 * get the list of features required to read blkptrs in the MOS from 4781 * the vdev label with the best uberblock and verify that our version 4782 * of zfs supports them all. 4783 */ 4784 error = spa_ld_select_uberblock(spa, type); 4785 if (error != 0) 4786 return (error); 4787 4788 /* 4789 * Pass that uberblock to the dsl_pool layer which will open the root 4790 * blkptr. This blkptr points to the latest version of the MOS and will 4791 * allow us to read its contents. 4792 */ 4793 error = spa_ld_open_rootbp(spa); 4794 if (error != 0) 4795 return (error); 4796 4797 return (0); 4798 } 4799 4800 static int 4801 spa_ld_checkpoint_rewind(spa_t *spa) 4802 { 4803 uberblock_t checkpoint; 4804 int error = 0; 4805 4806 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4807 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4808 4809 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4810 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4811 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4812 4813 if (error != 0) { 4814 spa_load_failed(spa, "unable to retrieve checkpointed " 4815 "uberblock from the MOS config [error=%d]", error); 4816 4817 if (error == ENOENT) 4818 error = ZFS_ERR_NO_CHECKPOINT; 4819 4820 return (error); 4821 } 4822 4823 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 4824 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 4825 4826 /* 4827 * We need to update the txg and timestamp of the checkpointed 4828 * uberblock to be higher than the latest one. This ensures that 4829 * the checkpointed uberblock is selected if we were to close and 4830 * reopen the pool right after we've written it in the vdev labels. 4831 * (also see block comment in vdev_uberblock_compare) 4832 */ 4833 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 4834 checkpoint.ub_timestamp = gethrestime_sec(); 4835 4836 /* 4837 * Set current uberblock to be the checkpointed uberblock. 4838 */ 4839 spa->spa_uberblock = checkpoint; 4840 4841 /* 4842 * If we are doing a normal rewind, then the pool is open for 4843 * writing and we sync the "updated" checkpointed uberblock to 4844 * disk. Once this is done, we've basically rewound the whole 4845 * pool and there is no way back. 4846 * 4847 * There are cases when we don't want to attempt and sync the 4848 * checkpointed uberblock to disk because we are opening a 4849 * pool as read-only. Specifically, verifying the checkpointed 4850 * state with zdb, and importing the checkpointed state to get 4851 * a "preview" of its content. 4852 */ 4853 if (spa_writeable(spa)) { 4854 vdev_t *rvd = spa->spa_root_vdev; 4855 4856 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4857 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 4858 int svdcount = 0; 4859 int children = rvd->vdev_children; 4860 int c0 = random_in_range(children); 4861 4862 for (int c = 0; c < children; c++) { 4863 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 4864 4865 /* Stop when revisiting the first vdev */ 4866 if (c > 0 && svd[0] == vd) 4867 break; 4868 4869 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 4870 !vdev_is_concrete(vd)) 4871 continue; 4872 4873 svd[svdcount++] = vd; 4874 if (svdcount == SPA_SYNC_MIN_VDEVS) 4875 break; 4876 } 4877 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg); 4878 if (error == 0) 4879 spa->spa_last_synced_guid = rvd->vdev_guid; 4880 spa_config_exit(spa, SCL_ALL, FTAG); 4881 4882 if (error != 0) { 4883 spa_load_failed(spa, "failed to write checkpointed " 4884 "uberblock to the vdev labels [error=%d]", error); 4885 return (error); 4886 } 4887 } 4888 4889 return (0); 4890 } 4891 4892 static int 4893 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 4894 boolean_t *update_config_cache) 4895 { 4896 int error; 4897 4898 /* 4899 * Parse the config for pool, open and validate vdevs, 4900 * select an uberblock, and use that uberblock to open 4901 * the MOS. 4902 */ 4903 error = spa_ld_mos_init(spa, type); 4904 if (error != 0) 4905 return (error); 4906 4907 /* 4908 * Retrieve the trusted config stored in the MOS and use it to create 4909 * a new, exact version of the vdev tree, then reopen all vdevs. 4910 */ 4911 error = spa_ld_trusted_config(spa, type, B_FALSE); 4912 if (error == EAGAIN) { 4913 if (update_config_cache != NULL) 4914 *update_config_cache = B_TRUE; 4915 4916 /* 4917 * Redo the loading process with the trusted config if it is 4918 * too different from the untrusted config. 4919 */ 4920 spa_ld_prepare_for_reload(spa); 4921 spa_load_note(spa, "RELOADING"); 4922 error = spa_ld_mos_init(spa, type); 4923 if (error != 0) 4924 return (error); 4925 4926 error = spa_ld_trusted_config(spa, type, B_TRUE); 4927 if (error != 0) 4928 return (error); 4929 4930 } else if (error != 0) { 4931 return (error); 4932 } 4933 4934 return (0); 4935 } 4936 4937 /* 4938 * Load an existing storage pool, using the config provided. This config 4939 * describes which vdevs are part of the pool and is later validated against 4940 * partial configs present in each vdev's label and an entire copy of the 4941 * config stored in the MOS. 4942 */ 4943 static int 4944 spa_load_impl(spa_t *spa, spa_import_type_t type, const char **ereport) 4945 { 4946 int error = 0; 4947 boolean_t missing_feat_write = B_FALSE; 4948 boolean_t checkpoint_rewind = 4949 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4950 boolean_t update_config_cache = B_FALSE; 4951 4952 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4953 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4954 4955 spa_load_note(spa, "LOADING"); 4956 4957 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 4958 if (error != 0) 4959 return (error); 4960 4961 /* 4962 * If we are rewinding to the checkpoint then we need to repeat 4963 * everything we've done so far in this function but this time 4964 * selecting the checkpointed uberblock and using that to open 4965 * the MOS. 4966 */ 4967 if (checkpoint_rewind) { 4968 /* 4969 * If we are rewinding to the checkpoint update config cache 4970 * anyway. 4971 */ 4972 update_config_cache = B_TRUE; 4973 4974 /* 4975 * Extract the checkpointed uberblock from the current MOS 4976 * and use this as the pool's uberblock from now on. If the 4977 * pool is imported as writeable we also write the checkpoint 4978 * uberblock to the labels, making the rewind permanent. 4979 */ 4980 error = spa_ld_checkpoint_rewind(spa); 4981 if (error != 0) 4982 return (error); 4983 4984 /* 4985 * Redo the loading process again with the 4986 * checkpointed uberblock. 4987 */ 4988 spa_ld_prepare_for_reload(spa); 4989 spa_load_note(spa, "LOADING checkpointed uberblock"); 4990 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 4991 if (error != 0) 4992 return (error); 4993 } 4994 4995 /* 4996 * Retrieve the checkpoint txg if the pool has a checkpoint. 4997 */ 4998 error = spa_ld_read_checkpoint_txg(spa); 4999 if (error != 0) 5000 return (error); 5001 5002 /* 5003 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 5004 * from the pool and their contents were re-mapped to other vdevs. Note 5005 * that everything that we read before this step must have been 5006 * rewritten on concrete vdevs after the last device removal was 5007 * initiated. Otherwise we could be reading from indirect vdevs before 5008 * we have loaded their mappings. 5009 */ 5010 error = spa_ld_open_indirect_vdev_metadata(spa); 5011 if (error != 0) 5012 return (error); 5013 5014 /* 5015 * Retrieve the full list of active features from the MOS and check if 5016 * they are all supported. 5017 */ 5018 error = spa_ld_check_features(spa, &missing_feat_write); 5019 if (error != 0) 5020 return (error); 5021 5022 /* 5023 * Load several special directories from the MOS needed by the dsl_pool 5024 * layer. 5025 */ 5026 error = spa_ld_load_special_directories(spa); 5027 if (error != 0) 5028 return (error); 5029 5030 /* 5031 * Retrieve pool properties from the MOS. 5032 */ 5033 error = spa_ld_get_props(spa); 5034 if (error != 0) 5035 return (error); 5036 5037 /* 5038 * Retrieve the list of auxiliary devices - cache devices and spares - 5039 * and open them. 5040 */ 5041 error = spa_ld_open_aux_vdevs(spa, type); 5042 if (error != 0) 5043 return (error); 5044 5045 /* 5046 * Load the metadata for all vdevs. Also check if unopenable devices 5047 * should be autoreplaced. 5048 */ 5049 error = spa_ld_load_vdev_metadata(spa); 5050 if (error != 0) 5051 return (error); 5052 5053 error = spa_ld_load_dedup_tables(spa); 5054 if (error != 0) 5055 return (error); 5056 5057 error = spa_ld_load_brt(spa); 5058 if (error != 0) 5059 return (error); 5060 5061 /* 5062 * Verify the logs now to make sure we don't have any unexpected errors 5063 * when we claim log blocks later. 5064 */ 5065 error = spa_ld_verify_logs(spa, type, ereport); 5066 if (error != 0) 5067 return (error); 5068 5069 if (missing_feat_write) { 5070 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 5071 5072 /* 5073 * At this point, we know that we can open the pool in 5074 * read-only mode but not read-write mode. We now have enough 5075 * information and can return to userland. 5076 */ 5077 return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 5078 ENOTSUP)); 5079 } 5080 5081 /* 5082 * Traverse the last txgs to make sure the pool was left off in a safe 5083 * state. When performing an extreme rewind, we verify the whole pool, 5084 * which can take a very long time. 5085 */ 5086 error = spa_ld_verify_pool_data(spa); 5087 if (error != 0) 5088 return (error); 5089 5090 /* 5091 * Calculate the deflated space for the pool. This must be done before 5092 * we write anything to the pool because we'd need to update the space 5093 * accounting using the deflated sizes. 5094 */ 5095 spa_update_dspace(spa); 5096 5097 /* 5098 * We have now retrieved all the information we needed to open the 5099 * pool. If we are importing the pool in read-write mode, a few 5100 * additional steps must be performed to finish the import. 5101 */ 5102 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 5103 spa->spa_load_max_txg == UINT64_MAX)) { 5104 uint64_t config_cache_txg = spa->spa_config_txg; 5105 5106 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 5107 5108 /* 5109 * Before we do any zio_write's, complete the raidz expansion 5110 * scratch space copying, if necessary. 5111 */ 5112 if (RRSS_GET_STATE(&spa->spa_uberblock) == RRSS_SCRATCH_VALID) 5113 vdev_raidz_reflow_copy_scratch(spa); 5114 5115 /* 5116 * In case of a checkpoint rewind, log the original txg 5117 * of the checkpointed uberblock. 5118 */ 5119 if (checkpoint_rewind) { 5120 spa_history_log_internal(spa, "checkpoint rewind", 5121 NULL, "rewound state to txg=%llu", 5122 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 5123 } 5124 5125 /* 5126 * Traverse the ZIL and claim all blocks. 5127 */ 5128 spa_ld_claim_log_blocks(spa); 5129 5130 /* 5131 * Kick-off the syncing thread. 5132 */ 5133 spa->spa_sync_on = B_TRUE; 5134 txg_sync_start(spa->spa_dsl_pool); 5135 mmp_thread_start(spa); 5136 5137 /* 5138 * Wait for all claims to sync. We sync up to the highest 5139 * claimed log block birth time so that claimed log blocks 5140 * don't appear to be from the future. spa_claim_max_txg 5141 * will have been set for us by ZIL traversal operations 5142 * performed above. 5143 */ 5144 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 5145 5146 /* 5147 * Check if we need to request an update of the config. On the 5148 * next sync, we would update the config stored in vdev labels 5149 * and the cachefile (by default /etc/zfs/zpool.cache). 5150 */ 5151 spa_ld_check_for_config_update(spa, config_cache_txg, 5152 update_config_cache); 5153 5154 /* 5155 * Check if a rebuild was in progress and if so resume it. 5156 * Then check all DTLs to see if anything needs resilvering. 5157 * The resilver will be deferred if a rebuild was started. 5158 */ 5159 if (vdev_rebuild_active(spa->spa_root_vdev)) { 5160 vdev_rebuild_restart(spa); 5161 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 5162 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 5163 spa_async_request(spa, SPA_ASYNC_RESILVER); 5164 } 5165 5166 /* 5167 * Log the fact that we booted up (so that we can detect if 5168 * we rebooted in the middle of an operation). 5169 */ 5170 spa_history_log_version(spa, "open", NULL); 5171 5172 spa_restart_removal(spa); 5173 spa_spawn_aux_threads(spa); 5174 5175 /* 5176 * Delete any inconsistent datasets. 5177 * 5178 * Note: 5179 * Since we may be issuing deletes for clones here, 5180 * we make sure to do so after we've spawned all the 5181 * auxiliary threads above (from which the livelist 5182 * deletion zthr is part of). 5183 */ 5184 (void) dmu_objset_find(spa_name(spa), 5185 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 5186 5187 /* 5188 * Clean up any stale temporary dataset userrefs. 5189 */ 5190 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 5191 5192 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5193 vdev_initialize_restart(spa->spa_root_vdev); 5194 vdev_trim_restart(spa->spa_root_vdev); 5195 vdev_autotrim_restart(spa); 5196 spa_config_exit(spa, SCL_CONFIG, FTAG); 5197 } 5198 5199 spa_import_progress_remove(spa_guid(spa)); 5200 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 5201 5202 spa_load_note(spa, "LOADED"); 5203 5204 return (0); 5205 } 5206 5207 static int 5208 spa_load_retry(spa_t *spa, spa_load_state_t state) 5209 { 5210 spa_mode_t mode = spa->spa_mode; 5211 5212 spa_unload(spa); 5213 spa_deactivate(spa); 5214 5215 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 5216 5217 spa_activate(spa, mode); 5218 spa_async_suspend(spa); 5219 5220 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 5221 (u_longlong_t)spa->spa_load_max_txg); 5222 5223 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 5224 } 5225 5226 /* 5227 * If spa_load() fails this function will try loading prior txg's. If 5228 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 5229 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 5230 * function will not rewind the pool and will return the same error as 5231 * spa_load(). 5232 */ 5233 static int 5234 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 5235 int rewind_flags) 5236 { 5237 nvlist_t *loadinfo = NULL; 5238 nvlist_t *config = NULL; 5239 int load_error, rewind_error; 5240 uint64_t safe_rewind_txg; 5241 uint64_t min_txg; 5242 5243 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 5244 spa->spa_load_max_txg = spa->spa_load_txg; 5245 spa_set_log_state(spa, SPA_LOG_CLEAR); 5246 } else { 5247 spa->spa_load_max_txg = max_request; 5248 if (max_request != UINT64_MAX) 5249 spa->spa_extreme_rewind = B_TRUE; 5250 } 5251 5252 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 5253 if (load_error == 0) 5254 return (0); 5255 if (load_error == ZFS_ERR_NO_CHECKPOINT) { 5256 /* 5257 * When attempting checkpoint-rewind on a pool with no 5258 * checkpoint, we should not attempt to load uberblocks 5259 * from previous txgs when spa_load fails. 5260 */ 5261 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5262 spa_import_progress_remove(spa_guid(spa)); 5263 return (load_error); 5264 } 5265 5266 if (spa->spa_root_vdev != NULL) 5267 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5268 5269 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 5270 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 5271 5272 if (rewind_flags & ZPOOL_NEVER_REWIND) { 5273 nvlist_free(config); 5274 spa_import_progress_remove(spa_guid(spa)); 5275 return (load_error); 5276 } 5277 5278 if (state == SPA_LOAD_RECOVER) { 5279 /* Price of rolling back is discarding txgs, including log */ 5280 spa_set_log_state(spa, SPA_LOG_CLEAR); 5281 } else { 5282 /* 5283 * If we aren't rolling back save the load info from our first 5284 * import attempt so that we can restore it after attempting 5285 * to rewind. 5286 */ 5287 loadinfo = spa->spa_load_info; 5288 spa->spa_load_info = fnvlist_alloc(); 5289 } 5290 5291 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 5292 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 5293 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 5294 TXG_INITIAL : safe_rewind_txg; 5295 5296 /* 5297 * Continue as long as we're finding errors, we're still within 5298 * the acceptable rewind range, and we're still finding uberblocks 5299 */ 5300 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 5301 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 5302 if (spa->spa_load_max_txg < safe_rewind_txg) 5303 spa->spa_extreme_rewind = B_TRUE; 5304 rewind_error = spa_load_retry(spa, state); 5305 } 5306 5307 spa->spa_extreme_rewind = B_FALSE; 5308 spa->spa_load_max_txg = UINT64_MAX; 5309 5310 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 5311 spa_config_set(spa, config); 5312 else 5313 nvlist_free(config); 5314 5315 if (state == SPA_LOAD_RECOVER) { 5316 ASSERT3P(loadinfo, ==, NULL); 5317 spa_import_progress_remove(spa_guid(spa)); 5318 return (rewind_error); 5319 } else { 5320 /* Store the rewind info as part of the initial load info */ 5321 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 5322 spa->spa_load_info); 5323 5324 /* Restore the initial load info */ 5325 fnvlist_free(spa->spa_load_info); 5326 spa->spa_load_info = loadinfo; 5327 5328 spa_import_progress_remove(spa_guid(spa)); 5329 return (load_error); 5330 } 5331 } 5332 5333 /* 5334 * Pool Open/Import 5335 * 5336 * The import case is identical to an open except that the configuration is sent 5337 * down from userland, instead of grabbed from the configuration cache. For the 5338 * case of an open, the pool configuration will exist in the 5339 * POOL_STATE_UNINITIALIZED state. 5340 * 5341 * The stats information (gen/count/ustats) is used to gather vdev statistics at 5342 * the same time open the pool, without having to keep around the spa_t in some 5343 * ambiguous state. 5344 */ 5345 static int 5346 spa_open_common(const char *pool, spa_t **spapp, const void *tag, 5347 nvlist_t *nvpolicy, nvlist_t **config) 5348 { 5349 spa_t *spa; 5350 spa_load_state_t state = SPA_LOAD_OPEN; 5351 int error; 5352 int locked = B_FALSE; 5353 int firstopen = B_FALSE; 5354 5355 *spapp = NULL; 5356 5357 /* 5358 * As disgusting as this is, we need to support recursive calls to this 5359 * function because dsl_dir_open() is called during spa_load(), and ends 5360 * up calling spa_open() again. The real fix is to figure out how to 5361 * avoid dsl_dir_open() calling this in the first place. 5362 */ 5363 if (MUTEX_NOT_HELD(&spa_namespace_lock)) { 5364 mutex_enter(&spa_namespace_lock); 5365 locked = B_TRUE; 5366 } 5367 5368 if ((spa = spa_lookup(pool)) == NULL) { 5369 if (locked) 5370 mutex_exit(&spa_namespace_lock); 5371 return (SET_ERROR(ENOENT)); 5372 } 5373 5374 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 5375 zpool_load_policy_t policy; 5376 5377 firstopen = B_TRUE; 5378 5379 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 5380 &policy); 5381 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 5382 state = SPA_LOAD_RECOVER; 5383 5384 spa_activate(spa, spa_mode_global); 5385 5386 if (state != SPA_LOAD_RECOVER) 5387 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 5388 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 5389 5390 zfs_dbgmsg("spa_open_common: opening %s", pool); 5391 error = spa_load_best(spa, state, policy.zlp_txg, 5392 policy.zlp_rewind); 5393 5394 if (error == EBADF) { 5395 /* 5396 * If vdev_validate() returns failure (indicated by 5397 * EBADF), it indicates that one of the vdevs indicates 5398 * that the pool has been exported or destroyed. If 5399 * this is the case, the config cache is out of sync and 5400 * we should remove the pool from the namespace. 5401 */ 5402 spa_unload(spa); 5403 spa_deactivate(spa); 5404 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 5405 spa_remove(spa); 5406 if (locked) 5407 mutex_exit(&spa_namespace_lock); 5408 return (SET_ERROR(ENOENT)); 5409 } 5410 5411 if (error) { 5412 /* 5413 * We can't open the pool, but we still have useful 5414 * information: the state of each vdev after the 5415 * attempted vdev_open(). Return this to the user. 5416 */ 5417 if (config != NULL && spa->spa_config) { 5418 *config = fnvlist_dup(spa->spa_config); 5419 fnvlist_add_nvlist(*config, 5420 ZPOOL_CONFIG_LOAD_INFO, 5421 spa->spa_load_info); 5422 } 5423 spa_unload(spa); 5424 spa_deactivate(spa); 5425 spa->spa_last_open_failed = error; 5426 if (locked) 5427 mutex_exit(&spa_namespace_lock); 5428 *spapp = NULL; 5429 return (error); 5430 } 5431 } 5432 5433 spa_open_ref(spa, tag); 5434 5435 if (config != NULL) 5436 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5437 5438 /* 5439 * If we've recovered the pool, pass back any information we 5440 * gathered while doing the load. 5441 */ 5442 if (state == SPA_LOAD_RECOVER && config != NULL) { 5443 fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 5444 spa->spa_load_info); 5445 } 5446 5447 if (locked) { 5448 spa->spa_last_open_failed = 0; 5449 spa->spa_last_ubsync_txg = 0; 5450 spa->spa_load_txg = 0; 5451 mutex_exit(&spa_namespace_lock); 5452 } 5453 5454 if (firstopen) 5455 zvol_create_minors_recursive(spa_name(spa)); 5456 5457 *spapp = spa; 5458 5459 return (0); 5460 } 5461 5462 int 5463 spa_open_rewind(const char *name, spa_t **spapp, const void *tag, 5464 nvlist_t *policy, nvlist_t **config) 5465 { 5466 return (spa_open_common(name, spapp, tag, policy, config)); 5467 } 5468 5469 int 5470 spa_open(const char *name, spa_t **spapp, const void *tag) 5471 { 5472 return (spa_open_common(name, spapp, tag, NULL, NULL)); 5473 } 5474 5475 /* 5476 * Lookup the given spa_t, incrementing the inject count in the process, 5477 * preventing it from being exported or destroyed. 5478 */ 5479 spa_t * 5480 spa_inject_addref(char *name) 5481 { 5482 spa_t *spa; 5483 5484 mutex_enter(&spa_namespace_lock); 5485 if ((spa = spa_lookup(name)) == NULL) { 5486 mutex_exit(&spa_namespace_lock); 5487 return (NULL); 5488 } 5489 spa->spa_inject_ref++; 5490 mutex_exit(&spa_namespace_lock); 5491 5492 return (spa); 5493 } 5494 5495 void 5496 spa_inject_delref(spa_t *spa) 5497 { 5498 mutex_enter(&spa_namespace_lock); 5499 spa->spa_inject_ref--; 5500 mutex_exit(&spa_namespace_lock); 5501 } 5502 5503 /* 5504 * Add spares device information to the nvlist. 5505 */ 5506 static void 5507 spa_add_spares(spa_t *spa, nvlist_t *config) 5508 { 5509 nvlist_t **spares; 5510 uint_t i, nspares; 5511 nvlist_t *nvroot; 5512 uint64_t guid; 5513 vdev_stat_t *vs; 5514 uint_t vsc; 5515 uint64_t pool; 5516 5517 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5518 5519 if (spa->spa_spares.sav_count == 0) 5520 return; 5521 5522 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 5523 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 5524 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 5525 if (nspares != 0) { 5526 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5527 (const nvlist_t * const *)spares, nspares); 5528 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5529 &spares, &nspares)); 5530 5531 /* 5532 * Go through and find any spares which have since been 5533 * repurposed as an active spare. If this is the case, update 5534 * their status appropriately. 5535 */ 5536 for (i = 0; i < nspares; i++) { 5537 guid = fnvlist_lookup_uint64(spares[i], 5538 ZPOOL_CONFIG_GUID); 5539 VERIFY0(nvlist_lookup_uint64_array(spares[i], 5540 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 5541 if (spa_spare_exists(guid, &pool, NULL) && 5542 pool != 0ULL) { 5543 vs->vs_state = VDEV_STATE_CANT_OPEN; 5544 vs->vs_aux = VDEV_AUX_SPARED; 5545 } else { 5546 vs->vs_state = 5547 spa->spa_spares.sav_vdevs[i]->vdev_state; 5548 } 5549 } 5550 } 5551 } 5552 5553 /* 5554 * Add l2cache device information to the nvlist, including vdev stats. 5555 */ 5556 static void 5557 spa_add_l2cache(spa_t *spa, nvlist_t *config) 5558 { 5559 nvlist_t **l2cache; 5560 uint_t i, j, nl2cache; 5561 nvlist_t *nvroot; 5562 uint64_t guid; 5563 vdev_t *vd; 5564 vdev_stat_t *vs; 5565 uint_t vsc; 5566 5567 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5568 5569 if (spa->spa_l2cache.sav_count == 0) 5570 return; 5571 5572 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 5573 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 5574 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 5575 if (nl2cache != 0) { 5576 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5577 (const nvlist_t * const *)l2cache, nl2cache); 5578 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5579 &l2cache, &nl2cache)); 5580 5581 /* 5582 * Update level 2 cache device stats. 5583 */ 5584 5585 for (i = 0; i < nl2cache; i++) { 5586 guid = fnvlist_lookup_uint64(l2cache[i], 5587 ZPOOL_CONFIG_GUID); 5588 5589 vd = NULL; 5590 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 5591 if (guid == 5592 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 5593 vd = spa->spa_l2cache.sav_vdevs[j]; 5594 break; 5595 } 5596 } 5597 ASSERT(vd != NULL); 5598 5599 VERIFY0(nvlist_lookup_uint64_array(l2cache[i], 5600 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 5601 vdev_get_stats(vd, vs); 5602 vdev_config_generate_stats(vd, l2cache[i]); 5603 5604 } 5605 } 5606 } 5607 5608 static void 5609 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 5610 { 5611 zap_cursor_t zc; 5612 zap_attribute_t za; 5613 5614 if (spa->spa_feat_for_read_obj != 0) { 5615 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5616 spa->spa_feat_for_read_obj); 5617 zap_cursor_retrieve(&zc, &za) == 0; 5618 zap_cursor_advance(&zc)) { 5619 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5620 za.za_num_integers == 1); 5621 VERIFY0(nvlist_add_uint64(features, za.za_name, 5622 za.za_first_integer)); 5623 } 5624 zap_cursor_fini(&zc); 5625 } 5626 5627 if (spa->spa_feat_for_write_obj != 0) { 5628 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5629 spa->spa_feat_for_write_obj); 5630 zap_cursor_retrieve(&zc, &za) == 0; 5631 zap_cursor_advance(&zc)) { 5632 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5633 za.za_num_integers == 1); 5634 VERIFY0(nvlist_add_uint64(features, za.za_name, 5635 za.za_first_integer)); 5636 } 5637 zap_cursor_fini(&zc); 5638 } 5639 } 5640 5641 static void 5642 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 5643 { 5644 int i; 5645 5646 for (i = 0; i < SPA_FEATURES; i++) { 5647 zfeature_info_t feature = spa_feature_table[i]; 5648 uint64_t refcount; 5649 5650 if (feature_get_refcount(spa, &feature, &refcount) != 0) 5651 continue; 5652 5653 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 5654 } 5655 } 5656 5657 /* 5658 * Store a list of pool features and their reference counts in the 5659 * config. 5660 * 5661 * The first time this is called on a spa, allocate a new nvlist, fetch 5662 * the pool features and reference counts from disk, then save the list 5663 * in the spa. In subsequent calls on the same spa use the saved nvlist 5664 * and refresh its values from the cached reference counts. This 5665 * ensures we don't block here on I/O on a suspended pool so 'zpool 5666 * clear' can resume the pool. 5667 */ 5668 static void 5669 spa_add_feature_stats(spa_t *spa, nvlist_t *config) 5670 { 5671 nvlist_t *features; 5672 5673 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5674 5675 mutex_enter(&spa->spa_feat_stats_lock); 5676 features = spa->spa_feat_stats; 5677 5678 if (features != NULL) { 5679 spa_feature_stats_from_cache(spa, features); 5680 } else { 5681 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 5682 spa->spa_feat_stats = features; 5683 spa_feature_stats_from_disk(spa, features); 5684 } 5685 5686 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 5687 features)); 5688 5689 mutex_exit(&spa->spa_feat_stats_lock); 5690 } 5691 5692 int 5693 spa_get_stats(const char *name, nvlist_t **config, 5694 char *altroot, size_t buflen) 5695 { 5696 int error; 5697 spa_t *spa; 5698 5699 *config = NULL; 5700 error = spa_open_common(name, &spa, FTAG, NULL, config); 5701 5702 if (spa != NULL) { 5703 /* 5704 * This still leaves a window of inconsistency where the spares 5705 * or l2cache devices could change and the config would be 5706 * self-inconsistent. 5707 */ 5708 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5709 5710 if (*config != NULL) { 5711 uint64_t loadtimes[2]; 5712 5713 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 5714 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 5715 fnvlist_add_uint64_array(*config, 5716 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2); 5717 5718 fnvlist_add_uint64(*config, 5719 ZPOOL_CONFIG_ERRCOUNT, 5720 spa_approx_errlog_size(spa)); 5721 5722 if (spa_suspended(spa)) { 5723 fnvlist_add_uint64(*config, 5724 ZPOOL_CONFIG_SUSPENDED, 5725 spa->spa_failmode); 5726 fnvlist_add_uint64(*config, 5727 ZPOOL_CONFIG_SUSPENDED_REASON, 5728 spa->spa_suspended); 5729 } 5730 5731 spa_add_spares(spa, *config); 5732 spa_add_l2cache(spa, *config); 5733 spa_add_feature_stats(spa, *config); 5734 } 5735 } 5736 5737 /* 5738 * We want to get the alternate root even for faulted pools, so we cheat 5739 * and call spa_lookup() directly. 5740 */ 5741 if (altroot) { 5742 if (spa == NULL) { 5743 mutex_enter(&spa_namespace_lock); 5744 spa = spa_lookup(name); 5745 if (spa) 5746 spa_altroot(spa, altroot, buflen); 5747 else 5748 altroot[0] = '\0'; 5749 spa = NULL; 5750 mutex_exit(&spa_namespace_lock); 5751 } else { 5752 spa_altroot(spa, altroot, buflen); 5753 } 5754 } 5755 5756 if (spa != NULL) { 5757 spa_config_exit(spa, SCL_CONFIG, FTAG); 5758 spa_close(spa, FTAG); 5759 } 5760 5761 return (error); 5762 } 5763 5764 /* 5765 * Validate that the auxiliary device array is well formed. We must have an 5766 * array of nvlists, each which describes a valid leaf vdev. If this is an 5767 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 5768 * specified, as long as they are well-formed. 5769 */ 5770 static int 5771 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 5772 spa_aux_vdev_t *sav, const char *config, uint64_t version, 5773 vdev_labeltype_t label) 5774 { 5775 nvlist_t **dev; 5776 uint_t i, ndev; 5777 vdev_t *vd; 5778 int error; 5779 5780 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5781 5782 /* 5783 * It's acceptable to have no devs specified. 5784 */ 5785 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 5786 return (0); 5787 5788 if (ndev == 0) 5789 return (SET_ERROR(EINVAL)); 5790 5791 /* 5792 * Make sure the pool is formatted with a version that supports this 5793 * device type. 5794 */ 5795 if (spa_version(spa) < version) 5796 return (SET_ERROR(ENOTSUP)); 5797 5798 /* 5799 * Set the pending device list so we correctly handle device in-use 5800 * checking. 5801 */ 5802 sav->sav_pending = dev; 5803 sav->sav_npending = ndev; 5804 5805 for (i = 0; i < ndev; i++) { 5806 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 5807 mode)) != 0) 5808 goto out; 5809 5810 if (!vd->vdev_ops->vdev_op_leaf) { 5811 vdev_free(vd); 5812 error = SET_ERROR(EINVAL); 5813 goto out; 5814 } 5815 5816 vd->vdev_top = vd; 5817 5818 if ((error = vdev_open(vd)) == 0 && 5819 (error = vdev_label_init(vd, crtxg, label)) == 0) { 5820 fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 5821 vd->vdev_guid); 5822 } 5823 5824 vdev_free(vd); 5825 5826 if (error && 5827 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 5828 goto out; 5829 else 5830 error = 0; 5831 } 5832 5833 out: 5834 sav->sav_pending = NULL; 5835 sav->sav_npending = 0; 5836 return (error); 5837 } 5838 5839 static int 5840 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 5841 { 5842 int error; 5843 5844 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5845 5846 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5847 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 5848 VDEV_LABEL_SPARE)) != 0) { 5849 return (error); 5850 } 5851 5852 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5853 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 5854 VDEV_LABEL_L2CACHE)); 5855 } 5856 5857 static void 5858 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 5859 const char *config) 5860 { 5861 int i; 5862 5863 if (sav->sav_config != NULL) { 5864 nvlist_t **olddevs; 5865 uint_t oldndevs; 5866 nvlist_t **newdevs; 5867 5868 /* 5869 * Generate new dev list by concatenating with the 5870 * current dev list. 5871 */ 5872 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config, 5873 &olddevs, &oldndevs)); 5874 5875 newdevs = kmem_alloc(sizeof (void *) * 5876 (ndevs + oldndevs), KM_SLEEP); 5877 for (i = 0; i < oldndevs; i++) 5878 newdevs[i] = fnvlist_dup(olddevs[i]); 5879 for (i = 0; i < ndevs; i++) 5880 newdevs[i + oldndevs] = fnvlist_dup(devs[i]); 5881 5882 fnvlist_remove(sav->sav_config, config); 5883 5884 fnvlist_add_nvlist_array(sav->sav_config, config, 5885 (const nvlist_t * const *)newdevs, ndevs + oldndevs); 5886 for (i = 0; i < oldndevs + ndevs; i++) 5887 nvlist_free(newdevs[i]); 5888 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 5889 } else { 5890 /* 5891 * Generate a new dev list. 5892 */ 5893 sav->sav_config = fnvlist_alloc(); 5894 fnvlist_add_nvlist_array(sav->sav_config, config, 5895 (const nvlist_t * const *)devs, ndevs); 5896 } 5897 } 5898 5899 /* 5900 * Stop and drop level 2 ARC devices 5901 */ 5902 void 5903 spa_l2cache_drop(spa_t *spa) 5904 { 5905 vdev_t *vd; 5906 int i; 5907 spa_aux_vdev_t *sav = &spa->spa_l2cache; 5908 5909 for (i = 0; i < sav->sav_count; i++) { 5910 uint64_t pool; 5911 5912 vd = sav->sav_vdevs[i]; 5913 ASSERT(vd != NULL); 5914 5915 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 5916 pool != 0ULL && l2arc_vdev_present(vd)) 5917 l2arc_remove_vdev(vd); 5918 } 5919 } 5920 5921 /* 5922 * Verify encryption parameters for spa creation. If we are encrypting, we must 5923 * have the encryption feature flag enabled. 5924 */ 5925 static int 5926 spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 5927 boolean_t has_encryption) 5928 { 5929 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 5930 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 5931 !has_encryption) 5932 return (SET_ERROR(ENOTSUP)); 5933 5934 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 5935 } 5936 5937 /* 5938 * Pool Creation 5939 */ 5940 int 5941 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 5942 nvlist_t *zplprops, dsl_crypto_params_t *dcp) 5943 { 5944 spa_t *spa; 5945 const char *altroot = NULL; 5946 vdev_t *rvd; 5947 dsl_pool_t *dp; 5948 dmu_tx_t *tx; 5949 int error = 0; 5950 uint64_t txg = TXG_INITIAL; 5951 nvlist_t **spares, **l2cache; 5952 uint_t nspares, nl2cache; 5953 uint64_t version, obj, ndraid = 0; 5954 boolean_t has_features; 5955 boolean_t has_encryption; 5956 boolean_t has_allocclass; 5957 spa_feature_t feat; 5958 const char *feat_name; 5959 const char *poolname; 5960 nvlist_t *nvl; 5961 5962 if (props == NULL || 5963 nvlist_lookup_string(props, "tname", &poolname) != 0) 5964 poolname = (char *)pool; 5965 5966 /* 5967 * If this pool already exists, return failure. 5968 */ 5969 mutex_enter(&spa_namespace_lock); 5970 if (spa_lookup(poolname) != NULL) { 5971 mutex_exit(&spa_namespace_lock); 5972 return (SET_ERROR(EEXIST)); 5973 } 5974 5975 /* 5976 * Allocate a new spa_t structure. 5977 */ 5978 nvl = fnvlist_alloc(); 5979 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 5980 (void) nvlist_lookup_string(props, 5981 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 5982 spa = spa_add(poolname, nvl, altroot); 5983 fnvlist_free(nvl); 5984 spa_activate(spa, spa_mode_global); 5985 5986 if (props && (error = spa_prop_validate(spa, props))) { 5987 spa_deactivate(spa); 5988 spa_remove(spa); 5989 mutex_exit(&spa_namespace_lock); 5990 return (error); 5991 } 5992 5993 /* 5994 * Temporary pool names should never be written to disk. 5995 */ 5996 if (poolname != pool) 5997 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 5998 5999 has_features = B_FALSE; 6000 has_encryption = B_FALSE; 6001 has_allocclass = B_FALSE; 6002 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 6003 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 6004 if (zpool_prop_feature(nvpair_name(elem))) { 6005 has_features = B_TRUE; 6006 6007 feat_name = strchr(nvpair_name(elem), '@') + 1; 6008 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 6009 if (feat == SPA_FEATURE_ENCRYPTION) 6010 has_encryption = B_TRUE; 6011 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 6012 has_allocclass = B_TRUE; 6013 } 6014 } 6015 6016 /* verify encryption params, if they were provided */ 6017 if (dcp != NULL) { 6018 error = spa_create_check_encryption_params(dcp, has_encryption); 6019 if (error != 0) { 6020 spa_deactivate(spa); 6021 spa_remove(spa); 6022 mutex_exit(&spa_namespace_lock); 6023 return (error); 6024 } 6025 } 6026 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 6027 spa_deactivate(spa); 6028 spa_remove(spa); 6029 mutex_exit(&spa_namespace_lock); 6030 return (ENOTSUP); 6031 } 6032 6033 if (has_features || nvlist_lookup_uint64(props, 6034 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 6035 version = SPA_VERSION; 6036 } 6037 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 6038 6039 spa->spa_first_txg = txg; 6040 spa->spa_uberblock.ub_txg = txg - 1; 6041 spa->spa_uberblock.ub_version = version; 6042 spa->spa_ubsync = spa->spa_uberblock; 6043 spa->spa_load_state = SPA_LOAD_CREATE; 6044 spa->spa_removing_phys.sr_state = DSS_NONE; 6045 spa->spa_removing_phys.sr_removing_vdev = -1; 6046 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 6047 spa->spa_indirect_vdevs_loaded = B_TRUE; 6048 6049 /* 6050 * Create "The Godfather" zio to hold all async IOs 6051 */ 6052 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 6053 KM_SLEEP); 6054 for (int i = 0; i < max_ncpus; i++) { 6055 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 6056 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 6057 ZIO_FLAG_GODFATHER); 6058 } 6059 6060 /* 6061 * Create the root vdev. 6062 */ 6063 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6064 6065 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 6066 6067 ASSERT(error != 0 || rvd != NULL); 6068 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 6069 6070 if (error == 0 && !zfs_allocatable_devs(nvroot)) 6071 error = SET_ERROR(EINVAL); 6072 6073 if (error == 0 && 6074 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 6075 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 && 6076 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 6077 /* 6078 * instantiate the metaslab groups (this will dirty the vdevs) 6079 * we can no longer error exit past this point 6080 */ 6081 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 6082 vdev_t *vd = rvd->vdev_child[c]; 6083 6084 vdev_metaslab_set_size(vd); 6085 vdev_expand(vd, txg); 6086 } 6087 } 6088 6089 spa_config_exit(spa, SCL_ALL, FTAG); 6090 6091 if (error != 0) { 6092 spa_unload(spa); 6093 spa_deactivate(spa); 6094 spa_remove(spa); 6095 mutex_exit(&spa_namespace_lock); 6096 return (error); 6097 } 6098 6099 /* 6100 * Get the list of spares, if specified. 6101 */ 6102 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6103 &spares, &nspares) == 0) { 6104 spa->spa_spares.sav_config = fnvlist_alloc(); 6105 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 6106 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 6107 nspares); 6108 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6109 spa_load_spares(spa); 6110 spa_config_exit(spa, SCL_ALL, FTAG); 6111 spa->spa_spares.sav_sync = B_TRUE; 6112 } 6113 6114 /* 6115 * Get the list of level 2 cache devices, if specified. 6116 */ 6117 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6118 &l2cache, &nl2cache) == 0) { 6119 VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config, 6120 NV_UNIQUE_NAME, KM_SLEEP)); 6121 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6122 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 6123 nl2cache); 6124 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6125 spa_load_l2cache(spa); 6126 spa_config_exit(spa, SCL_ALL, FTAG); 6127 spa->spa_l2cache.sav_sync = B_TRUE; 6128 } 6129 6130 spa->spa_is_initializing = B_TRUE; 6131 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 6132 spa->spa_is_initializing = B_FALSE; 6133 6134 /* 6135 * Create DDTs (dedup tables). 6136 */ 6137 ddt_create(spa); 6138 /* 6139 * Create BRT table and BRT table object. 6140 */ 6141 brt_create(spa); 6142 6143 spa_update_dspace(spa); 6144 6145 tx = dmu_tx_create_assigned(dp, txg); 6146 6147 /* 6148 * Create the pool's history object. 6149 */ 6150 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 6151 spa_history_create_obj(spa, tx); 6152 6153 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 6154 spa_history_log_version(spa, "create", tx); 6155 6156 /* 6157 * Create the pool config object. 6158 */ 6159 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 6160 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 6161 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 6162 6163 if (zap_add(spa->spa_meta_objset, 6164 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 6165 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 6166 cmn_err(CE_PANIC, "failed to add pool config"); 6167 } 6168 6169 if (zap_add(spa->spa_meta_objset, 6170 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 6171 sizeof (uint64_t), 1, &version, tx) != 0) { 6172 cmn_err(CE_PANIC, "failed to add pool version"); 6173 } 6174 6175 /* Newly created pools with the right version are always deflated. */ 6176 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 6177 spa->spa_deflate = TRUE; 6178 if (zap_add(spa->spa_meta_objset, 6179 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 6180 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 6181 cmn_err(CE_PANIC, "failed to add deflate"); 6182 } 6183 } 6184 6185 /* 6186 * Create the deferred-free bpobj. Turn off compression 6187 * because sync-to-convergence takes longer if the blocksize 6188 * keeps changing. 6189 */ 6190 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 6191 dmu_object_set_compress(spa->spa_meta_objset, obj, 6192 ZIO_COMPRESS_OFF, tx); 6193 if (zap_add(spa->spa_meta_objset, 6194 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 6195 sizeof (uint64_t), 1, &obj, tx) != 0) { 6196 cmn_err(CE_PANIC, "failed to add bpobj"); 6197 } 6198 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 6199 spa->spa_meta_objset, obj)); 6200 6201 /* 6202 * Generate some random noise for salted checksums to operate on. 6203 */ 6204 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 6205 sizeof (spa->spa_cksum_salt.zcs_bytes)); 6206 6207 /* 6208 * Set pool properties. 6209 */ 6210 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 6211 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 6212 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 6213 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 6214 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 6215 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 6216 6217 if (props != NULL) { 6218 spa_configfile_set(spa, props, B_FALSE); 6219 spa_sync_props(props, tx); 6220 } 6221 6222 for (int i = 0; i < ndraid; i++) 6223 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6224 6225 dmu_tx_commit(tx); 6226 6227 spa->spa_sync_on = B_TRUE; 6228 txg_sync_start(dp); 6229 mmp_thread_start(spa); 6230 txg_wait_synced(dp, txg); 6231 6232 spa_spawn_aux_threads(spa); 6233 6234 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 6235 6236 /* 6237 * Don't count references from objsets that are already closed 6238 * and are making their way through the eviction process. 6239 */ 6240 spa_evicting_os_wait(spa); 6241 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 6242 spa->spa_load_state = SPA_LOAD_NONE; 6243 6244 spa_import_os(spa); 6245 6246 mutex_exit(&spa_namespace_lock); 6247 6248 return (0); 6249 } 6250 6251 /* 6252 * Import a non-root pool into the system. 6253 */ 6254 int 6255 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 6256 { 6257 spa_t *spa; 6258 const char *altroot = NULL; 6259 spa_load_state_t state = SPA_LOAD_IMPORT; 6260 zpool_load_policy_t policy; 6261 spa_mode_t mode = spa_mode_global; 6262 uint64_t readonly = B_FALSE; 6263 int error; 6264 nvlist_t *nvroot; 6265 nvlist_t **spares, **l2cache; 6266 uint_t nspares, nl2cache; 6267 6268 /* 6269 * If a pool with this name exists, return failure. 6270 */ 6271 mutex_enter(&spa_namespace_lock); 6272 if (spa_lookup(pool) != NULL) { 6273 mutex_exit(&spa_namespace_lock); 6274 return (SET_ERROR(EEXIST)); 6275 } 6276 6277 /* 6278 * Create and initialize the spa structure. 6279 */ 6280 (void) nvlist_lookup_string(props, 6281 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6282 (void) nvlist_lookup_uint64(props, 6283 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 6284 if (readonly) 6285 mode = SPA_MODE_READ; 6286 spa = spa_add(pool, config, altroot); 6287 spa->spa_import_flags = flags; 6288 6289 /* 6290 * Verbatim import - Take a pool and insert it into the namespace 6291 * as if it had been loaded at boot. 6292 */ 6293 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 6294 if (props != NULL) 6295 spa_configfile_set(spa, props, B_FALSE); 6296 6297 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE); 6298 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6299 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 6300 mutex_exit(&spa_namespace_lock); 6301 return (0); 6302 } 6303 6304 spa_activate(spa, mode); 6305 6306 /* 6307 * Don't start async tasks until we know everything is healthy. 6308 */ 6309 spa_async_suspend(spa); 6310 6311 zpool_get_load_policy(config, &policy); 6312 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6313 state = SPA_LOAD_RECOVER; 6314 6315 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 6316 6317 if (state != SPA_LOAD_RECOVER) { 6318 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6319 zfs_dbgmsg("spa_import: importing %s", pool); 6320 } else { 6321 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 6322 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 6323 } 6324 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 6325 6326 /* 6327 * Propagate anything learned while loading the pool and pass it 6328 * back to caller (i.e. rewind info, missing devices, etc). 6329 */ 6330 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info); 6331 6332 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6333 /* 6334 * Toss any existing sparelist, as it doesn't have any validity 6335 * anymore, and conflicts with spa_has_spare(). 6336 */ 6337 if (spa->spa_spares.sav_config) { 6338 nvlist_free(spa->spa_spares.sav_config); 6339 spa->spa_spares.sav_config = NULL; 6340 spa_load_spares(spa); 6341 } 6342 if (spa->spa_l2cache.sav_config) { 6343 nvlist_free(spa->spa_l2cache.sav_config); 6344 spa->spa_l2cache.sav_config = NULL; 6345 spa_load_l2cache(spa); 6346 } 6347 6348 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 6349 spa_config_exit(spa, SCL_ALL, FTAG); 6350 6351 if (props != NULL) 6352 spa_configfile_set(spa, props, B_FALSE); 6353 6354 if (error != 0 || (props && spa_writeable(spa) && 6355 (error = spa_prop_set(spa, props)))) { 6356 spa_unload(spa); 6357 spa_deactivate(spa); 6358 spa_remove(spa); 6359 mutex_exit(&spa_namespace_lock); 6360 return (error); 6361 } 6362 6363 spa_async_resume(spa); 6364 6365 /* 6366 * Override any spares and level 2 cache devices as specified by 6367 * the user, as these may have correct device names/devids, etc. 6368 */ 6369 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6370 &spares, &nspares) == 0) { 6371 if (spa->spa_spares.sav_config) 6372 fnvlist_remove(spa->spa_spares.sav_config, 6373 ZPOOL_CONFIG_SPARES); 6374 else 6375 spa->spa_spares.sav_config = fnvlist_alloc(); 6376 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 6377 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 6378 nspares); 6379 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6380 spa_load_spares(spa); 6381 spa_config_exit(spa, SCL_ALL, FTAG); 6382 spa->spa_spares.sav_sync = B_TRUE; 6383 } 6384 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6385 &l2cache, &nl2cache) == 0) { 6386 if (spa->spa_l2cache.sav_config) 6387 fnvlist_remove(spa->spa_l2cache.sav_config, 6388 ZPOOL_CONFIG_L2CACHE); 6389 else 6390 spa->spa_l2cache.sav_config = fnvlist_alloc(); 6391 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6392 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 6393 nl2cache); 6394 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6395 spa_load_l2cache(spa); 6396 spa_config_exit(spa, SCL_ALL, FTAG); 6397 spa->spa_l2cache.sav_sync = B_TRUE; 6398 } 6399 6400 /* 6401 * Check for any removed devices. 6402 */ 6403 if (spa->spa_autoreplace) { 6404 spa_aux_check_removed(&spa->spa_spares); 6405 spa_aux_check_removed(&spa->spa_l2cache); 6406 } 6407 6408 if (spa_writeable(spa)) { 6409 /* 6410 * Update the config cache to include the newly-imported pool. 6411 */ 6412 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6413 } 6414 6415 /* 6416 * It's possible that the pool was expanded while it was exported. 6417 * We kick off an async task to handle this for us. 6418 */ 6419 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 6420 6421 spa_history_log_version(spa, "import", NULL); 6422 6423 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6424 6425 mutex_exit(&spa_namespace_lock); 6426 6427 zvol_create_minors_recursive(pool); 6428 6429 spa_import_os(spa); 6430 6431 return (0); 6432 } 6433 6434 nvlist_t * 6435 spa_tryimport(nvlist_t *tryconfig) 6436 { 6437 nvlist_t *config = NULL; 6438 const char *poolname, *cachefile; 6439 spa_t *spa; 6440 uint64_t state; 6441 int error; 6442 zpool_load_policy_t policy; 6443 6444 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 6445 return (NULL); 6446 6447 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 6448 return (NULL); 6449 6450 /* 6451 * Create and initialize the spa structure. 6452 */ 6453 mutex_enter(&spa_namespace_lock); 6454 spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL); 6455 spa_activate(spa, SPA_MODE_READ); 6456 6457 /* 6458 * Rewind pool if a max txg was provided. 6459 */ 6460 zpool_get_load_policy(spa->spa_config, &policy); 6461 if (policy.zlp_txg != UINT64_MAX) { 6462 spa->spa_load_max_txg = policy.zlp_txg; 6463 spa->spa_extreme_rewind = B_TRUE; 6464 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 6465 poolname, (longlong_t)policy.zlp_txg); 6466 } else { 6467 zfs_dbgmsg("spa_tryimport: importing %s", poolname); 6468 } 6469 6470 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 6471 == 0) { 6472 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 6473 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6474 } else { 6475 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 6476 } 6477 6478 /* 6479 * spa_import() relies on a pool config fetched by spa_try_import() 6480 * for spare/cache devices. Import flags are not passed to 6481 * spa_tryimport(), which makes it return early due to a missing log 6482 * device and missing retrieving the cache device and spare eventually. 6483 * Passing ZFS_IMPORT_MISSING_LOG to spa_tryimport() makes it fetch 6484 * the correct configuration regardless of the missing log device. 6485 */ 6486 spa->spa_import_flags |= ZFS_IMPORT_MISSING_LOG; 6487 6488 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 6489 6490 /* 6491 * If 'tryconfig' was at least parsable, return the current config. 6492 */ 6493 if (spa->spa_root_vdev != NULL) { 6494 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6495 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, poolname); 6496 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state); 6497 fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 6498 spa->spa_uberblock.ub_timestamp); 6499 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6500 spa->spa_load_info); 6501 fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 6502 spa->spa_errata); 6503 6504 /* 6505 * If the bootfs property exists on this pool then we 6506 * copy it out so that external consumers can tell which 6507 * pools are bootable. 6508 */ 6509 if ((!error || error == EEXIST) && spa->spa_bootfs) { 6510 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6511 6512 /* 6513 * We have to play games with the name since the 6514 * pool was opened as TRYIMPORT_NAME. 6515 */ 6516 if (dsl_dsobj_to_dsname(spa_name(spa), 6517 spa->spa_bootfs, tmpname) == 0) { 6518 char *cp; 6519 char *dsname; 6520 6521 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6522 6523 cp = strchr(tmpname, '/'); 6524 if (cp == NULL) { 6525 (void) strlcpy(dsname, tmpname, 6526 MAXPATHLEN); 6527 } else { 6528 (void) snprintf(dsname, MAXPATHLEN, 6529 "%s/%s", poolname, ++cp); 6530 } 6531 fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS, 6532 dsname); 6533 kmem_free(dsname, MAXPATHLEN); 6534 } 6535 kmem_free(tmpname, MAXPATHLEN); 6536 } 6537 6538 /* 6539 * Add the list of hot spares and level 2 cache devices. 6540 */ 6541 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6542 spa_add_spares(spa, config); 6543 spa_add_l2cache(spa, config); 6544 spa_config_exit(spa, SCL_CONFIG, FTAG); 6545 } 6546 6547 spa_unload(spa); 6548 spa_deactivate(spa); 6549 spa_remove(spa); 6550 mutex_exit(&spa_namespace_lock); 6551 6552 return (config); 6553 } 6554 6555 /* 6556 * Pool export/destroy 6557 * 6558 * The act of destroying or exporting a pool is very simple. We make sure there 6559 * is no more pending I/O and any references to the pool are gone. Then, we 6560 * update the pool state and sync all the labels to disk, removing the 6561 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 6562 * we don't sync the labels or remove the configuration cache. 6563 */ 6564 static int 6565 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 6566 boolean_t force, boolean_t hardforce) 6567 { 6568 int error; 6569 spa_t *spa; 6570 6571 if (oldconfig) 6572 *oldconfig = NULL; 6573 6574 if (!(spa_mode_global & SPA_MODE_WRITE)) 6575 return (SET_ERROR(EROFS)); 6576 6577 mutex_enter(&spa_namespace_lock); 6578 if ((spa = spa_lookup(pool)) == NULL) { 6579 mutex_exit(&spa_namespace_lock); 6580 return (SET_ERROR(ENOENT)); 6581 } 6582 6583 if (spa->spa_is_exporting) { 6584 /* the pool is being exported by another thread */ 6585 mutex_exit(&spa_namespace_lock); 6586 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 6587 } 6588 spa->spa_is_exporting = B_TRUE; 6589 6590 /* 6591 * Put a hold on the pool, drop the namespace lock, stop async tasks, 6592 * reacquire the namespace lock, and see if we can export. 6593 */ 6594 spa_open_ref(spa, FTAG); 6595 mutex_exit(&spa_namespace_lock); 6596 spa_async_suspend(spa); 6597 if (spa->spa_zvol_taskq) { 6598 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 6599 taskq_wait(spa->spa_zvol_taskq); 6600 } 6601 mutex_enter(&spa_namespace_lock); 6602 spa_close(spa, FTAG); 6603 6604 if (spa->spa_state == POOL_STATE_UNINITIALIZED) 6605 goto export_spa; 6606 /* 6607 * The pool will be in core if it's openable, in which case we can 6608 * modify its state. Objsets may be open only because they're dirty, 6609 * so we have to force it to sync before checking spa_refcnt. 6610 */ 6611 if (spa->spa_sync_on) { 6612 txg_wait_synced(spa->spa_dsl_pool, 0); 6613 spa_evicting_os_wait(spa); 6614 } 6615 6616 /* 6617 * A pool cannot be exported or destroyed if there are active 6618 * references. If we are resetting a pool, allow references by 6619 * fault injection handlers. 6620 */ 6621 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 6622 error = SET_ERROR(EBUSY); 6623 goto fail; 6624 } 6625 6626 if (spa->spa_sync_on) { 6627 vdev_t *rvd = spa->spa_root_vdev; 6628 /* 6629 * A pool cannot be exported if it has an active shared spare. 6630 * This is to prevent other pools stealing the active spare 6631 * from an exported pool. At user's own will, such pool can 6632 * be forcedly exported. 6633 */ 6634 if (!force && new_state == POOL_STATE_EXPORTED && 6635 spa_has_active_shared_spare(spa)) { 6636 error = SET_ERROR(EXDEV); 6637 goto fail; 6638 } 6639 6640 /* 6641 * We're about to export or destroy this pool. Make sure 6642 * we stop all initialization and trim activity here before 6643 * we set the spa_final_txg. This will ensure that all 6644 * dirty data resulting from the initialization is 6645 * committed to disk before we unload the pool. 6646 */ 6647 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 6648 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 6649 vdev_autotrim_stop_all(spa); 6650 vdev_rebuild_stop_all(spa); 6651 6652 /* 6653 * We want this to be reflected on every label, 6654 * so mark them all dirty. spa_unload() will do the 6655 * final sync that pushes these changes out. 6656 */ 6657 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6658 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6659 spa->spa_state = new_state; 6660 vdev_config_dirty(rvd); 6661 spa_config_exit(spa, SCL_ALL, FTAG); 6662 } 6663 6664 /* 6665 * If the log space map feature is enabled and the pool is 6666 * getting exported (but not destroyed), we want to spend some 6667 * time flushing as many metaslabs as we can in an attempt to 6668 * destroy log space maps and save import time. This has to be 6669 * done before we set the spa_final_txg, otherwise 6670 * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs. 6671 * spa_should_flush_logs_on_unload() should be called after 6672 * spa_state has been set to the new_state. 6673 */ 6674 if (spa_should_flush_logs_on_unload(spa)) 6675 spa_unload_log_sm_flush_all(spa); 6676 6677 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6678 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6679 spa->spa_final_txg = spa_last_synced_txg(spa) + 6680 TXG_DEFER_SIZE + 1; 6681 spa_config_exit(spa, SCL_ALL, FTAG); 6682 } 6683 } 6684 6685 export_spa: 6686 spa_export_os(spa); 6687 6688 if (new_state == POOL_STATE_DESTROYED) 6689 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 6690 else if (new_state == POOL_STATE_EXPORTED) 6691 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 6692 6693 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 6694 spa_unload(spa); 6695 spa_deactivate(spa); 6696 } 6697 6698 if (oldconfig && spa->spa_config) 6699 *oldconfig = fnvlist_dup(spa->spa_config); 6700 6701 if (new_state != POOL_STATE_UNINITIALIZED) { 6702 if (!hardforce) 6703 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 6704 spa_remove(spa); 6705 } else { 6706 /* 6707 * If spa_remove() is not called for this spa_t and 6708 * there is any possibility that it can be reused, 6709 * we make sure to reset the exporting flag. 6710 */ 6711 spa->spa_is_exporting = B_FALSE; 6712 } 6713 6714 mutex_exit(&spa_namespace_lock); 6715 return (0); 6716 6717 fail: 6718 spa->spa_is_exporting = B_FALSE; 6719 spa_async_resume(spa); 6720 mutex_exit(&spa_namespace_lock); 6721 return (error); 6722 } 6723 6724 /* 6725 * Destroy a storage pool. 6726 */ 6727 int 6728 spa_destroy(const char *pool) 6729 { 6730 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 6731 B_FALSE, B_FALSE)); 6732 } 6733 6734 /* 6735 * Export a storage pool. 6736 */ 6737 int 6738 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 6739 boolean_t hardforce) 6740 { 6741 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 6742 force, hardforce)); 6743 } 6744 6745 /* 6746 * Similar to spa_export(), this unloads the spa_t without actually removing it 6747 * from the namespace in any way. 6748 */ 6749 int 6750 spa_reset(const char *pool) 6751 { 6752 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 6753 B_FALSE, B_FALSE)); 6754 } 6755 6756 /* 6757 * ========================================================================== 6758 * Device manipulation 6759 * ========================================================================== 6760 */ 6761 6762 /* 6763 * This is called as a synctask to increment the draid feature flag 6764 */ 6765 static void 6766 spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 6767 { 6768 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 6769 int draid = (int)(uintptr_t)arg; 6770 6771 for (int c = 0; c < draid; c++) 6772 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6773 } 6774 6775 /* 6776 * Add a device to a storage pool. 6777 */ 6778 int 6779 spa_vdev_add(spa_t *spa, nvlist_t *nvroot) 6780 { 6781 uint64_t txg, ndraid = 0; 6782 int error; 6783 vdev_t *rvd = spa->spa_root_vdev; 6784 vdev_t *vd, *tvd; 6785 nvlist_t **spares, **l2cache; 6786 uint_t nspares, nl2cache; 6787 6788 ASSERT(spa_writeable(spa)); 6789 6790 txg = spa_vdev_enter(spa); 6791 6792 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 6793 VDEV_ALLOC_ADD)) != 0) 6794 return (spa_vdev_exit(spa, NULL, txg, error)); 6795 6796 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 6797 6798 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 6799 &nspares) != 0) 6800 nspares = 0; 6801 6802 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 6803 &nl2cache) != 0) 6804 nl2cache = 0; 6805 6806 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 6807 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6808 6809 if (vd->vdev_children != 0 && 6810 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 6811 return (spa_vdev_exit(spa, vd, txg, error)); 6812 } 6813 6814 /* 6815 * The virtual dRAID spares must be added after vdev tree is created 6816 * and the vdev guids are generated. The guid of their associated 6817 * dRAID is stored in the config and used when opening the spare. 6818 */ 6819 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 6820 rvd->vdev_children)) == 0) { 6821 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 6822 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 6823 nspares = 0; 6824 } else { 6825 return (spa_vdev_exit(spa, vd, txg, error)); 6826 } 6827 6828 /* 6829 * We must validate the spares and l2cache devices after checking the 6830 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 6831 */ 6832 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 6833 return (spa_vdev_exit(spa, vd, txg, error)); 6834 6835 /* 6836 * If we are in the middle of a device removal, we can only add 6837 * devices which match the existing devices in the pool. 6838 * If we are in the middle of a removal, or have some indirect 6839 * vdevs, we can not add raidz or dRAID top levels. 6840 */ 6841 if (spa->spa_vdev_removal != NULL || 6842 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 6843 for (int c = 0; c < vd->vdev_children; c++) { 6844 tvd = vd->vdev_child[c]; 6845 if (spa->spa_vdev_removal != NULL && 6846 tvd->vdev_ashift != spa->spa_max_ashift) { 6847 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6848 } 6849 /* Fail if top level vdev is raidz or a dRAID */ 6850 if (vdev_get_nparity(tvd) != 0) 6851 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6852 6853 /* 6854 * Need the top level mirror to be 6855 * a mirror of leaf vdevs only 6856 */ 6857 if (tvd->vdev_ops == &vdev_mirror_ops) { 6858 for (uint64_t cid = 0; 6859 cid < tvd->vdev_children; cid++) { 6860 vdev_t *cvd = tvd->vdev_child[cid]; 6861 if (!cvd->vdev_ops->vdev_op_leaf) { 6862 return (spa_vdev_exit(spa, vd, 6863 txg, EINVAL)); 6864 } 6865 } 6866 } 6867 } 6868 } 6869 6870 for (int c = 0; c < vd->vdev_children; c++) { 6871 tvd = vd->vdev_child[c]; 6872 vdev_remove_child(vd, tvd); 6873 tvd->vdev_id = rvd->vdev_children; 6874 vdev_add_child(rvd, tvd); 6875 vdev_config_dirty(tvd); 6876 } 6877 6878 if (nspares != 0) { 6879 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 6880 ZPOOL_CONFIG_SPARES); 6881 spa_load_spares(spa); 6882 spa->spa_spares.sav_sync = B_TRUE; 6883 } 6884 6885 if (nl2cache != 0) { 6886 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 6887 ZPOOL_CONFIG_L2CACHE); 6888 spa_load_l2cache(spa); 6889 spa->spa_l2cache.sav_sync = B_TRUE; 6890 } 6891 6892 /* 6893 * We can't increment a feature while holding spa_vdev so we 6894 * have to do it in a synctask. 6895 */ 6896 if (ndraid != 0) { 6897 dmu_tx_t *tx; 6898 6899 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 6900 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 6901 (void *)(uintptr_t)ndraid, tx); 6902 dmu_tx_commit(tx); 6903 } 6904 6905 /* 6906 * We have to be careful when adding new vdevs to an existing pool. 6907 * If other threads start allocating from these vdevs before we 6908 * sync the config cache, and we lose power, then upon reboot we may 6909 * fail to open the pool because there are DVAs that the config cache 6910 * can't translate. Therefore, we first add the vdevs without 6911 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 6912 * and then let spa_config_update() initialize the new metaslabs. 6913 * 6914 * spa_load() checks for added-but-not-initialized vdevs, so that 6915 * if we lose power at any point in this sequence, the remaining 6916 * steps will be completed the next time we load the pool. 6917 */ 6918 (void) spa_vdev_exit(spa, vd, txg, 0); 6919 6920 mutex_enter(&spa_namespace_lock); 6921 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6922 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 6923 mutex_exit(&spa_namespace_lock); 6924 6925 return (0); 6926 } 6927 6928 /* 6929 * Attach a device to a vdev specified by its guid. The vdev type can be 6930 * a mirror, a raidz, or a leaf device that is also a top-level (e.g. a 6931 * single device). When the vdev is a single device, a mirror vdev will be 6932 * automatically inserted. 6933 * 6934 * If 'replacing' is specified, the new device is intended to replace the 6935 * existing device; in this case the two devices are made into their own 6936 * mirror using the 'replacing' vdev, which is functionally identical to 6937 * the mirror vdev (it actually reuses all the same ops) but has a few 6938 * extra rules: you can't attach to it after it's been created, and upon 6939 * completion of resilvering, the first disk (the one being replaced) 6940 * is automatically detached. 6941 * 6942 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 6943 * should be performed instead of traditional healing reconstruction. From 6944 * an administrators perspective these are both resilver operations. 6945 */ 6946 int 6947 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 6948 int rebuild) 6949 { 6950 uint64_t txg, dtl_max_txg; 6951 vdev_t *rvd = spa->spa_root_vdev; 6952 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 6953 vdev_ops_t *pvops; 6954 char *oldvdpath, *newvdpath; 6955 int newvd_isspare = B_FALSE; 6956 int error; 6957 6958 ASSERT(spa_writeable(spa)); 6959 6960 txg = spa_vdev_enter(spa); 6961 6962 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 6963 6964 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6965 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6966 error = (spa_has_checkpoint(spa)) ? 6967 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6968 return (spa_vdev_exit(spa, NULL, txg, error)); 6969 } 6970 6971 if (rebuild) { 6972 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 6973 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6974 6975 if (dsl_scan_resilvering(spa_get_dsl(spa)) || 6976 dsl_scan_resilver_scheduled(spa_get_dsl(spa))) { 6977 return (spa_vdev_exit(spa, NULL, txg, 6978 ZFS_ERR_RESILVER_IN_PROGRESS)); 6979 } 6980 } else { 6981 if (vdev_rebuild_active(rvd)) 6982 return (spa_vdev_exit(spa, NULL, txg, 6983 ZFS_ERR_REBUILD_IN_PROGRESS)); 6984 } 6985 6986 if (spa->spa_vdev_removal != NULL) { 6987 return (spa_vdev_exit(spa, NULL, txg, 6988 ZFS_ERR_DEVRM_IN_PROGRESS)); 6989 } 6990 6991 if (oldvd == NULL) 6992 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6993 6994 boolean_t raidz = oldvd->vdev_ops == &vdev_raidz_ops; 6995 6996 if (raidz) { 6997 if (!spa_feature_is_enabled(spa, SPA_FEATURE_RAIDZ_EXPANSION)) 6998 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6999 7000 /* 7001 * Can't expand a raidz while prior expand is in progress. 7002 */ 7003 if (spa->spa_raidz_expand != NULL) { 7004 return (spa_vdev_exit(spa, NULL, txg, 7005 ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS)); 7006 } 7007 } else if (!oldvd->vdev_ops->vdev_op_leaf) { 7008 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7009 } 7010 7011 if (raidz) 7012 pvd = oldvd; 7013 else 7014 pvd = oldvd->vdev_parent; 7015 7016 if (spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 7017 VDEV_ALLOC_ATTACH) != 0) 7018 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7019 7020 if (newrootvd->vdev_children != 1) 7021 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 7022 7023 newvd = newrootvd->vdev_child[0]; 7024 7025 if (!newvd->vdev_ops->vdev_op_leaf) 7026 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 7027 7028 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 7029 return (spa_vdev_exit(spa, newrootvd, txg, error)); 7030 7031 /* 7032 * log, dedup and special vdevs should not be replaced by spares. 7033 */ 7034 if ((oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE || 7035 oldvd->vdev_top->vdev_islog) && newvd->vdev_isspare) { 7036 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7037 } 7038 7039 /* 7040 * A dRAID spare can only replace a child of its parent dRAID vdev. 7041 */ 7042 if (newvd->vdev_ops == &vdev_draid_spare_ops && 7043 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 7044 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7045 } 7046 7047 if (rebuild) { 7048 /* 7049 * For rebuilds, the top vdev must support reconstruction 7050 * using only space maps. This means the only allowable 7051 * vdevs types are the root vdev, a mirror, or dRAID. 7052 */ 7053 tvd = pvd; 7054 if (pvd->vdev_top != NULL) 7055 tvd = pvd->vdev_top; 7056 7057 if (tvd->vdev_ops != &vdev_mirror_ops && 7058 tvd->vdev_ops != &vdev_root_ops && 7059 tvd->vdev_ops != &vdev_draid_ops) { 7060 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7061 } 7062 } 7063 7064 if (!replacing) { 7065 /* 7066 * For attach, the only allowable parent is a mirror or the root 7067 * vdev. 7068 */ 7069 if (pvd->vdev_ops != &vdev_mirror_ops && 7070 pvd->vdev_ops != &vdev_raidz_ops && 7071 pvd->vdev_ops != &vdev_root_ops) 7072 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7073 7074 pvops = &vdev_mirror_ops; 7075 } else { 7076 /* 7077 * Active hot spares can only be replaced by inactive hot 7078 * spares. 7079 */ 7080 if (pvd->vdev_ops == &vdev_spare_ops && 7081 oldvd->vdev_isspare && 7082 !spa_has_spare(spa, newvd->vdev_guid)) 7083 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7084 7085 /* 7086 * If the source is a hot spare, and the parent isn't already a 7087 * spare, then we want to create a new hot spare. Otherwise, we 7088 * want to create a replacing vdev. The user is not allowed to 7089 * attach to a spared vdev child unless the 'isspare' state is 7090 * the same (spare replaces spare, non-spare replaces 7091 * non-spare). 7092 */ 7093 if (pvd->vdev_ops == &vdev_replacing_ops && 7094 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 7095 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7096 } else if (pvd->vdev_ops == &vdev_spare_ops && 7097 newvd->vdev_isspare != oldvd->vdev_isspare) { 7098 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7099 } 7100 7101 if (newvd->vdev_isspare) 7102 pvops = &vdev_spare_ops; 7103 else 7104 pvops = &vdev_replacing_ops; 7105 } 7106 7107 /* 7108 * Make sure the new device is big enough. 7109 */ 7110 vdev_t *min_vdev = raidz ? oldvd->vdev_child[0] : oldvd; 7111 if (newvd->vdev_asize < vdev_get_min_asize(min_vdev)) 7112 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 7113 7114 /* 7115 * The new device cannot have a higher alignment requirement 7116 * than the top-level vdev. 7117 */ 7118 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) 7119 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7120 7121 /* 7122 * RAIDZ-expansion-specific checks. 7123 */ 7124 if (raidz) { 7125 if (vdev_raidz_attach_check(newvd) != 0) 7126 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 7127 7128 /* 7129 * Fail early if a child is not healthy or being replaced 7130 */ 7131 for (int i = 0; i < oldvd->vdev_children; i++) { 7132 if (vdev_is_dead(oldvd->vdev_child[i]) || 7133 !oldvd->vdev_child[i]->vdev_ops->vdev_op_leaf) { 7134 return (spa_vdev_exit(spa, newrootvd, txg, 7135 ENXIO)); 7136 } 7137 /* Also fail if reserved boot area is in-use */ 7138 if (vdev_check_boot_reserve(spa, oldvd->vdev_child[i]) 7139 != 0) { 7140 return (spa_vdev_exit(spa, newrootvd, txg, 7141 EADDRINUSE)); 7142 } 7143 } 7144 } 7145 7146 if (raidz) { 7147 /* 7148 * Note: oldvdpath is freed by spa_strfree(), but 7149 * kmem_asprintf() is freed by kmem_strfree(), so we have to 7150 * move it to a spa_strdup-ed string. 7151 */ 7152 char *tmp = kmem_asprintf("raidz%u-%u", 7153 (uint_t)vdev_get_nparity(oldvd), (uint_t)oldvd->vdev_id); 7154 oldvdpath = spa_strdup(tmp); 7155 kmem_strfree(tmp); 7156 } else { 7157 oldvdpath = spa_strdup(oldvd->vdev_path); 7158 } 7159 newvdpath = spa_strdup(newvd->vdev_path); 7160 7161 /* 7162 * If this is an in-place replacement, update oldvd's path and devid 7163 * to make it distinguishable from newvd, and unopenable from now on. 7164 */ 7165 if (strcmp(oldvdpath, newvdpath) == 0) { 7166 spa_strfree(oldvd->vdev_path); 7167 oldvd->vdev_path = kmem_alloc(strlen(newvdpath) + 5, 7168 KM_SLEEP); 7169 (void) sprintf(oldvd->vdev_path, "%s/old", 7170 newvdpath); 7171 if (oldvd->vdev_devid != NULL) { 7172 spa_strfree(oldvd->vdev_devid); 7173 oldvd->vdev_devid = NULL; 7174 } 7175 spa_strfree(oldvdpath); 7176 oldvdpath = spa_strdup(oldvd->vdev_path); 7177 } 7178 7179 /* 7180 * If the parent is not a mirror, or if we're replacing, insert the new 7181 * mirror/replacing/spare vdev above oldvd. 7182 */ 7183 if (!raidz && pvd->vdev_ops != pvops) { 7184 pvd = vdev_add_parent(oldvd, pvops); 7185 ASSERT(pvd->vdev_ops == pvops); 7186 ASSERT(oldvd->vdev_parent == pvd); 7187 } 7188 7189 ASSERT(pvd->vdev_top->vdev_parent == rvd); 7190 7191 /* 7192 * Extract the new device from its root and add it to pvd. 7193 */ 7194 vdev_remove_child(newrootvd, newvd); 7195 newvd->vdev_id = pvd->vdev_children; 7196 newvd->vdev_crtxg = oldvd->vdev_crtxg; 7197 vdev_add_child(pvd, newvd); 7198 7199 /* 7200 * Reevaluate the parent vdev state. 7201 */ 7202 vdev_propagate_state(pvd); 7203 7204 tvd = newvd->vdev_top; 7205 ASSERT(pvd->vdev_top == tvd); 7206 ASSERT(tvd->vdev_parent == rvd); 7207 7208 vdev_config_dirty(tvd); 7209 7210 /* 7211 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 7212 * for any dmu_sync-ed blocks. It will propagate upward when 7213 * spa_vdev_exit() calls vdev_dtl_reassess(). 7214 */ 7215 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 7216 7217 if (raidz) { 7218 /* 7219 * Wait for the youngest allocations and frees to sync, 7220 * and then wait for the deferral of those frees to finish. 7221 */ 7222 spa_vdev_config_exit(spa, NULL, 7223 txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG); 7224 7225 vdev_initialize_stop_all(tvd, VDEV_INITIALIZE_ACTIVE); 7226 vdev_trim_stop_all(tvd, VDEV_TRIM_ACTIVE); 7227 vdev_autotrim_stop_wait(tvd); 7228 7229 dtl_max_txg = spa_vdev_config_enter(spa); 7230 7231 tvd->vdev_rz_expanding = B_TRUE; 7232 7233 vdev_dirty_leaves(tvd, VDD_DTL, dtl_max_txg); 7234 vdev_config_dirty(tvd); 7235 7236 dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool, 7237 dtl_max_txg); 7238 dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_raidz_attach_sync, 7239 newvd, tx); 7240 dmu_tx_commit(tx); 7241 } else { 7242 vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL, 7243 dtl_max_txg - TXG_INITIAL); 7244 7245 if (newvd->vdev_isspare) { 7246 spa_spare_activate(newvd); 7247 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 7248 } 7249 7250 newvd_isspare = newvd->vdev_isspare; 7251 7252 /* 7253 * Mark newvd's DTL dirty in this txg. 7254 */ 7255 vdev_dirty(tvd, VDD_DTL, newvd, txg); 7256 7257 /* 7258 * Schedule the resilver or rebuild to restart in the future. 7259 * We do this to ensure that dmu_sync-ed blocks have been 7260 * stitched into the respective datasets. 7261 */ 7262 if (rebuild) { 7263 newvd->vdev_rebuild_txg = txg; 7264 7265 vdev_rebuild(tvd); 7266 } else { 7267 newvd->vdev_resilver_txg = txg; 7268 7269 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 7270 spa_feature_is_enabled(spa, 7271 SPA_FEATURE_RESILVER_DEFER)) { 7272 vdev_defer_resilver(newvd); 7273 } else { 7274 dsl_scan_restart_resilver(spa->spa_dsl_pool, 7275 dtl_max_txg); 7276 } 7277 } 7278 } 7279 7280 if (spa->spa_bootfs) 7281 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 7282 7283 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 7284 7285 /* 7286 * Commit the config 7287 */ 7288 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 7289 7290 spa_history_log_internal(spa, "vdev attach", NULL, 7291 "%s vdev=%s %s vdev=%s", 7292 replacing && newvd_isspare ? "spare in" : 7293 replacing ? "replace" : "attach", newvdpath, 7294 replacing ? "for" : "to", oldvdpath); 7295 7296 spa_strfree(oldvdpath); 7297 spa_strfree(newvdpath); 7298 7299 return (0); 7300 } 7301 7302 /* 7303 * Detach a device from a mirror or replacing vdev. 7304 * 7305 * If 'replace_done' is specified, only detach if the parent 7306 * is a replacing or a spare vdev. 7307 */ 7308 int 7309 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 7310 { 7311 uint64_t txg; 7312 int error; 7313 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 7314 vdev_t *vd, *pvd, *cvd, *tvd; 7315 boolean_t unspare = B_FALSE; 7316 uint64_t unspare_guid = 0; 7317 char *vdpath; 7318 7319 ASSERT(spa_writeable(spa)); 7320 7321 txg = spa_vdev_detach_enter(spa, guid); 7322 7323 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7324 7325 /* 7326 * Besides being called directly from the userland through the 7327 * ioctl interface, spa_vdev_detach() can be potentially called 7328 * at the end of spa_vdev_resilver_done(). 7329 * 7330 * In the regular case, when we have a checkpoint this shouldn't 7331 * happen as we never empty the DTLs of a vdev during the scrub 7332 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 7333 * should never get here when we have a checkpoint. 7334 * 7335 * That said, even in a case when we checkpoint the pool exactly 7336 * as spa_vdev_resilver_done() calls this function everything 7337 * should be fine as the resilver will return right away. 7338 */ 7339 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7340 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7341 error = (spa_has_checkpoint(spa)) ? 7342 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7343 return (spa_vdev_exit(spa, NULL, txg, error)); 7344 } 7345 7346 if (vd == NULL) 7347 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 7348 7349 if (!vd->vdev_ops->vdev_op_leaf) 7350 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7351 7352 pvd = vd->vdev_parent; 7353 7354 /* 7355 * If the parent/child relationship is not as expected, don't do it. 7356 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 7357 * vdev that's replacing B with C. The user's intent in replacing 7358 * is to go from M(A,B) to M(A,C). If the user decides to cancel 7359 * the replace by detaching C, the expected behavior is to end up 7360 * M(A,B). But suppose that right after deciding to detach C, 7361 * the replacement of B completes. We would have M(A,C), and then 7362 * ask to detach C, which would leave us with just A -- not what 7363 * the user wanted. To prevent this, we make sure that the 7364 * parent/child relationship hasn't changed -- in this example, 7365 * that C's parent is still the replacing vdev R. 7366 */ 7367 if (pvd->vdev_guid != pguid && pguid != 0) 7368 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7369 7370 /* 7371 * Only 'replacing' or 'spare' vdevs can be replaced. 7372 */ 7373 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 7374 pvd->vdev_ops != &vdev_spare_ops) 7375 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7376 7377 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 7378 spa_version(spa) >= SPA_VERSION_SPARES); 7379 7380 /* 7381 * Only mirror, replacing, and spare vdevs support detach. 7382 */ 7383 if (pvd->vdev_ops != &vdev_replacing_ops && 7384 pvd->vdev_ops != &vdev_mirror_ops && 7385 pvd->vdev_ops != &vdev_spare_ops) 7386 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7387 7388 /* 7389 * If this device has the only valid copy of some data, 7390 * we cannot safely detach it. 7391 */ 7392 if (vdev_dtl_required(vd)) 7393 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7394 7395 ASSERT(pvd->vdev_children >= 2); 7396 7397 /* 7398 * If we are detaching the second disk from a replacing vdev, then 7399 * check to see if we changed the original vdev's path to have "/old" 7400 * at the end in spa_vdev_attach(). If so, undo that change now. 7401 */ 7402 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 7403 vd->vdev_path != NULL) { 7404 size_t len = strlen(vd->vdev_path); 7405 7406 for (int c = 0; c < pvd->vdev_children; c++) { 7407 cvd = pvd->vdev_child[c]; 7408 7409 if (cvd == vd || cvd->vdev_path == NULL) 7410 continue; 7411 7412 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 7413 strcmp(cvd->vdev_path + len, "/old") == 0) { 7414 spa_strfree(cvd->vdev_path); 7415 cvd->vdev_path = spa_strdup(vd->vdev_path); 7416 break; 7417 } 7418 } 7419 } 7420 7421 /* 7422 * If we are detaching the original disk from a normal spare, then it 7423 * implies that the spare should become a real disk, and be removed 7424 * from the active spare list for the pool. dRAID spares on the 7425 * other hand are coupled to the pool and thus should never be removed 7426 * from the spares list. 7427 */ 7428 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 7429 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7430 7431 if (last_cvd->vdev_isspare && 7432 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 7433 unspare = B_TRUE; 7434 } 7435 } 7436 7437 /* 7438 * Erase the disk labels so the disk can be used for other things. 7439 * This must be done after all other error cases are handled, 7440 * but before we disembowel vd (so we can still do I/O to it). 7441 * But if we can't do it, don't treat the error as fatal -- 7442 * it may be that the unwritability of the disk is the reason 7443 * it's being detached! 7444 */ 7445 (void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 7446 7447 /* 7448 * Remove vd from its parent and compact the parent's children. 7449 */ 7450 vdev_remove_child(pvd, vd); 7451 vdev_compact_children(pvd); 7452 7453 /* 7454 * Remember one of the remaining children so we can get tvd below. 7455 */ 7456 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7457 7458 /* 7459 * If we need to remove the remaining child from the list of hot spares, 7460 * do it now, marking the vdev as no longer a spare in the process. 7461 * We must do this before vdev_remove_parent(), because that can 7462 * change the GUID if it creates a new toplevel GUID. For a similar 7463 * reason, we must remove the spare now, in the same txg as the detach; 7464 * otherwise someone could attach a new sibling, change the GUID, and 7465 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 7466 */ 7467 if (unspare) { 7468 ASSERT(cvd->vdev_isspare); 7469 spa_spare_remove(cvd); 7470 unspare_guid = cvd->vdev_guid; 7471 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 7472 cvd->vdev_unspare = B_TRUE; 7473 } 7474 7475 /* 7476 * If the parent mirror/replacing vdev only has one child, 7477 * the parent is no longer needed. Remove it from the tree. 7478 */ 7479 if (pvd->vdev_children == 1) { 7480 if (pvd->vdev_ops == &vdev_spare_ops) 7481 cvd->vdev_unspare = B_FALSE; 7482 vdev_remove_parent(cvd); 7483 } 7484 7485 /* 7486 * We don't set tvd until now because the parent we just removed 7487 * may have been the previous top-level vdev. 7488 */ 7489 tvd = cvd->vdev_top; 7490 ASSERT(tvd->vdev_parent == rvd); 7491 7492 /* 7493 * Reevaluate the parent vdev state. 7494 */ 7495 vdev_propagate_state(cvd); 7496 7497 /* 7498 * If the 'autoexpand' property is set on the pool then automatically 7499 * try to expand the size of the pool. For example if the device we 7500 * just detached was smaller than the others, it may be possible to 7501 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 7502 * first so that we can obtain the updated sizes of the leaf vdevs. 7503 */ 7504 if (spa->spa_autoexpand) { 7505 vdev_reopen(tvd); 7506 vdev_expand(tvd, txg); 7507 } 7508 7509 vdev_config_dirty(tvd); 7510 7511 /* 7512 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 7513 * vd->vdev_detached is set and free vd's DTL object in syncing context. 7514 * But first make sure we're not on any *other* txg's DTL list, to 7515 * prevent vd from being accessed after it's freed. 7516 */ 7517 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 7518 for (int t = 0; t < TXG_SIZE; t++) 7519 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 7520 vd->vdev_detached = B_TRUE; 7521 vdev_dirty(tvd, VDD_DTL, vd, txg); 7522 7523 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 7524 spa_notify_waiters(spa); 7525 7526 /* hang on to the spa before we release the lock */ 7527 spa_open_ref(spa, FTAG); 7528 7529 error = spa_vdev_exit(spa, vd, txg, 0); 7530 7531 spa_history_log_internal(spa, "detach", NULL, 7532 "vdev=%s", vdpath); 7533 spa_strfree(vdpath); 7534 7535 /* 7536 * If this was the removal of the original device in a hot spare vdev, 7537 * then we want to go through and remove the device from the hot spare 7538 * list of every other pool. 7539 */ 7540 if (unspare) { 7541 spa_t *altspa = NULL; 7542 7543 mutex_enter(&spa_namespace_lock); 7544 while ((altspa = spa_next(altspa)) != NULL) { 7545 if (altspa->spa_state != POOL_STATE_ACTIVE || 7546 altspa == spa) 7547 continue; 7548 7549 spa_open_ref(altspa, FTAG); 7550 mutex_exit(&spa_namespace_lock); 7551 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 7552 mutex_enter(&spa_namespace_lock); 7553 spa_close(altspa, FTAG); 7554 } 7555 mutex_exit(&spa_namespace_lock); 7556 7557 /* search the rest of the vdevs for spares to remove */ 7558 spa_vdev_resilver_done(spa); 7559 } 7560 7561 /* all done with the spa; OK to release */ 7562 mutex_enter(&spa_namespace_lock); 7563 spa_close(spa, FTAG); 7564 mutex_exit(&spa_namespace_lock); 7565 7566 return (error); 7567 } 7568 7569 static int 7570 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7571 list_t *vd_list) 7572 { 7573 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7574 7575 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7576 7577 /* Look up vdev and ensure it's a leaf. */ 7578 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7579 if (vd == NULL || vd->vdev_detached) { 7580 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7581 return (SET_ERROR(ENODEV)); 7582 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7583 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7584 return (SET_ERROR(EINVAL)); 7585 } else if (!vdev_writeable(vd)) { 7586 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7587 return (SET_ERROR(EROFS)); 7588 } 7589 mutex_enter(&vd->vdev_initialize_lock); 7590 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7591 7592 /* 7593 * When we activate an initialize action we check to see 7594 * if the vdev_initialize_thread is NULL. We do this instead 7595 * of using the vdev_initialize_state since there might be 7596 * a previous initialization process which has completed but 7597 * the thread is not exited. 7598 */ 7599 if (cmd_type == POOL_INITIALIZE_START && 7600 (vd->vdev_initialize_thread != NULL || 7601 vd->vdev_top->vdev_removing || vd->vdev_top->vdev_rz_expanding)) { 7602 mutex_exit(&vd->vdev_initialize_lock); 7603 return (SET_ERROR(EBUSY)); 7604 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 7605 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 7606 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 7607 mutex_exit(&vd->vdev_initialize_lock); 7608 return (SET_ERROR(ESRCH)); 7609 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 7610 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 7611 mutex_exit(&vd->vdev_initialize_lock); 7612 return (SET_ERROR(ESRCH)); 7613 } else if (cmd_type == POOL_INITIALIZE_UNINIT && 7614 vd->vdev_initialize_thread != NULL) { 7615 mutex_exit(&vd->vdev_initialize_lock); 7616 return (SET_ERROR(EBUSY)); 7617 } 7618 7619 switch (cmd_type) { 7620 case POOL_INITIALIZE_START: 7621 vdev_initialize(vd); 7622 break; 7623 case POOL_INITIALIZE_CANCEL: 7624 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 7625 break; 7626 case POOL_INITIALIZE_SUSPEND: 7627 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 7628 break; 7629 case POOL_INITIALIZE_UNINIT: 7630 vdev_uninitialize(vd); 7631 break; 7632 default: 7633 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7634 } 7635 mutex_exit(&vd->vdev_initialize_lock); 7636 7637 return (0); 7638 } 7639 7640 int 7641 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 7642 nvlist_t *vdev_errlist) 7643 { 7644 int total_errors = 0; 7645 list_t vd_list; 7646 7647 list_create(&vd_list, sizeof (vdev_t), 7648 offsetof(vdev_t, vdev_initialize_node)); 7649 7650 /* 7651 * We hold the namespace lock through the whole function 7652 * to prevent any changes to the pool while we're starting or 7653 * stopping initialization. The config and state locks are held so that 7654 * we can properly assess the vdev state before we commit to 7655 * the initializing operation. 7656 */ 7657 mutex_enter(&spa_namespace_lock); 7658 7659 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7660 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7661 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7662 7663 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 7664 &vd_list); 7665 if (error != 0) { 7666 char guid_as_str[MAXNAMELEN]; 7667 7668 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7669 "%llu", (unsigned long long)vdev_guid); 7670 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7671 total_errors++; 7672 } 7673 } 7674 7675 /* Wait for all initialize threads to stop. */ 7676 vdev_initialize_stop_wait(spa, &vd_list); 7677 7678 /* Sync out the initializing state */ 7679 txg_wait_synced(spa->spa_dsl_pool, 0); 7680 mutex_exit(&spa_namespace_lock); 7681 7682 list_destroy(&vd_list); 7683 7684 return (total_errors); 7685 } 7686 7687 static int 7688 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7689 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 7690 { 7691 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7692 7693 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7694 7695 /* Look up vdev and ensure it's a leaf. */ 7696 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7697 if (vd == NULL || vd->vdev_detached) { 7698 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7699 return (SET_ERROR(ENODEV)); 7700 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7701 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7702 return (SET_ERROR(EINVAL)); 7703 } else if (!vdev_writeable(vd)) { 7704 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7705 return (SET_ERROR(EROFS)); 7706 } else if (!vd->vdev_has_trim) { 7707 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7708 return (SET_ERROR(EOPNOTSUPP)); 7709 } else if (secure && !vd->vdev_has_securetrim) { 7710 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7711 return (SET_ERROR(EOPNOTSUPP)); 7712 } 7713 mutex_enter(&vd->vdev_trim_lock); 7714 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7715 7716 /* 7717 * When we activate a TRIM action we check to see if the 7718 * vdev_trim_thread is NULL. We do this instead of using the 7719 * vdev_trim_state since there might be a previous TRIM process 7720 * which has completed but the thread is not exited. 7721 */ 7722 if (cmd_type == POOL_TRIM_START && 7723 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing || 7724 vd->vdev_top->vdev_rz_expanding)) { 7725 mutex_exit(&vd->vdev_trim_lock); 7726 return (SET_ERROR(EBUSY)); 7727 } else if (cmd_type == POOL_TRIM_CANCEL && 7728 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 7729 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 7730 mutex_exit(&vd->vdev_trim_lock); 7731 return (SET_ERROR(ESRCH)); 7732 } else if (cmd_type == POOL_TRIM_SUSPEND && 7733 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 7734 mutex_exit(&vd->vdev_trim_lock); 7735 return (SET_ERROR(ESRCH)); 7736 } 7737 7738 switch (cmd_type) { 7739 case POOL_TRIM_START: 7740 vdev_trim(vd, rate, partial, secure); 7741 break; 7742 case POOL_TRIM_CANCEL: 7743 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 7744 break; 7745 case POOL_TRIM_SUSPEND: 7746 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 7747 break; 7748 default: 7749 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7750 } 7751 mutex_exit(&vd->vdev_trim_lock); 7752 7753 return (0); 7754 } 7755 7756 /* 7757 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 7758 * TRIM threads for each child vdev. These threads pass over all of the free 7759 * space in the vdev's metaslabs and issues TRIM commands for that space. 7760 */ 7761 int 7762 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 7763 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 7764 { 7765 int total_errors = 0; 7766 list_t vd_list; 7767 7768 list_create(&vd_list, sizeof (vdev_t), 7769 offsetof(vdev_t, vdev_trim_node)); 7770 7771 /* 7772 * We hold the namespace lock through the whole function 7773 * to prevent any changes to the pool while we're starting or 7774 * stopping TRIM. The config and state locks are held so that 7775 * we can properly assess the vdev state before we commit to 7776 * the TRIM operation. 7777 */ 7778 mutex_enter(&spa_namespace_lock); 7779 7780 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7781 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7782 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7783 7784 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 7785 rate, partial, secure, &vd_list); 7786 if (error != 0) { 7787 char guid_as_str[MAXNAMELEN]; 7788 7789 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7790 "%llu", (unsigned long long)vdev_guid); 7791 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7792 total_errors++; 7793 } 7794 } 7795 7796 /* Wait for all TRIM threads to stop. */ 7797 vdev_trim_stop_wait(spa, &vd_list); 7798 7799 /* Sync out the TRIM state */ 7800 txg_wait_synced(spa->spa_dsl_pool, 0); 7801 mutex_exit(&spa_namespace_lock); 7802 7803 list_destroy(&vd_list); 7804 7805 return (total_errors); 7806 } 7807 7808 /* 7809 * Split a set of devices from their mirrors, and create a new pool from them. 7810 */ 7811 int 7812 spa_vdev_split_mirror(spa_t *spa, const char *newname, nvlist_t *config, 7813 nvlist_t *props, boolean_t exp) 7814 { 7815 int error = 0; 7816 uint64_t txg, *glist; 7817 spa_t *newspa; 7818 uint_t c, children, lastlog; 7819 nvlist_t **child, *nvl, *tmp; 7820 dmu_tx_t *tx; 7821 const char *altroot = NULL; 7822 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 7823 boolean_t activate_slog; 7824 7825 ASSERT(spa_writeable(spa)); 7826 7827 txg = spa_vdev_enter(spa); 7828 7829 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7830 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7831 error = (spa_has_checkpoint(spa)) ? 7832 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7833 return (spa_vdev_exit(spa, NULL, txg, error)); 7834 } 7835 7836 /* clear the log and flush everything up to now */ 7837 activate_slog = spa_passivate_log(spa); 7838 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7839 error = spa_reset_logs(spa); 7840 txg = spa_vdev_config_enter(spa); 7841 7842 if (activate_slog) 7843 spa_activate_log(spa); 7844 7845 if (error != 0) 7846 return (spa_vdev_exit(spa, NULL, txg, error)); 7847 7848 /* check new spa name before going any further */ 7849 if (spa_lookup(newname) != NULL) 7850 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 7851 7852 /* 7853 * scan through all the children to ensure they're all mirrors 7854 */ 7855 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 7856 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 7857 &children) != 0) 7858 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7859 7860 /* first, check to ensure we've got the right child count */ 7861 rvd = spa->spa_root_vdev; 7862 lastlog = 0; 7863 for (c = 0; c < rvd->vdev_children; c++) { 7864 vdev_t *vd = rvd->vdev_child[c]; 7865 7866 /* don't count the holes & logs as children */ 7867 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 7868 !vdev_is_concrete(vd))) { 7869 if (lastlog == 0) 7870 lastlog = c; 7871 continue; 7872 } 7873 7874 lastlog = 0; 7875 } 7876 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 7877 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7878 7879 /* next, ensure no spare or cache devices are part of the split */ 7880 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 7881 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 7882 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7883 7884 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 7885 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 7886 7887 /* then, loop over each vdev and validate it */ 7888 for (c = 0; c < children; c++) { 7889 uint64_t is_hole = 0; 7890 7891 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 7892 &is_hole); 7893 7894 if (is_hole != 0) { 7895 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 7896 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 7897 continue; 7898 } else { 7899 error = SET_ERROR(EINVAL); 7900 break; 7901 } 7902 } 7903 7904 /* deal with indirect vdevs */ 7905 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 7906 &vdev_indirect_ops) 7907 continue; 7908 7909 /* which disk is going to be split? */ 7910 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 7911 &glist[c]) != 0) { 7912 error = SET_ERROR(EINVAL); 7913 break; 7914 } 7915 7916 /* look it up in the spa */ 7917 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 7918 if (vml[c] == NULL) { 7919 error = SET_ERROR(ENODEV); 7920 break; 7921 } 7922 7923 /* make sure there's nothing stopping the split */ 7924 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 7925 vml[c]->vdev_islog || 7926 !vdev_is_concrete(vml[c]) || 7927 vml[c]->vdev_isspare || 7928 vml[c]->vdev_isl2cache || 7929 !vdev_writeable(vml[c]) || 7930 vml[c]->vdev_children != 0 || 7931 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 7932 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 7933 error = SET_ERROR(EINVAL); 7934 break; 7935 } 7936 7937 if (vdev_dtl_required(vml[c]) || 7938 vdev_resilver_needed(vml[c], NULL, NULL)) { 7939 error = SET_ERROR(EBUSY); 7940 break; 7941 } 7942 7943 /* we need certain info from the top level */ 7944 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 7945 vml[c]->vdev_top->vdev_ms_array); 7946 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 7947 vml[c]->vdev_top->vdev_ms_shift); 7948 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 7949 vml[c]->vdev_top->vdev_asize); 7950 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 7951 vml[c]->vdev_top->vdev_ashift); 7952 7953 /* transfer per-vdev ZAPs */ 7954 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 7955 VERIFY0(nvlist_add_uint64(child[c], 7956 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 7957 7958 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 7959 VERIFY0(nvlist_add_uint64(child[c], 7960 ZPOOL_CONFIG_VDEV_TOP_ZAP, 7961 vml[c]->vdev_parent->vdev_top_zap)); 7962 } 7963 7964 if (error != 0) { 7965 kmem_free(vml, children * sizeof (vdev_t *)); 7966 kmem_free(glist, children * sizeof (uint64_t)); 7967 return (spa_vdev_exit(spa, NULL, txg, error)); 7968 } 7969 7970 /* stop writers from using the disks */ 7971 for (c = 0; c < children; c++) { 7972 if (vml[c] != NULL) 7973 vml[c]->vdev_offline = B_TRUE; 7974 } 7975 vdev_reopen(spa->spa_root_vdev); 7976 7977 /* 7978 * Temporarily record the splitting vdevs in the spa config. This 7979 * will disappear once the config is regenerated. 7980 */ 7981 nvl = fnvlist_alloc(); 7982 fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children); 7983 kmem_free(glist, children * sizeof (uint64_t)); 7984 7985 mutex_enter(&spa->spa_props_lock); 7986 fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl); 7987 mutex_exit(&spa->spa_props_lock); 7988 spa->spa_config_splitting = nvl; 7989 vdev_config_dirty(spa->spa_root_vdev); 7990 7991 /* configure and create the new pool */ 7992 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname); 7993 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 7994 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE); 7995 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)); 7996 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg); 7997 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 7998 spa_generate_guid(NULL)); 7999 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 8000 (void) nvlist_lookup_string(props, 8001 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 8002 8003 /* add the new pool to the namespace */ 8004 newspa = spa_add(newname, config, altroot); 8005 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 8006 newspa->spa_config_txg = spa->spa_config_txg; 8007 spa_set_log_state(newspa, SPA_LOG_CLEAR); 8008 8009 /* release the spa config lock, retaining the namespace lock */ 8010 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 8011 8012 if (zio_injection_enabled) 8013 zio_handle_panic_injection(spa, FTAG, 1); 8014 8015 spa_activate(newspa, spa_mode_global); 8016 spa_async_suspend(newspa); 8017 8018 /* 8019 * Temporarily stop the initializing and TRIM activity. We set the 8020 * state to ACTIVE so that we know to resume initializing or TRIM 8021 * once the split has completed. 8022 */ 8023 list_t vd_initialize_list; 8024 list_create(&vd_initialize_list, sizeof (vdev_t), 8025 offsetof(vdev_t, vdev_initialize_node)); 8026 8027 list_t vd_trim_list; 8028 list_create(&vd_trim_list, sizeof (vdev_t), 8029 offsetof(vdev_t, vdev_trim_node)); 8030 8031 for (c = 0; c < children; c++) { 8032 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 8033 mutex_enter(&vml[c]->vdev_initialize_lock); 8034 vdev_initialize_stop(vml[c], 8035 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 8036 mutex_exit(&vml[c]->vdev_initialize_lock); 8037 8038 mutex_enter(&vml[c]->vdev_trim_lock); 8039 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 8040 mutex_exit(&vml[c]->vdev_trim_lock); 8041 } 8042 } 8043 8044 vdev_initialize_stop_wait(spa, &vd_initialize_list); 8045 vdev_trim_stop_wait(spa, &vd_trim_list); 8046 8047 list_destroy(&vd_initialize_list); 8048 list_destroy(&vd_trim_list); 8049 8050 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 8051 newspa->spa_is_splitting = B_TRUE; 8052 8053 /* create the new pool from the disks of the original pool */ 8054 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 8055 if (error) 8056 goto out; 8057 8058 /* if that worked, generate a real config for the new pool */ 8059 if (newspa->spa_root_vdev != NULL) { 8060 newspa->spa_config_splitting = fnvlist_alloc(); 8061 fnvlist_add_uint64(newspa->spa_config_splitting, 8062 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)); 8063 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 8064 B_TRUE)); 8065 } 8066 8067 /* set the props */ 8068 if (props != NULL) { 8069 spa_configfile_set(newspa, props, B_FALSE); 8070 error = spa_prop_set(newspa, props); 8071 if (error) 8072 goto out; 8073 } 8074 8075 /* flush everything */ 8076 txg = spa_vdev_config_enter(newspa); 8077 vdev_config_dirty(newspa->spa_root_vdev); 8078 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 8079 8080 if (zio_injection_enabled) 8081 zio_handle_panic_injection(spa, FTAG, 2); 8082 8083 spa_async_resume(newspa); 8084 8085 /* finally, update the original pool's config */ 8086 txg = spa_vdev_config_enter(spa); 8087 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 8088 error = dmu_tx_assign(tx, TXG_WAIT); 8089 if (error != 0) 8090 dmu_tx_abort(tx); 8091 for (c = 0; c < children; c++) { 8092 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 8093 vdev_t *tvd = vml[c]->vdev_top; 8094 8095 /* 8096 * Need to be sure the detachable VDEV is not 8097 * on any *other* txg's DTL list to prevent it 8098 * from being accessed after it's freed. 8099 */ 8100 for (int t = 0; t < TXG_SIZE; t++) { 8101 (void) txg_list_remove_this( 8102 &tvd->vdev_dtl_list, vml[c], t); 8103 } 8104 8105 vdev_split(vml[c]); 8106 if (error == 0) 8107 spa_history_log_internal(spa, "detach", tx, 8108 "vdev=%s", vml[c]->vdev_path); 8109 8110 vdev_free(vml[c]); 8111 } 8112 } 8113 spa->spa_avz_action = AVZ_ACTION_REBUILD; 8114 vdev_config_dirty(spa->spa_root_vdev); 8115 spa->spa_config_splitting = NULL; 8116 nvlist_free(nvl); 8117 if (error == 0) 8118 dmu_tx_commit(tx); 8119 (void) spa_vdev_exit(spa, NULL, txg, 0); 8120 8121 if (zio_injection_enabled) 8122 zio_handle_panic_injection(spa, FTAG, 3); 8123 8124 /* split is complete; log a history record */ 8125 spa_history_log_internal(newspa, "split", NULL, 8126 "from pool %s", spa_name(spa)); 8127 8128 newspa->spa_is_splitting = B_FALSE; 8129 kmem_free(vml, children * sizeof (vdev_t *)); 8130 8131 /* if we're not going to mount the filesystems in userland, export */ 8132 if (exp) 8133 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 8134 B_FALSE, B_FALSE); 8135 8136 return (error); 8137 8138 out: 8139 spa_unload(newspa); 8140 spa_deactivate(newspa); 8141 spa_remove(newspa); 8142 8143 txg = spa_vdev_config_enter(spa); 8144 8145 /* re-online all offlined disks */ 8146 for (c = 0; c < children; c++) { 8147 if (vml[c] != NULL) 8148 vml[c]->vdev_offline = B_FALSE; 8149 } 8150 8151 /* restart initializing or trimming disks as necessary */ 8152 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 8153 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 8154 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 8155 8156 vdev_reopen(spa->spa_root_vdev); 8157 8158 nvlist_free(spa->spa_config_splitting); 8159 spa->spa_config_splitting = NULL; 8160 (void) spa_vdev_exit(spa, NULL, txg, error); 8161 8162 kmem_free(vml, children * sizeof (vdev_t *)); 8163 return (error); 8164 } 8165 8166 /* 8167 * Find any device that's done replacing, or a vdev marked 'unspare' that's 8168 * currently spared, so we can detach it. 8169 */ 8170 static vdev_t * 8171 spa_vdev_resilver_done_hunt(vdev_t *vd) 8172 { 8173 vdev_t *newvd, *oldvd; 8174 8175 for (int c = 0; c < vd->vdev_children; c++) { 8176 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 8177 if (oldvd != NULL) 8178 return (oldvd); 8179 } 8180 8181 /* 8182 * Check for a completed replacement. We always consider the first 8183 * vdev in the list to be the oldest vdev, and the last one to be 8184 * the newest (see spa_vdev_attach() for how that works). In 8185 * the case where the newest vdev is faulted, we will not automatically 8186 * remove it after a resilver completes. This is OK as it will require 8187 * user intervention to determine which disk the admin wishes to keep. 8188 */ 8189 if (vd->vdev_ops == &vdev_replacing_ops) { 8190 ASSERT(vd->vdev_children > 1); 8191 8192 newvd = vd->vdev_child[vd->vdev_children - 1]; 8193 oldvd = vd->vdev_child[0]; 8194 8195 if (vdev_dtl_empty(newvd, DTL_MISSING) && 8196 vdev_dtl_empty(newvd, DTL_OUTAGE) && 8197 !vdev_dtl_required(oldvd)) 8198 return (oldvd); 8199 } 8200 8201 /* 8202 * Check for a completed resilver with the 'unspare' flag set. 8203 * Also potentially update faulted state. 8204 */ 8205 if (vd->vdev_ops == &vdev_spare_ops) { 8206 vdev_t *first = vd->vdev_child[0]; 8207 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 8208 8209 if (last->vdev_unspare) { 8210 oldvd = first; 8211 newvd = last; 8212 } else if (first->vdev_unspare) { 8213 oldvd = last; 8214 newvd = first; 8215 } else { 8216 oldvd = NULL; 8217 } 8218 8219 if (oldvd != NULL && 8220 vdev_dtl_empty(newvd, DTL_MISSING) && 8221 vdev_dtl_empty(newvd, DTL_OUTAGE) && 8222 !vdev_dtl_required(oldvd)) 8223 return (oldvd); 8224 8225 vdev_propagate_state(vd); 8226 8227 /* 8228 * If there are more than two spares attached to a disk, 8229 * and those spares are not required, then we want to 8230 * attempt to free them up now so that they can be used 8231 * by other pools. Once we're back down to a single 8232 * disk+spare, we stop removing them. 8233 */ 8234 if (vd->vdev_children > 2) { 8235 newvd = vd->vdev_child[1]; 8236 8237 if (newvd->vdev_isspare && last->vdev_isspare && 8238 vdev_dtl_empty(last, DTL_MISSING) && 8239 vdev_dtl_empty(last, DTL_OUTAGE) && 8240 !vdev_dtl_required(newvd)) 8241 return (newvd); 8242 } 8243 } 8244 8245 return (NULL); 8246 } 8247 8248 static void 8249 spa_vdev_resilver_done(spa_t *spa) 8250 { 8251 vdev_t *vd, *pvd, *ppvd; 8252 uint64_t guid, sguid, pguid, ppguid; 8253 8254 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 8255 8256 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 8257 pvd = vd->vdev_parent; 8258 ppvd = pvd->vdev_parent; 8259 guid = vd->vdev_guid; 8260 pguid = pvd->vdev_guid; 8261 ppguid = ppvd->vdev_guid; 8262 sguid = 0; 8263 /* 8264 * If we have just finished replacing a hot spared device, then 8265 * we need to detach the parent's first child (the original hot 8266 * spare) as well. 8267 */ 8268 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 8269 ppvd->vdev_children == 2) { 8270 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 8271 sguid = ppvd->vdev_child[1]->vdev_guid; 8272 } 8273 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 8274 8275 spa_config_exit(spa, SCL_ALL, FTAG); 8276 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 8277 return; 8278 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 8279 return; 8280 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 8281 } 8282 8283 spa_config_exit(spa, SCL_ALL, FTAG); 8284 8285 /* 8286 * If a detach was not performed above replace waiters will not have 8287 * been notified. In which case we must do so now. 8288 */ 8289 spa_notify_waiters(spa); 8290 } 8291 8292 /* 8293 * Update the stored path or FRU for this vdev. 8294 */ 8295 static int 8296 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 8297 boolean_t ispath) 8298 { 8299 vdev_t *vd; 8300 boolean_t sync = B_FALSE; 8301 8302 ASSERT(spa_writeable(spa)); 8303 8304 spa_vdev_state_enter(spa, SCL_ALL); 8305 8306 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 8307 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 8308 8309 if (!vd->vdev_ops->vdev_op_leaf) 8310 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 8311 8312 if (ispath) { 8313 if (strcmp(value, vd->vdev_path) != 0) { 8314 spa_strfree(vd->vdev_path); 8315 vd->vdev_path = spa_strdup(value); 8316 sync = B_TRUE; 8317 } 8318 } else { 8319 if (vd->vdev_fru == NULL) { 8320 vd->vdev_fru = spa_strdup(value); 8321 sync = B_TRUE; 8322 } else if (strcmp(value, vd->vdev_fru) != 0) { 8323 spa_strfree(vd->vdev_fru); 8324 vd->vdev_fru = spa_strdup(value); 8325 sync = B_TRUE; 8326 } 8327 } 8328 8329 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 8330 } 8331 8332 int 8333 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 8334 { 8335 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 8336 } 8337 8338 int 8339 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 8340 { 8341 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 8342 } 8343 8344 /* 8345 * ========================================================================== 8346 * SPA Scanning 8347 * ========================================================================== 8348 */ 8349 int 8350 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 8351 { 8352 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8353 8354 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8355 return (SET_ERROR(EBUSY)); 8356 8357 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 8358 } 8359 8360 int 8361 spa_scan_stop(spa_t *spa) 8362 { 8363 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8364 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 8365 return (SET_ERROR(EBUSY)); 8366 8367 return (dsl_scan_cancel(spa->spa_dsl_pool)); 8368 } 8369 8370 int 8371 spa_scan(spa_t *spa, pool_scan_func_t func) 8372 { 8373 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8374 8375 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 8376 return (SET_ERROR(ENOTSUP)); 8377 8378 if (func == POOL_SCAN_RESILVER && 8379 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 8380 return (SET_ERROR(ENOTSUP)); 8381 8382 /* 8383 * If a resilver was requested, but there is no DTL on a 8384 * writeable leaf device, we have nothing to do. 8385 */ 8386 if (func == POOL_SCAN_RESILVER && 8387 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 8388 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 8389 return (0); 8390 } 8391 8392 if (func == POOL_SCAN_ERRORSCRUB && 8393 !spa_feature_is_enabled(spa, SPA_FEATURE_HEAD_ERRLOG)) 8394 return (SET_ERROR(ENOTSUP)); 8395 8396 return (dsl_scan(spa->spa_dsl_pool, func)); 8397 } 8398 8399 /* 8400 * ========================================================================== 8401 * SPA async task processing 8402 * ========================================================================== 8403 */ 8404 8405 static void 8406 spa_async_remove(spa_t *spa, vdev_t *vd) 8407 { 8408 if (vd->vdev_remove_wanted) { 8409 vd->vdev_remove_wanted = B_FALSE; 8410 vd->vdev_delayed_close = B_FALSE; 8411 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 8412 8413 /* 8414 * We want to clear the stats, but we don't want to do a full 8415 * vdev_clear() as that will cause us to throw away 8416 * degraded/faulted state as well as attempt to reopen the 8417 * device, all of which is a waste. 8418 */ 8419 vd->vdev_stat.vs_read_errors = 0; 8420 vd->vdev_stat.vs_write_errors = 0; 8421 vd->vdev_stat.vs_checksum_errors = 0; 8422 8423 vdev_state_dirty(vd->vdev_top); 8424 8425 /* Tell userspace that the vdev is gone. */ 8426 zfs_post_remove(spa, vd); 8427 } 8428 8429 for (int c = 0; c < vd->vdev_children; c++) 8430 spa_async_remove(spa, vd->vdev_child[c]); 8431 } 8432 8433 static void 8434 spa_async_probe(spa_t *spa, vdev_t *vd) 8435 { 8436 if (vd->vdev_probe_wanted) { 8437 vd->vdev_probe_wanted = B_FALSE; 8438 vdev_reopen(vd); /* vdev_open() does the actual probe */ 8439 } 8440 8441 for (int c = 0; c < vd->vdev_children; c++) 8442 spa_async_probe(spa, vd->vdev_child[c]); 8443 } 8444 8445 static void 8446 spa_async_autoexpand(spa_t *spa, vdev_t *vd) 8447 { 8448 if (!spa->spa_autoexpand) 8449 return; 8450 8451 for (int c = 0; c < vd->vdev_children; c++) { 8452 vdev_t *cvd = vd->vdev_child[c]; 8453 spa_async_autoexpand(spa, cvd); 8454 } 8455 8456 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 8457 return; 8458 8459 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 8460 } 8461 8462 static __attribute__((noreturn)) void 8463 spa_async_thread(void *arg) 8464 { 8465 spa_t *spa = (spa_t *)arg; 8466 dsl_pool_t *dp = spa->spa_dsl_pool; 8467 int tasks; 8468 8469 ASSERT(spa->spa_sync_on); 8470 8471 mutex_enter(&spa->spa_async_lock); 8472 tasks = spa->spa_async_tasks; 8473 spa->spa_async_tasks = 0; 8474 mutex_exit(&spa->spa_async_lock); 8475 8476 /* 8477 * See if the config needs to be updated. 8478 */ 8479 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 8480 uint64_t old_space, new_space; 8481 8482 mutex_enter(&spa_namespace_lock); 8483 old_space = metaslab_class_get_space(spa_normal_class(spa)); 8484 old_space += metaslab_class_get_space(spa_special_class(spa)); 8485 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 8486 old_space += metaslab_class_get_space( 8487 spa_embedded_log_class(spa)); 8488 8489 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 8490 8491 new_space = metaslab_class_get_space(spa_normal_class(spa)); 8492 new_space += metaslab_class_get_space(spa_special_class(spa)); 8493 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 8494 new_space += metaslab_class_get_space( 8495 spa_embedded_log_class(spa)); 8496 mutex_exit(&spa_namespace_lock); 8497 8498 /* 8499 * If the pool grew as a result of the config update, 8500 * then log an internal history event. 8501 */ 8502 if (new_space != old_space) { 8503 spa_history_log_internal(spa, "vdev online", NULL, 8504 "pool '%s' size: %llu(+%llu)", 8505 spa_name(spa), (u_longlong_t)new_space, 8506 (u_longlong_t)(new_space - old_space)); 8507 } 8508 } 8509 8510 /* 8511 * See if any devices need to be marked REMOVED. 8512 */ 8513 if (tasks & SPA_ASYNC_REMOVE) { 8514 spa_vdev_state_enter(spa, SCL_NONE); 8515 spa_async_remove(spa, spa->spa_root_vdev); 8516 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 8517 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); 8518 for (int i = 0; i < spa->spa_spares.sav_count; i++) 8519 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); 8520 (void) spa_vdev_state_exit(spa, NULL, 0); 8521 } 8522 8523 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 8524 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8525 spa_async_autoexpand(spa, spa->spa_root_vdev); 8526 spa_config_exit(spa, SCL_CONFIG, FTAG); 8527 } 8528 8529 /* 8530 * See if any devices need to be probed. 8531 */ 8532 if (tasks & SPA_ASYNC_PROBE) { 8533 spa_vdev_state_enter(spa, SCL_NONE); 8534 spa_async_probe(spa, spa->spa_root_vdev); 8535 (void) spa_vdev_state_exit(spa, NULL, 0); 8536 } 8537 8538 /* 8539 * If any devices are done replacing, detach them. 8540 */ 8541 if (tasks & SPA_ASYNC_RESILVER_DONE || 8542 tasks & SPA_ASYNC_REBUILD_DONE || 8543 tasks & SPA_ASYNC_DETACH_SPARE) { 8544 spa_vdev_resilver_done(spa); 8545 } 8546 8547 /* 8548 * Kick off a resilver. 8549 */ 8550 if (tasks & SPA_ASYNC_RESILVER && 8551 !vdev_rebuild_active(spa->spa_root_vdev) && 8552 (!dsl_scan_resilvering(dp) || 8553 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 8554 dsl_scan_restart_resilver(dp, 0); 8555 8556 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 8557 mutex_enter(&spa_namespace_lock); 8558 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8559 vdev_initialize_restart(spa->spa_root_vdev); 8560 spa_config_exit(spa, SCL_CONFIG, FTAG); 8561 mutex_exit(&spa_namespace_lock); 8562 } 8563 8564 if (tasks & SPA_ASYNC_TRIM_RESTART) { 8565 mutex_enter(&spa_namespace_lock); 8566 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8567 vdev_trim_restart(spa->spa_root_vdev); 8568 spa_config_exit(spa, SCL_CONFIG, FTAG); 8569 mutex_exit(&spa_namespace_lock); 8570 } 8571 8572 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 8573 mutex_enter(&spa_namespace_lock); 8574 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8575 vdev_autotrim_restart(spa); 8576 spa_config_exit(spa, SCL_CONFIG, FTAG); 8577 mutex_exit(&spa_namespace_lock); 8578 } 8579 8580 /* 8581 * Kick off L2 cache whole device TRIM. 8582 */ 8583 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 8584 mutex_enter(&spa_namespace_lock); 8585 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8586 vdev_trim_l2arc(spa); 8587 spa_config_exit(spa, SCL_CONFIG, FTAG); 8588 mutex_exit(&spa_namespace_lock); 8589 } 8590 8591 /* 8592 * Kick off L2 cache rebuilding. 8593 */ 8594 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 8595 mutex_enter(&spa_namespace_lock); 8596 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 8597 l2arc_spa_rebuild_start(spa); 8598 spa_config_exit(spa, SCL_L2ARC, FTAG); 8599 mutex_exit(&spa_namespace_lock); 8600 } 8601 8602 /* 8603 * Let the world know that we're done. 8604 */ 8605 mutex_enter(&spa->spa_async_lock); 8606 spa->spa_async_thread = NULL; 8607 cv_broadcast(&spa->spa_async_cv); 8608 mutex_exit(&spa->spa_async_lock); 8609 thread_exit(); 8610 } 8611 8612 void 8613 spa_async_suspend(spa_t *spa) 8614 { 8615 mutex_enter(&spa->spa_async_lock); 8616 spa->spa_async_suspended++; 8617 while (spa->spa_async_thread != NULL) 8618 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 8619 mutex_exit(&spa->spa_async_lock); 8620 8621 spa_vdev_remove_suspend(spa); 8622 8623 zthr_t *condense_thread = spa->spa_condense_zthr; 8624 if (condense_thread != NULL) 8625 zthr_cancel(condense_thread); 8626 8627 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr; 8628 if (raidz_expand_thread != NULL) 8629 zthr_cancel(raidz_expand_thread); 8630 8631 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8632 if (discard_thread != NULL) 8633 zthr_cancel(discard_thread); 8634 8635 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8636 if (ll_delete_thread != NULL) 8637 zthr_cancel(ll_delete_thread); 8638 8639 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8640 if (ll_condense_thread != NULL) 8641 zthr_cancel(ll_condense_thread); 8642 } 8643 8644 void 8645 spa_async_resume(spa_t *spa) 8646 { 8647 mutex_enter(&spa->spa_async_lock); 8648 ASSERT(spa->spa_async_suspended != 0); 8649 spa->spa_async_suspended--; 8650 mutex_exit(&spa->spa_async_lock); 8651 spa_restart_removal(spa); 8652 8653 zthr_t *condense_thread = spa->spa_condense_zthr; 8654 if (condense_thread != NULL) 8655 zthr_resume(condense_thread); 8656 8657 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr; 8658 if (raidz_expand_thread != NULL) 8659 zthr_resume(raidz_expand_thread); 8660 8661 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8662 if (discard_thread != NULL) 8663 zthr_resume(discard_thread); 8664 8665 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8666 if (ll_delete_thread != NULL) 8667 zthr_resume(ll_delete_thread); 8668 8669 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8670 if (ll_condense_thread != NULL) 8671 zthr_resume(ll_condense_thread); 8672 } 8673 8674 static boolean_t 8675 spa_async_tasks_pending(spa_t *spa) 8676 { 8677 uint_t non_config_tasks; 8678 uint_t config_task; 8679 boolean_t config_task_suspended; 8680 8681 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 8682 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 8683 if (spa->spa_ccw_fail_time == 0) { 8684 config_task_suspended = B_FALSE; 8685 } else { 8686 config_task_suspended = 8687 (gethrtime() - spa->spa_ccw_fail_time) < 8688 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 8689 } 8690 8691 return (non_config_tasks || (config_task && !config_task_suspended)); 8692 } 8693 8694 static void 8695 spa_async_dispatch(spa_t *spa) 8696 { 8697 mutex_enter(&spa->spa_async_lock); 8698 if (spa_async_tasks_pending(spa) && 8699 !spa->spa_async_suspended && 8700 spa->spa_async_thread == NULL) 8701 spa->spa_async_thread = thread_create(NULL, 0, 8702 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 8703 mutex_exit(&spa->spa_async_lock); 8704 } 8705 8706 void 8707 spa_async_request(spa_t *spa, int task) 8708 { 8709 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); 8710 mutex_enter(&spa->spa_async_lock); 8711 spa->spa_async_tasks |= task; 8712 mutex_exit(&spa->spa_async_lock); 8713 } 8714 8715 int 8716 spa_async_tasks(spa_t *spa) 8717 { 8718 return (spa->spa_async_tasks); 8719 } 8720 8721 /* 8722 * ========================================================================== 8723 * SPA syncing routines 8724 * ========================================================================== 8725 */ 8726 8727 8728 static int 8729 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8730 dmu_tx_t *tx) 8731 { 8732 bpobj_t *bpo = arg; 8733 bpobj_enqueue(bpo, bp, bp_freed, tx); 8734 return (0); 8735 } 8736 8737 int 8738 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8739 { 8740 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 8741 } 8742 8743 int 8744 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8745 { 8746 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 8747 } 8748 8749 static int 8750 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8751 { 8752 zio_t *pio = arg; 8753 8754 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 8755 pio->io_flags)); 8756 return (0); 8757 } 8758 8759 static int 8760 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8761 dmu_tx_t *tx) 8762 { 8763 ASSERT(!bp_freed); 8764 return (spa_free_sync_cb(arg, bp, tx)); 8765 } 8766 8767 /* 8768 * Note: this simple function is not inlined to make it easier to dtrace the 8769 * amount of time spent syncing frees. 8770 */ 8771 static void 8772 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 8773 { 8774 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8775 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 8776 VERIFY(zio_wait(zio) == 0); 8777 } 8778 8779 /* 8780 * Note: this simple function is not inlined to make it easier to dtrace the 8781 * amount of time spent syncing deferred frees. 8782 */ 8783 static void 8784 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 8785 { 8786 if (spa_sync_pass(spa) != 1) 8787 return; 8788 8789 /* 8790 * Note: 8791 * If the log space map feature is active, we stop deferring 8792 * frees to the next TXG and therefore running this function 8793 * would be considered a no-op as spa_deferred_bpobj should 8794 * not have any entries. 8795 * 8796 * That said we run this function anyway (instead of returning 8797 * immediately) for the edge-case scenario where we just 8798 * activated the log space map feature in this TXG but we have 8799 * deferred frees from the previous TXG. 8800 */ 8801 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8802 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 8803 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 8804 VERIFY0(zio_wait(zio)); 8805 } 8806 8807 static void 8808 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 8809 { 8810 char *packed = NULL; 8811 size_t bufsize; 8812 size_t nvsize = 0; 8813 dmu_buf_t *db; 8814 8815 VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); 8816 8817 /* 8818 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 8819 * information. This avoids the dmu_buf_will_dirty() path and 8820 * saves us a pre-read to get data we don't actually care about. 8821 */ 8822 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 8823 packed = vmem_alloc(bufsize, KM_SLEEP); 8824 8825 VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 8826 KM_SLEEP) == 0); 8827 memset(packed + nvsize, 0, bufsize - nvsize); 8828 8829 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); 8830 8831 vmem_free(packed, bufsize); 8832 8833 VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 8834 dmu_buf_will_dirty(db, tx); 8835 *(uint64_t *)db->db_data = nvsize; 8836 dmu_buf_rele(db, FTAG); 8837 } 8838 8839 static void 8840 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 8841 const char *config, const char *entry) 8842 { 8843 nvlist_t *nvroot; 8844 nvlist_t **list; 8845 int i; 8846 8847 if (!sav->sav_sync) 8848 return; 8849 8850 /* 8851 * Update the MOS nvlist describing the list of available devices. 8852 * spa_validate_aux() will have already made sure this nvlist is 8853 * valid and the vdevs are labeled appropriately. 8854 */ 8855 if (sav->sav_object == 0) { 8856 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 8857 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 8858 sizeof (uint64_t), tx); 8859 VERIFY(zap_update(spa->spa_meta_objset, 8860 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 8861 &sav->sav_object, tx) == 0); 8862 } 8863 8864 nvroot = fnvlist_alloc(); 8865 if (sav->sav_count == 0) { 8866 fnvlist_add_nvlist_array(nvroot, config, 8867 (const nvlist_t * const *)NULL, 0); 8868 } else { 8869 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 8870 for (i = 0; i < sav->sav_count; i++) 8871 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 8872 B_FALSE, VDEV_CONFIG_L2CACHE); 8873 fnvlist_add_nvlist_array(nvroot, config, 8874 (const nvlist_t * const *)list, sav->sav_count); 8875 for (i = 0; i < sav->sav_count; i++) 8876 nvlist_free(list[i]); 8877 kmem_free(list, sav->sav_count * sizeof (void *)); 8878 } 8879 8880 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 8881 nvlist_free(nvroot); 8882 8883 sav->sav_sync = B_FALSE; 8884 } 8885 8886 /* 8887 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 8888 * The all-vdev ZAP must be empty. 8889 */ 8890 static void 8891 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 8892 { 8893 spa_t *spa = vd->vdev_spa; 8894 8895 if (vd->vdev_root_zap != 0 && 8896 spa_feature_is_active(spa, SPA_FEATURE_AVZ_V2)) { 8897 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8898 vd->vdev_root_zap, tx)); 8899 } 8900 if (vd->vdev_top_zap != 0) { 8901 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8902 vd->vdev_top_zap, tx)); 8903 } 8904 if (vd->vdev_leaf_zap != 0) { 8905 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8906 vd->vdev_leaf_zap, tx)); 8907 } 8908 for (uint64_t i = 0; i < vd->vdev_children; i++) { 8909 spa_avz_build(vd->vdev_child[i], avz, tx); 8910 } 8911 } 8912 8913 static void 8914 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 8915 { 8916 nvlist_t *config; 8917 8918 /* 8919 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 8920 * its config may not be dirty but we still need to build per-vdev ZAPs. 8921 * Similarly, if the pool is being assembled (e.g. after a split), we 8922 * need to rebuild the AVZ although the config may not be dirty. 8923 */ 8924 if (list_is_empty(&spa->spa_config_dirty_list) && 8925 spa->spa_avz_action == AVZ_ACTION_NONE) 8926 return; 8927 8928 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 8929 8930 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 8931 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 8932 spa->spa_all_vdev_zaps != 0); 8933 8934 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 8935 /* Make and build the new AVZ */ 8936 uint64_t new_avz = zap_create(spa->spa_meta_objset, 8937 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 8938 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 8939 8940 /* Diff old AVZ with new one */ 8941 zap_cursor_t zc; 8942 zap_attribute_t za; 8943 8944 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8945 spa->spa_all_vdev_zaps); 8946 zap_cursor_retrieve(&zc, &za) == 0; 8947 zap_cursor_advance(&zc)) { 8948 uint64_t vdzap = za.za_first_integer; 8949 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 8950 vdzap) == ENOENT) { 8951 /* 8952 * ZAP is listed in old AVZ but not in new one; 8953 * destroy it 8954 */ 8955 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 8956 tx)); 8957 } 8958 } 8959 8960 zap_cursor_fini(&zc); 8961 8962 /* Destroy the old AVZ */ 8963 VERIFY0(zap_destroy(spa->spa_meta_objset, 8964 spa->spa_all_vdev_zaps, tx)); 8965 8966 /* Replace the old AVZ in the dir obj with the new one */ 8967 VERIFY0(zap_update(spa->spa_meta_objset, 8968 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 8969 sizeof (new_avz), 1, &new_avz, tx)); 8970 8971 spa->spa_all_vdev_zaps = new_avz; 8972 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 8973 zap_cursor_t zc; 8974 zap_attribute_t za; 8975 8976 /* Walk through the AVZ and destroy all listed ZAPs */ 8977 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8978 spa->spa_all_vdev_zaps); 8979 zap_cursor_retrieve(&zc, &za) == 0; 8980 zap_cursor_advance(&zc)) { 8981 uint64_t zap = za.za_first_integer; 8982 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 8983 } 8984 8985 zap_cursor_fini(&zc); 8986 8987 /* Destroy and unlink the AVZ itself */ 8988 VERIFY0(zap_destroy(spa->spa_meta_objset, 8989 spa->spa_all_vdev_zaps, tx)); 8990 VERIFY0(zap_remove(spa->spa_meta_objset, 8991 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 8992 spa->spa_all_vdev_zaps = 0; 8993 } 8994 8995 if (spa->spa_all_vdev_zaps == 0) { 8996 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 8997 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 8998 DMU_POOL_VDEV_ZAP_MAP, tx); 8999 } 9000 spa->spa_avz_action = AVZ_ACTION_NONE; 9001 9002 /* Create ZAPs for vdevs that don't have them. */ 9003 vdev_construct_zaps(spa->spa_root_vdev, tx); 9004 9005 config = spa_config_generate(spa, spa->spa_root_vdev, 9006 dmu_tx_get_txg(tx), B_FALSE); 9007 9008 /* 9009 * If we're upgrading the spa version then make sure that 9010 * the config object gets updated with the correct version. 9011 */ 9012 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 9013 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 9014 spa->spa_uberblock.ub_version); 9015 9016 spa_config_exit(spa, SCL_STATE, FTAG); 9017 9018 nvlist_free(spa->spa_config_syncing); 9019 spa->spa_config_syncing = config; 9020 9021 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 9022 } 9023 9024 static void 9025 spa_sync_version(void *arg, dmu_tx_t *tx) 9026 { 9027 uint64_t *versionp = arg; 9028 uint64_t version = *versionp; 9029 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 9030 9031 /* 9032 * Setting the version is special cased when first creating the pool. 9033 */ 9034 ASSERT(tx->tx_txg != TXG_INITIAL); 9035 9036 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 9037 ASSERT(version >= spa_version(spa)); 9038 9039 spa->spa_uberblock.ub_version = version; 9040 vdev_config_dirty(spa->spa_root_vdev); 9041 spa_history_log_internal(spa, "set", tx, "version=%lld", 9042 (longlong_t)version); 9043 } 9044 9045 /* 9046 * Set zpool properties. 9047 */ 9048 static void 9049 spa_sync_props(void *arg, dmu_tx_t *tx) 9050 { 9051 nvlist_t *nvp = arg; 9052 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 9053 objset_t *mos = spa->spa_meta_objset; 9054 nvpair_t *elem = NULL; 9055 9056 mutex_enter(&spa->spa_props_lock); 9057 9058 while ((elem = nvlist_next_nvpair(nvp, elem))) { 9059 uint64_t intval; 9060 const char *strval, *fname; 9061 zpool_prop_t prop; 9062 const char *propname; 9063 const char *elemname = nvpair_name(elem); 9064 zprop_type_t proptype; 9065 spa_feature_t fid; 9066 9067 switch (prop = zpool_name_to_prop(elemname)) { 9068 case ZPOOL_PROP_VERSION: 9069 intval = fnvpair_value_uint64(elem); 9070 /* 9071 * The version is synced separately before other 9072 * properties and should be correct by now. 9073 */ 9074 ASSERT3U(spa_version(spa), >=, intval); 9075 break; 9076 9077 case ZPOOL_PROP_ALTROOT: 9078 /* 9079 * 'altroot' is a non-persistent property. It should 9080 * have been set temporarily at creation or import time. 9081 */ 9082 ASSERT(spa->spa_root != NULL); 9083 break; 9084 9085 case ZPOOL_PROP_READONLY: 9086 case ZPOOL_PROP_CACHEFILE: 9087 /* 9088 * 'readonly' and 'cachefile' are also non-persistent 9089 * properties. 9090 */ 9091 break; 9092 case ZPOOL_PROP_COMMENT: 9093 strval = fnvpair_value_string(elem); 9094 if (spa->spa_comment != NULL) 9095 spa_strfree(spa->spa_comment); 9096 spa->spa_comment = spa_strdup(strval); 9097 /* 9098 * We need to dirty the configuration on all the vdevs 9099 * so that their labels get updated. We also need to 9100 * update the cache file to keep it in sync with the 9101 * MOS version. It's unnecessary to do this for pool 9102 * creation since the vdev's configuration has already 9103 * been dirtied. 9104 */ 9105 if (tx->tx_txg != TXG_INITIAL) { 9106 vdev_config_dirty(spa->spa_root_vdev); 9107 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 9108 } 9109 spa_history_log_internal(spa, "set", tx, 9110 "%s=%s", elemname, strval); 9111 break; 9112 case ZPOOL_PROP_COMPATIBILITY: 9113 strval = fnvpair_value_string(elem); 9114 if (spa->spa_compatibility != NULL) 9115 spa_strfree(spa->spa_compatibility); 9116 spa->spa_compatibility = spa_strdup(strval); 9117 /* 9118 * Dirty the configuration on vdevs as above. 9119 */ 9120 if (tx->tx_txg != TXG_INITIAL) { 9121 vdev_config_dirty(spa->spa_root_vdev); 9122 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 9123 } 9124 9125 spa_history_log_internal(spa, "set", tx, 9126 "%s=%s", nvpair_name(elem), strval); 9127 break; 9128 9129 case ZPOOL_PROP_INVAL: 9130 if (zpool_prop_feature(elemname)) { 9131 fname = strchr(elemname, '@') + 1; 9132 VERIFY0(zfeature_lookup_name(fname, &fid)); 9133 9134 spa_feature_enable(spa, fid, tx); 9135 spa_history_log_internal(spa, "set", tx, 9136 "%s=enabled", elemname); 9137 break; 9138 } else if (!zfs_prop_user(elemname)) { 9139 ASSERT(zpool_prop_feature(elemname)); 9140 break; 9141 } 9142 zfs_fallthrough; 9143 default: 9144 /* 9145 * Set pool property values in the poolprops mos object. 9146 */ 9147 if (spa->spa_pool_props_object == 0) { 9148 spa->spa_pool_props_object = 9149 zap_create_link(mos, DMU_OT_POOL_PROPS, 9150 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 9151 tx); 9152 } 9153 9154 /* normalize the property name */ 9155 if (prop == ZPOOL_PROP_INVAL) { 9156 propname = elemname; 9157 proptype = PROP_TYPE_STRING; 9158 } else { 9159 propname = zpool_prop_to_name(prop); 9160 proptype = zpool_prop_get_type(prop); 9161 } 9162 9163 if (nvpair_type(elem) == DATA_TYPE_STRING) { 9164 ASSERT(proptype == PROP_TYPE_STRING); 9165 strval = fnvpair_value_string(elem); 9166 VERIFY0(zap_update(mos, 9167 spa->spa_pool_props_object, propname, 9168 1, strlen(strval) + 1, strval, tx)); 9169 spa_history_log_internal(spa, "set", tx, 9170 "%s=%s", elemname, strval); 9171 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 9172 intval = fnvpair_value_uint64(elem); 9173 9174 if (proptype == PROP_TYPE_INDEX) { 9175 const char *unused; 9176 VERIFY0(zpool_prop_index_to_string( 9177 prop, intval, &unused)); 9178 } 9179 VERIFY0(zap_update(mos, 9180 spa->spa_pool_props_object, propname, 9181 8, 1, &intval, tx)); 9182 spa_history_log_internal(spa, "set", tx, 9183 "%s=%lld", elemname, 9184 (longlong_t)intval); 9185 9186 switch (prop) { 9187 case ZPOOL_PROP_DELEGATION: 9188 spa->spa_delegation = intval; 9189 break; 9190 case ZPOOL_PROP_BOOTFS: 9191 spa->spa_bootfs = intval; 9192 break; 9193 case ZPOOL_PROP_FAILUREMODE: 9194 spa->spa_failmode = intval; 9195 break; 9196 case ZPOOL_PROP_AUTOTRIM: 9197 spa->spa_autotrim = intval; 9198 spa_async_request(spa, 9199 SPA_ASYNC_AUTOTRIM_RESTART); 9200 break; 9201 case ZPOOL_PROP_AUTOEXPAND: 9202 spa->spa_autoexpand = intval; 9203 if (tx->tx_txg != TXG_INITIAL) 9204 spa_async_request(spa, 9205 SPA_ASYNC_AUTOEXPAND); 9206 break; 9207 case ZPOOL_PROP_MULTIHOST: 9208 spa->spa_multihost = intval; 9209 break; 9210 default: 9211 break; 9212 } 9213 } else { 9214 ASSERT(0); /* not allowed */ 9215 } 9216 } 9217 9218 } 9219 9220 mutex_exit(&spa->spa_props_lock); 9221 } 9222 9223 /* 9224 * Perform one-time upgrade on-disk changes. spa_version() does not 9225 * reflect the new version this txg, so there must be no changes this 9226 * txg to anything that the upgrade code depends on after it executes. 9227 * Therefore this must be called after dsl_pool_sync() does the sync 9228 * tasks. 9229 */ 9230 static void 9231 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 9232 { 9233 if (spa_sync_pass(spa) != 1) 9234 return; 9235 9236 dsl_pool_t *dp = spa->spa_dsl_pool; 9237 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 9238 9239 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && 9240 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { 9241 dsl_pool_create_origin(dp, tx); 9242 9243 /* Keeping the origin open increases spa_minref */ 9244 spa->spa_minref += 3; 9245 } 9246 9247 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && 9248 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { 9249 dsl_pool_upgrade_clones(dp, tx); 9250 } 9251 9252 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && 9253 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { 9254 dsl_pool_upgrade_dir_clones(dp, tx); 9255 9256 /* Keeping the freedir open increases spa_minref */ 9257 spa->spa_minref += 3; 9258 } 9259 9260 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && 9261 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 9262 spa_feature_create_zap_objects(spa, tx); 9263 } 9264 9265 /* 9266 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 9267 * when possibility to use lz4 compression for metadata was added 9268 * Old pools that have this feature enabled must be upgraded to have 9269 * this feature active 9270 */ 9271 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 9272 boolean_t lz4_en = spa_feature_is_enabled(spa, 9273 SPA_FEATURE_LZ4_COMPRESS); 9274 boolean_t lz4_ac = spa_feature_is_active(spa, 9275 SPA_FEATURE_LZ4_COMPRESS); 9276 9277 if (lz4_en && !lz4_ac) 9278 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 9279 } 9280 9281 /* 9282 * If we haven't written the salt, do so now. Note that the 9283 * feature may not be activated yet, but that's fine since 9284 * the presence of this ZAP entry is backwards compatible. 9285 */ 9286 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 9287 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 9288 VERIFY0(zap_add(spa->spa_meta_objset, 9289 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 9290 sizeof (spa->spa_cksum_salt.zcs_bytes), 9291 spa->spa_cksum_salt.zcs_bytes, tx)); 9292 } 9293 9294 rrw_exit(&dp->dp_config_rwlock, FTAG); 9295 } 9296 9297 static void 9298 vdev_indirect_state_sync_verify(vdev_t *vd) 9299 { 9300 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 9301 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 9302 9303 if (vd->vdev_ops == &vdev_indirect_ops) { 9304 ASSERT(vim != NULL); 9305 ASSERT(vib != NULL); 9306 } 9307 9308 uint64_t obsolete_sm_object = 0; 9309 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 9310 if (obsolete_sm_object != 0) { 9311 ASSERT(vd->vdev_obsolete_sm != NULL); 9312 ASSERT(vd->vdev_removing || 9313 vd->vdev_ops == &vdev_indirect_ops); 9314 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 9315 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 9316 ASSERT3U(obsolete_sm_object, ==, 9317 space_map_object(vd->vdev_obsolete_sm)); 9318 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 9319 space_map_allocated(vd->vdev_obsolete_sm)); 9320 } 9321 ASSERT(vd->vdev_obsolete_segments != NULL); 9322 9323 /* 9324 * Since frees / remaps to an indirect vdev can only 9325 * happen in syncing context, the obsolete segments 9326 * tree must be empty when we start syncing. 9327 */ 9328 ASSERT0(range_tree_space(vd->vdev_obsolete_segments)); 9329 } 9330 9331 /* 9332 * Set the top-level vdev's max queue depth. Evaluate each top-level's 9333 * async write queue depth in case it changed. The max queue depth will 9334 * not change in the middle of syncing out this txg. 9335 */ 9336 static void 9337 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 9338 { 9339 ASSERT(spa_writeable(spa)); 9340 9341 vdev_t *rvd = spa->spa_root_vdev; 9342 uint32_t max_queue_depth = zfs_vdev_async_write_max_active * 9343 zfs_vdev_queue_depth_pct / 100; 9344 metaslab_class_t *normal = spa_normal_class(spa); 9345 metaslab_class_t *special = spa_special_class(spa); 9346 metaslab_class_t *dedup = spa_dedup_class(spa); 9347 9348 uint64_t slots_per_allocator = 0; 9349 for (int c = 0; c < rvd->vdev_children; c++) { 9350 vdev_t *tvd = rvd->vdev_child[c]; 9351 9352 metaslab_group_t *mg = tvd->vdev_mg; 9353 if (mg == NULL || !metaslab_group_initialized(mg)) 9354 continue; 9355 9356 metaslab_class_t *mc = mg->mg_class; 9357 if (mc != normal && mc != special && mc != dedup) 9358 continue; 9359 9360 /* 9361 * It is safe to do a lock-free check here because only async 9362 * allocations look at mg_max_alloc_queue_depth, and async 9363 * allocations all happen from spa_sync(). 9364 */ 9365 for (int i = 0; i < mg->mg_allocators; i++) { 9366 ASSERT0(zfs_refcount_count( 9367 &(mg->mg_allocator[i].mga_alloc_queue_depth))); 9368 } 9369 mg->mg_max_alloc_queue_depth = max_queue_depth; 9370 9371 for (int i = 0; i < mg->mg_allocators; i++) { 9372 mg->mg_allocator[i].mga_cur_max_alloc_queue_depth = 9373 zfs_vdev_def_queue_depth; 9374 } 9375 slots_per_allocator += zfs_vdev_def_queue_depth; 9376 } 9377 9378 for (int i = 0; i < spa->spa_alloc_count; i++) { 9379 ASSERT0(zfs_refcount_count(&normal->mc_allocator[i]. 9380 mca_alloc_slots)); 9381 ASSERT0(zfs_refcount_count(&special->mc_allocator[i]. 9382 mca_alloc_slots)); 9383 ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i]. 9384 mca_alloc_slots)); 9385 normal->mc_allocator[i].mca_alloc_max_slots = 9386 slots_per_allocator; 9387 special->mc_allocator[i].mca_alloc_max_slots = 9388 slots_per_allocator; 9389 dedup->mc_allocator[i].mca_alloc_max_slots = 9390 slots_per_allocator; 9391 } 9392 normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9393 special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9394 dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 9395 } 9396 9397 static void 9398 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 9399 { 9400 ASSERT(spa_writeable(spa)); 9401 9402 vdev_t *rvd = spa->spa_root_vdev; 9403 for (int c = 0; c < rvd->vdev_children; c++) { 9404 vdev_t *vd = rvd->vdev_child[c]; 9405 vdev_indirect_state_sync_verify(vd); 9406 9407 if (vdev_indirect_should_condense(vd)) { 9408 spa_condense_indirect_start_sync(vd, tx); 9409 break; 9410 } 9411 } 9412 } 9413 9414 static void 9415 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 9416 { 9417 objset_t *mos = spa->spa_meta_objset; 9418 dsl_pool_t *dp = spa->spa_dsl_pool; 9419 uint64_t txg = tx->tx_txg; 9420 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 9421 9422 do { 9423 int pass = ++spa->spa_sync_pass; 9424 9425 spa_sync_config_object(spa, tx); 9426 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 9427 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 9428 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 9429 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 9430 spa_errlog_sync(spa, txg); 9431 dsl_pool_sync(dp, txg); 9432 9433 if (pass < zfs_sync_pass_deferred_free || 9434 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 9435 /* 9436 * If the log space map feature is active we don't 9437 * care about deferred frees and the deferred bpobj 9438 * as the log space map should effectively have the 9439 * same results (i.e. appending only to one object). 9440 */ 9441 spa_sync_frees(spa, free_bpl, tx); 9442 } else { 9443 /* 9444 * We can not defer frees in pass 1, because 9445 * we sync the deferred frees later in pass 1. 9446 */ 9447 ASSERT3U(pass, >, 1); 9448 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 9449 &spa->spa_deferred_bpobj, tx); 9450 } 9451 9452 brt_sync(spa, txg); 9453 ddt_sync(spa, txg); 9454 dsl_scan_sync(dp, tx); 9455 dsl_errorscrub_sync(dp, tx); 9456 svr_sync(spa, tx); 9457 spa_sync_upgrades(spa, tx); 9458 9459 spa_flush_metaslabs(spa, tx); 9460 9461 vdev_t *vd = NULL; 9462 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 9463 != NULL) 9464 vdev_sync(vd, txg); 9465 9466 if (pass == 1) { 9467 /* 9468 * dsl_pool_sync() -> dp_sync_tasks may have dirtied 9469 * the config. If that happens, this txg should not 9470 * be a no-op. So we must sync the config to the MOS 9471 * before checking for no-op. 9472 * 9473 * Note that when the config is dirty, it will 9474 * be written to the MOS (i.e. the MOS will be 9475 * dirtied) every time we call spa_sync_config_object() 9476 * in this txg. Therefore we can't call this after 9477 * dsl_pool_sync() every pass, because it would 9478 * prevent us from converging, since we'd dirty 9479 * the MOS every pass. 9480 * 9481 * Sync tasks can only be processed in pass 1, so 9482 * there's no need to do this in later passes. 9483 */ 9484 spa_sync_config_object(spa, tx); 9485 } 9486 9487 /* 9488 * Note: We need to check if the MOS is dirty because we could 9489 * have marked the MOS dirty without updating the uberblock 9490 * (e.g. if we have sync tasks but no dirty user data). We need 9491 * to check the uberblock's rootbp because it is updated if we 9492 * have synced out dirty data (though in this case the MOS will 9493 * most likely also be dirty due to second order effects, we 9494 * don't want to rely on that here). 9495 */ 9496 if (pass == 1 && 9497 spa->spa_uberblock.ub_rootbp.blk_birth < txg && 9498 !dmu_objset_is_dirty(mos, txg)) { 9499 /* 9500 * Nothing changed on the first pass, therefore this 9501 * TXG is a no-op. Avoid syncing deferred frees, so 9502 * that we can keep this TXG as a no-op. 9503 */ 9504 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9505 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9506 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 9507 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 9508 break; 9509 } 9510 9511 spa_sync_deferred_frees(spa, tx); 9512 } while (dmu_objset_is_dirty(mos, txg)); 9513 } 9514 9515 /* 9516 * Rewrite the vdev configuration (which includes the uberblock) to 9517 * commit the transaction group. 9518 * 9519 * If there are no dirty vdevs, we sync the uberblock to a few random 9520 * top-level vdevs that are known to be visible in the config cache 9521 * (see spa_vdev_add() for a complete description). If there *are* dirty 9522 * vdevs, sync the uberblock to all vdevs. 9523 */ 9524 static void 9525 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 9526 { 9527 vdev_t *rvd = spa->spa_root_vdev; 9528 uint64_t txg = tx->tx_txg; 9529 9530 for (;;) { 9531 int error = 0; 9532 9533 /* 9534 * We hold SCL_STATE to prevent vdev open/close/etc. 9535 * while we're attempting to write the vdev labels. 9536 */ 9537 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9538 9539 if (list_is_empty(&spa->spa_config_dirty_list)) { 9540 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 9541 int svdcount = 0; 9542 int children = rvd->vdev_children; 9543 int c0 = random_in_range(children); 9544 9545 for (int c = 0; c < children; c++) { 9546 vdev_t *vd = 9547 rvd->vdev_child[(c0 + c) % children]; 9548 9549 /* Stop when revisiting the first vdev */ 9550 if (c > 0 && svd[0] == vd) 9551 break; 9552 9553 if (vd->vdev_ms_array == 0 || 9554 vd->vdev_islog || 9555 !vdev_is_concrete(vd)) 9556 continue; 9557 9558 svd[svdcount++] = vd; 9559 if (svdcount == SPA_SYNC_MIN_VDEVS) 9560 break; 9561 } 9562 error = vdev_config_sync(svd, svdcount, txg); 9563 } else { 9564 error = vdev_config_sync(rvd->vdev_child, 9565 rvd->vdev_children, txg); 9566 } 9567 9568 if (error == 0) 9569 spa->spa_last_synced_guid = rvd->vdev_guid; 9570 9571 spa_config_exit(spa, SCL_STATE, FTAG); 9572 9573 if (error == 0) 9574 break; 9575 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 9576 zio_resume_wait(spa); 9577 } 9578 } 9579 9580 /* 9581 * Sync the specified transaction group. New blocks may be dirtied as 9582 * part of the process, so we iterate until it converges. 9583 */ 9584 void 9585 spa_sync(spa_t *spa, uint64_t txg) 9586 { 9587 vdev_t *vd = NULL; 9588 9589 VERIFY(spa_writeable(spa)); 9590 9591 /* 9592 * Wait for i/os issued in open context that need to complete 9593 * before this txg syncs. 9594 */ 9595 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 9596 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 9597 ZIO_FLAG_CANFAIL); 9598 9599 /* 9600 * Now that there can be no more cloning in this transaction group, 9601 * but we are still before issuing frees, we can process pending BRT 9602 * updates. 9603 */ 9604 brt_pending_apply(spa, txg); 9605 9606 /* 9607 * Lock out configuration changes. 9608 */ 9609 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9610 9611 spa->spa_syncing_txg = txg; 9612 spa->spa_sync_pass = 0; 9613 9614 for (int i = 0; i < spa->spa_alloc_count; i++) { 9615 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9616 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9617 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9618 } 9619 9620 /* 9621 * If there are any pending vdev state changes, convert them 9622 * into config changes that go out with this transaction group. 9623 */ 9624 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9625 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9626 /* Avoid holding the write lock unless actually necessary */ 9627 if (vd->vdev_aux == NULL) { 9628 vdev_state_clean(vd); 9629 vdev_config_dirty(vd); 9630 continue; 9631 } 9632 /* 9633 * We need the write lock here because, for aux vdevs, 9634 * calling vdev_config_dirty() modifies sav_config. 9635 * This is ugly and will become unnecessary when we 9636 * eliminate the aux vdev wart by integrating all vdevs 9637 * into the root vdev tree. 9638 */ 9639 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9640 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 9641 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9642 vdev_state_clean(vd); 9643 vdev_config_dirty(vd); 9644 } 9645 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9646 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9647 } 9648 spa_config_exit(spa, SCL_STATE, FTAG); 9649 9650 dsl_pool_t *dp = spa->spa_dsl_pool; 9651 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 9652 9653 spa->spa_sync_starttime = gethrtime(); 9654 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9655 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 9656 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 9657 NSEC_TO_TICK(spa->spa_deadman_synctime)); 9658 9659 /* 9660 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 9661 * set spa_deflate if we have no raid-z vdevs. 9662 */ 9663 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 9664 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 9665 vdev_t *rvd = spa->spa_root_vdev; 9666 9667 int i; 9668 for (i = 0; i < rvd->vdev_children; i++) { 9669 vd = rvd->vdev_child[i]; 9670 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 9671 break; 9672 } 9673 if (i == rvd->vdev_children) { 9674 spa->spa_deflate = TRUE; 9675 VERIFY0(zap_add(spa->spa_meta_objset, 9676 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 9677 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 9678 } 9679 } 9680 9681 spa_sync_adjust_vdev_max_queue_depth(spa); 9682 9683 spa_sync_condense_indirect(spa, tx); 9684 9685 spa_sync_iterate_to_convergence(spa, tx); 9686 9687 #ifdef ZFS_DEBUG 9688 if (!list_is_empty(&spa->spa_config_dirty_list)) { 9689 /* 9690 * Make sure that the number of ZAPs for all the vdevs matches 9691 * the number of ZAPs in the per-vdev ZAP list. This only gets 9692 * called if the config is dirty; otherwise there may be 9693 * outstanding AVZ operations that weren't completed in 9694 * spa_sync_config_object. 9695 */ 9696 uint64_t all_vdev_zap_entry_count; 9697 ASSERT0(zap_count(spa->spa_meta_objset, 9698 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 9699 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 9700 all_vdev_zap_entry_count); 9701 } 9702 #endif 9703 9704 if (spa->spa_vdev_removal != NULL) { 9705 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 9706 } 9707 9708 spa_sync_rewrite_vdev_config(spa, tx); 9709 dmu_tx_commit(tx); 9710 9711 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9712 spa->spa_deadman_tqid = 0; 9713 9714 /* 9715 * Clear the dirty config list. 9716 */ 9717 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 9718 vdev_config_clean(vd); 9719 9720 /* 9721 * Now that the new config has synced transactionally, 9722 * let it become visible to the config cache. 9723 */ 9724 if (spa->spa_config_syncing != NULL) { 9725 spa_config_set(spa, spa->spa_config_syncing); 9726 spa->spa_config_txg = txg; 9727 spa->spa_config_syncing = NULL; 9728 } 9729 9730 dsl_pool_sync_done(dp, txg); 9731 9732 for (int i = 0; i < spa->spa_alloc_count; i++) { 9733 mutex_enter(&spa->spa_allocs[i].spaa_lock); 9734 VERIFY0(avl_numnodes(&spa->spa_allocs[i].spaa_tree)); 9735 mutex_exit(&spa->spa_allocs[i].spaa_lock); 9736 } 9737 9738 /* 9739 * Update usable space statistics. 9740 */ 9741 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 9742 != NULL) 9743 vdev_sync_done(vd, txg); 9744 9745 metaslab_class_evict_old(spa->spa_normal_class, txg); 9746 metaslab_class_evict_old(spa->spa_log_class, txg); 9747 9748 spa_sync_close_syncing_log_sm(spa); 9749 9750 spa_update_dspace(spa); 9751 9752 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) 9753 vdev_autotrim_kick(spa); 9754 9755 /* 9756 * It had better be the case that we didn't dirty anything 9757 * since vdev_config_sync(). 9758 */ 9759 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9760 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9761 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 9762 9763 while (zfs_pause_spa_sync) 9764 delay(1); 9765 9766 spa->spa_sync_pass = 0; 9767 9768 /* 9769 * Update the last synced uberblock here. We want to do this at 9770 * the end of spa_sync() so that consumers of spa_last_synced_txg() 9771 * will be guaranteed that all the processing associated with 9772 * that txg has been completed. 9773 */ 9774 spa->spa_ubsync = spa->spa_uberblock; 9775 spa_config_exit(spa, SCL_CONFIG, FTAG); 9776 9777 spa_handle_ignored_writes(spa); 9778 9779 /* 9780 * If any async tasks have been requested, kick them off. 9781 */ 9782 spa_async_dispatch(spa); 9783 } 9784 9785 /* 9786 * Sync all pools. We don't want to hold the namespace lock across these 9787 * operations, so we take a reference on the spa_t and drop the lock during the 9788 * sync. 9789 */ 9790 void 9791 spa_sync_allpools(void) 9792 { 9793 spa_t *spa = NULL; 9794 mutex_enter(&spa_namespace_lock); 9795 while ((spa = spa_next(spa)) != NULL) { 9796 if (spa_state(spa) != POOL_STATE_ACTIVE || 9797 !spa_writeable(spa) || spa_suspended(spa)) 9798 continue; 9799 spa_open_ref(spa, FTAG); 9800 mutex_exit(&spa_namespace_lock); 9801 txg_wait_synced(spa_get_dsl(spa), 0); 9802 mutex_enter(&spa_namespace_lock); 9803 spa_close(spa, FTAG); 9804 } 9805 mutex_exit(&spa_namespace_lock); 9806 } 9807 9808 taskq_t * 9809 spa_sync_tq_create(spa_t *spa, const char *name) 9810 { 9811 kthread_t **kthreads; 9812 9813 ASSERT(spa->spa_sync_tq == NULL); 9814 ASSERT3S(spa->spa_alloc_count, <=, boot_ncpus); 9815 9816 /* 9817 * - do not allow more allocators than cpus. 9818 * - there may be more cpus than allocators. 9819 * - do not allow more sync taskq threads than allocators or cpus. 9820 */ 9821 int nthreads = spa->spa_alloc_count; 9822 spa->spa_syncthreads = kmem_zalloc(sizeof (spa_syncthread_info_t) * 9823 nthreads, KM_SLEEP); 9824 9825 spa->spa_sync_tq = taskq_create_synced(name, nthreads, minclsyspri, 9826 nthreads, INT_MAX, TASKQ_PREPOPULATE, &kthreads); 9827 VERIFY(spa->spa_sync_tq != NULL); 9828 VERIFY(kthreads != NULL); 9829 9830 spa_taskqs_t *tqs = 9831 &spa->spa_zio_taskq[ZIO_TYPE_WRITE][ZIO_TASKQ_ISSUE]; 9832 9833 spa_syncthread_info_t *ti = spa->spa_syncthreads; 9834 for (int i = 0, w = 0; i < nthreads; i++, w++, ti++) { 9835 ti->sti_thread = kthreads[i]; 9836 if (w == tqs->stqs_count) { 9837 w = 0; 9838 } 9839 ti->sti_wr_iss_tq = tqs->stqs_taskq[w]; 9840 } 9841 9842 kmem_free(kthreads, sizeof (*kthreads) * nthreads); 9843 return (spa->spa_sync_tq); 9844 } 9845 9846 void 9847 spa_sync_tq_destroy(spa_t *spa) 9848 { 9849 ASSERT(spa->spa_sync_tq != NULL); 9850 9851 taskq_wait(spa->spa_sync_tq); 9852 taskq_destroy(spa->spa_sync_tq); 9853 kmem_free(spa->spa_syncthreads, 9854 sizeof (spa_syncthread_info_t) * spa->spa_alloc_count); 9855 spa->spa_sync_tq = NULL; 9856 } 9857 9858 void 9859 spa_select_allocator(zio_t *zio) 9860 { 9861 zbookmark_phys_t *bm = &zio->io_bookmark; 9862 spa_t *spa = zio->io_spa; 9863 9864 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 9865 9866 /* 9867 * A gang block (for example) may have inherited its parent's 9868 * allocator, in which case there is nothing further to do here. 9869 */ 9870 if (ZIO_HAS_ALLOCATOR(zio)) 9871 return; 9872 9873 ASSERT(spa != NULL); 9874 ASSERT(bm != NULL); 9875 9876 /* 9877 * First try to use an allocator assigned to the syncthread, and set 9878 * the corresponding write issue taskq for the allocator. 9879 * Note, we must have an open pool to do this. 9880 */ 9881 if (spa->spa_sync_tq != NULL) { 9882 spa_syncthread_info_t *ti = spa->spa_syncthreads; 9883 for (int i = 0; i < spa->spa_alloc_count; i++, ti++) { 9884 if (ti->sti_thread == curthread) { 9885 zio->io_allocator = i; 9886 zio->io_wr_iss_tq = ti->sti_wr_iss_tq; 9887 return; 9888 } 9889 } 9890 } 9891 9892 /* 9893 * We want to try to use as many allocators as possible to help improve 9894 * performance, but we also want logically adjacent IOs to be physically 9895 * adjacent to improve sequential read performance. We chunk each object 9896 * into 2^20 block regions, and then hash based on the objset, object, 9897 * level, and region to accomplish both of these goals. 9898 */ 9899 uint64_t hv = cityhash4(bm->zb_objset, bm->zb_object, bm->zb_level, 9900 bm->zb_blkid >> 20); 9901 9902 zio->io_allocator = (uint_t)hv % spa->spa_alloc_count; 9903 zio->io_wr_iss_tq = NULL; 9904 } 9905 9906 /* 9907 * ========================================================================== 9908 * Miscellaneous routines 9909 * ========================================================================== 9910 */ 9911 9912 /* 9913 * Remove all pools in the system. 9914 */ 9915 void 9916 spa_evict_all(void) 9917 { 9918 spa_t *spa; 9919 9920 /* 9921 * Remove all cached state. All pools should be closed now, 9922 * so every spa in the AVL tree should be unreferenced. 9923 */ 9924 mutex_enter(&spa_namespace_lock); 9925 while ((spa = spa_next(NULL)) != NULL) { 9926 /* 9927 * Stop async tasks. The async thread may need to detach 9928 * a device that's been replaced, which requires grabbing 9929 * spa_namespace_lock, so we must drop it here. 9930 */ 9931 spa_open_ref(spa, FTAG); 9932 mutex_exit(&spa_namespace_lock); 9933 spa_async_suspend(spa); 9934 mutex_enter(&spa_namespace_lock); 9935 spa_close(spa, FTAG); 9936 9937 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 9938 spa_unload(spa); 9939 spa_deactivate(spa); 9940 } 9941 spa_remove(spa); 9942 } 9943 mutex_exit(&spa_namespace_lock); 9944 } 9945 9946 vdev_t * 9947 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 9948 { 9949 vdev_t *vd; 9950 int i; 9951 9952 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 9953 return (vd); 9954 9955 if (aux) { 9956 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 9957 vd = spa->spa_l2cache.sav_vdevs[i]; 9958 if (vd->vdev_guid == guid) 9959 return (vd); 9960 } 9961 9962 for (i = 0; i < spa->spa_spares.sav_count; i++) { 9963 vd = spa->spa_spares.sav_vdevs[i]; 9964 if (vd->vdev_guid == guid) 9965 return (vd); 9966 } 9967 } 9968 9969 return (NULL); 9970 } 9971 9972 void 9973 spa_upgrade(spa_t *spa, uint64_t version) 9974 { 9975 ASSERT(spa_writeable(spa)); 9976 9977 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9978 9979 /* 9980 * This should only be called for a non-faulted pool, and since a 9981 * future version would result in an unopenable pool, this shouldn't be 9982 * possible. 9983 */ 9984 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 9985 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 9986 9987 spa->spa_uberblock.ub_version = version; 9988 vdev_config_dirty(spa->spa_root_vdev); 9989 9990 spa_config_exit(spa, SCL_ALL, FTAG); 9991 9992 txg_wait_synced(spa_get_dsl(spa), 0); 9993 } 9994 9995 static boolean_t 9996 spa_has_aux_vdev(spa_t *spa, uint64_t guid, spa_aux_vdev_t *sav) 9997 { 9998 (void) spa; 9999 int i; 10000 uint64_t vdev_guid; 10001 10002 for (i = 0; i < sav->sav_count; i++) 10003 if (sav->sav_vdevs[i]->vdev_guid == guid) 10004 return (B_TRUE); 10005 10006 for (i = 0; i < sav->sav_npending; i++) { 10007 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 10008 &vdev_guid) == 0 && vdev_guid == guid) 10009 return (B_TRUE); 10010 } 10011 10012 return (B_FALSE); 10013 } 10014 10015 boolean_t 10016 spa_has_l2cache(spa_t *spa, uint64_t guid) 10017 { 10018 return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache)); 10019 } 10020 10021 boolean_t 10022 spa_has_spare(spa_t *spa, uint64_t guid) 10023 { 10024 return (spa_has_aux_vdev(spa, guid, &spa->spa_spares)); 10025 } 10026 10027 /* 10028 * Check if a pool has an active shared spare device. 10029 * Note: reference count of an active spare is 2, as a spare and as a replace 10030 */ 10031 static boolean_t 10032 spa_has_active_shared_spare(spa_t *spa) 10033 { 10034 int i, refcnt; 10035 uint64_t pool; 10036 spa_aux_vdev_t *sav = &spa->spa_spares; 10037 10038 for (i = 0; i < sav->sav_count; i++) { 10039 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 10040 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 10041 refcnt > 2) 10042 return (B_TRUE); 10043 } 10044 10045 return (B_FALSE); 10046 } 10047 10048 uint64_t 10049 spa_total_metaslabs(spa_t *spa) 10050 { 10051 vdev_t *rvd = spa->spa_root_vdev; 10052 10053 uint64_t m = 0; 10054 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 10055 vdev_t *vd = rvd->vdev_child[c]; 10056 if (!vdev_is_concrete(vd)) 10057 continue; 10058 m += vd->vdev_ms_count; 10059 } 10060 return (m); 10061 } 10062 10063 /* 10064 * Notify any waiting threads that some activity has switched from being in- 10065 * progress to not-in-progress so that the thread can wake up and determine 10066 * whether it is finished waiting. 10067 */ 10068 void 10069 spa_notify_waiters(spa_t *spa) 10070 { 10071 /* 10072 * Acquiring spa_activities_lock here prevents the cv_broadcast from 10073 * happening between the waiting thread's check and cv_wait. 10074 */ 10075 mutex_enter(&spa->spa_activities_lock); 10076 cv_broadcast(&spa->spa_activities_cv); 10077 mutex_exit(&spa->spa_activities_lock); 10078 } 10079 10080 /* 10081 * Notify any waiting threads that the pool is exporting, and then block until 10082 * they are finished using the spa_t. 10083 */ 10084 void 10085 spa_wake_waiters(spa_t *spa) 10086 { 10087 mutex_enter(&spa->spa_activities_lock); 10088 spa->spa_waiters_cancel = B_TRUE; 10089 cv_broadcast(&spa->spa_activities_cv); 10090 while (spa->spa_waiters != 0) 10091 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 10092 spa->spa_waiters_cancel = B_FALSE; 10093 mutex_exit(&spa->spa_activities_lock); 10094 } 10095 10096 /* Whether the vdev or any of its descendants are being initialized/trimmed. */ 10097 static boolean_t 10098 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 10099 { 10100 spa_t *spa = vd->vdev_spa; 10101 10102 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 10103 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 10104 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 10105 activity == ZPOOL_WAIT_TRIM); 10106 10107 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 10108 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 10109 10110 mutex_exit(&spa->spa_activities_lock); 10111 mutex_enter(lock); 10112 mutex_enter(&spa->spa_activities_lock); 10113 10114 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 10115 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) : 10116 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE); 10117 mutex_exit(lock); 10118 10119 if (in_progress) 10120 return (B_TRUE); 10121 10122 for (int i = 0; i < vd->vdev_children; i++) { 10123 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 10124 activity)) 10125 return (B_TRUE); 10126 } 10127 10128 return (B_FALSE); 10129 } 10130 10131 /* 10132 * If use_guid is true, this checks whether the vdev specified by guid is 10133 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 10134 * is being initialized/trimmed. The caller must hold the config lock and 10135 * spa_activities_lock. 10136 */ 10137 static int 10138 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 10139 zpool_wait_activity_t activity, boolean_t *in_progress) 10140 { 10141 mutex_exit(&spa->spa_activities_lock); 10142 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 10143 mutex_enter(&spa->spa_activities_lock); 10144 10145 vdev_t *vd; 10146 if (use_guid) { 10147 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 10148 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 10149 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10150 return (EINVAL); 10151 } 10152 } else { 10153 vd = spa->spa_root_vdev; 10154 } 10155 10156 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 10157 10158 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10159 return (0); 10160 } 10161 10162 /* 10163 * Locking for waiting threads 10164 * --------------------------- 10165 * 10166 * Waiting threads need a way to check whether a given activity is in progress, 10167 * and then, if it is, wait for it to complete. Each activity will have some 10168 * in-memory representation of the relevant on-disk state which can be used to 10169 * determine whether or not the activity is in progress. The in-memory state and 10170 * the locking used to protect it will be different for each activity, and may 10171 * not be suitable for use with a cvar (e.g., some state is protected by the 10172 * config lock). To allow waiting threads to wait without any races, another 10173 * lock, spa_activities_lock, is used. 10174 * 10175 * When the state is checked, both the activity-specific lock (if there is one) 10176 * and spa_activities_lock are held. In some cases, the activity-specific lock 10177 * is acquired explicitly (e.g. the config lock). In others, the locking is 10178 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 10179 * thread releases the activity-specific lock and, if the activity is in 10180 * progress, then cv_waits using spa_activities_lock. 10181 * 10182 * The waiting thread is woken when another thread, one completing some 10183 * activity, updates the state of the activity and then calls 10184 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 10185 * needs to hold its activity-specific lock when updating the state, and this 10186 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 10187 * 10188 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 10189 * and because it is held when the waiting thread checks the state of the 10190 * activity, it can never be the case that the completing thread both updates 10191 * the activity state and cv_broadcasts in between the waiting thread's check 10192 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 10193 * 10194 * In order to prevent deadlock, when the waiting thread does its check, in some 10195 * cases it will temporarily drop spa_activities_lock in order to acquire the 10196 * activity-specific lock. The order in which spa_activities_lock and the 10197 * activity specific lock are acquired in the waiting thread is determined by 10198 * the order in which they are acquired in the completing thread; if the 10199 * completing thread calls spa_notify_waiters with the activity-specific lock 10200 * held, then the waiting thread must also acquire the activity-specific lock 10201 * first. 10202 */ 10203 10204 static int 10205 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 10206 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 10207 { 10208 int error = 0; 10209 10210 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 10211 10212 switch (activity) { 10213 case ZPOOL_WAIT_CKPT_DISCARD: 10214 *in_progress = 10215 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 10216 zap_contains(spa_meta_objset(spa), 10217 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 10218 ENOENT); 10219 break; 10220 case ZPOOL_WAIT_FREE: 10221 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 10222 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 10223 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 10224 spa_livelist_delete_check(spa)); 10225 break; 10226 case ZPOOL_WAIT_INITIALIZE: 10227 case ZPOOL_WAIT_TRIM: 10228 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 10229 activity, in_progress); 10230 break; 10231 case ZPOOL_WAIT_REPLACE: 10232 mutex_exit(&spa->spa_activities_lock); 10233 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 10234 mutex_enter(&spa->spa_activities_lock); 10235 10236 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 10237 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 10238 break; 10239 case ZPOOL_WAIT_REMOVE: 10240 *in_progress = (spa->spa_removing_phys.sr_state == 10241 DSS_SCANNING); 10242 break; 10243 case ZPOOL_WAIT_RESILVER: 10244 *in_progress = vdev_rebuild_active(spa->spa_root_vdev); 10245 if (*in_progress) 10246 break; 10247 zfs_fallthrough; 10248 case ZPOOL_WAIT_SCRUB: 10249 { 10250 boolean_t scanning, paused, is_scrub; 10251 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 10252 10253 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 10254 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 10255 paused = dsl_scan_is_paused_scrub(scn); 10256 *in_progress = (scanning && !paused && 10257 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 10258 break; 10259 } 10260 case ZPOOL_WAIT_RAIDZ_EXPAND: 10261 { 10262 vdev_raidz_expand_t *vre = spa->spa_raidz_expand; 10263 *in_progress = (vre != NULL && vre->vre_state == DSS_SCANNING); 10264 break; 10265 } 10266 default: 10267 panic("unrecognized value for activity %d", activity); 10268 } 10269 10270 return (error); 10271 } 10272 10273 static int 10274 spa_wait_common(const char *pool, zpool_wait_activity_t activity, 10275 boolean_t use_tag, uint64_t tag, boolean_t *waited) 10276 { 10277 /* 10278 * The tag is used to distinguish between instances of an activity. 10279 * 'initialize' and 'trim' are the only activities that we use this for. 10280 * The other activities can only have a single instance in progress in a 10281 * pool at one time, making the tag unnecessary. 10282 * 10283 * There can be multiple devices being replaced at once, but since they 10284 * all finish once resilvering finishes, we don't bother keeping track 10285 * of them individually, we just wait for them all to finish. 10286 */ 10287 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 10288 activity != ZPOOL_WAIT_TRIM) 10289 return (EINVAL); 10290 10291 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 10292 return (EINVAL); 10293 10294 spa_t *spa; 10295 int error = spa_open(pool, &spa, FTAG); 10296 if (error != 0) 10297 return (error); 10298 10299 /* 10300 * Increment the spa's waiter count so that we can call spa_close and 10301 * still ensure that the spa_t doesn't get freed before this thread is 10302 * finished with it when the pool is exported. We want to call spa_close 10303 * before we start waiting because otherwise the additional ref would 10304 * prevent the pool from being exported or destroyed throughout the 10305 * potentially long wait. 10306 */ 10307 mutex_enter(&spa->spa_activities_lock); 10308 spa->spa_waiters++; 10309 spa_close(spa, FTAG); 10310 10311 *waited = B_FALSE; 10312 for (;;) { 10313 boolean_t in_progress; 10314 error = spa_activity_in_progress(spa, activity, use_tag, tag, 10315 &in_progress); 10316 10317 if (error || !in_progress || spa->spa_waiters_cancel) 10318 break; 10319 10320 *waited = B_TRUE; 10321 10322 if (cv_wait_sig(&spa->spa_activities_cv, 10323 &spa->spa_activities_lock) == 0) { 10324 error = EINTR; 10325 break; 10326 } 10327 } 10328 10329 spa->spa_waiters--; 10330 cv_signal(&spa->spa_waiters_cv); 10331 mutex_exit(&spa->spa_activities_lock); 10332 10333 return (error); 10334 } 10335 10336 /* 10337 * Wait for a particular instance of the specified activity to complete, where 10338 * the instance is identified by 'tag' 10339 */ 10340 int 10341 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 10342 boolean_t *waited) 10343 { 10344 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 10345 } 10346 10347 /* 10348 * Wait for all instances of the specified activity complete 10349 */ 10350 int 10351 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 10352 { 10353 10354 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 10355 } 10356 10357 sysevent_t * 10358 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 10359 { 10360 sysevent_t *ev = NULL; 10361 #ifdef _KERNEL 10362 nvlist_t *resource; 10363 10364 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 10365 if (resource) { 10366 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 10367 ev->resource = resource; 10368 } 10369 #else 10370 (void) spa, (void) vd, (void) hist_nvl, (void) name; 10371 #endif 10372 return (ev); 10373 } 10374 10375 void 10376 spa_event_post(sysevent_t *ev) 10377 { 10378 #ifdef _KERNEL 10379 if (ev) { 10380 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 10381 kmem_free(ev, sizeof (*ev)); 10382 } 10383 #else 10384 (void) ev; 10385 #endif 10386 } 10387 10388 /* 10389 * Post a zevent corresponding to the given sysevent. The 'name' must be one 10390 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 10391 * filled in from the spa and (optionally) the vdev. This doesn't do anything 10392 * in the userland libzpool, as we don't want consumers to misinterpret ztest 10393 * or zdb as real changes. 10394 */ 10395 void 10396 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 10397 { 10398 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 10399 } 10400 10401 /* state manipulation functions */ 10402 EXPORT_SYMBOL(spa_open); 10403 EXPORT_SYMBOL(spa_open_rewind); 10404 EXPORT_SYMBOL(spa_get_stats); 10405 EXPORT_SYMBOL(spa_create); 10406 EXPORT_SYMBOL(spa_import); 10407 EXPORT_SYMBOL(spa_tryimport); 10408 EXPORT_SYMBOL(spa_destroy); 10409 EXPORT_SYMBOL(spa_export); 10410 EXPORT_SYMBOL(spa_reset); 10411 EXPORT_SYMBOL(spa_async_request); 10412 EXPORT_SYMBOL(spa_async_suspend); 10413 EXPORT_SYMBOL(spa_async_resume); 10414 EXPORT_SYMBOL(spa_inject_addref); 10415 EXPORT_SYMBOL(spa_inject_delref); 10416 EXPORT_SYMBOL(spa_scan_stat_init); 10417 EXPORT_SYMBOL(spa_scan_get_stats); 10418 10419 /* device manipulation */ 10420 EXPORT_SYMBOL(spa_vdev_add); 10421 EXPORT_SYMBOL(spa_vdev_attach); 10422 EXPORT_SYMBOL(spa_vdev_detach); 10423 EXPORT_SYMBOL(spa_vdev_setpath); 10424 EXPORT_SYMBOL(spa_vdev_setfru); 10425 EXPORT_SYMBOL(spa_vdev_split_mirror); 10426 10427 /* spare statech is global across all pools) */ 10428 EXPORT_SYMBOL(spa_spare_add); 10429 EXPORT_SYMBOL(spa_spare_remove); 10430 EXPORT_SYMBOL(spa_spare_exists); 10431 EXPORT_SYMBOL(spa_spare_activate); 10432 10433 /* L2ARC statech is global across all pools) */ 10434 EXPORT_SYMBOL(spa_l2cache_add); 10435 EXPORT_SYMBOL(spa_l2cache_remove); 10436 EXPORT_SYMBOL(spa_l2cache_exists); 10437 EXPORT_SYMBOL(spa_l2cache_activate); 10438 EXPORT_SYMBOL(spa_l2cache_drop); 10439 10440 /* scanning */ 10441 EXPORT_SYMBOL(spa_scan); 10442 EXPORT_SYMBOL(spa_scan_stop); 10443 10444 /* spa syncing */ 10445 EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 10446 EXPORT_SYMBOL(spa_sync_allpools); 10447 10448 /* properties */ 10449 EXPORT_SYMBOL(spa_prop_set); 10450 EXPORT_SYMBOL(spa_prop_get); 10451 EXPORT_SYMBOL(spa_prop_clear_bootfs); 10452 10453 /* asynchronous event notification */ 10454 EXPORT_SYMBOL(spa_event_notify); 10455 10456 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_pct, UINT, ZMOD_RW, 10457 "Percentage of CPUs to run a metaslab preload taskq"); 10458 10459 /* BEGIN CSTYLED */ 10460 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, UINT, ZMOD_RW, 10461 "log2 fraction of arc that can be used by inflight I/Os when " 10462 "verifying pool during import"); 10463 /* END CSTYLED */ 10464 10465 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 10466 "Set to traverse metadata on pool import"); 10467 10468 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 10469 "Set to traverse data on pool import"); 10470 10471 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 10472 "Print vdev tree to zfs_dbgmsg during pool import"); 10473 10474 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD, 10475 "Percentage of CPUs to run an IO worker thread"); 10476 10477 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD, 10478 "Number of threads per IO worker taskqueue"); 10479 10480 /* BEGIN CSTYLED */ 10481 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, U64, ZMOD_RW, 10482 "Allow importing pool with up to this number of missing top-level " 10483 "vdevs (in read-only mode)"); 10484 /* END CSTYLED */ 10485 10486 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, 10487 ZMOD_RW, "Set the livelist condense zthr to pause"); 10488 10489 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, 10490 ZMOD_RW, "Set the livelist condense synctask to pause"); 10491 10492 /* BEGIN CSTYLED */ 10493 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, 10494 INT, ZMOD_RW, 10495 "Whether livelist condensing was canceled in the synctask"); 10496 10497 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, 10498 INT, ZMOD_RW, 10499 "Whether livelist condensing was canceled in the zthr function"); 10500 10501 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, 10502 ZMOD_RW, 10503 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 10504 "was being condensed"); 10505 /* END CSTYLED */ 10506 10507 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_wr_iss_ncpus, UINT, ZMOD_RW, 10508 "Number of CPUs to run write issue taskqs"); 10509