1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 13 /* 14 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 15 * Copyright (c) 2011, 2024 by Delphix. All rights reserved. 16 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 17 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 18 * Copyright 2013 Saso Kiselkov. All rights reserved. 19 * Copyright (c) 2014 Integros [integros.com] 20 * Copyright 2016 Toomas Soome <tsoome@me.com> 21 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 22 * Copyright 2018 Joyent, Inc. 23 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 24 * Copyright 2017 Joyent, Inc. 25 * Copyright (c) 2017, Intel Corporation. 26 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 27 * Copyright (c) 2023 Hewlett Packard Enterprise Development LP. 28 * Copyright (c) 2023-2026, Klara, Inc. 29 * Copyright (c) 2026, TrueNAS. 30 * Copyright 2026 Edgecast Cloud LLC. 31 */ 32 33 /* 34 * SPA: Storage Pool Allocator 35 * 36 * This file contains all the routines used when modifying on-disk SPA state. 37 * This includes opening, importing, destroying, exporting a pool, and syncing a 38 * pool. 39 */ 40 41 #include <sys/zfs_context.h> 42 #include <sys/fm/fs/zfs.h> 43 #include <sys/spa_impl.h> 44 #include <sys/zio.h> 45 #include <sys/zio_checksum.h> 46 #include <sys/dmu.h> 47 #include <sys/dmu_tx.h> 48 #include <sys/zap.h> 49 #include <sys/zil.h> 50 #include <sys/brt.h> 51 #include <sys/ddt.h> 52 #include <sys/vdev_impl.h> 53 #include <sys/vdev_removal.h> 54 #include <sys/vdev_indirect_mapping.h> 55 #include <sys/vdev_indirect_births.h> 56 #include <sys/vdev_initialize.h> 57 #include <sys/vdev_rebuild.h> 58 #include <sys/vdev_trim.h> 59 #include <sys/vdev_disk.h> 60 #include <sys/vdev_raidz.h> 61 #include <sys/vdev_draid.h> 62 #include <sys/metaslab.h> 63 #include <sys/metaslab_impl.h> 64 #include <sys/mmp.h> 65 #include <sys/uberblock_impl.h> 66 #include <sys/txg.h> 67 #include <sys/avl.h> 68 #include <sys/bpobj.h> 69 #include <sys/dmu_traverse.h> 70 #include <sys/dmu_objset.h> 71 #include <sys/unique.h> 72 #include <sys/dsl_pool.h> 73 #include <sys/dsl_dataset.h> 74 #include <sys/dsl_dir.h> 75 #include <sys/dsl_prop.h> 76 #include <sys/dsl_synctask.h> 77 #include <sys/fs/zfs.h> 78 #include <sys/arc.h> 79 #include <sys/callb.h> 80 #include <sys/systeminfo.h> 81 #include <sys/zfs_ioctl.h> 82 #include <sys/dsl_scan.h> 83 #include <sys/zfeature.h> 84 #include <sys/dsl_destroy.h> 85 #include <sys/zvol.h> 86 87 #ifdef _KERNEL 88 #include <sys/fm/protocol.h> 89 #include <sys/fm/util.h> 90 #include <sys/callb.h> 91 #include <sys/zone.h> 92 #include <sys/vmsystm.h> 93 #endif /* _KERNEL */ 94 95 #include "zfs_crrd.h" 96 #include "zfs_prop.h" 97 #include "zfs_comutil.h" 98 #include <cityhash.h> 99 100 /* 101 * spa_thread() existed on Illumos as a parent thread for the various worker 102 * threads that actually run the pool, as a way to both reference the entire 103 * pool work as a single object, and to share properties like scheduling 104 * options. It has not yet been adapted to Linux or FreeBSD. This define is 105 * used to mark related parts of the code to make things easier for the reader, 106 * and to compile this code out. It can be removed when someone implements it, 107 * moves it to some Illumos-specific place, or removes it entirely. 108 */ 109 #undef HAVE_SPA_THREAD 110 111 /* 112 * The "System Duty Cycle" scheduling class is an Illumos feature to help 113 * prevent CPU-intensive kernel threads from affecting latency on interactive 114 * threads. It doesn't exist on Linux or FreeBSD, so the supporting code is 115 * gated behind a define. On Illumos SDC depends on spa_thread(), but 116 * spa_thread() also has other uses, so this is a separate define. 117 */ 118 #undef HAVE_SYSDC 119 120 /* 121 * The interval, in seconds, at which failed configuration cache file writes 122 * should be retried. 123 */ 124 int zfs_ccw_retry_interval = 300; 125 126 typedef enum zti_modes { 127 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 128 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 129 ZTI_MODE_SYNC, /* sync thread assigned */ 130 ZTI_MODE_NULL, /* don't create a taskq */ 131 ZTI_NMODES 132 } zti_modes_t; 133 134 #define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 135 #define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 136 #define ZTI_SCALE(min) { ZTI_MODE_SCALE, (min), 1 } 137 #define ZTI_SYNC { ZTI_MODE_SYNC, 0, 1 } 138 #define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 139 140 #define ZTI_N(n) ZTI_P(n, 1) 141 #define ZTI_ONE ZTI_N(1) 142 143 typedef struct zio_taskq_info { 144 zti_modes_t zti_mode; 145 uint_t zti_value; 146 uint_t zti_count; 147 } zio_taskq_info_t; 148 149 static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 150 "iss", "iss_h", "int", "int_h" 151 }; 152 153 /* 154 * This table defines the taskq settings for each ZFS I/O type. When 155 * initializing a pool, we use this table to create an appropriately sized 156 * taskq. Some operations are low volume and therefore have a small, static 157 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 158 * macros. Other operations process a large amount of data; the ZTI_SCALE 159 * macro causes us to create a taskq oriented for throughput. Some operations 160 * are so high frequency and short-lived that the taskq itself can become a 161 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 162 * additional degree of parallelism specified by the number of threads per- 163 * taskq and the number of taskqs; when dispatching an event in this case, the 164 * particular taskq is chosen at random. ZTI_SCALE uses a number of taskqs 165 * that scales with the number of CPUs. 166 * 167 * The different taskq priorities are to handle the different contexts (issue 168 * and interrupt) and then to reserve threads for high priority I/Os that 169 * need to be handled with minimum delay. Illumos taskq has unfair TQ_FRONT 170 * implementation, so separate high priority threads are used there. 171 */ 172 static zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 173 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 174 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 175 { ZTI_N(8), ZTI_NULL, ZTI_SCALE(0), ZTI_NULL }, /* READ */ 176 #ifdef illumos 177 { ZTI_SYNC, ZTI_N(5), ZTI_SCALE(0), ZTI_N(5) }, /* WRITE */ 178 #else 179 { ZTI_SYNC, ZTI_NULL, ZTI_SCALE(0), ZTI_NULL }, /* WRITE */ 180 #endif 181 { ZTI_SCALE(32), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 182 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 183 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FLUSH */ 184 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 185 }; 186 187 static void spa_sync_version(void *arg, dmu_tx_t *tx); 188 static void spa_sync_props(void *arg, dmu_tx_t *tx); 189 static boolean_t spa_has_active_shared_spare(spa_t *spa); 190 static int spa_load_impl(spa_t *spa, spa_import_type_t type, 191 const char **ereport); 192 static void spa_vdev_resilver_done(spa_t *spa); 193 194 /* 195 * Percentage of all CPUs that can be used by the metaslab preload taskq. 196 */ 197 static uint_t metaslab_preload_pct = 50; 198 199 static uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 200 static uint_t zio_taskq_batch_tpq; /* threads per taskq */ 201 202 #ifdef HAVE_SYSDC 203 static const boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 204 static const uint_t zio_taskq_basedc = 80; /* base duty cycle */ 205 #endif 206 207 #ifdef HAVE_SPA_THREAD 208 static const boolean_t spa_create_process = B_TRUE; /* no process => no sysdc */ 209 #endif 210 211 static uint_t zio_taskq_write_tpq = 16; 212 213 /* 214 * Report any spa_load_verify errors found, but do not fail spa_load. 215 * This is used by zdb to analyze non-idle pools. 216 */ 217 boolean_t spa_load_verify_dryrun = B_FALSE; 218 219 /* 220 * Allow read spacemaps in case of readonly import (spa_mode == SPA_MODE_READ). 221 * This is used by zdb for spacemaps verification. 222 */ 223 boolean_t spa_mode_readable_spacemaps = B_FALSE; 224 225 /* 226 * This (illegal) pool name is used when temporarily importing a spa_t in order 227 * to get the vdev stats associated with the imported devices. 228 */ 229 #define TRYIMPORT_NAME "$import" 230 231 /* 232 * For debugging purposes: print out vdev tree during pool import. 233 */ 234 static int spa_load_print_vdev_tree = B_FALSE; 235 236 /* 237 * A non-zero value for zfs_max_missing_tvds means that we allow importing 238 * pools with missing top-level vdevs. This is strictly intended for advanced 239 * pool recovery cases since missing data is almost inevitable. Pools with 240 * missing devices can only be imported read-only for safety reasons, and their 241 * fail-mode will be automatically set to "continue". 242 * 243 * With 1 missing vdev we should be able to import the pool and mount all 244 * datasets. User data that was not modified after the missing device has been 245 * added should be recoverable. This means that snapshots created prior to the 246 * addition of that device should be completely intact. 247 * 248 * With 2 missing vdevs, some datasets may fail to mount since there are 249 * dataset statistics that are stored as regular metadata. Some data might be 250 * recoverable if those vdevs were added recently. 251 * 252 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 253 * may be missing entirely. Chances of data recovery are very low. Note that 254 * there are also risks of performing an inadvertent rewind as we might be 255 * missing all the vdevs with the latest uberblocks. 256 */ 257 uint64_t zfs_max_missing_tvds = 0; 258 259 /* 260 * The parameters below are similar to zfs_max_missing_tvds but are only 261 * intended for a preliminary open of the pool with an untrusted config which 262 * might be incomplete or out-dated. 263 * 264 * We are more tolerant for pools opened from a cachefile since we could have 265 * an out-dated cachefile where a device removal was not registered. 266 * We could have set the limit arbitrarily high but in the case where devices 267 * are really missing we would want to return the proper error codes; we chose 268 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 269 * and we get a chance to retrieve the trusted config. 270 */ 271 uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 272 273 /* 274 * In the case where config was assembled by scanning device paths (/dev/dsks 275 * by default) we are less tolerant since all the existing devices should have 276 * been detected and we want spa_load to return the right error codes. 277 */ 278 uint64_t zfs_max_missing_tvds_scan = 0; 279 280 /* 281 * Debugging aid that pauses spa_sync() towards the end. 282 */ 283 static const boolean_t zfs_pause_spa_sync = B_FALSE; 284 285 /* 286 * Variables to indicate the livelist condense zthr func should wait at certain 287 * points for the livelist to be removed - used to test condense/destroy races 288 */ 289 static int zfs_livelist_condense_zthr_pause = 0; 290 static int zfs_livelist_condense_sync_pause = 0; 291 292 /* 293 * Variables to track whether or not condense cancellation has been 294 * triggered in testing. 295 */ 296 static int zfs_livelist_condense_sync_cancel = 0; 297 static int zfs_livelist_condense_zthr_cancel = 0; 298 299 /* 300 * Variable to track whether or not extra ALLOC blkptrs were added to a 301 * livelist entry while it was being condensed (caused by the way we track 302 * remapped blkptrs in dbuf_remap_impl) 303 */ 304 static int zfs_livelist_condense_new_alloc = 0; 305 306 /* 307 * Time variable to decide how often the txg should be added into the 308 * database (in seconds). 309 * The smallest available resolution is in minutes, which means an update occurs 310 * each time we reach `spa_note_txg_time` and the txg has changed. We provide 311 * a 256-slot ring buffer for minute-level resolution. The number is limited by 312 * the size of the structure we use and the maximum amount of bytes we can write 313 * into ZAP. Setting `spa_note_txg_time` to 10 minutes results in approximately 314 * 144 records per day. Given the 256 slots, this provides roughly 1.5 days of 315 * high-resolution data. 316 * 317 * The user can decrease `spa_note_txg_time` to increase resolution within 318 * a day, at the cost of retaining fewer days of data. Alternatively, increasing 319 * the interval allows storing data over a longer period, but with lower 320 * frequency. 321 * 322 * This parameter does not affect the daily or monthly databases, as those only 323 * store one record per day and per month, respectively. 324 */ 325 static uint_t spa_note_txg_time = 10 * 60; 326 327 /* 328 * How often flush txg database to a disk (in seconds). 329 * We flush data every time we write to it, making it the most reliable option. 330 * Since this happens every 10 minutes, it shouldn't introduce any noticeable 331 * overhead for the system. In case of failure, we will always have an 332 * up-to-date version of the database. 333 * 334 * The user can adjust the flush interval to a lower value, but it probably 335 * doesn't make sense to flush more often than the database is updated. 336 * The user can also increase the interval if they're concerned about the 337 * performance of writing the entire database to disk. 338 */ 339 static uint_t spa_flush_txg_time = 10 * 60; 340 341 /* 342 * ========================================================================== 343 * SPA properties routines 344 * ========================================================================== 345 */ 346 347 /* 348 * Add a (source=src, propname=propval) list to an nvlist. 349 */ 350 static void 351 spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, const char *strval, 352 uint64_t intval, zprop_source_t src) 353 { 354 const char *propname = zpool_prop_to_name(prop); 355 nvlist_t *propval; 356 357 propval = fnvlist_alloc(); 358 fnvlist_add_uint64(propval, ZPROP_SOURCE, src); 359 360 if (strval != NULL) 361 fnvlist_add_string(propval, ZPROP_VALUE, strval); 362 else 363 fnvlist_add_uint64(propval, ZPROP_VALUE, intval); 364 365 fnvlist_add_nvlist(nvl, propname, propval); 366 nvlist_free(propval); 367 } 368 369 static int 370 spa_prop_add(spa_t *spa, const char *propname, nvlist_t *outnvl) 371 { 372 zpool_prop_t prop = zpool_name_to_prop(propname); 373 zprop_source_t src = ZPROP_SRC_NONE; 374 uint64_t intval; 375 int err; 376 377 /* 378 * NB: Not all properties lookups via this API require 379 * the spa props lock, so they must explicitly grab it here. 380 */ 381 switch (prop) { 382 case ZPOOL_PROP_DEDUPCACHED: 383 err = ddt_get_pool_dedup_cached(spa, &intval); 384 if (err != 0) 385 return (SET_ERROR(err)); 386 break; 387 default: 388 return (SET_ERROR(EINVAL)); 389 } 390 391 spa_prop_add_list(outnvl, prop, NULL, intval, src); 392 393 return (0); 394 } 395 396 int 397 spa_prop_get_nvlist(spa_t *spa, char **props, unsigned int n_props, 398 nvlist_t *outnvl) 399 { 400 int err = 0; 401 402 if (props == NULL) 403 return (0); 404 405 for (unsigned int i = 0; i < n_props && err == 0; i++) { 406 err = spa_prop_add(spa, props[i], outnvl); 407 } 408 409 return (err); 410 } 411 412 /* 413 * Add metaslab class properties to an nvlist. 414 */ 415 static void 416 spa_prop_add_metaslab_class(nvlist_t *nv, metaslab_class_t *mc, 417 zpool_mc_props_t mcp, uint64_t *sizep, uint64_t *allocp, uint64_t *usablep, 418 uint64_t *usedp) 419 { 420 uint64_t size = metaslab_class_get_space(mc); 421 uint64_t alloc = metaslab_class_get_alloc(mc); 422 uint64_t dsize = metaslab_class_get_dspace(mc); 423 uint64_t dalloc = metaslab_class_get_dalloc(mc); 424 uint64_t cap = (size == 0) ? 0 : (alloc * 100 / size); 425 const zprop_source_t src = ZPROP_SRC_NONE; 426 427 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_SIZE, NULL, size, src); 428 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_ALLOCATED, NULL, alloc, src); 429 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_USABLE, NULL, dsize, src); 430 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_USED, NULL, dalloc, src); 431 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_FRAGMENTATION, NULL, 432 metaslab_class_fragmentation(mc), src); 433 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_EXPANDSZ, NULL, 434 metaslab_class_expandable_space(mc), src); 435 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_FREE, NULL, size - alloc, 436 src); 437 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_AVAILABLE, NULL, 438 dsize - dalloc, src); 439 spa_prop_add_list(nv, mcp + ZPOOL_MC_PROP_CAPACITY, NULL, cap, src); 440 if (sizep != NULL) 441 *sizep += size; 442 if (allocp != NULL) 443 *allocp += alloc; 444 if (usablep != NULL) 445 *usablep += dsize; 446 if (usedp != NULL) 447 *usedp += dalloc; 448 } 449 450 /* 451 * Add a user property (source=src, propname=propval) to an nvlist. 452 */ 453 static void 454 spa_prop_add_user(nvlist_t *nvl, const char *propname, char *strval, 455 zprop_source_t src) 456 { 457 nvlist_t *propval; 458 459 VERIFY0(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP)); 460 VERIFY0(nvlist_add_uint64(propval, ZPROP_SOURCE, src)); 461 VERIFY0(nvlist_add_string(propval, ZPROP_VALUE, strval)); 462 VERIFY0(nvlist_add_nvlist(nvl, propname, propval)); 463 nvlist_free(propval); 464 } 465 466 /* 467 * Get property values from the spa configuration. 468 */ 469 static void 470 spa_prop_get_config(spa_t *spa, nvlist_t *nv) 471 { 472 vdev_t *rvd = spa->spa_root_vdev; 473 dsl_pool_t *pool = spa->spa_dsl_pool; 474 uint64_t size, alloc, usable, used, cap, version; 475 const zprop_source_t src = ZPROP_SRC_NONE; 476 spa_config_dirent_t *dp; 477 metaslab_class_t *mc = spa_normal_class(spa); 478 479 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 480 481 if (rvd != NULL) { 482 spa_prop_add_list(nv, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 483 484 size = alloc = usable = used = 0; 485 spa_prop_add_metaslab_class(nv, mc, ZPOOL_MC_PROPS_NORMAL, 486 &size, &alloc, &usable, &used); 487 spa_prop_add_metaslab_class(nv, spa_special_class(spa), 488 ZPOOL_MC_PROPS_SPECIAL, &size, &alloc, &usable, &used); 489 spa_prop_add_metaslab_class(nv, spa_dedup_class(spa), 490 ZPOOL_MC_PROPS_DEDUP, &size, &alloc, &usable, &used); 491 spa_prop_add_metaslab_class(nv, spa_log_class(spa), 492 ZPOOL_MC_PROPS_LOG, NULL, NULL, NULL, NULL); 493 spa_prop_add_metaslab_class(nv, spa_embedded_log_class(spa), 494 ZPOOL_MC_PROPS_ELOG, &size, &alloc, &usable, &used); 495 spa_prop_add_metaslab_class(nv, 496 spa_special_embedded_log_class(spa), ZPOOL_MC_PROPS_SELOG, 497 &size, &alloc, &usable, &used); 498 499 spa_prop_add_list(nv, ZPOOL_PROP_SIZE, NULL, size, src); 500 spa_prop_add_list(nv, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 501 spa_prop_add_list(nv, ZPOOL_PROP_FREE, NULL, 502 size - alloc, src); 503 spa_prop_add_list(nv, ZPOOL_PROP_FRAGMENTATION, NULL, 504 metaslab_class_fragmentation(mc), src); 505 spa_prop_add_list(nv, ZPOOL_PROP_EXPANDSZ, NULL, 506 metaslab_class_expandable_space(mc), src); 507 cap = (size == 0) ? 0 : (alloc * 100 / size); 508 spa_prop_add_list(nv, ZPOOL_PROP_CAPACITY, NULL, cap, src); 509 spa_prop_add_list(nv, ZPOOL_PROP_AVAILABLE, NULL, usable - used, 510 src); 511 spa_prop_add_list(nv, ZPOOL_PROP_USABLE, NULL, usable, src); 512 spa_prop_add_list(nv, ZPOOL_PROP_USED, NULL, used, src); 513 514 spa_prop_add_list(nv, ZPOOL_PROP_CHECKPOINT, NULL, 515 spa->spa_checkpoint_info.sci_dspace, src); 516 spa_prop_add_list(nv, ZPOOL_PROP_READONLY, NULL, 517 (spa_mode(spa) == SPA_MODE_READ), src); 518 519 spa_prop_add_list(nv, ZPOOL_PROP_DEDUPRATIO, NULL, 520 ddt_get_pool_dedup_ratio(spa), src); 521 spa_prop_add_list(nv, ZPOOL_PROP_DEDUPUSED, NULL, 522 ddt_get_dedup_used(spa), src); 523 spa_prop_add_list(nv, ZPOOL_PROP_DEDUPSAVED, NULL, 524 ddt_get_dedup_saved(spa), src); 525 spa_prop_add_list(nv, ZPOOL_PROP_BCLONEUSED, NULL, 526 brt_get_used(spa), src); 527 spa_prop_add_list(nv, ZPOOL_PROP_BCLONESAVED, NULL, 528 brt_get_saved(spa), src); 529 spa_prop_add_list(nv, ZPOOL_PROP_BCLONERATIO, NULL, 530 brt_get_ratio(spa), src); 531 532 spa_prop_add_list(nv, ZPOOL_PROP_DEDUP_TABLE_SIZE, NULL, 533 ddt_get_ddt_dsize(spa), src); 534 spa_prop_add_list(nv, ZPOOL_PROP_HEALTH, NULL, 535 rvd->vdev_state, src); 536 spa_prop_add_list(nv, ZPOOL_PROP_LAST_SCRUBBED_TXG, NULL, 537 spa_get_last_scrubbed_txg(spa), src); 538 539 version = spa_version(spa); 540 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 541 spa_prop_add_list(nv, ZPOOL_PROP_VERSION, NULL, 542 version, ZPROP_SRC_DEFAULT); 543 } else { 544 spa_prop_add_list(nv, ZPOOL_PROP_VERSION, NULL, 545 version, ZPROP_SRC_LOCAL); 546 } 547 spa_prop_add_list(nv, ZPOOL_PROP_LOAD_GUID, 548 NULL, spa_load_guid(spa), src); 549 } 550 551 if (pool != NULL) { 552 /* 553 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 554 * when opening pools before this version freedir will be NULL. 555 */ 556 if (pool->dp_free_dir != NULL) { 557 spa_prop_add_list(nv, ZPOOL_PROP_FREEING, NULL, 558 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 559 src); 560 } else { 561 spa_prop_add_list(nv, ZPOOL_PROP_FREEING, 562 NULL, 0, src); 563 } 564 565 if (pool->dp_leak_dir != NULL) { 566 spa_prop_add_list(nv, ZPOOL_PROP_LEAKED, NULL, 567 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 568 src); 569 } else { 570 spa_prop_add_list(nv, ZPOOL_PROP_LEAKED, 571 NULL, 0, src); 572 } 573 } 574 575 spa_prop_add_list(nv, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 576 577 if (spa->spa_comment != NULL) { 578 spa_prop_add_list(nv, ZPOOL_PROP_COMMENT, spa->spa_comment, 579 0, ZPROP_SRC_LOCAL); 580 } 581 582 if (spa->spa_compatibility != NULL) { 583 spa_prop_add_list(nv, ZPOOL_PROP_COMPATIBILITY, 584 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 585 } 586 587 if (spa->spa_root != NULL) 588 spa_prop_add_list(nv, ZPOOL_PROP_ALTROOT, spa->spa_root, 589 0, ZPROP_SRC_LOCAL); 590 591 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 592 spa_prop_add_list(nv, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 593 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 594 } else { 595 spa_prop_add_list(nv, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 596 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 597 } 598 599 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 600 spa_prop_add_list(nv, ZPOOL_PROP_MAXDNODESIZE, NULL, 601 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 602 } else { 603 spa_prop_add_list(nv, ZPOOL_PROP_MAXDNODESIZE, NULL, 604 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 605 } 606 607 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 608 if (dp->scd_path == NULL) { 609 spa_prop_add_list(nv, ZPOOL_PROP_CACHEFILE, 610 "none", 0, ZPROP_SRC_LOCAL); 611 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 612 spa_prop_add_list(nv, ZPOOL_PROP_CACHEFILE, 613 dp->scd_path, 0, ZPROP_SRC_LOCAL); 614 } 615 } 616 } 617 618 /* 619 * Get zpool property values. 620 */ 621 int 622 spa_prop_get(spa_t *spa, nvlist_t *nv) 623 { 624 objset_t *mos = spa->spa_meta_objset; 625 zap_cursor_t zc; 626 zap_attribute_t *za; 627 dsl_pool_t *dp; 628 int err = 0; 629 630 dp = spa_get_dsl(spa); 631 dsl_pool_config_enter(dp, FTAG); 632 za = zap_attribute_alloc(); 633 mutex_enter(&spa->spa_props_lock); 634 635 /* 636 * Get properties from the spa config. 637 */ 638 spa_prop_get_config(spa, nv); 639 640 /* If no pool property object, no more prop to get. */ 641 if (mos == NULL || spa->spa_pool_props_object == 0) 642 goto out; 643 644 /* 645 * Get properties from the MOS pool property object. 646 */ 647 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 648 (err = zap_cursor_retrieve(&zc, za)) == 0; 649 zap_cursor_advance(&zc)) { 650 uint64_t intval = 0; 651 char *strval = NULL; 652 zprop_source_t src = ZPROP_SRC_DEFAULT; 653 zpool_prop_t prop; 654 655 if ((prop = zpool_name_to_prop(za->za_name)) == 656 ZPOOL_PROP_INVAL && !zfs_prop_user(za->za_name)) 657 continue; 658 659 switch (za->za_integer_length) { 660 case 8: 661 /* integer property */ 662 if (za->za_first_integer != 663 zpool_prop_default_numeric(prop)) 664 src = ZPROP_SRC_LOCAL; 665 666 if (prop == ZPOOL_PROP_BOOTFS) { 667 dsl_dataset_t *ds = NULL; 668 669 err = dsl_dataset_hold_obj(dp, 670 za->za_first_integer, FTAG, &ds); 671 if (err != 0) 672 break; 673 674 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 675 KM_SLEEP); 676 dsl_dataset_name(ds, strval); 677 dsl_dataset_rele(ds, FTAG); 678 } else { 679 strval = NULL; 680 intval = za->za_first_integer; 681 } 682 683 spa_prop_add_list(nv, prop, strval, intval, src); 684 685 if (strval != NULL) 686 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 687 688 break; 689 690 case 1: 691 /* string property */ 692 strval = kmem_alloc(za->za_num_integers, KM_SLEEP); 693 err = zap_lookup(mos, spa->spa_pool_props_object, 694 za->za_name, 1, za->za_num_integers, strval); 695 if (err) { 696 kmem_free(strval, za->za_num_integers); 697 break; 698 } 699 if (prop != ZPOOL_PROP_INVAL) { 700 spa_prop_add_list(nv, prop, strval, 0, src); 701 } else { 702 src = ZPROP_SRC_LOCAL; 703 spa_prop_add_user(nv, za->za_name, strval, 704 src); 705 } 706 kmem_free(strval, za->za_num_integers); 707 break; 708 709 default: 710 break; 711 } 712 } 713 zap_cursor_fini(&zc); 714 out: 715 mutex_exit(&spa->spa_props_lock); 716 dsl_pool_config_exit(dp, FTAG); 717 zap_attribute_free(za); 718 719 if (err && err != ENOENT) 720 return (err); 721 722 return (0); 723 } 724 725 /* 726 * Validate the given pool properties nvlist and modify the list 727 * for the property values to be set. 728 */ 729 static int 730 spa_prop_validate(spa_t *spa, nvlist_t *props) 731 { 732 nvpair_t *elem; 733 int error = 0, reset_bootfs = 0; 734 uint64_t objnum = 0; 735 boolean_t has_feature = B_FALSE; 736 737 elem = NULL; 738 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 739 uint64_t intval; 740 const char *strval, *slash, *check, *fname; 741 const char *propname = nvpair_name(elem); 742 zpool_prop_t prop = zpool_name_to_prop(propname); 743 744 switch (prop) { 745 case ZPOOL_PROP_INVAL: 746 /* 747 * Sanitize the input. 748 */ 749 if (zfs_prop_user(propname)) { 750 if (strlen(propname) >= ZAP_MAXNAMELEN) { 751 error = SET_ERROR(ENAMETOOLONG); 752 break; 753 } 754 755 if (strlen(fnvpair_value_string(elem)) >= 756 ZAP_MAXVALUELEN) { 757 error = SET_ERROR(E2BIG); 758 break; 759 } 760 } else if (zpool_prop_feature(propname)) { 761 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 762 error = SET_ERROR(EINVAL); 763 break; 764 } 765 766 if (nvpair_value_uint64(elem, &intval) != 0) { 767 error = SET_ERROR(EINVAL); 768 break; 769 } 770 771 if (intval != 0) { 772 error = SET_ERROR(EINVAL); 773 break; 774 } 775 776 fname = strchr(propname, '@') + 1; 777 if (zfeature_lookup_name(fname, NULL) != 0) { 778 error = SET_ERROR(EINVAL); 779 break; 780 } 781 782 has_feature = B_TRUE; 783 } else { 784 error = SET_ERROR(EINVAL); 785 break; 786 } 787 break; 788 789 case ZPOOL_PROP_VERSION: 790 error = nvpair_value_uint64(elem, &intval); 791 if (!error && 792 (intval < spa_version(spa) || 793 intval > SPA_VERSION_BEFORE_FEATURES || 794 has_feature)) 795 error = SET_ERROR(EINVAL); 796 break; 797 798 case ZPOOL_PROP_DEDUP_TABLE_QUOTA: 799 error = nvpair_value_uint64(elem, &intval); 800 break; 801 802 case ZPOOL_PROP_DELEGATION: 803 case ZPOOL_PROP_AUTOREPLACE: 804 case ZPOOL_PROP_LISTSNAPS: 805 case ZPOOL_PROP_AUTOEXPAND: 806 case ZPOOL_PROP_AUTOTRIM: 807 error = nvpair_value_uint64(elem, &intval); 808 if (!error && intval > 1) 809 error = SET_ERROR(EINVAL); 810 break; 811 812 case ZPOOL_PROP_MULTIHOST: 813 error = nvpair_value_uint64(elem, &intval); 814 if (!error && intval > 1) 815 error = SET_ERROR(EINVAL); 816 817 if (!error) { 818 uint32_t hostid = zone_get_hostid(NULL); 819 if (hostid) 820 spa->spa_hostid = hostid; 821 else 822 error = SET_ERROR(ENOTSUP); 823 } 824 825 break; 826 827 case ZPOOL_PROP_BOOTFS: 828 /* 829 * If the pool version is less than SPA_VERSION_BOOTFS, 830 * or the pool is still being created (version == 0), 831 * the bootfs property cannot be set. 832 */ 833 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 834 error = SET_ERROR(ENOTSUP); 835 break; 836 } 837 838 /* 839 * Make sure the vdev config is bootable 840 */ 841 if (!vdev_is_bootable(spa->spa_root_vdev)) { 842 error = SET_ERROR(ENOTSUP); 843 break; 844 } 845 846 reset_bootfs = 1; 847 848 error = nvpair_value_string(elem, &strval); 849 850 if (!error) { 851 objset_t *os; 852 853 if (strval == NULL || strval[0] == '\0') { 854 objnum = zpool_prop_default_numeric( 855 ZPOOL_PROP_BOOTFS); 856 break; 857 } 858 859 error = dmu_objset_hold(strval, FTAG, &os); 860 if (error != 0) 861 break; 862 863 /* Must be ZPL. */ 864 if (dmu_objset_type(os) != DMU_OST_ZFS) { 865 error = SET_ERROR(ENOTSUP); 866 } else { 867 objnum = dmu_objset_id(os); 868 } 869 dmu_objset_rele(os, FTAG); 870 } 871 break; 872 873 case ZPOOL_PROP_FAILUREMODE: 874 error = nvpair_value_uint64(elem, &intval); 875 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 876 error = SET_ERROR(EINVAL); 877 878 /* 879 * This is a special case which only occurs when 880 * the pool has completely failed. This allows 881 * the user to change the in-core failmode property 882 * without syncing it out to disk (I/Os might 883 * currently be blocked). We do this by returning 884 * EIO to the caller (spa_prop_set) to trick it 885 * into thinking we encountered a property validation 886 * error. 887 */ 888 if (!error && spa_suspended(spa)) { 889 spa->spa_failmode = intval; 890 error = SET_ERROR(EIO); 891 } 892 break; 893 894 case ZPOOL_PROP_CACHEFILE: 895 if ((error = nvpair_value_string(elem, &strval)) != 0) 896 break; 897 898 if (strval[0] == '\0') 899 break; 900 901 if (strcmp(strval, "none") == 0) 902 break; 903 904 if (strval[0] != '/') { 905 error = SET_ERROR(EINVAL); 906 break; 907 } 908 909 slash = strrchr(strval, '/'); 910 ASSERT(slash != NULL); 911 912 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 913 strcmp(slash, "/..") == 0) 914 error = SET_ERROR(EINVAL); 915 break; 916 917 case ZPOOL_PROP_COMMENT: 918 if ((error = nvpair_value_string(elem, &strval)) != 0) 919 break; 920 for (check = strval; *check != '\0'; check++) { 921 if (!isprint(*check)) { 922 error = SET_ERROR(EINVAL); 923 break; 924 } 925 } 926 if (strlen(strval) > ZPROP_MAX_COMMENT) 927 error = SET_ERROR(E2BIG); 928 break; 929 930 default: 931 break; 932 } 933 934 if (error) 935 break; 936 } 937 938 (void) nvlist_remove_all(props, 939 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 940 941 if (!error && reset_bootfs) { 942 error = nvlist_remove(props, 943 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 944 945 if (!error) { 946 error = nvlist_add_uint64(props, 947 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 948 } 949 } 950 951 return (error); 952 } 953 954 void 955 spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 956 { 957 const char *cachefile; 958 spa_config_dirent_t *dp; 959 960 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 961 &cachefile) != 0) 962 return; 963 964 dp = kmem_alloc(sizeof (spa_config_dirent_t), 965 KM_SLEEP); 966 967 if (cachefile[0] == '\0') 968 dp->scd_path = spa_strdup(spa_config_path); 969 else if (strcmp(cachefile, "none") == 0) 970 dp->scd_path = NULL; 971 else 972 dp->scd_path = spa_strdup(cachefile); 973 974 list_insert_head(&spa->spa_config_list, dp); 975 if (need_sync) 976 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 977 } 978 979 int 980 spa_prop_set(spa_t *spa, nvlist_t *nvp) 981 { 982 int error; 983 nvpair_t *elem = NULL; 984 boolean_t need_sync = B_FALSE; 985 986 if ((error = spa_prop_validate(spa, nvp)) != 0) 987 return (error); 988 989 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 990 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 991 992 if (prop == ZPOOL_PROP_CACHEFILE || 993 prop == ZPOOL_PROP_ALTROOT || 994 prop == ZPOOL_PROP_READONLY) 995 continue; 996 997 if (prop == ZPOOL_PROP_INVAL && 998 zfs_prop_user(nvpair_name(elem))) { 999 need_sync = B_TRUE; 1000 break; 1001 } 1002 1003 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 1004 uint64_t ver = 0; 1005 1006 if (prop == ZPOOL_PROP_VERSION) { 1007 VERIFY0(nvpair_value_uint64(elem, &ver)); 1008 } else { 1009 ASSERT(zpool_prop_feature(nvpair_name(elem))); 1010 ver = SPA_VERSION_FEATURES; 1011 need_sync = B_TRUE; 1012 } 1013 1014 /* Save time if the version is already set. */ 1015 if (ver == spa_version(spa)) 1016 continue; 1017 1018 /* 1019 * In addition to the pool directory object, we might 1020 * create the pool properties object, the features for 1021 * read object, the features for write object, or the 1022 * feature descriptions object. 1023 */ 1024 error = dsl_sync_task(spa->spa_name, NULL, 1025 spa_sync_version, &ver, 1026 6, ZFS_SPACE_CHECK_RESERVED); 1027 if (error) 1028 return (error); 1029 continue; 1030 } 1031 1032 need_sync = B_TRUE; 1033 break; 1034 } 1035 1036 if (need_sync) { 1037 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 1038 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 1039 } 1040 1041 return (0); 1042 } 1043 1044 /* 1045 * If the bootfs property value is dsobj, clear it. 1046 */ 1047 void 1048 spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 1049 { 1050 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 1051 VERIFY(zap_remove(spa->spa_meta_objset, 1052 spa->spa_pool_props_object, 1053 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 1054 spa->spa_bootfs = 0; 1055 } 1056 } 1057 1058 static int 1059 spa_change_guid_check(void *arg, dmu_tx_t *tx) 1060 { 1061 uint64_t *newguid __maybe_unused = arg; 1062 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 1063 vdev_t *rvd = spa->spa_root_vdev; 1064 uint64_t vdev_state; 1065 1066 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 1067 int error = (spa_has_checkpoint(spa)) ? 1068 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 1069 return (SET_ERROR(error)); 1070 } 1071 1072 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 1073 vdev_state = rvd->vdev_state; 1074 spa_config_exit(spa, SCL_STATE, FTAG); 1075 1076 if (vdev_state != VDEV_STATE_HEALTHY) 1077 return (SET_ERROR(ENXIO)); 1078 1079 ASSERT3U(spa_guid(spa), !=, *newguid); 1080 1081 return (0); 1082 } 1083 1084 static void 1085 spa_change_guid_sync(void *arg, dmu_tx_t *tx) 1086 { 1087 uint64_t *newguid = arg; 1088 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 1089 uint64_t oldguid; 1090 vdev_t *rvd = spa->spa_root_vdev; 1091 1092 oldguid = spa_guid(spa); 1093 1094 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 1095 rvd->vdev_guid = *newguid; 1096 rvd->vdev_guid_sum += (*newguid - oldguid); 1097 vdev_config_dirty(rvd); 1098 spa_config_exit(spa, SCL_STATE, FTAG); 1099 1100 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 1101 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 1102 } 1103 1104 /* 1105 * Change the GUID for the pool. This is done so that we can later 1106 * re-import a pool built from a clone of our own vdevs. We will modify 1107 * the root vdev's guid, our own pool guid, and then mark all of our 1108 * vdevs dirty. Note that we must make sure that all our vdevs are 1109 * online when we do this, or else any vdevs that weren't present 1110 * would be orphaned from our pool. We are also going to issue a 1111 * sysevent to update any watchers. 1112 * 1113 * The GUID of the pool will be changed to the value pointed to by guidp. 1114 * The GUID may not be set to the reserverd value of 0. 1115 * The new GUID will be generated if guidp is NULL. 1116 */ 1117 int 1118 spa_change_guid(spa_t *spa, const uint64_t *guidp) 1119 { 1120 uint64_t guid; 1121 int error; 1122 1123 mutex_enter(&spa->spa_vdev_top_lock); 1124 spa_namespace_enter(FTAG); 1125 1126 if (guidp != NULL) { 1127 guid = *guidp; 1128 if (guid == 0) { 1129 error = SET_ERROR(EINVAL); 1130 goto out; 1131 } 1132 1133 if (spa_guid_exists(guid, 0)) { 1134 error = SET_ERROR(EEXIST); 1135 goto out; 1136 } 1137 } else { 1138 guid = spa_generate_guid(NULL); 1139 } 1140 1141 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 1142 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 1143 1144 if (error == 0) { 1145 /* 1146 * Clear the kobj flag from all the vdevs to allow 1147 * vdev_cache_process_kobj_evt() to post events to all the 1148 * vdevs since GUID is updated. 1149 */ 1150 vdev_clear_kobj_evt(spa->spa_root_vdev); 1151 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 1152 vdev_clear_kobj_evt(spa->spa_l2cache.sav_vdevs[i]); 1153 1154 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 1155 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 1156 } 1157 1158 out: 1159 spa_namespace_exit(FTAG); 1160 mutex_exit(&spa->spa_vdev_top_lock); 1161 1162 return (error); 1163 } 1164 1165 /* 1166 * ========================================================================== 1167 * SPA state manipulation (open/create/destroy/import/export) 1168 * ========================================================================== 1169 */ 1170 1171 static int 1172 spa_error_entry_compare(const void *a, const void *b) 1173 { 1174 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 1175 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 1176 int ret; 1177 1178 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 1179 sizeof (zbookmark_phys_t)); 1180 1181 return (TREE_ISIGN(ret)); 1182 } 1183 1184 /* 1185 * Utility function which retrieves copies of the current logs and 1186 * re-initializes them in the process. 1187 */ 1188 void 1189 spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 1190 { 1191 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 1192 1193 memcpy(last, &spa->spa_errlist_last, sizeof (avl_tree_t)); 1194 memcpy(scrub, &spa->spa_errlist_scrub, sizeof (avl_tree_t)); 1195 1196 avl_create(&spa->spa_errlist_scrub, 1197 spa_error_entry_compare, sizeof (spa_error_entry_t), 1198 offsetof(spa_error_entry_t, se_avl)); 1199 avl_create(&spa->spa_errlist_last, 1200 spa_error_entry_compare, sizeof (spa_error_entry_t), 1201 offsetof(spa_error_entry_t, se_avl)); 1202 } 1203 1204 static void 1205 spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1206 { 1207 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 1208 enum zti_modes mode = ztip->zti_mode; 1209 uint_t value = ztip->zti_value; 1210 uint_t count = ztip->zti_count; 1211 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1212 uint_t cpus, threads, flags = TASKQ_DYNAMIC; 1213 1214 switch (mode) { 1215 case ZTI_MODE_FIXED: 1216 ASSERT3U(value, >, 0); 1217 break; 1218 1219 case ZTI_MODE_SYNC: 1220 1221 /* 1222 * Create one wr_iss taskq for every 'zio_taskq_write_tpq' CPUs, 1223 * not to exceed the number of spa allocators, and align to it. 1224 */ 1225 threads = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 1226 count = MAX(1, threads / MAX(1, zio_taskq_write_tpq)); 1227 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1228 count = MIN(count, spa->spa_alloc_count); 1229 while (spa->spa_alloc_count % count != 0 && 1230 spa->spa_alloc_count < count * 2) 1231 count--; 1232 1233 /* 1234 * zio_taskq_batch_pct is unbounded and may exceed 100%, but no 1235 * single taskq may have more threads than 100% of online cpus. 1236 */ 1237 value = (zio_taskq_batch_pct + count / 2) / count; 1238 value = MIN(value, 100); 1239 flags |= TASKQ_THREADS_CPU_PCT; 1240 break; 1241 1242 case ZTI_MODE_SCALE: 1243 /* 1244 * We want more taskqs to reduce lock contention, but we want 1245 * less for better request ordering and CPU utilization. 1246 */ 1247 threads = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 1248 threads = MAX(threads, value); 1249 if (zio_taskq_batch_tpq > 0) { 1250 count = MAX(1, (threads + zio_taskq_batch_tpq / 2) / 1251 zio_taskq_batch_tpq); 1252 } else { 1253 /* 1254 * Prefer 6 threads per taskq, but no more taskqs 1255 * than threads in them on large systems. For 80%: 1256 * 1257 * taskq taskq total 1258 * cpus taskqs percent threads threads 1259 * ------- ------- ------- ------- ------- 1260 * 1 1 80% 1 1 1261 * 2 1 80% 1 1 1262 * 4 1 80% 3 3 1263 * 8 2 40% 3 6 1264 * 16 3 27% 4 12 1265 * 32 5 16% 5 25 1266 * 64 7 11% 7 49 1267 * 128 10 8% 10 100 1268 * 256 14 6% 15 210 1269 */ 1270 cpus = MIN(threads, boot_ncpus); 1271 count = 1 + threads / 6; 1272 while (count * count > cpus) 1273 count--; 1274 } 1275 1276 /* 1277 * Try to represent the number of threads per taskq as percent 1278 * of online CPUs to allow scaling with later online/offline. 1279 * Fall back to absolute numbers if can't. 1280 */ 1281 value = (threads * 100 + boot_ncpus * count / 2) / 1282 (boot_ncpus * count); 1283 if (value < 5 || value > 100) 1284 value = MAX(1, (threads + count / 2) / count); 1285 else 1286 flags |= TASKQ_THREADS_CPU_PCT; 1287 break; 1288 1289 case ZTI_MODE_NULL: 1290 tqs->stqs_count = 0; 1291 tqs->stqs_taskq = NULL; 1292 return; 1293 1294 default: 1295 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1296 "spa_taskqs_init()", 1297 zio_type_name[t], zio_taskq_types[q], mode, value); 1298 break; 1299 } 1300 1301 ASSERT3U(count, >, 0); 1302 tqs->stqs_count = count; 1303 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1304 1305 for (uint_t i = 0; i < count; i++) { 1306 taskq_t *tq; 1307 char name[32]; 1308 1309 if (count > 1) 1310 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1311 zio_type_name[t], zio_taskq_types[q], i); 1312 else 1313 (void) snprintf(name, sizeof (name), "%s_%s", 1314 zio_type_name[t], zio_taskq_types[q]); 1315 1316 #ifdef HAVE_SYSDC 1317 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1318 (void) zio_taskq_basedc; 1319 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1320 spa->spa_proc, zio_taskq_basedc, flags); 1321 } else { 1322 #endif 1323 /* 1324 * The write issue taskq can be extremely CPU 1325 * intensive. Run it at slightly less important 1326 * priority than the other taskqs. 1327 */ 1328 const pri_t pri = (t == ZIO_TYPE_WRITE && 1329 q == ZIO_TASKQ_ISSUE) ? 1330 wtqclsyspri : maxclsyspri; 1331 tq = taskq_create_proc(name, value, pri, 50, 1332 INT_MAX, spa->spa_proc, flags); 1333 #ifdef HAVE_SYSDC 1334 } 1335 #endif 1336 1337 tqs->stqs_taskq[i] = tq; 1338 } 1339 } 1340 1341 static void 1342 spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1343 { 1344 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1345 1346 if (tqs->stqs_taskq == NULL) { 1347 ASSERT0(tqs->stqs_count); 1348 return; 1349 } 1350 1351 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1352 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1353 taskq_destroy(tqs->stqs_taskq[i]); 1354 } 1355 1356 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1357 tqs->stqs_taskq = NULL; 1358 } 1359 1360 #ifdef _KERNEL 1361 /* 1362 * The READ and WRITE rows of zio_taskqs are configurable at module load time 1363 * by setting zio_taskq_read or zio_taskq_write. 1364 * 1365 * Example (the defaults for READ and WRITE) 1366 * zio_taskq_read='fixed,1,8 null scale null' 1367 * zio_taskq_write='sync null scale null' 1368 * 1369 * Each sets the entire row at a time. 1370 * 1371 * 'fixed' is parameterised: fixed,Q,T where Q is number of taskqs, T is number 1372 * of threads per taskq. 1373 * 1374 * 'null' can only be set on the high-priority queues (queue selection for 1375 * high-priority queues will fall back to the regular queue if the high-pri 1376 * is NULL. 1377 */ 1378 static const char *const modes[ZTI_NMODES] = { 1379 "fixed", "scale", "sync", "null" 1380 }; 1381 1382 /* Parse the incoming config string. Modifies cfg */ 1383 static int 1384 spa_taskq_param_set(zio_type_t t, char *cfg) 1385 { 1386 int err = 0; 1387 1388 zio_taskq_info_t row[ZIO_TASKQ_TYPES] = {{0}}; 1389 1390 char *next = cfg, *tok, *c; 1391 1392 /* 1393 * Parse out each element from the string and fill `row`. The entire 1394 * row has to be set at once, so any errors are flagged by just 1395 * breaking out of this loop early. 1396 */ 1397 uint_t q; 1398 for (q = 0; q < ZIO_TASKQ_TYPES; q++) { 1399 /* `next` is the start of the config */ 1400 if (next == NULL) 1401 break; 1402 1403 /* Eat up leading space */ 1404 while (isspace(*next)) 1405 next++; 1406 if (*next == '\0') 1407 break; 1408 1409 /* Mode ends at space or end of string */ 1410 tok = next; 1411 next = strchr(tok, ' '); 1412 if (next != NULL) *next++ = '\0'; 1413 1414 /* Parameters start after a comma */ 1415 c = strchr(tok, ','); 1416 if (c != NULL) *c++ = '\0'; 1417 1418 /* Match mode string */ 1419 uint_t mode; 1420 for (mode = 0; mode < ZTI_NMODES; mode++) 1421 if (strcmp(tok, modes[mode]) == 0) 1422 break; 1423 if (mode == ZTI_NMODES) 1424 break; 1425 1426 /* Invalid canary */ 1427 row[q].zti_mode = ZTI_NMODES; 1428 1429 /* Per-mode setup */ 1430 switch (mode) { 1431 1432 /* 1433 * FIXED is parameterised: number of queues, and number of 1434 * threads per queue. 1435 */ 1436 case ZTI_MODE_FIXED: { 1437 /* No parameters? */ 1438 if (c == NULL || *c == '\0') 1439 break; 1440 1441 /* Find next parameter */ 1442 tok = c; 1443 c = strchr(tok, ','); 1444 if (c == NULL) 1445 break; 1446 1447 /* Take digits and convert */ 1448 unsigned long long nq; 1449 if (!(isdigit(*tok))) 1450 break; 1451 err = ddi_strtoull(tok, &tok, 10, &nq); 1452 /* Must succeed and also end at the next param sep */ 1453 if (err != 0 || tok != c) 1454 break; 1455 1456 /* Move past the comma */ 1457 tok++; 1458 /* Need another number */ 1459 if (!(isdigit(*tok))) 1460 break; 1461 /* Remember start to make sure we moved */ 1462 c = tok; 1463 1464 /* Take digits */ 1465 unsigned long long ntpq; 1466 err = ddi_strtoull(tok, &tok, 10, &ntpq); 1467 /* Must succeed, and moved forward */ 1468 if (err != 0 || tok == c || *tok != '\0') 1469 break; 1470 1471 /* 1472 * sanity; zero queues/threads make no sense, and 1473 * 16K is almost certainly more than anyone will ever 1474 * need and avoids silly numbers like UINT32_MAX 1475 */ 1476 if (nq == 0 || nq >= 16384 || 1477 ntpq == 0 || ntpq >= 16384) 1478 break; 1479 1480 const zio_taskq_info_t zti = ZTI_P(ntpq, nq); 1481 row[q] = zti; 1482 break; 1483 } 1484 1485 /* 1486 * SCALE is optionally parameterised by minimum number of 1487 * threads. 1488 */ 1489 case ZTI_MODE_SCALE: { 1490 unsigned long long mint = 0; 1491 if (c != NULL && *c != '\0') { 1492 /* Need a number */ 1493 if (!(isdigit(*c))) 1494 break; 1495 tok = c; 1496 1497 /* Take digits */ 1498 err = ddi_strtoull(tok, &tok, 10, &mint); 1499 /* Must succeed, and moved forward */ 1500 if (err != 0 || tok == c || *tok != '\0') 1501 break; 1502 1503 /* Sanity check */ 1504 if (mint >= 16384) 1505 break; 1506 } 1507 1508 const zio_taskq_info_t zti = ZTI_SCALE(mint); 1509 row[q] = zti; 1510 break; 1511 } 1512 1513 case ZTI_MODE_SYNC: { 1514 const zio_taskq_info_t zti = ZTI_SYNC; 1515 row[q] = zti; 1516 break; 1517 } 1518 1519 case ZTI_MODE_NULL: { 1520 /* 1521 * Can only null the high-priority queues; the general- 1522 * purpose ones have to exist. 1523 */ 1524 if (q != ZIO_TASKQ_ISSUE_HIGH && 1525 q != ZIO_TASKQ_INTERRUPT_HIGH) 1526 break; 1527 1528 const zio_taskq_info_t zti = ZTI_NULL; 1529 row[q] = zti; 1530 break; 1531 } 1532 1533 default: 1534 break; 1535 } 1536 1537 /* Ensure we set a mode */ 1538 if (row[q].zti_mode == ZTI_NMODES) 1539 break; 1540 } 1541 1542 /* Didn't get a full row, fail */ 1543 if (q < ZIO_TASKQ_TYPES) 1544 return (SET_ERROR(EINVAL)); 1545 1546 /* Eat trailing space */ 1547 if (next != NULL) 1548 while (isspace(*next)) 1549 next++; 1550 1551 /* If there's anything left over then fail */ 1552 if (next != NULL && *next != '\0') 1553 return (SET_ERROR(EINVAL)); 1554 1555 /* Success! Copy it into the real config */ 1556 for (q = 0; q < ZIO_TASKQ_TYPES; q++) 1557 zio_taskqs[t][q] = row[q]; 1558 1559 return (0); 1560 } 1561 1562 static int 1563 spa_taskq_param_get(zio_type_t t, char *buf, boolean_t add_newline) 1564 { 1565 int pos = 0; 1566 1567 /* Build paramater string from live config */ 1568 const char *sep = ""; 1569 for (uint_t q = 0; q < ZIO_TASKQ_TYPES; q++) { 1570 const zio_taskq_info_t *zti = &zio_taskqs[t][q]; 1571 if (zti->zti_mode == ZTI_MODE_FIXED) 1572 pos += sprintf(&buf[pos], "%s%s,%u,%u", sep, 1573 modes[zti->zti_mode], zti->zti_count, 1574 zti->zti_value); 1575 else if (zti->zti_mode == ZTI_MODE_SCALE && zti->zti_value > 0) 1576 pos += sprintf(&buf[pos], "%s%s,%u", sep, 1577 modes[zti->zti_mode], zti->zti_value); 1578 else 1579 pos += sprintf(&buf[pos], "%s%s", sep, 1580 modes[zti->zti_mode]); 1581 sep = " "; 1582 } 1583 1584 if (add_newline) 1585 buf[pos++] = '\n'; 1586 buf[pos] = '\0'; 1587 1588 return (pos); 1589 } 1590 1591 #ifdef __linux__ 1592 static int 1593 spa_taskq_read_param_set(const char *val, zfs_kernel_param_t *kp) 1594 { 1595 char *cfg = kmem_strdup(val); 1596 int err = spa_taskq_param_set(ZIO_TYPE_READ, cfg); 1597 kmem_strfree(cfg); 1598 return (-err); 1599 } 1600 1601 static int 1602 spa_taskq_read_param_get(char *buf, zfs_kernel_param_t *kp) 1603 { 1604 return (spa_taskq_param_get(ZIO_TYPE_READ, buf, TRUE)); 1605 } 1606 1607 static int 1608 spa_taskq_write_param_set(const char *val, zfs_kernel_param_t *kp) 1609 { 1610 char *cfg = kmem_strdup(val); 1611 int err = spa_taskq_param_set(ZIO_TYPE_WRITE, cfg); 1612 kmem_strfree(cfg); 1613 return (-err); 1614 } 1615 1616 static int 1617 spa_taskq_write_param_get(char *buf, zfs_kernel_param_t *kp) 1618 { 1619 return (spa_taskq_param_get(ZIO_TYPE_WRITE, buf, TRUE)); 1620 } 1621 1622 static int 1623 spa_taskq_free_param_set(const char *val, zfs_kernel_param_t *kp) 1624 { 1625 char *cfg = kmem_strdup(val); 1626 int err = spa_taskq_param_set(ZIO_TYPE_FREE, cfg); 1627 kmem_strfree(cfg); 1628 return (-err); 1629 } 1630 1631 static int 1632 spa_taskq_free_param_get(char *buf, zfs_kernel_param_t *kp) 1633 { 1634 return (spa_taskq_param_get(ZIO_TYPE_FREE, buf, TRUE)); 1635 } 1636 #else 1637 /* 1638 * On FreeBSD load-time parameters can be set up before malloc() is available, 1639 * so we have to do all the parsing work on the stack. 1640 */ 1641 #define SPA_TASKQ_PARAM_MAX (128) 1642 1643 static int 1644 spa_taskq_read_param(ZFS_MODULE_PARAM_ARGS) 1645 { 1646 char buf[SPA_TASKQ_PARAM_MAX]; 1647 int err; 1648 1649 (void) spa_taskq_param_get(ZIO_TYPE_READ, buf, FALSE); 1650 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 1651 if (err || req->newptr == NULL) 1652 return (err); 1653 return (spa_taskq_param_set(ZIO_TYPE_READ, buf)); 1654 } 1655 1656 static int 1657 spa_taskq_write_param(ZFS_MODULE_PARAM_ARGS) 1658 { 1659 char buf[SPA_TASKQ_PARAM_MAX]; 1660 int err; 1661 1662 (void) spa_taskq_param_get(ZIO_TYPE_WRITE, buf, FALSE); 1663 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 1664 if (err || req->newptr == NULL) 1665 return (err); 1666 return (spa_taskq_param_set(ZIO_TYPE_WRITE, buf)); 1667 } 1668 1669 static int 1670 spa_taskq_free_param(ZFS_MODULE_PARAM_ARGS) 1671 { 1672 char buf[SPA_TASKQ_PARAM_MAX]; 1673 int err; 1674 1675 (void) spa_taskq_param_get(ZIO_TYPE_FREE, buf, FALSE); 1676 err = sysctl_handle_string(oidp, buf, sizeof (buf), req); 1677 if (err || req->newptr == NULL) 1678 return (err); 1679 return (spa_taskq_param_set(ZIO_TYPE_FREE, buf)); 1680 } 1681 #endif 1682 #endif /* _KERNEL */ 1683 1684 /* 1685 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1686 * Note that a type may have multiple discrete taskqs to avoid lock contention 1687 * on the taskq itself. 1688 */ 1689 void 1690 spa_taskq_dispatch(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1691 task_func_t *func, zio_t *zio, boolean_t cutinline) 1692 { 1693 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1694 taskq_t *tq; 1695 1696 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1697 ASSERT3U(tqs->stqs_count, !=, 0); 1698 1699 /* 1700 * NB: We are assuming that the zio can only be dispatched 1701 * to a single taskq at a time. It would be a grievous error 1702 * to dispatch the zio to another taskq at the same time. 1703 */ 1704 ASSERT(zio); 1705 ASSERT(taskq_empty_ent(&zio->io_tqent)); 1706 1707 if (tqs->stqs_count == 1) { 1708 tq = tqs->stqs_taskq[0]; 1709 } else if ((t == ZIO_TYPE_WRITE) && (q == ZIO_TASKQ_ISSUE) && 1710 ZIO_HAS_ALLOCATOR(zio)) { 1711 tq = tqs->stqs_taskq[zio->io_allocator % tqs->stqs_count]; 1712 } else { 1713 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1714 } 1715 1716 taskq_dispatch_ent(tq, func, zio, cutinline ? TQ_FRONT : 0, 1717 &zio->io_tqent); 1718 } 1719 1720 static void 1721 spa_create_zio_taskqs(spa_t *spa) 1722 { 1723 for (int t = 0; t < ZIO_TYPES; t++) { 1724 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1725 spa_taskqs_init(spa, t, q); 1726 } 1727 } 1728 } 1729 1730 #if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1731 static void 1732 spa_thread(void *arg) 1733 { 1734 psetid_t zio_taskq_psrset_bind = PS_NONE; 1735 callb_cpr_t cprinfo; 1736 1737 spa_t *spa = arg; 1738 user_t *pu = PTOU(curproc); 1739 1740 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1741 spa->spa_name); 1742 1743 ASSERT(curproc != &p0); 1744 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1745 "zpool-%s", spa->spa_name); 1746 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1747 1748 /* bind this thread to the requested psrset */ 1749 if (zio_taskq_psrset_bind != PS_NONE) { 1750 pool_lock(); 1751 mutex_enter(&cpu_lock); 1752 mutex_enter(&pidlock); 1753 mutex_enter(&curproc->p_lock); 1754 1755 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1756 0, NULL, NULL) == 0) { 1757 curthread->t_bind_pset = zio_taskq_psrset_bind; 1758 } else { 1759 cmn_err(CE_WARN, 1760 "Couldn't bind process for zfs pool \"%s\" to " 1761 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1762 } 1763 1764 mutex_exit(&curproc->p_lock); 1765 mutex_exit(&pidlock); 1766 mutex_exit(&cpu_lock); 1767 pool_unlock(); 1768 } 1769 1770 #ifdef HAVE_SYSDC 1771 if (zio_taskq_sysdc) { 1772 sysdc_thread_enter(curthread, 100, 0); 1773 } 1774 #endif 1775 1776 spa->spa_proc = curproc; 1777 spa->spa_did = curthread->t_did; 1778 1779 spa_create_zio_taskqs(spa); 1780 1781 mutex_enter(&spa->spa_proc_lock); 1782 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1783 1784 spa->spa_proc_state = SPA_PROC_ACTIVE; 1785 cv_broadcast(&spa->spa_proc_cv); 1786 1787 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1788 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1789 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1790 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1791 1792 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1793 spa->spa_proc_state = SPA_PROC_GONE; 1794 spa->spa_proc = &p0; 1795 cv_broadcast(&spa->spa_proc_cv); 1796 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1797 1798 mutex_enter(&curproc->p_lock); 1799 lwp_exit(); 1800 } 1801 #endif 1802 1803 extern metaslab_ops_t *metaslab_allocator(spa_t *spa); 1804 1805 /* 1806 * Activate an uninitialized pool. 1807 */ 1808 static void 1809 spa_activate(spa_t *spa, spa_mode_t mode) 1810 { 1811 metaslab_ops_t *msp = metaslab_allocator(spa); 1812 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1813 1814 spa->spa_state = POOL_STATE_ACTIVE; 1815 spa->spa_final_txg = UINT64_MAX; 1816 spa->spa_mode = mode; 1817 spa->spa_read_spacemaps = spa_mode_readable_spacemaps; 1818 1819 spa->spa_normal_class = metaslab_class_create(spa, "normal", 1820 msp, B_FALSE); 1821 spa->spa_log_class = metaslab_class_create(spa, "log", msp, B_TRUE); 1822 spa->spa_embedded_log_class = metaslab_class_create(spa, 1823 "embedded_log", msp, B_TRUE); 1824 spa->spa_special_class = metaslab_class_create(spa, "special", 1825 msp, B_FALSE); 1826 spa->spa_special_embedded_log_class = metaslab_class_create(spa, 1827 "special_embedded_log", msp, B_TRUE); 1828 spa->spa_dedup_class = metaslab_class_create(spa, "dedup", 1829 msp, B_FALSE); 1830 1831 /* Try to create a covering process */ 1832 mutex_enter(&spa->spa_proc_lock); 1833 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1834 ASSERT(spa->spa_proc == &p0); 1835 spa->spa_did = 0; 1836 1837 #ifdef HAVE_SPA_THREAD 1838 /* Only create a process if we're going to be around a while. */ 1839 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1840 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1841 NULL, 0) == 0) { 1842 spa->spa_proc_state = SPA_PROC_CREATED; 1843 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1844 cv_wait(&spa->spa_proc_cv, 1845 &spa->spa_proc_lock); 1846 } 1847 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1848 ASSERT(spa->spa_proc != &p0); 1849 ASSERT(spa->spa_did != 0); 1850 } else { 1851 #ifdef _KERNEL 1852 cmn_err(CE_WARN, 1853 "Couldn't create process for zfs pool \"%s\"\n", 1854 spa->spa_name); 1855 #endif 1856 } 1857 } 1858 #endif /* HAVE_SPA_THREAD */ 1859 mutex_exit(&spa->spa_proc_lock); 1860 1861 /* If we didn't create a process, we need to create our taskqs. */ 1862 if (spa->spa_proc == &p0) { 1863 spa_create_zio_taskqs(spa); 1864 } 1865 1866 for (size_t i = 0; i < TXG_SIZE; i++) { 1867 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1868 ZIO_FLAG_CANFAIL); 1869 } 1870 1871 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1872 offsetof(vdev_t, vdev_config_dirty_node)); 1873 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1874 offsetof(objset_t, os_evicting_node)); 1875 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1876 offsetof(vdev_t, vdev_state_dirty_node)); 1877 1878 txg_list_create(&spa->spa_vdev_txg_list, spa, 1879 offsetof(struct vdev, vdev_txg_node)); 1880 1881 avl_create(&spa->spa_errlist_scrub, 1882 spa_error_entry_compare, sizeof (spa_error_entry_t), 1883 offsetof(spa_error_entry_t, se_avl)); 1884 avl_create(&spa->spa_errlist_last, 1885 spa_error_entry_compare, sizeof (spa_error_entry_t), 1886 offsetof(spa_error_entry_t, se_avl)); 1887 avl_create(&spa->spa_errlist_healed, 1888 spa_error_entry_compare, sizeof (spa_error_entry_t), 1889 offsetof(spa_error_entry_t, se_avl)); 1890 1891 spa_activate_os(spa); 1892 1893 spa_keystore_init(&spa->spa_keystore); 1894 1895 /* 1896 * This taskq is used to perform zvol-minor-related tasks 1897 * asynchronously. This has several advantages, including easy 1898 * resolution of various deadlocks. 1899 * 1900 * The taskq must be single threaded to ensure tasks are always 1901 * processed in the order in which they were dispatched. 1902 * 1903 * A taskq per pool allows one to keep the pools independent. 1904 * This way if one pool is suspended, it will not impact another. 1905 * 1906 * The preferred location to dispatch a zvol minor task is a sync 1907 * task. In this context, there is easy access to the spa_t and minimal 1908 * error handling is required because the sync task must succeed. 1909 */ 1910 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1911 1, INT_MAX, 0); 1912 1913 /* 1914 * The taskq to preload metaslabs. 1915 */ 1916 spa->spa_metaslab_taskq = taskq_create("z_metaslab", 1917 metaslab_preload_pct, maxclsyspri, 1, INT_MAX, 1918 TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1919 1920 /* 1921 * Taskq dedicated to prefetcher threads: this is used to prevent the 1922 * pool traverse code from monopolizing the global (and limited) 1923 * system_taskq by inappropriately scheduling long running tasks on it. 1924 */ 1925 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1926 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1927 1928 /* 1929 * The taskq to upgrade datasets in this pool. Currently used by 1930 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1931 */ 1932 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1933 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1934 } 1935 1936 /* 1937 * Opposite of spa_activate(). 1938 */ 1939 static void 1940 spa_deactivate(spa_t *spa) 1941 { 1942 if (spa->spa_create_info != NULL) { 1943 nvlist_free(spa->spa_create_info); 1944 spa->spa_create_info = NULL; 1945 } 1946 ASSERT(spa->spa_sync_on == B_FALSE); 1947 ASSERT0P(spa->spa_dsl_pool); 1948 ASSERT0P(spa->spa_root_vdev); 1949 ASSERT0P(spa->spa_async_zio_root); 1950 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1951 1952 spa_evicting_os_wait(spa); 1953 1954 if (spa->spa_zvol_taskq) { 1955 taskq_destroy(spa->spa_zvol_taskq); 1956 spa->spa_zvol_taskq = NULL; 1957 } 1958 1959 if (spa->spa_metaslab_taskq) { 1960 taskq_destroy(spa->spa_metaslab_taskq); 1961 spa->spa_metaslab_taskq = NULL; 1962 } 1963 1964 if (spa->spa_prefetch_taskq) { 1965 taskq_destroy(spa->spa_prefetch_taskq); 1966 spa->spa_prefetch_taskq = NULL; 1967 } 1968 1969 if (spa->spa_upgrade_taskq) { 1970 taskq_destroy(spa->spa_upgrade_taskq); 1971 spa->spa_upgrade_taskq = NULL; 1972 } 1973 1974 txg_list_destroy(&spa->spa_vdev_txg_list); 1975 1976 list_destroy(&spa->spa_config_dirty_list); 1977 list_destroy(&spa->spa_evicting_os_list); 1978 list_destroy(&spa->spa_state_dirty_list); 1979 1980 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid, B_TRUE); 1981 1982 for (int t = 0; t < ZIO_TYPES; t++) { 1983 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1984 spa_taskqs_fini(spa, t, q); 1985 } 1986 } 1987 1988 for (size_t i = 0; i < TXG_SIZE; i++) { 1989 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1990 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1991 spa->spa_txg_zio[i] = NULL; 1992 } 1993 1994 metaslab_class_destroy(spa->spa_normal_class); 1995 spa->spa_normal_class = NULL; 1996 1997 metaslab_class_destroy(spa->spa_log_class); 1998 spa->spa_log_class = NULL; 1999 2000 metaslab_class_destroy(spa->spa_embedded_log_class); 2001 spa->spa_embedded_log_class = NULL; 2002 2003 metaslab_class_destroy(spa->spa_special_class); 2004 spa->spa_special_class = NULL; 2005 2006 metaslab_class_destroy(spa->spa_special_embedded_log_class); 2007 spa->spa_special_embedded_log_class = NULL; 2008 2009 metaslab_class_destroy(spa->spa_dedup_class); 2010 spa->spa_dedup_class = NULL; 2011 2012 /* 2013 * If this was part of an import or the open otherwise failed, we may 2014 * still have errors left in the queues. Empty them just in case. 2015 */ 2016 spa_errlog_drain(spa); 2017 avl_destroy(&spa->spa_errlist_scrub); 2018 avl_destroy(&spa->spa_errlist_last); 2019 avl_destroy(&spa->spa_errlist_healed); 2020 2021 spa_keystore_fini(&spa->spa_keystore); 2022 2023 spa->spa_state = POOL_STATE_UNINITIALIZED; 2024 2025 mutex_enter(&spa->spa_proc_lock); 2026 if (spa->spa_proc_state != SPA_PROC_NONE) { 2027 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 2028 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 2029 cv_broadcast(&spa->spa_proc_cv); 2030 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 2031 ASSERT(spa->spa_proc != &p0); 2032 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 2033 } 2034 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 2035 spa->spa_proc_state = SPA_PROC_NONE; 2036 } 2037 ASSERT(spa->spa_proc == &p0); 2038 mutex_exit(&spa->spa_proc_lock); 2039 2040 /* 2041 * We want to make sure spa_thread() has actually exited the ZFS 2042 * module, so that the module can't be unloaded out from underneath 2043 * it. 2044 */ 2045 if (spa->spa_did != 0) { 2046 thread_join(spa->spa_did); 2047 spa->spa_did = 0; 2048 } 2049 2050 spa_deactivate_os(spa); 2051 2052 } 2053 2054 /* 2055 * Verify a pool configuration, and construct the vdev tree appropriately. This 2056 * will create all the necessary vdevs in the appropriate layout, with each vdev 2057 * in the CLOSED state. This will prep the pool before open/creation/import. 2058 * All vdev validation is done by the vdev_alloc() routine. 2059 */ 2060 int 2061 spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 2062 uint_t id, int atype) 2063 { 2064 nvlist_t **child; 2065 uint_t children; 2066 int error; 2067 2068 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 2069 return (error); 2070 2071 if ((*vdp)->vdev_ops->vdev_op_leaf) 2072 return (0); 2073 2074 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 2075 &child, &children); 2076 2077 if (error == ENOENT) 2078 return (0); 2079 2080 if (error) { 2081 vdev_free(*vdp); 2082 *vdp = NULL; 2083 return (SET_ERROR(EINVAL)); 2084 } 2085 2086 for (int c = 0; c < children; c++) { 2087 vdev_t *vd; 2088 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 2089 atype)) != 0) { 2090 vdev_free(*vdp); 2091 *vdp = NULL; 2092 return (error); 2093 } 2094 } 2095 2096 ASSERT(*vdp != NULL); 2097 2098 return (0); 2099 } 2100 2101 static boolean_t 2102 spa_should_flush_logs_on_unload(spa_t *spa) 2103 { 2104 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 2105 return (B_FALSE); 2106 2107 if (!spa_writeable(spa)) 2108 return (B_FALSE); 2109 2110 if (!spa->spa_sync_on) 2111 return (B_FALSE); 2112 2113 if (spa_state(spa) != POOL_STATE_EXPORTED) 2114 return (B_FALSE); 2115 2116 if (zfs_keep_log_spacemaps_at_export) 2117 return (B_FALSE); 2118 2119 return (B_TRUE); 2120 } 2121 2122 /* 2123 * Opens a transaction that will set the flag that will instruct 2124 * spa_sync to attempt to flush all the metaslabs for that txg. 2125 */ 2126 static void 2127 spa_unload_log_sm_flush_all(spa_t *spa) 2128 { 2129 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2130 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND)); 2131 2132 spa_log_flushall_start(spa, SPA_LOG_FLUSHALL_EXPORT, 2133 dmu_tx_get_txg(tx)); 2134 2135 dmu_tx_commit(tx); 2136 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 2137 } 2138 2139 static void 2140 spa_unload_log_sm_metadata(spa_t *spa) 2141 { 2142 void *cookie = NULL; 2143 spa_log_sm_t *sls; 2144 log_summary_entry_t *e; 2145 2146 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 2147 &cookie)) != NULL) { 2148 VERIFY0(sls->sls_mscount); 2149 kmem_free(sls, sizeof (spa_log_sm_t)); 2150 } 2151 2152 while ((e = list_remove_head(&spa->spa_log_summary)) != NULL) { 2153 VERIFY0(e->lse_mscount); 2154 kmem_free(e, sizeof (log_summary_entry_t)); 2155 } 2156 2157 spa->spa_unflushed_stats.sus_nblocks = 0; 2158 spa->spa_unflushed_stats.sus_memused = 0; 2159 spa->spa_unflushed_stats.sus_blocklimit = 0; 2160 spa->spa_unflushed_stats.sus_nmetaslabs = 0; 2161 2162 spa_log_sm_stats_update(spa); 2163 } 2164 2165 static void 2166 spa_destroy_aux_threads(spa_t *spa) 2167 { 2168 if (spa->spa_condense_zthr != NULL) { 2169 zthr_destroy(spa->spa_condense_zthr); 2170 spa->spa_condense_zthr = NULL; 2171 } 2172 if (spa->spa_checkpoint_discard_zthr != NULL) { 2173 zthr_destroy(spa->spa_checkpoint_discard_zthr); 2174 spa->spa_checkpoint_discard_zthr = NULL; 2175 } 2176 if (spa->spa_livelist_delete_zthr != NULL) { 2177 zthr_destroy(spa->spa_livelist_delete_zthr); 2178 spa->spa_livelist_delete_zthr = NULL; 2179 } 2180 if (spa->spa_livelist_condense_zthr != NULL) { 2181 zthr_destroy(spa->spa_livelist_condense_zthr); 2182 spa->spa_livelist_condense_zthr = NULL; 2183 } 2184 if (spa->spa_raidz_expand_zthr != NULL) { 2185 zthr_destroy(spa->spa_raidz_expand_zthr); 2186 spa->spa_raidz_expand_zthr = NULL; 2187 } 2188 } 2189 2190 static void 2191 spa_sync_time_logger(spa_t *spa, uint64_t txg, boolean_t force) 2192 { 2193 uint64_t curtime, dirty; 2194 dmu_tx_t *tx; 2195 dsl_pool_t *dp = spa->spa_dsl_pool; 2196 uint64_t idx = txg & TXG_MASK; 2197 2198 if (!spa_writeable(spa)) { 2199 return; 2200 } 2201 2202 curtime = gethrestime_sec(); 2203 if (txg > spa->spa_last_noted_txg && 2204 (force || 2205 curtime >= spa->spa_last_noted_txg_time + spa_note_txg_time)) { 2206 spa->spa_last_noted_txg_time = curtime; 2207 spa->spa_last_noted_txg = txg; 2208 2209 mutex_enter(&spa->spa_txg_log_time_lock); 2210 dbrrd_add(&spa->spa_txg_log_time, curtime, txg); 2211 mutex_exit(&spa->spa_txg_log_time_lock); 2212 } 2213 2214 if (!force && 2215 curtime < spa->spa_last_flush_txg_time + spa_flush_txg_time) { 2216 return; 2217 } 2218 if (txg > spa_final_dirty_txg(spa)) { 2219 return; 2220 } 2221 spa->spa_last_flush_txg_time = curtime; 2222 2223 mutex_enter(&dp->dp_lock); 2224 dirty = dp->dp_dirty_pertxg[idx]; 2225 mutex_exit(&dp->dp_lock); 2226 if (!force && dirty == 0) { 2227 return; 2228 } 2229 2230 spa->spa_last_flush_txg_time = curtime; 2231 tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg); 2232 2233 VERIFY0(zap_update(spa_meta_objset(spa), DMU_POOL_DIRECTORY_OBJECT, 2234 DMU_POOL_TXG_LOG_TIME_MINUTES, RRD_ENTRY_SIZE, RRD_STRUCT_ELEM, 2235 &spa->spa_txg_log_time.dbr_minutes, tx)); 2236 VERIFY0(zap_update(spa_meta_objset(spa), DMU_POOL_DIRECTORY_OBJECT, 2237 DMU_POOL_TXG_LOG_TIME_DAYS, RRD_ENTRY_SIZE, RRD_STRUCT_ELEM, 2238 &spa->spa_txg_log_time.dbr_days, tx)); 2239 VERIFY0(zap_update(spa_meta_objset(spa), DMU_POOL_DIRECTORY_OBJECT, 2240 DMU_POOL_TXG_LOG_TIME_MONTHS, RRD_ENTRY_SIZE, RRD_STRUCT_ELEM, 2241 &spa->spa_txg_log_time.dbr_months, tx)); 2242 dmu_tx_commit(tx); 2243 } 2244 2245 static void 2246 spa_unload_sync_time_logger(spa_t *spa) 2247 { 2248 uint64_t txg; 2249 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2250 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT)); 2251 2252 txg = dmu_tx_get_txg(tx); 2253 spa_sync_time_logger(spa, txg, B_TRUE); 2254 2255 dmu_tx_commit(tx); 2256 } 2257 2258 static void 2259 spa_load_txg_log_time(spa_t *spa) 2260 { 2261 int error; 2262 2263 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2264 DMU_POOL_TXG_LOG_TIME_MINUTES, RRD_ENTRY_SIZE, RRD_STRUCT_ELEM, 2265 &spa->spa_txg_log_time.dbr_minutes); 2266 if (error != 0 && error != ENOENT) { 2267 spa_load_note(spa, "unable to load a txg time database with " 2268 "minute resolution [error=%d]", error); 2269 } 2270 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2271 DMU_POOL_TXG_LOG_TIME_DAYS, RRD_ENTRY_SIZE, RRD_STRUCT_ELEM, 2272 &spa->spa_txg_log_time.dbr_days); 2273 if (error != 0 && error != ENOENT) { 2274 spa_load_note(spa, "unable to load a txg time database with " 2275 "day resolution [error=%d]", error); 2276 } 2277 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2278 DMU_POOL_TXG_LOG_TIME_MONTHS, RRD_ENTRY_SIZE, RRD_STRUCT_ELEM, 2279 &spa->spa_txg_log_time.dbr_months); 2280 if (error != 0 && error != ENOENT) { 2281 spa_load_note(spa, "unable to load a txg time database with " 2282 "month resolution [error=%d]", error); 2283 } 2284 } 2285 2286 static boolean_t 2287 spa_should_sync_time_logger_on_unload(spa_t *spa) 2288 { 2289 2290 if (!spa_writeable(spa)) 2291 return (B_FALSE); 2292 2293 if (!spa->spa_sync_on) 2294 return (B_FALSE); 2295 2296 if (spa_state(spa) != POOL_STATE_EXPORTED) 2297 return (B_FALSE); 2298 2299 if (spa->spa_last_noted_txg == 0) 2300 return (B_FALSE); 2301 2302 return (B_TRUE); 2303 } 2304 2305 2306 /* 2307 * Opposite of spa_load(). 2308 */ 2309 static void 2310 spa_unload(spa_t *spa) 2311 { 2312 ASSERT(spa_namespace_held() || 2313 spa->spa_export_thread == curthread); 2314 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 2315 2316 spa_import_progress_remove(spa_guid(spa)); 2317 spa_load_note(spa, "UNLOADING"); 2318 2319 spa_wake_waiters(spa); 2320 2321 /* 2322 * If we have set the spa_final_txg, we have already performed the 2323 * tasks below in spa_export_common(). We should not redo it here since 2324 * we delay the final TXGs beyond what spa_final_txg is set at. 2325 */ 2326 if (spa->spa_final_txg == UINT64_MAX) { 2327 if (spa_should_sync_time_logger_on_unload(spa)) 2328 spa_unload_sync_time_logger(spa); 2329 2330 /* 2331 * If the log space map feature is enabled and the pool is 2332 * getting exported (but not destroyed), we want to spend some 2333 * time flushing as many metaslabs as we can in an attempt to 2334 * destroy log space maps and save import time. 2335 */ 2336 if (spa_should_flush_logs_on_unload(spa)) 2337 spa_unload_log_sm_flush_all(spa); 2338 else 2339 spa_log_flushall_done(spa); 2340 2341 /* 2342 * Stop async tasks. 2343 */ 2344 spa_async_suspend(spa); 2345 2346 if (spa->spa_root_vdev) { 2347 vdev_t *root_vdev = spa->spa_root_vdev; 2348 vdev_initialize_stop_all(root_vdev, 2349 VDEV_INITIALIZE_ACTIVE); 2350 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 2351 vdev_autotrim_stop_all(spa); 2352 vdev_rebuild_stop_all(spa); 2353 l2arc_spa_rebuild_stop(spa); 2354 } 2355 2356 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 2357 spa->spa_final_txg = spa_last_synced_txg(spa) + 2358 TXG_DEFER_SIZE + 1; 2359 spa_config_exit(spa, SCL_ALL, FTAG); 2360 } 2361 2362 /* 2363 * Stop syncing. 2364 */ 2365 if (spa->spa_sync_on) { 2366 txg_sync_stop(spa->spa_dsl_pool); 2367 spa->spa_sync_on = B_FALSE; 2368 } 2369 2370 /* 2371 * This ensures that there is no async metaslab prefetching 2372 * while we attempt to unload the spa. 2373 */ 2374 taskq_wait(spa->spa_metaslab_taskq); 2375 2376 if (spa->spa_mmp.mmp_thread) 2377 mmp_thread_stop(spa); 2378 2379 /* 2380 * Wait for any outstanding async I/O to complete. 2381 */ 2382 if (spa->spa_async_zio_root != NULL) { 2383 for (int i = 0; i < max_ncpus; i++) 2384 (void) zio_wait(spa->spa_async_zio_root[i]); 2385 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 2386 spa->spa_async_zio_root = NULL; 2387 } 2388 2389 if (spa->spa_vdev_removal != NULL) { 2390 spa_vdev_removal_destroy(spa->spa_vdev_removal); 2391 spa->spa_vdev_removal = NULL; 2392 } 2393 2394 spa_destroy_aux_threads(spa); 2395 2396 spa_condense_fini(spa); 2397 2398 bpobj_close(&spa->spa_deferred_bpobj); 2399 2400 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 2401 2402 /* 2403 * Close all vdevs. 2404 */ 2405 if (spa->spa_root_vdev) 2406 vdev_free(spa->spa_root_vdev); 2407 ASSERT0P(spa->spa_root_vdev); 2408 2409 /* 2410 * Close the dsl pool. 2411 */ 2412 if (spa->spa_dsl_pool) { 2413 dsl_pool_close(spa->spa_dsl_pool); 2414 spa->spa_dsl_pool = NULL; 2415 spa->spa_meta_objset = NULL; 2416 } 2417 2418 ddt_unload(spa); 2419 brt_unload(spa); 2420 spa_unload_log_sm_metadata(spa); 2421 2422 /* 2423 * Drop and purge level 2 cache 2424 */ 2425 spa_l2cache_drop(spa); 2426 2427 if (spa->spa_spares.sav_vdevs) { 2428 for (int i = 0; i < spa->spa_spares.sav_count; i++) 2429 vdev_free(spa->spa_spares.sav_vdevs[i]); 2430 kmem_free(spa->spa_spares.sav_vdevs, 2431 spa->spa_spares.sav_count * sizeof (void *)); 2432 spa->spa_spares.sav_vdevs = NULL; 2433 } 2434 if (spa->spa_spares.sav_config) { 2435 nvlist_free(spa->spa_spares.sav_config); 2436 spa->spa_spares.sav_config = NULL; 2437 } 2438 spa->spa_spares.sav_count = 0; 2439 2440 if (spa->spa_l2cache.sav_vdevs) { 2441 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 2442 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 2443 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 2444 } 2445 kmem_free(spa->spa_l2cache.sav_vdevs, 2446 spa->spa_l2cache.sav_count * sizeof (void *)); 2447 spa->spa_l2cache.sav_vdevs = NULL; 2448 } 2449 if (spa->spa_l2cache.sav_config) { 2450 nvlist_free(spa->spa_l2cache.sav_config); 2451 spa->spa_l2cache.sav_config = NULL; 2452 } 2453 spa->spa_l2cache.sav_count = 0; 2454 2455 spa->spa_async_suspended = 0; 2456 2457 spa->spa_indirect_vdevs_loaded = B_FALSE; 2458 2459 if (spa->spa_comment != NULL) { 2460 spa_strfree(spa->spa_comment); 2461 spa->spa_comment = NULL; 2462 } 2463 if (spa->spa_compatibility != NULL) { 2464 spa_strfree(spa->spa_compatibility); 2465 spa->spa_compatibility = NULL; 2466 } 2467 2468 spa->spa_raidz_expand = NULL; 2469 spa->spa_checkpoint_txg = 0; 2470 2471 spa_config_exit(spa, SCL_ALL, spa); 2472 } 2473 2474 /* 2475 * Load (or re-load) the current list of vdevs describing the active spares for 2476 * this pool. When this is called, we have some form of basic information in 2477 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 2478 * then re-generate a more complete list including status information. 2479 */ 2480 void 2481 spa_load_spares(spa_t *spa) 2482 { 2483 nvlist_t **spares; 2484 uint_t nspares; 2485 int i; 2486 vdev_t *vd, *tvd; 2487 2488 #ifndef _KERNEL 2489 /* 2490 * zdb opens both the current state of the pool and the 2491 * checkpointed state (if present), with a different spa_t. 2492 * 2493 * As spare vdevs are shared among open pools, we skip loading 2494 * them when we load the checkpointed state of the pool. 2495 */ 2496 if (!spa_writeable(spa)) 2497 return; 2498 #endif 2499 2500 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 2501 2502 /* 2503 * First, close and free any existing spare vdevs. 2504 */ 2505 if (spa->spa_spares.sav_vdevs) { 2506 for (i = 0; i < spa->spa_spares.sav_count; i++) { 2507 vd = spa->spa_spares.sav_vdevs[i]; 2508 2509 /* Undo the call to spa_activate() below */ 2510 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 2511 B_FALSE)) != NULL && tvd->vdev_isspare) 2512 spa_spare_remove(tvd); 2513 vdev_close(vd); 2514 vdev_free(vd); 2515 } 2516 2517 kmem_free(spa->spa_spares.sav_vdevs, 2518 spa->spa_spares.sav_count * sizeof (void *)); 2519 } 2520 2521 if (spa->spa_spares.sav_config == NULL) 2522 nspares = 0; 2523 else 2524 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 2525 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 2526 2527 spa->spa_spares.sav_count = (int)nspares; 2528 spa->spa_spares.sav_vdevs = NULL; 2529 2530 if (nspares == 0) 2531 return; 2532 2533 /* 2534 * Construct the array of vdevs, opening them to get status in the 2535 * process. For each spare, there is potentially two different vdev_t 2536 * structures associated with it: one in the list of spares (used only 2537 * for basic validation purposes) and one in the active vdev 2538 * configuration (if it's spared in). During this phase we open and 2539 * validate each vdev on the spare list. If the vdev also exists in the 2540 * active configuration, then we also mark this vdev as an active spare. 2541 */ 2542 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 2543 KM_SLEEP); 2544 for (i = 0; i < spa->spa_spares.sav_count; i++) { 2545 VERIFY0(spa_config_parse(spa, &vd, spares[i], NULL, 0, 2546 VDEV_ALLOC_SPARE)); 2547 ASSERT(vd != NULL); 2548 2549 spa->spa_spares.sav_vdevs[i] = vd; 2550 2551 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 2552 B_FALSE)) != NULL) { 2553 if (!tvd->vdev_isspare) 2554 spa_spare_add(tvd); 2555 2556 /* 2557 * We only mark the spare active if we were successfully 2558 * able to load the vdev. Otherwise, importing a pool 2559 * with a bad active spare would result in strange 2560 * behavior, because multiple pool would think the spare 2561 * is actively in use. 2562 * 2563 * There is a vulnerability here to an equally bizarre 2564 * circumstance, where a dead active spare is later 2565 * brought back to life (onlined or otherwise). Given 2566 * the rarity of this scenario, and the extra complexity 2567 * it adds, we ignore the possibility. 2568 */ 2569 if (!vdev_is_dead(tvd)) 2570 spa_spare_activate(tvd); 2571 } 2572 2573 vd->vdev_top = vd; 2574 vd->vdev_aux = &spa->spa_spares; 2575 2576 if (vdev_open(vd, CRED()) != 0) 2577 continue; 2578 2579 if (vdev_validate_aux(vd) == 0) 2580 spa_spare_add(vd); 2581 } 2582 2583 /* 2584 * Recompute the stashed list of spares, with status information 2585 * this time. 2586 */ 2587 fnvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES); 2588 2589 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 2590 KM_SLEEP); 2591 for (i = 0; i < spa->spa_spares.sav_count; i++) 2592 spares[i] = vdev_config_generate(spa, 2593 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 2594 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 2595 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 2596 spa->spa_spares.sav_count); 2597 for (i = 0; i < spa->spa_spares.sav_count; i++) 2598 nvlist_free(spares[i]); 2599 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 2600 } 2601 2602 /* 2603 * Load (or re-load) the current list of vdevs describing the active l2cache for 2604 * this pool. When this is called, we have some form of basic information in 2605 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 2606 * then re-generate a more complete list including status information. 2607 * Devices which are already active have their details maintained, and are 2608 * not re-opened. 2609 */ 2610 void 2611 spa_load_l2cache(spa_t *spa) 2612 { 2613 nvlist_t **l2cache = NULL; 2614 uint_t nl2cache; 2615 int i, j, oldnvdevs; 2616 uint64_t guid; 2617 vdev_t *vd, **oldvdevs, **newvdevs; 2618 spa_aux_vdev_t *sav = &spa->spa_l2cache; 2619 2620 #ifndef _KERNEL 2621 /* 2622 * zdb opens both the current state of the pool and the 2623 * checkpointed state (if present), with a different spa_t. 2624 * 2625 * As L2 caches are part of the ARC which is shared among open 2626 * pools, we skip loading them when we load the checkpointed 2627 * state of the pool. 2628 */ 2629 if (!spa_writeable(spa)) 2630 return; 2631 #endif 2632 2633 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 2634 2635 oldvdevs = sav->sav_vdevs; 2636 oldnvdevs = sav->sav_count; 2637 sav->sav_vdevs = NULL; 2638 sav->sav_count = 0; 2639 2640 if (sav->sav_config == NULL) { 2641 nl2cache = 0; 2642 newvdevs = NULL; 2643 goto out; 2644 } 2645 2646 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, 2647 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 2648 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 2649 2650 /* 2651 * Process new nvlist of vdevs. 2652 */ 2653 for (i = 0; i < nl2cache; i++) { 2654 guid = fnvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID); 2655 2656 newvdevs[i] = NULL; 2657 for (j = 0; j < oldnvdevs; j++) { 2658 vd = oldvdevs[j]; 2659 if (vd != NULL && guid == vd->vdev_guid) { 2660 /* 2661 * Retain previous vdev for add/remove ops. 2662 */ 2663 newvdevs[i] = vd; 2664 oldvdevs[j] = NULL; 2665 break; 2666 } 2667 } 2668 2669 if (newvdevs[i] == NULL) { 2670 /* 2671 * Create new vdev 2672 */ 2673 VERIFY0(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 2674 VDEV_ALLOC_L2CACHE)); 2675 ASSERT(vd != NULL); 2676 newvdevs[i] = vd; 2677 2678 /* 2679 * Commit this vdev as an l2cache device, 2680 * even if it fails to open. 2681 */ 2682 spa_l2cache_add(vd); 2683 2684 vd->vdev_top = vd; 2685 vd->vdev_aux = sav; 2686 2687 spa_l2cache_activate(vd); 2688 2689 if (vdev_open(vd, CRED()) != 0) 2690 continue; 2691 2692 (void) vdev_validate_aux(vd); 2693 2694 if (!vdev_is_dead(vd)) 2695 l2arc_add_vdev(spa, vd); 2696 2697 /* 2698 * Upon cache device addition to a pool or pool 2699 * creation with a cache device or if the header 2700 * of the device is invalid we issue an async 2701 * TRIM command for the whole device which will 2702 * execute if l2arc_trim_ahead > 0. 2703 */ 2704 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 2705 } 2706 } 2707 2708 sav->sav_vdevs = newvdevs; 2709 sav->sav_count = (int)nl2cache; 2710 2711 /* 2712 * Recompute the stashed list of l2cache devices, with status 2713 * information this time. 2714 */ 2715 fnvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE); 2716 2717 if (sav->sav_count > 0) 2718 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 2719 KM_SLEEP); 2720 for (i = 0; i < sav->sav_count; i++) 2721 l2cache[i] = vdev_config_generate(spa, 2722 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 2723 fnvlist_add_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 2724 (const nvlist_t * const *)l2cache, sav->sav_count); 2725 2726 out: 2727 /* 2728 * Purge vdevs that were dropped 2729 */ 2730 if (oldvdevs) { 2731 for (i = 0; i < oldnvdevs; i++) { 2732 uint64_t pool; 2733 2734 vd = oldvdevs[i]; 2735 if (vd != NULL) { 2736 ASSERT(vd->vdev_isl2cache); 2737 2738 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 2739 pool != 0ULL && l2arc_vdev_present(vd)) 2740 l2arc_remove_vdev(vd); 2741 vdev_clear_stats(vd); 2742 vdev_free(vd); 2743 } 2744 } 2745 2746 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2747 } 2748 2749 for (i = 0; i < sav->sav_count; i++) 2750 nvlist_free(l2cache[i]); 2751 if (sav->sav_count) 2752 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2753 } 2754 2755 static int 2756 load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2757 { 2758 dmu_buf_t *db; 2759 char *packed = NULL; 2760 size_t nvsize = 0; 2761 int error; 2762 *value = NULL; 2763 2764 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2765 if (error) 2766 return (error); 2767 2768 nvsize = *(uint64_t *)db->db_data; 2769 dmu_buf_rele(db, FTAG); 2770 2771 packed = vmem_alloc(nvsize, KM_SLEEP); 2772 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2773 DMU_READ_PREFETCH); 2774 if (error == 0) 2775 error = nvlist_unpack(packed, nvsize, value, 0); 2776 vmem_free(packed, nvsize); 2777 2778 return (error); 2779 } 2780 2781 /* 2782 * Concrete top-level vdevs that are not missing and are not logs. At every 2783 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2784 */ 2785 static uint64_t 2786 spa_healthy_core_tvds(spa_t *spa) 2787 { 2788 vdev_t *rvd = spa->spa_root_vdev; 2789 uint64_t tvds = 0; 2790 2791 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2792 vdev_t *vd = rvd->vdev_child[i]; 2793 if (vd->vdev_islog) 2794 continue; 2795 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2796 tvds++; 2797 } 2798 2799 return (tvds); 2800 } 2801 2802 /* 2803 * Checks to see if the given vdev could not be opened, in which case we post a 2804 * sysevent to notify the autoreplace code that the device has been removed. 2805 */ 2806 static void 2807 spa_check_removed(vdev_t *vd) 2808 { 2809 for (uint64_t c = 0; c < vd->vdev_children; c++) 2810 spa_check_removed(vd->vdev_child[c]); 2811 2812 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2813 vdev_is_concrete(vd)) { 2814 zfs_post_autoreplace(vd->vdev_spa, vd); 2815 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2816 } 2817 } 2818 2819 static int 2820 spa_check_for_missing_logs(spa_t *spa) 2821 { 2822 vdev_t *rvd = spa->spa_root_vdev; 2823 2824 /* 2825 * If we're doing a normal import, then build up any additional 2826 * diagnostic information about missing log devices. 2827 * We'll pass this up to the user for further processing. 2828 */ 2829 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2830 nvlist_t **child, *nv; 2831 uint64_t idx = 0; 2832 2833 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2834 KM_SLEEP); 2835 nv = fnvlist_alloc(); 2836 2837 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2838 vdev_t *tvd = rvd->vdev_child[c]; 2839 2840 /* 2841 * We consider a device as missing only if it failed 2842 * to open (i.e. offline or faulted is not considered 2843 * as missing). 2844 */ 2845 if (tvd->vdev_islog && 2846 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2847 child[idx++] = vdev_config_generate(spa, tvd, 2848 B_FALSE, VDEV_CONFIG_MISSING); 2849 } 2850 } 2851 2852 if (idx > 0) { 2853 fnvlist_add_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 2854 (const nvlist_t * const *)child, idx); 2855 fnvlist_add_nvlist(spa->spa_load_info, 2856 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2857 2858 for (uint64_t i = 0; i < idx; i++) 2859 nvlist_free(child[i]); 2860 } 2861 nvlist_free(nv); 2862 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2863 2864 if (idx > 0) { 2865 spa_load_failed(spa, "some log devices are missing"); 2866 vdev_dbgmsg_print_tree(rvd, 2); 2867 return (SET_ERROR(ENXIO)); 2868 } 2869 } else { 2870 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2871 vdev_t *tvd = rvd->vdev_child[c]; 2872 2873 if (tvd->vdev_islog && 2874 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2875 spa_set_log_state(spa, SPA_LOG_CLEAR); 2876 spa_load_note(spa, "some log devices are " 2877 "missing, ZIL is dropped."); 2878 vdev_dbgmsg_print_tree(rvd, 2); 2879 break; 2880 } 2881 } 2882 } 2883 2884 return (0); 2885 } 2886 2887 /* 2888 * Check for missing log devices 2889 */ 2890 static boolean_t 2891 spa_check_logs(spa_t *spa) 2892 { 2893 boolean_t rv = B_FALSE; 2894 dsl_pool_t *dp = spa_get_dsl(spa); 2895 2896 switch (spa->spa_log_state) { 2897 default: 2898 break; 2899 case SPA_LOG_MISSING: 2900 /* need to recheck in case slog has been restored */ 2901 case SPA_LOG_UNKNOWN: 2902 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2903 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2904 if (rv) 2905 spa_set_log_state(spa, SPA_LOG_MISSING); 2906 break; 2907 } 2908 return (rv); 2909 } 2910 2911 /* 2912 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2913 */ 2914 static boolean_t 2915 spa_passivate_log(spa_t *spa) 2916 { 2917 vdev_t *rvd = spa->spa_root_vdev; 2918 boolean_t slog_found = B_FALSE; 2919 2920 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2921 2922 for (int c = 0; c < rvd->vdev_children; c++) { 2923 vdev_t *tvd = rvd->vdev_child[c]; 2924 2925 if (tvd->vdev_islog) { 2926 ASSERT0P(tvd->vdev_log_mg); 2927 metaslab_group_passivate(tvd->vdev_mg); 2928 slog_found = B_TRUE; 2929 } 2930 } 2931 2932 return (slog_found); 2933 } 2934 2935 /* 2936 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2937 */ 2938 static void 2939 spa_activate_log(spa_t *spa) 2940 { 2941 vdev_t *rvd = spa->spa_root_vdev; 2942 2943 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2944 2945 for (int c = 0; c < rvd->vdev_children; c++) { 2946 vdev_t *tvd = rvd->vdev_child[c]; 2947 2948 if (tvd->vdev_islog) { 2949 ASSERT0P(tvd->vdev_log_mg); 2950 metaslab_group_activate(tvd->vdev_mg); 2951 } 2952 } 2953 } 2954 2955 int 2956 spa_reset_logs(spa_t *spa) 2957 { 2958 int error; 2959 2960 error = dmu_objset_find(spa_name(spa), zil_reset, 2961 NULL, DS_FIND_CHILDREN); 2962 if (error == 0) { 2963 /* 2964 * We successfully offlined the log device, sync out the 2965 * current txg so that the "stubby" block can be removed 2966 * by zil_sync(). 2967 */ 2968 txg_wait_synced(spa->spa_dsl_pool, 0); 2969 } 2970 return (error); 2971 } 2972 2973 static void 2974 spa_aux_check_removed(spa_aux_vdev_t *sav) 2975 { 2976 for (int i = 0; i < sav->sav_count; i++) 2977 spa_check_removed(sav->sav_vdevs[i]); 2978 } 2979 2980 void 2981 spa_claim_notify(zio_t *zio) 2982 { 2983 spa_t *spa = zio->io_spa; 2984 2985 if (zio->io_error) 2986 return; 2987 2988 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2989 if (spa->spa_claim_max_txg < BP_GET_BIRTH(zio->io_bp)) 2990 spa->spa_claim_max_txg = BP_GET_BIRTH(zio->io_bp); 2991 mutex_exit(&spa->spa_props_lock); 2992 } 2993 2994 typedef struct spa_load_error { 2995 boolean_t sle_verify_data; 2996 boolean_t sle_relaxmeta; /* tolerate non-critical meta-data */ 2997 uint64_t sle_maxmeta; /* max acceptable meta-data errors */ 2998 uint64_t sle_maxdata; /* max acceptable data errors */ 2999 uint64_t sle_meta_count; 3000 uint64_t sle_data_count; 3001 } spa_load_error_t; 3002 3003 static void 3004 spa_load_verify_done(zio_t *zio) 3005 { 3006 blkptr_t *bp = zio->io_bp; 3007 spa_load_error_t *sle = zio->io_private; 3008 dmu_object_type_t type = BP_GET_TYPE(bp); 3009 int error = zio->io_error; 3010 spa_t *spa = zio->io_spa; 3011 3012 abd_free(zio->io_abd); 3013 if (error) { 3014 boolean_t meta; 3015 3016 if (type == DMU_OT_INTENT_LOG) { 3017 meta = B_FALSE; 3018 } else if (zio->io_bookmark.zb_objset == DMU_META_OBJSET) { 3019 meta = B_TRUE; 3020 } else if (sle->sle_relaxmeta) { 3021 /* 3022 * Losing a file or a directory costs us the affected 3023 * objects, but the pool as a whole remains operable. 3024 */ 3025 meta = DMU_OT_IS_CRITICAL(type, BP_GET_LEVEL(bp)); 3026 } else { 3027 meta = BP_GET_LEVEL(bp) != 0 || 3028 DMU_OT_IS_METADATA(type); 3029 } 3030 if (meta) 3031 atomic_inc_64(&sle->sle_meta_count); 3032 else 3033 atomic_inc_64(&sle->sle_data_count); 3034 } 3035 3036 mutex_enter(&spa->spa_scrub_lock); 3037 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 3038 cv_broadcast(&spa->spa_scrub_io_cv); 3039 mutex_exit(&spa->spa_scrub_lock); 3040 } 3041 3042 /* 3043 * Maximum number of inflight bytes is the log2 fraction of the arc size. 3044 * By default, we set it to 1/16th of the arc. 3045 */ 3046 static uint_t spa_load_verify_shift = 4; 3047 static int spa_load_verify_metadata = B_TRUE; 3048 static int spa_load_verify_data = B_TRUE; 3049 3050 static int 3051 spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 3052 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 3053 { 3054 zio_t *rio = arg; 3055 spa_load_error_t *sle = rio->io_private; 3056 3057 (void) zilog, (void) dnp; 3058 3059 /* 3060 * Note: normally this routine will not be called if 3061 * spa_load_verify_metadata is not set. However, it may be useful 3062 * to manually set the flag after the traversal has begun. 3063 */ 3064 if (!spa_load_verify_metadata) 3065 return (0); 3066 3067 /* 3068 * Stop the traversal as soon as the verdict is known, there is no 3069 * point in counting the errors we are not going to tolerate anyway. 3070 */ 3071 if (sle->sle_meta_count > sle->sle_maxmeta || 3072 sle->sle_data_count > sle->sle_maxdata) 3073 return (SET_ERROR(ECANCELED)); 3074 3075 /* 3076 * Sanity check the block pointer in order to detect obvious damage 3077 * before using the contents in subsequent checks or in zio_read(). 3078 * When damaged consider it to be a metadata error since we cannot 3079 * trust the BP_GET_TYPE and BP_GET_LEVEL values. 3080 */ 3081 if (zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) { 3082 atomic_inc_64(&sle->sle_meta_count); 3083 return (0); 3084 } 3085 3086 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 3087 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 3088 return (0); 3089 3090 if (!BP_IS_METADATA(bp) && 3091 (!spa_load_verify_data || !sle->sle_verify_data)) 3092 return (0); 3093 3094 uint64_t maxinflight_bytes = 3095 arc_target_bytes() >> spa_load_verify_shift; 3096 size_t size = BP_GET_PSIZE(bp); 3097 3098 mutex_enter(&spa->spa_scrub_lock); 3099 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 3100 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 3101 spa->spa_load_verify_bytes += size; 3102 mutex_exit(&spa->spa_scrub_lock); 3103 3104 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 3105 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 3106 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 3107 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 3108 return (0); 3109 } 3110 3111 static int 3112 verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 3113 { 3114 (void) dp, (void) arg; 3115 3116 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 3117 return (SET_ERROR(ENAMETOOLONG)); 3118 3119 return (0); 3120 } 3121 3122 static int 3123 spa_load_verify(spa_t *spa) 3124 { 3125 zio_t *rio; 3126 spa_load_error_t sle = { 0 }; 3127 zpool_load_policy_t policy; 3128 boolean_t verify_ok = B_FALSE, aborted = B_FALSE; 3129 int error = 0; 3130 3131 zpool_get_load_policy(spa->spa_config, &policy); 3132 3133 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND || 3134 policy.zlp_maxmeta == UINT64_MAX) 3135 return (0); 3136 3137 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 3138 error = dmu_objset_find_dp(spa->spa_dsl_pool, 3139 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 3140 DS_FIND_CHILDREN); 3141 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 3142 if (error != 0) 3143 return (error); 3144 3145 /* 3146 * Verify data only if somebody is going to look at the error count: 3147 * either the caller set a limit for it, or we are only searching for 3148 * the best txg without rewinding to it (zpool import -nF), which 3149 * reports the count back to the user. 3150 */ 3151 sle.sle_verify_data = policy.zlp_maxdata < UINT64_MAX || 3152 ((policy.zlp_rewind & ZPOOL_REWIND_MASK) && 3153 (spa_load_verify_dryrun || 3154 spa->spa_load_state != SPA_LOAD_RECOVER)); 3155 3156 sle.sle_relaxmeta = policy.zlp_relaxmeta; 3157 3158 /* 3159 * Dry run reports the errors instead of acting on them, so it needs 3160 * the complete counts. Otherwise stop counting once the thresholds 3161 * are exceeded, since the result can not change after that. 3162 */ 3163 if (spa_load_verify_dryrun) { 3164 sle.sle_maxmeta = sle.sle_maxdata = UINT64_MAX; 3165 } else { 3166 sle.sle_maxmeta = policy.zlp_maxmeta; 3167 sle.sle_maxdata = policy.zlp_maxdata; 3168 } 3169 3170 rio = zio_root(spa, NULL, &sle, 3171 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 3172 3173 if (spa_load_verify_metadata) { 3174 if (spa->spa_extreme_rewind) { 3175 spa_load_note(spa, "performing a complete scan of the " 3176 "pool since extreme rewind is on. This may take " 3177 "a very long time.\n (verifying metadata=%u, " 3178 "data=%u)", spa_load_verify_metadata, 3179 spa_load_verify_data && sle.sle_verify_data); 3180 } 3181 3182 error = traverse_pool(spa, spa->spa_verify_min_txg, 3183 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 3184 TRAVERSE_NO_DECRYPT | TRAVERSE_HARD, 3185 spa_load_verify_cb, rio); 3186 3187 /* 3188 * We aborted the traversal ourselves, so this is not a real 3189 * error, only the error counts below are now lower bounds. 3190 */ 3191 if (error == ECANCELED) { 3192 error = 0; 3193 aborted = B_TRUE; 3194 } 3195 } 3196 3197 (void) zio_wait(rio); 3198 ASSERT0(spa->spa_load_verify_bytes); 3199 3200 spa->spa_load_meta_errors = sle.sle_meta_count; 3201 spa->spa_load_data_errors = sle.sle_data_count; 3202 3203 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 3204 spa_load_note(spa, "spa_load_verify found %s%llu metadata " 3205 "errors and %llu data errors", 3206 aborted ? "at least " : "", 3207 (u_longlong_t)sle.sle_meta_count, 3208 (u_longlong_t)sle.sle_data_count); 3209 } 3210 3211 if (spa_load_verify_dryrun || 3212 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 3213 sle.sle_data_count <= policy.zlp_maxdata)) { 3214 verify_ok = B_TRUE; 3215 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 3216 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 3217 3218 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TXG, 3219 spa->spa_load_txg); 3220 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_LOAD_TIME, 3221 spa->spa_load_txg_ts); 3222 /* 3223 * The loss makes sense only for a fallback to an older 3224 * uberblock, which is the only case we know the newest one in. 3225 */ 3226 if (spa->spa_last_ubsync_txg_ts != 0) { 3227 fnvlist_add_int64(spa->spa_load_info, 3228 ZPOOL_CONFIG_REWIND_TIME, 3229 spa->spa_last_ubsync_txg_ts - 3230 spa->spa_load_txg_ts); 3231 } 3232 fnvlist_add_uint64(spa->spa_load_info, 3233 ZPOOL_CONFIG_LOAD_META_ERRORS, sle.sle_meta_count); 3234 fnvlist_add_uint64(spa->spa_load_info, 3235 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count); 3236 } else { 3237 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 3238 } 3239 3240 if (spa_load_verify_dryrun) 3241 return (0); 3242 3243 if (error) { 3244 if (error != ENXIO && error != EIO) 3245 error = SET_ERROR(EIO); 3246 return (error); 3247 } 3248 3249 return (verify_ok ? 0 : EIO); 3250 } 3251 3252 /* 3253 * Find a value in the pool props object. 3254 */ 3255 static void 3256 spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 3257 { 3258 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 3259 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 3260 } 3261 3262 /* 3263 * Find a value in the pool directory object. 3264 */ 3265 static int 3266 spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 3267 { 3268 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 3269 name, sizeof (uint64_t), 1, val); 3270 3271 if (error != 0 && (error != ENOENT || log_enoent)) { 3272 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 3273 "[error=%d]", name, error); 3274 } 3275 3276 return (error); 3277 } 3278 3279 static int 3280 spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 3281 { 3282 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 3283 return (SET_ERROR(err)); 3284 } 3285 3286 boolean_t 3287 spa_livelist_delete_check(spa_t *spa) 3288 { 3289 return (spa->spa_livelists_to_delete != 0); 3290 } 3291 3292 static boolean_t 3293 spa_livelist_delete_cb_check(void *arg, zthr_t *z) 3294 { 3295 (void) z; 3296 spa_t *spa = arg; 3297 return (spa_livelist_delete_check(spa)); 3298 } 3299 3300 static int 3301 delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 3302 { 3303 spa_t *spa = arg; 3304 zio_free(spa, tx->tx_txg, bp); 3305 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 3306 -bp_get_dsize_sync(spa, bp), 3307 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 3308 return (0); 3309 } 3310 3311 static int 3312 dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 3313 { 3314 int err; 3315 zap_cursor_t zc; 3316 zap_attribute_t *za = zap_attribute_alloc(); 3317 zap_cursor_init(&zc, os, zap_obj); 3318 err = zap_cursor_retrieve(&zc, za); 3319 zap_cursor_fini(&zc); 3320 if (err == 0) 3321 *llp = za->za_first_integer; 3322 zap_attribute_free(za); 3323 return (err); 3324 } 3325 3326 /* 3327 * Components of livelist deletion that must be performed in syncing 3328 * context: freeing block pointers and updating the pool-wide data 3329 * structures to indicate how much work is left to do 3330 */ 3331 typedef struct sublist_delete_arg { 3332 spa_t *spa; 3333 dsl_deadlist_t *ll; 3334 uint64_t key; 3335 bplist_t *to_free; 3336 } sublist_delete_arg_t; 3337 3338 static void 3339 sublist_delete_sync(void *arg, dmu_tx_t *tx) 3340 { 3341 sublist_delete_arg_t *sda = arg; 3342 spa_t *spa = sda->spa; 3343 dsl_deadlist_t *ll = sda->ll; 3344 uint64_t key = sda->key; 3345 bplist_t *to_free = sda->to_free; 3346 3347 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 3348 dsl_deadlist_remove_entry(ll, key, tx); 3349 } 3350 3351 typedef struct livelist_delete_arg { 3352 spa_t *spa; 3353 uint64_t ll_obj; 3354 uint64_t zap_obj; 3355 } livelist_delete_arg_t; 3356 3357 static void 3358 livelist_delete_sync(void *arg, dmu_tx_t *tx) 3359 { 3360 livelist_delete_arg_t *lda = arg; 3361 spa_t *spa = lda->spa; 3362 uint64_t ll_obj = lda->ll_obj; 3363 uint64_t zap_obj = lda->zap_obj; 3364 objset_t *mos = spa->spa_meta_objset; 3365 uint64_t count; 3366 3367 /* free the livelist and decrement the feature count */ 3368 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 3369 dsl_deadlist_free(mos, ll_obj, tx); 3370 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 3371 VERIFY0(zap_count(mos, zap_obj, &count)); 3372 if (count == 0) { 3373 /* no more livelists to delete */ 3374 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 3375 DMU_POOL_DELETED_CLONES, tx)); 3376 VERIFY0(zap_destroy(mos, zap_obj, tx)); 3377 spa->spa_livelists_to_delete = 0; 3378 spa_notify_waiters(spa); 3379 } 3380 } 3381 3382 /* 3383 * Load in the value for the livelist to be removed and open it. Then, 3384 * load its first sublist and determine which block pointers should actually 3385 * be freed. Then, call a synctask which performs the actual frees and updates 3386 * the pool-wide livelist data. 3387 */ 3388 static void 3389 spa_livelist_delete_cb(void *arg, zthr_t *z) 3390 { 3391 spa_t *spa = arg; 3392 uint64_t ll_obj = 0, count; 3393 objset_t *mos = spa->spa_meta_objset; 3394 uint64_t zap_obj = spa->spa_livelists_to_delete; 3395 /* 3396 * Determine the next livelist to delete. This function should only 3397 * be called if there is at least one deleted clone. 3398 */ 3399 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 3400 VERIFY0(zap_count(mos, ll_obj, &count)); 3401 if (count > 0) { 3402 dsl_deadlist_t *ll; 3403 dsl_deadlist_entry_t *dle; 3404 bplist_t to_free; 3405 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 3406 VERIFY0(dsl_deadlist_open(ll, mos, ll_obj)); 3407 dle = dsl_deadlist_first(ll); 3408 ASSERT3P(dle, !=, NULL); 3409 bplist_create(&to_free); 3410 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 3411 z, NULL); 3412 if (err == 0) { 3413 sublist_delete_arg_t sync_arg = { 3414 .spa = spa, 3415 .ll = ll, 3416 .key = dle->dle_mintxg, 3417 .to_free = &to_free 3418 }; 3419 zfs_dbgmsg("deleting sublist (id %llu) from" 3420 " livelist %llu, %lld remaining", 3421 (u_longlong_t)dle->dle_bpobj.bpo_object, 3422 (u_longlong_t)ll_obj, (longlong_t)count - 1); 3423 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 3424 sublist_delete_sync, &sync_arg, 0, 3425 ZFS_SPACE_CHECK_DESTROY)); 3426 } else { 3427 VERIFY3U(err, ==, EINTR); 3428 } 3429 bplist_clear(&to_free); 3430 bplist_destroy(&to_free); 3431 dsl_deadlist_close(ll); 3432 kmem_free(ll, sizeof (dsl_deadlist_t)); 3433 } else { 3434 livelist_delete_arg_t sync_arg = { 3435 .spa = spa, 3436 .ll_obj = ll_obj, 3437 .zap_obj = zap_obj 3438 }; 3439 zfs_dbgmsg("deletion of livelist %llu completed", 3440 (u_longlong_t)ll_obj); 3441 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 3442 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 3443 } 3444 } 3445 3446 static void 3447 spa_start_livelist_destroy_thread(spa_t *spa) 3448 { 3449 ASSERT0P(spa->spa_livelist_delete_zthr); 3450 spa->spa_livelist_delete_zthr = 3451 zthr_create("z_livelist_destroy", 3452 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa, 3453 minclsyspri); 3454 } 3455 3456 typedef struct livelist_new_arg { 3457 bplist_t *allocs; 3458 bplist_t *frees; 3459 } livelist_new_arg_t; 3460 3461 static int 3462 livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 3463 dmu_tx_t *tx) 3464 { 3465 ASSERT0P(tx); 3466 livelist_new_arg_t *lna = arg; 3467 if (bp_freed) { 3468 bplist_append(lna->frees, bp); 3469 } else { 3470 bplist_append(lna->allocs, bp); 3471 zfs_livelist_condense_new_alloc++; 3472 } 3473 return (0); 3474 } 3475 3476 typedef struct livelist_condense_arg { 3477 spa_t *spa; 3478 bplist_t to_keep; 3479 uint64_t first_size; 3480 uint64_t next_size; 3481 } livelist_condense_arg_t; 3482 3483 static void 3484 spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 3485 { 3486 livelist_condense_arg_t *lca = arg; 3487 spa_t *spa = lca->spa; 3488 bplist_t new_frees; 3489 dsl_dataset_t *ds = spa->spa_to_condense.ds; 3490 3491 /* Have we been cancelled? */ 3492 if (spa->spa_to_condense.cancelled) { 3493 zfs_livelist_condense_sync_cancel++; 3494 goto out; 3495 } 3496 3497 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 3498 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 3499 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 3500 3501 /* 3502 * It's possible that the livelist was changed while the zthr was 3503 * running. Therefore, we need to check for new blkptrs in the two 3504 * entries being condensed and continue to track them in the livelist. 3505 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 3506 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 3507 * we need to sort them into two different bplists. 3508 */ 3509 uint64_t first_obj = first->dle_bpobj.bpo_object; 3510 uint64_t next_obj = next->dle_bpobj.bpo_object; 3511 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 3512 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 3513 3514 bplist_create(&new_frees); 3515 livelist_new_arg_t new_bps = { 3516 .allocs = &lca->to_keep, 3517 .frees = &new_frees, 3518 }; 3519 3520 if (cur_first_size > lca->first_size) { 3521 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 3522 livelist_track_new_cb, &new_bps, lca->first_size)); 3523 } 3524 if (cur_next_size > lca->next_size) { 3525 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 3526 livelist_track_new_cb, &new_bps, lca->next_size)); 3527 } 3528 3529 dsl_deadlist_clear_entry(first, ll, tx); 3530 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 3531 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 3532 3533 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 3534 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 3535 bplist_destroy(&new_frees); 3536 3537 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 3538 dsl_dataset_name(ds, dsname); 3539 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 3540 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 3541 "(%llu blkptrs)", (u_longlong_t)tx->tx_txg, dsname, 3542 (u_longlong_t)ds->ds_object, (u_longlong_t)first_obj, 3543 (u_longlong_t)cur_first_size, (u_longlong_t)next_obj, 3544 (u_longlong_t)cur_next_size, 3545 (u_longlong_t)first->dle_bpobj.bpo_object, 3546 (u_longlong_t)first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 3547 out: 3548 dmu_buf_rele(ds->ds_dbuf, spa); 3549 spa->spa_to_condense.ds = NULL; 3550 bplist_clear(&lca->to_keep); 3551 bplist_destroy(&lca->to_keep); 3552 kmem_free(lca, sizeof (livelist_condense_arg_t)); 3553 spa->spa_to_condense.syncing = B_FALSE; 3554 } 3555 3556 static void 3557 spa_livelist_condense_cb(void *arg, zthr_t *t) 3558 { 3559 while (zfs_livelist_condense_zthr_pause && 3560 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 3561 delay(1); 3562 3563 spa_t *spa = arg; 3564 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 3565 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 3566 uint64_t first_size, next_size; 3567 3568 livelist_condense_arg_t *lca = 3569 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 3570 bplist_create(&lca->to_keep); 3571 3572 /* 3573 * Process the livelists (matching FREEs and ALLOCs) in open context 3574 * so we have minimal work in syncing context to condense. 3575 * 3576 * We save bpobj sizes (first_size and next_size) to use later in 3577 * syncing context to determine if entries were added to these sublists 3578 * while in open context. This is possible because the clone is still 3579 * active and open for normal writes and we want to make sure the new, 3580 * unprocessed blockpointers are inserted into the livelist normally. 3581 * 3582 * Note that dsl_process_sub_livelist() both stores the size number of 3583 * blockpointers and iterates over them while the bpobj's lock held, so 3584 * the sizes returned to us are consistent which what was actually 3585 * processed. 3586 */ 3587 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 3588 &first_size); 3589 if (err == 0) 3590 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 3591 t, &next_size); 3592 3593 if (err == 0) { 3594 while (zfs_livelist_condense_sync_pause && 3595 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 3596 delay(1); 3597 3598 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 3599 dmu_tx_mark_netfree(tx); 3600 dmu_tx_hold_space(tx, 1); 3601 err = dmu_tx_assign(tx, DMU_TX_NOWAIT | DMU_TX_NOTHROTTLE); 3602 if (err == 0) { 3603 /* 3604 * Prevent the condense zthr restarting before 3605 * the synctask completes. 3606 */ 3607 spa->spa_to_condense.syncing = B_TRUE; 3608 lca->spa = spa; 3609 lca->first_size = first_size; 3610 lca->next_size = next_size; 3611 dsl_sync_task_nowait(spa_get_dsl(spa), 3612 spa_livelist_condense_sync, lca, tx); 3613 dmu_tx_commit(tx); 3614 return; 3615 } 3616 } 3617 /* 3618 * Condensing can not continue: either it was externally stopped or 3619 * we were unable to assign to a tx because the pool has run out of 3620 * space. In the second case, we'll just end up trying to condense 3621 * again in a later txg. 3622 */ 3623 ASSERT(err != 0); 3624 bplist_clear(&lca->to_keep); 3625 bplist_destroy(&lca->to_keep); 3626 kmem_free(lca, sizeof (livelist_condense_arg_t)); 3627 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 3628 spa->spa_to_condense.ds = NULL; 3629 if (err == EINTR) 3630 zfs_livelist_condense_zthr_cancel++; 3631 } 3632 3633 /* 3634 * Check that there is something to condense but that a condense is not 3635 * already in progress and that condensing has not been cancelled. 3636 */ 3637 static boolean_t 3638 spa_livelist_condense_cb_check(void *arg, zthr_t *z) 3639 { 3640 (void) z; 3641 spa_t *spa = arg; 3642 if ((spa->spa_to_condense.ds != NULL) && 3643 (spa->spa_to_condense.syncing == B_FALSE) && 3644 (spa->spa_to_condense.cancelled == B_FALSE)) { 3645 return (B_TRUE); 3646 } 3647 return (B_FALSE); 3648 } 3649 3650 static void 3651 spa_start_livelist_condensing_thread(spa_t *spa) 3652 { 3653 spa->spa_to_condense.ds = NULL; 3654 spa->spa_to_condense.first = NULL; 3655 spa->spa_to_condense.next = NULL; 3656 spa->spa_to_condense.syncing = B_FALSE; 3657 spa->spa_to_condense.cancelled = B_FALSE; 3658 3659 ASSERT0P(spa->spa_livelist_condense_zthr); 3660 spa->spa_livelist_condense_zthr = 3661 zthr_create("z_livelist_condense", 3662 spa_livelist_condense_cb_check, 3663 spa_livelist_condense_cb, spa, minclsyspri); 3664 } 3665 3666 static void 3667 spa_spawn_aux_threads(spa_t *spa) 3668 { 3669 ASSERT(spa_writeable(spa)); 3670 3671 spa_start_raidz_expansion_thread(spa); 3672 spa_start_indirect_condensing_thread(spa); 3673 spa_start_livelist_destroy_thread(spa); 3674 spa_start_livelist_condensing_thread(spa); 3675 3676 ASSERT0P(spa->spa_checkpoint_discard_zthr); 3677 spa->spa_checkpoint_discard_zthr = 3678 zthr_create("z_checkpoint_discard", 3679 spa_checkpoint_discard_thread_check, 3680 spa_checkpoint_discard_thread, spa, minclsyspri); 3681 } 3682 3683 /* 3684 * Fix up config after a partly-completed split. This is done with the 3685 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 3686 * pool have that entry in their config, but only the splitting one contains 3687 * a list of all the guids of the vdevs that are being split off. 3688 * 3689 * This function determines what to do with that list: either rejoin 3690 * all the disks to the pool, or complete the splitting process. To attempt 3691 * the rejoin, each disk that is offlined is marked online again, and 3692 * we do a reopen() call. If the vdev label for every disk that was 3693 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 3694 * then we call vdev_split() on each disk, and complete the split. 3695 * 3696 * Otherwise we leave the config alone, with all the vdevs in place in 3697 * the original pool. 3698 */ 3699 static void 3700 spa_try_repair(spa_t *spa, nvlist_t *config) 3701 { 3702 uint_t extracted; 3703 uint64_t *glist; 3704 uint_t i, gcount; 3705 nvlist_t *nvl; 3706 vdev_t **vd; 3707 boolean_t attempt_reopen; 3708 3709 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 3710 return; 3711 3712 /* check that the config is complete */ 3713 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 3714 &glist, &gcount) != 0) 3715 return; 3716 3717 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 3718 3719 /* attempt to online all the vdevs & validate */ 3720 attempt_reopen = B_TRUE; 3721 for (i = 0; i < gcount; i++) { 3722 if (glist[i] == 0) /* vdev is hole */ 3723 continue; 3724 3725 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 3726 if (vd[i] == NULL) { 3727 /* 3728 * Don't bother attempting to reopen the disks; 3729 * just do the split. 3730 */ 3731 attempt_reopen = B_FALSE; 3732 } else { 3733 /* attempt to re-online it */ 3734 vd[i]->vdev_offline = B_FALSE; 3735 } 3736 } 3737 3738 if (attempt_reopen) { 3739 vdev_reopen(spa->spa_root_vdev); 3740 3741 /* check each device to see what state it's in */ 3742 for (extracted = 0, i = 0; i < gcount; i++) { 3743 if (vd[i] != NULL && 3744 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 3745 break; 3746 ++extracted; 3747 } 3748 } 3749 3750 /* 3751 * If every disk has been moved to the new pool, or if we never 3752 * even attempted to look at them, then we split them off for 3753 * good. 3754 */ 3755 if (!attempt_reopen || gcount == extracted) { 3756 for (i = 0; i < gcount; i++) 3757 if (vd[i] != NULL) 3758 vdev_split(vd[i]); 3759 vdev_reopen(spa->spa_root_vdev); 3760 } 3761 3762 kmem_free(vd, gcount * sizeof (vdev_t *)); 3763 } 3764 3765 static int 3766 spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 3767 { 3768 const char *ereport = FM_EREPORT_ZFS_POOL; 3769 int error; 3770 3771 spa->spa_load_state = state; 3772 (void) spa_import_progress_set_state(spa_guid(spa), 3773 spa_load_state(spa)); 3774 spa_import_progress_set_notes(spa, "spa_load()"); 3775 3776 gethrestime(&spa->spa_loaded_ts); 3777 error = spa_load_impl(spa, type, &ereport); 3778 3779 /* 3780 * Don't count references from objsets that are already closed 3781 * and are making their way through the eviction process. 3782 */ 3783 spa_evicting_os_wait(spa); 3784 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 3785 if (error) { 3786 if (error != EEXIST) { 3787 spa->spa_loaded_ts.tv_sec = 0; 3788 spa->spa_loaded_ts.tv_nsec = 0; 3789 } 3790 if (error != EBADF) { 3791 (void) zfs_ereport_post(ereport, spa, 3792 NULL, NULL, NULL, 0); 3793 } 3794 } 3795 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 3796 spa->spa_ena = 0; 3797 3798 (void) spa_import_progress_set_state(spa_guid(spa), 3799 spa_load_state(spa)); 3800 3801 return (error); 3802 } 3803 3804 #ifdef ZFS_DEBUG 3805 /* 3806 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 3807 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 3808 * spa's per-vdev ZAP list. 3809 */ 3810 static uint64_t 3811 vdev_count_verify_zaps(vdev_t *vd) 3812 { 3813 spa_t *spa = vd->vdev_spa; 3814 uint64_t total = 0; 3815 3816 if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) && 3817 vd->vdev_root_zap != 0) { 3818 total++; 3819 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3820 spa->spa_all_vdev_zaps, vd->vdev_root_zap)); 3821 } 3822 if (vd->vdev_top_zap != 0) { 3823 total++; 3824 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3825 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 3826 } 3827 if (vd->vdev_leaf_zap != 0) { 3828 total++; 3829 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 3830 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 3831 } 3832 3833 for (uint64_t i = 0; i < vd->vdev_children; i++) { 3834 total += vdev_count_verify_zaps(vd->vdev_child[i]); 3835 } 3836 3837 return (total); 3838 } 3839 #else 3840 #define vdev_count_verify_zaps(vd) ((void) sizeof (vd), 0) 3841 #endif 3842 3843 /* 3844 * Check the results load_info results from previous tryimport. 3845 * 3846 * error results: 3847 * 0 - Pool remains in an idle state 3848 * EREMOTEIO - Pool was known to be active on the other host 3849 * ENOENT - The config does not contain complete tryimport info 3850 */ 3851 static int 3852 spa_activity_verify_config(spa_t *spa, uberblock_t *ub) 3853 { 3854 uint64_t tryconfig_mmp_state = MMP_STATE_ACTIVE; 3855 uint64_t tryconfig_txg = 0; 3856 uint64_t tryconfig_timestamp = 0; 3857 uint16_t tryconfig_mmp_seq = 0; 3858 nvlist_t *nvinfo, *config = spa->spa_config; 3859 int error; 3860 3861 /* Simply a non-zero value to indicate the verify was done. */ 3862 spa->spa_mmp.mmp_import_ns = 1000; 3863 3864 error = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, &nvinfo); 3865 if (error) 3866 return (SET_ERROR(ENOENT)); 3867 3868 /* 3869 * If ZPOOL_CONFIG_MMP_STATE is present an activity check was performed 3870 * during the earlier tryimport. If the state recorded there isn't 3871 * MMP_STATE_INACTIVE the pool is known to be active on another host. 3872 */ 3873 error = nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_STATE, 3874 &tryconfig_mmp_state); 3875 if (error) 3876 return (SET_ERROR(ENOENT)); 3877 3878 if (tryconfig_mmp_state != MMP_STATE_INACTIVE) { 3879 spa_load_failed(spa, "mmp: pool is active on remote host, " 3880 "state=%llu", (u_longlong_t)tryconfig_mmp_state); 3881 return (SET_ERROR(EREMOTEIO)); 3882 } 3883 3884 /* 3885 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3886 * during the earlier tryimport. If the txg recorded there is 0 then 3887 * the pool is known to be active on another host. 3888 */ 3889 error = nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3890 &tryconfig_txg); 3891 if (error) 3892 return (SET_ERROR(ENOENT)); 3893 3894 if (tryconfig_txg == 0) { 3895 spa_load_failed(spa, "mmp: pool is active on remote host, " 3896 "tryconfig_txg=%llu", (u_longlong_t)tryconfig_txg); 3897 return (SET_ERROR(EREMOTEIO)); 3898 } 3899 3900 error = nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3901 &tryconfig_timestamp); 3902 if (error) 3903 return (SET_ERROR(ENOENT)); 3904 3905 error = nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3906 &tryconfig_mmp_seq); 3907 if (error) 3908 return (SET_ERROR(ENOENT)); 3909 3910 if (tryconfig_timestamp == ub->ub_timestamp && 3911 tryconfig_txg == ub->ub_txg && 3912 MMP_SEQ_VALID(ub) && tryconfig_mmp_seq == MMP_SEQ(ub)) { 3913 zfs_dbgmsg("mmp: verified pool mmp tryimport config, " 3914 "spa=%s", spa_load_name(spa)); 3915 return (0); 3916 } 3917 3918 spa_load_failed(spa, "mmp: pool is active on remote host, " 3919 "tc_timestamp=%llu ub_timestamp=%llu " 3920 "tc_txg=%llu ub_txg=%llu tc_seq=%llu ub_seq=%llu", 3921 (u_longlong_t)tryconfig_timestamp, (u_longlong_t)ub->ub_timestamp, 3922 (u_longlong_t)tryconfig_txg, (u_longlong_t)ub->ub_txg, 3923 (u_longlong_t)tryconfig_mmp_seq, (u_longlong_t)MMP_SEQ(ub)); 3924 3925 return (SET_ERROR(EREMOTEIO)); 3926 } 3927 3928 /* 3929 * Determine whether the activity check is required. 3930 */ 3931 static boolean_t 3932 spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label) 3933 { 3934 nvlist_t *config = spa->spa_config; 3935 uint64_t state = POOL_STATE_ACTIVE; 3936 uint64_t hostid = 0; 3937 3938 /* 3939 * Disable the MMP activity check - This is used by zdb which 3940 * is always read-only and intended to be used on potentially 3941 * active pools. 3942 */ 3943 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) { 3944 zfs_dbgmsg("mmp: skipping check ZFS_IMPORT_SKIP_MMP is set, " 3945 "spa=%s", spa_load_name(spa)); 3946 return (B_FALSE); 3947 } 3948 3949 /* 3950 * Skip the activity check when the MMP feature is disabled. 3951 * - MMP_MAGIC not set - Legacy pool predates the MMP feature, or 3952 * - MMP_MAGIC set && mmp_delay == 0 - MMP feature is disabled. 3953 */ 3954 if ((ub->ub_mmp_magic != MMP_MAGIC) || 3955 (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0)) { 3956 zfs_dbgmsg("mmp: skipping check: feature is disabled, " 3957 "spa=%s", spa_load_name(spa)); 3958 return (B_FALSE); 3959 } 3960 3961 /* 3962 * Allow the activity check to be skipped when importing a cleanly 3963 * exported pool on the same host which last imported it. Since the 3964 * hostid from configuration may be stale use the one read from the 3965 * label. Imports from other hostids must perform the activity check. 3966 */ 3967 if (label != NULL) { 3968 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3969 hostid = fnvlist_lookup_uint64(label, 3970 ZPOOL_CONFIG_HOSTID); 3971 3972 if (nvlist_exists(config, ZPOOL_CONFIG_POOL_STATE)) 3973 state = fnvlist_lookup_uint64(config, 3974 ZPOOL_CONFIG_POOL_STATE); 3975 3976 if (spa_get_hostid(spa) && hostid == spa_get_hostid(spa) && 3977 state == POOL_STATE_EXPORTED) { 3978 zfs_dbgmsg("mmp: skipping check: hostid matches " 3979 "and pool is exported, spa=%s, hostid=%llx", 3980 spa_load_name(spa), (u_longlong_t)hostid); 3981 return (B_FALSE); 3982 } 3983 3984 if (state == POOL_STATE_DESTROYED) { 3985 zfs_dbgmsg("mmp: skipping check: intentionally " 3986 "destroyed pool, spa=%s", spa_load_name(spa)); 3987 return (B_FALSE); 3988 } 3989 } 3990 3991 return (B_TRUE); 3992 } 3993 3994 /* 3995 * Nanoseconds the activity check must watch for changes on-disk. 3996 */ 3997 static uint64_t 3998 spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3999 { 4000 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 4001 uint64_t multihost_interval = MSEC2NSEC( 4002 MMP_INTERVAL_OK(zfs_multihost_interval)); 4003 uint64_t import_delay = MAX(NANOSEC, import_intervals * 4004 multihost_interval); 4005 4006 /* 4007 * Local tunables determine a minimum duration except for the case 4008 * where we know when the remote host will suspend the pool if MMP 4009 * writes do not land. 4010 * 4011 * See Big Theory comment at the top of mmp.c for the reasoning behind 4012 * these cases and times. 4013 */ 4014 4015 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 4016 4017 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 4018 MMP_FAIL_INT(ub) > 0) { 4019 4020 /* MMP on remote host will suspend pool after failed writes */ 4021 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 4022 MMP_IMPORT_SAFETY_FACTOR / 100; 4023 4024 zfs_dbgmsg("mmp: settings spa=%s fail_intvals>0 " 4025 "import_delay=%llu mmp_fails=%llu mmp_interval=%llu " 4026 "import_intervals=%llu", spa_load_name(spa), 4027 (u_longlong_t)import_delay, 4028 (u_longlong_t)MMP_FAIL_INT(ub), 4029 (u_longlong_t)MMP_INTERVAL(ub), 4030 (u_longlong_t)import_intervals); 4031 4032 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 4033 MMP_FAIL_INT(ub) == 0) { 4034 4035 /* MMP on remote host will never suspend pool */ 4036 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 4037 ub->ub_mmp_delay) * import_intervals); 4038 4039 zfs_dbgmsg("mmp: settings spa=%s fail_intvals=0 " 4040 "import_delay=%llu mmp_interval=%llu ub_mmp_delay=%llu " 4041 "import_intervals=%llu", spa_load_name(spa), 4042 (u_longlong_t)import_delay, 4043 (u_longlong_t)MMP_INTERVAL(ub), 4044 (u_longlong_t)ub->ub_mmp_delay, 4045 (u_longlong_t)import_intervals); 4046 4047 } else if (MMP_VALID(ub)) { 4048 /* 4049 * zfs-0.7 compatibility case 4050 */ 4051 4052 import_delay = MAX(import_delay, (multihost_interval + 4053 ub->ub_mmp_delay) * import_intervals); 4054 4055 zfs_dbgmsg("mmp: settings spa=%s import_delay=%llu " 4056 "ub_mmp_delay=%llu import_intervals=%llu leaves=%u", 4057 spa_load_name(spa), (u_longlong_t)import_delay, 4058 (u_longlong_t)ub->ub_mmp_delay, 4059 (u_longlong_t)import_intervals, 4060 vdev_count_leaves(spa)); 4061 } else { 4062 /* Using local tunings is the only reasonable option */ 4063 zfs_dbgmsg("mmp: pool last imported on non-MMP aware " 4064 "host using settings spa=%s import_delay=%llu " 4065 "multihost_interval=%llu import_intervals=%llu", 4066 spa_load_name(spa), (u_longlong_t)import_delay, 4067 (u_longlong_t)multihost_interval, 4068 (u_longlong_t)import_intervals); 4069 } 4070 4071 return (import_delay); 4072 } 4073 4074 /* 4075 * Store the observed pool status in spa->spa_load_info nvlist. If the 4076 * remote hostname or hostid are available from configuration read from 4077 * disk store them as well. Additionally, provide some diagnostic info 4078 * for which activity checks were run and their duration. This allows 4079 * 'zpool import' to generate a more useful message. 4080 * 4081 * Mandatory observed pool status 4082 * - ZPOOL_CONFIG_MMP_STATE - observed pool status (active/inactive) 4083 * - ZPOOL_CONFIG_MMP_TXG - observed pool txg number 4084 * - ZPOOL_CONFIG_MMP_SEQ - observed pool sequence id 4085 * 4086 * Optional information for detailed reporting 4087 * - ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 4088 * - ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 4089 * - ZPOOL_CONFIG_MMP_RESULT - set to result of activity check 4090 * - ZPOOL_CONFIG_MMP_TRYIMPORT_NS - tryimport duration in nanosec 4091 * - ZPOOL_CONFIG_MMP_IMPORT_NS - import duration in nanosec 4092 * - ZPOOL_CONFIG_MMP_CLAIM_NS - claim duration in nanosec 4093 * 4094 * ZPOOL_CONFIG_MMP_RESULT can be set to: 4095 * - ENXIO - system hostid not set 4096 * - ESRCH - activity check skipped 4097 * - EREMOTEIO - activity check detected active pool 4098 * - ENODEV - claim could not be written to a device the config expects 4099 * - EIO - claim writes were issued to present devices and failed 4100 * - EINTR - activity check interrupted 4101 * - 0 - activity check detected no activity 4102 * 4103 * ENODEV and EIO are reported with ZPOOL_CONFIG_MMP_STATE set to 4104 * MMP_STATE_ACTIVE even though no remote host was seen. Nothing is actually 4105 * active in either case, but an older zpool(8) knows only the two existing 4106 * states and reaches zfs_error_aux() with an uninitialized buffer for any 4107 * other value, so the state is kept as one it understands and the real cause 4108 * travels in the result. 4109 */ 4110 static void 4111 spa_activity_set_load_info(spa_t *spa, nvlist_t *label, mmp_state_t state, 4112 uint64_t txg, uint16_t seq, int error) 4113 { 4114 mmp_thread_t *mmp = &spa->spa_mmp; 4115 const char *hostname = NULL; 4116 uint64_t hostid = 0; 4117 4118 /* Always report a zero txg and seq id for active pools. */ 4119 if (state == MMP_STATE_ACTIVE) { 4120 ASSERT0(txg); 4121 ASSERT0(seq); 4122 } 4123 4124 if (label) { 4125 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTNAME)) { 4126 hostname = fnvlist_lookup_string(label, 4127 ZPOOL_CONFIG_HOSTNAME); 4128 fnvlist_add_string(spa->spa_load_info, 4129 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 4130 } 4131 4132 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) { 4133 hostid = fnvlist_lookup_uint64(label, 4134 ZPOOL_CONFIG_HOSTID); 4135 fnvlist_add_uint64(spa->spa_load_info, 4136 ZPOOL_CONFIG_MMP_HOSTID, hostid); 4137 } 4138 } 4139 4140 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_MMP_STATE, state); 4141 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_MMP_TXG, txg); 4142 fnvlist_add_uint16(spa->spa_load_info, ZPOOL_CONFIG_MMP_SEQ, seq); 4143 fnvlist_add_uint32(spa->spa_load_info, ZPOOL_CONFIG_MMP_RESULT, error); 4144 4145 if (mmp->mmp_tryimport_ns > 0) { 4146 fnvlist_add_uint64(spa->spa_load_info, 4147 ZPOOL_CONFIG_MMP_TRYIMPORT_NS, mmp->mmp_tryimport_ns); 4148 } 4149 4150 if (mmp->mmp_import_ns > 0) { 4151 fnvlist_add_uint64(spa->spa_load_info, 4152 ZPOOL_CONFIG_MMP_IMPORT_NS, mmp->mmp_import_ns); 4153 } 4154 4155 if (mmp->mmp_claim_ns > 0) { 4156 fnvlist_add_uint64(spa->spa_load_info, 4157 ZPOOL_CONFIG_MMP_CLAIM_NS, mmp->mmp_claim_ns); 4158 } 4159 4160 zfs_dbgmsg("mmp: set spa_load_info, spa=%s hostname=%s hostid=%llx " 4161 "state=%d txg=%llu seq=%llu tryimport_ns=%lld import_ns=%lld " 4162 "claim_ns=%lld", spa_load_name(spa), 4163 hostname != NULL ? hostname : "none", (u_longlong_t)hostid, 4164 (int)state, (u_longlong_t)txg, (u_longlong_t)seq, 4165 (longlong_t)mmp->mmp_tryimport_ns, (longlong_t)mmp->mmp_import_ns, 4166 (longlong_t)mmp->mmp_claim_ns); 4167 } 4168 4169 static int 4170 spa_ld_activity_result(spa_t *spa, int error, const char *state) 4171 { 4172 switch (error) { 4173 case ENXIO: 4174 cmn_err(CE_WARN, "pool '%s' system hostid not set, " 4175 "aborted import during %s", spa_load_name(spa), state); 4176 /* Userspace expects EREMOTEIO for no system hostid */ 4177 error = EREMOTEIO; 4178 break; 4179 case ENODEV: 4180 cmn_err(CE_WARN, "pool '%s' could not claim every device the " 4181 "config expects present, aborted import during %s; if a " 4182 "device is permanently gone see 'zhack mmp reclaim'", 4183 spa_load_name(spa), state); 4184 /* Userspace expects EREMOTEIO for a failed claim */ 4185 error = EREMOTEIO; 4186 break; 4187 case EIO: 4188 cmn_err(CE_WARN, "pool '%s' had I/O errors writing the claim, " 4189 "aborted import during %s", spa_load_name(spa), state); 4190 /* Userspace expects EREMOTEIO for a failed claim */ 4191 error = EREMOTEIO; 4192 break; 4193 case EREMOTEIO: 4194 cmn_err(CE_WARN, "pool '%s' activity detected, aborted " 4195 "import during %s", spa_load_name(spa), state); 4196 break; 4197 case EINTR: 4198 cmn_err(CE_WARN, "pool '%s' activity check, interrupted " 4199 "import during %s", spa_load_name(spa), state); 4200 break; 4201 case 0: 4202 cmn_err(CE_NOTE, "pool '%s' activity check completed " 4203 "successfully", spa_load_name(spa)); 4204 break; 4205 } 4206 4207 return (error); 4208 } 4209 4210 4211 /* 4212 * Remote host activity check. Performed during tryimport when the pool 4213 * has passed on the basic sanity check and is open read-only. 4214 * 4215 * error results: 4216 * 0 - no activity detected 4217 * EREMOTEIO - remote activity detected 4218 * EINTR - user canceled the operation 4219 */ 4220 static int 4221 spa_activity_check_tryimport(spa_t *spa, uberblock_t *spa_ub, 4222 boolean_t importing) 4223 { 4224 kcondvar_t cv; 4225 kmutex_t mtx; 4226 int error = 0; 4227 4228 cv_init(&cv, NULL, CV_DEFAULT, NULL); 4229 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 4230 mutex_enter(&mtx); 4231 4232 uint64_t import_delay = spa_activity_check_duration(spa, spa_ub); 4233 hrtime_t start_time = gethrtime(); 4234 4235 /* Add a small random factor in case of simultaneous imports (0-25%) */ 4236 import_delay += import_delay * random_in_range(250) / 1000; 4237 hrtime_t import_expire = gethrtime() + import_delay; 4238 4239 if (importing) { 4240 /* Console message includes tryimport and claim time */ 4241 hrtime_t extra_delay = MMP_IMPORT_VERIFY_ITERS * 4242 MSEC2NSEC(MMP_INTERVAL_VALID(spa_ub) ? 4243 MMP_INTERVAL(spa_ub) : MMP_MIN_INTERVAL); 4244 cmn_err(CE_NOTE, "pool '%s' activity check required, " 4245 "%llu seconds remaining", spa_load_name(spa), 4246 (u_longlong_t)MAX(NSEC2SEC(import_delay + extra_delay), 1)); 4247 spa_import_progress_set_notes(spa, "Checking MMP activity, " 4248 "waiting %llu ms", (u_longlong_t)NSEC2MSEC(import_delay)); 4249 } 4250 4251 hrtime_t now; 4252 nvlist_t *mmp_label = NULL; 4253 4254 while ((now = gethrtime()) < import_expire) { 4255 vdev_t *rvd = spa->spa_root_vdev; 4256 uberblock_t mmp_ub; 4257 4258 if (importing) { 4259 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 4260 NSEC2SEC(import_expire - gethrtime())); 4261 } 4262 4263 vdev_uberblock_load(rvd, &mmp_ub, &mmp_label); 4264 4265 if (vdev_uberblock_compare(spa_ub, &mmp_ub)) { 4266 spa_load_failed(spa, "mmp: activity detected during " 4267 "tryimport, spa_ub_txg=%llu mmp_ub_txg=%llu " 4268 "spa_ub_seq=%llu mmp_ub_seq=%llu " 4269 "spa_ub_timestamp=%llu mmp_ub_timestamp=%llu " 4270 "spa_ub_config=%#llx mmp_ub_config=%#llx", 4271 (u_longlong_t)spa_ub->ub_txg, 4272 (u_longlong_t)mmp_ub.ub_txg, 4273 (u_longlong_t)(MMP_SEQ_VALID(spa_ub) ? 4274 MMP_SEQ(spa_ub) : 0), 4275 (u_longlong_t)(MMP_SEQ_VALID(&mmp_ub) ? 4276 MMP_SEQ(&mmp_ub) : 0), 4277 (u_longlong_t)spa_ub->ub_timestamp, 4278 (u_longlong_t)mmp_ub.ub_timestamp, 4279 (u_longlong_t)spa_ub->ub_mmp_config, 4280 (u_longlong_t)mmp_ub.ub_mmp_config); 4281 error = SET_ERROR(EREMOTEIO); 4282 break; 4283 } 4284 4285 if (mmp_label) { 4286 nvlist_free(mmp_label); 4287 mmp_label = NULL; 4288 } 4289 4290 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 4291 if (error != -1) { 4292 error = SET_ERROR(EINTR); 4293 break; 4294 } 4295 error = 0; 4296 } 4297 4298 mutex_exit(&mtx); 4299 mutex_destroy(&mtx); 4300 cv_destroy(&cv); 4301 4302 if (mmp_label) 4303 nvlist_free(mmp_label); 4304 4305 if (spa->spa_load_state == SPA_LOAD_IMPORT || 4306 spa->spa_load_state == SPA_LOAD_OPEN) { 4307 spa->spa_mmp.mmp_import_ns = gethrtime() - start_time; 4308 } else { 4309 spa->spa_mmp.mmp_tryimport_ns = gethrtime() - start_time; 4310 } 4311 4312 return (error); 4313 } 4314 4315 /* 4316 * Remote host activity check. Performed during import when the pool has 4317 * passed most sanity check and has been reopened read/write. 4318 * 4319 * error results: 4320 * 0 - no activity detected 4321 * EREMOTEIO - remote activity detected 4322 * ENODEV - the claim could not be written to a device the config 4323 * expects to be present 4324 * EIO - the claim writes were issued to present devices and failed 4325 * EINTR - user canceled the operation 4326 */ 4327 static int 4328 spa_activity_check_claim(spa_t *spa) 4329 { 4330 vdev_t *rvd = spa->spa_root_vdev; 4331 nvlist_t *mmp_label; 4332 uberblock_t spa_ub; 4333 kcondvar_t cv; 4334 kmutex_t mtx; 4335 int error = 0; 4336 4337 cv_init(&cv, NULL, CV_DEFAULT, NULL); 4338 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 4339 mutex_enter(&mtx); 4340 4341 hrtime_t start_time = gethrtime(); 4342 4343 /* 4344 * Load the best uberblock and verify it matches the uberblock already 4345 * identified and stored as spa->spa_uberblock to verify the pool has 4346 * not changed. 4347 */ 4348 vdev_uberblock_load(rvd, &spa_ub, &mmp_label); 4349 4350 if (memcmp(&spa->spa_uberblock, &spa_ub, sizeof (uberblock_t))) { 4351 spa_load_failed(spa, "mmp: uberblock changed on disk"); 4352 error = SET_ERROR(EREMOTEIO); 4353 goto out; 4354 } 4355 4356 if (!MMP_VALID(&spa_ub) || !MMP_INTERVAL_VALID(&spa_ub) || 4357 !MMP_SEQ_VALID(&spa_ub) || !MMP_FAIL_INT_VALID(&spa_ub)) { 4358 spa_load_failed(spa, "mmp: is not enabled in spa uberblock"); 4359 error = SET_ERROR(EREMOTEIO); 4360 goto out; 4361 } 4362 4363 nvlist_free(mmp_label); 4364 mmp_label = NULL; 4365 4366 uint64_t spa_ub_interval = MMP_INTERVAL(&spa_ub); 4367 uint16_t spa_ub_seq = MMP_SEQ(&spa_ub); 4368 4369 /* 4370 * In the highly unlikely event the sequence numbers have been 4371 * exhaused reset the sequence to zero. As long as the MMP 4372 * uberblock is updated on all of the vdevs the activity will 4373 * still be detected. 4374 */ 4375 if (MMP_SEQ_MAX == spa_ub_seq) 4376 spa_ub_seq = 0; 4377 4378 spa_import_progress_set_notes(spa, 4379 "Establishing MMP claim, waiting %llu ms", 4380 (u_longlong_t)(MMP_IMPORT_VERIFY_ITERS * spa_ub_interval)); 4381 4382 /* 4383 * Repeatedly sync out an MMP uberblock with a randomly selected 4384 * sequence number, then read it back after the MMP interval. This 4385 * random value acts as a claim token and is visible on other hosts. 4386 * If the same random value is read back we can be certain no other 4387 * pool is attempting to import the pool. 4388 */ 4389 for (int i = MMP_IMPORT_VERIFY_ITERS; i > 0; i--) { 4390 uberblock_t set_ub, mmp_ub; 4391 uint16_t mmp_seq; 4392 4393 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 4394 NSEC2SEC(i * MSEC2NSEC(spa_ub_interval))); 4395 4396 set_ub = spa_ub; 4397 mmp_seq = spa_ub_seq + 1 + 4398 random_in_range(MMP_SEQ_MAX - spa_ub_seq); 4399 MMP_SEQ_CLEAR(&set_ub); 4400 set_ub.ub_mmp_config |= MMP_SEQ_SET(mmp_seq); 4401 4402 error = mmp_claim_uberblock(spa, rvd, &set_ub); 4403 if (error) { 4404 spa_load_failed(spa, "mmp: uberblock claim " 4405 "failed, error=%d", error); 4406 /* 4407 * ENODEV and EIO are both kept distinct from the 4408 * EREMOTEIO returned when another host is seen below. 4409 * Failing to write the claim is not evidence of a 4410 * remote host, and only the ENODEV case has a 4411 * recovery. 4412 */ 4413 break; 4414 } 4415 4416 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + 4417 MSEC_TO_TICK(spa_ub_interval)); 4418 if (error != -1) { 4419 error = SET_ERROR(EINTR); 4420 break; 4421 } 4422 4423 vdev_uberblock_load(rvd, &mmp_ub, &mmp_label); 4424 4425 if (vdev_uberblock_compare(&set_ub, &mmp_ub)) { 4426 spa_load_failed(spa, "mmp: activity detected during " 4427 "claim, set_ub_txg=%llu mmp_ub_txg=%llu " 4428 "set_ub_seq=%llu mmp_ub_seq=%llu " 4429 "set_ub_timestamp=%llu mmp_ub_timestamp=%llu " 4430 "set_ub_config=%#llx mmp_ub_config=%#llx", 4431 (u_longlong_t)set_ub.ub_txg, 4432 (u_longlong_t)mmp_ub.ub_txg, 4433 (u_longlong_t)(MMP_SEQ_VALID(&set_ub) ? 4434 MMP_SEQ(&set_ub) : 0), 4435 (u_longlong_t)(MMP_SEQ_VALID(&mmp_ub) ? 4436 MMP_SEQ(&mmp_ub) : 0), 4437 (u_longlong_t)set_ub.ub_timestamp, 4438 (u_longlong_t)mmp_ub.ub_timestamp, 4439 (u_longlong_t)set_ub.ub_mmp_config, 4440 (u_longlong_t)mmp_ub.ub_mmp_config); 4441 error = SET_ERROR(EREMOTEIO); 4442 break; 4443 } 4444 4445 if (mmp_label) { 4446 nvlist_free(mmp_label); 4447 mmp_label = NULL; 4448 } 4449 4450 error = 0; 4451 } 4452 out: 4453 spa->spa_mmp.mmp_claim_ns = gethrtime() - start_time; 4454 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 0); 4455 4456 /* 4457 * A claim shortfall reaches userspace as EREMOTEIO exactly as remote 4458 * activity does, so an older zpool(8) sees no change. The cause 4459 * travels in the result for a zpool(8) which knows to read it. 4460 */ 4461 if (error == EREMOTEIO || error == ENODEV || error == EIO) { 4462 spa_activity_set_load_info(spa, mmp_label, 4463 MMP_STATE_ACTIVE, 0, 0, error); 4464 } else { 4465 spa_activity_set_load_info(spa, mmp_label, 4466 MMP_STATE_INACTIVE, spa_ub.ub_txg, MMP_SEQ(&spa_ub), 0); 4467 } 4468 4469 /* 4470 * Restore the original sequence, this allows us to retry the 4471 * import procedure if a subsequent step fails during import. 4472 * Failure to restore it reduces the available sequence ids for 4473 * the next import but shouldn't be considered fatal. 4474 */ 4475 int restore_error = mmp_claim_uberblock(spa, rvd, &spa_ub); 4476 if (restore_error) { 4477 zfs_dbgmsg("mmp: uberblock restore failed, spa=%s error=%d", 4478 spa_load_name(spa), restore_error); 4479 } 4480 4481 if (mmp_label) 4482 nvlist_free(mmp_label); 4483 4484 mutex_exit(&mtx); 4485 mutex_destroy(&mtx); 4486 cv_destroy(&cv); 4487 4488 return (error); 4489 } 4490 4491 static int 4492 spa_ld_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *label) 4493 { 4494 vdev_t *rvd = spa->spa_root_vdev; 4495 int error; 4496 4497 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 4498 spa_get_hostid(spa) == 0) { 4499 spa_activity_set_load_info(spa, label, MMP_STATE_NO_HOSTID, 4500 ub->ub_txg, MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0, ENXIO); 4501 zfs_dbgmsg("mmp: system hostid not set, ub_mmp_magic=%llx " 4502 "ub_mmp_delay=%llu hostid=%llx", 4503 (u_longlong_t)ub->ub_mmp_magic, 4504 (u_longlong_t)ub->ub_mmp_delay, 4505 (u_longlong_t)spa_get_hostid(spa)); 4506 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, ENXIO)); 4507 } 4508 4509 switch (spa->spa_load_state) { 4510 case SPA_LOAD_TRYIMPORT: 4511 tryimport: 4512 error = spa_activity_check_tryimport(spa, ub, B_TRUE); 4513 if (error == EREMOTEIO) { 4514 spa_activity_set_load_info(spa, label, 4515 MMP_STATE_ACTIVE, 0, 0, EREMOTEIO); 4516 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4517 } else if (error) { 4518 ASSERT3S(error, ==, EINTR); 4519 spa_activity_set_load_info(spa, label, 4520 MMP_STATE_ACTIVE, 0, 0, EINTR); 4521 return (error); 4522 } 4523 4524 spa_activity_set_load_info(spa, label, MMP_STATE_INACTIVE, 4525 ub->ub_txg, MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0, 0); 4526 4527 break; 4528 4529 case SPA_LOAD_IMPORT: 4530 case SPA_LOAD_OPEN: 4531 error = spa_activity_verify_config(spa, ub); 4532 if (error == EREMOTEIO) { 4533 spa_activity_set_load_info(spa, label, 4534 MMP_STATE_ACTIVE, 0, 0, EREMOTEIO); 4535 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4536 } else if (error) { 4537 ASSERT3S(error, ==, ENOENT); 4538 goto tryimport; 4539 } 4540 4541 /* Load info set in spa_activity_check_claim() */ 4542 4543 break; 4544 4545 case SPA_LOAD_RECOVER: 4546 zfs_dbgmsg("mmp: skipping mmp check for rewind, spa=%s", 4547 spa_load_name(spa)); 4548 break; 4549 4550 default: 4551 spa_activity_set_load_info(spa, label, MMP_STATE_ACTIVE, 4552 0, 0, EREMOTEIO); 4553 zfs_dbgmsg("mmp: unreachable, spa=%s spa_load_state=%d", 4554 spa_load_name(spa), spa->spa_load_state); 4555 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4556 } 4557 4558 return (0); 4559 } 4560 4561 /* 4562 * Called from zfs_ioc_clear for a pool that was suspended 4563 * after failing mmp write checks. 4564 */ 4565 boolean_t 4566 spa_mmp_remote_host_activity(spa_t *spa) 4567 { 4568 ASSERT(spa_multihost(spa) && spa_suspended(spa)); 4569 4570 nvlist_t *best_label; 4571 uberblock_t best_ub; 4572 4573 /* 4574 * Locate the best uberblock on disk 4575 */ 4576 vdev_uberblock_load(spa->spa_root_vdev, &best_ub, &best_label); 4577 if (best_label) { 4578 /* 4579 * confirm that the best hostid matches our hostid 4580 */ 4581 if (nvlist_exists(best_label, ZPOOL_CONFIG_HOSTID) && 4582 spa_get_hostid(spa) != 4583 fnvlist_lookup_uint64(best_label, ZPOOL_CONFIG_HOSTID)) { 4584 nvlist_free(best_label); 4585 return (B_TRUE); 4586 } 4587 nvlist_free(best_label); 4588 } else { 4589 return (B_TRUE); 4590 } 4591 4592 if (!MMP_VALID(&best_ub) || 4593 !MMP_FAIL_INT_VALID(&best_ub) || 4594 MMP_FAIL_INT(&best_ub) == 0) { 4595 return (B_TRUE); 4596 } 4597 4598 if (best_ub.ub_txg != spa->spa_uberblock.ub_txg || 4599 best_ub.ub_timestamp != spa->spa_uberblock.ub_timestamp) { 4600 zfs_dbgmsg("mmp: txg mismatch detected during pool clear, " 4601 "spa=%s txg=%llu ub_txg=%llu timestamp=%llu " 4602 "ub_timestamp=%llu", spa_name(spa), 4603 (u_longlong_t)spa->spa_uberblock.ub_txg, 4604 (u_longlong_t)best_ub.ub_txg, 4605 (u_longlong_t)spa->spa_uberblock.ub_timestamp, 4606 (u_longlong_t)best_ub.ub_timestamp); 4607 return (B_TRUE); 4608 } 4609 4610 /* 4611 * Perform an activity check looking for any remote writer 4612 */ 4613 return (spa_activity_check_tryimport(spa, &best_ub, B_FALSE) != 0); 4614 } 4615 4616 static int 4617 spa_verify_host(spa_t *spa, nvlist_t *mos_config) 4618 { 4619 uint64_t hostid; 4620 const char *hostname; 4621 uint64_t myhostid = 0; 4622 4623 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 4624 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 4625 hostname = fnvlist_lookup_string(mos_config, 4626 ZPOOL_CONFIG_HOSTNAME); 4627 4628 myhostid = zone_get_hostid(NULL); 4629 4630 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 4631 cmn_err(CE_WARN, "pool '%s' could not be " 4632 "loaded as it was last accessed by " 4633 "another system (host: %s hostid: 0x%llx). " 4634 "See: https://openzfs.github.io/openzfs-docs/msg/" 4635 "ZFS-8000-EY", 4636 spa_name(spa), hostname, (u_longlong_t)hostid); 4637 spa_load_failed(spa, "hostid verification failed: pool " 4638 "last accessed by host: %s (hostid: 0x%llx)", 4639 hostname, (u_longlong_t)hostid); 4640 return (SET_ERROR(EBADF)); 4641 } 4642 } 4643 4644 return (0); 4645 } 4646 4647 static int 4648 spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 4649 { 4650 int error = 0; 4651 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 4652 int parse; 4653 vdev_t *rvd; 4654 uint64_t pool_guid; 4655 const char *comment; 4656 const char *compatibility; 4657 4658 /* 4659 * Versioning wasn't explicitly added to the label until later, so if 4660 * it's not present treat it as the initial version. 4661 */ 4662 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 4663 &spa->spa_ubsync.ub_version) != 0) 4664 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 4665 4666 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 4667 spa_load_failed(spa, "invalid config provided: '%s' missing", 4668 ZPOOL_CONFIG_POOL_GUID); 4669 return (SET_ERROR(EINVAL)); 4670 } 4671 4672 /* 4673 * If we are doing an import, ensure that the pool is not already 4674 * imported by checking if its pool guid already exists in the 4675 * spa namespace. 4676 * 4677 * The only case that we allow an already imported pool to be 4678 * imported again, is when the pool is checkpointed and we want to 4679 * look at its checkpointed state from userland tools like zdb. 4680 */ 4681 #ifdef _KERNEL 4682 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 4683 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 4684 spa_guid_exists(pool_guid, 0)) { 4685 #else 4686 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 4687 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 4688 spa_guid_exists(pool_guid, 0) && 4689 !spa_importing_readonly_checkpoint(spa)) { 4690 #endif 4691 spa_load_failed(spa, "a pool with guid %llu is already open", 4692 (u_longlong_t)pool_guid); 4693 return (SET_ERROR(EEXIST)); 4694 } 4695 4696 spa->spa_config_guid = pool_guid; 4697 4698 nvlist_free(spa->spa_load_info); 4699 spa->spa_load_info = fnvlist_alloc(); 4700 4701 ASSERT0P(spa->spa_comment); 4702 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 4703 spa->spa_comment = spa_strdup(comment); 4704 4705 ASSERT0P(spa->spa_compatibility); 4706 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 4707 &compatibility) == 0) 4708 spa->spa_compatibility = spa_strdup(compatibility); 4709 4710 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 4711 &spa->spa_config_txg); 4712 4713 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 4714 spa->spa_config_splitting = fnvlist_dup(nvl); 4715 4716 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 4717 spa_load_failed(spa, "invalid config provided: '%s' missing", 4718 ZPOOL_CONFIG_VDEV_TREE); 4719 return (SET_ERROR(EINVAL)); 4720 } 4721 4722 /* 4723 * Create "The Godfather" zio to hold all async IOs 4724 */ 4725 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 4726 KM_SLEEP); 4727 for (int i = 0; i < max_ncpus; i++) { 4728 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 4729 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 4730 ZIO_FLAG_GODFATHER); 4731 } 4732 4733 /* 4734 * Parse the configuration into a vdev tree. We explicitly set the 4735 * value that will be returned by spa_version() since parsing the 4736 * configuration requires knowing the version number. 4737 */ 4738 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4739 parse = (type == SPA_IMPORT_EXISTING ? 4740 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 4741 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 4742 spa_config_exit(spa, SCL_ALL, FTAG); 4743 4744 if (error != 0) { 4745 spa_load_failed(spa, "unable to parse config [error=%d]", 4746 error); 4747 return (error); 4748 } 4749 4750 ASSERT(spa->spa_root_vdev == rvd); 4751 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 4752 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 4753 4754 if (type != SPA_IMPORT_ASSEMBLE) { 4755 ASSERT(spa_guid(spa) == pool_guid); 4756 } 4757 4758 return (0); 4759 } 4760 4761 /* 4762 * Recursively open all vdevs in the vdev tree. This function is called twice: 4763 * first with the untrusted config, then with the trusted config. 4764 */ 4765 static int 4766 spa_ld_open_vdevs(spa_t *spa) 4767 { 4768 int error = 0; 4769 4770 /* 4771 * spa_missing_tvds_allowed defines how many top-level vdevs can be 4772 * missing/unopenable for the root vdev to be still considered openable. 4773 */ 4774 if (spa->spa_trust_config) { 4775 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 4776 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 4777 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 4778 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 4779 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 4780 } else { 4781 spa->spa_missing_tvds_allowed = 0; 4782 } 4783 4784 spa->spa_missing_tvds_allowed = 4785 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 4786 4787 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4788 error = vdev_open(spa->spa_root_vdev, CRED()); 4789 spa_config_exit(spa, SCL_ALL, FTAG); 4790 4791 if (spa->spa_missing_tvds != 0) { 4792 spa_load_note(spa, "vdev tree has %lld missing top-level " 4793 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 4794 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 4795 /* 4796 * Although theoretically we could allow users to open 4797 * incomplete pools in RW mode, we'd need to add a lot 4798 * of extra logic (e.g. adjust pool space to account 4799 * for missing vdevs). 4800 * This limitation also prevents users from accidentally 4801 * opening the pool in RW mode during data recovery and 4802 * damaging it further. 4803 */ 4804 spa_load_note(spa, "pools with missing top-level " 4805 "vdevs can only be opened in read-only mode."); 4806 error = SET_ERROR(ENXIO); 4807 } else { 4808 spa_load_note(spa, "current settings allow for maximum " 4809 "%lld missing top-level vdevs at this stage.", 4810 (u_longlong_t)spa->spa_missing_tvds_allowed); 4811 } 4812 } 4813 if (error != 0) { 4814 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 4815 error); 4816 } 4817 if (spa->spa_missing_tvds != 0 || error != 0) 4818 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 4819 4820 return (error); 4821 } 4822 4823 /* 4824 * We need to validate the vdev labels against the configuration that 4825 * we have in hand. This function is called twice: first with an untrusted 4826 * config, then with a trusted config. The validation is more strict when the 4827 * config is trusted. 4828 */ 4829 static int 4830 spa_ld_validate_vdevs(spa_t *spa) 4831 { 4832 int error = 0; 4833 vdev_t *rvd = spa->spa_root_vdev; 4834 4835 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4836 error = vdev_validate(rvd); 4837 spa_config_exit(spa, SCL_ALL, FTAG); 4838 4839 if (error != 0) { 4840 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 4841 return (error); 4842 } 4843 4844 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 4845 spa_load_failed(spa, "cannot open vdev tree after invalidating " 4846 "some vdevs"); 4847 vdev_dbgmsg_print_tree(rvd, 2); 4848 return (SET_ERROR(ENXIO)); 4849 } 4850 4851 return (0); 4852 } 4853 4854 static void 4855 spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 4856 { 4857 spa->spa_state = POOL_STATE_ACTIVE; 4858 spa->spa_ubsync = spa->spa_uberblock; 4859 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 4860 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 4861 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 4862 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 4863 spa->spa_claim_max_txg = spa->spa_first_txg; 4864 spa->spa_prev_software_version = ub->ub_software_version; 4865 } 4866 4867 static int 4868 spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 4869 { 4870 vdev_t *rvd = spa->spa_root_vdev; 4871 nvlist_t *label; 4872 uberblock_t *ub = &spa->spa_uberblock; 4873 4874 /* 4875 * If we are opening the checkpointed state of the pool by 4876 * rewinding to it, at this point we will have written the 4877 * checkpointed uberblock to the vdev labels, so searching 4878 * the labels will find the right uberblock. However, if 4879 * we are opening the checkpointed state read-only, we have 4880 * not modified the labels. Therefore, we must ignore the 4881 * labels and continue using the spa_uberblock that was set 4882 * by spa_ld_checkpoint_rewind. 4883 * 4884 * Note that it would be fine to ignore the labels when 4885 * rewinding (opening writeable) as well. However, if we 4886 * crash just after writing the labels, we will end up 4887 * searching the labels. Doing so in the common case means 4888 * that this code path gets exercised normally, rather than 4889 * just in the edge case. 4890 */ 4891 if (ub->ub_checkpoint_txg != 0 && 4892 spa_importing_readonly_checkpoint(spa)) { 4893 spa_ld_select_uberblock_done(spa, ub); 4894 return (0); 4895 } 4896 4897 /* 4898 * Find the best uberblock. 4899 */ 4900 vdev_uberblock_load(rvd, ub, &label); 4901 4902 /* 4903 * If we weren't able to find a single valid uberblock, return failure. 4904 */ 4905 if (ub->ub_txg == 0) { 4906 nvlist_free(label); 4907 spa_load_failed(spa, "no valid uberblock found"); 4908 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 4909 } 4910 4911 if (spa->spa_load_max_txg != UINT64_MAX) { 4912 (void) spa_import_progress_set_max_txg(spa_guid(spa), 4913 (u_longlong_t)spa->spa_load_max_txg); 4914 } 4915 spa_load_note(spa, "using uberblock with txg=%llu", 4916 (u_longlong_t)ub->ub_txg); 4917 if (ub->ub_raidz_reflow_info != 0) { 4918 spa_load_note(spa, "uberblock raidz_reflow_info: " 4919 "state=%u offset=%llu", 4920 (int)RRSS_GET_STATE(ub), 4921 (u_longlong_t)RRSS_GET_OFFSET(ub)); 4922 } 4923 4924 /* 4925 * For pools which have the multihost property on determine if the 4926 * pool is truly inactive and can be safely imported. Prevent 4927 * hosts which don't have a hostid set from importing the pool. 4928 */ 4929 spa->spa_activity_check = spa_activity_check_required(spa, ub, label); 4930 if (spa->spa_activity_check) { 4931 int error = spa_ld_activity_check(spa, ub, label); 4932 if (error) { 4933 spa_load_state_t state = spa->spa_load_state; 4934 error = spa_ld_activity_result(spa, error, 4935 state == SPA_LOAD_TRYIMPORT ? "tryimport" : 4936 state == SPA_LOAD_IMPORT ? "import" : "open"); 4937 nvlist_free(label); 4938 return (error); 4939 } 4940 } else { 4941 fnvlist_add_uint32(spa->spa_load_info, 4942 ZPOOL_CONFIG_MMP_RESULT, ESRCH); 4943 } 4944 4945 /* 4946 * If the pool has an unsupported version we can't open it. 4947 */ 4948 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 4949 nvlist_free(label); 4950 spa_load_failed(spa, "version %llu is not supported", 4951 (u_longlong_t)ub->ub_version); 4952 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 4953 } 4954 4955 if (ub->ub_version >= SPA_VERSION_FEATURES) { 4956 nvlist_t *features; 4957 4958 /* 4959 * If we weren't able to find what's necessary for reading the 4960 * MOS in the label, return failure. 4961 */ 4962 if (label == NULL) { 4963 spa_load_failed(spa, "label config unavailable"); 4964 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4965 ENXIO)); 4966 } 4967 4968 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 4969 &features) != 0) { 4970 nvlist_free(label); 4971 spa_load_failed(spa, "invalid label: '%s' missing", 4972 ZPOOL_CONFIG_FEATURES_FOR_READ); 4973 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4974 ENXIO)); 4975 } 4976 4977 /* 4978 * Update our in-core representation with the definitive values 4979 * from the label. 4980 */ 4981 nvlist_free(spa->spa_label_features); 4982 spa->spa_label_features = fnvlist_dup(features); 4983 } 4984 4985 nvlist_free(label); 4986 4987 /* 4988 * Look through entries in the label nvlist's features_for_read. If 4989 * there is a feature listed there which we don't understand then we 4990 * cannot open a pool. 4991 */ 4992 if (ub->ub_version >= SPA_VERSION_FEATURES) { 4993 nvlist_t *unsup_feat; 4994 4995 unsup_feat = fnvlist_alloc(); 4996 4997 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 4998 NULL); nvp != NULL; 4999 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 5000 if (!zfeature_is_supported(nvpair_name(nvp))) { 5001 fnvlist_add_string(unsup_feat, 5002 nvpair_name(nvp), ""); 5003 } 5004 } 5005 5006 if (!nvlist_empty(unsup_feat)) { 5007 fnvlist_add_nvlist(spa->spa_load_info, 5008 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 5009 nvlist_free(unsup_feat); 5010 spa_load_failed(spa, "some features are unsupported"); 5011 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 5012 ENOTSUP)); 5013 } 5014 5015 nvlist_free(unsup_feat); 5016 } 5017 5018 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 5019 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5020 spa_try_repair(spa, spa->spa_config); 5021 spa_config_exit(spa, SCL_ALL, FTAG); 5022 nvlist_free(spa->spa_config_splitting); 5023 spa->spa_config_splitting = NULL; 5024 } 5025 5026 /* 5027 * Initialize internal SPA structures. 5028 */ 5029 spa_ld_select_uberblock_done(spa, ub); 5030 5031 return (0); 5032 } 5033 5034 static int 5035 spa_ld_open_rootbp(spa_t *spa) 5036 { 5037 int error = 0; 5038 vdev_t *rvd = spa->spa_root_vdev; 5039 5040 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 5041 if (error != 0) { 5042 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 5043 "[error=%d]", error); 5044 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5045 } 5046 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 5047 5048 return (0); 5049 } 5050 5051 static int 5052 spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 5053 boolean_t reloading) 5054 { 5055 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 5056 nvlist_t *nv, *mos_config, *policy; 5057 int error = 0, copy_error; 5058 uint64_t healthy_tvds, healthy_tvds_mos; 5059 uint64_t mos_config_txg; 5060 5061 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 5062 != 0) 5063 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5064 5065 /* 5066 * If we're assembling a pool from a split, the config provided is 5067 * already trusted so there is nothing to do. 5068 */ 5069 if (type == SPA_IMPORT_ASSEMBLE) 5070 return (0); 5071 5072 healthy_tvds = spa_healthy_core_tvds(spa); 5073 5074 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 5075 != 0) { 5076 spa_load_failed(spa, "unable to retrieve MOS config"); 5077 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5078 } 5079 5080 /* 5081 * If we are doing an open, pool owner wasn't verified yet, thus do 5082 * the verification here. 5083 */ 5084 if (spa->spa_load_state == SPA_LOAD_OPEN) { 5085 error = spa_verify_host(spa, mos_config); 5086 if (error != 0) { 5087 nvlist_free(mos_config); 5088 return (error); 5089 } 5090 } 5091 5092 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 5093 5094 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5095 5096 /* 5097 * Build a new vdev tree from the trusted config 5098 */ 5099 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 5100 if (error != 0) { 5101 nvlist_free(mos_config); 5102 spa_config_exit(spa, SCL_ALL, FTAG); 5103 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 5104 error); 5105 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 5106 } 5107 5108 /* 5109 * Vdev paths in the MOS may be obsolete. If the untrusted config was 5110 * obtained by scanning /dev/dsk, then it will have the right vdev 5111 * paths. We update the trusted MOS config with this information. 5112 * We first try to copy the paths with vdev_copy_path_strict, which 5113 * succeeds only when both configs have exactly the same vdev tree. 5114 * If that fails, we fall back to a more flexible method that has a 5115 * best effort policy. 5116 */ 5117 copy_error = vdev_copy_path_strict(rvd, mrvd); 5118 if (copy_error != 0 || spa_load_print_vdev_tree) { 5119 spa_load_note(spa, "provided vdev tree:"); 5120 vdev_dbgmsg_print_tree(rvd, 2); 5121 spa_load_note(spa, "MOS vdev tree:"); 5122 vdev_dbgmsg_print_tree(mrvd, 2); 5123 } 5124 if (copy_error != 0) { 5125 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 5126 "back to vdev_copy_path_relaxed"); 5127 vdev_copy_path_relaxed(rvd, mrvd); 5128 } 5129 5130 vdev_close(rvd); 5131 vdev_free(rvd); 5132 spa->spa_root_vdev = mrvd; 5133 rvd = mrvd; 5134 spa_config_exit(spa, SCL_ALL, FTAG); 5135 5136 /* 5137 * If 'zpool import' used a cached config, then the on-disk hostid and 5138 * hostname may be different to the cached config in ways that should 5139 * prevent import. Userspace can't discover this without a scan, but 5140 * we know, so we add these values to LOAD_INFO so the caller can know 5141 * the difference. 5142 * 5143 * Note that we have to do this before the config is regenerated, 5144 * because the new config will have the hostid and hostname for this 5145 * host, in readiness for import. 5146 */ 5147 if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTID)) 5148 fnvlist_add_uint64(spa->spa_load_info, ZPOOL_CONFIG_HOSTID, 5149 fnvlist_lookup_uint64(mos_config, ZPOOL_CONFIG_HOSTID)); 5150 if (nvlist_exists(mos_config, ZPOOL_CONFIG_HOSTNAME)) 5151 fnvlist_add_string(spa->spa_load_info, ZPOOL_CONFIG_HOSTNAME, 5152 fnvlist_lookup_string(mos_config, ZPOOL_CONFIG_HOSTNAME)); 5153 5154 /* 5155 * We will use spa_config if we decide to reload the spa or if spa_load 5156 * fails and we rewind. We must thus regenerate the config using the 5157 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 5158 * pass settings on how to load the pool and is not stored in the MOS. 5159 * We copy it over to our new, trusted config. 5160 */ 5161 mos_config_txg = fnvlist_lookup_uint64(mos_config, 5162 ZPOOL_CONFIG_POOL_TXG); 5163 nvlist_free(mos_config); 5164 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 5165 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 5166 &policy) == 0) 5167 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 5168 spa_config_set(spa, mos_config); 5169 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 5170 5171 /* 5172 * Now that we got the config from the MOS, we should be more strict 5173 * in checking blkptrs and can make assumptions about the consistency 5174 * of the vdev tree. spa_trust_config must be set to true before opening 5175 * vdevs in order for them to be writeable. 5176 */ 5177 spa->spa_trust_config = B_TRUE; 5178 5179 /* 5180 * Open and validate the new vdev tree 5181 */ 5182 error = spa_ld_open_vdevs(spa); 5183 if (error != 0) 5184 return (error); 5185 5186 error = spa_ld_validate_vdevs(spa); 5187 if (error != 0) 5188 return (error); 5189 5190 if (copy_error != 0 || spa_load_print_vdev_tree) { 5191 spa_load_note(spa, "final vdev tree:"); 5192 vdev_dbgmsg_print_tree(rvd, 2); 5193 } 5194 5195 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 5196 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 5197 /* 5198 * Sanity check to make sure that we are indeed loading the 5199 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 5200 * in the config provided and they happened to be the only ones 5201 * to have the latest uberblock, we could involuntarily perform 5202 * an extreme rewind. 5203 */ 5204 healthy_tvds_mos = spa_healthy_core_tvds(spa); 5205 if (healthy_tvds_mos - healthy_tvds >= 5206 SPA_SYNC_MIN_VDEVS) { 5207 spa_load_note(spa, "config provided misses too many " 5208 "top-level vdevs compared to MOS (%lld vs %lld). ", 5209 (u_longlong_t)healthy_tvds, 5210 (u_longlong_t)healthy_tvds_mos); 5211 spa_load_note(spa, "vdev tree:"); 5212 vdev_dbgmsg_print_tree(rvd, 2); 5213 if (reloading) { 5214 spa_load_failed(spa, "config was already " 5215 "provided from MOS. Aborting."); 5216 return (spa_vdev_err(rvd, 5217 VDEV_AUX_CORRUPT_DATA, EIO)); 5218 } 5219 spa_load_note(spa, "spa must be reloaded using MOS " 5220 "config"); 5221 return (SET_ERROR(EAGAIN)); 5222 } 5223 } 5224 5225 /* 5226 * Final sanity check for multihost pools that no other host is 5227 * accessing the pool. All of the read-only check have passed at 5228 * this point, perform targetted updates to the mmp uberblocks to 5229 * safely force a visible change. 5230 */ 5231 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 5232 !spa->spa_extreme_rewind && spa->spa_activity_check) { 5233 5234 error = spa_activity_check_claim(spa); 5235 error = spa_ld_activity_result(spa, error, "claim"); 5236 5237 if (error == EREMOTEIO) 5238 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 5239 else if (error) 5240 return (error); 5241 } 5242 5243 error = spa_check_for_missing_logs(spa); 5244 if (error != 0) 5245 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 5246 5247 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 5248 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 5249 "guid sum (%llu != %llu)", 5250 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 5251 (u_longlong_t)rvd->vdev_guid_sum); 5252 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 5253 ENXIO)); 5254 } 5255 5256 return (0); 5257 } 5258 5259 static int 5260 spa_ld_open_indirect_vdev_metadata(spa_t *spa) 5261 { 5262 int error = 0; 5263 vdev_t *rvd = spa->spa_root_vdev; 5264 5265 /* 5266 * Everything that we read before spa_remove_init() must be stored 5267 * on concreted vdevs. Therefore we do this as early as possible. 5268 */ 5269 error = spa_remove_init(spa); 5270 if (error != 0) { 5271 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 5272 error); 5273 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5274 } 5275 5276 /* 5277 * Retrieve information needed to condense indirect vdev mappings. 5278 */ 5279 error = spa_condense_init(spa); 5280 if (error != 0) { 5281 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 5282 error); 5283 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 5284 } 5285 5286 return (0); 5287 } 5288 5289 static int 5290 spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 5291 { 5292 int error = 0; 5293 vdev_t *rvd = spa->spa_root_vdev; 5294 5295 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 5296 boolean_t missing_feat_read = B_FALSE; 5297 nvlist_t *unsup_feat, *enabled_feat; 5298 5299 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 5300 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 5301 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5302 } 5303 5304 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 5305 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 5306 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5307 } 5308 5309 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 5310 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 5311 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5312 } 5313 5314 enabled_feat = fnvlist_alloc(); 5315 unsup_feat = fnvlist_alloc(); 5316 5317 if (!spa_features_check(spa, B_FALSE, 5318 unsup_feat, enabled_feat)) 5319 missing_feat_read = B_TRUE; 5320 5321 if (spa_writeable(spa) || 5322 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 5323 if (!spa_features_check(spa, B_TRUE, 5324 unsup_feat, enabled_feat)) { 5325 *missing_feat_writep = B_TRUE; 5326 } 5327 } 5328 5329 fnvlist_add_nvlist(spa->spa_load_info, 5330 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 5331 5332 if (!nvlist_empty(unsup_feat)) { 5333 fnvlist_add_nvlist(spa->spa_load_info, 5334 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 5335 } 5336 5337 fnvlist_free(enabled_feat); 5338 fnvlist_free(unsup_feat); 5339 5340 if (!missing_feat_read) { 5341 fnvlist_add_boolean(spa->spa_load_info, 5342 ZPOOL_CONFIG_CAN_RDONLY); 5343 } 5344 5345 /* 5346 * If the state is SPA_LOAD_TRYIMPORT, our objective is 5347 * twofold: to determine whether the pool is available for 5348 * import in read-write mode and (if it is not) whether the 5349 * pool is available for import in read-only mode. If the pool 5350 * is available for import in read-write mode, it is displayed 5351 * as available in userland; if it is not available for import 5352 * in read-only mode, it is displayed as unavailable in 5353 * userland. If the pool is available for import in read-only 5354 * mode but not read-write mode, it is displayed as unavailable 5355 * in userland with a special note that the pool is actually 5356 * available for open in read-only mode. 5357 * 5358 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 5359 * missing a feature for write, we must first determine whether 5360 * the pool can be opened read-only before returning to 5361 * userland in order to know whether to display the 5362 * abovementioned note. 5363 */ 5364 if (missing_feat_read || (*missing_feat_writep && 5365 spa_writeable(spa))) { 5366 spa_load_failed(spa, "pool uses unsupported features"); 5367 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 5368 ENOTSUP)); 5369 } 5370 5371 /* 5372 * Load refcounts for ZFS features from disk into an in-memory 5373 * cache during SPA initialization. 5374 */ 5375 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 5376 uint64_t refcount; 5377 5378 error = feature_get_refcount_from_disk(spa, 5379 &spa_feature_table[i], &refcount); 5380 if (error == 0) { 5381 spa->spa_feat_refcount_cache[i] = refcount; 5382 } else if (error == ENOTSUP) { 5383 spa->spa_feat_refcount_cache[i] = 5384 SPA_FEATURE_DISABLED; 5385 } else { 5386 spa_load_failed(spa, "error getting refcount " 5387 "for feature %s [error=%d]", 5388 spa_feature_table[i].fi_guid, error); 5389 return (spa_vdev_err(rvd, 5390 VDEV_AUX_CORRUPT_DATA, EIO)); 5391 } 5392 } 5393 } 5394 5395 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 5396 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 5397 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 5398 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5399 } 5400 5401 /* 5402 * Encryption was added before bookmark_v2, even though bookmark_v2 5403 * is now a dependency. If this pool has encryption enabled without 5404 * bookmark_v2, trigger an errata message. 5405 */ 5406 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 5407 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 5408 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 5409 } 5410 5411 return (0); 5412 } 5413 5414 static int 5415 spa_ld_load_special_directories(spa_t *spa) 5416 { 5417 int error = 0; 5418 vdev_t *rvd = spa->spa_root_vdev; 5419 5420 spa->spa_is_initializing = B_TRUE; 5421 error = dsl_pool_open(spa->spa_dsl_pool); 5422 spa->spa_is_initializing = B_FALSE; 5423 if (error != 0) { 5424 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 5425 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5426 } 5427 5428 return (0); 5429 } 5430 5431 static int 5432 spa_ld_get_props(spa_t *spa) 5433 { 5434 int error = 0; 5435 uint64_t obj; 5436 vdev_t *rvd = spa->spa_root_vdev; 5437 5438 /* Grab the checksum salt from the MOS. */ 5439 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 5440 DMU_POOL_CHECKSUM_SALT, 1, 5441 sizeof (spa->spa_cksum_salt.zcs_bytes), 5442 spa->spa_cksum_salt.zcs_bytes); 5443 if (error == ENOENT) { 5444 /* Generate a new salt for subsequent use */ 5445 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 5446 sizeof (spa->spa_cksum_salt.zcs_bytes)); 5447 } else if (error != 0) { 5448 spa_load_failed(spa, "unable to retrieve checksum salt from " 5449 "MOS [error=%d]", error); 5450 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5451 } 5452 5453 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 5454 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5455 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 5456 if (error != 0) { 5457 spa_load_failed(spa, "error opening deferred-frees bpobj " 5458 "[error=%d]", error); 5459 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5460 } 5461 5462 /* 5463 * Load the bit that tells us to use the new accounting function 5464 * (raid-z deflation). If we have an older pool, this will not 5465 * be present. 5466 */ 5467 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 5468 if (error != 0 && error != ENOENT) 5469 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5470 5471 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 5472 &spa->spa_creation_version, B_FALSE); 5473 if (error != 0 && error != ENOENT) 5474 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5475 5476 /* Load time log */ 5477 spa_load_txg_log_time(spa); 5478 5479 /* 5480 * Load the persistent error log. If we have an older pool, this will 5481 * not be present. 5482 */ 5483 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 5484 B_FALSE); 5485 if (error != 0 && error != ENOENT) 5486 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5487 5488 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 5489 &spa->spa_errlog_scrub, B_FALSE); 5490 if (error != 0 && error != ENOENT) 5491 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5492 5493 /* Load the last scrubbed txg. */ 5494 error = spa_dir_prop(spa, DMU_POOL_LAST_SCRUBBED_TXG, 5495 &spa->spa_scrubbed_last_txg, B_FALSE); 5496 if (error != 0 && error != ENOENT) 5497 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5498 5499 /* 5500 * Load the livelist deletion field. If a livelist is queued for 5501 * deletion, indicate that in the spa 5502 */ 5503 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 5504 &spa->spa_livelists_to_delete, B_FALSE); 5505 if (error != 0 && error != ENOENT) 5506 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5507 5508 /* 5509 * Load the history object. If we have an older pool, this 5510 * will not be present. 5511 */ 5512 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 5513 if (error != 0 && error != ENOENT) 5514 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5515 5516 /* 5517 * Load the per-vdev ZAP map. If we have an older pool, this will not 5518 * be present; in this case, defer its creation to a later time to 5519 * avoid dirtying the MOS this early / out of sync context. See 5520 * spa_sync_config_object. 5521 */ 5522 5523 /* The sentinel is only available in the MOS config. */ 5524 nvlist_t *mos_config; 5525 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 5526 spa_load_failed(spa, "unable to retrieve MOS config"); 5527 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5528 } 5529 5530 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 5531 &spa->spa_all_vdev_zaps, B_FALSE); 5532 5533 if (error == ENOENT) { 5534 VERIFY(!nvlist_exists(mos_config, 5535 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 5536 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 5537 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 5538 } else if (error != 0) { 5539 nvlist_free(mos_config); 5540 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5541 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 5542 /* 5543 * An older version of ZFS overwrote the sentinel value, so 5544 * we have orphaned per-vdev ZAPs in the MOS. Defer their 5545 * destruction to later; see spa_sync_config_object. 5546 */ 5547 spa->spa_avz_action = AVZ_ACTION_DESTROY; 5548 /* 5549 * We're assuming that no vdevs have had their ZAPs created 5550 * before this. Better be sure of it. 5551 */ 5552 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 5553 } 5554 nvlist_free(mos_config); 5555 5556 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 5557 5558 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 5559 B_FALSE); 5560 if (error && error != ENOENT) 5561 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5562 5563 if (error == 0) { 5564 uint64_t autoreplace = 0; 5565 5566 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 5567 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 5568 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 5569 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 5570 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 5571 spa_prop_find(spa, ZPOOL_PROP_DEDUP_TABLE_QUOTA, 5572 &spa->spa_dedup_table_quota); 5573 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 5574 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 5575 spa->spa_autoreplace = (autoreplace != 0); 5576 } 5577 5578 /* 5579 * If we are importing a pool with missing top-level vdevs, 5580 * we enforce that the pool doesn't panic or get suspended on 5581 * error since the likelihood of missing data is extremely high. 5582 */ 5583 if (spa->spa_missing_tvds > 0 && 5584 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 5585 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 5586 spa_load_note(spa, "forcing failmode to 'continue' " 5587 "as some top level vdevs are missing"); 5588 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 5589 } 5590 5591 return (0); 5592 } 5593 5594 static int 5595 spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 5596 { 5597 int error = 0; 5598 vdev_t *rvd = spa->spa_root_vdev; 5599 5600 /* 5601 * If we're assembling the pool from the split-off vdevs of 5602 * an existing pool, we don't want to attach the spares & cache 5603 * devices. 5604 */ 5605 5606 /* 5607 * Load any hot spares for this pool. 5608 */ 5609 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 5610 B_FALSE); 5611 if (error != 0 && error != ENOENT) 5612 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5613 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 5614 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 5615 error = load_nvlist(spa, spa->spa_spares.sav_object, 5616 &spa->spa_spares.sav_config); 5617 if (error != 0) { 5618 if (!zfs_recover && spa_writeable(spa)) { 5619 spa_load_failed(spa, "error loading spares " 5620 "nvlist [error=%d]", error); 5621 return (spa_vdev_err(rvd, 5622 VDEV_AUX_CORRUPT_DATA, EIO)); 5623 } 5624 spa_load_note(spa, "ignoring spares nvlist " 5625 "[error=%d], no spares will be available", error); 5626 /* Leak the object, its dnode may be unreadable. */ 5627 spa->spa_spares.sav_object = 0; 5628 spa->spa_spares.sav_sync = B_TRUE; 5629 } else { 5630 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5631 spa_load_spares(spa); 5632 spa_config_exit(spa, SCL_ALL, FTAG); 5633 } 5634 } else if (error == 0) { 5635 spa->spa_spares.sav_sync = B_TRUE; 5636 } 5637 5638 /* 5639 * Load any level 2 ARC devices for this pool. 5640 */ 5641 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 5642 &spa->spa_l2cache.sav_object, B_FALSE); 5643 if (error != 0 && error != ENOENT) 5644 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5645 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 5646 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 5647 error = load_nvlist(spa, spa->spa_l2cache.sav_object, 5648 &spa->spa_l2cache.sav_config); 5649 if (error != 0) { 5650 if (!zfs_recover && spa_writeable(spa)) { 5651 spa_load_failed(spa, "error loading l2cache " 5652 "nvlist [error=%d]", error); 5653 return (spa_vdev_err(rvd, 5654 VDEV_AUX_CORRUPT_DATA, EIO)); 5655 } 5656 spa_load_note(spa, "ignoring l2cache nvlist " 5657 "[error=%d], no l2cache will be available", error); 5658 /* Leak the object, its dnode may be unreadable. */ 5659 spa->spa_l2cache.sav_object = 0; 5660 spa->spa_l2cache.sav_sync = B_TRUE; 5661 } else { 5662 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5663 spa_load_l2cache(spa); 5664 spa_config_exit(spa, SCL_ALL, FTAG); 5665 } 5666 } else if (error == 0) { 5667 spa->spa_l2cache.sav_sync = B_TRUE; 5668 } 5669 5670 return (0); 5671 } 5672 5673 static int 5674 spa_ld_load_vdev_metadata(spa_t *spa) 5675 { 5676 int error = 0; 5677 vdev_t *rvd = spa->spa_root_vdev; 5678 5679 /* 5680 * If the 'multihost' property is set, then never allow a pool to 5681 * be imported when the system hostid is zero. The exception to 5682 * this rule is zdb which is always allowed to access pools. 5683 */ 5684 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 5685 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 5686 fnvlist_add_uint64(spa->spa_load_info, 5687 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 5688 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 5689 } 5690 5691 /* 5692 * If the 'autoreplace' property is set, then post a resource notifying 5693 * the ZFS DE that it should not issue any faults for unopenable 5694 * devices. We also iterate over the vdevs, and post a sysevent for any 5695 * unopenable vdevs so that the normal autoreplace handler can take 5696 * over. 5697 */ 5698 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 5699 spa_check_removed(spa->spa_root_vdev); 5700 /* 5701 * For the import case, this is done in spa_import(), because 5702 * at this point we're using the spare definitions from 5703 * the MOS config, not necessarily from the userland config. 5704 */ 5705 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 5706 spa_aux_check_removed(&spa->spa_spares); 5707 spa_aux_check_removed(&spa->spa_l2cache); 5708 } 5709 } 5710 5711 /* 5712 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 5713 */ 5714 error = vdev_load(rvd); 5715 if (error != 0) { 5716 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 5717 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 5718 } 5719 5720 error = spa_ld_log_spacemaps(spa); 5721 if (error != 0) { 5722 spa_load_failed(spa, "spa_ld_log_spacemaps failed [error=%d]", 5723 error); 5724 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 5725 } 5726 5727 /* 5728 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 5729 */ 5730 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5731 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 5732 spa_config_exit(spa, SCL_ALL, FTAG); 5733 5734 return (0); 5735 } 5736 5737 static int 5738 spa_ld_load_dedup_tables(spa_t *spa) 5739 { 5740 int error = 0; 5741 vdev_t *rvd = spa->spa_root_vdev; 5742 5743 error = ddt_load(spa); 5744 if (error != 0) { 5745 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 5746 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5747 } 5748 5749 return (0); 5750 } 5751 5752 static int 5753 spa_ld_load_brt(spa_t *spa) 5754 { 5755 int error = 0; 5756 vdev_t *rvd = spa->spa_root_vdev; 5757 5758 error = brt_load(spa); 5759 if (error != 0) { 5760 spa_load_failed(spa, "brt_load failed [error=%d]", error); 5761 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 5762 } 5763 5764 return (0); 5765 } 5766 5767 static int 5768 spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, const char **ereport) 5769 { 5770 vdev_t *rvd = spa->spa_root_vdev; 5771 5772 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 5773 boolean_t missing = spa_check_logs(spa); 5774 if (missing) { 5775 if (spa->spa_missing_tvds != 0) { 5776 spa_load_note(spa, "spa_check_logs failed " 5777 "so dropping the logs"); 5778 } else { 5779 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 5780 spa_load_failed(spa, "spa_check_logs failed"); 5781 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 5782 ENXIO)); 5783 } 5784 } 5785 } 5786 5787 return (0); 5788 } 5789 5790 static int 5791 spa_ld_verify_pool_data(spa_t *spa) 5792 { 5793 int error = 0; 5794 vdev_t *rvd = spa->spa_root_vdev; 5795 5796 /* 5797 * We've successfully opened the pool, verify that we're ready 5798 * to start pushing transactions. 5799 */ 5800 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 5801 error = spa_load_verify(spa); 5802 if (error != 0) { 5803 spa_load_failed(spa, "spa_load_verify failed " 5804 "[error=%d]", error); 5805 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 5806 error)); 5807 } 5808 } 5809 5810 return (0); 5811 } 5812 5813 static void 5814 spa_ld_claim_log_blocks(spa_t *spa) 5815 { 5816 dmu_tx_t *tx; 5817 dsl_pool_t *dp = spa_get_dsl(spa); 5818 5819 /* 5820 * Claim log blocks that haven't been committed yet. 5821 * This must all happen in a single txg. 5822 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 5823 * invoked from zil_claim_log_block()'s i/o done callback. 5824 * Price of rollback is that we abandon the log. 5825 */ 5826 spa->spa_claiming = B_TRUE; 5827 5828 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 5829 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 5830 zil_claim, tx, DS_FIND_CHILDREN); 5831 dmu_tx_commit(tx); 5832 5833 spa->spa_claiming = B_FALSE; 5834 5835 spa_set_log_state(spa, SPA_LOG_GOOD); 5836 } 5837 5838 static void 5839 spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 5840 boolean_t update_config_cache) 5841 { 5842 vdev_t *rvd = spa->spa_root_vdev; 5843 int need_update = B_FALSE; 5844 5845 /* 5846 * If the config cache is stale, or we have uninitialized 5847 * metaslabs (see spa_vdev_add()), then update the config. 5848 * 5849 * If this is a verbatim import, trust the current 5850 * in-core spa_config and update the disk labels. 5851 */ 5852 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 5853 spa->spa_load_state == SPA_LOAD_IMPORT || 5854 spa->spa_load_state == SPA_LOAD_RECOVER || 5855 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 5856 need_update = B_TRUE; 5857 5858 for (int c = 0; c < rvd->vdev_children; c++) 5859 if (rvd->vdev_child[c]->vdev_ms_array == 0) 5860 need_update = B_TRUE; 5861 5862 /* 5863 * Update the config cache asynchronously in case we're the 5864 * root pool, in which case the config cache isn't writable yet. 5865 */ 5866 if (need_update) 5867 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 5868 } 5869 5870 static void 5871 spa_ld_prepare_for_reload(spa_t *spa) 5872 { 5873 spa_mode_t mode = spa->spa_mode; 5874 int async_suspended = spa->spa_async_suspended; 5875 5876 spa_unload(spa); 5877 spa_deactivate(spa); 5878 spa_activate(spa, mode); 5879 5880 /* 5881 * We save the value of spa_async_suspended as it gets reset to 0 by 5882 * spa_unload(). We want to restore it back to the original value before 5883 * returning as we might be calling spa_async_resume() later. 5884 */ 5885 spa->spa_async_suspended = async_suspended; 5886 } 5887 5888 static int 5889 spa_ld_read_checkpoint_txg(spa_t *spa) 5890 { 5891 uberblock_t checkpoint; 5892 int error = 0; 5893 5894 ASSERT0(spa->spa_checkpoint_txg); 5895 ASSERT(spa_namespace_held() || 5896 spa->spa_load_thread == curthread); 5897 5898 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 5899 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 5900 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 5901 5902 if (error == ENOENT) 5903 return (0); 5904 5905 if (error != 0) 5906 return (error); 5907 5908 ASSERT3U(checkpoint.ub_txg, !=, 0); 5909 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 5910 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 5911 spa->spa_checkpoint_txg = checkpoint.ub_txg; 5912 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 5913 5914 return (0); 5915 } 5916 5917 static int 5918 spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 5919 { 5920 int error = 0; 5921 5922 ASSERT(spa_namespace_held()); 5923 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 5924 5925 /* 5926 * Never trust the config that is provided unless we are assembling 5927 * a pool following a split. 5928 * This means don't trust blkptrs and the vdev tree in general. This 5929 * also effectively puts the spa in read-only mode since 5930 * spa_writeable() checks for spa_trust_config to be true. 5931 * We will later load a trusted config from the MOS. 5932 */ 5933 if (type != SPA_IMPORT_ASSEMBLE) 5934 spa->spa_trust_config = B_FALSE; 5935 5936 /* 5937 * Parse the config provided to create a vdev tree. 5938 */ 5939 error = spa_ld_parse_config(spa, type); 5940 if (error != 0) 5941 return (error); 5942 5943 spa_import_progress_add(spa); 5944 5945 /* 5946 * Now that we have the vdev tree, try to open each vdev. This involves 5947 * opening the underlying physical device, retrieving its geometry and 5948 * probing the vdev with a dummy I/O. The state of each vdev will be set 5949 * based on the success of those operations. After this we'll be ready 5950 * to read from the vdevs. 5951 */ 5952 error = spa_ld_open_vdevs(spa); 5953 if (error != 0) 5954 return (error); 5955 5956 /* 5957 * Read the label of each vdev and make sure that the GUIDs stored 5958 * there match the GUIDs in the config provided. 5959 * If we're assembling a new pool that's been split off from an 5960 * existing pool, the labels haven't yet been updated so we skip 5961 * validation for now. 5962 */ 5963 if (type != SPA_IMPORT_ASSEMBLE) { 5964 error = spa_ld_validate_vdevs(spa); 5965 if (error != 0) 5966 return (error); 5967 } 5968 5969 /* 5970 * Read all vdev labels to find the best uberblock (i.e. latest, 5971 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 5972 * get the list of features required to read blkptrs in the MOS from 5973 * the vdev label with the best uberblock and verify that our version 5974 * of zfs supports them all. 5975 */ 5976 error = spa_ld_select_uberblock(spa, type); 5977 if (error != 0) 5978 return (error); 5979 5980 /* 5981 * Pass that uberblock to the dsl_pool layer which will open the root 5982 * blkptr. This blkptr points to the latest version of the MOS and will 5983 * allow us to read its contents. 5984 */ 5985 error = spa_ld_open_rootbp(spa); 5986 if (error != 0) 5987 return (error); 5988 5989 return (0); 5990 } 5991 5992 static int 5993 spa_ld_checkpoint_rewind(spa_t *spa) 5994 { 5995 uberblock_t checkpoint; 5996 int error = 0; 5997 5998 ASSERT(spa_namespace_held()); 5999 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 6000 6001 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 6002 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 6003 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 6004 6005 if (error != 0) { 6006 spa_load_failed(spa, "unable to retrieve checkpointed " 6007 "uberblock from the MOS config [error=%d]", error); 6008 6009 if (error == ENOENT) 6010 error = ZFS_ERR_NO_CHECKPOINT; 6011 6012 return (error); 6013 } 6014 6015 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 6016 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 6017 6018 /* 6019 * We need to update the txg and timestamp of the checkpointed 6020 * uberblock to be higher than the latest one. This ensures that 6021 * the checkpointed uberblock is selected if we were to close and 6022 * reopen the pool right after we've written it in the vdev labels. 6023 * (also see block comment in vdev_uberblock_compare) 6024 */ 6025 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 6026 checkpoint.ub_timestamp = gethrestime_sec(); 6027 6028 /* 6029 * Set current uberblock to be the checkpointed uberblock. 6030 */ 6031 spa->spa_uberblock = checkpoint; 6032 6033 /* 6034 * If we are doing a normal rewind, then the pool is open for 6035 * writing and we sync the "updated" checkpointed uberblock to 6036 * disk. Once this is done, we've basically rewound the whole 6037 * pool and there is no way back. 6038 * 6039 * There are cases when we don't want to attempt and sync the 6040 * checkpointed uberblock to disk because we are opening a 6041 * pool as read-only. Specifically, verifying the checkpointed 6042 * state with zdb, and importing the checkpointed state to get 6043 * a "preview" of its content. 6044 */ 6045 if (spa_writeable(spa)) { 6046 vdev_t *rvd = spa->spa_root_vdev; 6047 6048 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6049 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 6050 int svdcount = 0; 6051 int children = rvd->vdev_children; 6052 int c0 = random_in_range(children); 6053 6054 for (int c = 0; c < children; c++) { 6055 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 6056 6057 /* Stop when revisiting the first vdev */ 6058 if (c > 0 && svd[0] == vd) 6059 break; 6060 6061 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 6062 !vdev_is_concrete(vd)) 6063 continue; 6064 6065 svd[svdcount++] = vd; 6066 if (svdcount == SPA_SYNC_MIN_VDEVS) 6067 break; 6068 } 6069 error = vdev_config_sync(spa, svd, svdcount, 6070 spa->spa_first_txg); 6071 if (error == 0) 6072 spa->spa_last_synced_guid = rvd->vdev_guid; 6073 spa_config_exit(spa, SCL_ALL, FTAG); 6074 6075 if (error != 0) { 6076 spa_load_failed(spa, "failed to write checkpointed " 6077 "uberblock to the vdev labels [error=%d]", error); 6078 return (error); 6079 } 6080 } 6081 6082 return (0); 6083 } 6084 6085 static int 6086 spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 6087 boolean_t *update_config_cache) 6088 { 6089 int error; 6090 6091 /* 6092 * Parse the config for pool, open and validate vdevs, 6093 * select an uberblock, and use that uberblock to open 6094 * the MOS. 6095 */ 6096 error = spa_ld_mos_init(spa, type); 6097 if (error != 0) 6098 return (error); 6099 6100 /* 6101 * Retrieve the trusted config stored in the MOS and use it to create 6102 * a new, exact version of the vdev tree, then reopen all vdevs. 6103 */ 6104 error = spa_ld_trusted_config(spa, type, B_FALSE); 6105 if (error == EAGAIN) { 6106 if (update_config_cache != NULL) 6107 *update_config_cache = B_TRUE; 6108 6109 /* 6110 * Redo the loading process with the trusted config if it is 6111 * too different from the untrusted config. 6112 */ 6113 spa_ld_prepare_for_reload(spa); 6114 spa_load_note(spa, "RELOADING"); 6115 error = spa_ld_mos_init(spa, type); 6116 if (error != 0) 6117 return (error); 6118 6119 error = spa_ld_trusted_config(spa, type, B_TRUE); 6120 if (error != 0) 6121 return (error); 6122 6123 } else if (error != 0) { 6124 return (error); 6125 } 6126 6127 return (0); 6128 } 6129 6130 /* 6131 * Load an existing storage pool, using the config provided. This config 6132 * describes which vdevs are part of the pool and is later validated against 6133 * partial configs present in each vdev's label and an entire copy of the 6134 * config stored in the MOS. 6135 */ 6136 static int 6137 spa_load_impl(spa_t *spa, spa_import_type_t type, const char **ereport) 6138 { 6139 int error = 0; 6140 boolean_t missing_feat_write = B_FALSE; 6141 boolean_t checkpoint_rewind = 6142 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 6143 boolean_t update_config_cache = B_FALSE; 6144 hrtime_t load_start = gethrtime(); 6145 6146 ASSERT(spa_namespace_held()); 6147 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 6148 6149 spa_load_note(spa, "LOADING"); 6150 6151 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 6152 if (error != 0) 6153 return (error); 6154 6155 /* 6156 * If we are rewinding to the checkpoint then we need to repeat 6157 * everything we've done so far in this function but this time 6158 * selecting the checkpointed uberblock and using that to open 6159 * the MOS. 6160 */ 6161 if (checkpoint_rewind) { 6162 /* 6163 * If we are rewinding to the checkpoint update config cache 6164 * anyway. 6165 */ 6166 update_config_cache = B_TRUE; 6167 6168 /* 6169 * Extract the checkpointed uberblock from the current MOS 6170 * and use this as the pool's uberblock from now on. If the 6171 * pool is imported as writeable we also write the checkpoint 6172 * uberblock to the labels, making the rewind permanent. 6173 */ 6174 error = spa_ld_checkpoint_rewind(spa); 6175 if (error != 0) 6176 return (error); 6177 6178 /* 6179 * Redo the loading process again with the 6180 * checkpointed uberblock. 6181 */ 6182 spa_ld_prepare_for_reload(spa); 6183 spa_load_note(spa, "LOADING checkpointed uberblock"); 6184 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 6185 if (error != 0) 6186 return (error); 6187 } 6188 6189 /* 6190 * Drop the namespace lock for the rest of the function. 6191 */ 6192 spa->spa_load_thread = curthread; 6193 spa_namespace_exit(FTAG); 6194 6195 /* 6196 * Retrieve the checkpoint txg if the pool has a checkpoint. 6197 */ 6198 spa_import_progress_set_notes(spa, "Loading checkpoint txg"); 6199 error = spa_ld_read_checkpoint_txg(spa); 6200 if (error != 0) 6201 goto fail; 6202 6203 /* 6204 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 6205 * from the pool and their contents were re-mapped to other vdevs. Note 6206 * that everything that we read before this step must have been 6207 * rewritten on concrete vdevs after the last device removal was 6208 * initiated. Otherwise we could be reading from indirect vdevs before 6209 * we have loaded their mappings. 6210 */ 6211 spa_import_progress_set_notes(spa, "Loading indirect vdev metadata"); 6212 error = spa_ld_open_indirect_vdev_metadata(spa); 6213 if (error != 0) 6214 goto fail; 6215 6216 /* 6217 * Retrieve the full list of active features from the MOS and check if 6218 * they are all supported. 6219 */ 6220 spa_import_progress_set_notes(spa, "Checking feature flags"); 6221 error = spa_ld_check_features(spa, &missing_feat_write); 6222 if (error != 0) 6223 goto fail; 6224 6225 /* 6226 * Load several special directories from the MOS needed by the dsl_pool 6227 * layer. 6228 */ 6229 spa_import_progress_set_notes(spa, "Loading special MOS directories"); 6230 error = spa_ld_load_special_directories(spa); 6231 if (error != 0) 6232 goto fail; 6233 6234 /* 6235 * Retrieve pool properties from the MOS. 6236 */ 6237 spa_import_progress_set_notes(spa, "Loading properties"); 6238 error = spa_ld_get_props(spa); 6239 if (error != 0) 6240 goto fail; 6241 6242 /* 6243 * Retrieve the list of auxiliary devices - cache devices and spares - 6244 * and open them. 6245 */ 6246 spa_import_progress_set_notes(spa, "Loading AUX vdevs"); 6247 error = spa_ld_open_aux_vdevs(spa, type); 6248 if (error != 0) 6249 goto fail; 6250 6251 /* 6252 * Load the metadata for all vdevs. Also check if unopenable devices 6253 * should be autoreplaced. 6254 */ 6255 spa_import_progress_set_notes(spa, "Loading vdev metadata"); 6256 error = spa_ld_load_vdev_metadata(spa); 6257 if (error != 0) 6258 goto fail; 6259 6260 spa_import_progress_set_notes(spa, "Loading dedup tables"); 6261 error = spa_ld_load_dedup_tables(spa); 6262 if (error != 0) 6263 goto fail; 6264 6265 spa_import_progress_set_notes(spa, "Loading BRT"); 6266 error = spa_ld_load_brt(spa); 6267 if (error != 0) 6268 goto fail; 6269 6270 /* 6271 * Verify the logs now to make sure we don't have any unexpected errors 6272 * when we claim log blocks later. 6273 */ 6274 spa_import_progress_set_notes(spa, "Verifying Log Devices"); 6275 error = spa_ld_verify_logs(spa, type, ereport); 6276 if (error != 0) 6277 goto fail; 6278 6279 if (missing_feat_write) { 6280 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 6281 6282 /* 6283 * At this point, we know that we can open the pool in 6284 * read-only mode but not read-write mode. We now have enough 6285 * information and can return to userland. 6286 */ 6287 error = spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 6288 ENOTSUP); 6289 goto fail; 6290 } 6291 6292 /* 6293 * Traverse the last txgs to make sure the pool was left off in a safe 6294 * state. When performing an extreme rewind, we verify the whole pool, 6295 * which can take a very long time. 6296 */ 6297 spa_import_progress_set_notes(spa, "Verifying pool data"); 6298 error = spa_ld_verify_pool_data(spa); 6299 if (error != 0) 6300 goto fail; 6301 6302 /* 6303 * Calculate the deflated space for the pool. This must be done before 6304 * we write anything to the pool because we'd need to update the space 6305 * accounting using the deflated sizes. 6306 */ 6307 spa_import_progress_set_notes(spa, "Calculating deflated space"); 6308 spa_update_dspace(spa); 6309 6310 /* 6311 * We have now retrieved all the information we needed to open the 6312 * pool. If we are importing the pool in read-write mode, a few 6313 * additional steps must be performed to finish the import. 6314 */ 6315 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 6316 spa->spa_load_max_txg == UINT64_MAX)) { 6317 uint64_t config_cache_txg = spa->spa_config_txg; 6318 6319 spa_import_progress_set_notes(spa, "Starting import"); 6320 6321 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 6322 6323 /* 6324 * Before we do any zio_write's, complete the raidz expansion 6325 * scratch space copying, if necessary. 6326 */ 6327 if (RRSS_GET_STATE(&spa->spa_uberblock) == RRSS_SCRATCH_VALID) 6328 vdev_raidz_reflow_copy_scratch(spa); 6329 6330 /* 6331 * In case of a checkpoint rewind, log the original txg 6332 * of the checkpointed uberblock. 6333 */ 6334 if (checkpoint_rewind) { 6335 spa_history_log_internal(spa, "checkpoint rewind", 6336 NULL, "rewound state to txg=%llu", 6337 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 6338 } 6339 6340 spa_import_progress_set_notes(spa, "Claiming ZIL blocks"); 6341 /* 6342 * Traverse the ZIL and claim all blocks. 6343 */ 6344 spa_ld_claim_log_blocks(spa); 6345 6346 /* 6347 * Kick-off the syncing thread. 6348 */ 6349 spa->spa_sync_on = B_TRUE; 6350 txg_sync_start(spa->spa_dsl_pool); 6351 mmp_thread_start(spa); 6352 6353 /* 6354 * Wait for all claims to sync. We sync up to the highest 6355 * claimed log block birth time so that claimed log blocks 6356 * don't appear to be from the future. spa_claim_max_txg 6357 * will have been set for us by ZIL traversal operations 6358 * performed above. 6359 */ 6360 spa_import_progress_set_notes(spa, "Syncing ZIL claims"); 6361 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 6362 6363 /* 6364 * Check if we need to request an update of the config. On the 6365 * next sync, we would update the config stored in vdev labels 6366 * and the cachefile (by default /etc/zfs/zpool.cache). 6367 */ 6368 spa_import_progress_set_notes(spa, "Updating configs"); 6369 spa_ld_check_for_config_update(spa, config_cache_txg, 6370 update_config_cache); 6371 6372 /* 6373 * Check if a rebuild was in progress and if so resume it. 6374 * Then check all DTLs to see if anything needs resilvering. 6375 * The resilver will be deferred if a rebuild was started. 6376 */ 6377 spa_import_progress_set_notes(spa, "Starting resilvers"); 6378 if (vdev_rebuild_active(spa->spa_root_vdev)) { 6379 vdev_rebuild_restart(spa); 6380 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 6381 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 6382 spa_async_request(spa, SPA_ASYNC_RESILVER); 6383 } 6384 6385 /* 6386 * Log the fact that we booted up (so that we can detect if 6387 * we rebooted in the middle of an operation). 6388 */ 6389 spa_history_log_version(spa, "open", NULL); 6390 6391 spa_import_progress_set_notes(spa, 6392 "Restarting device removals"); 6393 spa_restart_removal(spa); 6394 spa_spawn_aux_threads(spa); 6395 6396 /* 6397 * Delete any inconsistent datasets. 6398 * 6399 * Note: 6400 * Since we may be issuing deletes for clones here, 6401 * we make sure to do so after we've spawned all the 6402 * auxiliary threads above (from which the livelist 6403 * deletion zthr is part of). 6404 */ 6405 spa_import_progress_set_notes(spa, 6406 "Cleaning up inconsistent objsets"); 6407 (void) dmu_objset_find(spa_name(spa), 6408 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 6409 6410 /* 6411 * Clean up any stale temporary dataset userrefs. 6412 */ 6413 spa_import_progress_set_notes(spa, 6414 "Cleaning up temporary userrefs"); 6415 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 6416 6417 /* 6418 * Anything still marked for deferred destruction because a 6419 * mount was holding it was left that way by a crash or an 6420 * export, and nothing is holding it now. The sweep walks 6421 * every snapshot, so leave it to the async thread rather than 6422 * spending import time on it. 6423 */ 6424 spa_async_request(spa, SPA_ASYNC_DEFER_DESTROY); 6425 6426 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6427 spa_import_progress_set_notes(spa, "Restarting initialize"); 6428 vdev_initialize_restart(spa->spa_root_vdev); 6429 spa_import_progress_set_notes(spa, "Restarting TRIM"); 6430 vdev_trim_restart(spa->spa_root_vdev); 6431 vdev_autotrim_restart(spa); 6432 spa_config_exit(spa, SCL_CONFIG, FTAG); 6433 spa_import_progress_set_notes(spa, "Finished importing"); 6434 } 6435 zio_handle_import_delay(spa, gethrtime() - load_start); 6436 6437 spa_import_progress_remove(spa_guid(spa)); 6438 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 6439 6440 spa_load_note(spa, "LOADED"); 6441 fail: 6442 spa_namespace_enter(FTAG); 6443 spa->spa_load_thread = NULL; 6444 spa_namespace_broadcast(); 6445 6446 return (error); 6447 6448 } 6449 6450 static int 6451 spa_load_retry(spa_t *spa, spa_load_state_t state) 6452 { 6453 spa_mode_t mode = spa->spa_mode; 6454 6455 spa_unload(spa); 6456 spa_deactivate(spa); 6457 6458 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 6459 6460 spa_activate(spa, mode); 6461 spa_async_suspend(spa); 6462 6463 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 6464 (u_longlong_t)spa->spa_load_max_txg); 6465 6466 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 6467 } 6468 6469 /* 6470 * If spa_load() fails this function will try loading prior txg's. If 6471 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 6472 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 6473 * function will not rewind the pool and will return the same error as 6474 * spa_load(), or ECANCELED if the load only probed the requested txg 6475 * instead of bringing the pool up. 6476 */ 6477 static int 6478 spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 6479 int rewind_flags) 6480 { 6481 nvlist_t *loadinfo = NULL; 6482 nvlist_t *config = NULL; 6483 int load_error, rewind_error; 6484 uint64_t safe_rewind_txg; 6485 uint64_t min_txg; 6486 6487 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 6488 spa->spa_load_max_txg = spa->spa_load_txg; 6489 spa_set_log_state(spa, SPA_LOG_CLEAR); 6490 } else { 6491 spa->spa_load_max_txg = max_request; 6492 if (max_request != UINT64_MAX) 6493 spa->spa_extreme_rewind = B_TRUE; 6494 } 6495 6496 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 6497 if (load_error == 0) { 6498 /* 6499 * A load of an explicitly requested txg may only probe 6500 * whether that txg is usable, finishing without a syncing 6501 * thread. Such a pool is not functional, so it can not be 6502 * handed to the caller no matter how well it loaded. Report 6503 * the probed txg the same way an actual rewind would. 6504 */ 6505 if (spa_writeable(spa) && !spa->spa_sync_on) { 6506 loadinfo = fnvlist_alloc(); 6507 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 6508 spa->spa_load_info); 6509 fnvlist_free(spa->spa_load_info); 6510 spa->spa_load_info = loadinfo; 6511 spa_import_progress_remove(spa_guid(spa)); 6512 return (SET_ERROR(ECANCELED)); 6513 } 6514 return (0); 6515 } 6516 6517 /* Do not attempt to load uberblocks from previous txgs when: */ 6518 switch (load_error) { 6519 case ZFS_ERR_NO_CHECKPOINT: 6520 /* Attempting checkpoint-rewind on a pool with no checkpoint */ 6521 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 6522 zfs_fallthrough; 6523 case EREMOTEIO: 6524 /* MMP determines the pool is active on another host */ 6525 zfs_fallthrough; 6526 case EBADF: 6527 /* The config cache is out of sync (vdevs or hostid) */ 6528 zfs_fallthrough; 6529 case EINTR: 6530 /* The user interactively interrupted the import */ 6531 spa_import_progress_remove(spa_guid(spa)); 6532 return (load_error); 6533 } 6534 6535 if (spa->spa_root_vdev != NULL) 6536 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6537 6538 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 6539 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 6540 6541 if (rewind_flags & ZPOOL_NEVER_REWIND) { 6542 nvlist_free(config); 6543 spa_import_progress_remove(spa_guid(spa)); 6544 return (load_error); 6545 } 6546 6547 if (state == SPA_LOAD_RECOVER) { 6548 /* Price of rolling back is discarding txgs, including log */ 6549 spa_set_log_state(spa, SPA_LOG_CLEAR); 6550 } else { 6551 /* 6552 * If we aren't rolling back save the load info from our first 6553 * import attempt so that we can restore it after attempting 6554 * to rewind. 6555 */ 6556 loadinfo = spa->spa_load_info; 6557 spa->spa_load_info = fnvlist_alloc(); 6558 } 6559 6560 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 6561 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 6562 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 6563 TXG_INITIAL : safe_rewind_txg; 6564 6565 /* 6566 * Continue as long as we're finding errors, we're still within 6567 * the acceptable rewind range, and we're still finding uberblocks 6568 */ 6569 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 6570 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 6571 if (spa->spa_load_max_txg < safe_rewind_txg) 6572 spa->spa_extreme_rewind = B_TRUE; 6573 rewind_error = spa_load_retry(spa, state); 6574 } 6575 6576 spa->spa_extreme_rewind = B_FALSE; 6577 spa->spa_load_max_txg = UINT64_MAX; 6578 6579 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 6580 spa_config_set(spa, config); 6581 else 6582 nvlist_free(config); 6583 6584 if (state == SPA_LOAD_RECOVER) { 6585 ASSERT0P(loadinfo); 6586 spa_import_progress_remove(spa_guid(spa)); 6587 return (rewind_error); 6588 } else { 6589 /* Store the rewind info as part of the initial load info */ 6590 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 6591 spa->spa_load_info); 6592 6593 /* Restore the initial load info */ 6594 fnvlist_free(spa->spa_load_info); 6595 spa->spa_load_info = loadinfo; 6596 6597 spa_import_progress_remove(spa_guid(spa)); 6598 return (load_error); 6599 } 6600 } 6601 6602 /* 6603 * Pool Open/Import 6604 * 6605 * The import case is identical to an open except that the configuration is sent 6606 * down from userland, instead of grabbed from the configuration cache. For the 6607 * case of an open, the pool configuration will exist in the 6608 * POOL_STATE_UNINITIALIZED state. 6609 * 6610 * The stats information (gen/count/ustats) is used to gather vdev statistics at 6611 * the same time open the pool, without having to keep around the spa_t in some 6612 * ambiguous state. 6613 */ 6614 static int 6615 spa_open_common(const char *pool, spa_t **spapp, const void *tag, 6616 nvlist_t *nvpolicy, nvlist_t **config) 6617 { 6618 spa_t *spa; 6619 spa_load_state_t state = SPA_LOAD_OPEN; 6620 int error; 6621 int locked = B_FALSE; 6622 int firstopen = B_FALSE; 6623 6624 *spapp = NULL; 6625 6626 /* 6627 * As disgusting as this is, we need to support recursive calls to this 6628 * function because dsl_dir_open() is called during spa_load(), and ends 6629 * up calling spa_open() again. The real fix is to figure out how to 6630 * avoid dsl_dir_open() calling this in the first place. 6631 */ 6632 if (!spa_namespace_held()) { 6633 spa_namespace_enter(FTAG); 6634 locked = B_TRUE; 6635 } 6636 6637 if ((spa = spa_lookup(pool)) == NULL) { 6638 if (locked) 6639 spa_namespace_exit(FTAG); 6640 return (SET_ERROR(ENOENT)); 6641 } 6642 6643 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 6644 zpool_load_policy_t policy; 6645 6646 firstopen = B_TRUE; 6647 6648 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 6649 &policy); 6650 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6651 state = SPA_LOAD_RECOVER; 6652 6653 spa_activate(spa, spa_mode_global); 6654 6655 if (state != SPA_LOAD_RECOVER) 6656 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6657 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6658 6659 zfs_dbgmsg("spa_open_common: opening %s", pool); 6660 error = spa_load_best(spa, state, policy.zlp_txg, 6661 policy.zlp_rewind); 6662 6663 if (error == EBADF) { 6664 /* 6665 * If vdev_validate() returns failure (indicated by 6666 * EBADF), it indicates that one of the vdevs indicates 6667 * that the pool has been exported or destroyed. If 6668 * this is the case, the config cache is out of sync and 6669 * we should remove the pool from the namespace. 6670 */ 6671 spa_unload(spa); 6672 spa_deactivate(spa); 6673 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 6674 spa_remove(spa); 6675 if (locked) 6676 spa_namespace_exit(FTAG); 6677 return (SET_ERROR(ENOENT)); 6678 } 6679 6680 if (error) { 6681 /* 6682 * We can't open the pool, but we still have useful 6683 * information: the state of each vdev after the 6684 * attempted vdev_open(). Return this to the user. 6685 */ 6686 if (config != NULL && spa->spa_config) { 6687 *config = fnvlist_dup(spa->spa_config); 6688 fnvlist_add_nvlist(*config, 6689 ZPOOL_CONFIG_LOAD_INFO, 6690 spa->spa_load_info); 6691 } 6692 spa_unload(spa); 6693 spa_deactivate(spa); 6694 spa->spa_last_open_failed = error; 6695 if (locked) 6696 spa_namespace_exit(FTAG); 6697 *spapp = NULL; 6698 return (error); 6699 } 6700 } 6701 6702 spa_open_ref(spa, tag); 6703 6704 if (config != NULL) 6705 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6706 6707 /* 6708 * If we've recovered the pool, pass back any information we 6709 * gathered while doing the load. 6710 */ 6711 if (state == SPA_LOAD_RECOVER && config != NULL) { 6712 fnvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 6713 spa->spa_load_info); 6714 } 6715 6716 if (locked) { 6717 spa->spa_last_open_failed = 0; 6718 spa->spa_last_ubsync_txg = 0; 6719 spa->spa_load_txg = 0; 6720 spa_namespace_exit(FTAG); 6721 } 6722 6723 if (firstopen) 6724 zvol_create_minors(spa_name(spa)); 6725 6726 *spapp = spa; 6727 6728 return (0); 6729 } 6730 6731 int 6732 spa_open_rewind(const char *name, spa_t **spapp, const void *tag, 6733 nvlist_t *policy, nvlist_t **config) 6734 { 6735 return (spa_open_common(name, spapp, tag, policy, config)); 6736 } 6737 6738 int 6739 spa_open(const char *name, spa_t **spapp, const void *tag) 6740 { 6741 return (spa_open_common(name, spapp, tag, NULL, NULL)); 6742 } 6743 6744 /* 6745 * Lookup the given spa_t, incrementing the inject count in the process, 6746 * preventing it from being exported or destroyed. 6747 */ 6748 spa_t * 6749 spa_inject_addref(char *name) 6750 { 6751 spa_t *spa; 6752 6753 spa_namespace_enter(FTAG); 6754 if ((spa = spa_lookup(name)) == NULL) { 6755 spa_namespace_exit(FTAG); 6756 return (NULL); 6757 } 6758 spa->spa_inject_ref++; 6759 spa_namespace_exit(FTAG); 6760 6761 return (spa); 6762 } 6763 6764 void 6765 spa_inject_delref(spa_t *spa) 6766 { 6767 spa_namespace_enter(FTAG); 6768 spa->spa_inject_ref--; 6769 spa_namespace_exit(FTAG); 6770 } 6771 6772 /* 6773 * Add spares device information to the nvlist. 6774 */ 6775 static void 6776 spa_add_spares(spa_t *spa, nvlist_t *config) 6777 { 6778 nvlist_t **spares; 6779 uint_t i, nspares; 6780 nvlist_t *nvroot; 6781 uint64_t guid; 6782 vdev_stat_t *vs; 6783 uint_t vsc; 6784 uint64_t pool; 6785 6786 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 6787 6788 if (spa->spa_spares.sav_count == 0) 6789 return; 6790 6791 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 6792 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 6793 ZPOOL_CONFIG_SPARES, &spares, &nspares)); 6794 if (nspares != 0) { 6795 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6796 (const nvlist_t * const *)spares, nspares); 6797 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6798 &spares, &nspares)); 6799 6800 /* 6801 * Go through and find any spares which have since been 6802 * repurposed as an active spare. If this is the case, update 6803 * their status appropriately. 6804 */ 6805 for (i = 0; i < nspares; i++) { 6806 guid = fnvlist_lookup_uint64(spares[i], 6807 ZPOOL_CONFIG_GUID); 6808 VERIFY0(nvlist_lookup_uint64_array(spares[i], 6809 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 6810 if (spa_spare_exists(guid, &pool, NULL) && 6811 pool != 0ULL) { 6812 vs->vs_state = VDEV_STATE_CANT_OPEN; 6813 vs->vs_aux = VDEV_AUX_SPARED; 6814 } else { 6815 vs->vs_state = 6816 spa->spa_spares.sav_vdevs[i]->vdev_state; 6817 } 6818 } 6819 } 6820 } 6821 6822 /* 6823 * Add l2cache device information to the nvlist, including vdev stats. 6824 */ 6825 static void 6826 spa_add_l2cache(spa_t *spa, nvlist_t *config) 6827 { 6828 nvlist_t **l2cache; 6829 uint_t i, j, nl2cache; 6830 nvlist_t *nvroot; 6831 uint64_t guid; 6832 vdev_t *vd; 6833 vdev_stat_t *vs; 6834 uint_t vsc; 6835 6836 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 6837 6838 if (spa->spa_l2cache.sav_count == 0) 6839 return; 6840 6841 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 6842 VERIFY0(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 6843 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache)); 6844 if (nl2cache != 0) { 6845 fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6846 (const nvlist_t * const *)l2cache, nl2cache); 6847 VERIFY0(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6848 &l2cache, &nl2cache)); 6849 6850 /* 6851 * Update level 2 cache device stats. 6852 */ 6853 6854 for (i = 0; i < nl2cache; i++) { 6855 guid = fnvlist_lookup_uint64(l2cache[i], 6856 ZPOOL_CONFIG_GUID); 6857 6858 vd = NULL; 6859 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 6860 if (guid == 6861 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 6862 vd = spa->spa_l2cache.sav_vdevs[j]; 6863 break; 6864 } 6865 } 6866 ASSERT(vd != NULL); 6867 6868 VERIFY0(nvlist_lookup_uint64_array(l2cache[i], 6869 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc)); 6870 vdev_get_stats(vd, vs); 6871 vdev_config_generate_stats(vd, l2cache[i]); 6872 6873 } 6874 } 6875 } 6876 6877 static void 6878 spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 6879 { 6880 zap_cursor_t zc; 6881 zap_attribute_t *za = zap_attribute_alloc(); 6882 6883 if (spa->spa_feat_for_read_obj != 0) { 6884 for (zap_cursor_init(&zc, spa->spa_meta_objset, 6885 spa->spa_feat_for_read_obj); 6886 zap_cursor_retrieve(&zc, za) == 0; 6887 zap_cursor_advance(&zc)) { 6888 ASSERT(za->za_integer_length == sizeof (uint64_t) && 6889 za->za_num_integers == 1); 6890 VERIFY0(nvlist_add_uint64(features, za->za_name, 6891 za->za_first_integer)); 6892 } 6893 zap_cursor_fini(&zc); 6894 } 6895 6896 if (spa->spa_feat_for_write_obj != 0) { 6897 for (zap_cursor_init(&zc, spa->spa_meta_objset, 6898 spa->spa_feat_for_write_obj); 6899 zap_cursor_retrieve(&zc, za) == 0; 6900 zap_cursor_advance(&zc)) { 6901 ASSERT(za->za_integer_length == sizeof (uint64_t) && 6902 za->za_num_integers == 1); 6903 VERIFY0(nvlist_add_uint64(features, za->za_name, 6904 za->za_first_integer)); 6905 } 6906 zap_cursor_fini(&zc); 6907 } 6908 zap_attribute_free(za); 6909 } 6910 6911 static void 6912 spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 6913 { 6914 int i; 6915 6916 for (i = 0; i < SPA_FEATURES; i++) { 6917 zfeature_info_t feature = spa_feature_table[i]; 6918 uint64_t refcount; 6919 6920 if (feature_get_refcount(spa, &feature, &refcount) != 0) 6921 continue; 6922 6923 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 6924 } 6925 } 6926 6927 /* 6928 * Store a list of pool features and their reference counts in the 6929 * config. 6930 * 6931 * The first time this is called on a spa, allocate a new nvlist, fetch 6932 * the pool features and reference counts from disk, then save the list 6933 * in the spa. In subsequent calls on the same spa use the saved nvlist 6934 * and refresh its values from the cached reference counts. This 6935 * ensures we don't block here on I/O on a suspended pool so 'zpool 6936 * clear' can resume the pool. 6937 */ 6938 static void 6939 spa_add_feature_stats(spa_t *spa, nvlist_t *config) 6940 { 6941 nvlist_t *features; 6942 6943 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 6944 6945 mutex_enter(&spa->spa_feat_stats_lock); 6946 features = spa->spa_feat_stats; 6947 6948 if (features != NULL) { 6949 spa_feature_stats_from_cache(spa, features); 6950 } else { 6951 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 6952 spa->spa_feat_stats = features; 6953 spa_feature_stats_from_disk(spa, features); 6954 } 6955 6956 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 6957 features)); 6958 6959 mutex_exit(&spa->spa_feat_stats_lock); 6960 } 6961 6962 int 6963 spa_get_stats(const char *name, nvlist_t **config, 6964 char *altroot, size_t buflen) 6965 { 6966 int error; 6967 spa_t *spa; 6968 6969 *config = NULL; 6970 error = spa_open_common(name, &spa, FTAG, NULL, config); 6971 6972 if (spa != NULL) { 6973 /* 6974 * This still leaves a window of inconsistency where the spares 6975 * or l2cache devices could change and the config would be 6976 * self-inconsistent. 6977 */ 6978 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6979 6980 if (*config != NULL) { 6981 uint64_t loadtimes[2]; 6982 6983 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 6984 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 6985 fnvlist_add_uint64_array(*config, 6986 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2); 6987 6988 fnvlist_add_uint64(*config, 6989 ZPOOL_CONFIG_ERRCOUNT, 6990 spa_approx_errlog_size(spa)); 6991 6992 if (spa_suspended(spa)) { 6993 fnvlist_add_uint64(*config, 6994 ZPOOL_CONFIG_SUSPENDED, 6995 spa->spa_failmode); 6996 fnvlist_add_uint64(*config, 6997 ZPOOL_CONFIG_SUSPENDED_REASON, 6998 spa->spa_suspended); 6999 } 7000 7001 spa_add_spares(spa, *config); 7002 spa_add_l2cache(spa, *config); 7003 spa_add_feature_stats(spa, *config); 7004 } 7005 } 7006 7007 /* 7008 * We want to get the alternate root even for faulted pools, so we cheat 7009 * and call spa_lookup() directly. 7010 */ 7011 if (altroot) { 7012 if (spa == NULL) { 7013 spa_namespace_enter(FTAG); 7014 spa = spa_lookup(name); 7015 if (spa) 7016 spa_altroot(spa, altroot, buflen); 7017 else 7018 altroot[0] = '\0'; 7019 spa = NULL; 7020 spa_namespace_exit(FTAG); 7021 } else { 7022 spa_altroot(spa, altroot, buflen); 7023 } 7024 } 7025 7026 if (spa != NULL) { 7027 spa_config_exit(spa, SCL_CONFIG, FTAG); 7028 spa_close(spa, FTAG); 7029 } 7030 7031 return (error); 7032 } 7033 7034 /* 7035 * Validate that the auxiliary device array is well formed. We must have an 7036 * array of nvlists, each which describes a valid leaf vdev. If this is an 7037 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 7038 * specified, as long as they are well-formed. 7039 */ 7040 static int 7041 spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 7042 spa_aux_vdev_t *sav, const char *config, uint64_t version, 7043 vdev_labeltype_t label) 7044 { 7045 nvlist_t **dev; 7046 uint_t i, ndev; 7047 vdev_t *vd; 7048 int error; 7049 7050 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 7051 7052 /* 7053 * It's acceptable to have no devs specified. 7054 */ 7055 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 7056 return (0); 7057 7058 if (ndev == 0) 7059 return (SET_ERROR(EINVAL)); 7060 7061 /* 7062 * Make sure the pool is formatted with a version that supports this 7063 * device type. 7064 */ 7065 if (spa_version(spa) < version) 7066 return (SET_ERROR(ENOTSUP)); 7067 7068 /* 7069 * Set the pending device list so we correctly handle device in-use 7070 * checking. 7071 */ 7072 sav->sav_pending = dev; 7073 sav->sav_npending = ndev; 7074 7075 for (i = 0; i < ndev; i++) { 7076 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 7077 mode)) != 0) 7078 goto out; 7079 7080 if (!vd->vdev_ops->vdev_op_leaf) { 7081 vdev_free(vd); 7082 error = SET_ERROR(EINVAL); 7083 goto out; 7084 } 7085 7086 vd->vdev_top = vd; 7087 7088 if ((error = vdev_open(vd, CRED())) == 0 && 7089 (error = vdev_label_init(vd, crtxg, label)) == 0) { 7090 fnvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 7091 vd->vdev_guid); 7092 } 7093 7094 vdev_free(vd); 7095 7096 if (error && 7097 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 7098 goto out; 7099 else 7100 error = 0; 7101 } 7102 7103 out: 7104 sav->sav_pending = NULL; 7105 sav->sav_npending = 0; 7106 return (error); 7107 } 7108 7109 static int 7110 spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 7111 { 7112 int error; 7113 7114 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 7115 7116 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 7117 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 7118 VDEV_LABEL_SPARE)) != 0) { 7119 return (error); 7120 } 7121 7122 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 7123 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 7124 VDEV_LABEL_L2CACHE)); 7125 } 7126 7127 static void 7128 spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 7129 const char *config) 7130 { 7131 int i; 7132 7133 if (sav->sav_config != NULL) { 7134 nvlist_t **olddevs; 7135 uint_t oldndevs; 7136 nvlist_t **newdevs; 7137 7138 /* 7139 * Generate new dev list by concatenating with the 7140 * current dev list. 7141 */ 7142 VERIFY0(nvlist_lookup_nvlist_array(sav->sav_config, config, 7143 &olddevs, &oldndevs)); 7144 7145 newdevs = kmem_alloc(sizeof (void *) * 7146 (ndevs + oldndevs), KM_SLEEP); 7147 for (i = 0; i < oldndevs; i++) 7148 newdevs[i] = fnvlist_dup(olddevs[i]); 7149 for (i = 0; i < ndevs; i++) 7150 newdevs[i + oldndevs] = fnvlist_dup(devs[i]); 7151 7152 fnvlist_remove(sav->sav_config, config); 7153 7154 fnvlist_add_nvlist_array(sav->sav_config, config, 7155 (const nvlist_t * const *)newdevs, ndevs + oldndevs); 7156 for (i = 0; i < oldndevs + ndevs; i++) 7157 nvlist_free(newdevs[i]); 7158 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 7159 } else { 7160 /* 7161 * Generate a new dev list. 7162 */ 7163 sav->sav_config = fnvlist_alloc(); 7164 fnvlist_add_nvlist_array(sav->sav_config, config, 7165 (const nvlist_t * const *)devs, ndevs); 7166 } 7167 } 7168 7169 /* 7170 * Stop and drop level 2 ARC devices 7171 */ 7172 void 7173 spa_l2cache_drop(spa_t *spa) 7174 { 7175 vdev_t *vd; 7176 int i; 7177 spa_aux_vdev_t *sav = &spa->spa_l2cache; 7178 7179 for (i = 0; i < sav->sav_count; i++) { 7180 uint64_t pool; 7181 7182 vd = sav->sav_vdevs[i]; 7183 ASSERT(vd != NULL); 7184 7185 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 7186 pool != 0ULL && l2arc_vdev_present(vd)) 7187 l2arc_remove_vdev(vd); 7188 } 7189 } 7190 7191 /* 7192 * Verify encryption parameters for spa creation. If we are encrypting, we must 7193 * have the encryption feature flag enabled. 7194 */ 7195 static int 7196 spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 7197 boolean_t has_encryption) 7198 { 7199 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 7200 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 7201 !has_encryption) 7202 return (SET_ERROR(ENOTSUP)); 7203 7204 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 7205 } 7206 7207 /* 7208 * Pool Creation 7209 */ 7210 int 7211 spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 7212 nvlist_t *zplprops, dsl_crypto_params_t *dcp, nvlist_t **errinfo) 7213 { 7214 spa_t *spa; 7215 const char *altroot = NULL; 7216 vdev_t *rvd; 7217 dsl_pool_t *dp; 7218 dmu_tx_t *tx; 7219 int error = 0; 7220 uint64_t txg = TXG_INITIAL; 7221 nvlist_t **spares, **l2cache; 7222 uint_t nspares, nl2cache; 7223 uint64_t version, obj, ndraid = 0, draid_nfgroup = 0; 7224 boolean_t has_features; 7225 boolean_t has_encryption; 7226 boolean_t has_allocclass; 7227 boolean_t has_draid; 7228 boolean_t has_draid_fdomains; 7229 spa_feature_t feat; 7230 const char *feat_name; 7231 const char *poolname; 7232 nvlist_t *nvl; 7233 7234 if (props == NULL || 7235 nvlist_lookup_string(props, 7236 zpool_prop_to_name(ZPOOL_PROP_TNAME), &poolname) != 0) 7237 poolname = (char *)pool; 7238 7239 /* 7240 * If this pool already exists, return failure. 7241 */ 7242 spa_namespace_enter(FTAG); 7243 if (spa_lookup(poolname) != NULL) { 7244 spa_namespace_exit(FTAG); 7245 return (SET_ERROR(EEXIST)); 7246 } 7247 7248 /* 7249 * Allocate a new spa_t structure. 7250 */ 7251 nvl = fnvlist_alloc(); 7252 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 7253 (void) nvlist_lookup_string(props, 7254 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 7255 spa = spa_add(poolname, nvl, altroot); 7256 fnvlist_free(nvl); 7257 spa_activate(spa, spa_mode_global); 7258 7259 if (props && (error = spa_prop_validate(spa, props))) { 7260 spa_deactivate(spa); 7261 spa_remove(spa); 7262 spa_namespace_exit(FTAG); 7263 return (error); 7264 } 7265 7266 /* 7267 * Temporary pool names should never be written to disk. 7268 */ 7269 if (poolname != pool) 7270 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 7271 7272 has_features = B_FALSE; 7273 has_encryption = B_FALSE; 7274 has_allocclass = B_FALSE; 7275 has_draid = B_FALSE; 7276 has_draid_fdomains = B_FALSE; 7277 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 7278 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 7279 if (zpool_prop_feature(nvpair_name(elem))) { 7280 has_features = B_TRUE; 7281 7282 feat_name = strchr(nvpair_name(elem), '@') + 1; 7283 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 7284 if (feat == SPA_FEATURE_ENCRYPTION) 7285 has_encryption = B_TRUE; 7286 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 7287 has_allocclass = B_TRUE; 7288 if (feat == SPA_FEATURE_DRAID) 7289 has_draid = B_TRUE; 7290 if (feat == SPA_FEATURE_DRAID_FAIL_DOMAINS) 7291 has_draid_fdomains = B_TRUE; 7292 } 7293 } 7294 7295 /* verify encryption params, if they were provided */ 7296 if (dcp != NULL) { 7297 error = spa_create_check_encryption_params(dcp, has_encryption); 7298 if (error != 0) { 7299 spa_deactivate(spa); 7300 spa_remove(spa); 7301 spa_namespace_exit(FTAG); 7302 return (error); 7303 } 7304 } 7305 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 7306 spa_deactivate(spa); 7307 spa_remove(spa); 7308 spa_namespace_exit(FTAG); 7309 return (ENOTSUP); 7310 } 7311 7312 if (has_features || nvlist_lookup_uint64(props, 7313 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 7314 version = SPA_VERSION; 7315 } 7316 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 7317 7318 spa->spa_first_txg = txg; 7319 spa->spa_uberblock.ub_txg = txg - 1; 7320 spa->spa_uberblock.ub_version = version; 7321 spa->spa_ubsync = spa->spa_uberblock; 7322 spa->spa_load_state = SPA_LOAD_CREATE; 7323 spa->spa_removing_phys.sr_state = DSS_NONE; 7324 spa->spa_removing_phys.sr_removing_vdev = -1; 7325 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 7326 spa->spa_indirect_vdevs_loaded = B_TRUE; 7327 spa->spa_deflate = (version >= SPA_VERSION_RAIDZ_DEFLATE); 7328 7329 /* 7330 * Create "The Godfather" zio to hold all async IOs 7331 */ 7332 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 7333 KM_SLEEP); 7334 for (int i = 0; i < max_ncpus; i++) { 7335 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 7336 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 7337 ZIO_FLAG_GODFATHER); 7338 } 7339 7340 /* 7341 * Create the root vdev. 7342 */ 7343 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7344 7345 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 7346 7347 ASSERT(error != 0 || rvd != NULL); 7348 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 7349 7350 if (error == 0 && !zfs_allocatable_devs(nvroot)) 7351 error = SET_ERROR(EINVAL); 7352 7353 if (error == 0 && 7354 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 7355 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 7356 &draid_nfgroup, 0)) == 0 && 7357 (ndraid == 0 || has_draid || (error = SET_ERROR(ENOTSUP))) && 7358 (draid_nfgroup == 0 || has_draid_fdomains || 7359 (error = SET_ERROR(ENOTSUP))) && error == 0 && 7360 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 7361 /* 7362 * instantiate the metaslab groups (this will dirty the vdevs) 7363 * we can no longer error exit past this point 7364 */ 7365 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 7366 vdev_t *vd = rvd->vdev_child[c]; 7367 7368 vdev_metaslab_set_size(vd); 7369 vdev_expand(vd, txg); 7370 } 7371 } 7372 7373 spa_config_exit(spa, SCL_ALL, FTAG); 7374 7375 if (error != 0) { 7376 if (errinfo != NULL) { 7377 *errinfo = spa->spa_create_info; 7378 spa->spa_create_info = NULL; 7379 } 7380 spa_unload(spa); 7381 spa_deactivate(spa); 7382 spa_remove(spa); 7383 spa_namespace_exit(FTAG); 7384 return (error); 7385 } 7386 7387 /* 7388 * Get the list of spares, if specified. 7389 */ 7390 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 7391 &spares, &nspares) == 0) { 7392 spa->spa_spares.sav_config = fnvlist_alloc(); 7393 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 7394 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 7395 nspares); 7396 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7397 spa_load_spares(spa); 7398 spa_config_exit(spa, SCL_ALL, FTAG); 7399 spa->spa_spares.sav_sync = B_TRUE; 7400 } 7401 7402 /* 7403 * Get the list of level 2 cache devices, if specified. 7404 */ 7405 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 7406 &l2cache, &nl2cache) == 0) { 7407 VERIFY0(nvlist_alloc(&spa->spa_l2cache.sav_config, 7408 NV_UNIQUE_NAME, KM_SLEEP)); 7409 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 7410 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 7411 nl2cache); 7412 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7413 spa_load_l2cache(spa); 7414 spa_config_exit(spa, SCL_ALL, FTAG); 7415 spa->spa_l2cache.sav_sync = B_TRUE; 7416 } 7417 7418 spa->spa_is_initializing = B_TRUE; 7419 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 7420 spa->spa_is_initializing = B_FALSE; 7421 7422 /* 7423 * Create DDTs (dedup tables). 7424 */ 7425 ddt_create(spa); 7426 /* 7427 * Create BRT table and BRT table object. 7428 */ 7429 brt_create(spa); 7430 7431 spa_update_dspace(spa); 7432 7433 tx = dmu_tx_create_assigned(dp, txg); 7434 7435 /* 7436 * Create the pool's history object. 7437 */ 7438 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 7439 spa_history_create_obj(spa, tx); 7440 7441 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 7442 spa_history_log_version(spa, "create", tx); 7443 7444 /* 7445 * Create the pool config object. 7446 */ 7447 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 7448 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 7449 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 7450 7451 if (zap_add(spa->spa_meta_objset, 7452 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 7453 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 7454 cmn_err(CE_PANIC, "failed to add pool config"); 7455 } 7456 7457 if (zap_add(spa->spa_meta_objset, 7458 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 7459 sizeof (uint64_t), 1, &version, tx) != 0) { 7460 cmn_err(CE_PANIC, "failed to add pool version"); 7461 } 7462 7463 /* Newly created pools with the right version are always deflated. */ 7464 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 7465 if (zap_add(spa->spa_meta_objset, 7466 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 7467 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 7468 cmn_err(CE_PANIC, "failed to add deflate"); 7469 } 7470 } 7471 7472 /* 7473 * Create the deferred-free bpobj. Turn off compression 7474 * because sync-to-convergence takes longer if the blocksize 7475 * keeps changing. 7476 */ 7477 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 7478 dmu_object_set_compress(spa->spa_meta_objset, obj, 7479 ZIO_COMPRESS_OFF, tx); 7480 if (zap_add(spa->spa_meta_objset, 7481 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 7482 sizeof (uint64_t), 1, &obj, tx) != 0) { 7483 cmn_err(CE_PANIC, "failed to add bpobj"); 7484 } 7485 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 7486 spa->spa_meta_objset, obj)); 7487 7488 /* 7489 * Generate some random noise for salted checksums to operate on. 7490 */ 7491 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 7492 sizeof (spa->spa_cksum_salt.zcs_bytes)); 7493 7494 /* 7495 * Set pool properties. 7496 */ 7497 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 7498 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 7499 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 7500 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 7501 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 7502 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 7503 spa->spa_dedup_table_quota = 7504 zpool_prop_default_numeric(ZPOOL_PROP_DEDUP_TABLE_QUOTA); 7505 7506 if (props != NULL) { 7507 spa_configfile_set(spa, props, B_FALSE); 7508 spa_sync_props(props, tx); 7509 } 7510 7511 for (int i = 0; i < ndraid; i++) 7512 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 7513 7514 for (int i = 0; i < draid_nfgroup; i++) 7515 spa_feature_incr(spa, SPA_FEATURE_DRAID_FAIL_DOMAINS, tx); 7516 7517 dmu_tx_commit(tx); 7518 7519 spa->spa_sync_on = B_TRUE; 7520 txg_sync_start(dp); 7521 mmp_thread_start(spa); 7522 txg_wait_synced(dp, txg); 7523 7524 spa_spawn_aux_threads(spa); 7525 7526 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_TRUE); 7527 7528 /* 7529 * Don't count references from objsets that are already closed 7530 * and are making their way through the eviction process. 7531 */ 7532 spa_evicting_os_wait(spa); 7533 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 7534 spa->spa_load_state = SPA_LOAD_NONE; 7535 7536 spa_import_os(spa); 7537 7538 spa_namespace_exit(FTAG); 7539 7540 return (0); 7541 } 7542 7543 /* 7544 * Import a non-root pool into the system. 7545 */ 7546 int 7547 spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 7548 { 7549 spa_t *spa; 7550 const char *altroot = NULL; 7551 spa_load_state_t state = SPA_LOAD_IMPORT; 7552 zpool_load_policy_t policy; 7553 spa_mode_t mode = spa_mode_global; 7554 uint64_t readonly = B_FALSE; 7555 int error; 7556 nvlist_t *nvroot; 7557 nvlist_t **spares, **l2cache; 7558 uint_t nspares, nl2cache; 7559 7560 /* 7561 * If a pool with this name exists, return failure. 7562 */ 7563 spa_namespace_enter(FTAG); 7564 if (spa_lookup(pool) != NULL) { 7565 spa_namespace_exit(FTAG); 7566 return (SET_ERROR(EEXIST)); 7567 } 7568 7569 /* 7570 * Create and initialize the spa structure. 7571 */ 7572 (void) nvlist_lookup_string(props, 7573 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 7574 (void) nvlist_lookup_uint64(props, 7575 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 7576 if (readonly) 7577 mode = SPA_MODE_READ; 7578 spa = spa_add(pool, config, altroot); 7579 spa->spa_import_flags = flags; 7580 7581 /* 7582 * Verbatim import - Take a pool and insert it into the namespace 7583 * as if it had been loaded at boot. 7584 */ 7585 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 7586 if (props != NULL) 7587 spa_configfile_set(spa, props, B_FALSE); 7588 7589 spa_write_cachefile(spa, B_FALSE, B_TRUE, B_FALSE); 7590 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 7591 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 7592 spa_namespace_exit(FTAG); 7593 return (0); 7594 } 7595 7596 spa_activate(spa, mode); 7597 7598 /* 7599 * Don't start async tasks until we know everything is healthy. 7600 */ 7601 spa_async_suspend(spa); 7602 7603 zpool_get_load_policy(config, &policy); 7604 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 7605 state = SPA_LOAD_RECOVER; 7606 7607 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 7608 7609 if (state != SPA_LOAD_RECOVER) { 7610 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 7611 zfs_dbgmsg("spa_import: importing %s", pool); 7612 } else { 7613 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 7614 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 7615 } 7616 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 7617 7618 /* 7619 * Propagate anything learned while loading the pool and pass it 7620 * back to caller (i.e. rewind info, missing devices, etc). 7621 */ 7622 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info); 7623 7624 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7625 /* 7626 * Toss any existing sparelist, as it doesn't have any validity 7627 * anymore, and conflicts with spa_has_spare(). 7628 */ 7629 if (spa->spa_spares.sav_config) { 7630 nvlist_free(spa->spa_spares.sav_config); 7631 spa->spa_spares.sav_config = NULL; 7632 spa_load_spares(spa); 7633 } 7634 if (spa->spa_l2cache.sav_config) { 7635 nvlist_free(spa->spa_l2cache.sav_config); 7636 spa->spa_l2cache.sav_config = NULL; 7637 spa_load_l2cache(spa); 7638 } 7639 7640 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE); 7641 spa_config_exit(spa, SCL_ALL, FTAG); 7642 7643 if (props != NULL) 7644 spa_configfile_set(spa, props, B_FALSE); 7645 7646 if (error != 0 || (props && spa_writeable(spa) && 7647 (error = spa_prop_set(spa, props)))) { 7648 spa_unload(spa); 7649 spa_deactivate(spa); 7650 spa_remove(spa); 7651 spa_namespace_exit(FTAG); 7652 return (error); 7653 } 7654 7655 spa_async_resume(spa); 7656 7657 /* 7658 * Override any spares and level 2 cache devices as specified by 7659 * the user, as these may have correct device names/devids, etc. 7660 */ 7661 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 7662 &spares, &nspares) == 0) { 7663 if (spa->spa_spares.sav_config) 7664 fnvlist_remove(spa->spa_spares.sav_config, 7665 ZPOOL_CONFIG_SPARES); 7666 else 7667 spa->spa_spares.sav_config = fnvlist_alloc(); 7668 fnvlist_add_nvlist_array(spa->spa_spares.sav_config, 7669 ZPOOL_CONFIG_SPARES, (const nvlist_t * const *)spares, 7670 nspares); 7671 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7672 spa_load_spares(spa); 7673 spa_config_exit(spa, SCL_ALL, FTAG); 7674 spa->spa_spares.sav_sync = B_TRUE; 7675 spa->spa_spares.sav_label_sync = B_TRUE; 7676 } 7677 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 7678 &l2cache, &nl2cache) == 0) { 7679 if (spa->spa_l2cache.sav_config) 7680 fnvlist_remove(spa->spa_l2cache.sav_config, 7681 ZPOOL_CONFIG_L2CACHE); 7682 else 7683 spa->spa_l2cache.sav_config = fnvlist_alloc(); 7684 fnvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 7685 ZPOOL_CONFIG_L2CACHE, (const nvlist_t * const *)l2cache, 7686 nl2cache); 7687 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7688 spa_load_l2cache(spa); 7689 spa_config_exit(spa, SCL_ALL, FTAG); 7690 spa->spa_l2cache.sav_sync = B_TRUE; 7691 spa->spa_l2cache.sav_label_sync = B_TRUE; 7692 } 7693 7694 /* 7695 * Check for any removed devices. 7696 */ 7697 if (spa->spa_autoreplace) { 7698 spa_aux_check_removed(&spa->spa_spares); 7699 spa_aux_check_removed(&spa->spa_l2cache); 7700 } 7701 7702 if (spa_writeable(spa)) { 7703 /* 7704 * Update the config cache to include the newly-imported pool. 7705 */ 7706 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 7707 } 7708 7709 /* 7710 * It's possible that the pool was expanded while it was exported. 7711 * We kick off an async task to handle this for us. 7712 */ 7713 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 7714 7715 spa_history_log_version(spa, "import", NULL); 7716 7717 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 7718 7719 spa_namespace_exit(FTAG); 7720 7721 zvol_create_minors(pool); 7722 7723 spa_import_os(spa); 7724 7725 return (0); 7726 } 7727 7728 nvlist_t * 7729 spa_tryimport(nvlist_t *tryconfig) 7730 { 7731 nvlist_t *config = NULL; 7732 const char *poolname, *cachefile; 7733 spa_t *spa; 7734 uint64_t state; 7735 int error; 7736 zpool_load_policy_t policy; 7737 7738 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 7739 return (NULL); 7740 7741 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 7742 return (NULL); 7743 7744 /* 7745 * Create and initialize the spa structure. 7746 */ 7747 char *name = kmem_alloc(MAXPATHLEN, KM_SLEEP); 7748 (void) snprintf(name, MAXPATHLEN, "%s-%llx-%s", 7749 TRYIMPORT_NAME, (u_longlong_t)(uintptr_t)curthread, poolname); 7750 7751 spa_namespace_enter(FTAG); 7752 spa = spa_add(name, tryconfig, NULL); 7753 spa_activate(spa, SPA_MODE_READ); 7754 kmem_free(name, MAXPATHLEN); 7755 7756 spa->spa_load_name = spa_strdup(poolname); 7757 7758 /* 7759 * Rewind pool if a max txg was provided. 7760 */ 7761 zpool_get_load_policy(spa->spa_config, &policy); 7762 if (policy.zlp_txg != UINT64_MAX) { 7763 spa->spa_load_max_txg = policy.zlp_txg; 7764 spa->spa_extreme_rewind = B_TRUE; 7765 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 7766 spa_load_name(spa), (longlong_t)policy.zlp_txg); 7767 } else { 7768 zfs_dbgmsg("spa_tryimport: importing %s", spa_load_name(spa)); 7769 } 7770 7771 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 7772 == 0) { 7773 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 7774 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 7775 } else { 7776 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 7777 } 7778 7779 /* 7780 * spa_import() relies on a pool config fetched by spa_try_import() 7781 * for spare/cache devices. Import flags are not passed to 7782 * spa_tryimport(), which makes it return early due to a missing log 7783 * device and missing retrieving the cache device and spare eventually. 7784 * Passing ZFS_IMPORT_MISSING_LOG to spa_tryimport() makes it fetch 7785 * the correct configuration regardless of the missing log device. 7786 */ 7787 spa->spa_import_flags |= ZFS_IMPORT_MISSING_LOG; 7788 7789 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 7790 7791 /* 7792 * If 'tryconfig' was at least parsable, return the current config. 7793 */ 7794 if (spa->spa_root_vdev != NULL) { 7795 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 7796 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, 7797 spa_load_name(spa)); 7798 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, state); 7799 fnvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 7800 spa->spa_uberblock.ub_timestamp); 7801 fnvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 7802 spa->spa_load_info); 7803 fnvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 7804 spa->spa_errata); 7805 7806 /* 7807 * If the bootfs property exists on this pool then we 7808 * copy it out so that external consumers can tell which 7809 * pools are bootable. 7810 */ 7811 if ((!error || error == EEXIST) && spa->spa_bootfs) { 7812 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 7813 7814 /* 7815 * We have to play games with the name since the 7816 * pool was opened as TRYIMPORT_NAME. 7817 */ 7818 if (dsl_dsobj_to_dsname(spa_name(spa), 7819 spa->spa_bootfs, tmpname) == 0) { 7820 char *cp; 7821 char *dsname; 7822 7823 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 7824 7825 cp = strchr(tmpname, '/'); 7826 if (cp == NULL) { 7827 (void) strlcpy(dsname, tmpname, 7828 MAXPATHLEN); 7829 } else { 7830 (void) snprintf(dsname, MAXPATHLEN, 7831 "%s/%s", spa_load_name(spa), ++cp); 7832 } 7833 fnvlist_add_string(config, ZPOOL_CONFIG_BOOTFS, 7834 dsname); 7835 kmem_free(dsname, MAXPATHLEN); 7836 } 7837 kmem_free(tmpname, MAXPATHLEN); 7838 } 7839 7840 /* 7841 * Add the list of hot spares and level 2 cache devices. 7842 */ 7843 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 7844 spa_add_spares(spa, config); 7845 spa_add_l2cache(spa, config); 7846 spa_config_exit(spa, SCL_CONFIG, FTAG); 7847 } 7848 7849 spa_unload(spa); 7850 spa_deactivate(spa); 7851 spa_remove(spa); 7852 spa_namespace_exit(FTAG); 7853 7854 return (config); 7855 } 7856 7857 /* 7858 * Pool export/destroy 7859 * 7860 * The act of destroying or exporting a pool is very simple. We make sure there 7861 * is no more pending I/O and any references to the pool are gone. Then, we 7862 * update the pool state and sync all the labels to disk, removing the 7863 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 7864 * we don't sync the labels or remove the configuration cache. 7865 */ 7866 static int 7867 spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 7868 boolean_t force, boolean_t hardforce) 7869 { 7870 int error = 0; 7871 spa_t *spa; 7872 hrtime_t export_start = gethrtime(); 7873 7874 if (oldconfig) 7875 *oldconfig = NULL; 7876 7877 if (!(spa_mode_global & SPA_MODE_WRITE)) 7878 return (SET_ERROR(EROFS)); 7879 7880 spa_namespace_enter(FTAG); 7881 if ((spa = spa_lookup(pool)) == NULL) { 7882 spa_namespace_exit(FTAG); 7883 return (SET_ERROR(ENOENT)); 7884 } 7885 7886 if (spa->spa_is_exporting) { 7887 /* the pool is being exported by another thread */ 7888 spa_namespace_exit(FTAG); 7889 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 7890 } 7891 spa->spa_is_exporting = B_TRUE; 7892 7893 /* 7894 * Put a hold on the pool, drop the namespace lock, stop async tasks 7895 * and see if we can export. 7896 */ 7897 spa_open_ref(spa, FTAG); 7898 spa_namespace_exit(FTAG); 7899 #ifdef ZFS_DEBUG 7900 spa_condense_debug_cancel(spa); 7901 #endif 7902 spa_async_suspend(spa); 7903 7904 spa_namespace_enter(FTAG); 7905 spa->spa_export_thread = curthread; 7906 spa_close(spa, FTAG); 7907 7908 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 7909 spa_namespace_exit(FTAG); 7910 goto export_spa; 7911 } 7912 7913 /* 7914 * The pool will be in core if it's openable, in which case we can 7915 * modify its state. Objsets may be open only because they're dirty, 7916 * so we have to force it to sync before checking spa_refcnt. 7917 */ 7918 if (spa->spa_sync_on) { 7919 txg_wait_synced(spa->spa_dsl_pool, 0); 7920 spa_evicting_os_wait(spa); 7921 } 7922 7923 /* 7924 * A pool cannot be exported or destroyed if there are active 7925 * references. If we are resetting a pool, allow references by 7926 * fault injection handlers. 7927 */ 7928 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 7929 error = SET_ERROR(EBUSY); 7930 goto fail; 7931 } 7932 7933 spa_namespace_exit(FTAG); 7934 /* 7935 * At this point we no longer hold the spa_namespace_lock and 7936 * there were no references on the spa. Future spa_lookups will 7937 * notice the spa->spa_export_thread and wait until we signal 7938 * that we are finshed. 7939 */ 7940 7941 if (spa->spa_zvol_taskq) { 7942 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 7943 taskq_wait(spa->spa_zvol_taskq); 7944 } 7945 7946 if (spa->spa_sync_on) { 7947 vdev_t *rvd = spa->spa_root_vdev; 7948 /* 7949 * A pool cannot be exported if it has an active shared spare. 7950 * This is to prevent other pools stealing the active spare 7951 * from an exported pool. At user's own will, such pool can 7952 * be forcedly exported. 7953 */ 7954 if (!force && new_state == POOL_STATE_EXPORTED && 7955 spa_has_active_shared_spare(spa)) { 7956 error = SET_ERROR(EXDEV); 7957 spa_namespace_enter(FTAG); 7958 goto fail; 7959 } 7960 7961 /* 7962 * We're about to export or destroy this pool. Make sure 7963 * we stop all initialization and trim activity here before 7964 * we set the spa_final_txg. This will ensure that all 7965 * dirty data resulting from the initialization is 7966 * committed to disk before we unload the pool. 7967 */ 7968 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 7969 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 7970 vdev_autotrim_stop_all(spa); 7971 vdev_rebuild_stop_all(spa); 7972 l2arc_spa_rebuild_stop(spa); 7973 7974 /* 7975 * We want this to be reflected on every label, 7976 * so mark them all dirty. spa_unload() will do the 7977 * final sync that pushes these changes out. 7978 */ 7979 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 7980 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7981 spa->spa_state = new_state; 7982 vdev_config_dirty(rvd); 7983 spa_config_exit(spa, SCL_ALL, FTAG); 7984 } 7985 7986 if (spa_should_sync_time_logger_on_unload(spa)) 7987 spa_unload_sync_time_logger(spa); 7988 7989 /* 7990 * If the log space map feature is enabled and the pool is 7991 * getting exported (but not destroyed), we want to spend some 7992 * time flushing as many metaslabs as we can in an attempt to 7993 * destroy log space maps and save import time. This has to be 7994 * done before we set the spa_final_txg, otherwise 7995 * spa_sync() -> spa_flush_metaslabs() may dirty the final TXGs. 7996 * spa_should_flush_logs_on_unload() should be called after 7997 * spa_state has been set to the new_state. 7998 */ 7999 if (spa_should_flush_logs_on_unload(spa)) 8000 spa_unload_log_sm_flush_all(spa); 8001 8002 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 8003 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 8004 spa->spa_final_txg = spa_last_synced_txg(spa) + 8005 TXG_DEFER_SIZE + 1; 8006 spa_config_exit(spa, SCL_ALL, FTAG); 8007 } 8008 } 8009 8010 export_spa: 8011 spa_export_os(spa); 8012 8013 if (new_state == POOL_STATE_DESTROYED) 8014 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 8015 else if (new_state == POOL_STATE_EXPORTED) 8016 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 8017 8018 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 8019 spa_unload(spa); 8020 spa_deactivate(spa); 8021 } 8022 8023 if (oldconfig && spa->spa_config) 8024 *oldconfig = fnvlist_dup(spa->spa_config); 8025 8026 if (new_state == POOL_STATE_EXPORTED) 8027 zio_handle_export_delay(spa, gethrtime() - export_start); 8028 8029 /* 8030 * Take the namespace lock for the actual spa_t removal 8031 */ 8032 spa_namespace_enter(FTAG); 8033 if (new_state != POOL_STATE_UNINITIALIZED) { 8034 if (!hardforce) 8035 spa_write_cachefile(spa, B_TRUE, B_TRUE, B_FALSE); 8036 spa_remove(spa); 8037 } else { 8038 /* 8039 * If spa_remove() is not called for this spa_t and 8040 * there is any possibility that it can be reused, 8041 * we make sure to reset the exporting flag. 8042 */ 8043 spa->spa_is_exporting = B_FALSE; 8044 spa->spa_export_thread = NULL; 8045 } 8046 8047 /* 8048 * Wake up any waiters in spa_lookup() 8049 */ 8050 spa_namespace_broadcast(); 8051 spa_namespace_exit(FTAG); 8052 return (0); 8053 8054 fail: 8055 spa->spa_is_exporting = B_FALSE; 8056 spa->spa_export_thread = NULL; 8057 8058 spa_async_resume(spa); 8059 /* 8060 * Wake up any waiters in spa_lookup() 8061 */ 8062 spa_namespace_broadcast(); 8063 spa_namespace_exit(FTAG); 8064 return (error); 8065 } 8066 8067 /* 8068 * Destroy a storage pool. 8069 */ 8070 int 8071 spa_destroy(const char *pool) 8072 { 8073 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 8074 B_FALSE, B_FALSE)); 8075 } 8076 8077 /* 8078 * Export a storage pool. 8079 */ 8080 int 8081 spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 8082 boolean_t hardforce) 8083 { 8084 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 8085 force, hardforce)); 8086 } 8087 8088 /* 8089 * Similar to spa_export(), this unloads the spa_t without actually removing it 8090 * from the namespace in any way. 8091 */ 8092 int 8093 spa_reset(const char *pool) 8094 { 8095 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 8096 B_FALSE, B_FALSE)); 8097 } 8098 8099 /* 8100 * ========================================================================== 8101 * Device manipulation 8102 * ========================================================================== 8103 */ 8104 8105 /* 8106 * This is called as a synctask to increment the draid feature flag 8107 */ 8108 static void 8109 spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 8110 { 8111 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8112 int draid = (int)(uintptr_t)arg; 8113 8114 for (int c = 0; c < draid; c++) 8115 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 8116 } 8117 8118 /* 8119 * This is called as a synctask to increment the draid_fail_domains feature flag 8120 */ 8121 static void 8122 spa_draid_fdomains_feature_incr(void *arg, dmu_tx_t *tx) 8123 { 8124 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8125 int nfgrp = (int)(uintptr_t)arg; 8126 8127 for (int c = 0; c < nfgrp; c++) 8128 spa_feature_incr(spa, SPA_FEATURE_DRAID_FAIL_DOMAINS, tx); 8129 } 8130 8131 /* 8132 * Add a device to a storage pool. 8133 */ 8134 int 8135 spa_vdev_add(spa_t *spa, nvlist_t *nvroot, boolean_t check_ashift) 8136 { 8137 uint64_t txg, ndraid = 0, draid_nfgroup = 0; 8138 int error; 8139 vdev_t *rvd = spa->spa_root_vdev; 8140 vdev_t *vd, *tvd; 8141 nvlist_t **spares, **l2cache; 8142 uint_t nspares, nl2cache; 8143 8144 ASSERT(spa_writeable(spa)); 8145 8146 txg = spa_vdev_enter(spa); 8147 8148 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 8149 VDEV_ALLOC_ADD)) != 0) 8150 return (spa_vdev_exit(spa, NULL, txg, error)); 8151 8152 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 8153 8154 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 8155 &nspares) != 0) 8156 nspares = 0; 8157 8158 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 8159 &nl2cache) != 0) 8160 nl2cache = 0; 8161 8162 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 8163 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 8164 8165 if (vd->vdev_children != 0 && 8166 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 8167 return (spa_vdev_exit(spa, vd, txg, error)); 8168 } 8169 8170 /* 8171 * The virtual dRAID spares must be added after vdev tree is created 8172 * and the vdev guids are generated. The guid of their associated 8173 * dRAID is stored in the config and used when opening the spare. 8174 */ 8175 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 8176 &draid_nfgroup, rvd->vdev_children)) == 0) { 8177 8178 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 8179 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 8180 nspares = 0; 8181 8182 if (draid_nfgroup > 0 && !spa_feature_is_enabled(spa, 8183 SPA_FEATURE_DRAID_FAIL_DOMAINS)) 8184 return (spa_vdev_exit(spa, vd, txg, ENOTSUP)); 8185 } else { 8186 return (spa_vdev_exit(spa, vd, txg, error)); 8187 } 8188 8189 /* 8190 * We must validate the spares and l2cache devices after checking the 8191 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 8192 */ 8193 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 8194 return (spa_vdev_exit(spa, vd, txg, error)); 8195 8196 /* 8197 * If we are in the middle of a device removal, we can only add 8198 * devices which match the existing devices in the pool. 8199 * If we are in the middle of a removal, or have some indirect 8200 * vdevs, we can not add raidz or dRAID top levels. 8201 */ 8202 if (spa->spa_vdev_removal != NULL || 8203 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 8204 for (int c = 0; c < vd->vdev_children; c++) { 8205 tvd = vd->vdev_child[c]; 8206 if (spa->spa_vdev_removal != NULL && 8207 tvd->vdev_ashift != spa->spa_max_ashift) { 8208 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 8209 } 8210 /* Fail if top level vdev is raidz or a dRAID */ 8211 if (vdev_get_nparity(tvd) != 0) 8212 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 8213 8214 /* 8215 * Need the top level mirror to be 8216 * a mirror of leaf vdevs only 8217 */ 8218 if (tvd->vdev_ops == &vdev_mirror_ops) { 8219 for (uint64_t cid = 0; 8220 cid < tvd->vdev_children; cid++) { 8221 vdev_t *cvd = tvd->vdev_child[cid]; 8222 if (!cvd->vdev_ops->vdev_op_leaf) { 8223 return (spa_vdev_exit(spa, vd, 8224 txg, EINVAL)); 8225 } 8226 } 8227 } 8228 } 8229 } 8230 8231 if (check_ashift && spa->spa_max_ashift == spa->spa_min_ashift) { 8232 for (int c = 0; c < vd->vdev_children; c++) { 8233 tvd = vd->vdev_child[c]; 8234 if (tvd->vdev_ashift != spa->spa_max_ashift) { 8235 return (spa_vdev_exit(spa, vd, txg, 8236 ZFS_ERR_ASHIFT_MISMATCH)); 8237 } 8238 } 8239 } 8240 8241 for (int c = 0; c < vd->vdev_children; c++) { 8242 tvd = vd->vdev_child[c]; 8243 vdev_remove_child(vd, tvd); 8244 tvd->vdev_id = rvd->vdev_children; 8245 vdev_add_child(rvd, tvd); 8246 vdev_config_dirty(tvd); 8247 } 8248 8249 if (nspares != 0) { 8250 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 8251 ZPOOL_CONFIG_SPARES); 8252 spa_load_spares(spa); 8253 spa->spa_spares.sav_sync = B_TRUE; 8254 } 8255 8256 if (nl2cache != 0) { 8257 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 8258 ZPOOL_CONFIG_L2CACHE); 8259 spa_load_l2cache(spa); 8260 spa->spa_l2cache.sav_sync = B_TRUE; 8261 } 8262 8263 /* 8264 * We can't increment a feature while holding spa_vdev so we 8265 * have to do it in a synctask. 8266 */ 8267 if (ndraid != 0) { 8268 dmu_tx_t *tx; 8269 8270 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 8271 8272 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 8273 (void *)(uintptr_t)ndraid, tx); 8274 8275 if (draid_nfgroup > 0) 8276 dsl_sync_task_nowait(spa->spa_dsl_pool, 8277 spa_draid_fdomains_feature_incr, 8278 (void *)(uintptr_t)draid_nfgroup, tx); 8279 8280 dmu_tx_commit(tx); 8281 } 8282 8283 /* 8284 * We have to be careful when adding new vdevs to an existing pool. 8285 * If other threads start allocating from these vdevs before we 8286 * sync the config cache, and we lose power, then upon reboot we may 8287 * fail to open the pool because there are DVAs that the config cache 8288 * can't translate. Therefore, we first add the vdevs without 8289 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 8290 * and then let spa_config_update() initialize the new metaslabs. 8291 * 8292 * spa_load() checks for added-but-not-initialized vdevs, so that 8293 * if we lose power at any point in this sequence, the remaining 8294 * steps will be completed the next time we load the pool. 8295 */ 8296 (void) spa_vdev_exit(spa, vd, txg, 0); 8297 8298 spa_namespace_enter(FTAG); 8299 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 8300 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 8301 spa_namespace_exit(FTAG); 8302 8303 return (0); 8304 } 8305 8306 /* 8307 * Given a vdev to be replaced and its parent, check for a possible 8308 * "double spare" condition if a vdev is to be replaced by a spare. When this 8309 * happens, you can get two spares assigned to one failed vdev. 8310 * 8311 * To trigger a double spare condition: 8312 * 8313 * 1. disk1 fails 8314 * 2. 1st spare is kicked in for disk1 and it resilvers 8315 * 3. Someone replaces disk1 with a new blank disk 8316 * 4. New blank disk starts resilvering 8317 * 5. While resilvering, new blank disk has IO errors and faults 8318 * 6. 2nd spare is kicked in for new blank disk 8319 * 7. At this point two spares are kicked in for the original disk1. 8320 * 8321 * It looks like this: 8322 * 8323 * NAME STATE READ WRITE CKSUM 8324 * tank2 DEGRADED 0 0 0 8325 * draid2:6d:10c:2s-0 DEGRADED 0 0 0 8326 * scsi-0QEMU_QEMU_HARDDISK_d1 ONLINE 0 0 0 8327 * scsi-0QEMU_QEMU_HARDDISK_d2 ONLINE 0 0 0 8328 * scsi-0QEMU_QEMU_HARDDISK_d3 ONLINE 0 0 0 8329 * scsi-0QEMU_QEMU_HARDDISK_d4 ONLINE 0 0 0 8330 * scsi-0QEMU_QEMU_HARDDISK_d5 ONLINE 0 0 0 8331 * scsi-0QEMU_QEMU_HARDDISK_d6 ONLINE 0 0 0 8332 * scsi-0QEMU_QEMU_HARDDISK_d7 ONLINE 0 0 0 8333 * scsi-0QEMU_QEMU_HARDDISK_d8 ONLINE 0 0 0 8334 * scsi-0QEMU_QEMU_HARDDISK_d9 ONLINE 0 0 0 8335 * spare-9 DEGRADED 0 0 0 8336 * replacing-0 DEGRADED 0 93 0 8337 * scsi-0QEMU_QEMU_HARDDISK_d10-part1/old UNAVAIL 0 0 0 8338 * spare-1 DEGRADED 0 0 0 8339 * scsi-0QEMU_QEMU_HARDDISK_d10 REMOVED 0 0 0 8340 * draid2-0-0 ONLINE 0 0 0 8341 * draid2-0-1 ONLINE 0 0 0 8342 * spares 8343 * draid2-0-0 INUSE currently in use 8344 * draid2-0-1 INUSE currently in use 8345 * 8346 * ARGS: 8347 * 8348 * newvd: New spare disk 8349 * pvd: Parent vdev_t the spare should attach to 8350 * 8351 * This function returns B_TRUE if adding the new vdev would create a double 8352 * spare condition, B_FALSE otherwise. 8353 */ 8354 static boolean_t 8355 spa_vdev_new_spare_would_cause_double_spares(vdev_t *newvd, vdev_t *pvd) 8356 { 8357 vdev_t *ppvd; 8358 8359 ppvd = pvd->vdev_parent; 8360 if (ppvd == NULL) 8361 return (B_FALSE); 8362 8363 /* 8364 * To determine if this configuration would cause a double spare, we 8365 * look at the vdev_op of the parent vdev, and of the parent's parent 8366 * vdev. We also look at vdev_isspare on the new disk. A double spare 8367 * condition looks like this: 8368 * 8369 * 1. parent of parent's op is a spare or draid spare 8370 * 2. parent's op is replacing 8371 * 3. new disk is a spare 8372 */ 8373 if ((ppvd->vdev_ops == &vdev_spare_ops) || 8374 (ppvd->vdev_ops == &vdev_draid_spare_ops)) 8375 if (pvd->vdev_ops == &vdev_replacing_ops) 8376 if (newvd->vdev_isspare) 8377 return (B_TRUE); 8378 8379 return (B_FALSE); 8380 } 8381 8382 /* 8383 * Attach a device to a vdev specified by its guid. The vdev type can be 8384 * a mirror, a raidz, or a leaf device that is also a top-level (e.g. a 8385 * single device). When the vdev is a single device, a mirror vdev will be 8386 * automatically inserted. 8387 * 8388 * If 'replacing' is specified, the new device is intended to replace the 8389 * existing device; in this case the two devices are made into their own 8390 * mirror using the 'replacing' vdev, which is functionally identical to 8391 * the mirror vdev (it actually reuses all the same ops) but has a few 8392 * extra rules: you can't attach to it after it's been created, and upon 8393 * completion of resilvering, the first disk (the one being replaced) 8394 * is automatically detached. 8395 * 8396 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 8397 * should be performed instead of traditional healing reconstruction. From 8398 * an administrators perspective these are both resilver operations. 8399 */ 8400 int 8401 spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 8402 int rebuild) 8403 { 8404 uint64_t txg, dtl_max_txg; 8405 vdev_t *rvd = spa->spa_root_vdev; 8406 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 8407 vdev_ops_t *pvops; 8408 char *oldvdpath, *newvdpath; 8409 int newvd_isspare = B_FALSE; 8410 int error; 8411 8412 ASSERT(spa_writeable(spa)); 8413 8414 txg = spa_vdev_enter(spa); 8415 8416 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 8417 8418 ASSERT(spa_namespace_held()); 8419 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 8420 error = (spa_has_checkpoint(spa)) ? 8421 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 8422 return (spa_vdev_exit(spa, NULL, txg, error)); 8423 } 8424 8425 if (rebuild) { 8426 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 8427 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 8428 8429 if (dsl_scan_resilvering(spa_get_dsl(spa)) || 8430 dsl_scan_resilver_scheduled(spa_get_dsl(spa))) { 8431 return (spa_vdev_exit(spa, NULL, txg, 8432 ZFS_ERR_RESILVER_IN_PROGRESS)); 8433 } 8434 } else { 8435 if (vdev_rebuild_active(rvd)) 8436 return (spa_vdev_exit(spa, NULL, txg, 8437 ZFS_ERR_REBUILD_IN_PROGRESS)); 8438 } 8439 8440 if (spa->spa_vdev_removal != NULL) { 8441 return (spa_vdev_exit(spa, NULL, txg, 8442 ZFS_ERR_DEVRM_IN_PROGRESS)); 8443 } 8444 8445 if (oldvd == NULL) 8446 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 8447 8448 boolean_t raidz = oldvd->vdev_ops == &vdev_raidz_ops; 8449 8450 if (raidz) { 8451 if (!spa_feature_is_enabled(spa, SPA_FEATURE_RAIDZ_EXPANSION)) 8452 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 8453 8454 /* 8455 * Can't expand a raidz while prior expand is in progress. 8456 */ 8457 if (spa->spa_raidz_expand != NULL) { 8458 return (spa_vdev_exit(spa, NULL, txg, 8459 ZFS_ERR_RAIDZ_EXPAND_IN_PROGRESS)); 8460 } 8461 } else if (!oldvd->vdev_ops->vdev_op_leaf) { 8462 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 8463 } 8464 8465 if (raidz) 8466 pvd = oldvd; 8467 else 8468 pvd = oldvd->vdev_parent; 8469 8470 if (spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 8471 VDEV_ALLOC_ATTACH) != 0) 8472 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 8473 8474 if (newrootvd->vdev_children != 1) 8475 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 8476 8477 newvd = newrootvd->vdev_child[0]; 8478 8479 if (!newvd->vdev_ops->vdev_op_leaf) 8480 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 8481 8482 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 8483 return (spa_vdev_exit(spa, newrootvd, txg, error)); 8484 8485 /* 8486 * Spares can't replace logs 8487 */ 8488 if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare) 8489 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8490 8491 /* 8492 * For special and dedup vdevs a spare must have matching rotational 8493 * characteristics. A rotating spare replacing a non-rotating vdev 8494 * would silently degrade pool performance, so we reject the mismatch. 8495 */ 8496 if (newvd->vdev_isspare && 8497 oldvd->vdev_top->vdev_alloc_bias != VDEV_BIAS_NONE && 8498 newvd->vdev_nonrot != oldvd->vdev_nonrot) 8499 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8500 8501 /* 8502 * A dRAID spare can only replace a child of its parent dRAID vdev. 8503 */ 8504 if (newvd->vdev_ops == &vdev_draid_spare_ops && 8505 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 8506 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8507 } 8508 8509 if (rebuild) { 8510 /* 8511 * For rebuilds, the top vdev must support reconstruction 8512 * using only space maps. This means the only allowable 8513 * vdevs types are the root vdev, a mirror, or dRAID. 8514 */ 8515 tvd = pvd; 8516 if (pvd->vdev_top != NULL) 8517 tvd = pvd->vdev_top; 8518 8519 if (tvd->vdev_ops != &vdev_mirror_ops && 8520 tvd->vdev_ops != &vdev_root_ops && 8521 tvd->vdev_ops != &vdev_draid_ops) { 8522 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8523 } 8524 } 8525 8526 if (!replacing) { 8527 /* 8528 * For attach, the only allowable parent is a mirror or 8529 * the root vdev. A raidz vdev can be attached to, but 8530 * you cannot attach to a raidz child. 8531 */ 8532 if (pvd->vdev_ops != &vdev_mirror_ops && 8533 pvd->vdev_ops != &vdev_root_ops && 8534 !raidz) 8535 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8536 8537 pvops = &vdev_mirror_ops; 8538 } else { 8539 /* 8540 * Active hot spares can only be replaced by inactive hot 8541 * spares. 8542 */ 8543 if (pvd->vdev_ops == &vdev_spare_ops && 8544 oldvd->vdev_isspare && 8545 !spa_has_spare(spa, newvd->vdev_guid)) 8546 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8547 8548 /* 8549 * If the source is a hot spare, and the parent isn't already a 8550 * spare, then we want to create a new hot spare. Otherwise, we 8551 * want to create a replacing vdev. The user is not allowed to 8552 * attach to a spared vdev child unless the 'isspare' state is 8553 * the same (spare replaces spare, non-spare replaces 8554 * non-spare). 8555 */ 8556 if (pvd->vdev_ops == &vdev_replacing_ops && 8557 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 8558 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8559 } else if (pvd->vdev_ops == &vdev_spare_ops && 8560 newvd->vdev_isspare != oldvd->vdev_isspare) { 8561 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8562 } 8563 8564 if (spa_vdev_new_spare_would_cause_double_spares(newvd, pvd)) { 8565 vdev_dbgmsg(newvd, 8566 "disk would create double spares, ignore."); 8567 return (spa_vdev_exit(spa, newrootvd, txg, EEXIST)); 8568 } 8569 8570 if (newvd->vdev_isspare) 8571 pvops = &vdev_spare_ops; 8572 else 8573 pvops = &vdev_replacing_ops; 8574 } 8575 8576 /* 8577 * Make sure the new device is big enough. 8578 */ 8579 vdev_t *min_vdev = raidz ? oldvd->vdev_child[0] : oldvd; 8580 if (newvd->vdev_asize < vdev_get_min_asize(min_vdev)) 8581 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 8582 8583 /* 8584 * The new device cannot have a higher alignment requirement 8585 * than the top-level vdev. 8586 */ 8587 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) { 8588 return (spa_vdev_exit(spa, newrootvd, txg, 8589 ZFS_ERR_ASHIFT_MISMATCH)); 8590 } 8591 8592 /* 8593 * RAIDZ-expansion-specific checks. 8594 */ 8595 if (raidz) { 8596 if (vdev_raidz_attach_check(newvd) != 0) 8597 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 8598 8599 /* 8600 * Fail early if a child is not healthy or being replaced 8601 */ 8602 for (int i = 0; i < oldvd->vdev_children; i++) { 8603 if (vdev_is_dead(oldvd->vdev_child[i]) || 8604 !oldvd->vdev_child[i]->vdev_ops->vdev_op_leaf) { 8605 return (spa_vdev_exit(spa, newrootvd, txg, 8606 ENXIO)); 8607 } 8608 /* Also fail if reserved boot area is in-use */ 8609 if (vdev_check_boot_reserve(spa, oldvd->vdev_child[i]) 8610 != 0) { 8611 return (spa_vdev_exit(spa, newrootvd, txg, 8612 EADDRINUSE)); 8613 } 8614 } 8615 } 8616 8617 if (raidz) { 8618 /* 8619 * Note: oldvdpath is freed by spa_strfree(), but 8620 * kmem_asprintf() is freed by kmem_strfree(), so we have to 8621 * move it to a spa_strdup-ed string. 8622 */ 8623 char *tmp = kmem_asprintf("raidz%u-%u", 8624 (uint_t)vdev_get_nparity(oldvd), (uint_t)oldvd->vdev_id); 8625 oldvdpath = spa_strdup(tmp); 8626 kmem_strfree(tmp); 8627 } else { 8628 oldvdpath = spa_strdup(oldvd->vdev_path); 8629 } 8630 newvdpath = spa_strdup(newvd->vdev_path); 8631 8632 /* 8633 * If this is an in-place replacement, update oldvd's path and devid 8634 * to make it distinguishable from newvd, and unopenable from now on. 8635 */ 8636 if (strcmp(oldvdpath, newvdpath) == 0) { 8637 spa_strfree(oldvd->vdev_path); 8638 oldvd->vdev_path = kmem_alloc(strlen(newvdpath) + 5, 8639 KM_SLEEP); 8640 (void) sprintf(oldvd->vdev_path, "%s/old", 8641 newvdpath); 8642 if (oldvd->vdev_devid != NULL) { 8643 spa_strfree(oldvd->vdev_devid); 8644 oldvd->vdev_devid = NULL; 8645 } 8646 spa_strfree(oldvdpath); 8647 oldvdpath = spa_strdup(oldvd->vdev_path); 8648 } 8649 8650 /* 8651 * If the parent is not a mirror, or if we're replacing, insert the new 8652 * mirror/replacing/spare vdev above oldvd. 8653 */ 8654 if (!raidz && pvd->vdev_ops != pvops) { 8655 pvd = vdev_add_parent(oldvd, pvops); 8656 ASSERT(pvd->vdev_ops == pvops); 8657 ASSERT(oldvd->vdev_parent == pvd); 8658 } 8659 8660 ASSERT(pvd->vdev_top->vdev_parent == rvd); 8661 8662 /* 8663 * Extract the new device from its root and add it to pvd. 8664 */ 8665 vdev_remove_child(newrootvd, newvd); 8666 newvd->vdev_id = pvd->vdev_children; 8667 newvd->vdev_crtxg = oldvd->vdev_crtxg; 8668 vdev_add_child(pvd, newvd); 8669 8670 /* 8671 * Reevaluate the parent vdev state. 8672 */ 8673 vdev_propagate_state(pvd); 8674 8675 tvd = newvd->vdev_top; 8676 ASSERT(pvd->vdev_top == tvd); 8677 ASSERT(tvd->vdev_parent == rvd); 8678 8679 vdev_config_dirty(tvd); 8680 8681 /* 8682 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 8683 * for any dmu_sync-ed blocks. It will propagate upward when 8684 * spa_vdev_exit() calls vdev_dtl_reassess(). 8685 */ 8686 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 8687 8688 if (raidz) { 8689 dmu_tx_t *tx = dmu_tx_create_assigned(spa->spa_dsl_pool, 8690 txg); 8691 dsl_sync_task_nowait(spa->spa_dsl_pool, vdev_raidz_attach_sync, 8692 newvd, tx); 8693 dmu_tx_commit(tx); 8694 8695 /* 8696 * Wait for the youngest allocations and frees to sync, 8697 * and then wait for the deferral of those frees to finish. 8698 */ 8699 spa_vdev_config_exit(spa, NULL, 8700 txg + TXG_CONCURRENT_STATES + TXG_DEFER_SIZE, 0, FTAG); 8701 8702 vdev_initialize_stop_all(tvd, VDEV_INITIALIZE_ACTIVE); 8703 vdev_trim_stop_all(tvd, VDEV_TRIM_ACTIVE); 8704 vdev_autotrim_stop_wait(tvd); 8705 8706 dtl_max_txg = spa_vdev_config_enter(spa); 8707 8708 tvd->vdev_rz_expanding = B_TRUE; 8709 8710 vdev_dirty_leaves(tvd, VDD_DTL, dtl_max_txg); 8711 vdev_config_dirty(tvd); 8712 zthr_wakeup(spa->spa_raidz_expand_zthr); 8713 } else { 8714 vdev_dtl_dirty(newvd, DTL_MISSING, TXG_INITIAL, 8715 dtl_max_txg - TXG_INITIAL); 8716 8717 if (newvd->vdev_isspare) { 8718 spa_spare_activate(newvd); 8719 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 8720 } 8721 8722 newvd_isspare = newvd->vdev_isspare; 8723 8724 /* 8725 * Mark newvd's DTL dirty in this txg. 8726 */ 8727 vdev_dirty(tvd, VDD_DTL, newvd, txg); 8728 8729 /* 8730 * Schedule the resilver or rebuild to restart in the future. 8731 * We do this to ensure that dmu_sync-ed blocks have been 8732 * stitched into the respective datasets. 8733 */ 8734 if (rebuild) { 8735 newvd->vdev_rebuild_txg = txg; 8736 8737 vdev_rebuild(tvd, txg); 8738 } else { 8739 newvd->vdev_resilver_txg = txg; 8740 8741 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 8742 spa_feature_is_enabled(spa, 8743 SPA_FEATURE_RESILVER_DEFER)) { 8744 vdev_defer_resilver(newvd); 8745 } else { 8746 dsl_scan_restart_resilver(spa->spa_dsl_pool, 8747 dtl_max_txg); 8748 } 8749 } 8750 } 8751 8752 if (spa->spa_bootfs) 8753 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 8754 8755 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 8756 8757 /* 8758 * Commit the config 8759 */ 8760 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 8761 8762 spa_history_log_internal(spa, "vdev attach", NULL, 8763 "%s vdev=%s %s vdev=%s", 8764 replacing && newvd_isspare ? "spare in" : 8765 replacing ? "replace" : "attach", newvdpath, 8766 replacing ? "for" : "to", oldvdpath); 8767 8768 spa_strfree(oldvdpath); 8769 spa_strfree(newvdpath); 8770 8771 return (0); 8772 } 8773 8774 /* 8775 * Detach a device from a mirror or replacing vdev. 8776 * 8777 * If 'replace_done' is specified, only detach if the parent 8778 * is a replacing or a spare vdev. 8779 */ 8780 int 8781 spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 8782 { 8783 uint64_t txg; 8784 int error; 8785 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 8786 vdev_t *vd, *pvd, *cvd, *tvd; 8787 boolean_t unspare = B_FALSE; 8788 uint64_t unspare_guid = 0; 8789 char *vdpath; 8790 8791 ASSERT(spa_writeable(spa)); 8792 8793 txg = spa_vdev_detach_enter(spa, guid); 8794 8795 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 8796 8797 /* 8798 * Besides being called directly from the userland through the 8799 * ioctl interface, spa_vdev_detach() can be potentially called 8800 * at the end of spa_vdev_resilver_done(). 8801 * 8802 * In the regular case, when we have a checkpoint this shouldn't 8803 * happen as we never empty the DTLs of a vdev during the scrub 8804 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 8805 * should never get here when we have a checkpoint. 8806 * 8807 * That said, even in a case when we checkpoint the pool exactly 8808 * as spa_vdev_resilver_done() calls this function everything 8809 * should be fine as the resilver will return right away. 8810 */ 8811 ASSERT(spa_namespace_held()); 8812 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 8813 error = (spa_has_checkpoint(spa)) ? 8814 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 8815 return (spa_vdev_exit(spa, NULL, txg, error)); 8816 } 8817 8818 if (vd == NULL) 8819 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 8820 8821 if (!vd->vdev_ops->vdev_op_leaf) 8822 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 8823 8824 pvd = vd->vdev_parent; 8825 8826 /* 8827 * If the parent/child relationship is not as expected, don't do it. 8828 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 8829 * vdev that's replacing B with C. The user's intent in replacing 8830 * is to go from M(A,B) to M(A,C). If the user decides to cancel 8831 * the replace by detaching C, the expected behavior is to end up 8832 * M(A,B). But suppose that right after deciding to detach C, 8833 * the replacement of B completes. We would have M(A,C), and then 8834 * ask to detach C, which would leave us with just A -- not what 8835 * the user wanted. To prevent this, we make sure that the 8836 * parent/child relationship hasn't changed -- in this example, 8837 * that C's parent is still the replacing vdev R. 8838 */ 8839 if (pvd->vdev_guid != pguid && pguid != 0) 8840 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 8841 8842 /* 8843 * Only 'replacing' or 'spare' vdevs can be replaced. 8844 */ 8845 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 8846 pvd->vdev_ops != &vdev_spare_ops) 8847 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 8848 8849 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 8850 spa_version(spa) >= SPA_VERSION_SPARES); 8851 8852 /* 8853 * Only mirror, replacing, and spare vdevs support detach. 8854 */ 8855 if (pvd->vdev_ops != &vdev_replacing_ops && 8856 pvd->vdev_ops != &vdev_mirror_ops && 8857 pvd->vdev_ops != &vdev_spare_ops) 8858 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 8859 8860 /* 8861 * If this device has the only valid copy of some data, 8862 * we cannot safely detach it. 8863 */ 8864 if (vdev_dtl_required(vd)) 8865 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 8866 8867 ASSERT(pvd->vdev_children >= 2); 8868 8869 /* 8870 * If we are detaching the second disk from a replacing vdev, then 8871 * check to see if we changed the original vdev's path to have "/old" 8872 * at the end in spa_vdev_attach(). If so, undo that change now. 8873 */ 8874 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 8875 vd->vdev_path != NULL) { 8876 size_t len = strlen(vd->vdev_path); 8877 8878 for (int c = 0; c < pvd->vdev_children; c++) { 8879 cvd = pvd->vdev_child[c]; 8880 8881 if (cvd == vd || cvd->vdev_path == NULL) 8882 continue; 8883 8884 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 8885 strcmp(cvd->vdev_path + len, "/old") == 0) { 8886 spa_strfree(cvd->vdev_path); 8887 cvd->vdev_path = spa_strdup(vd->vdev_path); 8888 break; 8889 } 8890 } 8891 } 8892 8893 /* 8894 * If we are detaching the original disk from a normal spare, then it 8895 * implies that the spare should become a real disk, and be removed 8896 * from the active spare list for the pool. dRAID spares on the 8897 * other hand are coupled to the pool and thus should never be removed 8898 * from the spares list. 8899 */ 8900 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 8901 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 8902 8903 if (last_cvd->vdev_isspare && 8904 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 8905 unspare = B_TRUE; 8906 } 8907 } 8908 8909 /* 8910 * Erase the disk labels so the disk can be used for other things. 8911 * This must be done after all other error cases are handled, 8912 * but before we disembowel vd (so we can still do I/O to it). 8913 * But if we can't do it, don't treat the error as fatal -- 8914 * it may be that the unwritability of the disk is the reason 8915 * it's being detached! 8916 */ 8917 (void) vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 8918 8919 /* 8920 * Remove vd from its parent and compact the parent's children. 8921 */ 8922 vdev_remove_child(pvd, vd); 8923 vdev_compact_children(pvd); 8924 8925 /* 8926 * Remember one of the remaining children so we can get tvd below. 8927 */ 8928 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 8929 8930 /* 8931 * If we need to remove the remaining child from the list of hot spares, 8932 * do it now, marking the vdev as no longer a spare in the process. 8933 * We must do this before vdev_remove_parent(), because that can 8934 * change the GUID if it creates a new toplevel GUID. For a similar 8935 * reason, we must remove the spare now, in the same txg as the detach; 8936 * otherwise someone could attach a new sibling, change the GUID, and 8937 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 8938 */ 8939 if (unspare) { 8940 ASSERT(cvd->vdev_isspare); 8941 spa_spare_remove(cvd); 8942 unspare_guid = cvd->vdev_guid; 8943 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 8944 cvd->vdev_unspare = B_TRUE; 8945 } 8946 8947 /* 8948 * If the parent mirror/replacing vdev only has one child, 8949 * the parent is no longer needed. Remove it from the tree. 8950 */ 8951 if (pvd->vdev_children == 1) { 8952 if (pvd->vdev_ops == &vdev_spare_ops) 8953 cvd->vdev_unspare = B_FALSE; 8954 vdev_remove_parent(cvd); 8955 } 8956 8957 /* 8958 * We don't set tvd until now because the parent we just removed 8959 * may have been the previous top-level vdev. 8960 */ 8961 tvd = cvd->vdev_top; 8962 ASSERT(tvd->vdev_parent == rvd); 8963 8964 /* 8965 * Reevaluate the parent vdev state. 8966 */ 8967 vdev_propagate_state(cvd); 8968 8969 /* 8970 * If the 'autoexpand' property is set on the pool then automatically 8971 * try to expand the size of the pool. For example if the device we 8972 * just detached was smaller than the others, it may be possible to 8973 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 8974 * first so that we can obtain the updated sizes of the leaf vdevs. 8975 */ 8976 if (spa->spa_autoexpand) { 8977 vdev_reopen(tvd); 8978 vdev_expand(tvd, txg); 8979 } 8980 8981 vdev_config_dirty(tvd); 8982 8983 /* 8984 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 8985 * vd->vdev_detached is set and free vd's DTL object in syncing context. 8986 * But first make sure we're not on any *other* txg's DTL list, to 8987 * prevent vd from being accessed after it's freed. 8988 */ 8989 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 8990 for (int t = 0; t < TXG_SIZE; t++) 8991 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 8992 vd->vdev_detached = B_TRUE; 8993 vdev_dirty(tvd, VDD_DTL, vd, txg); 8994 8995 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 8996 spa_notify_waiters(spa); 8997 8998 /* hang on to the spa before we release the lock */ 8999 spa_open_ref(spa, FTAG); 9000 9001 error = spa_vdev_exit(spa, vd, txg, 0); 9002 9003 spa_history_log_internal(spa, "detach", NULL, 9004 "vdev=%s", vdpath); 9005 spa_strfree(vdpath); 9006 9007 /* 9008 * If this was the removal of the original device in a hot spare vdev, 9009 * then we want to go through and remove the device from the hot spare 9010 * list of every other pool. 9011 */ 9012 if (unspare) { 9013 spa_t *altspa = NULL; 9014 9015 spa_namespace_enter(FTAG); 9016 while ((altspa = spa_next(altspa)) != NULL) { 9017 if (altspa->spa_state != POOL_STATE_ACTIVE || 9018 altspa == spa) 9019 continue; 9020 9021 spa_open_ref(altspa, FTAG); 9022 spa_namespace_exit(FTAG); 9023 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 9024 spa_namespace_enter(FTAG); 9025 spa_close(altspa, FTAG); 9026 } 9027 spa_namespace_exit(FTAG); 9028 9029 /* search the rest of the vdevs for spares to remove */ 9030 spa_vdev_resilver_done(spa); 9031 } 9032 9033 /* all done with the spa; OK to release */ 9034 spa_namespace_enter(FTAG); 9035 spa_close(spa, FTAG); 9036 spa_namespace_exit(FTAG); 9037 9038 return (error); 9039 } 9040 9041 static int 9042 spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 9043 uint64_t value, boolean_t value_provided, list_t *vd_list) 9044 { 9045 ASSERT(spa_namespace_held()); 9046 9047 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9048 9049 /* Look up vdev and ensure it's a leaf. */ 9050 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 9051 if (vd == NULL || vd->vdev_detached) { 9052 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9053 return (SET_ERROR(ENODEV)); 9054 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 9055 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9056 return (SET_ERROR(EINVAL)); 9057 } else if (!vdev_writeable(vd)) { 9058 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9059 return (SET_ERROR(EROFS)); 9060 } 9061 mutex_enter(&vd->vdev_initialize_lock); 9062 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9063 9064 /* 9065 * When we activate an initialize action we check to see 9066 * if the vdev_initialize_thread is NULL. We do this instead 9067 * of using the vdev_initialize_state since there might be 9068 * a previous initialization process which has completed but 9069 * the thread is not exited. 9070 */ 9071 if (cmd_type == POOL_INITIALIZE_START && 9072 (vd->vdev_initialize_thread != NULL || 9073 vd->vdev_top->vdev_removing || vd->vdev_top->vdev_rz_expanding)) { 9074 mutex_exit(&vd->vdev_initialize_lock); 9075 return (SET_ERROR(EBUSY)); 9076 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 9077 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 9078 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 9079 mutex_exit(&vd->vdev_initialize_lock); 9080 return (SET_ERROR(ESRCH)); 9081 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 9082 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 9083 mutex_exit(&vd->vdev_initialize_lock); 9084 return (SET_ERROR(ESRCH)); 9085 } else if (cmd_type == POOL_INITIALIZE_UNINIT && 9086 vd->vdev_initialize_thread != NULL) { 9087 mutex_exit(&vd->vdev_initialize_lock); 9088 return (SET_ERROR(EBUSY)); 9089 } 9090 9091 switch (cmd_type) { 9092 case POOL_INITIALIZE_START: 9093 vdev_initialize(vd, value, value_provided); 9094 break; 9095 case POOL_INITIALIZE_CANCEL: 9096 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 9097 break; 9098 case POOL_INITIALIZE_SUSPEND: 9099 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 9100 break; 9101 case POOL_INITIALIZE_UNINIT: 9102 vdev_uninitialize(vd); 9103 break; 9104 default: 9105 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 9106 } 9107 mutex_exit(&vd->vdev_initialize_lock); 9108 9109 return (0); 9110 } 9111 9112 int 9113 spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 9114 uint64_t value, boolean_t value_provided, nvlist_t *vdev_errlist) 9115 { 9116 int total_errors = 0; 9117 list_t vd_list; 9118 9119 list_create(&vd_list, sizeof (vdev_t), 9120 offsetof(vdev_t, vdev_initialize_node)); 9121 9122 /* 9123 * We hold the namespace lock through the whole function 9124 * to prevent any changes to the pool while we're starting or 9125 * stopping initialization. The config and state locks are held so that 9126 * we can properly assess the vdev state before we commit to 9127 * the initializing operation. 9128 */ 9129 spa_namespace_enter(FTAG); 9130 9131 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 9132 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 9133 uint64_t vdev_guid = fnvpair_value_uint64(pair); 9134 9135 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 9136 value, value_provided, &vd_list); 9137 if (error != 0) { 9138 char guid_as_str[MAXNAMELEN]; 9139 9140 (void) snprintf(guid_as_str, sizeof (guid_as_str), 9141 "%llu", (unsigned long long)vdev_guid); 9142 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 9143 total_errors++; 9144 } 9145 } 9146 9147 /* Wait for all initialize threads to stop. */ 9148 vdev_initialize_stop_wait(spa, &vd_list); 9149 9150 /* Sync out the initializing state */ 9151 txg_wait_synced(spa->spa_dsl_pool, 0); 9152 spa_namespace_exit(FTAG); 9153 9154 list_destroy(&vd_list); 9155 9156 return (total_errors); 9157 } 9158 9159 static int 9160 spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 9161 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 9162 { 9163 ASSERT(spa_namespace_held()); 9164 9165 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9166 9167 /* Look up vdev and ensure it's a leaf. */ 9168 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 9169 if (vd == NULL || vd->vdev_detached) { 9170 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9171 return (SET_ERROR(ENODEV)); 9172 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 9173 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9174 return (SET_ERROR(EINVAL)); 9175 } else if (!vdev_writeable(vd)) { 9176 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9177 return (SET_ERROR(EROFS)); 9178 } else if (!vd->vdev_has_trim) { 9179 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9180 return (SET_ERROR(EOPNOTSUPP)); 9181 } else if (secure && !vd->vdev_has_securetrim) { 9182 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9183 return (SET_ERROR(EOPNOTSUPP)); 9184 } 9185 mutex_enter(&vd->vdev_trim_lock); 9186 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9187 9188 /* 9189 * When we activate a TRIM action we check to see if the 9190 * vdev_trim_thread is NULL. We do this instead of using the 9191 * vdev_trim_state since there might be a previous TRIM process 9192 * which has completed but the thread is not exited. 9193 */ 9194 if (cmd_type == POOL_TRIM_START && 9195 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing || 9196 vd->vdev_top->vdev_rz_expanding)) { 9197 mutex_exit(&vd->vdev_trim_lock); 9198 return (SET_ERROR(EBUSY)); 9199 } else if (cmd_type == POOL_TRIM_CANCEL && 9200 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 9201 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 9202 mutex_exit(&vd->vdev_trim_lock); 9203 return (SET_ERROR(ESRCH)); 9204 } else if (cmd_type == POOL_TRIM_SUSPEND && 9205 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 9206 mutex_exit(&vd->vdev_trim_lock); 9207 return (SET_ERROR(ESRCH)); 9208 } 9209 9210 switch (cmd_type) { 9211 case POOL_TRIM_START: 9212 vdev_trim(vd, rate, partial, secure); 9213 break; 9214 case POOL_TRIM_CANCEL: 9215 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 9216 break; 9217 case POOL_TRIM_SUSPEND: 9218 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 9219 break; 9220 default: 9221 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 9222 } 9223 mutex_exit(&vd->vdev_trim_lock); 9224 9225 return (0); 9226 } 9227 9228 /* 9229 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 9230 * TRIM threads for each child vdev. These threads pass over all of the free 9231 * space in the vdev's metaslabs and issues TRIM commands for that space. 9232 */ 9233 int 9234 spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 9235 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 9236 { 9237 int total_errors = 0; 9238 list_t vd_list; 9239 9240 list_create(&vd_list, sizeof (vdev_t), 9241 offsetof(vdev_t, vdev_trim_node)); 9242 9243 /* 9244 * We hold the namespace lock through the whole function 9245 * to prevent any changes to the pool while we're starting or 9246 * stopping TRIM. The config and state locks are held so that 9247 * we can properly assess the vdev state before we commit to 9248 * the TRIM operation. 9249 */ 9250 spa_namespace_enter(FTAG); 9251 9252 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 9253 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 9254 uint64_t vdev_guid = fnvpair_value_uint64(pair); 9255 9256 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 9257 rate, partial, secure, &vd_list); 9258 if (error != 0) { 9259 char guid_as_str[MAXNAMELEN]; 9260 9261 (void) snprintf(guid_as_str, sizeof (guid_as_str), 9262 "%llu", (unsigned long long)vdev_guid); 9263 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 9264 total_errors++; 9265 } 9266 } 9267 9268 /* Wait for all TRIM threads to stop. */ 9269 vdev_trim_stop_wait(spa, &vd_list); 9270 9271 /* Sync out the TRIM state */ 9272 txg_wait_synced(spa->spa_dsl_pool, 0); 9273 spa_namespace_exit(FTAG); 9274 9275 list_destroy(&vd_list); 9276 9277 return (total_errors); 9278 } 9279 9280 typedef struct spa_split_dtl_arg { 9281 spa_t *ssda_spa; /* the new pool */ 9282 uint64_t *ssda_objs; /* original DTL space map objects */ 9283 uint_t ssda_count; /* nitems in ssda_objs */ 9284 } spa_split_dtl_arg_t; 9285 9286 /* 9287 * Record the DTL space map object of every leaf that has one into objs[], 9288 * advancing *idxp. These are the objects that will be carried, via the 9289 * copied MOS, onto the split disks. 9290 */ 9291 static void 9292 spa_split_collect_dtl(vdev_t *vd, uint64_t *objs, uint_t *idxp) 9293 { 9294 if (vd->vdev_ops->vdev_op_leaf) { 9295 if (vd->vdev_dtl_sm != NULL) 9296 objs[(*idxp)++] = space_map_object(vd->vdev_dtl_sm); 9297 return; 9298 } 9299 for (uint64_t c = 0; c < vd->vdev_children; c++) 9300 spa_split_collect_dtl(vd->vdev_child[c], objs, idxp); 9301 } 9302 9303 /* 9304 * Callback that frees the inherited DTL space map objects from the new 9305 * pool MOS. The new pool MOS is a byte copy of the original pool, so it 9306 * contains a DTL space map object for every leaf of the original pool. 9307 * The new pool references none of them because split leaves 9308 * start with an empty DTL and allocate their own on demand. 9309 */ 9310 static void 9311 spa_split_dtl_free_sync(void *arg, dmu_tx_t *tx) 9312 { 9313 spa_split_dtl_arg_t *ssda = arg; 9314 objset_t *mos = ssda->ssda_spa->spa_meta_objset; 9315 9316 for (uint_t i = 0; i < ssda->ssda_count; i++) 9317 space_map_free_obj(mos, ssda->ssda_objs[i], tx); 9318 } 9319 9320 /* 9321 * Split a set of devices from their mirrors, and create a new pool from them. 9322 */ 9323 int 9324 spa_vdev_split_mirror(spa_t *spa, const char *newname, nvlist_t *config, 9325 nvlist_t *props, boolean_t exp) 9326 { 9327 int error = 0; 9328 uint64_t txg, *glist; 9329 spa_t *newspa; 9330 uint_t c, children, lastlog; 9331 nvlist_t **child, *nvl, *tmp; 9332 dmu_tx_t *tx; 9333 const char *altroot = NULL; 9334 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 9335 uint64_t *dtl_objs = NULL; /* DTL objs from original pool */ 9336 uint_t ndtl = 0, nleaves; 9337 boolean_t activate_slog; 9338 9339 ASSERT(spa_writeable(spa)); 9340 9341 txg = spa_vdev_enter(spa); 9342 9343 ASSERT(spa_namespace_held()); 9344 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 9345 error = (spa_has_checkpoint(spa)) ? 9346 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 9347 return (spa_vdev_exit(spa, NULL, txg, error)); 9348 } 9349 9350 /* clear the log and flush everything up to now */ 9351 activate_slog = spa_passivate_log(spa); 9352 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 9353 error = spa_reset_logs(spa); 9354 txg = spa_vdev_config_enter(spa); 9355 9356 if (activate_slog) 9357 spa_activate_log(spa); 9358 9359 if (error != 0) 9360 return (spa_vdev_exit(spa, NULL, txg, error)); 9361 9362 /* check new spa name before going any further */ 9363 if (spa_lookup(newname) != NULL) 9364 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 9365 9366 /* 9367 * scan through all the children to ensure they're all mirrors 9368 */ 9369 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 9370 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 9371 &children) != 0) 9372 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 9373 9374 /* first, check to ensure we've got the right child count */ 9375 rvd = spa->spa_root_vdev; 9376 lastlog = 0; 9377 for (c = 0; c < rvd->vdev_children; c++) { 9378 vdev_t *vd = rvd->vdev_child[c]; 9379 9380 /* don't count the holes & logs as children */ 9381 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 9382 !vdev_is_concrete(vd))) { 9383 if (lastlog == 0) 9384 lastlog = c; 9385 continue; 9386 } 9387 9388 lastlog = 0; 9389 } 9390 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 9391 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 9392 9393 /* next, ensure no spare or cache devices are part of the split */ 9394 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 9395 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 9396 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 9397 9398 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 9399 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 9400 9401 /* then, loop over each vdev and validate it */ 9402 for (c = 0; c < children; c++) { 9403 uint64_t is_hole = 0; 9404 9405 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 9406 &is_hole); 9407 9408 if (is_hole != 0) { 9409 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 9410 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 9411 continue; 9412 } else { 9413 error = SET_ERROR(EINVAL); 9414 break; 9415 } 9416 } 9417 9418 /* deal with indirect vdevs */ 9419 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 9420 &vdev_indirect_ops) 9421 continue; 9422 9423 /* which disk is going to be split? */ 9424 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 9425 &glist[c]) != 0) { 9426 error = SET_ERROR(EINVAL); 9427 break; 9428 } 9429 9430 /* look it up in the spa */ 9431 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 9432 if (vml[c] == NULL) { 9433 error = SET_ERROR(ENODEV); 9434 break; 9435 } 9436 9437 /* make sure there's nothing stopping the split */ 9438 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 9439 vml[c]->vdev_islog || 9440 !vdev_is_concrete(vml[c]) || 9441 vml[c]->vdev_isspare || 9442 vml[c]->vdev_isl2cache || 9443 !vdev_writeable(vml[c]) || 9444 vml[c]->vdev_children != 0 || 9445 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 9446 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 9447 error = SET_ERROR(EINVAL); 9448 break; 9449 } 9450 9451 if (vdev_dtl_required(vml[c]) || 9452 vdev_resilver_needed(vml[c], NULL, NULL)) { 9453 error = SET_ERROR(EBUSY); 9454 break; 9455 } 9456 9457 /* we need certain info from the top level */ 9458 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 9459 vml[c]->vdev_top->vdev_ms_array); 9460 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 9461 vml[c]->vdev_top->vdev_ms_shift); 9462 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 9463 vml[c]->vdev_top->vdev_asize); 9464 fnvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 9465 vml[c]->vdev_top->vdev_ashift); 9466 9467 /* transfer per-vdev ZAPs */ 9468 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 9469 VERIFY0(nvlist_add_uint64(child[c], 9470 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 9471 9472 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 9473 VERIFY0(nvlist_add_uint64(child[c], 9474 ZPOOL_CONFIG_VDEV_TOP_ZAP, 9475 vml[c]->vdev_parent->vdev_top_zap)); 9476 } 9477 9478 if (error != 0) { 9479 kmem_free(vml, children * sizeof (vdev_t *)); 9480 kmem_free(glist, children * sizeof (uint64_t)); 9481 return (spa_vdev_exit(spa, NULL, txg, error)); 9482 } 9483 9484 /* Create array of DTL objects. */ 9485 nleaves = vdev_count_leaves(spa); 9486 dtl_objs = kmem_zalloc(nleaves * sizeof (uint64_t), KM_SLEEP); 9487 spa_split_collect_dtl(spa->spa_root_vdev, dtl_objs, &ndtl); 9488 9489 /* stop writers from using the disks */ 9490 for (c = 0; c < children; c++) { 9491 if (vml[c] != NULL) 9492 vml[c]->vdev_offline = B_TRUE; 9493 } 9494 vdev_reopen(spa->spa_root_vdev); 9495 9496 /* 9497 * Temporarily record the splitting vdevs in the spa config. This 9498 * will disappear once the config is regenerated. 9499 */ 9500 nvl = fnvlist_alloc(); 9501 fnvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children); 9502 kmem_free(glist, children * sizeof (uint64_t)); 9503 9504 mutex_enter(&spa->spa_props_lock); 9505 fnvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, nvl); 9506 mutex_exit(&spa->spa_props_lock); 9507 spa->spa_config_splitting = nvl; 9508 vdev_config_dirty(spa->spa_root_vdev); 9509 9510 /* configure and create the new pool */ 9511 fnvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname); 9512 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 9513 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE); 9514 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)); 9515 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, spa->spa_config_txg); 9516 fnvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 9517 spa_generate_guid(NULL)); 9518 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 9519 (void) nvlist_lookup_string(props, 9520 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 9521 9522 /* add the new pool to the namespace */ 9523 newspa = spa_add(newname, config, altroot); 9524 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 9525 newspa->spa_config_txg = spa->spa_config_txg; 9526 spa_set_log_state(newspa, SPA_LOG_CLEAR); 9527 9528 /* release the spa config lock, retaining the namespace lock */ 9529 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 9530 9531 if (zio_injection_enabled) 9532 zio_handle_panic_injection(spa, FTAG, 1); 9533 9534 spa_activate(newspa, spa_mode_global); 9535 spa_async_suspend(newspa); 9536 9537 /* 9538 * Temporarily stop the initializing and TRIM activity. We set the 9539 * state to ACTIVE so that we know to resume initializing or TRIM 9540 * once the split has completed. 9541 */ 9542 list_t vd_initialize_list; 9543 list_create(&vd_initialize_list, sizeof (vdev_t), 9544 offsetof(vdev_t, vdev_initialize_node)); 9545 9546 list_t vd_trim_list; 9547 list_create(&vd_trim_list, sizeof (vdev_t), 9548 offsetof(vdev_t, vdev_trim_node)); 9549 9550 for (c = 0; c < children; c++) { 9551 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 9552 mutex_enter(&vml[c]->vdev_initialize_lock); 9553 vdev_initialize_stop(vml[c], 9554 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 9555 mutex_exit(&vml[c]->vdev_initialize_lock); 9556 9557 mutex_enter(&vml[c]->vdev_trim_lock); 9558 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 9559 mutex_exit(&vml[c]->vdev_trim_lock); 9560 } 9561 } 9562 9563 vdev_initialize_stop_wait(spa, &vd_initialize_list); 9564 vdev_trim_stop_wait(spa, &vd_trim_list); 9565 9566 list_destroy(&vd_initialize_list); 9567 list_destroy(&vd_trim_list); 9568 9569 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 9570 newspa->spa_is_splitting = B_TRUE; 9571 9572 /* create the new pool from the disks of the original pool */ 9573 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 9574 if (error) 9575 goto out; 9576 9577 /* if that worked, generate a real config for the new pool */ 9578 if (newspa->spa_root_vdev != NULL) { 9579 newspa->spa_config_splitting = fnvlist_alloc(); 9580 fnvlist_add_uint64(newspa->spa_config_splitting, 9581 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)); 9582 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 9583 B_TRUE)); 9584 } 9585 9586 /* 9587 * Free the DTL space map objects inherited from the original pool 9588 * MOS so we won't leak them. 9589 */ 9590 if (ndtl != 0) { 9591 spa_split_dtl_arg_t ssda; 9592 9593 ssda.ssda_spa = newspa; 9594 ssda.ssda_objs = dtl_objs; 9595 ssda.ssda_count = ndtl; 9596 VERIFY0(dsl_sync_task(spa_name(newspa), NULL, 9597 spa_split_dtl_free_sync, &ssda, 0, ZFS_SPACE_CHECK_NONE)); 9598 } 9599 9600 /* set the props */ 9601 if (props != NULL) { 9602 spa_configfile_set(newspa, props, B_FALSE); 9603 error = spa_prop_set(newspa, props); 9604 if (error) 9605 goto out; 9606 } 9607 9608 /* flush everything */ 9609 txg = spa_vdev_config_enter(newspa); 9610 vdev_config_dirty(newspa->spa_root_vdev); 9611 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 9612 9613 if (zio_injection_enabled) 9614 zio_handle_panic_injection(spa, FTAG, 2); 9615 9616 spa_async_resume(newspa); 9617 9618 /* finally, update the original pool's config */ 9619 txg = spa_vdev_config_enter(spa); 9620 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 9621 error = dmu_tx_assign(tx, DMU_TX_WAIT); 9622 if (error != 0) 9623 dmu_tx_abort(tx); 9624 for (c = 0; c < children; c++) { 9625 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 9626 vdev_t *tvd = vml[c]->vdev_top; 9627 9628 /* 9629 * Need to be sure the detachable VDEV is not 9630 * on any *other* txg's DTL list to prevent it 9631 * from being accessed after it's freed. 9632 */ 9633 for (int t = 0; t < TXG_SIZE; t++) { 9634 (void) txg_list_remove_this( 9635 &tvd->vdev_dtl_list, vml[c], t); 9636 } 9637 9638 vdev_split(vml[c]); 9639 9640 /* 9641 * As in spa_vdev_detach(), mark the vdev detached 9642 * and dirty its DTL, so that vdev_dtl_sync() frees 9643 * the leaf's DTL space map object. 9644 */ 9645 vml[c]->vdev_detached = B_TRUE; 9646 9647 /* 9648 * The leaf ZAP was transferred to the new pool 9649 * and this pool's copy is destroyed by the AVZ 9650 * rebuild below, so clear it to keep 9651 * vdev_dtl_sync() from destroying it again. 9652 */ 9653 vml[c]->vdev_leaf_zap = 0; 9654 9655 /* 9656 * vml[c]->vdev_top may be stale; the 9657 * surviving top-level vdev is rvd->vdev_child[c]. 9658 */ 9659 if (vml[c]->vdev_dtl_sm != NULL) 9660 vdev_dirty(rvd->vdev_child[c], VDD_DTL, 9661 vml[c], txg); 9662 9663 if (error == 0) 9664 spa_history_log_internal(spa, "detach", tx, 9665 "vdev=%s", vml[c]->vdev_path); 9666 } 9667 } 9668 spa->spa_avz_action = AVZ_ACTION_REBUILD; 9669 vdev_config_dirty(spa->spa_root_vdev); 9670 spa->spa_config_splitting = NULL; 9671 nvlist_free(nvl); 9672 if (error == 0) 9673 dmu_tx_commit(tx); 9674 (void) spa_vdev_exit(spa, NULL, txg, 0); 9675 9676 /* 9677 * txg is synced, free vdevs. 9678 */ 9679 spa_config_enter(spa, SCL_STATE_ALL, spa, RW_WRITER); 9680 for (c = 0; c < children; c++) { 9681 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 9682 ASSERT0P(vml[c]->vdev_dtl_sm); 9683 vdev_free(vml[c]); 9684 } 9685 } 9686 spa_config_exit(spa, SCL_STATE_ALL, spa); 9687 9688 if (zio_injection_enabled) 9689 zio_handle_panic_injection(spa, FTAG, 3); 9690 9691 /* split is complete; log a history record */ 9692 spa_history_log_internal(newspa, "split", NULL, 9693 "from pool %s", spa_name(spa)); 9694 9695 newspa->spa_is_splitting = B_FALSE; 9696 kmem_free(vml, children * sizeof (vdev_t *)); 9697 if (dtl_objs != NULL) 9698 kmem_free(dtl_objs, nleaves * sizeof (uint64_t)); 9699 9700 /* if we're not going to mount the filesystems in userland, export */ 9701 if (exp) 9702 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 9703 B_FALSE, B_FALSE); 9704 9705 return (error); 9706 9707 out: 9708 spa_unload(newspa); 9709 spa_deactivate(newspa); 9710 spa_remove(newspa); 9711 9712 txg = spa_vdev_config_enter(spa); 9713 9714 /* re-online all offlined disks */ 9715 for (c = 0; c < children; c++) { 9716 if (vml[c] != NULL) 9717 vml[c]->vdev_offline = B_FALSE; 9718 } 9719 9720 /* restart initializing or trimming disks as necessary */ 9721 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 9722 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 9723 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 9724 9725 vdev_reopen(spa->spa_root_vdev); 9726 9727 nvlist_free(spa->spa_config_splitting); 9728 spa->spa_config_splitting = NULL; 9729 (void) spa_vdev_exit(spa, NULL, txg, error); 9730 9731 kmem_free(vml, children * sizeof (vdev_t *)); 9732 if (dtl_objs != NULL) 9733 kmem_free(dtl_objs, nleaves * sizeof (uint64_t)); 9734 9735 return (error); 9736 } 9737 9738 /* 9739 * Find any device that's done replacing, or a vdev marked 'unspare' that's 9740 * currently spared, so we can detach it. 9741 */ 9742 static vdev_t * 9743 spa_vdev_resilver_done_hunt(vdev_t *vd) 9744 { 9745 vdev_t *newvd, *oldvd; 9746 9747 for (int c = 0; c < vd->vdev_children; c++) { 9748 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 9749 if (oldvd != NULL) 9750 return (oldvd); 9751 } 9752 9753 /* 9754 * Check for a completed replacement. We always consider the first 9755 * vdev in the list to be the oldest vdev, and the last one to be 9756 * the newest (see spa_vdev_attach() for how that works). In 9757 * the case where the newest vdev is faulted, we will not automatically 9758 * remove it after a resilver completes. This is OK as it will require 9759 * user intervention to determine which disk the admin wishes to keep. 9760 */ 9761 if (vd->vdev_ops == &vdev_replacing_ops) { 9762 ASSERT(vd->vdev_children > 1); 9763 9764 newvd = vd->vdev_child[vd->vdev_children - 1]; 9765 oldvd = vd->vdev_child[0]; 9766 9767 if (vdev_dtl_empty(newvd, DTL_MISSING) && 9768 vdev_dtl_empty(newvd, DTL_OUTAGE) && 9769 !vdev_dtl_required(oldvd)) 9770 return (oldvd); 9771 } 9772 9773 /* 9774 * Check for a completed resilver with the 'unspare' flag set. 9775 * Also potentially update faulted state. 9776 */ 9777 if (vd->vdev_ops == &vdev_spare_ops) { 9778 vdev_t *first = vd->vdev_child[0]; 9779 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 9780 9781 if (last->vdev_unspare) { 9782 oldvd = first; 9783 newvd = last; 9784 } else if (first->vdev_unspare) { 9785 oldvd = last; 9786 newvd = first; 9787 } else { 9788 oldvd = NULL; 9789 } 9790 9791 if (oldvd != NULL && 9792 vdev_dtl_empty(newvd, DTL_MISSING) && 9793 vdev_dtl_empty(newvd, DTL_OUTAGE) && 9794 !vdev_dtl_required(oldvd)) 9795 return (oldvd); 9796 9797 vdev_propagate_state(vd); 9798 9799 /* 9800 * If there are more than two spares attached to a disk, 9801 * and those spares are not required, then we want to 9802 * attempt to free them up now so that they can be used 9803 * by other pools. Once we're back down to a single 9804 * disk+spare, we stop removing them. 9805 */ 9806 if (vd->vdev_children > 2) { 9807 newvd = vd->vdev_child[1]; 9808 9809 if (newvd->vdev_isspare && last->vdev_isspare && 9810 vdev_dtl_empty(last, DTL_MISSING) && 9811 vdev_dtl_empty(last, DTL_OUTAGE) && 9812 !vdev_dtl_required(newvd)) 9813 return (newvd); 9814 } 9815 } 9816 9817 return (NULL); 9818 } 9819 9820 static void 9821 spa_vdev_resilver_done(spa_t *spa) 9822 { 9823 vdev_t *vd, *pvd, *ppvd; 9824 uint64_t guid, sguid, pguid, ppguid; 9825 9826 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9827 9828 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 9829 pvd = vd->vdev_parent; 9830 ppvd = pvd->vdev_parent; 9831 guid = vd->vdev_guid; 9832 pguid = pvd->vdev_guid; 9833 ppguid = ppvd->vdev_guid; 9834 sguid = 0; 9835 /* 9836 * If we have just finished replacing a hot spared device, then 9837 * we need to detach the parent's first child (the original hot 9838 * spare) as well. 9839 */ 9840 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 9841 ppvd->vdev_children == 2) { 9842 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 9843 sguid = ppvd->vdev_child[1]->vdev_guid; 9844 } 9845 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 9846 9847 spa_config_exit(spa, SCL_ALL, FTAG); 9848 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 9849 return; 9850 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 9851 return; 9852 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9853 } 9854 9855 spa_config_exit(spa, SCL_ALL, FTAG); 9856 9857 /* 9858 * If a detach was not performed above replace waiters will not have 9859 * been notified. In which case we must do so now. 9860 */ 9861 spa_notify_waiters(spa); 9862 } 9863 9864 /* 9865 * Update the stored path or FRU for this vdev. 9866 */ 9867 static int 9868 spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 9869 boolean_t ispath) 9870 { 9871 vdev_t *vd; 9872 boolean_t sync = B_FALSE; 9873 9874 ASSERT(spa_writeable(spa)); 9875 9876 spa_vdev_state_enter(spa, SCL_ALL); 9877 9878 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 9879 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 9880 9881 if (!vd->vdev_ops->vdev_op_leaf) 9882 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 9883 9884 if (ispath) { 9885 if (strcmp(value, vd->vdev_path) != 0) { 9886 spa_strfree(vd->vdev_path); 9887 vd->vdev_path = spa_strdup(value); 9888 sync = B_TRUE; 9889 } 9890 } else { 9891 if (vd->vdev_fru == NULL) { 9892 vd->vdev_fru = spa_strdup(value); 9893 sync = B_TRUE; 9894 } else if (strcmp(value, vd->vdev_fru) != 0) { 9895 spa_strfree(vd->vdev_fru); 9896 vd->vdev_fru = spa_strdup(value); 9897 sync = B_TRUE; 9898 } 9899 } 9900 9901 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 9902 } 9903 9904 int 9905 spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 9906 { 9907 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 9908 } 9909 9910 int 9911 spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 9912 { 9913 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 9914 } 9915 9916 /* 9917 * ========================================================================== 9918 * SPA Scanning 9919 * ========================================================================== 9920 */ 9921 int 9922 spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 9923 { 9924 ASSERT0(spa_config_held(spa, SCL_ALL, RW_WRITER)); 9925 9926 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 9927 return (SET_ERROR(EBUSY)); 9928 9929 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 9930 } 9931 9932 int 9933 spa_scan_stop(spa_t *spa) 9934 { 9935 ASSERT0(spa_config_held(spa, SCL_ALL, RW_WRITER)); 9936 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 9937 return (SET_ERROR(EBUSY)); 9938 9939 return (dsl_scan_cancel(spa->spa_dsl_pool)); 9940 } 9941 9942 int 9943 spa_scan(spa_t *spa, pool_scan_func_t func, pool_scrub_flags_t flags) 9944 { 9945 return (spa_scan_range(spa, func, 0, 0, flags)); 9946 } 9947 9948 int 9949 spa_scan_range(spa_t *spa, pool_scan_func_t func, uint64_t txgstart, 9950 uint64_t txgend, pool_scrub_flags_t flags) 9951 { 9952 dsl_scan_flags_t dsl_flags = 0; 9953 9954 ASSERT0(spa_config_held(spa, SCL_ALL, RW_WRITER)); 9955 9956 if (flags & POOL_SCRUB_THOROUGH) 9957 dsl_flags |= DSF_SCRUB_THOROUGH; 9958 9959 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 9960 return (SET_ERROR(ENOTSUP)); 9961 9962 if (func == POOL_SCAN_RESILVER && 9963 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 9964 return (SET_ERROR(ENOTSUP)); 9965 9966 if (func != POOL_SCAN_SCRUB && (txgstart != 0 || txgend != 0)) 9967 return (SET_ERROR(ENOTSUP)); 9968 9969 /* 9970 * If a resilver was requested, but there is no DTL on a 9971 * writeable leaf device, we have nothing to do. 9972 */ 9973 if (func == POOL_SCAN_RESILVER && 9974 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 9975 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 9976 return (0); 9977 } 9978 9979 if (func == POOL_SCAN_ERRORSCRUB && 9980 !spa_feature_is_enabled(spa, SPA_FEATURE_HEAD_ERRLOG)) 9981 return (SET_ERROR(ENOTSUP)); 9982 9983 return (dsl_scan(spa->spa_dsl_pool, func, txgstart, txgend, dsl_flags)); 9984 } 9985 9986 /* 9987 * ========================================================================== 9988 * SPA async task processing 9989 * ========================================================================== 9990 */ 9991 9992 static void 9993 spa_async_remove(spa_t *spa, vdev_t *vd, boolean_t by_kernel) 9994 { 9995 if (vd->vdev_remove_wanted) { 9996 vd->vdev_remove_wanted = B_FALSE; 9997 vd->vdev_delayed_close = B_FALSE; 9998 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 9999 10000 /* 10001 * We want to clear the stats, but we don't want to do a full 10002 * vdev_clear() as that will cause us to throw away 10003 * degraded/faulted state as well as attempt to reopen the 10004 * device, all of which is a waste. 10005 */ 10006 vd->vdev_stat.vs_read_errors = 0; 10007 vd->vdev_stat.vs_write_errors = 0; 10008 vd->vdev_stat.vs_checksum_errors = 0; 10009 10010 vdev_state_dirty(vd->vdev_top); 10011 10012 /* Tell userspace that the vdev is gone. */ 10013 zfs_post_remove(spa, vd, by_kernel); 10014 } 10015 10016 for (int c = 0; c < vd->vdev_children; c++) 10017 spa_async_remove(spa, vd->vdev_child[c], by_kernel); 10018 } 10019 10020 static void 10021 spa_async_fault_vdev(vdev_t *vd, boolean_t *suspend) 10022 { 10023 if (vd->vdev_fault_wanted) { 10024 vdev_state_t newstate = VDEV_STATE_FAULTED; 10025 vd->vdev_fault_wanted = B_FALSE; 10026 10027 /* 10028 * If this device has the only valid copy of the data, then 10029 * back off and simply mark the vdev as degraded instead. 10030 */ 10031 if (!vd->vdev_top->vdev_islog && vd->vdev_aux == NULL && 10032 vdev_dtl_required(vd)) { 10033 newstate = VDEV_STATE_DEGRADED; 10034 /* A required disk is missing so suspend the pool */ 10035 *suspend = B_TRUE; 10036 } 10037 vdev_set_state(vd, B_TRUE, newstate, VDEV_AUX_ERR_EXCEEDED); 10038 } 10039 for (int c = 0; c < vd->vdev_children; c++) 10040 spa_async_fault_vdev(vd->vdev_child[c], suspend); 10041 } 10042 10043 static void 10044 spa_async_autoexpand(spa_t *spa, vdev_t *vd) 10045 { 10046 if (!spa->spa_autoexpand) 10047 return; 10048 10049 for (int c = 0; c < vd->vdev_children; c++) { 10050 vdev_t *cvd = vd->vdev_child[c]; 10051 spa_async_autoexpand(spa, cvd); 10052 } 10053 10054 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 10055 return; 10056 10057 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 10058 } 10059 10060 static __attribute__((noreturn)) void 10061 spa_async_thread(void *arg) 10062 { 10063 spa_t *spa = (spa_t *)arg; 10064 dsl_pool_t *dp = spa->spa_dsl_pool; 10065 uint32_t tasks; 10066 10067 ASSERT(spa->spa_sync_on); 10068 10069 mutex_enter(&spa->spa_async_lock); 10070 tasks = spa->spa_async_tasks; 10071 spa->spa_async_tasks = 0; 10072 mutex_exit(&spa->spa_async_lock); 10073 10074 /* 10075 * See if the config needs to be updated. 10076 */ 10077 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 10078 uint64_t old_space, new_space; 10079 10080 spa_namespace_enter(FTAG); 10081 old_space = metaslab_class_get_space(spa_normal_class(spa)); 10082 old_space += metaslab_class_get_space(spa_special_class(spa)); 10083 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 10084 old_space += metaslab_class_get_space( 10085 spa_embedded_log_class(spa)); 10086 old_space += metaslab_class_get_space( 10087 spa_special_embedded_log_class(spa)); 10088 10089 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 10090 10091 new_space = metaslab_class_get_space(spa_normal_class(spa)); 10092 new_space += metaslab_class_get_space(spa_special_class(spa)); 10093 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 10094 new_space += metaslab_class_get_space( 10095 spa_embedded_log_class(spa)); 10096 new_space += metaslab_class_get_space( 10097 spa_special_embedded_log_class(spa)); 10098 spa_namespace_exit(FTAG); 10099 10100 /* 10101 * If the pool grew as a result of the config update, 10102 * then log an internal history event. 10103 */ 10104 if (new_space != old_space) { 10105 spa_history_log_internal(spa, "vdev online", NULL, 10106 "pool '%s' size: %llu(+%llu)", 10107 spa_name(spa), (u_longlong_t)new_space, 10108 (u_longlong_t)(new_space - old_space)); 10109 } 10110 } 10111 10112 /* 10113 * See if any devices need to be marked REMOVED. 10114 */ 10115 if (tasks & (SPA_ASYNC_REMOVE | SPA_ASYNC_REMOVE_BY_USER)) { 10116 boolean_t by_kernel = B_TRUE; 10117 if (tasks & SPA_ASYNC_REMOVE_BY_USER) 10118 by_kernel = B_FALSE; 10119 spa_vdev_state_enter(spa, SCL_NONE); 10120 spa_async_remove(spa, spa->spa_root_vdev, by_kernel); 10121 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 10122 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i], 10123 by_kernel); 10124 for (int i = 0; i < spa->spa_spares.sav_count; i++) 10125 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i], 10126 by_kernel); 10127 (void) spa_vdev_state_exit(spa, NULL, 0); 10128 } 10129 10130 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 10131 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 10132 spa_async_autoexpand(spa, spa->spa_root_vdev); 10133 spa_config_exit(spa, SCL_CONFIG, FTAG); 10134 } 10135 10136 /* 10137 * See if any devices need to be marked faulted. 10138 */ 10139 if (tasks & SPA_ASYNC_FAULT_VDEV) { 10140 spa_vdev_state_enter(spa, SCL_NONE); 10141 boolean_t suspend = B_FALSE; 10142 spa_async_fault_vdev(spa->spa_root_vdev, &suspend); 10143 (void) spa_vdev_state_exit(spa, NULL, 0); 10144 if (suspend) 10145 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 10146 } 10147 10148 /* 10149 * If any devices are done replacing, detach them. 10150 */ 10151 if (tasks & SPA_ASYNC_RESILVER_DONE || 10152 tasks & SPA_ASYNC_REBUILD_DONE || 10153 tasks & SPA_ASYNC_DETACH_SPARE) { 10154 spa_vdev_resilver_done(spa); 10155 } 10156 10157 /* 10158 * Kick off a resilver. 10159 */ 10160 if (tasks & SPA_ASYNC_RESILVER && 10161 !vdev_rebuild_active(spa->spa_root_vdev) && 10162 (!dsl_scan_resilvering(dp) || 10163 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 10164 dsl_scan_restart_resilver(dp, 0); 10165 10166 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 10167 spa_namespace_enter(FTAG); 10168 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 10169 vdev_initialize_restart(spa->spa_root_vdev); 10170 spa_config_exit(spa, SCL_CONFIG, FTAG); 10171 spa_namespace_exit(FTAG); 10172 } 10173 10174 if (tasks & SPA_ASYNC_TRIM_RESTART) { 10175 spa_namespace_enter(FTAG); 10176 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 10177 vdev_trim_restart(spa->spa_root_vdev); 10178 spa_config_exit(spa, SCL_CONFIG, FTAG); 10179 spa_namespace_exit(FTAG); 10180 } 10181 10182 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 10183 spa_namespace_enter(FTAG); 10184 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 10185 vdev_autotrim_restart(spa); 10186 spa_config_exit(spa, SCL_CONFIG, FTAG); 10187 spa_namespace_exit(FTAG); 10188 } 10189 10190 /* 10191 * Kick off L2 cache whole device TRIM. 10192 */ 10193 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 10194 spa_namespace_enter(FTAG); 10195 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 10196 vdev_trim_l2arc(spa); 10197 spa_config_exit(spa, SCL_CONFIG, FTAG); 10198 spa_namespace_exit(FTAG); 10199 } 10200 10201 /* 10202 * Kick off L2 cache rebuilding. 10203 */ 10204 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 10205 spa_namespace_enter(FTAG); 10206 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 10207 l2arc_spa_rebuild_start(spa); 10208 spa_config_exit(spa, SCL_L2ARC, FTAG); 10209 spa_namespace_exit(FTAG); 10210 } 10211 10212 /* 10213 * Finish off snapshots whose deferred destruction was waiting on 10214 * something that has since let go of them. The destroy is a sync 10215 * task, which a suspended pool would never come back from, and the 10216 * export path waits on this thread, so leave the mark where it is 10217 * and pick it up on the next request or at the next import. 10218 */ 10219 if ((tasks & SPA_ASYNC_DEFER_DESTROY) && spa_writeable(spa) && 10220 !spa_suspended(spa)) 10221 dsl_destroy_snapshot_deferred(spa_name(spa)); 10222 10223 /* 10224 * Let the world know that we're done. 10225 */ 10226 mutex_enter(&spa->spa_async_lock); 10227 spa->spa_async_thread = NULL; 10228 cv_broadcast(&spa->spa_async_cv); 10229 mutex_exit(&spa->spa_async_lock); 10230 thread_exit(); 10231 } 10232 10233 void 10234 spa_async_suspend(spa_t *spa) 10235 { 10236 mutex_enter(&spa->spa_async_lock); 10237 spa->spa_async_suspended++; 10238 while (spa->spa_async_thread != NULL) 10239 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 10240 mutex_exit(&spa->spa_async_lock); 10241 10242 spa_vdev_remove_suspend(spa); 10243 10244 zthr_t *condense_thread = spa->spa_condense_zthr; 10245 if (condense_thread != NULL) 10246 zthr_cancel(condense_thread); 10247 10248 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr; 10249 if (raidz_expand_thread != NULL) 10250 zthr_cancel(raidz_expand_thread); 10251 10252 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 10253 if (discard_thread != NULL) 10254 zthr_cancel(discard_thread); 10255 10256 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 10257 if (ll_delete_thread != NULL) 10258 zthr_cancel(ll_delete_thread); 10259 10260 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 10261 if (ll_condense_thread != NULL) 10262 zthr_cancel(ll_condense_thread); 10263 } 10264 10265 void 10266 spa_async_resume(spa_t *spa) 10267 { 10268 mutex_enter(&spa->spa_async_lock); 10269 ASSERT(spa->spa_async_suspended != 0); 10270 spa->spa_async_suspended--; 10271 mutex_exit(&spa->spa_async_lock); 10272 spa_restart_removal(spa); 10273 10274 zthr_t *condense_thread = spa->spa_condense_zthr; 10275 if (condense_thread != NULL) 10276 zthr_resume(condense_thread); 10277 10278 zthr_t *raidz_expand_thread = spa->spa_raidz_expand_zthr; 10279 if (raidz_expand_thread != NULL) 10280 zthr_resume(raidz_expand_thread); 10281 10282 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 10283 if (discard_thread != NULL) 10284 zthr_resume(discard_thread); 10285 10286 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 10287 if (ll_delete_thread != NULL) 10288 zthr_resume(ll_delete_thread); 10289 10290 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 10291 if (ll_condense_thread != NULL) 10292 zthr_resume(ll_condense_thread); 10293 } 10294 10295 static boolean_t 10296 spa_async_tasks_pending(spa_t *spa) 10297 { 10298 uint_t non_config_tasks; 10299 uint_t config_task; 10300 boolean_t config_task_suspended; 10301 10302 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 10303 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 10304 if (spa->spa_ccw_fail_time == 0) { 10305 config_task_suspended = B_FALSE; 10306 } else { 10307 config_task_suspended = 10308 (gethrtime() - spa->spa_ccw_fail_time) < 10309 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 10310 } 10311 10312 return (non_config_tasks || (config_task && !config_task_suspended)); 10313 } 10314 10315 static void 10316 spa_async_dispatch(spa_t *spa) 10317 { 10318 mutex_enter(&spa->spa_async_lock); 10319 if (spa_async_tasks_pending(spa) && 10320 !spa->spa_async_suspended && 10321 spa->spa_async_thread == NULL) 10322 spa->spa_async_thread = thread_create(NULL, 0, 10323 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 10324 mutex_exit(&spa->spa_async_lock); 10325 } 10326 10327 void 10328 spa_async_request(spa_t *spa, int task) 10329 { 10330 zfs_dbgmsg("spa=%s async request task=%u", spa_load_name(spa), task); 10331 mutex_enter(&spa->spa_async_lock); 10332 spa->spa_async_tasks |= task; 10333 mutex_exit(&spa->spa_async_lock); 10334 } 10335 10336 int 10337 spa_async_tasks(spa_t *spa) 10338 { 10339 return (spa->spa_async_tasks); 10340 } 10341 10342 /* 10343 * ========================================================================== 10344 * SPA syncing routines 10345 * ========================================================================== 10346 */ 10347 10348 10349 static int 10350 bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 10351 dmu_tx_t *tx) 10352 { 10353 bpobj_t *bpo = arg; 10354 bpobj_enqueue(bpo, bp, bp_freed, tx); 10355 return (0); 10356 } 10357 10358 int 10359 bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 10360 { 10361 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 10362 } 10363 10364 int 10365 bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 10366 { 10367 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 10368 } 10369 10370 static int 10371 spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 10372 { 10373 zio_t *pio = arg; 10374 10375 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 10376 pio->io_flags)); 10377 return (0); 10378 } 10379 10380 static int 10381 bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 10382 dmu_tx_t *tx) 10383 { 10384 ASSERT(!bp_freed); 10385 return (spa_free_sync_cb(arg, bp, tx)); 10386 } 10387 10388 /* 10389 * Note: this simple function is not inlined to make it easier to dtrace the 10390 * amount of time spent syncing frees. 10391 */ 10392 static void 10393 spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 10394 { 10395 zio_t *zio = zio_root(spa, NULL, NULL, 0); 10396 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 10397 VERIFY0(zio_wait(zio)); 10398 } 10399 10400 /* 10401 * Note: this simple function is not inlined to make it easier to dtrace the 10402 * amount of time spent syncing deferred frees. 10403 */ 10404 static void 10405 spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 10406 { 10407 if (spa_sync_pass(spa) != 1) 10408 return; 10409 10410 /* 10411 * Note: 10412 * If the log space map feature is active, we stop deferring 10413 * frees to the next TXG and therefore running this function 10414 * would be considered a no-op as spa_deferred_bpobj should 10415 * not have any entries. 10416 * 10417 * That said we run this function anyway (instead of returning 10418 * immediately) for the edge-case scenario where we just 10419 * activated the log space map feature in this TXG but we have 10420 * deferred frees from the previous TXG. 10421 */ 10422 zio_t *zio = zio_root(spa, NULL, NULL, 0); 10423 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 10424 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 10425 VERIFY0(zio_wait(zio)); 10426 } 10427 10428 static void 10429 spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 10430 { 10431 char *packed = NULL; 10432 size_t bufsize; 10433 size_t nvsize = 0; 10434 dmu_buf_t *db; 10435 10436 VERIFY0(nvlist_size(nv, &nvsize, NV_ENCODE_XDR)); 10437 10438 /* 10439 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 10440 * information. This avoids the dmu_buf_will_dirty() path and 10441 * saves us a pre-read to get data we don't actually care about. 10442 */ 10443 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 10444 packed = vmem_alloc(bufsize, KM_SLEEP); 10445 10446 VERIFY0(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 10447 KM_SLEEP)); 10448 memset(packed + nvsize, 0, bufsize - nvsize); 10449 10450 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx, 10451 DMU_READ_NO_PREFETCH); 10452 10453 vmem_free(packed, bufsize); 10454 10455 VERIFY0(dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 10456 dmu_buf_will_dirty(db, tx); 10457 *(uint64_t *)db->db_data = nvsize; 10458 dmu_buf_rele(db, FTAG); 10459 } 10460 10461 static void 10462 spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 10463 const char *config, const char *entry) 10464 { 10465 nvlist_t *nvroot; 10466 nvlist_t **list; 10467 int i; 10468 10469 if (!sav->sav_sync) 10470 return; 10471 10472 /* 10473 * Update the MOS nvlist describing the list of available devices. 10474 * spa_validate_aux() will have already made sure this nvlist is 10475 * valid and the vdevs are labeled appropriately. 10476 */ 10477 if (sav->sav_object == 0) { 10478 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 10479 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 10480 sizeof (uint64_t), tx); 10481 VERIFY(zap_update(spa->spa_meta_objset, 10482 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 10483 &sav->sav_object, tx) == 0); 10484 } 10485 10486 nvroot = fnvlist_alloc(); 10487 if (sav->sav_count == 0) { 10488 fnvlist_add_nvlist_array(nvroot, config, 10489 (const nvlist_t * const *)NULL, 0); 10490 } else { 10491 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 10492 for (i = 0; i < sav->sav_count; i++) 10493 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 10494 B_FALSE, VDEV_CONFIG_L2CACHE); 10495 fnvlist_add_nvlist_array(nvroot, config, 10496 (const nvlist_t * const *)list, sav->sav_count); 10497 for (i = 0; i < sav->sav_count; i++) 10498 nvlist_free(list[i]); 10499 kmem_free(list, sav->sav_count * sizeof (void *)); 10500 } 10501 10502 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 10503 nvlist_free(nvroot); 10504 10505 sav->sav_sync = B_FALSE; 10506 } 10507 10508 /* 10509 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 10510 * The all-vdev ZAP must be empty. 10511 */ 10512 static void 10513 spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 10514 { 10515 spa_t *spa = vd->vdev_spa; 10516 10517 if (vd->vdev_root_zap != 0 && 10518 spa_feature_is_active(spa, SPA_FEATURE_AVZ_V2)) { 10519 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 10520 vd->vdev_root_zap, tx)); 10521 } 10522 if (vd->vdev_top_zap != 0) { 10523 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 10524 vd->vdev_top_zap, tx)); 10525 } 10526 if (vd->vdev_leaf_zap != 0) { 10527 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 10528 vd->vdev_leaf_zap, tx)); 10529 } 10530 for (uint64_t i = 0; i < vd->vdev_children; i++) { 10531 spa_avz_build(vd->vdev_child[i], avz, tx); 10532 } 10533 } 10534 10535 static void 10536 spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 10537 { 10538 nvlist_t *config; 10539 10540 /* 10541 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 10542 * its config may not be dirty but we still need to build per-vdev ZAPs. 10543 * Similarly, if the pool is being assembled (e.g. after a split), we 10544 * need to rebuild the AVZ although the config may not be dirty. 10545 */ 10546 if (list_is_empty(&spa->spa_config_dirty_list) && 10547 spa->spa_avz_action == AVZ_ACTION_NONE) 10548 return; 10549 10550 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 10551 10552 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 10553 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 10554 spa->spa_all_vdev_zaps != 0); 10555 10556 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 10557 /* Make and build the new AVZ */ 10558 uint64_t new_avz = zap_create(spa->spa_meta_objset, 10559 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 10560 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 10561 10562 /* Diff old AVZ with new one */ 10563 zap_cursor_t zc; 10564 zap_attribute_t *za = zap_attribute_alloc(); 10565 10566 for (zap_cursor_init(&zc, spa->spa_meta_objset, 10567 spa->spa_all_vdev_zaps); 10568 zap_cursor_retrieve(&zc, za) == 0; 10569 zap_cursor_advance(&zc)) { 10570 uint64_t vdzap = za->za_first_integer; 10571 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 10572 vdzap) == ENOENT) { 10573 /* 10574 * ZAP is listed in old AVZ but not in new one; 10575 * destroy it 10576 */ 10577 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 10578 tx)); 10579 } 10580 } 10581 10582 zap_cursor_fini(&zc); 10583 zap_attribute_free(za); 10584 10585 /* Destroy the old AVZ */ 10586 VERIFY0(zap_destroy(spa->spa_meta_objset, 10587 spa->spa_all_vdev_zaps, tx)); 10588 10589 /* Replace the old AVZ in the dir obj with the new one */ 10590 VERIFY0(zap_update(spa->spa_meta_objset, 10591 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 10592 sizeof (new_avz), 1, &new_avz, tx)); 10593 10594 spa->spa_all_vdev_zaps = new_avz; 10595 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 10596 zap_cursor_t zc; 10597 zap_attribute_t *za = zap_attribute_alloc(); 10598 10599 /* Walk through the AVZ and destroy all listed ZAPs */ 10600 for (zap_cursor_init(&zc, spa->spa_meta_objset, 10601 spa->spa_all_vdev_zaps); 10602 zap_cursor_retrieve(&zc, za) == 0; 10603 zap_cursor_advance(&zc)) { 10604 uint64_t zap = za->za_first_integer; 10605 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 10606 } 10607 10608 zap_cursor_fini(&zc); 10609 zap_attribute_free(za); 10610 10611 /* Destroy and unlink the AVZ itself */ 10612 VERIFY0(zap_destroy(spa->spa_meta_objset, 10613 spa->spa_all_vdev_zaps, tx)); 10614 VERIFY0(zap_remove(spa->spa_meta_objset, 10615 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 10616 spa->spa_all_vdev_zaps = 0; 10617 } 10618 10619 if (spa->spa_all_vdev_zaps == 0) { 10620 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 10621 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 10622 DMU_POOL_VDEV_ZAP_MAP, tx); 10623 } 10624 spa->spa_avz_action = AVZ_ACTION_NONE; 10625 10626 /* Create ZAPs for vdevs that don't have them. */ 10627 vdev_construct_zaps(spa->spa_root_vdev, tx); 10628 10629 config = spa_config_generate(spa, spa->spa_root_vdev, 10630 dmu_tx_get_txg(tx), B_FALSE); 10631 10632 /* 10633 * If we're upgrading the spa version then make sure that 10634 * the config object gets updated with the correct version. 10635 */ 10636 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 10637 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 10638 spa->spa_uberblock.ub_version); 10639 10640 spa_config_exit(spa, SCL_STATE, FTAG); 10641 10642 nvlist_free(spa->spa_config_syncing); 10643 spa->spa_config_syncing = config; 10644 10645 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 10646 } 10647 10648 static void 10649 spa_sync_version(void *arg, dmu_tx_t *tx) 10650 { 10651 uint64_t *versionp = arg; 10652 uint64_t version = *versionp; 10653 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 10654 10655 /* 10656 * Setting the version is special cased when first creating the pool. 10657 */ 10658 ASSERT(tx->tx_txg != TXG_INITIAL); 10659 10660 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 10661 ASSERT(version >= spa_version(spa)); 10662 10663 spa->spa_uberblock.ub_version = version; 10664 vdev_config_dirty(spa->spa_root_vdev); 10665 spa_history_log_internal(spa, "set", tx, "version=%lld", 10666 (longlong_t)version); 10667 } 10668 10669 /* 10670 * Set zpool properties. 10671 */ 10672 static void 10673 spa_sync_props(void *arg, dmu_tx_t *tx) 10674 { 10675 nvlist_t *nvp = arg; 10676 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 10677 objset_t *mos = spa->spa_meta_objset; 10678 nvpair_t *elem = NULL; 10679 10680 mutex_enter(&spa->spa_props_lock); 10681 10682 while ((elem = nvlist_next_nvpair(nvp, elem))) { 10683 uint64_t intval; 10684 const char *strval, *fname; 10685 zpool_prop_t prop; 10686 const char *propname; 10687 const char *elemname = nvpair_name(elem); 10688 zprop_type_t proptype; 10689 spa_feature_t fid; 10690 10691 switch (prop = zpool_name_to_prop(elemname)) { 10692 case ZPOOL_PROP_VERSION: 10693 intval = fnvpair_value_uint64(elem); 10694 /* 10695 * The version is synced separately before other 10696 * properties and should be correct by now. 10697 */ 10698 ASSERT3U(spa_version(spa), >=, intval); 10699 break; 10700 10701 case ZPOOL_PROP_ALTROOT: 10702 /* 10703 * 'altroot' is a non-persistent property. It should 10704 * have been set temporarily at creation or import time. 10705 */ 10706 ASSERT(spa->spa_root != NULL); 10707 break; 10708 10709 case ZPOOL_PROP_READONLY: 10710 case ZPOOL_PROP_CACHEFILE: 10711 /* 10712 * 'readonly' and 'cachefile' are also non-persistent 10713 * properties. 10714 */ 10715 break; 10716 case ZPOOL_PROP_COMMENT: 10717 strval = fnvpair_value_string(elem); 10718 if (spa->spa_comment != NULL) 10719 spa_strfree(spa->spa_comment); 10720 spa->spa_comment = spa_strdup(strval); 10721 /* 10722 * We need to dirty the configuration on all the vdevs 10723 * so that their labels get updated. We also need to 10724 * update the cache file to keep it in sync with the 10725 * MOS version. It's unnecessary to do this for pool 10726 * creation since the vdev's configuration has already 10727 * been dirtied. 10728 */ 10729 if (tx->tx_txg != TXG_INITIAL) { 10730 vdev_config_dirty(spa->spa_root_vdev); 10731 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 10732 } 10733 spa_history_log_internal(spa, "set", tx, 10734 "%s=%s", elemname, strval); 10735 break; 10736 case ZPOOL_PROP_COMPATIBILITY: 10737 strval = fnvpair_value_string(elem); 10738 if (spa->spa_compatibility != NULL) 10739 spa_strfree(spa->spa_compatibility); 10740 spa->spa_compatibility = spa_strdup(strval); 10741 /* 10742 * Dirty the configuration on vdevs as above. 10743 */ 10744 if (tx->tx_txg != TXG_INITIAL) { 10745 vdev_config_dirty(spa->spa_root_vdev); 10746 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 10747 } 10748 10749 spa_history_log_internal(spa, "set", tx, 10750 "%s=%s", nvpair_name(elem), strval); 10751 break; 10752 10753 case ZPOOL_PROP_INVAL: 10754 if (zpool_prop_feature(elemname)) { 10755 fname = strchr(elemname, '@') + 1; 10756 VERIFY0(zfeature_lookup_name(fname, &fid)); 10757 10758 spa_feature_enable(spa, fid, tx); 10759 spa_history_log_internal(spa, "set", tx, 10760 "%s=enabled", elemname); 10761 break; 10762 } else if (!zfs_prop_user(elemname)) { 10763 ASSERT(zpool_prop_feature(elemname)); 10764 break; 10765 } 10766 zfs_fallthrough; 10767 default: 10768 /* 10769 * Set pool property values in the poolprops mos object. 10770 */ 10771 if (spa->spa_pool_props_object == 0) { 10772 spa->spa_pool_props_object = 10773 zap_create_link(mos, DMU_OT_POOL_PROPS, 10774 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 10775 tx); 10776 } 10777 10778 /* normalize the property name */ 10779 if (prop == ZPOOL_PROP_INVAL) { 10780 propname = elemname; 10781 proptype = PROP_TYPE_STRING; 10782 } else { 10783 propname = zpool_prop_to_name(prop); 10784 proptype = zpool_prop_get_type(prop); 10785 } 10786 10787 if (nvpair_type(elem) == DATA_TYPE_STRING) { 10788 ASSERT(proptype == PROP_TYPE_STRING); 10789 strval = fnvpair_value_string(elem); 10790 if (strlen(strval) == 0) { 10791 /* remove the property if value == "" */ 10792 (void) zap_remove(mos, 10793 spa->spa_pool_props_object, 10794 propname, tx); 10795 } else { 10796 VERIFY0(zap_update(mos, 10797 spa->spa_pool_props_object, 10798 propname, 1, strlen(strval) + 1, 10799 strval, tx)); 10800 } 10801 spa_history_log_internal(spa, "set", tx, 10802 "%s=%s", elemname, strval); 10803 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 10804 intval = fnvpair_value_uint64(elem); 10805 10806 if (proptype == PROP_TYPE_INDEX) { 10807 const char *unused; 10808 VERIFY0(zpool_prop_index_to_string( 10809 prop, intval, &unused)); 10810 } 10811 VERIFY0(zap_update(mos, 10812 spa->spa_pool_props_object, propname, 10813 8, 1, &intval, tx)); 10814 spa_history_log_internal(spa, "set", tx, 10815 "%s=%lld", elemname, 10816 (longlong_t)intval); 10817 10818 switch (prop) { 10819 case ZPOOL_PROP_DELEGATION: 10820 spa->spa_delegation = intval; 10821 break; 10822 case ZPOOL_PROP_BOOTFS: 10823 spa->spa_bootfs = intval; 10824 break; 10825 case ZPOOL_PROP_FAILUREMODE: 10826 spa->spa_failmode = intval; 10827 break; 10828 case ZPOOL_PROP_AUTOTRIM: 10829 spa->spa_autotrim = intval; 10830 spa_async_request(spa, 10831 SPA_ASYNC_AUTOTRIM_RESTART); 10832 break; 10833 case ZPOOL_PROP_AUTOEXPAND: 10834 spa->spa_autoexpand = intval; 10835 if (tx->tx_txg != TXG_INITIAL) 10836 spa_async_request(spa, 10837 SPA_ASYNC_AUTOEXPAND); 10838 break; 10839 case ZPOOL_PROP_MULTIHOST: 10840 spa->spa_multihost = intval; 10841 break; 10842 case ZPOOL_PROP_DEDUP_TABLE_QUOTA: 10843 spa->spa_dedup_table_quota = intval; 10844 break; 10845 default: 10846 break; 10847 } 10848 } else { 10849 ASSERT(0); /* not allowed */ 10850 } 10851 } 10852 10853 } 10854 10855 mutex_exit(&spa->spa_props_lock); 10856 } 10857 10858 /* 10859 * Perform one-time upgrade on-disk changes. spa_version() does not 10860 * reflect the new version this txg, so there must be no changes this 10861 * txg to anything that the upgrade code depends on after it executes. 10862 * Therefore this must be called after dsl_pool_sync() does the sync 10863 * tasks. 10864 */ 10865 static void 10866 spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 10867 { 10868 if (spa_sync_pass(spa) != 1) 10869 return; 10870 10871 uint64_t oldver = spa->spa_ubsync.ub_version; 10872 uint64_t newver = spa->spa_uberblock.ub_version; 10873 10874 /* 10875 * These upgrades change DSL namespace, so they need the 10876 * writer lock. 10877 */ 10878 boolean_t need_origin = oldver < SPA_VERSION_ORIGIN && 10879 newver >= SPA_VERSION_ORIGIN; 10880 boolean_t need_clones = oldver < SPA_VERSION_NEXT_CLONES && 10881 newver >= SPA_VERSION_NEXT_CLONES; 10882 boolean_t need_dir_clones = oldver < SPA_VERSION_DIR_CLONES && 10883 newver >= SPA_VERSION_DIR_CLONES; 10884 10885 if (need_origin || need_clones || need_dir_clones) { 10886 dsl_pool_t *dp = spa->spa_dsl_pool; 10887 10888 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 10889 10890 if (need_origin) { 10891 dsl_pool_create_origin(dp, tx); 10892 10893 /* Keeping the origin open increases spa_minref */ 10894 spa->spa_minref += 3; 10895 } 10896 10897 if (need_clones) { 10898 dsl_pool_upgrade_clones(dp, tx); 10899 } 10900 10901 if (need_dir_clones) { 10902 dsl_pool_upgrade_dir_clones(dp, tx); 10903 10904 /* Keeping the freedir open increases spa_minref */ 10905 spa->spa_minref += 3; 10906 } 10907 10908 rrw_exit(&dp->dp_config_rwlock, FTAG); 10909 } 10910 10911 /* Remaining upgrades do not need dp_config_rwlock */ 10912 10913 if (oldver < SPA_VERSION_FEATURES && newver >= SPA_VERSION_FEATURES) { 10914 spa_feature_create_zap_objects(spa, tx); 10915 } 10916 10917 /* 10918 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 10919 * when possibility to use lz4 compression for metadata was added 10920 * Old pools that have this feature enabled must be upgraded to have 10921 * this feature active 10922 */ 10923 if (newver >= SPA_VERSION_FEATURES) { 10924 boolean_t lz4_en = spa_feature_is_enabled(spa, 10925 SPA_FEATURE_LZ4_COMPRESS); 10926 boolean_t lz4_ac = spa_feature_is_active(spa, 10927 SPA_FEATURE_LZ4_COMPRESS); 10928 10929 if (lz4_en && !lz4_ac) 10930 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 10931 } 10932 10933 /* 10934 * If we haven't written the salt, do so now. Note that the 10935 * feature may not be activated yet, but that's fine since 10936 * the presence of this ZAP entry is backwards compatible. 10937 */ 10938 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 10939 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 10940 VERIFY0(zap_add(spa->spa_meta_objset, 10941 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 10942 sizeof (spa->spa_cksum_salt.zcs_bytes), 10943 spa->spa_cksum_salt.zcs_bytes, tx)); 10944 } 10945 } 10946 10947 static void 10948 vdev_indirect_state_sync_verify(vdev_t *vd) 10949 { 10950 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 10951 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 10952 10953 if (vd->vdev_ops == &vdev_indirect_ops) { 10954 ASSERT(vim != NULL); 10955 ASSERT(vib != NULL); 10956 } 10957 10958 uint64_t obsolete_sm_object = 0; 10959 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 10960 if (obsolete_sm_object != 0) { 10961 ASSERT(vd->vdev_obsolete_sm != NULL); 10962 ASSERT(vd->vdev_removing || 10963 vd->vdev_ops == &vdev_indirect_ops); 10964 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 10965 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 10966 ASSERT3U(obsolete_sm_object, ==, 10967 space_map_object(vd->vdev_obsolete_sm)); 10968 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 10969 space_map_allocated(vd->vdev_obsolete_sm)); 10970 } 10971 ASSERT(vd->vdev_obsolete_segments != NULL); 10972 10973 /* 10974 * Since frees / remaps to an indirect vdev can only 10975 * happen in syncing context, the obsolete segments 10976 * tree must be empty when we start syncing. 10977 */ 10978 ASSERT0(zfs_range_tree_space(vd->vdev_obsolete_segments)); 10979 } 10980 10981 /* 10982 * Set the top-level vdev's max queue depth. Evaluate each top-level's 10983 * async write queue depth in case it changed. The max queue depth will 10984 * not change in the middle of syncing out this txg. 10985 */ 10986 static void 10987 spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 10988 { 10989 ASSERT(spa_writeable(spa)); 10990 10991 metaslab_class_balance(spa_normal_class(spa), B_TRUE); 10992 metaslab_class_balance(spa_special_class(spa), B_TRUE); 10993 metaslab_class_balance(spa_dedup_class(spa), B_TRUE); 10994 } 10995 10996 static void 10997 spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 10998 { 10999 ASSERT(spa_writeable(spa)); 11000 11001 vdev_t *rvd = spa->spa_root_vdev; 11002 for (int c = 0; c < rvd->vdev_children; c++) { 11003 vdev_t *vd = rvd->vdev_child[c]; 11004 vdev_indirect_state_sync_verify(vd); 11005 11006 if (vdev_indirect_should_condense(vd)) { 11007 spa_condense_indirect_start_sync(vd, tx); 11008 break; 11009 } 11010 } 11011 } 11012 11013 static void 11014 spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 11015 { 11016 objset_t *mos = spa->spa_meta_objset; 11017 dsl_pool_t *dp = spa->spa_dsl_pool; 11018 uint64_t txg = tx->tx_txg; 11019 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 11020 11021 do { 11022 int pass = ++spa->spa_sync_pass; 11023 11024 spa_sync_config_object(spa, tx); 11025 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 11026 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 11027 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 11028 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 11029 spa_errlog_sync(spa, txg); 11030 dsl_pool_sync(dp, txg); 11031 11032 if (pass < zfs_sync_pass_deferred_free || 11033 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 11034 /* 11035 * If the log space map feature is active we don't 11036 * care about deferred frees and the deferred bpobj 11037 * as the log space map should effectively have the 11038 * same results (i.e. appending only to one object). 11039 */ 11040 spa_sync_frees(spa, free_bpl, tx); 11041 } else { 11042 /* 11043 * We can not defer frees in pass 1, because 11044 * we sync the deferred frees later in pass 1. 11045 */ 11046 ASSERT3U(pass, >, 1); 11047 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 11048 &spa->spa_deferred_bpobj, tx); 11049 } 11050 11051 brt_sync(spa, txg); 11052 ddt_sync(spa, txg); 11053 dsl_scan_sync(dp, tx); 11054 dsl_errorscrub_sync(dp, tx); 11055 svr_sync(spa, tx); 11056 spa_sync_upgrades(spa, tx); 11057 11058 spa_flush_metaslabs(spa, tx); 11059 11060 vdev_t *vd = NULL; 11061 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 11062 != NULL) 11063 vdev_sync(vd, txg); 11064 11065 if (pass == 1) { 11066 /* 11067 * dsl_pool_sync() -> dp_sync_tasks may have dirtied 11068 * the config. If that happens, this txg should not 11069 * be a no-op. So we must sync the config to the MOS 11070 * before checking for no-op. 11071 * 11072 * Note that when the config is dirty, it will 11073 * be written to the MOS (i.e. the MOS will be 11074 * dirtied) every time we call spa_sync_config_object() 11075 * in this txg. Therefore we can't call this after 11076 * dsl_pool_sync() every pass, because it would 11077 * prevent us from converging, since we'd dirty 11078 * the MOS every pass. 11079 * 11080 * Sync tasks can only be processed in pass 1, so 11081 * there's no need to do this in later passes. 11082 */ 11083 spa_sync_config_object(spa, tx); 11084 } 11085 11086 /* 11087 * Note: We need to check if the MOS is dirty because we could 11088 * have marked the MOS dirty without updating the uberblock 11089 * (e.g. if we have sync tasks but no dirty user data). We need 11090 * to check the uberblock's rootbp because it is updated if we 11091 * have synced out dirty data (though in this case the MOS will 11092 * most likely also be dirty due to second order effects, we 11093 * don't want to rely on that here). 11094 */ 11095 if (pass == 1 && 11096 BP_GET_LOGICAL_BIRTH(&spa->spa_uberblock.ub_rootbp) < txg && 11097 !dmu_objset_is_dirty(mos, txg)) { 11098 /* 11099 * Nothing changed on the first pass, therefore this 11100 * TXG is a no-op. Avoid syncing deferred frees, so 11101 * that we can keep this TXG as a no-op. 11102 */ 11103 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 11104 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 11105 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 11106 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 11107 break; 11108 } 11109 11110 spa_sync_deferred_frees(spa, tx); 11111 } while (dmu_objset_is_dirty(mos, txg)); 11112 } 11113 11114 /* 11115 * Select up to SPA_SYNC_MIN_VDEVS top-level vdevs to write the uberblock to. 11116 * First take the ones written during this txg, so that the idle ones may stay 11117 * asleep. If there are not enough, top up from special and dedup vdevs, which 11118 * are expected to have no seek penalty. Pools having none of those keep the 11119 * old behavior of topping up from any vdev. 11120 */ 11121 static int 11122 spa_select_uberblock_vdevs(spa_t *spa, vdev_t **svd, uint64_t txg) 11123 { 11124 vdev_t *rvd = spa->spa_root_vdev; 11125 uint64_t children = rvd->vdev_children; 11126 uint64_t c0 = random_in_range(children); 11127 boolean_t tiered = spa_has_special(spa) || spa_has_dedup(spa); 11128 int svdcount = 0; 11129 11130 for (int pass = 0; pass < 3; pass++) { 11131 if (pass == 2 && svdcount > 0 && tiered) 11132 break; 11133 11134 for (uint64_t c = 0; c < children && 11135 svdcount < SPA_SYNC_MIN_VDEVS; c++) { 11136 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 11137 boolean_t dup = B_FALSE; 11138 11139 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 11140 !vdev_is_concrete(vd)) 11141 continue; 11142 11143 if (pass == 0 && !txg_list_member( 11144 &spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg))) 11145 continue; 11146 11147 if (pass == 1) { 11148 metaslab_class_t *mc = vd->vdev_mg != NULL ? 11149 vd->vdev_mg->mg_class : NULL; 11150 if (mc != spa_special_class(spa) && 11151 mc != spa_dedup_class(spa)) 11152 continue; 11153 } 11154 11155 for (int i = 0; i < svdcount; i++) 11156 dup |= (svd[i] == vd); 11157 if (dup) 11158 continue; 11159 11160 svd[svdcount++] = vd; 11161 } 11162 } 11163 11164 return (svdcount); 11165 } 11166 11167 /* 11168 * Rewrite the vdev configuration (which includes the uberblock) to 11169 * commit the transaction group. 11170 * 11171 * If there are no dirty vdevs, we sync the uberblock to a few random 11172 * top-level vdevs that are known to be visible in the config cache 11173 * (see spa_vdev_add() for a complete description). If there *are* dirty 11174 * vdevs, sync the uberblock to all vdevs. 11175 */ 11176 static void 11177 spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 11178 { 11179 vdev_t *rvd = spa->spa_root_vdev; 11180 uint64_t txg = tx->tx_txg; 11181 11182 for (;;) { 11183 int error = 0; 11184 11185 /* 11186 * We hold SCL_STATE to prevent vdev open/close/etc. 11187 * while we're attempting to write the vdev labels. 11188 */ 11189 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 11190 11191 if (list_is_empty(&spa->spa_config_dirty_list)) { 11192 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 11193 int svdcount = spa_select_uberblock_vdevs(spa, svd, 11194 txg); 11195 11196 error = vdev_config_sync(spa, svd, svdcount, txg); 11197 } else { 11198 error = vdev_config_sync(spa, rvd->vdev_child, 11199 rvd->vdev_children, txg); 11200 } 11201 11202 if (error == 0) 11203 spa->spa_last_synced_guid = rvd->vdev_guid; 11204 11205 spa_config_exit(spa, SCL_STATE, FTAG); 11206 11207 if (error == 0) 11208 break; 11209 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 11210 zio_resume_wait(spa); 11211 } 11212 } 11213 11214 /* 11215 * Sync the specified transaction group. New blocks may be dirtied as 11216 * part of the process, so we iterate until it converges. 11217 */ 11218 void 11219 spa_sync(spa_t *spa, uint64_t txg) 11220 { 11221 vdev_t *vd = NULL; 11222 11223 VERIFY(spa_writeable(spa)); 11224 11225 /* 11226 * Wait for i/os issued in open context that need to complete 11227 * before this txg syncs. 11228 */ 11229 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 11230 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 11231 ZIO_FLAG_CANFAIL); 11232 11233 /* 11234 * Now that there can be no more cloning in this transaction group, 11235 * but we are still before issuing frees, we can process pending BRT 11236 * updates. 11237 */ 11238 brt_pending_apply(spa, txg); 11239 11240 spa_sync_time_logger(spa, txg, B_FALSE); 11241 11242 /* 11243 * Lock out configuration changes. 11244 */ 11245 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 11246 11247 spa->spa_syncing_txg = txg; 11248 spa->spa_sync_pass = 0; 11249 11250 /* 11251 * If there are any pending vdev state changes, convert them 11252 * into config changes that go out with this transaction group. 11253 */ 11254 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 11255 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 11256 /* Avoid holding the write lock unless actually necessary */ 11257 if (vd->vdev_aux == NULL) { 11258 vdev_state_clean(vd); 11259 vdev_config_dirty(vd); 11260 continue; 11261 } 11262 /* 11263 * We need the write lock here because, for aux vdevs, 11264 * calling vdev_config_dirty() modifies sav_config. 11265 * This is ugly and will become unnecessary when we 11266 * eliminate the aux vdev wart by integrating all vdevs 11267 * into the root vdev tree. 11268 */ 11269 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 11270 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 11271 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 11272 vdev_state_clean(vd); 11273 vdev_config_dirty(vd); 11274 } 11275 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 11276 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 11277 } 11278 spa_config_exit(spa, SCL_STATE, FTAG); 11279 11280 dsl_pool_t *dp = spa->spa_dsl_pool; 11281 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 11282 11283 spa->spa_sync_starttime = getlrtime(); 11284 11285 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid, B_TRUE); 11286 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 11287 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 11288 NSEC_TO_TICK(spa->spa_deadman_synctime)); 11289 11290 /* 11291 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 11292 * set spa_deflate if we have no raid-z vdevs. 11293 */ 11294 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 11295 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 11296 vdev_t *rvd = spa->spa_root_vdev; 11297 11298 int i; 11299 for (i = 0; i < rvd->vdev_children; i++) { 11300 vd = rvd->vdev_child[i]; 11301 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 11302 break; 11303 } 11304 if (i == rvd->vdev_children) { 11305 spa->spa_deflate = TRUE; 11306 VERIFY0(zap_add(spa->spa_meta_objset, 11307 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 11308 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 11309 } 11310 } 11311 11312 spa_sync_adjust_vdev_max_queue_depth(spa); 11313 11314 spa_sync_condense_indirect(spa, tx); 11315 11316 spa_sync_iterate_to_convergence(spa, tx); 11317 11318 #ifdef ZFS_DEBUG 11319 if (!list_is_empty(&spa->spa_config_dirty_list)) { 11320 /* 11321 * Make sure that the number of ZAPs for all the vdevs matches 11322 * the number of ZAPs in the per-vdev ZAP list. This only gets 11323 * called if the config is dirty; otherwise there may be 11324 * outstanding AVZ operations that weren't completed in 11325 * spa_sync_config_object. 11326 */ 11327 uint64_t all_vdev_zap_entry_count; 11328 ASSERT0(zap_count(spa->spa_meta_objset, 11329 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 11330 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 11331 all_vdev_zap_entry_count); 11332 } 11333 #endif 11334 11335 if (spa->spa_vdev_removal != NULL) { 11336 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 11337 } 11338 11339 for (vd = txg_list_head(&spa->spa_vdev_txg_list, TXG_CLEAN(txg)); vd; 11340 vd = txg_list_next(&spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg))) 11341 vdev_sync_dispatch(vd, txg); 11342 11343 spa_sync_rewrite_vdev_config(spa, tx); 11344 dmu_tx_commit(tx); 11345 11346 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid, B_TRUE); 11347 spa->spa_deadman_tqid = 0; 11348 11349 /* 11350 * Clear the dirty config list. 11351 */ 11352 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 11353 vdev_config_clean(vd); 11354 11355 /* 11356 * Now that the new config has synced transactionally, 11357 * let it become visible to the config cache. 11358 */ 11359 if (spa->spa_config_syncing != NULL) { 11360 spa_config_set(spa, spa->spa_config_syncing); 11361 spa->spa_config_txg = txg; 11362 spa->spa_config_syncing = NULL; 11363 } 11364 11365 dsl_pool_sync_done(dp, txg); 11366 11367 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 11368 != NULL) 11369 vdev_sync_done(vd, txg); 11370 11371 metaslab_class_evict_old(spa->spa_normal_class, txg); 11372 metaslab_class_evict_old(spa->spa_log_class, txg); 11373 /* Embedded log classes have only one metaslab per vdev. */ 11374 metaslab_class_evict_old(spa->spa_special_class, txg); 11375 metaslab_class_evict_old(spa->spa_dedup_class, txg); 11376 11377 spa_sync_close_syncing_log_sm(spa); 11378 11379 spa_update_dspace(spa); 11380 spa_log_sm_stats_update(spa); 11381 11382 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) 11383 vdev_autotrim_kick(spa); 11384 11385 /* 11386 * It had better be the case that we didn't dirty anything 11387 * since vdev_config_sync(). 11388 */ 11389 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 11390 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 11391 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 11392 11393 while (zfs_pause_spa_sync) 11394 delay(1); 11395 11396 spa->spa_sync_pass = 0; 11397 11398 /* 11399 * Update the last synced uberblock here. We want to do this at 11400 * the end of spa_sync() so that consumers of spa_last_synced_txg() 11401 * will be guaranteed that all the processing associated with 11402 * that txg has been completed. 11403 */ 11404 spa->spa_ubsync = spa->spa_uberblock; 11405 spa_config_exit(spa, SCL_CONFIG, FTAG); 11406 11407 /* 11408 * An activity that ended in this txg is only over for a reader of 11409 * the pool now that the txg is on disk, so let the waiters look 11410 * again (see spa_activity_in_progress()). 11411 */ 11412 spa_notify_waiters(spa); 11413 11414 spa_handle_ignored_writes(spa); 11415 11416 /* 11417 * If any async tasks have been requested, kick them off. 11418 */ 11419 spa_async_dispatch(spa); 11420 } 11421 11422 /* 11423 * Sync all pools. We don't want to hold the namespace lock across these 11424 * operations, so we take a reference on the spa_t and drop the lock during the 11425 * sync. 11426 */ 11427 void 11428 spa_sync_allpools(void) 11429 { 11430 spa_t *spa = NULL; 11431 spa_namespace_enter(FTAG); 11432 while ((spa = spa_next(spa)) != NULL) { 11433 if (spa_state(spa) != POOL_STATE_ACTIVE || 11434 !spa_writeable(spa) || spa_suspended(spa)) 11435 continue; 11436 spa_open_ref(spa, FTAG); 11437 spa_namespace_exit(FTAG); 11438 txg_wait_synced(spa_get_dsl(spa), 0); 11439 spa_namespace_enter(FTAG); 11440 spa_close(spa, FTAG); 11441 } 11442 spa_namespace_exit(FTAG); 11443 } 11444 11445 taskq_t * 11446 spa_sync_tq_create(spa_t *spa, const char *name) 11447 { 11448 kthread_t **kthreads; 11449 11450 ASSERT0P(spa->spa_sync_tq); 11451 ASSERT3S(spa->spa_alloc_count, <=, boot_ncpus); 11452 11453 /* 11454 * - do not allow more allocators than cpus. 11455 * - there may be more cpus than allocators. 11456 * - do not allow more sync taskq threads than allocators or cpus. 11457 */ 11458 int nthreads = spa->spa_alloc_count; 11459 spa->spa_syncthreads = kmem_zalloc(sizeof (spa_syncthread_info_t) * 11460 nthreads, KM_SLEEP); 11461 11462 spa->spa_sync_tq = taskq_create_synced(name, nthreads, minclsyspri, 11463 nthreads, INT_MAX, TASKQ_PREPOPULATE, &kthreads); 11464 VERIFY(spa->spa_sync_tq != NULL); 11465 VERIFY(kthreads != NULL); 11466 11467 spa_syncthread_info_t *ti = spa->spa_syncthreads; 11468 for (int i = 0; i < nthreads; i++, ti++) { 11469 ti->sti_thread = kthreads[i]; 11470 ti->sti_allocator = i; 11471 } 11472 11473 kmem_free(kthreads, sizeof (*kthreads) * nthreads); 11474 return (spa->spa_sync_tq); 11475 } 11476 11477 void 11478 spa_sync_tq_destroy(spa_t *spa) 11479 { 11480 ASSERT(spa->spa_sync_tq != NULL); 11481 11482 taskq_wait(spa->spa_sync_tq); 11483 taskq_destroy(spa->spa_sync_tq); 11484 kmem_free(spa->spa_syncthreads, 11485 sizeof (spa_syncthread_info_t) * spa->spa_alloc_count); 11486 spa->spa_sync_tq = NULL; 11487 } 11488 11489 uint_t 11490 spa_acq_allocator(spa_t *spa) 11491 { 11492 int i; 11493 11494 if (spa->spa_alloc_count == 1) 11495 return (0); 11496 11497 mutex_enter(&spa->spa_allocs_use->sau_lock); 11498 uint_t r = spa->spa_allocs_use->sau_rotor; 11499 do { 11500 if (++r == spa->spa_alloc_count) 11501 r = 0; 11502 } while (spa->spa_allocs_use->sau_inuse[r]); 11503 spa->spa_allocs_use->sau_inuse[r] = B_TRUE; 11504 spa->spa_allocs_use->sau_rotor = r; 11505 mutex_exit(&spa->spa_allocs_use->sau_lock); 11506 11507 spa_syncthread_info_t *ti = spa->spa_syncthreads; 11508 for (i = 0; i < spa->spa_alloc_count; i++, ti++) { 11509 if (ti->sti_thread == curthread) { 11510 ti->sti_allocator = r; 11511 break; 11512 } 11513 } 11514 ASSERT3S(i, <, spa->spa_alloc_count); 11515 return (r); 11516 } 11517 11518 void 11519 spa_rel_allocator(spa_t *spa, uint_t allocator) 11520 { 11521 if (spa->spa_alloc_count > 1) 11522 spa->spa_allocs_use->sau_inuse[allocator] = B_FALSE; 11523 } 11524 11525 void 11526 spa_select_allocator(zio_t *zio) 11527 { 11528 zbookmark_phys_t *bm = &zio->io_bookmark; 11529 spa_t *spa = zio->io_spa; 11530 11531 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 11532 11533 /* 11534 * A gang block (for example) may have inherited its parent's 11535 * allocator, in which case there is nothing further to do here. 11536 */ 11537 if (ZIO_HAS_ALLOCATOR(zio)) 11538 return; 11539 11540 ASSERT(spa != NULL); 11541 ASSERT(bm != NULL); 11542 11543 /* 11544 * First try to use an allocator assigned to the syncthread, and set 11545 * the corresponding write issue taskq for the allocator. 11546 * Note, we must have an open pool to do this. 11547 */ 11548 if (spa->spa_sync_tq != NULL) { 11549 spa_syncthread_info_t *ti = spa->spa_syncthreads; 11550 for (int i = 0; i < spa->spa_alloc_count; i++, ti++) { 11551 if (ti->sti_thread == curthread) { 11552 zio->io_allocator = ti->sti_allocator; 11553 return; 11554 } 11555 } 11556 } 11557 11558 /* 11559 * We want to try to use as many allocators as possible to help improve 11560 * performance, but we also want logically adjacent IOs to be physically 11561 * adjacent to improve sequential read performance. We chunk each object 11562 * into 2^20 block regions, and then hash based on the objset, object, 11563 * level, and region to accomplish both of these goals. 11564 */ 11565 uint64_t hv = cityhash4(bm->zb_objset, bm->zb_object, bm->zb_level, 11566 bm->zb_blkid >> 20); 11567 11568 zio->io_allocator = (uint_t)hv % spa->spa_alloc_count; 11569 } 11570 11571 /* 11572 * ========================================================================== 11573 * Miscellaneous routines 11574 * ========================================================================== 11575 */ 11576 11577 /* 11578 * Remove all pools in the system. 11579 */ 11580 void 11581 spa_evict_all(void) 11582 { 11583 spa_t *spa; 11584 11585 /* 11586 * Remove all cached state. All pools should be closed now, 11587 * so every spa in the AVL tree should be unreferenced. 11588 */ 11589 spa_namespace_enter(FTAG); 11590 while ((spa = spa_next(NULL)) != NULL) { 11591 /* 11592 * Stop async tasks. The async thread may need to detach 11593 * a device that's been replaced, which requires grabbing 11594 * spa_namespace_lock, so we must drop it here. 11595 */ 11596 spa_open_ref(spa, FTAG); 11597 spa_namespace_exit(FTAG); 11598 spa_async_suspend(spa); 11599 spa_namespace_enter(FTAG); 11600 spa_close(spa, FTAG); 11601 11602 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 11603 spa_unload(spa); 11604 spa_deactivate(spa); 11605 } 11606 spa_remove(spa); 11607 } 11608 spa_namespace_exit(FTAG); 11609 } 11610 11611 vdev_t * 11612 spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 11613 { 11614 vdev_t *vd; 11615 int i; 11616 11617 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 11618 return (vd); 11619 11620 if (aux) { 11621 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 11622 vd = spa->spa_l2cache.sav_vdevs[i]; 11623 if (vd->vdev_guid == guid) 11624 return (vd); 11625 } 11626 11627 for (i = 0; i < spa->spa_spares.sav_count; i++) { 11628 vd = spa->spa_spares.sav_vdevs[i]; 11629 if (vd->vdev_guid == guid) 11630 return (vd); 11631 } 11632 } 11633 11634 return (NULL); 11635 } 11636 11637 void 11638 spa_upgrade(spa_t *spa, uint64_t version) 11639 { 11640 ASSERT(spa_writeable(spa)); 11641 11642 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 11643 11644 /* 11645 * This should only be called for a non-faulted pool, and since a 11646 * future version would result in an unopenable pool, this shouldn't be 11647 * possible. 11648 */ 11649 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 11650 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 11651 11652 spa->spa_uberblock.ub_version = version; 11653 vdev_config_dirty(spa->spa_root_vdev); 11654 11655 spa_config_exit(spa, SCL_ALL, FTAG); 11656 11657 txg_wait_synced(spa_get_dsl(spa), 0); 11658 } 11659 11660 static boolean_t 11661 spa_has_aux_vdev(spa_t *spa, uint64_t guid, spa_aux_vdev_t *sav) 11662 { 11663 (void) spa; 11664 int i; 11665 uint64_t vdev_guid; 11666 11667 for (i = 0; i < sav->sav_count; i++) 11668 if (sav->sav_vdevs[i]->vdev_guid == guid) 11669 return (B_TRUE); 11670 11671 for (i = 0; i < sav->sav_npending; i++) { 11672 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 11673 &vdev_guid) == 0 && vdev_guid == guid) 11674 return (B_TRUE); 11675 } 11676 11677 return (B_FALSE); 11678 } 11679 11680 boolean_t 11681 spa_has_l2cache(spa_t *spa, uint64_t guid) 11682 { 11683 return (spa_has_aux_vdev(spa, guid, &spa->spa_l2cache)); 11684 } 11685 11686 boolean_t 11687 spa_has_spare(spa_t *spa, uint64_t guid) 11688 { 11689 return (spa_has_aux_vdev(spa, guid, &spa->spa_spares)); 11690 } 11691 11692 /* 11693 * Check if a pool has an active shared spare device. 11694 * Note: reference count of an active spare is 2, as a spare and as a replace 11695 */ 11696 static boolean_t 11697 spa_has_active_shared_spare(spa_t *spa) 11698 { 11699 int i, refcnt; 11700 uint64_t pool; 11701 spa_aux_vdev_t *sav = &spa->spa_spares; 11702 11703 for (i = 0; i < sav->sav_count; i++) { 11704 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 11705 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 11706 refcnt > 2) 11707 return (B_TRUE); 11708 } 11709 11710 return (B_FALSE); 11711 } 11712 11713 uint64_t 11714 spa_total_metaslabs(spa_t *spa) 11715 { 11716 vdev_t *rvd = spa->spa_root_vdev; 11717 11718 uint64_t m = 0; 11719 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 11720 vdev_t *vd = rvd->vdev_child[c]; 11721 if (!vdev_is_concrete(vd)) 11722 continue; 11723 m += vd->vdev_ms_count; 11724 } 11725 return (m); 11726 } 11727 11728 /* 11729 * Notify any waiting threads that some activity has switched from being in- 11730 * progress to not-in-progress so that the thread can wake up and determine 11731 * whether it is finished waiting. 11732 */ 11733 void 11734 spa_notify_waiters(spa_t *spa) 11735 { 11736 /* 11737 * Acquiring spa_activities_lock here prevents the cv_broadcast from 11738 * happening between the waiting thread's check and cv_wait. 11739 */ 11740 mutex_enter(&spa->spa_activities_lock); 11741 cv_broadcast(&spa->spa_activities_cv); 11742 mutex_exit(&spa->spa_activities_lock); 11743 } 11744 11745 /* 11746 * Notify any waiting threads that the pool is exporting, and then block until 11747 * they are finished using the spa_t. 11748 */ 11749 void 11750 spa_wake_waiters(spa_t *spa) 11751 { 11752 mutex_enter(&spa->spa_activities_lock); 11753 spa->spa_waiters_cancel = B_TRUE; 11754 cv_broadcast(&spa->spa_activities_cv); 11755 while (spa->spa_waiters != 0) 11756 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 11757 spa->spa_waiters_cancel = B_FALSE; 11758 mutex_exit(&spa->spa_activities_lock); 11759 } 11760 11761 /* Whether the vdev or any of its descendants are being initialized/trimmed. */ 11762 static boolean_t 11763 spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 11764 { 11765 spa_t *spa = vd->vdev_spa; 11766 11767 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 11768 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 11769 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 11770 activity == ZPOOL_WAIT_TRIM); 11771 11772 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 11773 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 11774 11775 mutex_exit(&spa->spa_activities_lock); 11776 mutex_enter(lock); 11777 mutex_enter(&spa->spa_activities_lock); 11778 11779 /* 11780 * A thread that has finished still has to sync out the new state 11781 * before it exits, and until it does the vdev cannot be initialized 11782 * or trimmed again. Wait for the thread itself, not just the state, 11783 * so that a command issued after the wait returns does not fail with 11784 * EBUSY. 11785 */ 11786 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 11787 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE || 11788 vd->vdev_initialize_thread != NULL) : 11789 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE || 11790 vd->vdev_trim_thread != NULL); 11791 mutex_exit(lock); 11792 11793 if (in_progress) 11794 return (B_TRUE); 11795 11796 for (int i = 0; i < vd->vdev_children; i++) { 11797 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 11798 activity)) 11799 return (B_TRUE); 11800 } 11801 11802 return (B_FALSE); 11803 } 11804 11805 /* 11806 * If use_guid is true, this checks whether the vdev specified by guid is 11807 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 11808 * is being initialized/trimmed. The caller must hold the config lock and 11809 * spa_activities_lock. 11810 */ 11811 static int 11812 spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 11813 zpool_wait_activity_t activity, boolean_t *in_progress) 11814 { 11815 mutex_exit(&spa->spa_activities_lock); 11816 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 11817 mutex_enter(&spa->spa_activities_lock); 11818 11819 vdev_t *vd; 11820 if (use_guid) { 11821 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 11822 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 11823 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 11824 return (EINVAL); 11825 } 11826 } else { 11827 vd = spa->spa_root_vdev; 11828 } 11829 11830 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 11831 11832 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 11833 return (0); 11834 } 11835 11836 /* 11837 * Locking for waiting threads 11838 * --------------------------- 11839 * 11840 * Waiting threads need a way to check whether a given activity is in progress, 11841 * and then, if it is, wait for it to complete. Each activity will have some 11842 * in-memory representation of the relevant on-disk state which can be used to 11843 * determine whether or not the activity is in progress. The in-memory state and 11844 * the locking used to protect it will be different for each activity, and may 11845 * not be suitable for use with a cvar (e.g., some state is protected by the 11846 * config lock). To allow waiting threads to wait without any races, another 11847 * lock, spa_activities_lock, is used. 11848 * 11849 * When the state is checked, both the activity-specific lock (if there is one) 11850 * and spa_activities_lock are held. In some cases, the activity-specific lock 11851 * is acquired explicitly (e.g. the config lock). In others, the locking is 11852 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 11853 * thread releases the activity-specific lock and, if the activity is in 11854 * progress, then cv_waits using spa_activities_lock. 11855 * 11856 * The waiting thread is woken when another thread, one completing some 11857 * activity, updates the state of the activity and then calls 11858 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 11859 * needs to hold its activity-specific lock when updating the state, and this 11860 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 11861 * 11862 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 11863 * and because it is held when the waiting thread checks the state of the 11864 * activity, it can never be the case that the completing thread both updates 11865 * the activity state and cv_broadcasts in between the waiting thread's check 11866 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 11867 * 11868 * In order to prevent deadlock, when the waiting thread does its check, in some 11869 * cases it will temporarily drop spa_activities_lock in order to acquire the 11870 * activity-specific lock. The order in which spa_activities_lock and the 11871 * activity specific lock are acquired in the waiting thread is determined by 11872 * the order in which they are acquired in the completing thread; if the 11873 * completing thread calls spa_notify_waiters with the activity-specific lock 11874 * held, then the waiting thread must also acquire the activity-specific lock 11875 * first. 11876 */ 11877 11878 static int 11879 spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 11880 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 11881 { 11882 int error = 0; 11883 11884 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 11885 11886 switch (activity) { 11887 case ZPOOL_WAIT_CKPT_DISCARD: 11888 *in_progress = 11889 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 11890 zap_contains(spa_meta_objset(spa), 11891 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 11892 ENOENT); 11893 break; 11894 case ZPOOL_WAIT_FREE: 11895 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 11896 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 11897 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 11898 spa_livelist_delete_check(spa)); 11899 break; 11900 case ZPOOL_WAIT_INITIALIZE: 11901 case ZPOOL_WAIT_TRIM: 11902 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 11903 activity, in_progress); 11904 break; 11905 case ZPOOL_WAIT_REPLACE: 11906 mutex_exit(&spa->spa_activities_lock); 11907 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 11908 mutex_enter(&spa->spa_activities_lock); 11909 11910 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 11911 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 11912 break; 11913 case ZPOOL_WAIT_REMOVE: 11914 *in_progress = (spa->spa_removing_phys.sr_state == 11915 DSS_SCANNING); 11916 break; 11917 case ZPOOL_WAIT_RESILVER: 11918 *in_progress = vdev_rebuild_active(spa->spa_root_vdev); 11919 if (*in_progress) 11920 break; 11921 zfs_fallthrough; 11922 case ZPOOL_WAIT_SCRUB: 11923 { 11924 boolean_t scanning, paused, is_scrub, finishing; 11925 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 11926 11927 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 11928 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 11929 paused = dsl_scan_is_paused_scrub(scn); 11930 11931 /* 11932 * dsl_scan_done() marks the scan finished in syncing 11933 * context, ahead of the config and label writes that the 11934 * same txg carries, so the scan is not over for anyone 11935 * reading the pool until that txg has synced. Keep 11936 * reporting it as in progress until then, the way the 11937 * initialize and trim waits cover the whole operation. 11938 */ 11939 finishing = (scn->scn_finished_txg != 0 && 11940 spa_last_synced_txg(spa) < scn->scn_finished_txg); 11941 11942 *in_progress = ((scanning || finishing) && !paused && 11943 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 11944 break; 11945 } 11946 case ZPOOL_WAIT_RAIDZ_EXPAND: 11947 { 11948 vdev_raidz_expand_t *vre = spa->spa_raidz_expand; 11949 *in_progress = (vre != NULL && vre->vre_state == DSS_SCANNING); 11950 break; 11951 } 11952 case ZPOOL_WAIT_CONDENSE: { 11953 *in_progress = B_FALSE; 11954 spa_condense_stat_t *scns; 11955 11956 for (spa_condense_type_t type = 0; 11957 type < SPA_CONDENSE_TYPES; type++) { 11958 scns = &spa->spa_condense_stats[type]; 11959 if (scns->scns_start_time > 0 && 11960 scns->scns_end_time == 0) { 11961 *in_progress = B_TRUE; 11962 break; 11963 } 11964 } 11965 break; 11966 } 11967 default: 11968 panic("unrecognized value for activity %d", activity); 11969 } 11970 11971 return (error); 11972 } 11973 11974 static int 11975 spa_wait_common(const char *pool, zpool_wait_activity_t activity, 11976 boolean_t use_tag, uint64_t tag, boolean_t *waited) 11977 { 11978 /* 11979 * The tag is used to distinguish between instances of an activity. 11980 * 'initialize' and 'trim' are the only activities that we use this for. 11981 * The other activities can only have a single instance in progress in a 11982 * pool at one time, making the tag unnecessary. 11983 * 11984 * There can be multiple devices being replaced at once, but since they 11985 * all finish once resilvering finishes, we don't bother keeping track 11986 * of them individually, we just wait for them all to finish. 11987 */ 11988 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 11989 activity != ZPOOL_WAIT_TRIM) 11990 return (EINVAL); 11991 11992 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 11993 return (EINVAL); 11994 11995 spa_t *spa; 11996 int error = spa_open(pool, &spa, FTAG); 11997 if (error != 0) 11998 return (error); 11999 12000 /* 12001 * Increment the spa's waiter count so that we can call spa_close and 12002 * still ensure that the spa_t doesn't get freed before this thread is 12003 * finished with it when the pool is exported. We want to call spa_close 12004 * before we start waiting because otherwise the additional ref would 12005 * prevent the pool from being exported or destroyed throughout the 12006 * potentially long wait. 12007 */ 12008 mutex_enter(&spa->spa_activities_lock); 12009 spa->spa_waiters++; 12010 spa_close(spa, FTAG); 12011 12012 *waited = B_FALSE; 12013 for (;;) { 12014 boolean_t in_progress; 12015 error = spa_activity_in_progress(spa, activity, use_tag, tag, 12016 &in_progress); 12017 12018 if (error || !in_progress || spa->spa_waiters_cancel) 12019 break; 12020 12021 *waited = B_TRUE; 12022 12023 if (cv_wait_sig(&spa->spa_activities_cv, 12024 &spa->spa_activities_lock) == 0) { 12025 error = EINTR; 12026 break; 12027 } 12028 } 12029 12030 spa->spa_waiters--; 12031 cv_signal(&spa->spa_waiters_cv); 12032 mutex_exit(&spa->spa_activities_lock); 12033 12034 return (error); 12035 } 12036 12037 /* 12038 * Wait for a particular instance of the specified activity to complete, where 12039 * the instance is identified by 'tag' 12040 */ 12041 int 12042 spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 12043 boolean_t *waited) 12044 { 12045 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 12046 } 12047 12048 /* 12049 * Wait for all instances of the specified activity complete 12050 */ 12051 int 12052 spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 12053 { 12054 12055 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 12056 } 12057 12058 sysevent_t * 12059 spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 12060 { 12061 sysevent_t *ev = NULL; 12062 #ifdef _KERNEL 12063 nvlist_t *resource; 12064 12065 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 12066 if (resource) { 12067 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 12068 ev->resource = resource; 12069 } 12070 #else 12071 (void) spa, (void) vd, (void) hist_nvl, (void) name; 12072 #endif 12073 return (ev); 12074 } 12075 12076 void 12077 spa_event_post(sysevent_t *ev) 12078 { 12079 #ifdef _KERNEL 12080 if (ev) { 12081 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 12082 kmem_free(ev, sizeof (*ev)); 12083 } 12084 #else 12085 (void) ev; 12086 #endif 12087 } 12088 12089 /* 12090 * Post a zevent corresponding to the given sysevent. The 'name' must be one 12091 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 12092 * filled in from the spa and (optionally) the vdev. This doesn't do anything 12093 * in the userland libzpool, as we don't want consumers to misinterpret ztest 12094 * or zdb as real changes. 12095 */ 12096 void 12097 spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 12098 { 12099 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 12100 } 12101 12102 #ifdef ZFS_DEBUG 12103 /* 12104 * This runs the "debug" condense type, which does nothing, just updates the 12105 * condense counters every second for ten seconds. This exists entirely for 12106 * testing and debugging the condense system itself, which is why it is 12107 * compiled out of production builds. 12108 */ 12109 #define SPA_CONDENSE_DEBUG_STEP (10) 12110 12111 static void 12112 spa_condense_debug_task(void *arg) 12113 { 12114 spa_t *spa = arg; 12115 spa_condense_stat_t *scns = 12116 &spa->spa_condense_stats[SPA_CONDENSE_DEBUG]; 12117 12118 mutex_enter(&spa->spa_condense_stats_lock); 12119 12120 if (spa->spa_condense_debug_tqid == TASKQID_INVALID) { 12121 /* 12122 * Task no longer required, probably cancelled by 12123 * spa_condense_debug_cancel(). Just exit. 12124 */ 12125 mutex_exit(&spa->spa_condense_stats_lock); 12126 return; 12127 } 12128 12129 spa->spa_condense_debug_tqid = TASKQID_INVALID; 12130 12131 /* Move the condense progress along a bit. */ 12132 scns->scns_processed = MIN(scns->scns_total, scns->scns_processed + 12133 (scns->scns_total / SPA_CONDENSE_DEBUG_STEP)); 12134 if (scns->scns_processed == scns->scns_total) { 12135 /* 12136 * Reached the end. Set the end time to "complete" the 12137 * condense, signal waiters, release resources and we're done. 12138 */ 12139 scns->scns_end_time = gethrestime_sec(); 12140 mutex_exit(&spa->spa_condense_stats_lock); 12141 spa_notify_waiters(spa); 12142 spa_close(spa, scns); 12143 return; 12144 } 12145 12146 /* More to do, re-arm the timer for another round. */ 12147 spa->spa_condense_debug_tqid = taskq_dispatch_delay(system_delay_taskq, 12148 spa_condense_debug_task, spa, TQ_SLEEP, 12149 ddi_get_lbolt() + SEC_TO_TICK(1)); 12150 mutex_exit(&spa->spa_condense_stats_lock); 12151 } 12152 12153 void 12154 spa_condense_debug_start(spa_t *spa) 12155 { 12156 uint32_t nitems = 10 + random_in_range(90) * SPA_CONDENSE_DEBUG_STEP; 12157 12158 spa_condense_stat_t *scns = 12159 &spa->spa_condense_stats[SPA_CONDENSE_DEBUG]; 12160 12161 mutex_enter(&spa->spa_condense_stats_lock); 12162 12163 if (scns->scns_start_time == 0 || scns->scns_end_time > 0) { 12164 /* Previous run finished, or no previous run. Start fresh. */ 12165 scns->scns_start_time = gethrestime_sec(); 12166 scns->scns_end_time = 0; 12167 scns->scns_processed = 0; 12168 scns->scns_total = nitems; 12169 } else { 12170 /* In progress, just add some more work. */ 12171 scns->scns_total += nitems; 12172 } 12173 12174 if (spa->spa_condense_debug_tqid == TASKQID_INVALID) { 12175 spa_open_ref(spa, scns); 12176 spa->spa_condense_debug_tqid = taskq_dispatch_delay( 12177 system_delay_taskq, spa_condense_debug_task, spa, TQ_SLEEP, 12178 ddi_get_lbolt() + SEC_TO_TICK(1)); 12179 } 12180 12181 mutex_exit(&spa->spa_condense_stats_lock); 12182 } 12183 12184 void 12185 spa_condense_debug_cancel(spa_t *spa) 12186 { 12187 spa_condense_stat_t *scns = 12188 &spa->spa_condense_stats[SPA_CONDENSE_DEBUG]; 12189 12190 mutex_enter(&spa->spa_condense_stats_lock); 12191 12192 /* "Cancel" by just setting the end time. */ 12193 if (scns->scns_end_time == 0) 12194 scns->scns_end_time = gethrestime_sec(); 12195 12196 if (spa->spa_condense_debug_tqid == TASKQID_INVALID) { 12197 /* No task, so nothing else to do. */ 12198 mutex_exit(&spa->spa_condense_stats_lock); 12199 spa_notify_waiters(spa); 12200 return; 12201 } 12202 12203 /* 12204 * Task is either waiting to run, or running and waiting to take 12205 * spa_condense_stats_lock. Clear the tqid, so if it does run after we 12206 * drop the lock, it will immediately exit. 12207 */ 12208 taskqid_t tqid = spa->spa_condense_debug_tqid; 12209 spa->spa_condense_debug_tqid = TASKQID_INVALID; 12210 12211 mutex_exit(&spa->spa_condense_stats_lock); 12212 12213 /* 12214 * Cancel the task. If its running, wait for it to complete (ie do 12215 * nothing, per above). 12216 */ 12217 taskq_cancel_id(system_delay_taskq, tqid, B_TRUE); 12218 12219 /* 12220 * Task didn't run or aborted, so it never cleaned up. We do it on its 12221 * behalf. 12222 */ 12223 spa_notify_waiters(spa); 12224 spa_close(spa, scns); 12225 } 12226 #endif 12227 12228 /* state manipulation functions */ 12229 EXPORT_SYMBOL(spa_open); 12230 EXPORT_SYMBOL(spa_open_rewind); 12231 EXPORT_SYMBOL(spa_get_stats); 12232 EXPORT_SYMBOL(spa_create); 12233 EXPORT_SYMBOL(spa_import); 12234 EXPORT_SYMBOL(spa_tryimport); 12235 EXPORT_SYMBOL(spa_destroy); 12236 EXPORT_SYMBOL(spa_export); 12237 EXPORT_SYMBOL(spa_reset); 12238 EXPORT_SYMBOL(spa_async_request); 12239 EXPORT_SYMBOL(spa_async_suspend); 12240 EXPORT_SYMBOL(spa_async_resume); 12241 EXPORT_SYMBOL(spa_inject_addref); 12242 EXPORT_SYMBOL(spa_inject_delref); 12243 EXPORT_SYMBOL(spa_scan_stat_init); 12244 EXPORT_SYMBOL(spa_scan_get_stats); 12245 12246 /* device manipulation */ 12247 EXPORT_SYMBOL(spa_vdev_add); 12248 EXPORT_SYMBOL(spa_vdev_attach); 12249 EXPORT_SYMBOL(spa_vdev_detach); 12250 EXPORT_SYMBOL(spa_vdev_setpath); 12251 EXPORT_SYMBOL(spa_vdev_setfru); 12252 EXPORT_SYMBOL(spa_vdev_split_mirror); 12253 12254 /* spare statech is global across all pools) */ 12255 EXPORT_SYMBOL(spa_spare_add); 12256 EXPORT_SYMBOL(spa_spare_remove); 12257 EXPORT_SYMBOL(spa_spare_exists); 12258 EXPORT_SYMBOL(spa_spare_activate); 12259 12260 /* L2ARC statech is global across all pools) */ 12261 EXPORT_SYMBOL(spa_l2cache_add); 12262 EXPORT_SYMBOL(spa_l2cache_remove); 12263 EXPORT_SYMBOL(spa_l2cache_exists); 12264 EXPORT_SYMBOL(spa_l2cache_activate); 12265 EXPORT_SYMBOL(spa_l2cache_drop); 12266 12267 /* scanning */ 12268 EXPORT_SYMBOL(spa_scan); 12269 EXPORT_SYMBOL(spa_scan_range); 12270 EXPORT_SYMBOL(spa_scan_stop); 12271 12272 /* spa syncing */ 12273 EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 12274 EXPORT_SYMBOL(spa_sync_allpools); 12275 12276 /* properties */ 12277 EXPORT_SYMBOL(spa_prop_set); 12278 EXPORT_SYMBOL(spa_prop_get); 12279 EXPORT_SYMBOL(spa_prop_clear_bootfs); 12280 12281 /* asynchronous event notification */ 12282 EXPORT_SYMBOL(spa_event_notify); 12283 12284 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_pct, UINT, ZMOD_RW, 12285 "Percentage of CPUs to run a metaslab preload taskq"); 12286 12287 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, UINT, ZMOD_RW, 12288 "log2 fraction of arc that can be used by inflight I/Os when " 12289 "verifying pool during import"); 12290 12291 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 12292 "Set to traverse metadata on pool import"); 12293 12294 ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 12295 "Set to traverse data on pool import"); 12296 12297 ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 12298 "Print vdev tree to zfs_dbgmsg during pool import"); 12299 12300 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RW, 12301 "Percentage of CPUs to run an IO worker thread"); 12302 12303 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RW, 12304 "Number of threads per IO worker taskqueue"); 12305 12306 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, U64, ZMOD_RW, 12307 "Allow importing pool with up to this number of missing top-level " 12308 "vdevs (in read-only mode)"); 12309 12310 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds_cachefile, U64, ZMOD_RW, 12311 "Allow importing pools with missing top-level vdevs in cache file"); 12312 12313 ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds_scan, U64, ZMOD_RW, 12314 "Allow importing pools with missing top-level vdevs during scan"); 12315 12316 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, 12317 ZMOD_RW, "Set the livelist condense zthr to pause"); 12318 12319 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, 12320 ZMOD_RW, "Set the livelist condense synctask to pause"); 12321 12322 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, 12323 INT, ZMOD_RW, 12324 "Whether livelist condensing was canceled in the synctask"); 12325 12326 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, 12327 INT, ZMOD_RW, 12328 "Whether livelist condensing was canceled in the zthr function"); 12329 12330 ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, 12331 ZMOD_RW, 12332 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 12333 "was being condensed"); 12334 12335 ZFS_MODULE_PARAM(zfs_spa, spa_, note_txg_time, UINT, ZMOD_RW, 12336 "How frequently TXG timestamps are stored internally (in seconds)"); 12337 12338 ZFS_MODULE_PARAM(zfs_spa, spa_, flush_txg_time, UINT, ZMOD_RW, 12339 "How frequently the TXG timestamps database should be flushed " 12340 "to disk (in seconds)"); 12341 12342 #ifdef _KERNEL 12343 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_read, 12344 spa_taskq_read_param_set, spa_taskq_read_param_get, ZMOD_RW, 12345 "Configure IO queues for read IO"); 12346 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_write, 12347 spa_taskq_write_param_set, spa_taskq_write_param_get, ZMOD_RW, 12348 "Configure IO queues for write IO"); 12349 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs_zio, zio_, taskq_free, 12350 spa_taskq_free_param_set, spa_taskq_free_param_get, ZMOD_RW, 12351 "Configure IO queues for free IO"); 12352 #endif 12353 12354 ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_write_tpq, UINT, ZMOD_RW, 12355 "Number of CPUs per write issue taskq"); 12356 12357 ZFS_MODULE_PARAM(zfs, zfs_, ccw_retry_interval, INT, ZMOD_RW, 12358 "Configuration cache file write, retry after failure, interval " 12359 "(seconds)"); 12360