1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 13 /* 14 * Copyright (c) 2016, 2024 by Delphix. All rights reserved. 15 */ 16 17 #include <sys/spa.h> 18 #include <sys/spa_impl.h> 19 #include <sys/txg.h> 20 #include <sys/vdev_impl.h> 21 #include <sys/metaslab_impl.h> 22 #include <sys/dsl_synctask.h> 23 #include <sys/zap.h> 24 #include <sys/dmu_tx.h> 25 #include <sys/vdev_initialize.h> 26 27 /* 28 * Value that is written to disk during initialization. 29 */ 30 static uint64_t zfs_initialize_value = 0xdeadbeefdeadbeeeULL; 31 32 /* maximum number of I/Os outstanding per leaf vdev */ 33 static const int zfs_initialize_limit = 1; 34 35 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */ 36 static uint64_t zfs_initialize_chunk_size = 1024 * 1024; 37 38 static boolean_t 39 vdev_initialize_should_stop(vdev_t *vd) 40 { 41 return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) || 42 vd->vdev_detached || vd->vdev_top->vdev_removing || 43 vd->vdev_top->vdev_rz_expanding); 44 } 45 46 static void 47 vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx) 48 { 49 /* 50 * We pass in the guid instead of the vdev_t since the vdev may 51 * have been freed prior to the sync task being processed. This 52 * happens when a vdev is detached as we call spa_config_vdev_exit(), 53 * stop the initializing thread, schedule the sync task, and free 54 * the vdev. Later when the scheduled sync task is invoked, it would 55 * find that the vdev has been freed. 56 */ 57 uint64_t guid = *(uint64_t *)arg; 58 uint64_t txg = dmu_tx_get_txg(tx); 59 kmem_free(arg, sizeof (uint64_t)); 60 61 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE); 62 if (vd == NULL || vd->vdev_top->vdev_removing || 63 !vdev_is_concrete(vd) || vd->vdev_top->vdev_rz_expanding) 64 return; 65 66 uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK]; 67 vd->vdev_initialize_offset[txg & TXG_MASK] = 0; 68 69 VERIFY(vd->vdev_leaf_zap != 0); 70 71 objset_t *mos = vd->vdev_spa->spa_meta_objset; 72 73 if (last_offset > 0) { 74 vd->vdev_initialize_last_offset = last_offset; 75 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 76 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, 77 sizeof (last_offset), 1, &last_offset, tx)); 78 } 79 if (vd->vdev_initialize_action_time > 0) { 80 uint64_t val = (uint64_t)vd->vdev_initialize_action_time; 81 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 82 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val), 83 1, &val, tx)); 84 } 85 86 uint64_t initialize_state = vd->vdev_initialize_state; 87 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 88 VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1, 89 &initialize_state, tx)); 90 91 uint64_t initialize_value = vd->vdev_initialize_value; 92 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 93 VDEV_LEAF_ZAP_INITIALIZE_VALUE, sizeof (initialize_value), 1, 94 &initialize_value, tx)); 95 } 96 97 static void 98 vdev_initialize_zap_remove_sync(void *arg, dmu_tx_t *tx) 99 { 100 uint64_t guid = *(uint64_t *)arg; 101 102 kmem_free(arg, sizeof (uint64_t)); 103 104 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE); 105 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd)) 106 return; 107 108 ASSERT3S(vd->vdev_initialize_state, ==, VDEV_INITIALIZE_NONE); 109 ASSERT3U(vd->vdev_leaf_zap, !=, 0); 110 111 vd->vdev_initialize_last_offset = 0; 112 vd->vdev_initialize_action_time = 0; 113 114 objset_t *mos = vd->vdev_spa->spa_meta_objset; 115 int error; 116 117 error = zap_remove(mos, vd->vdev_leaf_zap, 118 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, tx); 119 VERIFY(error == 0 || error == ENOENT); 120 121 error = zap_remove(mos, vd->vdev_leaf_zap, 122 VDEV_LEAF_ZAP_INITIALIZE_STATE, tx); 123 VERIFY(error == 0 || error == ENOENT); 124 125 error = zap_remove(mos, vd->vdev_leaf_zap, 126 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, tx); 127 VERIFY(error == 0 || error == ENOENT); 128 129 error = zap_remove(mos, vd->vdev_leaf_zap, 130 VDEV_LEAF_ZAP_INITIALIZE_VALUE, tx); 131 VERIFY(error == 0 || error == ENOENT); 132 } 133 134 static void 135 vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state) 136 { 137 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 138 spa_t *spa = vd->vdev_spa; 139 140 if (new_state == vd->vdev_initialize_state) 141 return; 142 143 /* 144 * Copy the vd's guid, this will be freed by the sync task. 145 */ 146 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 147 *guid = vd->vdev_guid; 148 149 /* 150 * If we're suspending, then preserving the original start time. 151 */ 152 if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) { 153 vd->vdev_initialize_action_time = gethrestime_sec(); 154 } 155 156 vdev_initializing_state_t old_state = vd->vdev_initialize_state; 157 vd->vdev_initialize_state = new_state; 158 159 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 160 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND)); 161 162 if (new_state != VDEV_INITIALIZE_NONE) { 163 dsl_sync_task_nowait(spa_get_dsl(spa), 164 vdev_initialize_zap_update_sync, guid, tx); 165 } else { 166 dsl_sync_task_nowait(spa_get_dsl(spa), 167 vdev_initialize_zap_remove_sync, guid, tx); 168 } 169 170 switch (new_state) { 171 case VDEV_INITIALIZE_ACTIVE: 172 spa_history_log_internal(spa, "initialize", tx, 173 "vdev=%s activated", vd->vdev_path); 174 break; 175 case VDEV_INITIALIZE_SUSPENDED: 176 spa_history_log_internal(spa, "initialize", tx, 177 "vdev=%s suspended", vd->vdev_path); 178 break; 179 case VDEV_INITIALIZE_CANCELED: 180 if (old_state == VDEV_INITIALIZE_ACTIVE || 181 old_state == VDEV_INITIALIZE_SUSPENDED) 182 spa_history_log_internal(spa, "initialize", tx, 183 "vdev=%s canceled", vd->vdev_path); 184 break; 185 case VDEV_INITIALIZE_COMPLETE: 186 spa_history_log_internal(spa, "initialize", tx, 187 "vdev=%s complete", vd->vdev_path); 188 break; 189 case VDEV_INITIALIZE_NONE: 190 spa_history_log_internal(spa, "uninitialize", tx, 191 "vdev=%s", vd->vdev_path); 192 break; 193 default: 194 panic("invalid state %llu", (unsigned long long)new_state); 195 } 196 197 dmu_tx_commit(tx); 198 199 if (new_state != VDEV_INITIALIZE_ACTIVE) 200 spa_notify_waiters(spa); 201 } 202 203 static void 204 vdev_initialize_cb(zio_t *zio) 205 { 206 vdev_t *vd = zio->io_vd; 207 mutex_enter(&vd->vdev_initialize_io_lock); 208 if (zio->io_error == ENXIO && !vdev_writeable(vd)) { 209 /* 210 * The I/O failed because the vdev was unavailable; roll the 211 * last offset back. (This works because spa_sync waits on 212 * spa_txg_zio before it runs sync tasks.) 213 */ 214 uint64_t *off = 215 &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK]; 216 *off = MIN(*off, zio->io_offset); 217 } else { 218 /* 219 * Since initializing is best-effort, we ignore I/O errors and 220 * rely on vdev_probe to determine if the errors are more 221 * critical. 222 */ 223 if (zio->io_error != 0) 224 vd->vdev_stat.vs_initialize_errors++; 225 226 vd->vdev_initialize_bytes_done += zio->io_orig_size; 227 } 228 ASSERT3U(vd->vdev_initialize_inflight, >, 0); 229 vd->vdev_initialize_inflight--; 230 cv_broadcast(&vd->vdev_initialize_io_cv); 231 mutex_exit(&vd->vdev_initialize_io_lock); 232 233 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 234 } 235 236 /* Takes care of physical writing and limiting # of concurrent ZIOs. */ 237 static int 238 vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data) 239 { 240 spa_t *spa = vd->vdev_spa; 241 242 /* Limit inflight initializing I/Os */ 243 mutex_enter(&vd->vdev_initialize_io_lock); 244 while (vd->vdev_initialize_inflight >= zfs_initialize_limit) { 245 cv_wait(&vd->vdev_initialize_io_cv, 246 &vd->vdev_initialize_io_lock); 247 } 248 vd->vdev_initialize_inflight++; 249 mutex_exit(&vd->vdev_initialize_io_lock); 250 251 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 252 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND)); 253 uint64_t txg = dmu_tx_get_txg(tx); 254 255 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER); 256 mutex_enter(&vd->vdev_initialize_lock); 257 258 if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) { 259 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 260 *guid = vd->vdev_guid; 261 262 /* This is the first write of this txg. */ 263 dsl_sync_task_nowait(spa_get_dsl(spa), 264 vdev_initialize_zap_update_sync, guid, tx); 265 } 266 267 /* 268 * We know the vdev struct will still be around since all 269 * consumers of vdev_free must stop the initialization first. 270 */ 271 if (vdev_initialize_should_stop(vd)) { 272 mutex_enter(&vd->vdev_initialize_io_lock); 273 ASSERT3U(vd->vdev_initialize_inflight, >, 0); 274 vd->vdev_initialize_inflight--; 275 mutex_exit(&vd->vdev_initialize_io_lock); 276 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 277 mutex_exit(&vd->vdev_initialize_lock); 278 dmu_tx_commit(tx); 279 return (SET_ERROR(EINTR)); 280 } 281 mutex_exit(&vd->vdev_initialize_lock); 282 283 vd->vdev_initialize_offset[txg & TXG_MASK] = start + size; 284 zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start, 285 size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL, 286 ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE)); 287 /* vdev_initialize_cb releases SCL_STATE_ALL */ 288 289 dmu_tx_commit(tx); 290 291 return (0); 292 } 293 294 /* 295 * Callback to fill each ABD chunk with the requested fill value. len must be 296 * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD 297 * allocation will guarantee these for us. 298 */ 299 static int 300 vdev_initialize_block_fill(void *buf, size_t len, void *arg) 301 { 302 uint64_t value = *(uint64_t *)arg; 303 304 ASSERT0(len % sizeof (uint64_t)); 305 for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) { 306 *(uint64_t *)((char *)(buf) + i) = value; 307 } 308 return (0); 309 } 310 311 static abd_t * 312 vdev_initialize_block_alloc(uint64_t value) 313 { 314 /* Allocate ABD for filler data */ 315 abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE); 316 317 ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t)); 318 (void) abd_iterate_func(data, 0, zfs_initialize_chunk_size, 319 vdev_initialize_block_fill, &value); 320 321 return (data); 322 } 323 324 static void 325 vdev_initialize_block_free(abd_t *data) 326 { 327 abd_free(data); 328 } 329 330 static int 331 vdev_initialize_ranges(vdev_t *vd, abd_t *data) 332 { 333 zfs_range_tree_t *rt = vd->vdev_initialize_tree; 334 zfs_btree_t *bt = &rt->rt_root; 335 zfs_btree_index_t where; 336 337 for (zfs_range_seg_t *rs = zfs_btree_first(bt, &where); rs != NULL; 338 rs = zfs_btree_next(bt, &where, &where)) { 339 uint64_t size = zfs_rs_get_end(rs, rt) - 340 zfs_rs_get_start(rs, rt); 341 342 /* Split range into legally-sized physical chunks */ 343 uint64_t writes_required = 344 ((size - 1) / zfs_initialize_chunk_size) + 1; 345 346 for (uint64_t w = 0; w < writes_required; w++) { 347 int error; 348 349 error = vdev_initialize_write(vd, 350 VDEV_LABEL_START_SIZE + zfs_rs_get_start(rs, rt) + 351 (w * zfs_initialize_chunk_size), 352 MIN(size - (w * zfs_initialize_chunk_size), 353 zfs_initialize_chunk_size), data); 354 if (error != 0) 355 return (error); 356 } 357 } 358 return (0); 359 } 360 361 static void 362 vdev_initialize_xlate_last_rs_end(void *arg, zfs_range_seg64_t *physical_rs) 363 { 364 uint64_t *last_rs_end = (uint64_t *)arg; 365 366 if (physical_rs->rs_end > *last_rs_end) 367 *last_rs_end = physical_rs->rs_end; 368 } 369 370 static void 371 vdev_initialize_xlate_progress(void *arg, zfs_range_seg64_t *physical_rs) 372 { 373 vdev_t *vd = (vdev_t *)arg; 374 375 uint64_t size = physical_rs->rs_end - physical_rs->rs_start; 376 vd->vdev_initialize_bytes_est += size; 377 378 if (vd->vdev_initialize_last_offset > physical_rs->rs_end) { 379 vd->vdev_initialize_bytes_done += size; 380 } else if (vd->vdev_initialize_last_offset > physical_rs->rs_start && 381 vd->vdev_initialize_last_offset < physical_rs->rs_end) { 382 vd->vdev_initialize_bytes_done += 383 vd->vdev_initialize_last_offset - physical_rs->rs_start; 384 } 385 } 386 387 static void 388 vdev_initialize_calculate_progress(vdev_t *vd) 389 { 390 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 391 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 392 ASSERT(vd->vdev_leaf_zap != 0); 393 394 vd->vdev_initialize_bytes_est = 0; 395 vd->vdev_initialize_bytes_done = 0; 396 397 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) { 398 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 399 mutex_enter(&msp->ms_lock); 400 401 uint64_t ms_free = (msp->ms_size - 402 metaslab_allocated_space(msp)) / 403 vdev_get_ndisks(vd->vdev_top); 404 405 /* 406 * Convert the metaslab range to a physical range 407 * on our vdev. We use this to determine if we are 408 * in the middle of this metaslab range. 409 */ 410 zfs_range_seg64_t logical_rs, physical_rs, remain_rs; 411 logical_rs.rs_start = msp->ms_start; 412 logical_rs.rs_end = msp->ms_start + msp->ms_size; 413 414 /* Metaslab space after this offset has not been initialized */ 415 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs); 416 if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) { 417 vd->vdev_initialize_bytes_est += ms_free; 418 mutex_exit(&msp->ms_lock); 419 continue; 420 } 421 422 /* Metaslab space before this offset has been initialized */ 423 uint64_t last_rs_end = physical_rs.rs_end; 424 if (!vdev_xlate_is_empty(&remain_rs)) { 425 vdev_xlate_walk(vd, &remain_rs, 426 vdev_initialize_xlate_last_rs_end, &last_rs_end); 427 } 428 429 if (vd->vdev_initialize_last_offset > last_rs_end) { 430 vd->vdev_initialize_bytes_done += ms_free; 431 vd->vdev_initialize_bytes_est += ms_free; 432 mutex_exit(&msp->ms_lock); 433 continue; 434 } 435 436 /* 437 * If we get here, we're in the middle of initializing this 438 * metaslab. Load it and walk the free tree for more accurate 439 * progress estimation. 440 */ 441 VERIFY0(metaslab_load(msp)); 442 443 zfs_btree_index_t where; 444 zfs_range_tree_t *rt = msp->ms_allocatable; 445 for (zfs_range_seg_t *rs = 446 zfs_btree_first(&rt->rt_root, &where); rs; 447 rs = zfs_btree_next(&rt->rt_root, &where, 448 &where)) { 449 logical_rs.rs_start = zfs_rs_get_start(rs, rt); 450 logical_rs.rs_end = zfs_rs_get_end(rs, rt); 451 452 vdev_xlate_walk(vd, &logical_rs, 453 vdev_initialize_xlate_progress, vd); 454 } 455 mutex_exit(&msp->ms_lock); 456 } 457 } 458 459 static int 460 vdev_initialize_load(vdev_t *vd) 461 { 462 int err = 0; 463 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 464 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 465 ASSERT(vd->vdev_leaf_zap != 0); 466 467 if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE || 468 vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) { 469 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 470 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, 471 sizeof (vd->vdev_initialize_last_offset), 1, 472 &vd->vdev_initialize_last_offset); 473 if (err == ENOENT) { 474 vd->vdev_initialize_last_offset = 0; 475 err = 0; 476 } 477 } 478 479 vdev_initialize_calculate_progress(vd); 480 return (err); 481 } 482 483 static void 484 vdev_initialize_xlate_range_add(void *arg, zfs_range_seg64_t *physical_rs) 485 { 486 vdev_t *vd = arg; 487 488 /* Only add segments that we have not visited yet */ 489 if (physical_rs->rs_end <= vd->vdev_initialize_last_offset) 490 return; 491 492 /* Pick up where we left off mid-range. */ 493 if (vd->vdev_initialize_last_offset > physical_rs->rs_start) { 494 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to " 495 "(%llu, %llu)", vd->vdev_path, 496 (u_longlong_t)physical_rs->rs_start, 497 (u_longlong_t)physical_rs->rs_end, 498 (u_longlong_t)vd->vdev_initialize_last_offset, 499 (u_longlong_t)physical_rs->rs_end); 500 ASSERT3U(physical_rs->rs_end, >, 501 vd->vdev_initialize_last_offset); 502 physical_rs->rs_start = vd->vdev_initialize_last_offset; 503 } 504 505 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start); 506 507 zfs_range_tree_add(vd->vdev_initialize_tree, physical_rs->rs_start, 508 physical_rs->rs_end - physical_rs->rs_start); 509 } 510 511 /* 512 * Convert the logical range into a physical range and add it to our 513 * avl tree. 514 */ 515 static void 516 vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size) 517 { 518 vdev_t *vd = arg; 519 zfs_range_seg64_t logical_rs; 520 logical_rs.rs_start = start; 521 logical_rs.rs_end = start + size; 522 523 ASSERT(vd->vdev_ops->vdev_op_leaf); 524 vdev_xlate_walk(vd, &logical_rs, vdev_initialize_xlate_range_add, arg); 525 } 526 527 static __attribute__((noreturn)) void 528 vdev_initialize_thread(void *arg) 529 { 530 vdev_t *vd = arg; 531 spa_t *spa = vd->vdev_spa; 532 int error = 0; 533 uint64_t ms_count = 0; 534 535 ASSERT(vdev_is_concrete(vd)); 536 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 537 538 vd->vdev_initialize_last_offset = 0; 539 VERIFY0(vdev_initialize_load(vd)); 540 541 abd_t *deadbeef = 542 vdev_initialize_block_alloc(vd->vdev_initialize_value); 543 544 vd->vdev_initialize_tree = zfs_range_tree_create_flags( 545 NULL, ZFS_RANGE_SEG64, NULL, 0, 0, 546 ZFS_RT_F_DYN_NAME, vdev_rt_name(vd, "vdev_initialize_tree")); 547 548 for (uint64_t i = 0; !vd->vdev_detached && 549 i < vd->vdev_top->vdev_ms_count; i++) { 550 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 551 boolean_t unload_when_done = B_FALSE; 552 553 /* 554 * If we've expanded the top-level vdev or it's our 555 * first pass, calculate our progress. 556 */ 557 if (vd->vdev_top->vdev_ms_count != ms_count) { 558 vdev_initialize_calculate_progress(vd); 559 ms_count = vd->vdev_top->vdev_ms_count; 560 } 561 562 spa_config_exit(spa, SCL_CONFIG, FTAG); 563 metaslab_disable(msp); 564 mutex_enter(&msp->ms_lock); 565 if (!msp->ms_loaded && !msp->ms_loading) 566 unload_when_done = B_TRUE; 567 VERIFY0(metaslab_load(msp)); 568 569 zfs_range_tree_walk(msp->ms_allocatable, 570 vdev_initialize_range_add, vd); 571 mutex_exit(&msp->ms_lock); 572 573 error = vdev_initialize_ranges(vd, deadbeef); 574 metaslab_enable(msp, B_TRUE, unload_when_done); 575 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 576 577 zfs_range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL); 578 if (error != 0) 579 break; 580 } 581 582 spa_config_exit(spa, SCL_CONFIG, FTAG); 583 mutex_enter(&vd->vdev_initialize_io_lock); 584 while (vd->vdev_initialize_inflight > 0) { 585 cv_wait(&vd->vdev_initialize_io_cv, 586 &vd->vdev_initialize_io_lock); 587 } 588 mutex_exit(&vd->vdev_initialize_io_lock); 589 590 zfs_range_tree_destroy(vd->vdev_initialize_tree); 591 vdev_initialize_block_free(deadbeef); 592 vd->vdev_initialize_tree = NULL; 593 594 mutex_enter(&vd->vdev_initialize_lock); 595 if (!vd->vdev_initialize_exit_wanted) { 596 if (vdev_writeable(vd)) { 597 vdev_initialize_change_state(vd, 598 VDEV_INITIALIZE_COMPLETE); 599 } else if (vd->vdev_faulted) { 600 vdev_initialize_change_state(vd, 601 VDEV_INITIALIZE_CANCELED); 602 } 603 } 604 ASSERT(vd->vdev_initialize_thread != NULL || 605 vd->vdev_initialize_inflight == 0); 606 607 /* 608 * Drop the vdev_initialize_lock while we sync out the 609 * txg since it's possible that a device might be trying to 610 * come online and must check to see if it needs to restart an 611 * initialization. That thread will be holding the spa_config_lock 612 * which would prevent the txg_wait_synced from completing. 613 */ 614 mutex_exit(&vd->vdev_initialize_lock); 615 txg_wait_synced(spa_get_dsl(spa), 0); 616 mutex_enter(&vd->vdev_initialize_lock); 617 618 vd->vdev_initialize_thread = NULL; 619 cv_broadcast(&vd->vdev_initialize_cv); 620 spa_notify_waiters(spa); 621 mutex_exit(&vd->vdev_initialize_lock); 622 623 thread_exit(); 624 } 625 626 /* 627 * Initiates a device. Caller must hold vdev_initialize_lock. 628 * Device must be a leaf and not already be initializing. 629 */ 630 void 631 vdev_initialize(vdev_t *vd, uint64_t value, boolean_t value_provided) 632 { 633 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 634 ASSERT(vd->vdev_ops->vdev_op_leaf); 635 ASSERT(vdev_is_concrete(vd)); 636 ASSERT0P(vd->vdev_initialize_thread); 637 ASSERT(!vd->vdev_detached); 638 ASSERT(!vd->vdev_initialize_exit_wanted); 639 ASSERT(!vd->vdev_top->vdev_removing); 640 ASSERT(!vd->vdev_top->vdev_rz_expanding); 641 642 /* 643 * Fix the fill value for the whole run so it is stable across a 644 * suspend/resume (it is persisted to the leaf ZAP). An explicit value 645 * always wins. Otherwise, when resuming an existing run keep the value 646 * chosen when it started (already loaded from the leaf ZAP); only a 647 * fresh run falls back to the module default. 648 */ 649 if (value_provided) { 650 vd->vdev_initialize_value = value; 651 } else if (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 652 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) { 653 vd->vdev_initialize_value = zfs_initialize_value; 654 } 655 656 vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE); 657 vd->vdev_initialize_thread = thread_create(NULL, 0, 658 vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri); 659 } 660 661 /* 662 * Uninitializes a device. Caller must hold vdev_initialize_lock. 663 * Device must be a leaf and not already be initializing. 664 */ 665 void 666 vdev_uninitialize(vdev_t *vd) 667 { 668 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 669 ASSERT(vd->vdev_ops->vdev_op_leaf); 670 ASSERT(vdev_is_concrete(vd)); 671 ASSERT0P(vd->vdev_initialize_thread); 672 ASSERT(!vd->vdev_detached); 673 ASSERT(!vd->vdev_initialize_exit_wanted); 674 ASSERT(!vd->vdev_top->vdev_removing); 675 676 vdev_initialize_change_state(vd, VDEV_INITIALIZE_NONE); 677 } 678 679 /* 680 * Wait for the initialize thread to be terminated (cancelled or stopped). 681 */ 682 static void 683 vdev_initialize_stop_wait_impl(vdev_t *vd) 684 { 685 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 686 687 while (vd->vdev_initialize_thread != NULL) 688 cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock); 689 690 ASSERT0P(vd->vdev_initialize_thread); 691 vd->vdev_initialize_exit_wanted = B_FALSE; 692 } 693 694 /* 695 * Wait for vdev initialize threads which were either to cleanly exit. 696 */ 697 void 698 vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list) 699 { 700 (void) spa; 701 vdev_t *vd; 702 703 ASSERT(spa_namespace_held() || 704 spa->spa_export_thread == curthread); 705 706 while ((vd = list_remove_head(vd_list)) != NULL) { 707 mutex_enter(&vd->vdev_initialize_lock); 708 vdev_initialize_stop_wait_impl(vd); 709 mutex_exit(&vd->vdev_initialize_lock); 710 } 711 } 712 713 /* 714 * Stop initializing a device, with the resultant initializing state being 715 * tgt_state. For blocking behavior pass NULL for vd_list. Otherwise, when 716 * a list_t is provided the stopping vdev is inserted in to the list. Callers 717 * are then required to call vdev_initialize_stop_wait() to block for all the 718 * initialization threads to exit. The caller must hold vdev_initialize_lock 719 * and must not be writing to the spa config, as the initializing thread may 720 * try to enter the config as a reader before exiting. 721 */ 722 void 723 vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state, 724 list_t *vd_list) 725 { 726 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER)); 727 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 728 ASSERT(vd->vdev_ops->vdev_op_leaf); 729 ASSERT(vdev_is_concrete(vd)); 730 731 /* 732 * Allow cancel requests to proceed even if the initialize thread 733 * has stopped. 734 */ 735 if (vd->vdev_initialize_thread == NULL && 736 tgt_state != VDEV_INITIALIZE_CANCELED) { 737 return; 738 } 739 740 vdev_initialize_change_state(vd, tgt_state); 741 vd->vdev_initialize_exit_wanted = B_TRUE; 742 743 if (vd_list == NULL) { 744 vdev_initialize_stop_wait_impl(vd); 745 } else { 746 ASSERT(spa_namespace_held() || 747 vd->vdev_spa->spa_export_thread == curthread); 748 list_insert_tail(vd_list, vd); 749 } 750 } 751 752 static void 753 vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state, 754 list_t *vd_list) 755 { 756 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) { 757 mutex_enter(&vd->vdev_initialize_lock); 758 vdev_initialize_stop(vd, tgt_state, vd_list); 759 mutex_exit(&vd->vdev_initialize_lock); 760 return; 761 } 762 763 for (uint64_t i = 0; i < vd->vdev_children; i++) { 764 vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state, 765 vd_list); 766 } 767 } 768 769 /* 770 * Convenience function to stop initializing of a vdev tree and set all 771 * initialize thread pointers to NULL. 772 */ 773 void 774 vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state) 775 { 776 spa_t *spa = vd->vdev_spa; 777 list_t vd_list; 778 779 ASSERT(spa_namespace_held() || 780 spa->spa_export_thread == curthread); 781 782 list_create(&vd_list, sizeof (vdev_t), 783 offsetof(vdev_t, vdev_initialize_node)); 784 785 vdev_initialize_stop_all_impl(vd, tgt_state, &vd_list); 786 vdev_initialize_stop_wait(spa, &vd_list); 787 788 if (vd->vdev_spa->spa_sync_on) { 789 /* Make sure that our state has been synced to disk */ 790 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 791 } 792 793 list_destroy(&vd_list); 794 } 795 796 void 797 vdev_initialize_restart(vdev_t *vd) 798 { 799 ASSERT(spa_namespace_held() || 800 vd->vdev_spa->spa_load_thread == curthread); 801 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER)); 802 803 if (vd->vdev_leaf_zap != 0) { 804 mutex_enter(&vd->vdev_initialize_lock); 805 uint64_t initialize_state = VDEV_INITIALIZE_NONE; 806 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, 807 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE, 808 sizeof (initialize_state), 1, &initialize_state); 809 ASSERT(err == 0 || err == ENOENT); 810 vd->vdev_initialize_state = initialize_state; 811 812 uint64_t timestamp = 0; 813 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 814 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, 815 sizeof (timestamp), 1, ×tamp); 816 ASSERT(err == 0 || err == ENOENT); 817 vd->vdev_initialize_action_time = timestamp; 818 819 /* 820 * Restore the fill value chosen when this run started. Pools 821 * initialized before this field existed fall back to the 822 * module default, preserving their previous behavior. 823 */ 824 uint64_t value = zfs_initialize_value; 825 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 826 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_VALUE, 827 sizeof (value), 1, &value); 828 ASSERT(err == 0 || err == ENOENT); 829 vd->vdev_initialize_value = value; 830 831 if ((vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED || 832 vd->vdev_offline) && !vd->vdev_top->vdev_rz_expanding) { 833 /* load progress for reporting, but don't resume */ 834 VERIFY0(vdev_initialize_load(vd)); 835 } else if (vd->vdev_initialize_state == 836 VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd) && 837 !vd->vdev_top->vdev_removing && 838 !vd->vdev_top->vdev_rz_expanding && 839 vd->vdev_initialize_thread == NULL) { 840 vdev_initialize(vd, value, B_TRUE); 841 } 842 843 mutex_exit(&vd->vdev_initialize_lock); 844 } 845 846 for (uint64_t i = 0; i < vd->vdev_children; i++) { 847 vdev_initialize_restart(vd->vdev_child[i]); 848 } 849 } 850 851 EXPORT_SYMBOL(vdev_initialize); 852 EXPORT_SYMBOL(vdev_uninitialize); 853 EXPORT_SYMBOL(vdev_initialize_stop); 854 EXPORT_SYMBOL(vdev_initialize_stop_all); 855 EXPORT_SYMBOL(vdev_initialize_stop_wait); 856 EXPORT_SYMBOL(vdev_initialize_restart); 857 858 ZFS_MODULE_PARAM(zfs, zfs_, initialize_value, U64, ZMOD_RW, 859 "Value written during zpool initialize"); 860 861 ZFS_MODULE_PARAM(zfs, zfs_, initialize_chunk_size, U64, ZMOD_RW, 862 "Size in bytes of writes by zpool initialize"); 863