1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2016, 2024 by Delphix. All rights reserved. 24 */ 25 26 #include <sys/spa.h> 27 #include <sys/spa_impl.h> 28 #include <sys/txg.h> 29 #include <sys/vdev_impl.h> 30 #include <sys/metaslab_impl.h> 31 #include <sys/dsl_synctask.h> 32 #include <sys/zap.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/vdev_initialize.h> 35 36 /* 37 * Value that is written to disk during initialization. 38 */ 39 static uint64_t zfs_initialize_value = 0xdeadbeefdeadbeeeULL; 40 41 /* maximum number of I/Os outstanding per leaf vdev */ 42 static const int zfs_initialize_limit = 1; 43 44 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */ 45 static uint64_t zfs_initialize_chunk_size = 1024 * 1024; 46 47 static boolean_t 48 vdev_initialize_should_stop(vdev_t *vd) 49 { 50 return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) || 51 vd->vdev_detached || vd->vdev_top->vdev_removing || 52 vd->vdev_top->vdev_rz_expanding); 53 } 54 55 static void 56 vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx) 57 { 58 /* 59 * We pass in the guid instead of the vdev_t since the vdev may 60 * have been freed prior to the sync task being processed. This 61 * happens when a vdev is detached as we call spa_config_vdev_exit(), 62 * stop the initializing thread, schedule the sync task, and free 63 * the vdev. Later when the scheduled sync task is invoked, it would 64 * find that the vdev has been freed. 65 */ 66 uint64_t guid = *(uint64_t *)arg; 67 uint64_t txg = dmu_tx_get_txg(tx); 68 kmem_free(arg, sizeof (uint64_t)); 69 70 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE); 71 if (vd == NULL || vd->vdev_top->vdev_removing || 72 !vdev_is_concrete(vd) || vd->vdev_top->vdev_rz_expanding) 73 return; 74 75 uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK]; 76 vd->vdev_initialize_offset[txg & TXG_MASK] = 0; 77 78 VERIFY(vd->vdev_leaf_zap != 0); 79 80 objset_t *mos = vd->vdev_spa->spa_meta_objset; 81 82 if (last_offset > 0) { 83 vd->vdev_initialize_last_offset = last_offset; 84 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 85 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, 86 sizeof (last_offset), 1, &last_offset, tx)); 87 } 88 if (vd->vdev_initialize_action_time > 0) { 89 uint64_t val = (uint64_t)vd->vdev_initialize_action_time; 90 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 91 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val), 92 1, &val, tx)); 93 } 94 95 uint64_t initialize_state = vd->vdev_initialize_state; 96 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, 97 VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1, 98 &initialize_state, tx)); 99 } 100 101 static void 102 vdev_initialize_zap_remove_sync(void *arg, dmu_tx_t *tx) 103 { 104 uint64_t guid = *(uint64_t *)arg; 105 106 kmem_free(arg, sizeof (uint64_t)); 107 108 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE); 109 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd)) 110 return; 111 112 ASSERT3S(vd->vdev_initialize_state, ==, VDEV_INITIALIZE_NONE); 113 ASSERT3U(vd->vdev_leaf_zap, !=, 0); 114 115 vd->vdev_initialize_last_offset = 0; 116 vd->vdev_initialize_action_time = 0; 117 118 objset_t *mos = vd->vdev_spa->spa_meta_objset; 119 int error; 120 121 error = zap_remove(mos, vd->vdev_leaf_zap, 122 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, tx); 123 VERIFY(error == 0 || error == ENOENT); 124 125 error = zap_remove(mos, vd->vdev_leaf_zap, 126 VDEV_LEAF_ZAP_INITIALIZE_STATE, tx); 127 VERIFY(error == 0 || error == ENOENT); 128 129 error = zap_remove(mos, vd->vdev_leaf_zap, 130 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, tx); 131 VERIFY(error == 0 || error == ENOENT); 132 } 133 134 static void 135 vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state) 136 { 137 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 138 spa_t *spa = vd->vdev_spa; 139 140 if (new_state == vd->vdev_initialize_state) 141 return; 142 143 /* 144 * Copy the vd's guid, this will be freed by the sync task. 145 */ 146 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 147 *guid = vd->vdev_guid; 148 149 /* 150 * If we're suspending, then preserving the original start time. 151 */ 152 if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) { 153 vd->vdev_initialize_action_time = gethrestime_sec(); 154 } 155 156 vdev_initializing_state_t old_state = vd->vdev_initialize_state; 157 vd->vdev_initialize_state = new_state; 158 159 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 160 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 161 162 if (new_state != VDEV_INITIALIZE_NONE) { 163 dsl_sync_task_nowait(spa_get_dsl(spa), 164 vdev_initialize_zap_update_sync, guid, tx); 165 } else { 166 dsl_sync_task_nowait(spa_get_dsl(spa), 167 vdev_initialize_zap_remove_sync, guid, tx); 168 } 169 170 switch (new_state) { 171 case VDEV_INITIALIZE_ACTIVE: 172 spa_history_log_internal(spa, "initialize", tx, 173 "vdev=%s activated", vd->vdev_path); 174 break; 175 case VDEV_INITIALIZE_SUSPENDED: 176 spa_history_log_internal(spa, "initialize", tx, 177 "vdev=%s suspended", vd->vdev_path); 178 break; 179 case VDEV_INITIALIZE_CANCELED: 180 if (old_state == VDEV_INITIALIZE_ACTIVE || 181 old_state == VDEV_INITIALIZE_SUSPENDED) 182 spa_history_log_internal(spa, "initialize", tx, 183 "vdev=%s canceled", vd->vdev_path); 184 break; 185 case VDEV_INITIALIZE_COMPLETE: 186 spa_history_log_internal(spa, "initialize", tx, 187 "vdev=%s complete", vd->vdev_path); 188 break; 189 case VDEV_INITIALIZE_NONE: 190 spa_history_log_internal(spa, "uninitialize", tx, 191 "vdev=%s", vd->vdev_path); 192 break; 193 default: 194 panic("invalid state %llu", (unsigned long long)new_state); 195 } 196 197 dmu_tx_commit(tx); 198 199 if (new_state != VDEV_INITIALIZE_ACTIVE) 200 spa_notify_waiters(spa); 201 } 202 203 static void 204 vdev_initialize_cb(zio_t *zio) 205 { 206 vdev_t *vd = zio->io_vd; 207 mutex_enter(&vd->vdev_initialize_io_lock); 208 if (zio->io_error == ENXIO && !vdev_writeable(vd)) { 209 /* 210 * The I/O failed because the vdev was unavailable; roll the 211 * last offset back. (This works because spa_sync waits on 212 * spa_txg_zio before it runs sync tasks.) 213 */ 214 uint64_t *off = 215 &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK]; 216 *off = MIN(*off, zio->io_offset); 217 } else { 218 /* 219 * Since initializing is best-effort, we ignore I/O errors and 220 * rely on vdev_probe to determine if the errors are more 221 * critical. 222 */ 223 if (zio->io_error != 0) 224 vd->vdev_stat.vs_initialize_errors++; 225 226 vd->vdev_initialize_bytes_done += zio->io_orig_size; 227 } 228 ASSERT3U(vd->vdev_initialize_inflight, >, 0); 229 vd->vdev_initialize_inflight--; 230 cv_broadcast(&vd->vdev_initialize_io_cv); 231 mutex_exit(&vd->vdev_initialize_io_lock); 232 233 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 234 } 235 236 /* Takes care of physical writing and limiting # of concurrent ZIOs. */ 237 static int 238 vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data) 239 { 240 spa_t *spa = vd->vdev_spa; 241 242 /* Limit inflight initializing I/Os */ 243 mutex_enter(&vd->vdev_initialize_io_lock); 244 while (vd->vdev_initialize_inflight >= zfs_initialize_limit) { 245 cv_wait(&vd->vdev_initialize_io_cv, 246 &vd->vdev_initialize_io_lock); 247 } 248 vd->vdev_initialize_inflight++; 249 mutex_exit(&vd->vdev_initialize_io_lock); 250 251 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 252 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 253 uint64_t txg = dmu_tx_get_txg(tx); 254 255 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER); 256 mutex_enter(&vd->vdev_initialize_lock); 257 258 if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) { 259 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); 260 *guid = vd->vdev_guid; 261 262 /* This is the first write of this txg. */ 263 dsl_sync_task_nowait(spa_get_dsl(spa), 264 vdev_initialize_zap_update_sync, guid, tx); 265 } 266 267 /* 268 * We know the vdev struct will still be around since all 269 * consumers of vdev_free must stop the initialization first. 270 */ 271 if (vdev_initialize_should_stop(vd)) { 272 mutex_enter(&vd->vdev_initialize_io_lock); 273 ASSERT3U(vd->vdev_initialize_inflight, >, 0); 274 vd->vdev_initialize_inflight--; 275 mutex_exit(&vd->vdev_initialize_io_lock); 276 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd); 277 mutex_exit(&vd->vdev_initialize_lock); 278 dmu_tx_commit(tx); 279 return (SET_ERROR(EINTR)); 280 } 281 mutex_exit(&vd->vdev_initialize_lock); 282 283 vd->vdev_initialize_offset[txg & TXG_MASK] = start + size; 284 zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start, 285 size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL, 286 ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE)); 287 /* vdev_initialize_cb releases SCL_STATE_ALL */ 288 289 dmu_tx_commit(tx); 290 291 return (0); 292 } 293 294 /* 295 * Callback to fill each ABD chunk with zfs_initialize_value. len must be 296 * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD 297 * allocation will guarantee these for us. 298 */ 299 static int 300 vdev_initialize_block_fill(void *buf, size_t len, void *unused) 301 { 302 (void) unused; 303 304 ASSERT0(len % sizeof (uint64_t)); 305 for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) { 306 *(uint64_t *)((char *)(buf) + i) = zfs_initialize_value; 307 } 308 return (0); 309 } 310 311 static abd_t * 312 vdev_initialize_block_alloc(void) 313 { 314 /* Allocate ABD for filler data */ 315 abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE); 316 317 ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t)); 318 (void) abd_iterate_func(data, 0, zfs_initialize_chunk_size, 319 vdev_initialize_block_fill, NULL); 320 321 return (data); 322 } 323 324 static void 325 vdev_initialize_block_free(abd_t *data) 326 { 327 abd_free(data); 328 } 329 330 static int 331 vdev_initialize_ranges(vdev_t *vd, abd_t *data) 332 { 333 zfs_range_tree_t *rt = vd->vdev_initialize_tree; 334 zfs_btree_t *bt = &rt->rt_root; 335 zfs_btree_index_t where; 336 337 for (zfs_range_seg_t *rs = zfs_btree_first(bt, &where); rs != NULL; 338 rs = zfs_btree_next(bt, &where, &where)) { 339 uint64_t size = zfs_rs_get_end(rs, rt) - 340 zfs_rs_get_start(rs, rt); 341 342 /* Split range into legally-sized physical chunks */ 343 uint64_t writes_required = 344 ((size - 1) / zfs_initialize_chunk_size) + 1; 345 346 for (uint64_t w = 0; w < writes_required; w++) { 347 int error; 348 349 error = vdev_initialize_write(vd, 350 VDEV_LABEL_START_SIZE + zfs_rs_get_start(rs, rt) + 351 (w * zfs_initialize_chunk_size), 352 MIN(size - (w * zfs_initialize_chunk_size), 353 zfs_initialize_chunk_size), data); 354 if (error != 0) 355 return (error); 356 } 357 } 358 return (0); 359 } 360 361 static void 362 vdev_initialize_xlate_last_rs_end(void *arg, zfs_range_seg64_t *physical_rs) 363 { 364 uint64_t *last_rs_end = (uint64_t *)arg; 365 366 if (physical_rs->rs_end > *last_rs_end) 367 *last_rs_end = physical_rs->rs_end; 368 } 369 370 static void 371 vdev_initialize_xlate_progress(void *arg, zfs_range_seg64_t *physical_rs) 372 { 373 vdev_t *vd = (vdev_t *)arg; 374 375 uint64_t size = physical_rs->rs_end - physical_rs->rs_start; 376 vd->vdev_initialize_bytes_est += size; 377 378 if (vd->vdev_initialize_last_offset > physical_rs->rs_end) { 379 vd->vdev_initialize_bytes_done += size; 380 } else if (vd->vdev_initialize_last_offset > physical_rs->rs_start && 381 vd->vdev_initialize_last_offset < physical_rs->rs_end) { 382 vd->vdev_initialize_bytes_done += 383 vd->vdev_initialize_last_offset - physical_rs->rs_start; 384 } 385 } 386 387 static void 388 vdev_initialize_calculate_progress(vdev_t *vd) 389 { 390 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 391 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 392 ASSERT(vd->vdev_leaf_zap != 0); 393 394 vd->vdev_initialize_bytes_est = 0; 395 vd->vdev_initialize_bytes_done = 0; 396 397 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) { 398 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 399 mutex_enter(&msp->ms_lock); 400 401 uint64_t ms_free = (msp->ms_size - 402 metaslab_allocated_space(msp)) / 403 vdev_get_ndisks(vd->vdev_top); 404 405 /* 406 * Convert the metaslab range to a physical range 407 * on our vdev. We use this to determine if we are 408 * in the middle of this metaslab range. 409 */ 410 zfs_range_seg64_t logical_rs, physical_rs, remain_rs; 411 logical_rs.rs_start = msp->ms_start; 412 logical_rs.rs_end = msp->ms_start + msp->ms_size; 413 414 /* Metaslab space after this offset has not been initialized */ 415 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs); 416 if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) { 417 vd->vdev_initialize_bytes_est += ms_free; 418 mutex_exit(&msp->ms_lock); 419 continue; 420 } 421 422 /* Metaslab space before this offset has been initialized */ 423 uint64_t last_rs_end = physical_rs.rs_end; 424 if (!vdev_xlate_is_empty(&remain_rs)) { 425 vdev_xlate_walk(vd, &remain_rs, 426 vdev_initialize_xlate_last_rs_end, &last_rs_end); 427 } 428 429 if (vd->vdev_initialize_last_offset > last_rs_end) { 430 vd->vdev_initialize_bytes_done += ms_free; 431 vd->vdev_initialize_bytes_est += ms_free; 432 mutex_exit(&msp->ms_lock); 433 continue; 434 } 435 436 /* 437 * If we get here, we're in the middle of initializing this 438 * metaslab. Load it and walk the free tree for more accurate 439 * progress estimation. 440 */ 441 VERIFY0(metaslab_load(msp)); 442 443 zfs_btree_index_t where; 444 zfs_range_tree_t *rt = msp->ms_allocatable; 445 for (zfs_range_seg_t *rs = 446 zfs_btree_first(&rt->rt_root, &where); rs; 447 rs = zfs_btree_next(&rt->rt_root, &where, 448 &where)) { 449 logical_rs.rs_start = zfs_rs_get_start(rs, rt); 450 logical_rs.rs_end = zfs_rs_get_end(rs, rt); 451 452 vdev_xlate_walk(vd, &logical_rs, 453 vdev_initialize_xlate_progress, vd); 454 } 455 mutex_exit(&msp->ms_lock); 456 } 457 } 458 459 static int 460 vdev_initialize_load(vdev_t *vd) 461 { 462 int err = 0; 463 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) || 464 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER)); 465 ASSERT(vd->vdev_leaf_zap != 0); 466 467 if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE || 468 vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) { 469 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 470 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, 471 sizeof (vd->vdev_initialize_last_offset), 1, 472 &vd->vdev_initialize_last_offset); 473 if (err == ENOENT) { 474 vd->vdev_initialize_last_offset = 0; 475 err = 0; 476 } 477 } 478 479 vdev_initialize_calculate_progress(vd); 480 return (err); 481 } 482 483 static void 484 vdev_initialize_xlate_range_add(void *arg, zfs_range_seg64_t *physical_rs) 485 { 486 vdev_t *vd = arg; 487 488 /* Only add segments that we have not visited yet */ 489 if (physical_rs->rs_end <= vd->vdev_initialize_last_offset) 490 return; 491 492 /* Pick up where we left off mid-range. */ 493 if (vd->vdev_initialize_last_offset > physical_rs->rs_start) { 494 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to " 495 "(%llu, %llu)", vd->vdev_path, 496 (u_longlong_t)physical_rs->rs_start, 497 (u_longlong_t)physical_rs->rs_end, 498 (u_longlong_t)vd->vdev_initialize_last_offset, 499 (u_longlong_t)physical_rs->rs_end); 500 ASSERT3U(physical_rs->rs_end, >, 501 vd->vdev_initialize_last_offset); 502 physical_rs->rs_start = vd->vdev_initialize_last_offset; 503 } 504 505 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start); 506 507 zfs_range_tree_add(vd->vdev_initialize_tree, physical_rs->rs_start, 508 physical_rs->rs_end - physical_rs->rs_start); 509 } 510 511 /* 512 * Convert the logical range into a physical range and add it to our 513 * avl tree. 514 */ 515 static void 516 vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size) 517 { 518 vdev_t *vd = arg; 519 zfs_range_seg64_t logical_rs; 520 logical_rs.rs_start = start; 521 logical_rs.rs_end = start + size; 522 523 ASSERT(vd->vdev_ops->vdev_op_leaf); 524 vdev_xlate_walk(vd, &logical_rs, vdev_initialize_xlate_range_add, arg); 525 } 526 527 static __attribute__((noreturn)) void 528 vdev_initialize_thread(void *arg) 529 { 530 vdev_t *vd = arg; 531 spa_t *spa = vd->vdev_spa; 532 int error = 0; 533 uint64_t ms_count = 0; 534 535 ASSERT(vdev_is_concrete(vd)); 536 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 537 538 vd->vdev_initialize_last_offset = 0; 539 VERIFY0(vdev_initialize_load(vd)); 540 541 abd_t *deadbeef = vdev_initialize_block_alloc(); 542 543 vd->vdev_initialize_tree = zfs_range_tree_create(NULL, ZFS_RANGE_SEG64, 544 NULL, 0, 0); 545 546 for (uint64_t i = 0; !vd->vdev_detached && 547 i < vd->vdev_top->vdev_ms_count; i++) { 548 metaslab_t *msp = vd->vdev_top->vdev_ms[i]; 549 boolean_t unload_when_done = B_FALSE; 550 551 /* 552 * If we've expanded the top-level vdev or it's our 553 * first pass, calculate our progress. 554 */ 555 if (vd->vdev_top->vdev_ms_count != ms_count) { 556 vdev_initialize_calculate_progress(vd); 557 ms_count = vd->vdev_top->vdev_ms_count; 558 } 559 560 spa_config_exit(spa, SCL_CONFIG, FTAG); 561 metaslab_disable(msp); 562 mutex_enter(&msp->ms_lock); 563 if (!msp->ms_loaded && !msp->ms_loading) 564 unload_when_done = B_TRUE; 565 VERIFY0(metaslab_load(msp)); 566 567 zfs_range_tree_walk(msp->ms_allocatable, 568 vdev_initialize_range_add, vd); 569 mutex_exit(&msp->ms_lock); 570 571 error = vdev_initialize_ranges(vd, deadbeef); 572 metaslab_enable(msp, B_TRUE, unload_when_done); 573 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 574 575 zfs_range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL); 576 if (error != 0) 577 break; 578 } 579 580 spa_config_exit(spa, SCL_CONFIG, FTAG); 581 mutex_enter(&vd->vdev_initialize_io_lock); 582 while (vd->vdev_initialize_inflight > 0) { 583 cv_wait(&vd->vdev_initialize_io_cv, 584 &vd->vdev_initialize_io_lock); 585 } 586 mutex_exit(&vd->vdev_initialize_io_lock); 587 588 zfs_range_tree_destroy(vd->vdev_initialize_tree); 589 vdev_initialize_block_free(deadbeef); 590 vd->vdev_initialize_tree = NULL; 591 592 mutex_enter(&vd->vdev_initialize_lock); 593 if (!vd->vdev_initialize_exit_wanted) { 594 if (vdev_writeable(vd)) { 595 vdev_initialize_change_state(vd, 596 VDEV_INITIALIZE_COMPLETE); 597 } else if (vd->vdev_faulted) { 598 vdev_initialize_change_state(vd, 599 VDEV_INITIALIZE_CANCELED); 600 } 601 } 602 ASSERT(vd->vdev_initialize_thread != NULL || 603 vd->vdev_initialize_inflight == 0); 604 605 /* 606 * Drop the vdev_initialize_lock while we sync out the 607 * txg since it's possible that a device might be trying to 608 * come online and must check to see if it needs to restart an 609 * initialization. That thread will be holding the spa_config_lock 610 * which would prevent the txg_wait_synced from completing. 611 */ 612 mutex_exit(&vd->vdev_initialize_lock); 613 txg_wait_synced(spa_get_dsl(spa), 0); 614 mutex_enter(&vd->vdev_initialize_lock); 615 616 vd->vdev_initialize_thread = NULL; 617 cv_broadcast(&vd->vdev_initialize_cv); 618 mutex_exit(&vd->vdev_initialize_lock); 619 620 thread_exit(); 621 } 622 623 /* 624 * Initiates a device. Caller must hold vdev_initialize_lock. 625 * Device must be a leaf and not already be initializing. 626 */ 627 void 628 vdev_initialize(vdev_t *vd) 629 { 630 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 631 ASSERT(vd->vdev_ops->vdev_op_leaf); 632 ASSERT(vdev_is_concrete(vd)); 633 ASSERT3P(vd->vdev_initialize_thread, ==, NULL); 634 ASSERT(!vd->vdev_detached); 635 ASSERT(!vd->vdev_initialize_exit_wanted); 636 ASSERT(!vd->vdev_top->vdev_removing); 637 ASSERT(!vd->vdev_top->vdev_rz_expanding); 638 639 vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE); 640 vd->vdev_initialize_thread = thread_create(NULL, 0, 641 vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri); 642 } 643 644 /* 645 * Uninitializes a device. Caller must hold vdev_initialize_lock. 646 * Device must be a leaf and not already be initializing. 647 */ 648 void 649 vdev_uninitialize(vdev_t *vd) 650 { 651 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 652 ASSERT(vd->vdev_ops->vdev_op_leaf); 653 ASSERT(vdev_is_concrete(vd)); 654 ASSERT3P(vd->vdev_initialize_thread, ==, NULL); 655 ASSERT(!vd->vdev_detached); 656 ASSERT(!vd->vdev_initialize_exit_wanted); 657 ASSERT(!vd->vdev_top->vdev_removing); 658 659 vdev_initialize_change_state(vd, VDEV_INITIALIZE_NONE); 660 } 661 662 /* 663 * Wait for the initialize thread to be terminated (cancelled or stopped). 664 */ 665 static void 666 vdev_initialize_stop_wait_impl(vdev_t *vd) 667 { 668 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 669 670 while (vd->vdev_initialize_thread != NULL) 671 cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock); 672 673 ASSERT3P(vd->vdev_initialize_thread, ==, NULL); 674 vd->vdev_initialize_exit_wanted = B_FALSE; 675 } 676 677 /* 678 * Wait for vdev initialize threads which were either to cleanly exit. 679 */ 680 void 681 vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list) 682 { 683 (void) spa; 684 vdev_t *vd; 685 686 ASSERT(MUTEX_HELD(&spa_namespace_lock) || 687 spa->spa_export_thread == curthread); 688 689 while ((vd = list_remove_head(vd_list)) != NULL) { 690 mutex_enter(&vd->vdev_initialize_lock); 691 vdev_initialize_stop_wait_impl(vd); 692 mutex_exit(&vd->vdev_initialize_lock); 693 } 694 } 695 696 /* 697 * Stop initializing a device, with the resultant initializing state being 698 * tgt_state. For blocking behavior pass NULL for vd_list. Otherwise, when 699 * a list_t is provided the stopping vdev is inserted in to the list. Callers 700 * are then required to call vdev_initialize_stop_wait() to block for all the 701 * initialization threads to exit. The caller must hold vdev_initialize_lock 702 * and must not be writing to the spa config, as the initializing thread may 703 * try to enter the config as a reader before exiting. 704 */ 705 void 706 vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state, 707 list_t *vd_list) 708 { 709 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER)); 710 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock)); 711 ASSERT(vd->vdev_ops->vdev_op_leaf); 712 ASSERT(vdev_is_concrete(vd)); 713 714 /* 715 * Allow cancel requests to proceed even if the initialize thread 716 * has stopped. 717 */ 718 if (vd->vdev_initialize_thread == NULL && 719 tgt_state != VDEV_INITIALIZE_CANCELED) { 720 return; 721 } 722 723 vdev_initialize_change_state(vd, tgt_state); 724 vd->vdev_initialize_exit_wanted = B_TRUE; 725 726 if (vd_list == NULL) { 727 vdev_initialize_stop_wait_impl(vd); 728 } else { 729 ASSERT(MUTEX_HELD(&spa_namespace_lock) || 730 vd->vdev_spa->spa_export_thread == curthread); 731 list_insert_tail(vd_list, vd); 732 } 733 } 734 735 static void 736 vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state, 737 list_t *vd_list) 738 { 739 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) { 740 mutex_enter(&vd->vdev_initialize_lock); 741 vdev_initialize_stop(vd, tgt_state, vd_list); 742 mutex_exit(&vd->vdev_initialize_lock); 743 return; 744 } 745 746 for (uint64_t i = 0; i < vd->vdev_children; i++) { 747 vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state, 748 vd_list); 749 } 750 } 751 752 /* 753 * Convenience function to stop initializing of a vdev tree and set all 754 * initialize thread pointers to NULL. 755 */ 756 void 757 vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state) 758 { 759 spa_t *spa = vd->vdev_spa; 760 list_t vd_list; 761 762 ASSERT(MUTEX_HELD(&spa_namespace_lock) || 763 spa->spa_export_thread == curthread); 764 765 list_create(&vd_list, sizeof (vdev_t), 766 offsetof(vdev_t, vdev_initialize_node)); 767 768 vdev_initialize_stop_all_impl(vd, tgt_state, &vd_list); 769 vdev_initialize_stop_wait(spa, &vd_list); 770 771 if (vd->vdev_spa->spa_sync_on) { 772 /* Make sure that our state has been synced to disk */ 773 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0); 774 } 775 776 list_destroy(&vd_list); 777 } 778 779 void 780 vdev_initialize_restart(vdev_t *vd) 781 { 782 ASSERT(MUTEX_HELD(&spa_namespace_lock) || 783 vd->vdev_spa->spa_load_thread == curthread); 784 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER)); 785 786 if (vd->vdev_leaf_zap != 0) { 787 mutex_enter(&vd->vdev_initialize_lock); 788 uint64_t initialize_state = VDEV_INITIALIZE_NONE; 789 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, 790 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE, 791 sizeof (initialize_state), 1, &initialize_state); 792 ASSERT(err == 0 || err == ENOENT); 793 vd->vdev_initialize_state = initialize_state; 794 795 uint64_t timestamp = 0; 796 err = zap_lookup(vd->vdev_spa->spa_meta_objset, 797 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, 798 sizeof (timestamp), 1, ×tamp); 799 ASSERT(err == 0 || err == ENOENT); 800 vd->vdev_initialize_action_time = timestamp; 801 802 if ((vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED || 803 vd->vdev_offline) && !vd->vdev_top->vdev_rz_expanding) { 804 /* load progress for reporting, but don't resume */ 805 VERIFY0(vdev_initialize_load(vd)); 806 } else if (vd->vdev_initialize_state == 807 VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd) && 808 !vd->vdev_top->vdev_removing && 809 !vd->vdev_top->vdev_rz_expanding && 810 vd->vdev_initialize_thread == NULL) { 811 vdev_initialize(vd); 812 } 813 814 mutex_exit(&vd->vdev_initialize_lock); 815 } 816 817 for (uint64_t i = 0; i < vd->vdev_children; i++) { 818 vdev_initialize_restart(vd->vdev_child[i]); 819 } 820 } 821 822 EXPORT_SYMBOL(vdev_initialize); 823 EXPORT_SYMBOL(vdev_uninitialize); 824 EXPORT_SYMBOL(vdev_initialize_stop); 825 EXPORT_SYMBOL(vdev_initialize_stop_all); 826 EXPORT_SYMBOL(vdev_initialize_stop_wait); 827 EXPORT_SYMBOL(vdev_initialize_restart); 828 829 ZFS_MODULE_PARAM(zfs, zfs_, initialize_value, U64, ZMOD_RW, 830 "Value written during zpool initialize"); 831 832 ZFS_MODULE_PARAM(zfs, zfs_, initialize_chunk_size, U64, ZMOD_RW, 833 "Size in bytes of writes by zpool initialize"); 834