1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12
13 /*
14 * Copyright (c) 2016, 2024 by Delphix. All rights reserved.
15 */
16
17 #include <sys/spa.h>
18 #include <sys/spa_impl.h>
19 #include <sys/txg.h>
20 #include <sys/vdev_impl.h>
21 #include <sys/metaslab_impl.h>
22 #include <sys/dsl_synctask.h>
23 #include <sys/zap.h>
24 #include <sys/dmu_tx.h>
25 #include <sys/vdev_initialize.h>
26
27 /*
28 * Value that is written to disk during initialization.
29 */
30 static uint64_t zfs_initialize_value = 0xdeadbeefdeadbeeeULL;
31
32 /* maximum number of I/Os outstanding per leaf vdev */
33 static const int zfs_initialize_limit = 1;
34
35 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */
36 static uint64_t zfs_initialize_chunk_size = 1024 * 1024;
37
38 static boolean_t
vdev_initialize_should_stop(vdev_t * vd)39 vdev_initialize_should_stop(vdev_t *vd)
40 {
41 return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) ||
42 vd->vdev_detached || vd->vdev_top->vdev_removing ||
43 vd->vdev_top->vdev_rz_expanding);
44 }
45
46 static void
vdev_initialize_zap_update_sync(void * arg,dmu_tx_t * tx)47 vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx)
48 {
49 /*
50 * We pass in the guid instead of the vdev_t since the vdev may
51 * have been freed prior to the sync task being processed. This
52 * happens when a vdev is detached as we call spa_config_vdev_exit(),
53 * stop the initializing thread, schedule the sync task, and free
54 * the vdev. Later when the scheduled sync task is invoked, it would
55 * find that the vdev has been freed.
56 */
57 uint64_t guid = *(uint64_t *)arg;
58 uint64_t txg = dmu_tx_get_txg(tx);
59 kmem_free(arg, sizeof (uint64_t));
60
61 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
62 if (vd == NULL || vd->vdev_top->vdev_removing ||
63 !vdev_is_concrete(vd) || vd->vdev_top->vdev_rz_expanding)
64 return;
65
66 uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK];
67 vd->vdev_initialize_offset[txg & TXG_MASK] = 0;
68
69 VERIFY(vd->vdev_leaf_zap != 0);
70
71 objset_t *mos = vd->vdev_spa->spa_meta_objset;
72
73 if (last_offset > 0) {
74 vd->vdev_initialize_last_offset = last_offset;
75 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
76 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
77 sizeof (last_offset), 1, &last_offset, tx));
78 }
79 if (vd->vdev_initialize_action_time > 0) {
80 uint64_t val = (uint64_t)vd->vdev_initialize_action_time;
81 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
82 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val),
83 1, &val, tx));
84 }
85
86 uint64_t initialize_state = vd->vdev_initialize_state;
87 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
88 VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1,
89 &initialize_state, tx));
90
91 uint64_t initialize_value = vd->vdev_initialize_value;
92 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
93 VDEV_LEAF_ZAP_INITIALIZE_VALUE, sizeof (initialize_value), 1,
94 &initialize_value, tx));
95 }
96
97 static void
vdev_initialize_zap_remove_sync(void * arg,dmu_tx_t * tx)98 vdev_initialize_zap_remove_sync(void *arg, dmu_tx_t *tx)
99 {
100 uint64_t guid = *(uint64_t *)arg;
101
102 kmem_free(arg, sizeof (uint64_t));
103
104 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
105 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
106 return;
107
108 ASSERT3S(vd->vdev_initialize_state, ==, VDEV_INITIALIZE_NONE);
109 ASSERT3U(vd->vdev_leaf_zap, !=, 0);
110
111 vd->vdev_initialize_last_offset = 0;
112 vd->vdev_initialize_action_time = 0;
113
114 objset_t *mos = vd->vdev_spa->spa_meta_objset;
115 int error;
116
117 error = zap_remove(mos, vd->vdev_leaf_zap,
118 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET, tx);
119 VERIFY(error == 0 || error == ENOENT);
120
121 error = zap_remove(mos, vd->vdev_leaf_zap,
122 VDEV_LEAF_ZAP_INITIALIZE_STATE, tx);
123 VERIFY(error == 0 || error == ENOENT);
124
125 error = zap_remove(mos, vd->vdev_leaf_zap,
126 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, tx);
127 VERIFY(error == 0 || error == ENOENT);
128
129 error = zap_remove(mos, vd->vdev_leaf_zap,
130 VDEV_LEAF_ZAP_INITIALIZE_VALUE, tx);
131 VERIFY(error == 0 || error == ENOENT);
132 }
133
134 static void
vdev_initialize_change_state(vdev_t * vd,vdev_initializing_state_t new_state)135 vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state)
136 {
137 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
138 spa_t *spa = vd->vdev_spa;
139
140 if (new_state == vd->vdev_initialize_state)
141 return;
142
143 /*
144 * Copy the vd's guid, this will be freed by the sync task.
145 */
146 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
147 *guid = vd->vdev_guid;
148
149 /*
150 * If we're suspending, then preserving the original start time.
151 */
152 if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) {
153 vd->vdev_initialize_action_time = gethrestime_sec();
154 }
155
156 vdev_initializing_state_t old_state = vd->vdev_initialize_state;
157 vd->vdev_initialize_state = new_state;
158
159 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
160 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND));
161
162 if (new_state != VDEV_INITIALIZE_NONE) {
163 dsl_sync_task_nowait(spa_get_dsl(spa),
164 vdev_initialize_zap_update_sync, guid, tx);
165 } else {
166 dsl_sync_task_nowait(spa_get_dsl(spa),
167 vdev_initialize_zap_remove_sync, guid, tx);
168 }
169
170 switch (new_state) {
171 case VDEV_INITIALIZE_ACTIVE:
172 spa_history_log_internal(spa, "initialize", tx,
173 "vdev=%s activated", vd->vdev_path);
174 break;
175 case VDEV_INITIALIZE_SUSPENDED:
176 spa_history_log_internal(spa, "initialize", tx,
177 "vdev=%s suspended", vd->vdev_path);
178 break;
179 case VDEV_INITIALIZE_CANCELED:
180 if (old_state == VDEV_INITIALIZE_ACTIVE ||
181 old_state == VDEV_INITIALIZE_SUSPENDED)
182 spa_history_log_internal(spa, "initialize", tx,
183 "vdev=%s canceled", vd->vdev_path);
184 break;
185 case VDEV_INITIALIZE_COMPLETE:
186 spa_history_log_internal(spa, "initialize", tx,
187 "vdev=%s complete", vd->vdev_path);
188 break;
189 case VDEV_INITIALIZE_NONE:
190 spa_history_log_internal(spa, "uninitialize", tx,
191 "vdev=%s", vd->vdev_path);
192 break;
193 default:
194 panic("invalid state %llu", (unsigned long long)new_state);
195 }
196
197 dmu_tx_commit(tx);
198
199 if (new_state != VDEV_INITIALIZE_ACTIVE)
200 spa_notify_waiters(spa);
201 }
202
203 static void
vdev_initialize_cb(zio_t * zio)204 vdev_initialize_cb(zio_t *zio)
205 {
206 vdev_t *vd = zio->io_vd;
207 mutex_enter(&vd->vdev_initialize_io_lock);
208 if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
209 /*
210 * The I/O failed because the vdev was unavailable; roll the
211 * last offset back. (This works because spa_sync waits on
212 * spa_txg_zio before it runs sync tasks.)
213 */
214 uint64_t *off =
215 &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK];
216 *off = MIN(*off, zio->io_offset);
217 } else {
218 /*
219 * Since initializing is best-effort, we ignore I/O errors and
220 * rely on vdev_probe to determine if the errors are more
221 * critical.
222 */
223 if (zio->io_error != 0)
224 vd->vdev_stat.vs_initialize_errors++;
225
226 vd->vdev_initialize_bytes_done += zio->io_orig_size;
227 }
228 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
229 vd->vdev_initialize_inflight--;
230 cv_broadcast(&vd->vdev_initialize_io_cv);
231 mutex_exit(&vd->vdev_initialize_io_lock);
232
233 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
234 }
235
236 /* Takes care of physical writing and limiting # of concurrent ZIOs. */
237 static int
vdev_initialize_write(vdev_t * vd,uint64_t start,uint64_t size,abd_t * data)238 vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data)
239 {
240 spa_t *spa = vd->vdev_spa;
241
242 /* Limit inflight initializing I/Os */
243 mutex_enter(&vd->vdev_initialize_io_lock);
244 while (vd->vdev_initialize_inflight >= zfs_initialize_limit) {
245 cv_wait(&vd->vdev_initialize_io_cv,
246 &vd->vdev_initialize_io_lock);
247 }
248 vd->vdev_initialize_inflight++;
249 mutex_exit(&vd->vdev_initialize_io_lock);
250
251 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
252 VERIFY0(dmu_tx_assign(tx, DMU_TX_WAIT | DMU_TX_SUSPEND));
253 uint64_t txg = dmu_tx_get_txg(tx);
254
255 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
256 mutex_enter(&vd->vdev_initialize_lock);
257
258 if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) {
259 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
260 *guid = vd->vdev_guid;
261
262 /* This is the first write of this txg. */
263 dsl_sync_task_nowait(spa_get_dsl(spa),
264 vdev_initialize_zap_update_sync, guid, tx);
265 }
266
267 /*
268 * We know the vdev struct will still be around since all
269 * consumers of vdev_free must stop the initialization first.
270 */
271 if (vdev_initialize_should_stop(vd)) {
272 mutex_enter(&vd->vdev_initialize_io_lock);
273 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
274 vd->vdev_initialize_inflight--;
275 mutex_exit(&vd->vdev_initialize_io_lock);
276 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
277 mutex_exit(&vd->vdev_initialize_lock);
278 dmu_tx_commit(tx);
279 return (SET_ERROR(EINTR));
280 }
281 mutex_exit(&vd->vdev_initialize_lock);
282
283 vd->vdev_initialize_offset[txg & TXG_MASK] = start + size;
284 zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start,
285 size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL,
286 ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE));
287 /* vdev_initialize_cb releases SCL_STATE_ALL */
288
289 dmu_tx_commit(tx);
290
291 return (0);
292 }
293
294 /*
295 * Callback to fill each ABD chunk with the requested fill value. len must be
296 * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD
297 * allocation will guarantee these for us.
298 */
299 static int
vdev_initialize_block_fill(void * buf,size_t len,void * arg)300 vdev_initialize_block_fill(void *buf, size_t len, void *arg)
301 {
302 uint64_t value = *(uint64_t *)arg;
303
304 ASSERT0(len % sizeof (uint64_t));
305 for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) {
306 *(uint64_t *)((char *)(buf) + i) = value;
307 }
308 return (0);
309 }
310
311 static abd_t *
vdev_initialize_block_alloc(uint64_t value)312 vdev_initialize_block_alloc(uint64_t value)
313 {
314 /* Allocate ABD for filler data */
315 abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE);
316
317 ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t));
318 (void) abd_iterate_func(data, 0, zfs_initialize_chunk_size,
319 vdev_initialize_block_fill, &value);
320
321 return (data);
322 }
323
324 static void
vdev_initialize_block_free(abd_t * data)325 vdev_initialize_block_free(abd_t *data)
326 {
327 abd_free(data);
328 }
329
330 static int
vdev_initialize_ranges(vdev_t * vd,abd_t * data)331 vdev_initialize_ranges(vdev_t *vd, abd_t *data)
332 {
333 zfs_range_tree_t *rt = vd->vdev_initialize_tree;
334 zfs_btree_t *bt = &rt->rt_root;
335 zfs_btree_index_t where;
336
337 for (zfs_range_seg_t *rs = zfs_btree_first(bt, &where); rs != NULL;
338 rs = zfs_btree_next(bt, &where, &where)) {
339 uint64_t size = zfs_rs_get_end(rs, rt) -
340 zfs_rs_get_start(rs, rt);
341
342 /* Split range into legally-sized physical chunks */
343 uint64_t writes_required =
344 ((size - 1) / zfs_initialize_chunk_size) + 1;
345
346 for (uint64_t w = 0; w < writes_required; w++) {
347 int error;
348
349 error = vdev_initialize_write(vd,
350 VDEV_LABEL_START_SIZE + zfs_rs_get_start(rs, rt) +
351 (w * zfs_initialize_chunk_size),
352 MIN(size - (w * zfs_initialize_chunk_size),
353 zfs_initialize_chunk_size), data);
354 if (error != 0)
355 return (error);
356 }
357 }
358 return (0);
359 }
360
361 static void
vdev_initialize_xlate_last_rs_end(void * arg,zfs_range_seg64_t * physical_rs)362 vdev_initialize_xlate_last_rs_end(void *arg, zfs_range_seg64_t *physical_rs)
363 {
364 uint64_t *last_rs_end = (uint64_t *)arg;
365
366 if (physical_rs->rs_end > *last_rs_end)
367 *last_rs_end = physical_rs->rs_end;
368 }
369
370 static void
vdev_initialize_xlate_progress(void * arg,zfs_range_seg64_t * physical_rs)371 vdev_initialize_xlate_progress(void *arg, zfs_range_seg64_t *physical_rs)
372 {
373 vdev_t *vd = (vdev_t *)arg;
374
375 uint64_t size = physical_rs->rs_end - physical_rs->rs_start;
376 vd->vdev_initialize_bytes_est += size;
377
378 if (vd->vdev_initialize_last_offset > physical_rs->rs_end) {
379 vd->vdev_initialize_bytes_done += size;
380 } else if (vd->vdev_initialize_last_offset > physical_rs->rs_start &&
381 vd->vdev_initialize_last_offset < physical_rs->rs_end) {
382 vd->vdev_initialize_bytes_done +=
383 vd->vdev_initialize_last_offset - physical_rs->rs_start;
384 }
385 }
386
387 static void
vdev_initialize_calculate_progress(vdev_t * vd)388 vdev_initialize_calculate_progress(vdev_t *vd)
389 {
390 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
391 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
392 ASSERT(vd->vdev_leaf_zap != 0);
393
394 vd->vdev_initialize_bytes_est = 0;
395 vd->vdev_initialize_bytes_done = 0;
396
397 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
398 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
399 mutex_enter(&msp->ms_lock);
400
401 uint64_t ms_free = (msp->ms_size -
402 metaslab_allocated_space(msp)) /
403 vdev_get_ndisks(vd->vdev_top);
404
405 /*
406 * Convert the metaslab range to a physical range
407 * on our vdev. We use this to determine if we are
408 * in the middle of this metaslab range.
409 */
410 zfs_range_seg64_t logical_rs, physical_rs, remain_rs;
411 logical_rs.rs_start = msp->ms_start;
412 logical_rs.rs_end = msp->ms_start + msp->ms_size;
413
414 /* Metaslab space after this offset has not been initialized */
415 vdev_xlate(vd, &logical_rs, &physical_rs, &remain_rs);
416 if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) {
417 vd->vdev_initialize_bytes_est += ms_free;
418 mutex_exit(&msp->ms_lock);
419 continue;
420 }
421
422 /* Metaslab space before this offset has been initialized */
423 uint64_t last_rs_end = physical_rs.rs_end;
424 if (!vdev_xlate_is_empty(&remain_rs)) {
425 vdev_xlate_walk(vd, &remain_rs,
426 vdev_initialize_xlate_last_rs_end, &last_rs_end);
427 }
428
429 if (vd->vdev_initialize_last_offset > last_rs_end) {
430 vd->vdev_initialize_bytes_done += ms_free;
431 vd->vdev_initialize_bytes_est += ms_free;
432 mutex_exit(&msp->ms_lock);
433 continue;
434 }
435
436 /*
437 * If we get here, we're in the middle of initializing this
438 * metaslab. Load it and walk the free tree for more accurate
439 * progress estimation.
440 */
441 VERIFY0(metaslab_load(msp));
442
443 zfs_btree_index_t where;
444 zfs_range_tree_t *rt = msp->ms_allocatable;
445 for (zfs_range_seg_t *rs =
446 zfs_btree_first(&rt->rt_root, &where); rs;
447 rs = zfs_btree_next(&rt->rt_root, &where,
448 &where)) {
449 logical_rs.rs_start = zfs_rs_get_start(rs, rt);
450 logical_rs.rs_end = zfs_rs_get_end(rs, rt);
451
452 vdev_xlate_walk(vd, &logical_rs,
453 vdev_initialize_xlate_progress, vd);
454 }
455 mutex_exit(&msp->ms_lock);
456 }
457 }
458
459 static int
vdev_initialize_load(vdev_t * vd)460 vdev_initialize_load(vdev_t *vd)
461 {
462 int err = 0;
463 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
464 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
465 ASSERT(vd->vdev_leaf_zap != 0);
466
467 if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE ||
468 vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) {
469 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
470 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
471 sizeof (vd->vdev_initialize_last_offset), 1,
472 &vd->vdev_initialize_last_offset);
473 if (err == ENOENT) {
474 vd->vdev_initialize_last_offset = 0;
475 err = 0;
476 }
477 }
478
479 vdev_initialize_calculate_progress(vd);
480 return (err);
481 }
482
483 static void
vdev_initialize_xlate_range_add(void * arg,zfs_range_seg64_t * physical_rs)484 vdev_initialize_xlate_range_add(void *arg, zfs_range_seg64_t *physical_rs)
485 {
486 vdev_t *vd = arg;
487
488 /* Only add segments that we have not visited yet */
489 if (physical_rs->rs_end <= vd->vdev_initialize_last_offset)
490 return;
491
492 /* Pick up where we left off mid-range. */
493 if (vd->vdev_initialize_last_offset > physical_rs->rs_start) {
494 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to "
495 "(%llu, %llu)", vd->vdev_path,
496 (u_longlong_t)physical_rs->rs_start,
497 (u_longlong_t)physical_rs->rs_end,
498 (u_longlong_t)vd->vdev_initialize_last_offset,
499 (u_longlong_t)physical_rs->rs_end);
500 ASSERT3U(physical_rs->rs_end, >,
501 vd->vdev_initialize_last_offset);
502 physical_rs->rs_start = vd->vdev_initialize_last_offset;
503 }
504
505 ASSERT3U(physical_rs->rs_end, >, physical_rs->rs_start);
506
507 zfs_range_tree_add(vd->vdev_initialize_tree, physical_rs->rs_start,
508 physical_rs->rs_end - physical_rs->rs_start);
509 }
510
511 /*
512 * Convert the logical range into a physical range and add it to our
513 * avl tree.
514 */
515 static void
vdev_initialize_range_add(void * arg,uint64_t start,uint64_t size)516 vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size)
517 {
518 vdev_t *vd = arg;
519 zfs_range_seg64_t logical_rs;
520 logical_rs.rs_start = start;
521 logical_rs.rs_end = start + size;
522
523 ASSERT(vd->vdev_ops->vdev_op_leaf);
524 vdev_xlate_walk(vd, &logical_rs, vdev_initialize_xlate_range_add, arg);
525 }
526
527 static __attribute__((noreturn)) void
vdev_initialize_thread(void * arg)528 vdev_initialize_thread(void *arg)
529 {
530 vdev_t *vd = arg;
531 spa_t *spa = vd->vdev_spa;
532 int error = 0;
533 uint64_t ms_count = 0;
534
535 ASSERT(vdev_is_concrete(vd));
536 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
537
538 vd->vdev_initialize_last_offset = 0;
539 VERIFY0(vdev_initialize_load(vd));
540
541 abd_t *deadbeef =
542 vdev_initialize_block_alloc(vd->vdev_initialize_value);
543
544 vd->vdev_initialize_tree = zfs_range_tree_create_flags(
545 NULL, ZFS_RANGE_SEG64, NULL, 0, 0,
546 ZFS_RT_F_DYN_NAME, vdev_rt_name(vd, "vdev_initialize_tree"));
547
548 for (uint64_t i = 0; !vd->vdev_detached &&
549 i < vd->vdev_top->vdev_ms_count; i++) {
550 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
551 boolean_t unload_when_done = B_FALSE;
552
553 /*
554 * If we've expanded the top-level vdev or it's our
555 * first pass, calculate our progress.
556 */
557 if (vd->vdev_top->vdev_ms_count != ms_count) {
558 vdev_initialize_calculate_progress(vd);
559 ms_count = vd->vdev_top->vdev_ms_count;
560 }
561
562 spa_config_exit(spa, SCL_CONFIG, FTAG);
563 metaslab_disable(msp);
564 mutex_enter(&msp->ms_lock);
565 if (!msp->ms_loaded && !msp->ms_loading)
566 unload_when_done = B_TRUE;
567 VERIFY0(metaslab_load(msp));
568
569 zfs_range_tree_walk(msp->ms_allocatable,
570 vdev_initialize_range_add, vd);
571 mutex_exit(&msp->ms_lock);
572
573 error = vdev_initialize_ranges(vd, deadbeef);
574 metaslab_enable(msp, B_TRUE, unload_when_done);
575 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
576
577 zfs_range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL);
578 if (error != 0)
579 break;
580 }
581
582 spa_config_exit(spa, SCL_CONFIG, FTAG);
583 mutex_enter(&vd->vdev_initialize_io_lock);
584 while (vd->vdev_initialize_inflight > 0) {
585 cv_wait(&vd->vdev_initialize_io_cv,
586 &vd->vdev_initialize_io_lock);
587 }
588 mutex_exit(&vd->vdev_initialize_io_lock);
589
590 zfs_range_tree_destroy(vd->vdev_initialize_tree);
591 vdev_initialize_block_free(deadbeef);
592 vd->vdev_initialize_tree = NULL;
593
594 mutex_enter(&vd->vdev_initialize_lock);
595 if (!vd->vdev_initialize_exit_wanted) {
596 if (vdev_writeable(vd)) {
597 vdev_initialize_change_state(vd,
598 VDEV_INITIALIZE_COMPLETE);
599 } else if (vd->vdev_faulted) {
600 vdev_initialize_change_state(vd,
601 VDEV_INITIALIZE_CANCELED);
602 }
603 }
604 ASSERT(vd->vdev_initialize_thread != NULL ||
605 vd->vdev_initialize_inflight == 0);
606
607 /*
608 * Drop the vdev_initialize_lock while we sync out the
609 * txg since it's possible that a device might be trying to
610 * come online and must check to see if it needs to restart an
611 * initialization. That thread will be holding the spa_config_lock
612 * which would prevent the txg_wait_synced from completing.
613 */
614 mutex_exit(&vd->vdev_initialize_lock);
615 txg_wait_synced(spa_get_dsl(spa), 0);
616 mutex_enter(&vd->vdev_initialize_lock);
617
618 vd->vdev_initialize_thread = NULL;
619 cv_broadcast(&vd->vdev_initialize_cv);
620 mutex_exit(&vd->vdev_initialize_lock);
621
622 thread_exit();
623 }
624
625 /*
626 * Initiates a device. Caller must hold vdev_initialize_lock.
627 * Device must be a leaf and not already be initializing.
628 */
629 void
vdev_initialize(vdev_t * vd,uint64_t value,boolean_t value_provided)630 vdev_initialize(vdev_t *vd, uint64_t value, boolean_t value_provided)
631 {
632 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
633 ASSERT(vd->vdev_ops->vdev_op_leaf);
634 ASSERT(vdev_is_concrete(vd));
635 ASSERT0P(vd->vdev_initialize_thread);
636 ASSERT(!vd->vdev_detached);
637 ASSERT(!vd->vdev_initialize_exit_wanted);
638 ASSERT(!vd->vdev_top->vdev_removing);
639 ASSERT(!vd->vdev_top->vdev_rz_expanding);
640
641 /*
642 * Fix the fill value for the whole run so it is stable across a
643 * suspend/resume (it is persisted to the leaf ZAP). An explicit value
644 * always wins. Otherwise, when resuming an existing run keep the value
645 * chosen when it started (already loaded from the leaf ZAP); only a
646 * fresh run falls back to the module default.
647 */
648 if (value_provided) {
649 vd->vdev_initialize_value = value;
650 } else if (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE &&
651 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) {
652 vd->vdev_initialize_value = zfs_initialize_value;
653 }
654
655 vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE);
656 vd->vdev_initialize_thread = thread_create(NULL, 0,
657 vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
658 }
659
660 /*
661 * Uninitializes a device. Caller must hold vdev_initialize_lock.
662 * Device must be a leaf and not already be initializing.
663 */
664 void
vdev_uninitialize(vdev_t * vd)665 vdev_uninitialize(vdev_t *vd)
666 {
667 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
668 ASSERT(vd->vdev_ops->vdev_op_leaf);
669 ASSERT(vdev_is_concrete(vd));
670 ASSERT0P(vd->vdev_initialize_thread);
671 ASSERT(!vd->vdev_detached);
672 ASSERT(!vd->vdev_initialize_exit_wanted);
673 ASSERT(!vd->vdev_top->vdev_removing);
674
675 vdev_initialize_change_state(vd, VDEV_INITIALIZE_NONE);
676 }
677
678 /*
679 * Wait for the initialize thread to be terminated (cancelled or stopped).
680 */
681 static void
vdev_initialize_stop_wait_impl(vdev_t * vd)682 vdev_initialize_stop_wait_impl(vdev_t *vd)
683 {
684 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
685
686 while (vd->vdev_initialize_thread != NULL)
687 cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock);
688
689 ASSERT0P(vd->vdev_initialize_thread);
690 vd->vdev_initialize_exit_wanted = B_FALSE;
691 }
692
693 /*
694 * Wait for vdev initialize threads which were either to cleanly exit.
695 */
696 void
vdev_initialize_stop_wait(spa_t * spa,list_t * vd_list)697 vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list)
698 {
699 (void) spa;
700 vdev_t *vd;
701
702 ASSERT(spa_namespace_held() ||
703 spa->spa_export_thread == curthread);
704
705 while ((vd = list_remove_head(vd_list)) != NULL) {
706 mutex_enter(&vd->vdev_initialize_lock);
707 vdev_initialize_stop_wait_impl(vd);
708 mutex_exit(&vd->vdev_initialize_lock);
709 }
710 }
711
712 /*
713 * Stop initializing a device, with the resultant initializing state being
714 * tgt_state. For blocking behavior pass NULL for vd_list. Otherwise, when
715 * a list_t is provided the stopping vdev is inserted in to the list. Callers
716 * are then required to call vdev_initialize_stop_wait() to block for all the
717 * initialization threads to exit. The caller must hold vdev_initialize_lock
718 * and must not be writing to the spa config, as the initializing thread may
719 * try to enter the config as a reader before exiting.
720 */
721 void
vdev_initialize_stop(vdev_t * vd,vdev_initializing_state_t tgt_state,list_t * vd_list)722 vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state,
723 list_t *vd_list)
724 {
725 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
726 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
727 ASSERT(vd->vdev_ops->vdev_op_leaf);
728 ASSERT(vdev_is_concrete(vd));
729
730 /*
731 * Allow cancel requests to proceed even if the initialize thread
732 * has stopped.
733 */
734 if (vd->vdev_initialize_thread == NULL &&
735 tgt_state != VDEV_INITIALIZE_CANCELED) {
736 return;
737 }
738
739 vdev_initialize_change_state(vd, tgt_state);
740 vd->vdev_initialize_exit_wanted = B_TRUE;
741
742 if (vd_list == NULL) {
743 vdev_initialize_stop_wait_impl(vd);
744 } else {
745 ASSERT(spa_namespace_held() ||
746 vd->vdev_spa->spa_export_thread == curthread);
747 list_insert_tail(vd_list, vd);
748 }
749 }
750
751 static void
vdev_initialize_stop_all_impl(vdev_t * vd,vdev_initializing_state_t tgt_state,list_t * vd_list)752 vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state,
753 list_t *vd_list)
754 {
755 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
756 mutex_enter(&vd->vdev_initialize_lock);
757 vdev_initialize_stop(vd, tgt_state, vd_list);
758 mutex_exit(&vd->vdev_initialize_lock);
759 return;
760 }
761
762 for (uint64_t i = 0; i < vd->vdev_children; i++) {
763 vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state,
764 vd_list);
765 }
766 }
767
768 /*
769 * Convenience function to stop initializing of a vdev tree and set all
770 * initialize thread pointers to NULL.
771 */
772 void
vdev_initialize_stop_all(vdev_t * vd,vdev_initializing_state_t tgt_state)773 vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state)
774 {
775 spa_t *spa = vd->vdev_spa;
776 list_t vd_list;
777
778 ASSERT(spa_namespace_held() ||
779 spa->spa_export_thread == curthread);
780
781 list_create(&vd_list, sizeof (vdev_t),
782 offsetof(vdev_t, vdev_initialize_node));
783
784 vdev_initialize_stop_all_impl(vd, tgt_state, &vd_list);
785 vdev_initialize_stop_wait(spa, &vd_list);
786
787 if (vd->vdev_spa->spa_sync_on) {
788 /* Make sure that our state has been synced to disk */
789 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
790 }
791
792 list_destroy(&vd_list);
793 }
794
795 void
vdev_initialize_restart(vdev_t * vd)796 vdev_initialize_restart(vdev_t *vd)
797 {
798 ASSERT(spa_namespace_held() ||
799 vd->vdev_spa->spa_load_thread == curthread);
800 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
801
802 if (vd->vdev_leaf_zap != 0) {
803 mutex_enter(&vd->vdev_initialize_lock);
804 uint64_t initialize_state = VDEV_INITIALIZE_NONE;
805 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
806 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE,
807 sizeof (initialize_state), 1, &initialize_state);
808 ASSERT(err == 0 || err == ENOENT);
809 vd->vdev_initialize_state = initialize_state;
810
811 uint64_t timestamp = 0;
812 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
813 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME,
814 sizeof (timestamp), 1, ×tamp);
815 ASSERT(err == 0 || err == ENOENT);
816 vd->vdev_initialize_action_time = timestamp;
817
818 /*
819 * Restore the fill value chosen when this run started. Pools
820 * initialized before this field existed fall back to the
821 * module default, preserving their previous behavior.
822 */
823 uint64_t value = zfs_initialize_value;
824 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
825 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_VALUE,
826 sizeof (value), 1, &value);
827 ASSERT(err == 0 || err == ENOENT);
828 vd->vdev_initialize_value = value;
829
830 if ((vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
831 vd->vdev_offline) && !vd->vdev_top->vdev_rz_expanding) {
832 /* load progress for reporting, but don't resume */
833 VERIFY0(vdev_initialize_load(vd));
834 } else if (vd->vdev_initialize_state ==
835 VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd) &&
836 !vd->vdev_top->vdev_removing &&
837 !vd->vdev_top->vdev_rz_expanding &&
838 vd->vdev_initialize_thread == NULL) {
839 vdev_initialize(vd, value, B_TRUE);
840 }
841
842 mutex_exit(&vd->vdev_initialize_lock);
843 }
844
845 for (uint64_t i = 0; i < vd->vdev_children; i++) {
846 vdev_initialize_restart(vd->vdev_child[i]);
847 }
848 }
849
850 EXPORT_SYMBOL(vdev_initialize);
851 EXPORT_SYMBOL(vdev_uninitialize);
852 EXPORT_SYMBOL(vdev_initialize_stop);
853 EXPORT_SYMBOL(vdev_initialize_stop_all);
854 EXPORT_SYMBOL(vdev_initialize_stop_wait);
855 EXPORT_SYMBOL(vdev_initialize_restart);
856
857 ZFS_MODULE_PARAM(zfs, zfs_, initialize_value, U64, ZMOD_RW,
858 "Value written during zpool initialize");
859
860 ZFS_MODULE_PARAM(zfs, zfs_, initialize_chunk_size, U64, ZMOD_RW,
861 "Size in bytes of writes by zpool initialize");
862