1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
24 */
25
26 #include <sys/spa.h>
27 #include <sys/spa_impl.h>
28 #include <sys/txg.h>
29 #include <sys/vdev_impl.h>
30 #include <sys/refcount.h>
31 #include <sys/metaslab_impl.h>
32 #include <sys/dsl_synctask.h>
33 #include <sys/zap.h>
34 #include <sys/dmu_tx.h>
35
36 /*
37 * Value that is written to disk during initialization.
38 */
39 uint64_t zfs_initialize_value = 0xdeadbeefdeadbeefULL;
40
41 /* maximum number of I/Os outstanding per leaf vdev */
42 int zfs_initialize_limit = 1;
43
44 /* size of initializing writes; default 1MiB, see zfs_remove_max_segment */
45 unsigned long zfs_initialize_chunk_size = 1024 * 1024;
46
47 static boolean_t
vdev_initialize_should_stop(vdev_t * vd)48 vdev_initialize_should_stop(vdev_t *vd)
49 {
50 return (vd->vdev_initialize_exit_wanted || !vdev_writeable(vd) ||
51 vd->vdev_detached || vd->vdev_top->vdev_removing);
52 }
53
54 static void
vdev_initialize_zap_update_sync(void * arg,dmu_tx_t * tx)55 vdev_initialize_zap_update_sync(void *arg, dmu_tx_t *tx)
56 {
57 /*
58 * We pass in the guid instead of the vdev_t since the vdev may
59 * have been freed prior to the sync task being processed. This
60 * happens when a vdev is detached as we call spa_config_vdev_exit(),
61 * stop the initializing thread, schedule the sync task, and free
62 * the vdev. Later when the scheduled sync task is invoked, it would
63 * find that the vdev has been freed.
64 */
65 uint64_t guid = *(uint64_t *)arg;
66 uint64_t txg = dmu_tx_get_txg(tx);
67 kmem_free(arg, sizeof (uint64_t));
68
69 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
70 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
71 return;
72
73 uint64_t last_offset = vd->vdev_initialize_offset[txg & TXG_MASK];
74 vd->vdev_initialize_offset[txg & TXG_MASK] = 0;
75
76 VERIFY(vd->vdev_leaf_zap != 0);
77
78 objset_t *mos = vd->vdev_spa->spa_meta_objset;
79
80 if (last_offset > 0) {
81 vd->vdev_initialize_last_offset = last_offset;
82 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
83 VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
84 sizeof (last_offset), 1, &last_offset, tx));
85 }
86 if (vd->vdev_initialize_action_time > 0) {
87 uint64_t val = (uint64_t)vd->vdev_initialize_action_time;
88 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
89 VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME, sizeof (val),
90 1, &val, tx));
91 }
92
93 uint64_t initialize_state = vd->vdev_initialize_state;
94 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
95 VDEV_LEAF_ZAP_INITIALIZE_STATE, sizeof (initialize_state), 1,
96 &initialize_state, tx));
97 }
98
99 static void
vdev_initialize_change_state(vdev_t * vd,vdev_initializing_state_t new_state)100 vdev_initialize_change_state(vdev_t *vd, vdev_initializing_state_t new_state)
101 {
102 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
103 spa_t *spa = vd->vdev_spa;
104
105 if (new_state == vd->vdev_initialize_state)
106 return;
107
108 /*
109 * Copy the vd's guid, this will be freed by the sync task.
110 */
111 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
112 *guid = vd->vdev_guid;
113
114 /*
115 * If we're suspending, then preserving the original start time.
116 */
117 if (vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED) {
118 vd->vdev_initialize_action_time = gethrestime_sec();
119 }
120 vd->vdev_initialize_state = new_state;
121
122 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
123 VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
124 dsl_sync_task_nowait(spa_get_dsl(spa), vdev_initialize_zap_update_sync,
125 guid, 2, ZFS_SPACE_CHECK_NONE, tx);
126
127 switch (new_state) {
128 case VDEV_INITIALIZE_ACTIVE:
129 spa_history_log_internal(spa, "initialize", tx,
130 "vdev=%s activated", vd->vdev_path);
131 break;
132 case VDEV_INITIALIZE_SUSPENDED:
133 spa_history_log_internal(spa, "initialize", tx,
134 "vdev=%s suspended", vd->vdev_path);
135 break;
136 case VDEV_INITIALIZE_CANCELED:
137 spa_history_log_internal(spa, "initialize", tx,
138 "vdev=%s canceled", vd->vdev_path);
139 break;
140 case VDEV_INITIALIZE_COMPLETE:
141 spa_history_log_internal(spa, "initialize", tx,
142 "vdev=%s complete", vd->vdev_path);
143 break;
144 default:
145 panic("invalid state %llu", (unsigned long long)new_state);
146 }
147
148 dmu_tx_commit(tx);
149
150 if (new_state != VDEV_INITIALIZE_ACTIVE)
151 spa_notify_waiters(spa);
152 }
153
154 static void
vdev_initialize_cb(zio_t * zio)155 vdev_initialize_cb(zio_t *zio)
156 {
157 vdev_t *vd = zio->io_vd;
158 mutex_enter(&vd->vdev_initialize_io_lock);
159 if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
160 /*
161 * The I/O failed because the vdev was unavailable; roll the
162 * last offset back. (This works because spa_sync waits on
163 * spa_txg_zio before it runs sync tasks.)
164 */
165 uint64_t *off =
166 &vd->vdev_initialize_offset[zio->io_txg & TXG_MASK];
167 *off = MIN(*off, zio->io_offset);
168 } else {
169 /*
170 * Since initializing is best-effort, we ignore I/O errors and
171 * rely on vdev_probe to determine if the errors are more
172 * critical.
173 */
174 if (zio->io_error != 0)
175 vd->vdev_stat.vs_initialize_errors++;
176
177 vd->vdev_initialize_bytes_done += zio->io_orig_size;
178 }
179 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
180 vd->vdev_initialize_inflight--;
181 cv_broadcast(&vd->vdev_initialize_io_cv);
182 mutex_exit(&vd->vdev_initialize_io_lock);
183
184 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
185 }
186
187 /* Takes care of physical writing and limiting # of concurrent ZIOs. */
188 static int
vdev_initialize_write(vdev_t * vd,uint64_t start,uint64_t size,abd_t * data)189 vdev_initialize_write(vdev_t *vd, uint64_t start, uint64_t size, abd_t *data)
190 {
191 spa_t *spa = vd->vdev_spa;
192
193 /* Limit inflight initializing I/Os */
194 mutex_enter(&vd->vdev_initialize_io_lock);
195 while (vd->vdev_initialize_inflight >= zfs_initialize_limit) {
196 cv_wait(&vd->vdev_initialize_io_cv,
197 &vd->vdev_initialize_io_lock);
198 }
199 vd->vdev_initialize_inflight++;
200 mutex_exit(&vd->vdev_initialize_io_lock);
201
202 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
203 VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
204 uint64_t txg = dmu_tx_get_txg(tx);
205
206 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
207 mutex_enter(&vd->vdev_initialize_lock);
208
209 if (vd->vdev_initialize_offset[txg & TXG_MASK] == 0) {
210 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
211 *guid = vd->vdev_guid;
212
213 /* This is the first write of this txg. */
214 dsl_sync_task_nowait(spa_get_dsl(spa),
215 vdev_initialize_zap_update_sync, guid, 2,
216 ZFS_SPACE_CHECK_RESERVED, tx);
217 }
218
219 /*
220 * We know the vdev struct will still be around since all
221 * consumers of vdev_free must stop the initialization first.
222 */
223 if (vdev_initialize_should_stop(vd)) {
224 mutex_enter(&vd->vdev_initialize_io_lock);
225 ASSERT3U(vd->vdev_initialize_inflight, >, 0);
226 vd->vdev_initialize_inflight--;
227 mutex_exit(&vd->vdev_initialize_io_lock);
228 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
229 mutex_exit(&vd->vdev_initialize_lock);
230 dmu_tx_commit(tx);
231 return (SET_ERROR(EINTR));
232 }
233 mutex_exit(&vd->vdev_initialize_lock);
234
235 vd->vdev_initialize_offset[txg & TXG_MASK] = start + size;
236 zio_nowait(zio_write_phys(spa->spa_txg_zio[txg & TXG_MASK], vd, start,
237 size, data, ZIO_CHECKSUM_OFF, vdev_initialize_cb, NULL,
238 ZIO_PRIORITY_INITIALIZING, ZIO_FLAG_CANFAIL, B_FALSE));
239 /* vdev_initialize_cb releases SCL_STATE_ALL */
240
241 dmu_tx_commit(tx);
242
243 return (0);
244 }
245
246 /*
247 * Callback to fill each ABD chunk with zfs_initialize_value. len must be
248 * divisible by sizeof (uint64_t), and buf must be 8-byte aligned. The ABD
249 * allocation will guarantee these for us.
250 */
251 /* ARGSUSED */
252 static int
vdev_initialize_block_fill(void * buf,size_t len,void * unused)253 vdev_initialize_block_fill(void *buf, size_t len, void *unused)
254 {
255 ASSERT0(len % sizeof (uint64_t));
256 for (uint64_t i = 0; i < len; i += sizeof (uint64_t)) {
257 *(uint64_t *)((char *)(buf) + i) = zfs_initialize_value;
258 }
259 return (0);
260 }
261
262 static abd_t *
vdev_initialize_block_alloc()263 vdev_initialize_block_alloc()
264 {
265 /* Allocate ABD for filler data */
266 abd_t *data = abd_alloc_for_io(zfs_initialize_chunk_size, B_FALSE);
267
268 ASSERT0(zfs_initialize_chunk_size % sizeof (uint64_t));
269 (void) abd_iterate_func(data, 0, zfs_initialize_chunk_size,
270 vdev_initialize_block_fill, NULL);
271
272 return (data);
273 }
274
275 static void
vdev_initialize_block_free(abd_t * data)276 vdev_initialize_block_free(abd_t *data)
277 {
278 abd_free(data);
279 }
280
281 static int
vdev_initialize_ranges(vdev_t * vd,abd_t * data)282 vdev_initialize_ranges(vdev_t *vd, abd_t *data)
283 {
284 range_tree_t *rt = vd->vdev_initialize_tree;
285 zfs_btree_t *bt = &rt->rt_root;
286 zfs_btree_index_t where;
287
288 for (range_seg_t *rs = zfs_btree_first(bt, &where); rs != NULL;
289 rs = zfs_btree_next(bt, &where, &where)) {
290 uint64_t size = rs_get_end(rs, rt) - rs_get_start(rs, rt);
291
292 /* Split range into legally-sized physical chunks */
293 uint64_t writes_required =
294 ((size - 1) / zfs_initialize_chunk_size) + 1;
295
296 for (uint64_t w = 0; w < writes_required; w++) {
297 int error;
298
299 error = vdev_initialize_write(vd,
300 VDEV_LABEL_START_SIZE + rs_get_start(rs, rt) +
301 (w * zfs_initialize_chunk_size),
302 MIN(size - (w * zfs_initialize_chunk_size),
303 zfs_initialize_chunk_size), data);
304 if (error != 0)
305 return (error);
306 }
307 }
308 return (0);
309 }
310
311 static void
vdev_initialize_calculate_progress(vdev_t * vd)312 vdev_initialize_calculate_progress(vdev_t *vd)
313 {
314 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
315 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
316 ASSERT(vd->vdev_leaf_zap != 0);
317
318 vd->vdev_initialize_bytes_est = 0;
319 vd->vdev_initialize_bytes_done = 0;
320
321 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
322 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
323 mutex_enter(&msp->ms_lock);
324
325 uint64_t ms_free = msp->ms_size -
326 metaslab_allocated_space(msp);
327
328 if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
329 ms_free /= vd->vdev_top->vdev_children;
330
331 /*
332 * Convert the metaslab range to a physical range
333 * on our vdev. We use this to determine if we are
334 * in the middle of this metaslab range.
335 */
336 range_seg64_t logical_rs, physical_rs;
337 logical_rs.rs_start = msp->ms_start;
338 logical_rs.rs_end = msp->ms_start + msp->ms_size;
339 vdev_xlate(vd, &logical_rs, &physical_rs);
340
341 if (vd->vdev_initialize_last_offset <= physical_rs.rs_start) {
342 vd->vdev_initialize_bytes_est += ms_free;
343 mutex_exit(&msp->ms_lock);
344 continue;
345 } else if (vd->vdev_initialize_last_offset >
346 physical_rs.rs_end) {
347 vd->vdev_initialize_bytes_done += ms_free;
348 vd->vdev_initialize_bytes_est += ms_free;
349 mutex_exit(&msp->ms_lock);
350 continue;
351 }
352
353 /*
354 * If we get here, we're in the middle of initializing this
355 * metaslab. Load it and walk the free tree for more accurate
356 * progress estimation.
357 */
358 VERIFY0(metaslab_load(msp));
359
360 zfs_btree_index_t where;
361 range_tree_t *rt = msp->ms_allocatable;
362 for (range_seg_t *rs =
363 zfs_btree_first(&rt->rt_root, &where); rs;
364 rs = zfs_btree_next(&rt->rt_root, &where,
365 &where)) {
366 logical_rs.rs_start = rs_get_start(rs, rt);
367 logical_rs.rs_end = rs_get_end(rs, rt);
368 vdev_xlate(vd, &logical_rs, &physical_rs);
369
370 uint64_t size = physical_rs.rs_end -
371 physical_rs.rs_start;
372 vd->vdev_initialize_bytes_est += size;
373 if (vd->vdev_initialize_last_offset >
374 physical_rs.rs_end) {
375 vd->vdev_initialize_bytes_done += size;
376 } else if (vd->vdev_initialize_last_offset >
377 physical_rs.rs_start &&
378 vd->vdev_initialize_last_offset <
379 physical_rs.rs_end) {
380 vd->vdev_initialize_bytes_done +=
381 vd->vdev_initialize_last_offset -
382 physical_rs.rs_start;
383 }
384 }
385 mutex_exit(&msp->ms_lock);
386 }
387 }
388
389 static int
vdev_initialize_load(vdev_t * vd)390 vdev_initialize_load(vdev_t *vd)
391 {
392 int err = 0;
393 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
394 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
395 ASSERT(vd->vdev_leaf_zap != 0);
396
397 if (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE ||
398 vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED) {
399 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
400 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_LAST_OFFSET,
401 sizeof (vd->vdev_initialize_last_offset), 1,
402 &vd->vdev_initialize_last_offset);
403 if (err == ENOENT) {
404 vd->vdev_initialize_last_offset = 0;
405 err = 0;
406 }
407 }
408
409 vdev_initialize_calculate_progress(vd);
410 return (err);
411 }
412
413
414 /*
415 * Convert the logical range into a physical range and add it to our
416 * avl tree.
417 */
418 void
vdev_initialize_range_add(void * arg,uint64_t start,uint64_t size)419 vdev_initialize_range_add(void *arg, uint64_t start, uint64_t size)
420 {
421 vdev_t *vd = arg;
422 range_seg64_t logical_rs, physical_rs;
423 logical_rs.rs_start = start;
424 logical_rs.rs_end = start + size;
425
426 ASSERT(vd->vdev_ops->vdev_op_leaf);
427 vdev_xlate(vd, &logical_rs, &physical_rs);
428
429 IMPLY(vd->vdev_top == vd,
430 logical_rs.rs_start == physical_rs.rs_start);
431 IMPLY(vd->vdev_top == vd,
432 logical_rs.rs_end == physical_rs.rs_end);
433
434 /* Only add segments that we have not visited yet */
435 if (physical_rs.rs_end <= vd->vdev_initialize_last_offset)
436 return;
437
438 /* Pick up where we left off mid-range. */
439 if (vd->vdev_initialize_last_offset > physical_rs.rs_start) {
440 zfs_dbgmsg("range write: vd %s changed (%llu, %llu) to "
441 "(%llu, %llu)", vd->vdev_path,
442 (u_longlong_t)physical_rs.rs_start,
443 (u_longlong_t)physical_rs.rs_end,
444 (u_longlong_t)vd->vdev_initialize_last_offset,
445 (u_longlong_t)physical_rs.rs_end);
446 ASSERT3U(physical_rs.rs_end, >,
447 vd->vdev_initialize_last_offset);
448 physical_rs.rs_start = vd->vdev_initialize_last_offset;
449 }
450 ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
451
452 /*
453 * With raidz, it's possible that the logical range does not live on
454 * this leaf vdev. We only add the physical range to this vdev's if it
455 * has a length greater than 0.
456 */
457 if (physical_rs.rs_end > physical_rs.rs_start) {
458 range_tree_add(vd->vdev_initialize_tree, physical_rs.rs_start,
459 physical_rs.rs_end - physical_rs.rs_start);
460 } else {
461 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
462 }
463 }
464
465 static void
vdev_initialize_thread(void * arg)466 vdev_initialize_thread(void *arg)
467 {
468 vdev_t *vd = arg;
469 spa_t *spa = vd->vdev_spa;
470 int error = 0;
471 uint64_t ms_count = 0;
472
473 ASSERT(vdev_is_concrete(vd));
474 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
475
476 vd->vdev_initialize_last_offset = 0;
477 VERIFY0(vdev_initialize_load(vd));
478
479 abd_t *deadbeef = vdev_initialize_block_alloc();
480
481 vd->vdev_initialize_tree = range_tree_create(NULL, RANGE_SEG64, NULL,
482 0, 0);
483
484 for (uint64_t i = 0; !vd->vdev_detached &&
485 i < vd->vdev_top->vdev_ms_count; i++) {
486 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
487 boolean_t unload_when_done = B_FALSE;
488
489 /*
490 * If we've expanded the top-level vdev or it's our
491 * first pass, calculate our progress.
492 */
493 if (vd->vdev_top->vdev_ms_count != ms_count) {
494 vdev_initialize_calculate_progress(vd);
495 ms_count = vd->vdev_top->vdev_ms_count;
496 }
497
498 spa_config_exit(spa, SCL_CONFIG, FTAG);
499 metaslab_disable(msp);
500 mutex_enter(&msp->ms_lock);
501 if (!msp->ms_loaded && !msp->ms_loading)
502 unload_when_done = B_TRUE;
503 VERIFY0(metaslab_load(msp));
504
505 range_tree_walk(msp->ms_allocatable, vdev_initialize_range_add,
506 vd);
507 mutex_exit(&msp->ms_lock);
508
509 error = vdev_initialize_ranges(vd, deadbeef);
510 metaslab_enable(msp, B_TRUE, unload_when_done);
511 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
512
513 range_tree_vacate(vd->vdev_initialize_tree, NULL, NULL);
514 if (error != 0)
515 break;
516 }
517
518 spa_config_exit(spa, SCL_CONFIG, FTAG);
519 mutex_enter(&vd->vdev_initialize_io_lock);
520 while (vd->vdev_initialize_inflight > 0) {
521 cv_wait(&vd->vdev_initialize_io_cv,
522 &vd->vdev_initialize_io_lock);
523 }
524 mutex_exit(&vd->vdev_initialize_io_lock);
525
526 range_tree_destroy(vd->vdev_initialize_tree);
527 vdev_initialize_block_free(deadbeef);
528 vd->vdev_initialize_tree = NULL;
529
530 mutex_enter(&vd->vdev_initialize_lock);
531 if (!vd->vdev_initialize_exit_wanted && vdev_writeable(vd)) {
532 vdev_initialize_change_state(vd, VDEV_INITIALIZE_COMPLETE);
533 }
534 ASSERT(vd->vdev_initialize_thread != NULL ||
535 vd->vdev_initialize_inflight == 0);
536
537 /*
538 * Drop the vdev_initialize_lock while we sync out the
539 * txg since it's possible that a device might be trying to
540 * come online and must check to see if it needs to restart an
541 * initialization. That thread will be holding the spa_config_lock
542 * which would prevent the txg_wait_synced from completing.
543 */
544 mutex_exit(&vd->vdev_initialize_lock);
545 txg_wait_synced(spa_get_dsl(spa), 0);
546 mutex_enter(&vd->vdev_initialize_lock);
547
548 vd->vdev_initialize_thread = NULL;
549 cv_broadcast(&vd->vdev_initialize_cv);
550 mutex_exit(&vd->vdev_initialize_lock);
551 }
552
553 /*
554 * Initiates a device. Caller must hold vdev_initialize_lock.
555 * Device must be a leaf and not already be initializing.
556 */
557 void
vdev_initialize(vdev_t * vd)558 vdev_initialize(vdev_t *vd)
559 {
560 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
561 ASSERT(vd->vdev_ops->vdev_op_leaf);
562 ASSERT(vdev_is_concrete(vd));
563 ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
564 ASSERT(!vd->vdev_detached);
565 ASSERT(!vd->vdev_initialize_exit_wanted);
566 ASSERT(!vd->vdev_top->vdev_removing);
567
568 vdev_initialize_change_state(vd, VDEV_INITIALIZE_ACTIVE);
569 vd->vdev_initialize_thread = thread_create(NULL, 0,
570 vdev_initialize_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
571 }
572
573 /*
574 * Wait for the initialize thread to be terminated (cancelled or stopped).
575 */
576 static void
vdev_initialize_stop_wait_impl(vdev_t * vd)577 vdev_initialize_stop_wait_impl(vdev_t *vd)
578 {
579 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
580
581 while (vd->vdev_initialize_thread != NULL)
582 cv_wait(&vd->vdev_initialize_cv, &vd->vdev_initialize_lock);
583
584 ASSERT3P(vd->vdev_initialize_thread, ==, NULL);
585 vd->vdev_initialize_exit_wanted = B_FALSE;
586 }
587
588 /*
589 * Wait for vdev initialize threads which were either to cleanly exit.
590 */
591 void
vdev_initialize_stop_wait(spa_t * spa,list_t * vd_list)592 vdev_initialize_stop_wait(spa_t *spa, list_t *vd_list)
593 {
594 vdev_t *vd;
595
596 ASSERT(MUTEX_HELD(&spa_namespace_lock));
597
598 while ((vd = list_remove_head(vd_list)) != NULL) {
599 mutex_enter(&vd->vdev_initialize_lock);
600 vdev_initialize_stop_wait_impl(vd);
601 mutex_exit(&vd->vdev_initialize_lock);
602 }
603 }
604
605 /*
606 * Stop initializing a device, with the resultant initializing state being
607 * tgt_state. For blocking behavior pass NULL for vd_list. Otherwise, when
608 * a list_t is provided the stopping vdev is inserted in to the list. Callers
609 * are then required to call vdev_initialize_stop_wait() to block for all the
610 * initialization threads to exit. The caller must hold vdev_initialize_lock
611 * and must not be writing to the spa config, as the initializing thread may
612 * try to enter the config as a reader before exiting.
613 */
614 void
vdev_initialize_stop(vdev_t * vd,vdev_initializing_state_t tgt_state,list_t * vd_list)615 vdev_initialize_stop(vdev_t *vd, vdev_initializing_state_t tgt_state,
616 list_t *vd_list)
617 {
618 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
619 ASSERT(MUTEX_HELD(&vd->vdev_initialize_lock));
620 ASSERT(vd->vdev_ops->vdev_op_leaf);
621 ASSERT(vdev_is_concrete(vd));
622
623 /*
624 * Allow cancel requests to proceed even if the initialize thread
625 * has stopped.
626 */
627 if (vd->vdev_initialize_thread == NULL &&
628 tgt_state != VDEV_INITIALIZE_CANCELED) {
629 return;
630 }
631
632 vdev_initialize_change_state(vd, tgt_state);
633 vd->vdev_initialize_exit_wanted = B_TRUE;
634
635 if (vd_list == NULL) {
636 vdev_initialize_stop_wait_impl(vd);
637 } else {
638 ASSERT(MUTEX_HELD(&spa_namespace_lock));
639 list_insert_tail(vd_list, vd);
640 }
641 }
642
643 static void
vdev_initialize_stop_all_impl(vdev_t * vd,vdev_initializing_state_t tgt_state,list_t * vd_list)644 vdev_initialize_stop_all_impl(vdev_t *vd, vdev_initializing_state_t tgt_state,
645 list_t *vd_list)
646 {
647 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
648 mutex_enter(&vd->vdev_initialize_lock);
649 vdev_initialize_stop(vd, tgt_state, vd_list);
650 mutex_exit(&vd->vdev_initialize_lock);
651 return;
652 }
653
654 for (uint64_t i = 0; i < vd->vdev_children; i++) {
655 vdev_initialize_stop_all_impl(vd->vdev_child[i], tgt_state,
656 vd_list);
657 }
658 }
659
660 /*
661 * Convenience function to stop initializing of a vdev tree and set all
662 * initialize thread pointers to NULL.
663 */
664 void
vdev_initialize_stop_all(vdev_t * vd,vdev_initializing_state_t tgt_state)665 vdev_initialize_stop_all(vdev_t *vd, vdev_initializing_state_t tgt_state)
666 {
667 spa_t *spa = vd->vdev_spa;
668 list_t vd_list;
669
670 ASSERT(MUTEX_HELD(&spa_namespace_lock));
671
672 list_create(&vd_list, sizeof (vdev_t),
673 offsetof(vdev_t, vdev_initialize_node));
674
675 vdev_initialize_stop_all_impl(vd, tgt_state, &vd_list);
676 vdev_initialize_stop_wait(spa, &vd_list);
677
678 if (vd->vdev_spa->spa_sync_on) {
679 /* Make sure that our state has been synced to disk */
680 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
681 }
682
683 list_destroy(&vd_list);
684 }
685
686 void
vdev_initialize_restart(vdev_t * vd)687 vdev_initialize_restart(vdev_t *vd)
688 {
689 ASSERT(MUTEX_HELD(&spa_namespace_lock));
690 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
691
692 if (vd->vdev_leaf_zap != 0) {
693 mutex_enter(&vd->vdev_initialize_lock);
694 uint64_t initialize_state = VDEV_INITIALIZE_NONE;
695 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
696 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_STATE,
697 sizeof (initialize_state), 1, &initialize_state);
698 ASSERT(err == 0 || err == ENOENT);
699 vd->vdev_initialize_state = initialize_state;
700
701 uint64_t timestamp = 0;
702 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
703 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_INITIALIZE_ACTION_TIME,
704 sizeof (timestamp), 1, ×tamp);
705 ASSERT(err == 0 || err == ENOENT);
706 vd->vdev_initialize_action_time = (time_t)timestamp;
707
708 if (vd->vdev_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
709 vd->vdev_offline) {
710 /* load progress for reporting, but don't resume */
711 VERIFY0(vdev_initialize_load(vd));
712 } else if (vd->vdev_initialize_state ==
713 VDEV_INITIALIZE_ACTIVE && vdev_writeable(vd) &&
714 !vd->vdev_top->vdev_removing &&
715 vd->vdev_initialize_thread == NULL) {
716 vdev_initialize(vd);
717 }
718
719 mutex_exit(&vd->vdev_initialize_lock);
720 }
721
722 for (uint64_t i = 0; i < vd->vdev_children; i++) {
723 vdev_initialize_restart(vd->vdev_child[i]);
724 }
725 }
726