1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
24 */
25
26 #include <sys/abd.h>
27 #include <sys/mmp.h>
28 #include <sys/spa.h>
29 #include <sys/spa_impl.h>
30 #include <sys/time.h>
31 #include <sys/vdev.h>
32 #include <sys/vdev_impl.h>
33 #include <sys/zfs_context.h>
34 #include <sys/callb.h>
35
36 /*
37 * Multi-Modifier Protection (MMP) attempts to prevent a user from importing
38 * or opening a pool on more than one host at a time. In particular, it
39 * prevents "zpool import -f" on a host from succeeding while the pool is
40 * already imported on another host. There are many other ways in which a
41 * device could be used by two hosts for different purposes at the same time
42 * resulting in pool damage. This implementation does not attempt to detect
43 * those cases.
44 *
45 * MMP operates by ensuring there are frequent visible changes on disk (a
46 * "heartbeat") at all times. And by altering the import process to check
47 * for these changes and failing the import when they are detected. This
48 * functionality is enabled by setting the 'multihost' pool property to on.
49 *
50 * Uberblocks written by the txg_sync thread always go into the first
51 * (N-MMP_BLOCKS_PER_LABEL) slots, the remaining slots are reserved for MMP.
52 * They are used to hold uberblocks which are exactly the same as the last
53 * synced uberblock except that the ub_timestamp and mmp_config are frequently
54 * updated. Like all other uberblocks, the slot is written with an embedded
55 * checksum, and slots with invalid checksums are ignored. This provides the
56 * "heartbeat", with no risk of overwriting good uberblocks that must be
57 * preserved, e.g. previous txgs and associated block pointers.
58 *
59 * Three optional fields are added to uberblock structure; ub_mmp_magic,
60 * ub_mmp_config, and ub_mmp_delay. The ub_mmp_magic value allows zfs to tell
61 * whether the other ub_mmp_* fields are valid. The ub_mmp_config field tells
62 * the importing host the settings of zfs_multihost_interval and
63 * zfs_multihost_fail_intervals on the host which last had (or currently has)
64 * the pool imported. These determine how long a host must wait to detect
65 * activity in the pool, before concluding the pool is not in use. The
66 * mmp_delay field is a decaying average of the amount of time between
67 * completion of successive MMP writes, in nanoseconds. It indicates whether
68 * MMP is enabled.
69 *
70 * During import an activity test may now be performed to determine if
71 * the pool is in use. The activity test is typically required if the
72 * ZPOOL_CONFIG_HOSTID does not match the system hostid, the pool state is
73 * POOL_STATE_ACTIVE, and the pool is not a root pool.
74 *
75 * The activity test finds the "best" uberblock (highest txg, timestamp, and, if
76 * ub_mmp_magic is valid, sequence number from ub_mmp_config). It then waits
77 * some time, and finds the "best" uberblock again. If any of the mentioned
78 * fields have different values in the newly read uberblock, the pool is in use
79 * by another host and the import fails. In order to assure the accuracy of the
80 * activity test, the default values result in an activity test duration of 20x
81 * the mmp write interval.
82 *
83 * The duration of the "zpool import" activity test depends on the information
84 * available in the "best" uberblock:
85 *
86 * 1) If uberblock was written by zfs-0.8 or newer and fail_intervals > 0:
87 * ub_mmp_config.fail_intervals * ub_mmp_config.multihost_interval * 2
88 *
89 * In this case, a weak guarantee is provided. Since the host which last had
90 * the pool imported will suspend the pool if no mmp writes land within
91 * fail_intervals * multihost_interval ms, the absence of writes during that
92 * time means either the pool is not imported, or it is imported but the pool
93 * is suspended and no further writes will occur.
94 *
95 * Note that resuming the suspended pool on the remote host would invalidate
96 * this guarantee, and so it is not allowed.
97 *
98 * The factor of 2 provides a conservative safety factor and derives from
99 * MMP_IMPORT_SAFETY_FACTOR;
100 *
101 * 2) If uberblock was written by zfs-0.8 or newer and fail_intervals == 0:
102 * (ub_mmp_config.multihost_interval + ub_mmp_delay) *
103 * zfs_multihost_import_intervals
104 *
105 * In this case no guarantee can provided. However, as long as some devices
106 * are healthy and connected, it is likely that at least one write will land
107 * within (multihost_interval + mmp_delay) because multihost_interval is
108 * enough time for a write to be attempted to each leaf vdev, and mmp_delay
109 * is enough for one to land, based on past delays. Multiplying by
110 * zfs_multihost_import_intervals provides a conservative safety factor.
111 *
112 * 3) If uberblock was written by zfs-0.7:
113 * (zfs_multihost_interval + ub_mmp_delay) * zfs_multihost_import_intervals
114 *
115 * The same logic as case #2 applies, but we do not know remote tunables.
116 *
117 * We use the local value for zfs_multihost_interval because the original MMP
118 * did not record this value in the uberblock.
119 *
120 * ub_mmp_delay >= (zfs_multihost_interval / leaves), so if the other host
121 * has a much larger zfs_multihost_interval set, ub_mmp_delay will reflect
122 * that. We will have waited enough time for zfs_multihost_import_intervals
123 * writes to be issued and all but one to land.
124 *
125 * single device pool example delays
126 *
127 * import_delay = (1 + 1) * 20 = 40s #defaults, no I/O delay
128 * import_delay = (1 + 10) * 20 = 220s #defaults, 10s I/O delay
129 * import_delay = (10 + 10) * 20 = 400s #10s multihost_interval,
130 * no I/O delay
131 * 100 device pool example delays
132 *
133 * import_delay = (1 + .01) * 20 = 20s #defaults, no I/O delay
134 * import_delay = (1 + 10) * 20 = 220s #defaults, 10s I/O delay
135 * import_delay = (10 + .1) * 20 = 202s #10s multihost_interval,
136 * no I/O delay
137 *
138 * 4) Otherwise, this uberblock was written by a pre-MMP zfs:
139 * zfs_multihost_import_intervals * zfs_multihost_interval
140 *
141 * In this case local tunables are used. By default this product = 10s, long
142 * enough for a pool with any activity at all to write at least one
143 * uberblock. No guarantee can be provided.
144 *
145 * Additionally, the duration is then extended by a random 25% to attempt to to
146 * detect simultaneous imports. For example, if both partner hosts are rebooted
147 * at the same time and automatically attempt to import the pool.
148 *
149 * Once the read-only activity check completes and the pool is determined to
150 * be inactive a second check is performed to claim the pool. During this
151 * phase the host writes out MMP uberblocks to each of the devices which are
152 * identical to the best uberblock but with a randomly selected sequence id.
153 * The "best" uberblock is then read back and it must contain this new sequence
154 * number. This check is performed multiple times to ensure that there is
155 * no window where a concurrently importing system can incorrectly determine
156 * the pool to be inactive.
157 */
158
159 /*
160 * Used to control the frequency of mmp writes which are performed when the
161 * 'multihost' pool property is on. This is one factor used to determine the
162 * length of the activity check during import.
163 *
164 * On average an mmp write will be issued for each leaf vdev every
165 * zfs_multihost_interval milliseconds. In practice, the observed period can
166 * vary with the I/O load and this observed value is the ub_mmp_delay which is
167 * stored in the uberblock. The minimum allowed value is 100 ms.
168 */
169 uint64_t zfs_multihost_interval = MMP_DEFAULT_INTERVAL;
170
171 /*
172 * Used to control the duration of the activity test on import. Smaller values
173 * of zfs_multihost_import_intervals will reduce the import time but increase
174 * the risk of failing to detect an active pool. The total activity check time
175 * is never allowed to drop below one second. A value of 0 is ignored and
176 * treated as if it was set to 1.
177 */
178 uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS;
179
180 /*
181 * Controls the behavior of the pool when mmp write failures or delays are
182 * detected.
183 *
184 * When zfs_multihost_fail_intervals = 0, mmp write failures or delays are
185 * ignored. The failures will still be reported to the ZED which depending on
186 * its configuration may take action such as suspending the pool or taking a
187 * device offline.
188 *
189 * When zfs_multihost_fail_intervals > 0, the pool will be suspended if
190 * zfs_multihost_fail_intervals * zfs_multihost_interval milliseconds pass
191 * without a successful mmp write. This guarantees the activity test will see
192 * mmp writes if the pool is imported. A value of 1 is ignored and treated as
193 * if it was set to 2, because a single leaf vdev pool will issue a write once
194 * per multihost_interval and thus any variation in latency would cause the
195 * pool to be suspended.
196 */
197 uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS;
198
199 static const void *const mmp_tag = "mmp_write_uberblock";
200 static __attribute__((noreturn)) void mmp_thread(void *arg);
201
202 void
mmp_init(spa_t * spa)203 mmp_init(spa_t *spa)
204 {
205 mmp_thread_t *mmp = &spa->spa_mmp;
206
207 mutex_init(&mmp->mmp_thread_lock, NULL, MUTEX_DEFAULT, NULL);
208 cv_init(&mmp->mmp_thread_cv, NULL, CV_DEFAULT, NULL);
209 mutex_init(&mmp->mmp_io_lock, NULL, MUTEX_DEFAULT, NULL);
210 mmp->mmp_kstat_id = 1;
211 }
212
213 void
mmp_fini(spa_t * spa)214 mmp_fini(spa_t *spa)
215 {
216 mmp_thread_t *mmp = &spa->spa_mmp;
217
218 mutex_destroy(&mmp->mmp_thread_lock);
219 cv_destroy(&mmp->mmp_thread_cv);
220 mutex_destroy(&mmp->mmp_io_lock);
221 }
222
223 static void
mmp_thread_enter(mmp_thread_t * mmp,callb_cpr_t * cpr)224 mmp_thread_enter(mmp_thread_t *mmp, callb_cpr_t *cpr)
225 {
226 CALLB_CPR_INIT(cpr, &mmp->mmp_thread_lock, callb_generic_cpr, FTAG);
227 mutex_enter(&mmp->mmp_thread_lock);
228 }
229
230 static void
mmp_thread_exit(mmp_thread_t * mmp,kthread_t ** mpp,callb_cpr_t * cpr)231 mmp_thread_exit(mmp_thread_t *mmp, kthread_t **mpp, callb_cpr_t *cpr)
232 {
233 ASSERT(*mpp != NULL);
234 *mpp = NULL;
235 cv_broadcast(&mmp->mmp_thread_cv);
236 CALLB_CPR_EXIT(cpr); /* drops &mmp->mmp_thread_lock */
237 }
238
239 void
mmp_thread_start(spa_t * spa)240 mmp_thread_start(spa_t *spa)
241 {
242 mmp_thread_t *mmp = &spa->spa_mmp;
243
244 if (spa_writeable(spa)) {
245 mutex_enter(&mmp->mmp_thread_lock);
246 if (!mmp->mmp_thread) {
247 mmp->mmp_thread = thread_create(NULL, 0, mmp_thread,
248 spa, 0, &p0, TS_RUN, defclsyspri);
249 zfs_dbgmsg("mmp: mmp thread started spa=%s "
250 "gethrtime=%llu", spa_name(spa), gethrtime());
251 }
252 mutex_exit(&mmp->mmp_thread_lock);
253 }
254 }
255
256 void
mmp_thread_stop(spa_t * spa)257 mmp_thread_stop(spa_t *spa)
258 {
259 mmp_thread_t *mmp = &spa->spa_mmp;
260
261 mutex_enter(&mmp->mmp_thread_lock);
262 mmp->mmp_thread_exiting = 1;
263 cv_broadcast(&mmp->mmp_thread_cv);
264
265 while (mmp->mmp_thread) {
266 cv_wait(&mmp->mmp_thread_cv, &mmp->mmp_thread_lock);
267 }
268 mutex_exit(&mmp->mmp_thread_lock);
269 zfs_dbgmsg("mmp: mmp thread stopped spa=%s gethrtime=%llu",
270 spa_name(spa), gethrtime());
271
272 ASSERT0P(mmp->mmp_thread);
273 mmp->mmp_thread_exiting = 0;
274 }
275
276 typedef enum mmp_vdev_state_flag {
277 MMP_FAIL_NOT_WRITABLE = (1 << 0),
278 MMP_FAIL_WRITE_PENDING = (1 << 1),
279 } mmp_vdev_state_flag_t;
280
281 /*
282 * Find a leaf vdev to write an MMP block to. It must not have an outstanding
283 * mmp write (if so a new write will also likely block). If there is no usable
284 * leaf, a nonzero error value is returned. The error value returned is a bit
285 * field.
286 *
287 * MMP_FAIL_WRITE_PENDING One or more leaf vdevs are writeable, but have an
288 * outstanding MMP write.
289 * MMP_FAIL_NOT_WRITABLE One or more leaf vdevs are not writeable.
290 */
291
292 static int
mmp_next_leaf(spa_t * spa)293 mmp_next_leaf(spa_t *spa)
294 {
295 vdev_t *leaf;
296 vdev_t *starting_leaf;
297 int fail_mask = 0;
298
299 ASSERT(MUTEX_HELD(&spa->spa_mmp.mmp_io_lock));
300 ASSERT(spa_config_held(spa, SCL_STATE, RW_READER));
301 ASSERT(list_link_active(&spa->spa_leaf_list.list_head) == B_TRUE);
302 ASSERT(!list_is_empty(&spa->spa_leaf_list));
303
304 if (spa->spa_mmp.mmp_leaf_last_gen != spa->spa_leaf_list_gen) {
305 spa->spa_mmp.mmp_last_leaf = list_head(&spa->spa_leaf_list);
306 spa->spa_mmp.mmp_leaf_last_gen = spa->spa_leaf_list_gen;
307 }
308
309 leaf = spa->spa_mmp.mmp_last_leaf;
310 if (leaf == NULL)
311 leaf = list_head(&spa->spa_leaf_list);
312 starting_leaf = leaf;
313
314 do {
315 leaf = list_next(&spa->spa_leaf_list, leaf);
316 if (leaf == NULL) {
317 leaf = list_head(&spa->spa_leaf_list);
318 ASSERT3P(leaf, !=, NULL);
319 }
320
321 /*
322 * We skip unwritable, offline, detached, and dRAID spare
323 * devices as they are either not legal targets or the write
324 * may fail or not be seen by other hosts. Skipped dRAID
325 * spares can never be written so the fail mask is not set.
326 */
327 if (!vdev_writeable(leaf) || leaf->vdev_offline ||
328 leaf->vdev_detached) {
329 fail_mask |= MMP_FAIL_NOT_WRITABLE;
330 } else if (leaf->vdev_ops == &vdev_draid_spare_ops) {
331 continue;
332 } else if (leaf->vdev_mmp_pending != 0) {
333 fail_mask |= MMP_FAIL_WRITE_PENDING;
334 } else {
335 spa->spa_mmp.mmp_last_leaf = leaf;
336 return (0);
337 }
338 } while (leaf != starting_leaf);
339
340 ASSERT(fail_mask);
341
342 return (fail_mask);
343 }
344
345 /*
346 * MMP writes are issued on a fixed schedule, but may complete at variable,
347 * much longer, intervals. The mmp_delay captures long periods between
348 * successful writes for any reason, including disk latency, scheduling delays,
349 * etc.
350 *
351 * The mmp_delay is usually calculated as a decaying average, but if the latest
352 * delay is higher we do not average it, so that we do not hide sudden spikes
353 * which the importing host must wait for.
354 *
355 * If writes are occurring frequently, such as due to a high rate of txg syncs,
356 * the mmp_delay could become very small. Since those short delays depend on
357 * activity we cannot count on, we never allow mmp_delay to get lower than rate
358 * expected if only mmp_thread writes occur.
359 *
360 * If an mmp write was skipped or fails, and we have already waited longer than
361 * mmp_delay, we need to update it so the next write reflects the longer delay.
362 *
363 * Do not set mmp_delay if the multihost property is not on, so as not to
364 * trigger an activity check on import.
365 */
366 static void
mmp_delay_update(spa_t * spa,boolean_t write_completed)367 mmp_delay_update(spa_t *spa, boolean_t write_completed)
368 {
369 mmp_thread_t *mts = &spa->spa_mmp;
370 hrtime_t delay = gethrtime() - mts->mmp_last_write;
371
372 ASSERT(MUTEX_HELD(&mts->mmp_io_lock));
373
374 if (spa_multihost(spa) == B_FALSE) {
375 mts->mmp_delay = 0;
376 return;
377 }
378
379 if (delay > mts->mmp_delay)
380 mts->mmp_delay = delay;
381
382 if (write_completed == B_FALSE)
383 return;
384
385 mts->mmp_last_write = gethrtime();
386
387 /*
388 * strictly less than, in case delay was changed above.
389 */
390 if (delay < mts->mmp_delay) {
391 hrtime_t min_delay =
392 MSEC2NSEC(MMP_INTERVAL_OK(zfs_multihost_interval)) /
393 MAX(1, vdev_count_leaves(spa));
394 mts->mmp_delay = MAX(((delay + mts->mmp_delay * 127) / 128),
395 min_delay);
396 }
397 }
398
399 static void
mmp_write_done(zio_t * zio)400 mmp_write_done(zio_t *zio)
401 {
402 spa_t *spa = zio->io_spa;
403 vdev_t *vd = zio->io_vd;
404 mmp_thread_t *mts = zio->io_private;
405
406 mutex_enter(&mts->mmp_io_lock);
407 uint64_t mmp_kstat_id = vd->vdev_mmp_kstat_id;
408 hrtime_t mmp_write_duration = gethrtime() - vd->vdev_mmp_pending;
409
410 mmp_delay_update(spa, (zio->io_error == 0));
411
412 vd->vdev_mmp_pending = 0;
413 vd->vdev_mmp_kstat_id = 0;
414
415 mutex_exit(&mts->mmp_io_lock);
416 spa_config_exit(spa, SCL_STATE, mmp_tag);
417
418 spa_mmp_history_set(spa, mmp_kstat_id, zio->io_error,
419 mmp_write_duration);
420
421 abd_free(zio->io_abd);
422 }
423
424 /*
425 * When the uberblock on-disk is updated by a spa_sync,
426 * creating a new "best" uberblock, update the one stored
427 * in the mmp thread state, used for mmp writes.
428 */
429 void
mmp_update_uberblock(spa_t * spa,uberblock_t * ub)430 mmp_update_uberblock(spa_t *spa, uberblock_t *ub)
431 {
432 mmp_thread_t *mmp = &spa->spa_mmp;
433
434 mutex_enter(&mmp->mmp_io_lock);
435 mmp->mmp_ub = *ub;
436 mmp->mmp_seq = 1;
437 mmp->mmp_ub.ub_timestamp = gethrestime_sec();
438 mmp_delay_update(spa, B_TRUE);
439 mutex_exit(&mmp->mmp_io_lock);
440 }
441
442 /*
443 * Choose a random vdev, label, and MMP block, and write over it
444 * with a copy of the last-synced uberblock, whose timestamp
445 * has been updated to reflect that the pool is in use.
446 */
447 static void
mmp_write_uberblock(spa_t * spa)448 mmp_write_uberblock(spa_t *spa)
449 {
450 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
451 mmp_thread_t *mmp = &spa->spa_mmp;
452 uberblock_t *ub;
453 vdev_t *vd = NULL;
454 int label, error;
455 uint64_t offset;
456
457 hrtime_t lock_acquire_time = gethrtime();
458 spa_config_enter_priority(spa, SCL_STATE, mmp_tag, RW_READER);
459 lock_acquire_time = gethrtime() - lock_acquire_time;
460 if (lock_acquire_time > (MSEC2NSEC(MMP_MIN_INTERVAL) / 10))
461 zfs_dbgmsg("mmp: long SCL_STATE acquisition, spa=%s "
462 "acquire_time=%llu gethrtime=%llu", spa_name(spa),
463 lock_acquire_time, gethrtime());
464
465 mutex_enter(&mmp->mmp_io_lock);
466
467 error = mmp_next_leaf(spa);
468
469 /*
470 * spa_mmp_history has two types of entries:
471 * Issued MMP write: records time issued, error status, etc.
472 * Skipped MMP write: an MMP write could not be issued because no
473 * suitable leaf vdev was available. See comment above struct
474 * spa_mmp_history for details.
475 */
476
477 if (error) {
478 mmp_delay_update(spa, B_FALSE);
479 if (mmp->mmp_skip_error == error) {
480 spa_mmp_history_set_skip(spa, mmp->mmp_kstat_id - 1);
481 } else {
482 mmp->mmp_skip_error = error;
483 spa_mmp_history_add(spa, mmp->mmp_ub.ub_txg,
484 gethrestime_sec(), mmp->mmp_delay, NULL, 0,
485 mmp->mmp_kstat_id++, error);
486 zfs_dbgmsg("mmp: error choosing leaf, spa=%s "
487 "gethrtime=%llu fail_mask=%#x", spa_name(spa),
488 gethrtime(), error);
489 }
490 mutex_exit(&mmp->mmp_io_lock);
491 spa_config_exit(spa, SCL_STATE, mmp_tag);
492 return;
493 }
494
495 vd = spa->spa_mmp.mmp_last_leaf;
496 if (mmp->mmp_skip_error != 0) {
497 zfs_dbgmsg("mmp: write after skipping due to unavailable "
498 "leaves, spa=%s gethrtime=%llu vdev=%llu error=%d",
499 spa_name(spa), (u_longlong_t)gethrtime(),
500 (u_longlong_t)vd->vdev_guid, mmp->mmp_skip_error);
501 mmp->mmp_skip_error = 0;
502 }
503
504 if (mmp->mmp_zio_root == NULL)
505 mmp->mmp_zio_root = zio_root(spa, NULL, NULL,
506 flags | ZIO_FLAG_GODFATHER);
507
508 if (mmp->mmp_ub.ub_timestamp != gethrestime_sec()) {
509 /*
510 * Want to reset mmp_seq when timestamp advances because after
511 * an mmp_seq wrap new values will not be chosen by
512 * uberblock_compare() as the "best".
513 */
514 mmp->mmp_ub.ub_timestamp = gethrestime_sec();
515 mmp->mmp_seq = 1;
516 }
517
518 ub = &mmp->mmp_ub;
519 ub->ub_mmp_magic = MMP_MAGIC;
520 ub->ub_mmp_delay = mmp->mmp_delay;
521 ub->ub_mmp_config = MMP_SEQ_SET(mmp->mmp_seq) |
522 MMP_INTERVAL_SET(MMP_INTERVAL_OK(zfs_multihost_interval)) |
523 MMP_FAIL_INT_SET(MMP_FAIL_INTVS_OK(
524 zfs_multihost_fail_intervals));
525 vd->vdev_mmp_pending = gethrtime();
526 vd->vdev_mmp_kstat_id = mmp->mmp_kstat_id;
527
528 zio_t *zio = zio_null(mmp->mmp_zio_root, spa, NULL, NULL, NULL, flags);
529 abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
530 abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t));
531 abd_zero_off(ub_abd, sizeof (uberblock_t),
532 VDEV_UBERBLOCK_SIZE(vd) - sizeof (uberblock_t));
533
534 mmp->mmp_seq++;
535 mmp->mmp_kstat_id++;
536 mutex_exit(&mmp->mmp_io_lock);
537
538 offset = VDEV_UBERBLOCK_OFFSET(vd, VDEV_UBERBLOCK_COUNT(vd) -
539 MMP_BLOCKS_PER_LABEL + random_in_range(MMP_BLOCKS_PER_LABEL));
540
541 label = random_in_range(VDEV_LABELS);
542 vdev_label_write(zio, vd, label, ub_abd, offset,
543 VDEV_UBERBLOCK_SIZE(vd), mmp_write_done, mmp,
544 flags | ZIO_FLAG_DONT_PROPAGATE);
545
546 (void) spa_mmp_history_add(spa, ub->ub_txg, ub->ub_timestamp,
547 ub->ub_mmp_delay, vd, label, vd->vdev_mmp_kstat_id, 0);
548
549 zio_nowait(zio);
550 }
551
552 static void
mmp_claim_uberblock_sync_done(zio_t * zio)553 mmp_claim_uberblock_sync_done(zio_t *zio)
554 {
555 uint64_t *good_writes = zio->io_private;
556
557 if (zio->io_error == 0 && zio->io_vd->vdev_top->vdev_ms_array != 0)
558 atomic_inc_64(good_writes);
559 }
560
561 /*
562 * Write the uberblock to the first label of all leaves of the specified vdev.
563 * Two writes required for each mirror, one for a singleton, and parity+1 for
564 * raidz or draid vdevs.
565 */
566 static void
mmp_claim_uberblock_sync(zio_t * zio,uint64_t * good_writes,uint64_t * req_writes,uberblock_t * ub,vdev_t * vd,int flags)567 mmp_claim_uberblock_sync(zio_t *zio, uint64_t *good_writes,
568 uint64_t *req_writes, uberblock_t *ub, vdev_t *vd, int flags)
569 {
570 for (uint64_t c = 0; c < vd->vdev_children; c++) {
571 vdev_t *cvd = vd->vdev_child[c];
572
573 if (cvd->vdev_islog || cvd->vdev_isspare || cvd->vdev_isl2cache)
574 continue;
575
576 if (cvd->vdev_top == cvd) {
577 uint64_t nparity = vdev_get_nparity(cvd);
578 if (nparity) {
579 *req_writes += nparity + 1;
580 } else {
581 *req_writes +=
582 MIN(MAX(cvd->vdev_children, 1), 2);
583 }
584 }
585
586 mmp_claim_uberblock_sync(zio, good_writes, req_writes,
587 ub, cvd, flags);
588 }
589
590 if (!vd->vdev_ops->vdev_op_leaf)
591 return;
592
593 if (!vdev_writeable(vd))
594 return;
595
596 if (vd->vdev_ops == &vdev_draid_spare_ops)
597 return;
598
599 abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
600 abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t));
601 abd_zero_off(ub_abd, sizeof (uberblock_t),
602 VDEV_UBERBLOCK_SIZE(vd) - sizeof (uberblock_t));
603
604 vdev_label_write(zio, vd, 0, ub_abd,
605 VDEV_UBERBLOCK_OFFSET(vd, VDEV_UBERBLOCK_COUNT(vd) -
606 MMP_BLOCKS_PER_LABEL), VDEV_UBERBLOCK_SIZE(vd),
607 mmp_claim_uberblock_sync_done, good_writes,
608 flags | ZIO_FLAG_DONT_PROPAGATE);
609
610 abd_free(ub_abd);
611 }
612
613 int
mmp_claim_uberblock(spa_t * spa,vdev_t * vd,uberblock_t * ub)614 mmp_claim_uberblock(spa_t *spa, vdev_t *vd, uberblock_t *ub)
615 {
616 int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
617 uint64_t good_writes = 0;
618 uint64_t req_writes = 0;
619 zio_t *zio;
620
621 ASSERT(MMP_VALID(ub));
622 ASSERT(MMP_SEQ_VALID(ub));
623
624 spa_config_enter(spa, SCL_ALL, mmp_tag, RW_WRITER);
625
626 /* Sync the uberblock to all writeable leaves */
627 zio = zio_root(spa, NULL, NULL, flags);
628 mmp_claim_uberblock_sync(zio, &good_writes, &req_writes, ub, vd, flags);
629 (void) zio_wait(zio);
630
631 /* Flush the new uberblocks so they're immediately visible */
632 zio = zio_root(spa, NULL, NULL, flags);
633 zio_flush(zio, vd);
634 (void) zio_wait(zio);
635
636 spa_config_exit(spa, SCL_ALL, mmp_tag);
637
638 zfs_dbgmsg("mmp: claiming uberblock, spa=%s txg=%llu seq=%llu "
639 "req_writes=%llu good_writes=%llu", spa_load_name(spa),
640 (u_longlong_t)ub->ub_txg, (u_longlong_t)MMP_SEQ(ub),
641 (u_longlong_t)req_writes, (u_longlong_t)good_writes);
642
643 /*
644 * To guarantee visibility from a remote host we require a minimum
645 * number of good writes. For raidz/draid vdevs parity+1 writes, for
646 * mirrors 2 writes, and for singletons 1 write.
647 */
648 if (req_writes == 0 || good_writes < req_writes)
649 return (SET_ERROR(EIO));
650
651 return (0);
652 }
653
654 static __attribute__((noreturn)) void
mmp_thread(void * arg)655 mmp_thread(void *arg)
656 {
657 spa_t *spa = (spa_t *)arg;
658 mmp_thread_t *mmp = &spa->spa_mmp;
659 boolean_t suspended = spa_suspended(spa);
660 boolean_t multihost = spa_multihost(spa);
661 uint64_t mmp_interval = MSEC2NSEC(MMP_INTERVAL_OK(
662 zfs_multihost_interval));
663 uint32_t mmp_fail_intervals = MMP_FAIL_INTVS_OK(
664 zfs_multihost_fail_intervals);
665 hrtime_t mmp_fail_ns = mmp_fail_intervals * mmp_interval;
666 boolean_t last_spa_suspended;
667 boolean_t last_spa_multihost;
668 uint64_t last_mmp_interval;
669 uint32_t last_mmp_fail_intervals;
670 hrtime_t last_mmp_fail_ns;
671 callb_cpr_t cpr;
672 int skip_wait = 0;
673
674 mmp_thread_enter(mmp, &cpr);
675
676 /*
677 * There have been no MMP writes yet. Setting mmp_last_write here gives
678 * us one mmp_fail_ns period, which is consistent with the activity
679 * check duration, to try to land an MMP write before MMP suspends the
680 * pool (if so configured).
681 */
682
683 mutex_enter(&mmp->mmp_io_lock);
684 mmp->mmp_last_write = gethrtime();
685 mmp->mmp_delay = MSEC2NSEC(MMP_INTERVAL_OK(zfs_multihost_interval));
686 mutex_exit(&mmp->mmp_io_lock);
687
688 while (!mmp->mmp_thread_exiting) {
689 hrtime_t next_time = gethrtime() +
690 MSEC2NSEC(MMP_DEFAULT_INTERVAL);
691 int leaves = MAX(vdev_count_leaves(spa), 1);
692
693 /* Detect changes in tunables or state */
694
695 last_spa_suspended = suspended;
696 last_spa_multihost = multihost;
697 suspended = spa_suspended(spa);
698 multihost = spa_multihost(spa);
699
700 last_mmp_interval = mmp_interval;
701 last_mmp_fail_intervals = mmp_fail_intervals;
702 last_mmp_fail_ns = mmp_fail_ns;
703 mmp_interval = MSEC2NSEC(MMP_INTERVAL_OK(
704 zfs_multihost_interval));
705 mmp_fail_intervals = MMP_FAIL_INTVS_OK(
706 zfs_multihost_fail_intervals);
707
708 /* Smooth so pool is not suspended when reducing tunables */
709 if (mmp_fail_intervals * mmp_interval < mmp_fail_ns) {
710 mmp_fail_ns = (mmp_fail_ns * 31 +
711 mmp_fail_intervals * mmp_interval) / 32;
712 } else {
713 mmp_fail_ns = mmp_fail_intervals *
714 mmp_interval;
715 }
716
717 if (mmp_interval != last_mmp_interval ||
718 mmp_fail_intervals != last_mmp_fail_intervals) {
719 /*
720 * We want other hosts to see new tunables as quickly as
721 * possible. Write out at higher frequency than usual.
722 */
723 skip_wait += leaves;
724 }
725
726 if (multihost)
727 next_time = gethrtime() + mmp_interval / leaves;
728
729 if (mmp_fail_ns != last_mmp_fail_ns) {
730 zfs_dbgmsg("mmp: interval change, spa=%s "
731 "gethrtime=%llu last_mmp_interval=%llu "
732 "mmp_interval=%llu last_mmp_fail_intervals=%u "
733 "mmp_fail_intervals=%u mmp_fail_ns=%llu "
734 "skip_wait=%d leaves=%d next_time=%llu",
735 spa_name(spa), (u_longlong_t)gethrtime(),
736 (u_longlong_t)last_mmp_interval,
737 (u_longlong_t)mmp_interval, last_mmp_fail_intervals,
738 mmp_fail_intervals, (u_longlong_t)mmp_fail_ns,
739 skip_wait, leaves, (u_longlong_t)next_time);
740 }
741
742 /*
743 * MMP off => on, or suspended => !suspended:
744 * No writes occurred recently. Update mmp_last_write to give
745 * us some time to try.
746 */
747 if ((!last_spa_multihost && multihost) ||
748 (last_spa_suspended && !suspended)) {
749 zfs_dbgmsg("mmp: state change spa=%s: gethrtime=%llu "
750 "last_spa_multihost=%u multihost=%u "
751 "last_spa_suspended=%u suspended=%u",
752 spa_name(spa), (u_longlong_t)gethrtime(),
753 last_spa_multihost, multihost, last_spa_suspended,
754 suspended);
755 mutex_enter(&mmp->mmp_io_lock);
756 mmp->mmp_last_write = gethrtime();
757 mmp->mmp_delay = mmp_interval;
758 mutex_exit(&mmp->mmp_io_lock);
759 }
760
761 /*
762 * MMP on => off:
763 * mmp_delay == 0 tells importing node to skip activity check.
764 */
765 if (last_spa_multihost && !multihost) {
766 mutex_enter(&mmp->mmp_io_lock);
767 mmp->mmp_delay = 0;
768 mutex_exit(&mmp->mmp_io_lock);
769 }
770
771 /*
772 * Suspend the pool if no MMP write has succeeded in over
773 * mmp_interval * mmp_fail_intervals nanoseconds.
774 */
775 if (multihost && !suspended && mmp_fail_intervals &&
776 (gethrtime() - mmp->mmp_last_write) > mmp_fail_ns) {
777 zfs_dbgmsg("mmp: suspending pool, spa=%s "
778 "gethrtime=%llu mmp_last_write=%llu "
779 "mmp_interval=%llu mmp_fail_intervals=%llu "
780 "mmp_fail_ns=%llu txg=%llu",
781 spa_name(spa), (u_longlong_t)gethrtime(),
782 (u_longlong_t)mmp->mmp_last_write,
783 (u_longlong_t)mmp_interval,
784 (u_longlong_t)mmp_fail_intervals,
785 (u_longlong_t)mmp_fail_ns,
786 (u_longlong_t)spa->spa_uberblock.ub_txg);
787 cmn_err(CE_WARN, "MMP writes to pool '%s' have not "
788 "succeeded in over %llu ms; suspending pool. "
789 "Hrtime %llu",
790 spa_name(spa),
791 NSEC2MSEC(gethrtime() - mmp->mmp_last_write),
792 gethrtime());
793 zio_suspend(spa, NULL, ZIO_SUSPEND_MMP);
794 }
795
796 if (multihost && !suspended)
797 mmp_write_uberblock(spa);
798
799 if (skip_wait > 0) {
800 next_time = gethrtime() + MSEC2NSEC(MMP_MIN_INTERVAL) /
801 leaves;
802 skip_wait--;
803 }
804
805 CALLB_CPR_SAFE_BEGIN(&cpr);
806 (void) cv_timedwait_idle_hires(&mmp->mmp_thread_cv,
807 &mmp->mmp_thread_lock, next_time, USEC2NSEC(100),
808 CALLOUT_FLAG_ABSOLUTE);
809 CALLB_CPR_SAFE_END(&cpr, &mmp->mmp_thread_lock);
810 }
811
812 /* Outstanding writes are allowed to complete. */
813 zio_wait(mmp->mmp_zio_root);
814
815 mmp->mmp_zio_root = NULL;
816 mmp_thread_exit(mmp, &mmp->mmp_thread, &cpr);
817
818 thread_exit();
819 }
820
821 /*
822 * Signal the MMP thread to wake it, when it is sleeping on
823 * its cv. Used when some module parameter has changed and
824 * we want the thread to know about it.
825 * Only signal if the pool is active and mmp thread is
826 * running, otherwise there is no thread to wake.
827 */
828 static void
mmp_signal_thread(spa_t * spa)829 mmp_signal_thread(spa_t *spa)
830 {
831 mmp_thread_t *mmp = &spa->spa_mmp;
832
833 mutex_enter(&mmp->mmp_thread_lock);
834 if (mmp->mmp_thread)
835 cv_broadcast(&mmp->mmp_thread_cv);
836 mutex_exit(&mmp->mmp_thread_lock);
837 }
838
839 void
mmp_signal_all_threads(void)840 mmp_signal_all_threads(void)
841 {
842 spa_t *spa = NULL;
843
844 spa_namespace_enter(FTAG);
845 while ((spa = spa_next(spa))) {
846 if (spa->spa_state == POOL_STATE_ACTIVE)
847 mmp_signal_thread(spa);
848 }
849 spa_namespace_exit(FTAG);
850 }
851
852 ZFS_MODULE_PARAM_CALL(zfs_multihost, zfs_multihost_, interval,
853 param_set_multihost_interval, spl_param_get_u64, ZMOD_RW,
854 "Milliseconds between mmp writes to each leaf");
855
856 ZFS_MODULE_PARAM(zfs_multihost, zfs_multihost_, fail_intervals, UINT, ZMOD_RW,
857 "Max allowed period without a successful mmp write");
858
859 ZFS_MODULE_PARAM(zfs_multihost, zfs_multihost_, import_intervals, UINT, ZMOD_RW,
860 "Number of zfs_multihost_interval periods to wait for activity");
861