xref: /freebsd/sys/contrib/openzfs/module/zfs/mmp.c (revision e92ffd9b626833ebdbf2742c8ffddc6cd94b963e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
23  */
24 
25 #include <sys/abd.h>
26 #include <sys/mmp.h>
27 #include <sys/spa.h>
28 #include <sys/spa_impl.h>
29 #include <sys/time.h>
30 #include <sys/vdev.h>
31 #include <sys/vdev_impl.h>
32 #include <sys/zfs_context.h>
33 #include <sys/callb.h>
34 
35 /*
36  * Multi-Modifier Protection (MMP) attempts to prevent a user from importing
37  * or opening a pool on more than one host at a time.  In particular, it
38  * prevents "zpool import -f" on a host from succeeding while the pool is
39  * already imported on another host.  There are many other ways in which a
40  * device could be used by two hosts for different purposes at the same time
41  * resulting in pool damage.  This implementation does not attempt to detect
42  * those cases.
43  *
44  * MMP operates by ensuring there are frequent visible changes on disk (a
45  * "heartbeat") at all times.  And by altering the import process to check
46  * for these changes and failing the import when they are detected.  This
47  * functionality is enabled by setting the 'multihost' pool property to on.
48  *
49  * Uberblocks written by the txg_sync thread always go into the first
50  * (N-MMP_BLOCKS_PER_LABEL) slots, the remaining slots are reserved for MMP.
51  * They are used to hold uberblocks which are exactly the same as the last
52  * synced uberblock except that the ub_timestamp and mmp_config are frequently
53  * updated.  Like all other uberblocks, the slot is written with an embedded
54  * checksum, and slots with invalid checksums are ignored.  This provides the
55  * "heartbeat", with no risk of overwriting good uberblocks that must be
56  * preserved, e.g. previous txgs and associated block pointers.
57  *
58  * Three optional fields are added to uberblock structure; ub_mmp_magic,
59  * ub_mmp_config, and ub_mmp_delay.  The ub_mmp_magic value allows zfs to tell
60  * whether the other ub_mmp_* fields are valid.  The ub_mmp_config field tells
61  * the importing host the settings of zfs_multihost_interval and
62  * zfs_multihost_fail_intervals on the host which last had (or currently has)
63  * the pool imported.  These determine how long a host must wait to detect
64  * activity in the pool, before concluding the pool is not in use.  The
65  * mmp_delay field is a decaying average of the amount of time between
66  * completion of successive MMP writes, in nanoseconds.  It indicates whether
67  * MMP is enabled.
68  *
69  * During import an activity test may now be performed to determine if
70  * the pool is in use.  The activity test is typically required if the
71  * ZPOOL_CONFIG_HOSTID does not match the system hostid, the pool state is
72  * POOL_STATE_ACTIVE, and the pool is not a root pool.
73  *
74  * The activity test finds the "best" uberblock (highest txg, timestamp, and, if
75  * ub_mmp_magic is valid, sequence number from ub_mmp_config).  It then waits
76  * some time, and finds the "best" uberblock again.  If any of the mentioned
77  * fields have different values in the newly read uberblock, the pool is in use
78  * by another host and the import fails.  In order to assure the accuracy of the
79  * activity test, the default values result in an activity test duration of 20x
80  * the mmp write interval.
81  *
82  * The duration of the "zpool import" activity test depends on the information
83  * available in the "best" uberblock:
84  *
85  * 1) If uberblock was written by zfs-0.8 or newer and fail_intervals > 0:
86  *    ub_mmp_config.fail_intervals * ub_mmp_config.multihost_interval * 2
87  *
88  *    In this case, a weak guarantee is provided.  Since the host which last had
89  *    the pool imported will suspend the pool if no mmp writes land within
90  *    fail_intervals * multihost_interval ms, the absence of writes during that
91  *    time means either the pool is not imported, or it is imported but the pool
92  *    is suspended and no further writes will occur.
93  *
94  *    Note that resuming the suspended pool on the remote host would invalidate
95  *    this guarantee, and so it is not allowed.
96  *
97  *    The factor of 2 provides a conservative safety factor and derives from
98  *    MMP_IMPORT_SAFETY_FACTOR;
99  *
100  * 2) If uberblock was written by zfs-0.8 or newer and fail_intervals == 0:
101  *    (ub_mmp_config.multihost_interval + ub_mmp_delay) *
102  *        zfs_multihost_import_intervals
103  *
104  *    In this case no guarantee can provided.  However, as long as some devices
105  *    are healthy and connected, it is likely that at least one write will land
106  *    within (multihost_interval + mmp_delay) because multihost_interval is
107  *    enough time for a write to be attempted to each leaf vdev, and mmp_delay
108  *    is enough for one to land, based on past delays.  Multiplying by
109  *    zfs_multihost_import_intervals provides a conservative safety factor.
110  *
111  * 3) If uberblock was written by zfs-0.7:
112  *    (zfs_multihost_interval + ub_mmp_delay) * zfs_multihost_import_intervals
113  *
114  *    The same logic as case #2 applies, but we do not know remote tunables.
115  *
116  *    We use the local value for zfs_multihost_interval because the original MMP
117  *    did not record this value in the uberblock.
118  *
119  *    ub_mmp_delay >= (zfs_multihost_interval / leaves), so if the other host
120  *    has a much larger zfs_multihost_interval set, ub_mmp_delay will reflect
121  *    that.  We will have waited enough time for zfs_multihost_import_intervals
122  *    writes to be issued and all but one to land.
123  *
124  *    single device pool example delays
125  *
126  *    import_delay = (1 + 1) * 20   =  40s #defaults, no I/O delay
127  *    import_delay = (1 + 10) * 20  = 220s #defaults, 10s I/O delay
128  *    import_delay = (10 + 10) * 20 = 400s #10s multihost_interval,
129  *                                          no I/O delay
130  *    100 device pool example delays
131  *
132  *    import_delay = (1 + .01) * 20 =  20s #defaults, no I/O delay
133  *    import_delay = (1 + 10) * 20  = 220s #defaults, 10s I/O delay
134  *    import_delay = (10 + .1) * 20 = 202s #10s multihost_interval,
135  *                                          no I/O delay
136  *
137  * 4) Otherwise, this uberblock was written by a pre-MMP zfs:
138  *    zfs_multihost_import_intervals * zfs_multihost_interval
139  *
140  *    In this case local tunables are used.  By default this product = 10s, long
141  *    enough for a pool with any activity at all to write at least one
142  *    uberblock.  No guarantee can be provided.
143  *
144  * Additionally, the duration is then extended by a random 25% to attempt to to
145  * detect simultaneous imports.  For example, if both partner hosts are rebooted
146  * at the same time and automatically attempt to import the pool.
147  */
148 
149 /*
150  * Used to control the frequency of mmp writes which are performed when the
151  * 'multihost' pool property is on.  This is one factor used to determine the
152  * length of the activity check during import.
153  *
154  * On average an mmp write will be issued for each leaf vdev every
155  * zfs_multihost_interval milliseconds.  In practice, the observed period can
156  * vary with the I/O load and this observed value is the ub_mmp_delay which is
157  * stored in the uberblock.  The minimum allowed value is 100 ms.
158  */
159 ulong_t zfs_multihost_interval = MMP_DEFAULT_INTERVAL;
160 
161 /*
162  * Used to control the duration of the activity test on import.  Smaller values
163  * of zfs_multihost_import_intervals will reduce the import time but increase
164  * the risk of failing to detect an active pool.  The total activity check time
165  * is never allowed to drop below one second.  A value of 0 is ignored and
166  * treated as if it was set to 1.
167  */
168 uint_t zfs_multihost_import_intervals = MMP_DEFAULT_IMPORT_INTERVALS;
169 
170 /*
171  * Controls the behavior of the pool when mmp write failures or delays are
172  * detected.
173  *
174  * When zfs_multihost_fail_intervals = 0, mmp write failures or delays are
175  * ignored.  The failures will still be reported to the ZED which depending on
176  * its configuration may take action such as suspending the pool or taking a
177  * device offline.
178  *
179  * When zfs_multihost_fail_intervals > 0, the pool will be suspended if
180  * zfs_multihost_fail_intervals * zfs_multihost_interval milliseconds pass
181  * without a successful mmp write.  This guarantees the activity test will see
182  * mmp writes if the pool is imported.  A value of 1 is ignored and treated as
183  * if it was set to 2, because a single leaf vdev pool will issue a write once
184  * per multihost_interval and thus any variation in latency would cause the
185  * pool to be suspended.
186  */
187 uint_t zfs_multihost_fail_intervals = MMP_DEFAULT_FAIL_INTERVALS;
188 
189 static void *const mmp_tag = "mmp_write_uberblock";
190 static void mmp_thread(void *arg);
191 
192 void
193 mmp_init(spa_t *spa)
194 {
195 	mmp_thread_t *mmp = &spa->spa_mmp;
196 
197 	mutex_init(&mmp->mmp_thread_lock, NULL, MUTEX_DEFAULT, NULL);
198 	cv_init(&mmp->mmp_thread_cv, NULL, CV_DEFAULT, NULL);
199 	mutex_init(&mmp->mmp_io_lock, NULL, MUTEX_DEFAULT, NULL);
200 	mmp->mmp_kstat_id = 1;
201 }
202 
203 void
204 mmp_fini(spa_t *spa)
205 {
206 	mmp_thread_t *mmp = &spa->spa_mmp;
207 
208 	mutex_destroy(&mmp->mmp_thread_lock);
209 	cv_destroy(&mmp->mmp_thread_cv);
210 	mutex_destroy(&mmp->mmp_io_lock);
211 }
212 
213 static void
214 mmp_thread_enter(mmp_thread_t *mmp, callb_cpr_t *cpr)
215 {
216 	CALLB_CPR_INIT(cpr, &mmp->mmp_thread_lock, callb_generic_cpr, FTAG);
217 	mutex_enter(&mmp->mmp_thread_lock);
218 }
219 
220 static void
221 mmp_thread_exit(mmp_thread_t *mmp, kthread_t **mpp, callb_cpr_t *cpr)
222 {
223 	ASSERT(*mpp != NULL);
224 	*mpp = NULL;
225 	cv_broadcast(&mmp->mmp_thread_cv);
226 	CALLB_CPR_EXIT(cpr);		/* drops &mmp->mmp_thread_lock */
227 	thread_exit();
228 }
229 
230 void
231 mmp_thread_start(spa_t *spa)
232 {
233 	mmp_thread_t *mmp = &spa->spa_mmp;
234 
235 	if (spa_writeable(spa)) {
236 		mutex_enter(&mmp->mmp_thread_lock);
237 		if (!mmp->mmp_thread) {
238 			mmp->mmp_thread = thread_create(NULL, 0, mmp_thread,
239 			    spa, 0, &p0, TS_RUN, defclsyspri);
240 			zfs_dbgmsg("MMP thread started pool '%s' "
241 			    "gethrtime %llu", spa_name(spa), gethrtime());
242 		}
243 		mutex_exit(&mmp->mmp_thread_lock);
244 	}
245 }
246 
247 void
248 mmp_thread_stop(spa_t *spa)
249 {
250 	mmp_thread_t *mmp = &spa->spa_mmp;
251 
252 	mutex_enter(&mmp->mmp_thread_lock);
253 	mmp->mmp_thread_exiting = 1;
254 	cv_broadcast(&mmp->mmp_thread_cv);
255 
256 	while (mmp->mmp_thread) {
257 		cv_wait(&mmp->mmp_thread_cv, &mmp->mmp_thread_lock);
258 	}
259 	mutex_exit(&mmp->mmp_thread_lock);
260 	zfs_dbgmsg("MMP thread stopped pool '%s' gethrtime %llu",
261 	    spa_name(spa), gethrtime());
262 
263 	ASSERT(mmp->mmp_thread == NULL);
264 	mmp->mmp_thread_exiting = 0;
265 }
266 
267 typedef enum mmp_vdev_state_flag {
268 	MMP_FAIL_NOT_WRITABLE	= (1 << 0),
269 	MMP_FAIL_WRITE_PENDING	= (1 << 1),
270 } mmp_vdev_state_flag_t;
271 
272 /*
273  * Find a leaf vdev to write an MMP block to.  It must not have an outstanding
274  * mmp write (if so a new write will also likely block).  If there is no usable
275  * leaf, a nonzero error value is returned. The error value returned is a bit
276  * field.
277  *
278  * MMP_FAIL_WRITE_PENDING   One or more leaf vdevs are writeable, but have an
279  *                          outstanding MMP write.
280  * MMP_FAIL_NOT_WRITABLE    One or more leaf vdevs are not writeable.
281  */
282 
283 static int
284 mmp_next_leaf(spa_t *spa)
285 {
286 	vdev_t *leaf;
287 	vdev_t *starting_leaf;
288 	int fail_mask = 0;
289 
290 	ASSERT(MUTEX_HELD(&spa->spa_mmp.mmp_io_lock));
291 	ASSERT(spa_config_held(spa, SCL_STATE, RW_READER));
292 	ASSERT(list_link_active(&spa->spa_leaf_list.list_head) == B_TRUE);
293 	ASSERT(!list_is_empty(&spa->spa_leaf_list));
294 
295 	if (spa->spa_mmp.mmp_leaf_last_gen != spa->spa_leaf_list_gen) {
296 		spa->spa_mmp.mmp_last_leaf = list_head(&spa->spa_leaf_list);
297 		spa->spa_mmp.mmp_leaf_last_gen = spa->spa_leaf_list_gen;
298 	}
299 
300 	leaf = spa->spa_mmp.mmp_last_leaf;
301 	if (leaf == NULL)
302 		leaf = list_head(&spa->spa_leaf_list);
303 	starting_leaf = leaf;
304 
305 	do {
306 		leaf = list_next(&spa->spa_leaf_list, leaf);
307 		if (leaf == NULL)
308 			leaf = list_head(&spa->spa_leaf_list);
309 
310 		/*
311 		 * We skip unwritable, offline, detached, and dRAID spare
312 		 * devices as they are either not legal targets or the write
313 		 * may fail or not be seen by other hosts.  Skipped dRAID
314 		 * spares can never be written so the fail mask is not set.
315 		 */
316 		if (!vdev_writeable(leaf) || leaf->vdev_offline ||
317 		    leaf->vdev_detached) {
318 			fail_mask |= MMP_FAIL_NOT_WRITABLE;
319 		} else if (leaf->vdev_ops == &vdev_draid_spare_ops) {
320 			continue;
321 		} else if (leaf->vdev_mmp_pending != 0) {
322 			fail_mask |= MMP_FAIL_WRITE_PENDING;
323 		} else {
324 			spa->spa_mmp.mmp_last_leaf = leaf;
325 			return (0);
326 		}
327 	} while (leaf != starting_leaf);
328 
329 	ASSERT(fail_mask);
330 
331 	return (fail_mask);
332 }
333 
334 /*
335  * MMP writes are issued on a fixed schedule, but may complete at variable,
336  * much longer, intervals.  The mmp_delay captures long periods between
337  * successful writes for any reason, including disk latency, scheduling delays,
338  * etc.
339  *
340  * The mmp_delay is usually calculated as a decaying average, but if the latest
341  * delay is higher we do not average it, so that we do not hide sudden spikes
342  * which the importing host must wait for.
343  *
344  * If writes are occurring frequently, such as due to a high rate of txg syncs,
345  * the mmp_delay could become very small.  Since those short delays depend on
346  * activity we cannot count on, we never allow mmp_delay to get lower than rate
347  * expected if only mmp_thread writes occur.
348  *
349  * If an mmp write was skipped or fails, and we have already waited longer than
350  * mmp_delay, we need to update it so the next write reflects the longer delay.
351  *
352  * Do not set mmp_delay if the multihost property is not on, so as not to
353  * trigger an activity check on import.
354  */
355 static void
356 mmp_delay_update(spa_t *spa, boolean_t write_completed)
357 {
358 	mmp_thread_t *mts = &spa->spa_mmp;
359 	hrtime_t delay = gethrtime() - mts->mmp_last_write;
360 
361 	ASSERT(MUTEX_HELD(&mts->mmp_io_lock));
362 
363 	if (spa_multihost(spa) == B_FALSE) {
364 		mts->mmp_delay = 0;
365 		return;
366 	}
367 
368 	if (delay > mts->mmp_delay)
369 		mts->mmp_delay = delay;
370 
371 	if (write_completed == B_FALSE)
372 		return;
373 
374 	mts->mmp_last_write = gethrtime();
375 
376 	/*
377 	 * strictly less than, in case delay was changed above.
378 	 */
379 	if (delay < mts->mmp_delay) {
380 		hrtime_t min_delay =
381 		    MSEC2NSEC(MMP_INTERVAL_OK(zfs_multihost_interval)) /
382 		    MAX(1, vdev_count_leaves(spa));
383 		mts->mmp_delay = MAX(((delay + mts->mmp_delay * 127) / 128),
384 		    min_delay);
385 	}
386 }
387 
388 static void
389 mmp_write_done(zio_t *zio)
390 {
391 	spa_t *spa = zio->io_spa;
392 	vdev_t *vd = zio->io_vd;
393 	mmp_thread_t *mts = zio->io_private;
394 
395 	mutex_enter(&mts->mmp_io_lock);
396 	uint64_t mmp_kstat_id = vd->vdev_mmp_kstat_id;
397 	hrtime_t mmp_write_duration = gethrtime() - vd->vdev_mmp_pending;
398 
399 	mmp_delay_update(spa, (zio->io_error == 0));
400 
401 	vd->vdev_mmp_pending = 0;
402 	vd->vdev_mmp_kstat_id = 0;
403 
404 	mutex_exit(&mts->mmp_io_lock);
405 	spa_config_exit(spa, SCL_STATE, mmp_tag);
406 
407 	spa_mmp_history_set(spa, mmp_kstat_id, zio->io_error,
408 	    mmp_write_duration);
409 
410 	abd_free(zio->io_abd);
411 }
412 
413 /*
414  * When the uberblock on-disk is updated by a spa_sync,
415  * creating a new "best" uberblock, update the one stored
416  * in the mmp thread state, used for mmp writes.
417  */
418 void
419 mmp_update_uberblock(spa_t *spa, uberblock_t *ub)
420 {
421 	mmp_thread_t *mmp = &spa->spa_mmp;
422 
423 	mutex_enter(&mmp->mmp_io_lock);
424 	mmp->mmp_ub = *ub;
425 	mmp->mmp_seq = 1;
426 	mmp->mmp_ub.ub_timestamp = gethrestime_sec();
427 	mmp_delay_update(spa, B_TRUE);
428 	mutex_exit(&mmp->mmp_io_lock);
429 }
430 
431 /*
432  * Choose a random vdev, label, and MMP block, and write over it
433  * with a copy of the last-synced uberblock, whose timestamp
434  * has been updated to reflect that the pool is in use.
435  */
436 static void
437 mmp_write_uberblock(spa_t *spa)
438 {
439 	int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
440 	mmp_thread_t *mmp = &spa->spa_mmp;
441 	uberblock_t *ub;
442 	vdev_t *vd = NULL;
443 	int label, error;
444 	uint64_t offset;
445 
446 	hrtime_t lock_acquire_time = gethrtime();
447 	spa_config_enter(spa, SCL_STATE, mmp_tag, RW_READER);
448 	lock_acquire_time = gethrtime() - lock_acquire_time;
449 	if (lock_acquire_time > (MSEC2NSEC(MMP_MIN_INTERVAL) / 10))
450 		zfs_dbgmsg("MMP SCL_STATE acquisition pool '%s' took %llu ns "
451 		    "gethrtime %llu", spa_name(spa), lock_acquire_time,
452 		    gethrtime());
453 
454 	mutex_enter(&mmp->mmp_io_lock);
455 
456 	error = mmp_next_leaf(spa);
457 
458 	/*
459 	 * spa_mmp_history has two types of entries:
460 	 * Issued MMP write: records time issued, error status, etc.
461 	 * Skipped MMP write: an MMP write could not be issued because no
462 	 * suitable leaf vdev was available.  See comment above struct
463 	 * spa_mmp_history for details.
464 	 */
465 
466 	if (error) {
467 		mmp_delay_update(spa, B_FALSE);
468 		if (mmp->mmp_skip_error == error) {
469 			spa_mmp_history_set_skip(spa, mmp->mmp_kstat_id - 1);
470 		} else {
471 			mmp->mmp_skip_error = error;
472 			spa_mmp_history_add(spa, mmp->mmp_ub.ub_txg,
473 			    gethrestime_sec(), mmp->mmp_delay, NULL, 0,
474 			    mmp->mmp_kstat_id++, error);
475 			zfs_dbgmsg("MMP error choosing leaf pool '%s' "
476 			    "gethrtime %llu fail_mask %#x", spa_name(spa),
477 			    gethrtime(), error);
478 		}
479 		mutex_exit(&mmp->mmp_io_lock);
480 		spa_config_exit(spa, SCL_STATE, mmp_tag);
481 		return;
482 	}
483 
484 	vd = spa->spa_mmp.mmp_last_leaf;
485 	if (mmp->mmp_skip_error != 0) {
486 		mmp->mmp_skip_error = 0;
487 		zfs_dbgmsg("MMP write after skipping due to unavailable "
488 		    "leaves, pool '%s' gethrtime %llu leaf %llu",
489 		    spa_name(spa), (u_longlong_t)gethrtime(),
490 		    (u_longlong_t)vd->vdev_guid);
491 	}
492 
493 	if (mmp->mmp_zio_root == NULL)
494 		mmp->mmp_zio_root = zio_root(spa, NULL, NULL,
495 		    flags | ZIO_FLAG_GODFATHER);
496 
497 	if (mmp->mmp_ub.ub_timestamp != gethrestime_sec()) {
498 		/*
499 		 * Want to reset mmp_seq when timestamp advances because after
500 		 * an mmp_seq wrap new values will not be chosen by
501 		 * uberblock_compare() as the "best".
502 		 */
503 		mmp->mmp_ub.ub_timestamp = gethrestime_sec();
504 		mmp->mmp_seq = 1;
505 	}
506 
507 	ub = &mmp->mmp_ub;
508 	ub->ub_mmp_magic = MMP_MAGIC;
509 	ub->ub_mmp_delay = mmp->mmp_delay;
510 	ub->ub_mmp_config = MMP_SEQ_SET(mmp->mmp_seq) |
511 	    MMP_INTERVAL_SET(MMP_INTERVAL_OK(zfs_multihost_interval)) |
512 	    MMP_FAIL_INT_SET(MMP_FAIL_INTVS_OK(
513 	    zfs_multihost_fail_intervals));
514 	vd->vdev_mmp_pending = gethrtime();
515 	vd->vdev_mmp_kstat_id = mmp->mmp_kstat_id;
516 
517 	zio_t *zio  = zio_null(mmp->mmp_zio_root, spa, NULL, NULL, NULL, flags);
518 	abd_t *ub_abd = abd_alloc_for_io(VDEV_UBERBLOCK_SIZE(vd), B_TRUE);
519 	abd_zero(ub_abd, VDEV_UBERBLOCK_SIZE(vd));
520 	abd_copy_from_buf(ub_abd, ub, sizeof (uberblock_t));
521 
522 	mmp->mmp_seq++;
523 	mmp->mmp_kstat_id++;
524 	mutex_exit(&mmp->mmp_io_lock);
525 
526 	offset = VDEV_UBERBLOCK_OFFSET(vd, VDEV_UBERBLOCK_COUNT(vd) -
527 	    MMP_BLOCKS_PER_LABEL + random_in_range(MMP_BLOCKS_PER_LABEL));
528 
529 	label = random_in_range(VDEV_LABELS);
530 	vdev_label_write(zio, vd, label, ub_abd, offset,
531 	    VDEV_UBERBLOCK_SIZE(vd), mmp_write_done, mmp,
532 	    flags | ZIO_FLAG_DONT_PROPAGATE);
533 
534 	(void) spa_mmp_history_add(spa, ub->ub_txg, ub->ub_timestamp,
535 	    ub->ub_mmp_delay, vd, label, vd->vdev_mmp_kstat_id, 0);
536 
537 	zio_nowait(zio);
538 }
539 
540 static void
541 mmp_thread(void *arg)
542 {
543 	spa_t *spa = (spa_t *)arg;
544 	mmp_thread_t *mmp = &spa->spa_mmp;
545 	boolean_t suspended = spa_suspended(spa);
546 	boolean_t multihost = spa_multihost(spa);
547 	uint64_t mmp_interval = MSEC2NSEC(MMP_INTERVAL_OK(
548 	    zfs_multihost_interval));
549 	uint32_t mmp_fail_intervals = MMP_FAIL_INTVS_OK(
550 	    zfs_multihost_fail_intervals);
551 	hrtime_t mmp_fail_ns = mmp_fail_intervals * mmp_interval;
552 	boolean_t last_spa_suspended = suspended;
553 	boolean_t last_spa_multihost = multihost;
554 	uint64_t last_mmp_interval = mmp_interval;
555 	uint32_t last_mmp_fail_intervals = mmp_fail_intervals;
556 	hrtime_t last_mmp_fail_ns = mmp_fail_ns;
557 	callb_cpr_t cpr;
558 	int skip_wait = 0;
559 
560 	mmp_thread_enter(mmp, &cpr);
561 
562 	/*
563 	 * There have been no MMP writes yet.  Setting mmp_last_write here gives
564 	 * us one mmp_fail_ns period, which is consistent with the activity
565 	 * check duration, to try to land an MMP write before MMP suspends the
566 	 * pool (if so configured).
567 	 */
568 
569 	mutex_enter(&mmp->mmp_io_lock);
570 	mmp->mmp_last_write = gethrtime();
571 	mmp->mmp_delay = MSEC2NSEC(MMP_INTERVAL_OK(zfs_multihost_interval));
572 	mutex_exit(&mmp->mmp_io_lock);
573 
574 	while (!mmp->mmp_thread_exiting) {
575 		hrtime_t next_time = gethrtime() +
576 		    MSEC2NSEC(MMP_DEFAULT_INTERVAL);
577 		int leaves = MAX(vdev_count_leaves(spa), 1);
578 
579 		/* Detect changes in tunables or state */
580 
581 		last_spa_suspended = suspended;
582 		last_spa_multihost = multihost;
583 		suspended = spa_suspended(spa);
584 		multihost = spa_multihost(spa);
585 
586 		last_mmp_interval = mmp_interval;
587 		last_mmp_fail_intervals = mmp_fail_intervals;
588 		last_mmp_fail_ns = mmp_fail_ns;
589 		mmp_interval = MSEC2NSEC(MMP_INTERVAL_OK(
590 		    zfs_multihost_interval));
591 		mmp_fail_intervals = MMP_FAIL_INTVS_OK(
592 		    zfs_multihost_fail_intervals);
593 
594 		/* Smooth so pool is not suspended when reducing tunables */
595 		if (mmp_fail_intervals * mmp_interval < mmp_fail_ns) {
596 			mmp_fail_ns = (mmp_fail_ns * 31 +
597 			    mmp_fail_intervals * mmp_interval) / 32;
598 		} else {
599 			mmp_fail_ns = mmp_fail_intervals *
600 			    mmp_interval;
601 		}
602 
603 		if (mmp_interval != last_mmp_interval ||
604 		    mmp_fail_intervals != last_mmp_fail_intervals) {
605 			/*
606 			 * We want other hosts to see new tunables as quickly as
607 			 * possible.  Write out at higher frequency than usual.
608 			 */
609 			skip_wait += leaves;
610 		}
611 
612 		if (multihost)
613 			next_time = gethrtime() + mmp_interval / leaves;
614 
615 		if (mmp_fail_ns != last_mmp_fail_ns) {
616 			zfs_dbgmsg("MMP interval change pool '%s' "
617 			    "gethrtime %llu last_mmp_interval %llu "
618 			    "mmp_interval %llu last_mmp_fail_intervals %u "
619 			    "mmp_fail_intervals %u mmp_fail_ns %llu "
620 			    "skip_wait %d leaves %d next_time %llu",
621 			    spa_name(spa), (u_longlong_t)gethrtime(),
622 			    (u_longlong_t)last_mmp_interval,
623 			    (u_longlong_t)mmp_interval, last_mmp_fail_intervals,
624 			    mmp_fail_intervals, (u_longlong_t)mmp_fail_ns,
625 			    skip_wait, leaves, (u_longlong_t)next_time);
626 		}
627 
628 		/*
629 		 * MMP off => on, or suspended => !suspended:
630 		 * No writes occurred recently.  Update mmp_last_write to give
631 		 * us some time to try.
632 		 */
633 		if ((!last_spa_multihost && multihost) ||
634 		    (last_spa_suspended && !suspended)) {
635 			zfs_dbgmsg("MMP state change pool '%s': gethrtime %llu "
636 			    "last_spa_multihost %u multihost %u "
637 			    "last_spa_suspended %u suspended %u",
638 			    spa_name(spa), (u_longlong_t)gethrtime(),
639 			    last_spa_multihost, multihost, last_spa_suspended,
640 			    suspended);
641 			mutex_enter(&mmp->mmp_io_lock);
642 			mmp->mmp_last_write = gethrtime();
643 			mmp->mmp_delay = mmp_interval;
644 			mutex_exit(&mmp->mmp_io_lock);
645 		}
646 
647 		/*
648 		 * MMP on => off:
649 		 * mmp_delay == 0 tells importing node to skip activity check.
650 		 */
651 		if (last_spa_multihost && !multihost) {
652 			mutex_enter(&mmp->mmp_io_lock);
653 			mmp->mmp_delay = 0;
654 			mutex_exit(&mmp->mmp_io_lock);
655 		}
656 
657 		/*
658 		 * Suspend the pool if no MMP write has succeeded in over
659 		 * mmp_interval * mmp_fail_intervals nanoseconds.
660 		 */
661 		if (multihost && !suspended && mmp_fail_intervals &&
662 		    (gethrtime() - mmp->mmp_last_write) > mmp_fail_ns) {
663 			zfs_dbgmsg("MMP suspending pool '%s': gethrtime %llu "
664 			    "mmp_last_write %llu mmp_interval %llu "
665 			    "mmp_fail_intervals %llu mmp_fail_ns %llu",
666 			    spa_name(spa), (u_longlong_t)gethrtime(),
667 			    (u_longlong_t)mmp->mmp_last_write,
668 			    (u_longlong_t)mmp_interval,
669 			    (u_longlong_t)mmp_fail_intervals,
670 			    (u_longlong_t)mmp_fail_ns);
671 			cmn_err(CE_WARN, "MMP writes to pool '%s' have not "
672 			    "succeeded in over %llu ms; suspending pool. "
673 			    "Hrtime %llu",
674 			    spa_name(spa),
675 			    NSEC2MSEC(gethrtime() - mmp->mmp_last_write),
676 			    gethrtime());
677 			zio_suspend(spa, NULL, ZIO_SUSPEND_MMP);
678 		}
679 
680 		if (multihost && !suspended)
681 			mmp_write_uberblock(spa);
682 
683 		if (skip_wait > 0) {
684 			next_time = gethrtime() + MSEC2NSEC(MMP_MIN_INTERVAL) /
685 			    leaves;
686 			skip_wait--;
687 		}
688 
689 		CALLB_CPR_SAFE_BEGIN(&cpr);
690 		(void) cv_timedwait_idle_hires(&mmp->mmp_thread_cv,
691 		    &mmp->mmp_thread_lock, next_time, USEC2NSEC(100),
692 		    CALLOUT_FLAG_ABSOLUTE);
693 		CALLB_CPR_SAFE_END(&cpr, &mmp->mmp_thread_lock);
694 	}
695 
696 	/* Outstanding writes are allowed to complete. */
697 	zio_wait(mmp->mmp_zio_root);
698 
699 	mmp->mmp_zio_root = NULL;
700 	mmp_thread_exit(mmp, &mmp->mmp_thread, &cpr);
701 }
702 
703 /*
704  * Signal the MMP thread to wake it, when it is sleeping on
705  * its cv.  Used when some module parameter has changed and
706  * we want the thread to know about it.
707  * Only signal if the pool is active and mmp thread is
708  * running, otherwise there is no thread to wake.
709  */
710 static void
711 mmp_signal_thread(spa_t *spa)
712 {
713 	mmp_thread_t *mmp = &spa->spa_mmp;
714 
715 	mutex_enter(&mmp->mmp_thread_lock);
716 	if (mmp->mmp_thread)
717 		cv_broadcast(&mmp->mmp_thread_cv);
718 	mutex_exit(&mmp->mmp_thread_lock);
719 }
720 
721 void
722 mmp_signal_all_threads(void)
723 {
724 	spa_t *spa = NULL;
725 
726 	mutex_enter(&spa_namespace_lock);
727 	while ((spa = spa_next(spa))) {
728 		if (spa->spa_state == POOL_STATE_ACTIVE)
729 			mmp_signal_thread(spa);
730 	}
731 	mutex_exit(&spa_namespace_lock);
732 }
733 
734 /* BEGIN CSTYLED */
735 ZFS_MODULE_PARAM_CALL(zfs_multihost, zfs_multihost_, interval,
736 	param_set_multihost_interval, param_get_ulong, ZMOD_RW,
737 	"Milliseconds between mmp writes to each leaf");
738 /* END CSTYLED */
739 
740 ZFS_MODULE_PARAM(zfs_multihost, zfs_multihost_, fail_intervals, UINT, ZMOD_RW,
741 	"Max allowed period without a successful mmp write");
742 
743 ZFS_MODULE_PARAM(zfs_multihost, zfs_multihost_, import_intervals, UINT, ZMOD_RW,
744 	"Number of zfs_multihost_interval periods to wait for activity");
745