xref: /titanic_41/usr/src/cmd/lvm/util/metaclust.c (revision c0dd49bdd68c0d758a67d56f07826f3b45cfc664)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <meta.h>
#include <sdssc.h>
#include <signal.h>
#include <syslog.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/lvm/md_mirror.h>
#include <metad.h>

#define	MY_VERSION		"1.0"	/* the highest supported version */
#define	MAX_DEBUG_LEVEL		5	/* maximum verbosity level */

#define	RESET_OWNER		0x0001
#define	CHOOSE_OWNER		0x0002
#define	RESET_ABR		0x0004
#define	UPDATE_ABR		0x0008
#define	GET_MIRROR_STATE	0x0010

#define	SET_INFO_NO_WR	0x0002
#define	SET_INFO_MN	0x0004

/*
 * This table defines all the metaclust reconfig steps we understand
 */
typedef enum stpnum {
	MC_UNK = 0,
	MC_START,
	MC_STOP,
	MC_ABORT,
	MC_RETURN,
	MC_STEP1,
	MC_STEP2,
	MC_STEP3,
	MC_STEP4
} stepnum_t;

/*
 * Structure for step_name -> step_number mapping
 */
struct step_t {
	char		*step_nam;
	stepnum_t	step_num;
};

/*
 * Step name to step number mapping table
 * This table MUST be sorted alphabetically in ascending order of step name
 */
static struct step_t step_table[] = {
	{ "abort",	MC_ABORT },
	{ "return",	MC_RETURN },
	{ "start",	MC_START },
	{ "step1",	MC_STEP1 },
	{ "step2",	MC_STEP2 },
	{ "step3",	MC_STEP3 },
	{ "step4",	MC_STEP4 },
	{ "stop",	MC_STOP }
};

/*
 * If support for a different version is added, the new version number should
 * be appended to the version_table below. This list will be searched to
 * determine if a version requested via the -V option is supported or not.
 */
static char *version_table[] = {
	MY_VERSION
};

uint_t	timeout = 0;			/* disable timeout by default */
char	*version = MY_VERSION;		/* use latest version by default */
int	stepnum = MC_UNK;		/* reconfiguration step number */
pid_t	c_pid;				/* child process id */

/*
 * Binary search comparison routine
 */
static int
mc_compare(const void *stp1, const void *stp2)
{
	return (strcmp((const char *)stp1,
	    ((const struct step_t *)stp2)->step_nam));
}

/*
 * Timeout expiry alarm signal handler
 */
/*ARGSUSED*/
static void
sigalarmhandler(int sig)
{
	int	i, n, ret, stat_loc = 0;
	FILE	*pgcore;
	char	corecmd[256];

	n = sizeof (step_table) / sizeof (step_table[0]);
	for (i = 0; i < n; i++) {
		if (stepnum == step_table[i].step_num)
			break;
	}

	assert(i != n);

	meta_mc_log(MC_LOG1, gettext("Timeout expired in %s: %s"),
	    step_table[i].step_nam,
	    meta_print_hrtime(gethrtime() - start_time));

	/*
	 * See what the child was actually doing when the timeout expired.
	 * A core-dump of this would be _really_ good, so let's just
	 * try a 'gcore -g c_pid' and hope
	 */

	(void) memset(corecmd, 0, sizeof (corecmd));
	(void) snprintf(corecmd, sizeof (corecmd),
	    "/bin/gcore -g %d >/dev/null 2>&1", (int)c_pid);

	pgcore = popen(corecmd, "r");

	if (pgcore == NULL) {
		meta_mc_log(MC_LOG1, gettext("Could not grab core for pid %s"),
		    c_pid);
	} else {
		(void) pclose(pgcore);
	}

	if ((ret = kill(c_pid, SIGKILL)) == 0) {
		/*
		 * The child will wait forever until the status is retrieved
		 * so get it now. Keep retrying if the call is interrupted.
		 *
		 * The possible results are,
		 *
		 *	- child killed successfully
		 *	- signal sent but child not killed
		 *	- waitpid failed/interrupted
		 */
		(void) sleep(2);
		while ((ret = waitpid(c_pid, &stat_loc, WNOHANG)) < 0) {
			if (errno != EINTR) {
				break;
			}
		}
		if ((ret == c_pid) || (errno == ECHILD)) {
			ret = 0;
		} else {
			ret = 1;
		}
	} else if (errno == ESRCH) {
		/*
		 * If the kill did not catch the child then it means the child
		 * exited immediately after the timeout occured.
		 */
		ret = 0;
	}

	/*
	 * make sure not to exit with 205 for any steps other than step1-step4.
	 * Suncluster reconfiguration can't handle it otherwise.
	 */
	switch (stepnum) {
	case MC_STEP1:
	case MC_STEP2:
	case MC_STEP3:
	case MC_STEP4:
		/*
		 * If the child was killed successfully return 205 for a
		 * new reconfig cycle otherwise send 1 to panic the node.
		 */
		if (ret != 0) {
			md_eprintf(gettext("Could not kill child\n"));
			exit(1);
		} else {
			exit(205);
		}
		break;
	case MC_START:
	case MC_STOP:
	case MC_ABORT:
	case MC_RETURN:
	default:
		exit(1);
		break;
	}
}

/*
 * Attempt to load local set.
 * Returns:
 *	pointer to mdsetname_t for local set (local_sp) is successful.
 *	0 if failure
 *		if there are no local set mddbs, no error message is printed.
 *		Otherwise, error message is printed so that user
 *		can determine why the local set didn't start.
 */
mdsetname_t *
load_local_set(md_error_t *ep)
{
	mdsetname_t	*local_sp = NULL;

	/* Does local set exist? If not, give no error */
	if ((local_sp = metasetname(MD_LOCAL_NAME, ep)) == NULL) {
		return (0);
	}

	/*
	 * snarf local set
	 * If fails with MDE_DB_NODB, then just return 1 printing
	 * no failure.
	 * Otherwise, print error message, and return 1.
	 */
	if (meta_setup_db_locations(ep) != 0) {
		if (!(mdismddberror(ep, MDE_DB_NODB)))
			mde_perror(ep, "");
		return (0);
	}

	/* local set loaded successfully */
	return (local_sp);
}

/*
 * Purpose:	Compose a full path name for a metadevice
 *
 * On entry:	sp	- setname pointer
 *		mnum	- minor number of metadevice
 *		pathname - pointer to array to return path string
 *		pathlen	- max length of pathname array
 */
static int
compose_path(mdsetname_t *sp, int mnum, char *pathname, int pathlen)
{
	int	rtn;
	mdname_t	*np;
	md_error_t	status = mdnullerror;

	if (MD_MIN2SET(mnum) != sp->setno) {
		md_eprintf(gettext("minor number 0x%x invalid for set %d\n"),
		    mnum, sp->setno);
		return (-1);
	}

	if ((np = metamnumname(&sp, mnum, 0, &status)) == NULL) {
		return (-1);
	}

	rtn = snprintf(pathname, pathlen, "%s", np->rname);

	if ((pathname[0] == '\0') || (rtn >= pathlen)) {
		md_eprintf(gettext(
		    "Could not create path for device %s\n"),
		    get_mdname(sp, mnum));
		return (-1);
	}
	return (0);
}

/*
 * Purpose:	Walk through all the devices specified for the given set
 *		and do the action specified in mode
 */
static int
reset_state(uint_t mode, mdsetname_t *sp, char *drivername, md_error_t *ep)
{
	mdnamelist_t			*devnlp = NULL;
	mdnamelist_t			*p;
	mdname_t			*devnp = NULL;
	md_set_mmown_params_t		ownpar_p;
	md_set_mmown_params_t		*ownpar = &ownpar_p;
	md_unit_t			*mm;
	int				mirror_dev = 0;
	mndiskset_membershiplist_t	*nl;
	int				cnt;
	int				has_parent;
	md_mn_get_mir_state_t		mir_state_p;
	md_mn_get_mir_state_t		*mir_state = &mir_state_p;

	/*
	 * if we are choosing or resetting the owners then make sure
	 * we are only doing it for mirror devices
	 */
	mirror_dev = (strcmp(MD_MIRROR, drivername) == 0);
	if ((mode & (RESET_OWNER | CHOOSE_OWNER)) && !mirror_dev) {
		return (-1);
	}

	/* get a list of all the metadevices for current set */
	if (mirror_dev && meta_get_mirror_names(sp, &devnlp, 0, ep) < 0) {
		mde_perror(ep, gettext("Could not get mirrors for set %s"),
		    sp->setname);
		return (-1);
	} else if (meta_get_sp_names(sp, &devnlp, 0, ep) < 0) {
		mde_perror(ep, gettext(
		    "Could not get soft partitions for set %s"), sp->setname);
		return (-1);
	}

	/* If resetting the owner, get the known membership list */
	if (mode & RESET_OWNER) {
		if (meta_read_nodelist(&cnt, &nl, ep)) {
			mde_perror(ep, "Could not get nodelist");
			return (-1);
		}
	}

	/* for each metadevice */
	for (p = devnlp; (p != NULL); p = p->next) {
		devnp = p->namep;

		/*
		 * Get the current setting for mirror ABR state and all of the
		 * submirror state and flags from the master node. We only
		 * perform this when going through a 'start' cycle.
		 */
		if ((mode & GET_MIRROR_STATE) && mirror_dev) {
			char	*miscname;

			/*
			 * Ensure that we ignore soft-parts that are returned
			 * from the meta_get_mirror_names() call
			 */
			if ((miscname = metagetmiscname(devnp, ep)) == NULL)
				goto out;
			if (strcmp(miscname, MD_MIRROR) != 0)
				continue;

			mir_state->mnum = meta_getminor(devnp->dev);
			MD_SETDRIVERNAME(mir_state, MD_MIRROR, sp->setno);
			meta_mc_log(MC_LOG4, gettext("Getting mirror state"
			    " for %s: %s"), get_mdname(sp, mir_state->mnum),
			    meta_print_hrtime(gethrtime() - start_time));

			if (metaioctl(MD_MN_GET_MIRROR_STATE, mir_state, ep,
			    "MD_MN_GET_MIRROR_STATE") != 0) {
				mde_perror(ep, gettext("Unable to get "
				    "mirror state for %s"),
				    get_mdname(sp, mir_state->mnum));
				goto out;
			} else {
				continue;
			}
		}

		/* check if this is a top level metadevice */
		if ((mm = meta_get_mdunit(sp, devnp, ep)) == NULL)
			goto out;
		if (MD_HAS_PARENT(MD_PARENT(mm))) {
			has_parent = 1;
		} else {
			has_parent = 0;
		}
		Free(mm);

		if (mode & (RESET_OWNER | CHOOSE_OWNER)) {
			char	*miscname;

			/*
			 * we can only do these for mirrors so make sure we
			 * really have a mirror device and not a softpartition
			 * imitating one. meta_get_mirror_names seems to think
			 * softparts on top of a mirror are mirrors!
			 */
			if ((miscname = metagetmiscname(devnp, ep)) == NULL)
				goto out;
			if (strcmp(miscname, MD_MIRROR) != 0)
				continue;

			(void) memset(ownpar, 0, sizeof (*ownpar));
			ownpar->d.mnum = meta_getminor(devnp->dev);
			MD_SETDRIVERNAME(ownpar, MD_MIRROR, sp->setno);

			meta_mc_log(MC_LOG4, gettext("Setting owner "
			    "for %s: %s"), get_mdname(sp, ownpar->d.mnum),
			    meta_print_hrtime(gethrtime() - start_time));

			/* get the current owner id */
			if (metaioctl(MD_MN_GET_MM_OWNER, ownpar, ep,
			    "MD_MN_GET_MM_OWNER") != 0) {
				mde_perror(ep, gettext("Unable to get "
				    "mirror owner for %s"),
				    get_mdname(sp, ownpar->d.mnum));
				goto out;
			}
		}

		if (mode & RESET_OWNER) {
			if (ownpar->d.owner == MD_MN_MIRROR_UNOWNED) {
				mdclrerror(ep);
				continue;
			}

			/*
			 * reset owner only if the current owner is
			 * not in the membership list
			 * Also kill the resync thread so that when the resync
			 * is started, it will perform an optimized resync
			 * for any resync regions that were dirty when the
			 * current owner left the membership.
			 */
			if (meta_is_member(NULL, ownpar->d.owner, nl) != 1) {
				if (meta_mn_change_owner(&ownpar,
				    sp->setno, ownpar->d.mnum,
				    MD_MN_MIRROR_UNOWNED,
				    MD_MN_MM_ALLOW_CHANGE) == -1) {
					md_eprintf(gettext(
					    "Unable to reset mirror owner "
					    "for %s\n"),
					    get_mdname(sp, ownpar->d.mnum));
					goto out;
				}
				if (meta_mirror_resync(sp, devnp, 0, ep,
				    MD_RESYNC_KILL_NO_WAIT) != 0) {
					md_eprintf(gettext(
					    "Unable to kill resync for"
					    " %s\n"),
					    get_mdname(sp, ownpar->d.mnum));
					goto out;
				}
			}
		}

		if (mode & CHOOSE_OWNER) {
			/*
			 * only orphaned resyncs will have no owner.
			 * if that is the case choose a new owner. Otherwise
			 * re-establish the existing owner. This covers the
			 * case where a node that owned the mirror
			 * reboots/panics and comes back into the cluster before
			 * the reconfig cycle has completed. In this case the
			 * other cluster nodes will have the mirror owner marked
			 * as the rebooted node while it has the owner marked
			 * as 'None'. We have to reestablish the ownership so
			 * that the subsequent resync can continue.
			 */
			if (meta_mn_change_owner(&ownpar, sp->setno,
			    ownpar->d.mnum, ownpar->d.owner,
			    MD_MN_MM_CHOOSE_OWNER) == -1) {
				md_eprintf(gettext("Unable to choose "
				    "mirror owner for %s\n"),
				    get_mdname(sp, ownpar->d.mnum));
				goto out;
			}
		}

		/*
		 * For RESET_ABR and UPDATE_ABR - only handle top
		 * level metadevices.
		 */
		if (has_parent)
			continue;

		if (mode & RESET_ABR) {
			/*
			 * Reset the ABR (application based recovery)
			 * value on all nodes. We are dealing with
			 * the possibility that we have ABR set but the
			 * only node that had the device open with ABR has
			 * left the cluster. We simply open and close the
			 * device and if this is the last close in the
			 * cluster, ABR will be cleared on all nodes.
			 */
			char		*miscname;
			char		name[MAXPATHLEN];
			int		mnum, fd;

			name[0] = '\0';
			mnum = meta_getminor(devnp->dev);

			/*
			 * Ensure that we don't include soft-parts in the
			 * mirror-only call to RESET_ABR. meta_get_mirror_names
			 * returns a bogus list that includes all soft-parts
			 * built on mirrors.
			 */
			if ((miscname = metagetmiscname(devnp, ep)) == NULL)
				goto out;
			if (mirror_dev && (strcmp(miscname, MD_MIRROR) != 0))
				continue;

			meta_mc_log(MC_LOG4, gettext("Re-setting ABR state "
			    "for %s: %s"), get_mdname(sp, mnum),
			    meta_print_hrtime(gethrtime() - start_time));

			/* compose the absolute device path and open it */
			if (compose_path(sp, mnum, &name[0],
			    sizeof (name)) != 0)
				goto out;
			if ((fd = open(name, O_RDWR, 0)) < 0) {
				md_perror(gettext("Could not open device %s"),
				    name);
				continue;
			}

			(void) close(fd);
		}

		if (mode & UPDATE_ABR) {
			/*
			 * Update the ABR value on this node. We obtain the
			 * current ABR state from the master node.
			 */

			char		*miscname;
			char		name[MAXPATHLEN];
			int		mnum, fd;
			volcap_t	vc;
			uint_t		tstate;

			name[0] = '\0';
			mnum = meta_getminor(devnp->dev);

			/*
			 * Ensure that we don't include soft-parts in the
			 * mirror-only call to UPDATE_ABR. meta_get_mirror_names
			 * returns a bogus list that includes all soft-parts
			 * built on mirrors.
			 */
			if ((miscname = metagetmiscname(devnp, ep)) == NULL)
				goto out;
			if (mirror_dev && (strcmp(miscname, MD_MIRROR) != 0))
				continue;

			/* Get tstate from Master */
			if (meta_mn_send_get_tstate(devnp->dev, &tstate, ep)
			    != 0)
				continue;
			/* If not set on the master, nothing to do */
			if (!(tstate & MD_ABR_CAP))
				continue;

			meta_mc_log(MC_LOG4, gettext("Updating ABR state "
			    "for %s: %s"), get_mdname(sp, mnum),
			    meta_print_hrtime(gethrtime() - start_time));

			/* compose the absolute device path and open it */
			if (compose_path(sp, mnum, &name[0],
			    sizeof (name)) != 0)
				goto out;
			if ((fd = open(name, O_RDWR, 0)) < 0) {
				md_perror(gettext("Could not open device %s"),
				    name);
				continue;
			}

			/* set ABR state */
			vc.vc_info = 0;
			vc.vc_set = 0;
			if (ioctl(fd, DKIOCGETVOLCAP, &vc) < 0) {
				/*
				 * Ignore if device does not support this
				 * ioctl
				 */
				if ((errno != ENOTTY) && (errno != ENOTSUP)) {
					md_perror(gettext("Could not get "
					    "ABR/DMR state for device %s"),
					    name);
				}
				(void) close(fd);
				continue;
			}
			if (!(vc.vc_info & (DKV_ABR_CAP | DKV_DMR_CAP))) {
				(void) close(fd);
				continue;
			}

			vc.vc_set = DKV_ABR_CAP;
			if (ioctl(fd, DKIOCSETVOLCAP, &vc) < 0) {
				md_perror(gettext(
				    "Could not set ABR state for "
				    "device %s"), name);
				(void) close(fd);
				goto out;
			} else {
				md_eprintf(gettext(
				    "Setting ABR state on device %s\n"), name);
			}

			(void) close(fd);
		}
	}

	/* cleanup */
	if (mode & RESET_OWNER) {
		meta_free_nodelist(nl);
	}
	metafreenamelist(devnlp);
	return (0);

out:
	/* cleanup */
	if (mode & RESET_OWNER) {
		meta_free_nodelist(nl);
	}
	metafreenamelist(devnlp);
	return (-1);
}

/*
 * Print usage message
 */
static void
usage(mdsetname_t *sp, int eval)
{
	(void) fprintf(stderr, gettext("usage:"
	    "\t%s [-V version] [-t timeout] [-d level] start localnodeid\n"
	    "\t%s [-V version] [-t timeout] [-d level] step nodelist...\n"
	    "\t%s [-V version] [-t timeout] [-d level] abort | stop\n"
	    "\t%s [-V | -? | -h]\n"),
	    myname, myname, myname, myname);
	if (!eval) {
		(void) fprintf(stderr, gettext("\n"
		    "\tValid debug (-d) levels are 1-%d for increasing "
		    "verbosity.\n\tDefault is -d 3.\n\n"
		    "\tValid step values are: return | step1 | step2 | "
		    "step3 | step4\n\n"
		    "\tNodelist is a space-separated list of node id's\n\n"),
		    MAX_DEBUG_LEVEL);
	}
	md_exit(sp, eval);
}

/*
 * Input:	Input takes a config step name followed by a list of
 *		possible node id's.
 *
 * Returns:	  0 - Success
 *		  1 - Fail
 *			Node will be removed from cluster membership
 *			by forcing node to panic.
 *		205 - Unsuccessful. Start another reconfig cycle.
 *			Problem was encountered that could be fixed by
 *			running another reconfig cycle.
 *			Problem could be a result of a failure to read
 *			the nodelist file or that all work could not be
 *			accomplished in a reconfig step in the amount of
 *			time given so another reconfig cycle is needed in
 *			order to finish the current step.
 */
int
main(int argc, char **argv)
{
	mdsetname_t		*sp = NULL;
	md_error_t		status = mdnullerror;
	md_error_t		*ep = &status;
	set_t			max_sets, setno;
	int			c, clust = 0;
	struct sigaction	nsa, osa;
	struct step_t		*step_ptr;
	mdsetname_t		*local_sp = NULL;
	md_drive_desc		*dd;
	int			rval = 0;
	md_set_desc		*sd;
	mddb_block_parm_t	mbp;
	uint_t			debug = 3; /* log upto MC_LOG3 by default */
	int			version_table_size;
	mddb_setflags_config_t	sf;
	int			ret_val;
	mddb_config_t		cfg;
	int			set_info[MD_MAXSETS];
	long			commd_timeout = 0;

	/*
	 * Get the locale set up before calling any other routines
	 * with messages to ouput.  Just in case we're not in a build
	 * environment, make sure that TEXT_DOMAIN gets set to
	 * something.
	 */
#if !defined(TEXT_DOMAIN)
#define	TEXT_DOMAIN "SYS_TEST"
#endif
	(void) setlocale(LC_ALL, "");
	(void) textdomain(TEXT_DOMAIN);

	if ((clust = sdssc_bind_library()) == SDSSC_ERROR) {
		md_eprintf(gettext("Interface error with libsds_sc.so\n"));
		exit(1);
	}

	if (md_init(argc, argv, 1, 1, ep) != 0 || meta_check_root(ep) != 0) {
		mde_perror(ep, "");
		md_exit(sp, 1);
	}

	/*
	 * open log and enable libmeta logging. Do it here explicitly
	 * rather than letting md_init() do it because we are not really
	 * a daemon and that is what md_init() opens the log as.
	 */
	openlog("metaclust", LOG_CONS, LOG_USER);

	version_table_size = sizeof (version_table) / sizeof (version_table[0]);

	optind = 1;
	opterr = 0;
	while ((c = getopt(argc, argv, "hd:V:t:?")) != -1) {
		switch (c) {
		case 'h':
			usage(sp, 0);
			break;

		case 'd':
			if (sscanf(optarg, "%u", &debug) != 1) {
				md_eprintf(gettext("Invalid debug level\n"));
				md_exit(sp, 1);
			} else if ((debug < 1) || (debug > MAX_DEBUG_LEVEL)) {
				debug = min(max(debug, 1), MAX_DEBUG_LEVEL);
				md_eprintf(gettext("Debug level must be "
				    "between 1 and %d inclusive.\n"),
				    MAX_DEBUG_LEVEL);
				md_eprintf(gettext("Debug level set to %d.\n"),
				    debug);
			}
			break;

		case 'V':
			version = Strdup(optarg);
			break;

		case 't':
			if (sscanf(optarg, "%u", &timeout) != 1) {
				md_eprintf(gettext("Invalid timeout value\n"));
				md_exit(sp, 1);
			}
			break;

		case '?':
			if (optopt == '?') {
				usage(sp, 0);
			} else if (optopt == 'V') {
				int	i;

				(void) fprintf(stdout, gettext(
				    "%s: Versions Supported:"), myname);
				for (i = 0; i < version_table_size; i++) {
					(void) fprintf(stdout, " %s",
					    version_table[i]);
				}
				(void) fprintf(stdout, "\n");
				md_exit(sp, 0);
			}
			/*FALLTHROUGH*/

		default:
			usage(sp, 1);
			break;
		}
	}

	/* initialise the debug level and start time */
	setup_mc_log(debug);

	/*
	 * check that the version specified (if any) is supported.
	 */
	if (version != NULL) {
		int	i, found = 0;

		for (i = 0; i < version_table_size; i++) {
			if (strcmp(version, version_table[i]) == 0) {
				found = 1;
				break;
			}
		}
		if (!found) {
			md_eprintf(gettext("Version %s not supported\n"),
			    version);
			md_exit(sp, 1);
		}
	}

	argc -= optind;
	argv += optind;

	/* parse arguments */
	if (argc <= 0) {
		usage(sp, 1);
	}

	/* convert the step name to the corresponding number */
	step_ptr = bsearch(argv[0], step_table, (sizeof (step_table) /
	    sizeof (step_table[0])), sizeof (step_table[0]), mc_compare);
	if (step_ptr != NULL) {
		stepnum = step_ptr->step_num;
	}

	--argc;
	++argv;

	/* set timeout alarm signal, a value of 0 will disable timeout */
	if (timeout > 0) {
		int	stat_loc = 0;
		commd_timeout = (long)(timeout * .75);

		c_pid = fork();

		if (c_pid == (pid_t)-1) {
			md_perror(gettext("Unable to fork"));
			md_exit(sp, 1);
		} else if (c_pid) {
			/* parent */
			nsa.sa_flags = 0;
			if (sigfillset(&nsa.sa_mask) < 0) {
				md_perror(gettext("Unable to set signal mask"));
				md_exit(sp, 1);
			}

			nsa.sa_handler = sigalarmhandler;
			if (sigaction(SIGALRM, &nsa, &osa) == -1) {
				md_perror(gettext("Unable to set alarm "
				    "handler"));
				md_exit(sp, 1);
			}

			(void) alarm(timeout);

			/*
			 * wait for child to exit or timeout to expire.
			 * keep retrying if the call is interrupted
			 */
			while ((ret_val = waitpid(c_pid, &stat_loc, 0)) < 0) {
				if (errno != EINTR) {
					break;
				}
			}
			if (ret_val == c_pid) {
				/* exit with the childs exit value */
				exit(WEXITSTATUS(stat_loc));
			} else if (errno == ECHILD) {
				md_exit(sp, 0);
			} else {
				perror(myname);
				md_exit(sp, 1);
			}
		}
	}

	/*
	 * If a timeout value is given, everything from this point onwards is
	 * executed in the child process.
	 */

	switch (stepnum) {
	case MC_START:
		/*
		 * Start Step
		 *
		 * - Suspend all rpc.mdcommd messages
		 */

		/* expect the local node id to be given only */
		if (argc != 1)
			usage(sp, 1);

		meta_mc_log(MC_LOG2, gettext("Starting Start step: %s"),
		    meta_print_hrtime(0));

		/*
		 * With multinode disksets configured we need to
		 * update all replicas on all cluster nodes to have
		 * the same status. If local replicas on a cluster
		 * node are not accessible we need to panic this
		 * node, otherwise we abort in the reconfig cycle
		 * and failfast/reboot the "good" cluster node too.
		 * To avoid a total cluster outage in the above case
		 * we panic only the failing node via md_exit(.., 1).
		 */
		if ((local_sp = load_local_set(ep)) == NULL) {
			/* panic the node */
			md_exit(local_sp, 1);
		}

		if ((max_sets = get_max_sets(ep)) == 0) {
			mde_perror(ep, "");
			md_exit(sp, 1);
		}

		/* start walking through all possible disksets */
		for (setno = 1; setno < max_sets; setno++) {
			if ((sp = metasetnosetname(setno, ep)) == NULL) {
				if (mdiserror(ep, MDE_NO_SET)) {
					/* No set for this setno - continue */
					mdclrerror(ep);
					continue;
				} else {
					mde_perror(ep, gettext("Unable to "
					    "get set %d information"), setno);
					md_exit(sp, 1);
				}
			}

			/* only check multi-node disksets */
			if (!meta_is_mn_set(sp, ep)) {
				mdclrerror(ep);
				continue;
			}

			meta_mc_log(MC_LOG3, gettext("Start - block parse "
			    "messages for set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));

			/*
			 * Mddb parse messages are sent amongst the nodes
			 * in a diskset whenever the locator block or
			 * locator names structure has been changed.
			 * A locator block change could occur as a result
			 * of a disk failure during the reconfig cycle,
			 * so block the mddb parse messages while the
			 * rpc.mdcommd is suspended during the reconfig cycle.
			 */
			if (s_ownset(sp->setno, ep) == MD_SETOWNER_YES) {
				(void) memset(&mbp, 0, sizeof (mbp));
				mbp.c_setno = setno;
				mbp.c_blk_flags = MDDB_BLOCK_PARSE;
				if (metaioctl(MD_MN_MDDB_BLOCK, &mbp,
				    &mbp.c_mde, NULL)) {
					(void) mdstealerror(ep, &mbp.c_mde);
					mde_perror(ep, gettext("Could not "
					    "block set %s"), sp->setname);
					md_exit(sp, 1);
				}
			}

			/* suspend commd and spin waiting for drain */
			while ((ret_val = mdmn_suspend(setno,
			    MD_COMM_ALL_CLASSES, commd_timeout)) ==
			    MDE_DS_COMMDCTL_SUSPEND_NYD) {
				(void) sleep(1);
			}

			if (ret_val) {
				md_eprintf(gettext("Could not suspend "
				    "rpc.mdcommd for set %s\n"), sp->setname);
				md_exit(sp, 1);
			}

			/*
			 * Set start step flag for set. This is set to indicate
			 * that this node entered the reconfig cycle through
			 * the start step.  This is used during the reconfig
			 * cycle to determine whether the node had entered
			 * through the start step or the return step.
			 */
			(void) memset(&sf, 0, sizeof (sf));
			sf.sf_setno = sp->setno;
			sf.sf_setflags = MD_SET_MN_START_RC;
			sf.sf_flags = MDDB_NM_SET;
			/* Use magic to help protect ioctl against attack. */
			sf.sf_magic = MDDB_SETFLAGS_MAGIC;
			if (metaioctl(MD_MN_SET_SETFLAGS, &sf,
			    &sf.sf_mde, NULL)) {
				(void) mdstealerror(ep, &sf.sf_mde);
				mde_perror(ep, gettext("Could not set "
				    "start_step flag for set %s"), sp->setname);
				md_exit(sp, 1);
			}

		}

		meta_mc_log(MC_LOG2, gettext("Start step completed: %s"),
		    meta_print_hrtime(gethrtime() - start_time));

		break;

	case MC_STOP:
		/*
		 * Stop Step
		 *
		 * - ???
		 */

		/* don't expect any more arguments to follow the step name */
		if (argc != 0)
			usage(sp, 1);

		break;

	case MC_ABORT:
		/*
		 * Abort Step
		 *
		 * - Abort rpc.mdcommd
		 */

		/* don't expect any more arguments to follow the step name */
		if (argc != 0)
			usage(sp, 1);

		meta_mc_log(MC_LOG2, gettext("Starting Abort step: %s"),
		    meta_print_hrtime(0));

		/*
		 * Does local set exist? If not, exit with 0
		 * since there's no reason to have this node panic if
		 * the local set cannot be started.
		 */
		if ((local_sp = load_local_set(ep)) == NULL) {
			md_exit(local_sp, 0);
		}

		/*
		 * abort the rpc.mdcommd.  The abort is only issued on this node
		 * meaning that the abort reconfig step is called on this
		 * node before a panic while the rest of the cluster will
		 * undergo a reconfig cycle.
		 * There is no time relation between this node running a
		 * reconfig abort and the the rest of the cluster
		 * running a reconfig cycle meaning that this node may
		 * panic before, during or after the cluster has run
		 * a reconfig cycle.
		 */
		mdmn_abort();

		meta_mc_log(MC_LOG2, gettext("Abort step completed: %s"),
		    meta_print_hrtime(gethrtime() - start_time));

		break;

	case MC_RETURN:
		/*
		 * Return Step
		 *
		 * - Grab local set lock, issue rpc.mdcommd DRAIN ALL
		 *   and release local set lock.  Grabbing the local set
		 *   lock allows any active metaset/metadb commands to
		 *   terminate gracefully and will keep a metaset/metadb
		 *   command from starting until the DRAIN ALL is issued.
		 *   The metaset/metadb commands can issue
		 *   DRAIN ALL/RESUME ALL commands to rpc.mdcommd,
		 *   so the return step must not issue the DRAIN ALL command
		 *   until metaset/metadb have finished or metaset may issue
		 *   a RESUME ALL after this return reconfig step has issued
		 *   the DRAIN ALL command.
		 *   After this reconfig step has issued the DRAIN_ALL and
		 *   released the local set lock, metaset/metadb will fail
		 *   when attempting to contact the rpc.mdcommd and will
		 *   terminate without making any configuration changes.
		 *   The DRAIN ALL command will keep all other meta* commands
		 *   from running during the reconfig cycle (these commands
		 *   will wait until the rpc.mdcommd is resumed) since the
		 *   reconfig cycle may be changing the diskset configuration.
		 */

		/* expect the nodelist to follow the step name */
		if (argc < 1)
			usage(sp, 1);

		meta_mc_log(MC_LOG2, gettext("Starting Return step: %s"),
		    meta_print_hrtime(0));

		/*
		 * Does local set exist? If not, exit with 0
		 * since there's no reason to have this node panic if
		 * the local set cannot be started.
		 */
		if ((local_sp = load_local_set(ep)) == NULL) {
			md_exit(local_sp, 0);
		}

		/*
		 * Suspend any mirror resyncs that are in progress. This
		 * stops unnecessary timeouts.
		 */
		meta_mirror_resync_block_all();

		if (meta_lock(local_sp, TRUE, ep) != 0) {
			mde_perror(ep, "");
			md_exit(local_sp, 1);
		}

		/*
		 * All metaset and metadb commands on this node have now
		 * terminated gracefully.  Now, issue a drain all to
		 * the rpc.mdcommd.  Any meta command issued after the
		 * drain all will either spin sending the command to the
		 * master until after the reconfig cycle has finished OR
		 * will terminate gracefully (metaset/metadb).
		 */
		if ((max_sets = get_max_sets(ep)) == 0) {
			mde_perror(ep, "");
			md_exit(sp, 1);
		}

		/* start walking through all possible disksets */
		for (setno = 1; setno < max_sets; setno++) {
			if ((sp = metasetnosetname(setno, ep)) == NULL) {
				if (mdiserror(ep, MDE_NO_SET)) {
					/* No set for this setno - continue */
					mdclrerror(ep);
					continue;
				} else {
					mde_perror(ep, gettext("Unable to "
					    "get set %d information"), setno);
					md_exit(sp, 1);
				}
			}

			/* only check multi-node disksets */
			if (!meta_is_mn_set(sp, ep)) {
				mdclrerror(ep);
				continue;
			}

			meta_mc_log(MC_LOG3, gettext("Return - block parse "
			    "messages for set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));

			/*
			 * Mddb parse messages are sent amongst the nodes
			 * in a diskset whenever the locator block or
			 * locator names structure has been changed.
			 * A locator block change could occur as a result
			 * of a disk failure during the reconfig cycle,
			 * so block the mddb parse messages while the
			 * rpc.commd is suspended during the reconfig cycle.
			 */
			if (s_ownset(sp->setno, ep) == MD_SETOWNER_YES) {
				(void) memset(&mbp, 0, sizeof (mbp));
				mbp.c_setno = setno;
				mbp.c_blk_flags = MDDB_BLOCK_PARSE;
				if (metaioctl(MD_MN_MDDB_BLOCK, &mbp,
				    &mbp.c_mde, NULL)) {
					(void) mdstealerror(ep, &mbp.c_mde);
					mde_perror(ep, gettext("Could not "
					    "block set %s"), sp->setname);
					md_exit(sp, 1);
				}
			}

			/* suspend commd and spin waiting for drain */
			while ((ret_val = mdmn_suspend(setno,
			    MD_COMM_ALL_CLASSES, commd_timeout)) ==
			    MDE_DS_COMMDCTL_SUSPEND_NYD) {
				(void) sleep(1);
			}

			if (ret_val) {
				md_eprintf(gettext("Could not suspend "
				    "rpc.mdcommd for set %s\n"), sp->setname);
				md_exit(sp, 1);
			}
		}
		/*
		 * Resume all I/Os for this node for all MN sets in
		 * case master node had suspended I/Os but panic'd
		 * before resuming I/Os.  In case of failure, exit
		 * with a 1 since unable to resume I/Os on this node.
		 */
		if (clnt_mn_susp_res_io(mynode(), 0, MN_RES_IO, ep)) {
			mde_perror(ep, gettext(
			    "Unable to resume I/O on node %s for all sets"),
			    mynode());
			md_exit(sp, 1);
		}


		/*
		 * Can now unlock local set lock.  New metaset/metadb
		 * commands are now held off using drain all.
		 */
		(void) meta_unlock(local_sp, ep);

		meta_mc_log(MC_LOG2, gettext("Return step completed: %s"),
		    meta_print_hrtime(gethrtime() - start_time));

		break;

	case MC_STEP1:
		/*
		 * Step 1
		 *
		 * - Populate nodelist file if we are on clustering
		 *   and pick a master node for each MN diskset.
		 */

		/* expect the nodelist to follow the step name */
		if (argc < 1)
			usage(sp, 1);

		meta_mc_log(MC_LOG2, gettext("Starting Step1: %s"),
		    meta_print_hrtime(0));

		/* Always write nodelist file even if no local set exists */
		if (clust == SDSSC_OKAY) {
			/* skip to the nodelist args */
			if (meta_write_nodelist(argc, argv, ep) != 0) {
				mde_perror(ep, gettext(
				    "Could not populate nodelist file"));
				md_exit(sp, 1);
			}
		}

		/*
		 * Does local set exist? If not, exit with 0
		 * since there's no reason to have this node panic if
		 * the local set cannot be started.
		 */
		if ((local_sp = load_local_set(ep)) == NULL) {
			md_exit(local_sp, 0);
		}

		/*
		 * At this point, all meta* commands are blocked across
		 * all disksets since the master rpc.mdcommd has drained or
		 * the master node has died.
		 * If a metaset or metadb command had been in progress
		 * at the start of the reconfig cycle, this command has
		 * either completed or it has been terminated due to
		 * the death of the master node.
		 *
		 * This means that that it is now ok to remove any
		 * outstanding clnt_locks associated with multinode
		 * disksets on this node due to a node panic during
		 * a metaset operation.  This allows the routines that
		 * choose the master to use rpc.metad to determine the
		 * master of the diskset.
		 */
		if (clnt_clr_mnsetlock(mynode(), ep) != 0) {
			meta_mc_log(MC_LOG2, gettext("Step1 aborted:"
			    "clear locks failed %s"),
			    meta_print_hrtime(gethrtime() - start_time));
			md_exit(local_sp, 1);
		}

		/*
		 * Call reconfig_choose_master to choose a master for
		 * each MN diskset, update the nodelist for each diskset
		 * given the member information and send a reinit message
		 * to rpc.mdcommd to reload the nodelist.
		 */
		rval = meta_reconfig_choose_master(commd_timeout, ep);
		if (rval == 205) {
			/*
			 * NOTE: Should issue call to reboot remote host that
			 * is causing the RPC failure.  Clustering to
			 * provide interface in the future.  This should
			 * stop a never-ending set of 205 reconfig cycles.
			 * Remote host causing failure is stored in
			 * ep->host if ep is an RPC error.
			 * if (mdanyrpcerror(ep))
			 * 	reboot (ep->host);
			 */
			meta_mc_log(MC_LOG2, gettext("Step1 aborted:"
			    "choose master failure of 205 %s"),
			    meta_print_hrtime(gethrtime() - start_time));
			md_exit(local_sp, 205);
		} else if (rval != 0) {
			meta_mc_log(MC_LOG2, gettext("Step1 failure: "
			    "choose master failure %s"),
			    meta_print_hrtime(gethrtime() - start_time));
			md_exit(local_sp, 1);
		}

		meta_mc_log(MC_LOG2, gettext("Step1 completed: %s"),
		    meta_print_hrtime(gethrtime() - start_time));

		md_exit(local_sp, rval);
		break;

	case MC_STEP2:
		/*
		 * Step 2
		 *
		 * In Step 2, each node walks the list of disksets.  If a
		 * node is a master of a MN diskset, it synchronizes
		 * the local set USER records for that diskset.
		 *
		 * If disks exist in the diskset and there is a joined
		 * (owner) node in the diskset, the master will also:
		 *	- synchronize the diskset mddbs to the master
		 *	- play the change log
		 *
		 * The master node will now attempt to join any unjoined
		 * nodes that are currently members in the membership list.
		 */

		/* expect the nodelist to follow the step name */
		if (argc < 1)
			usage(sp, 1);

		meta_mc_log(MC_LOG2, gettext("Starting Step2: %s"),
		    meta_print_hrtime(0));

		/*
		 * Does local set exist? If not, exit with 0
		 * since there's no reason to have this node panic if
		 * the local set cannot be started.
		 */
		if ((local_sp = load_local_set(ep)) == NULL) {
			md_exit(local_sp, 0);
		}

		if ((max_sets = get_max_sets(ep)) == 0) {
			mde_perror(ep, "");
			md_exit(local_sp, 1);
		}

		/* start walking through all possible disksets */
		for (setno = 1; setno < max_sets; setno++) {
			if ((sp = metasetnosetname(setno, ep)) == NULL) {
				if (mdiserror(ep, MDE_NO_SET)) {
					/* No set for this setno - continue */
					mdclrerror(ep);
					continue;
				} else if (mdanyrpcerror(ep)) {
					/* Fail on RPC failure to self */
					mde_perror(ep, gettext(
					    "Unable to get information for "
					    "set number %d"), setno);
					md_exit(local_sp, 1);
				} else {
					mde_perror(ep, gettext(
					    "Unable to get information for "
					    "set number %d"), setno);
					mdclrerror(ep);
					continue;
				}
			}

			if ((sd = metaget_setdesc(sp, ep)) == NULL) {
				if (mdanyrpcerror(ep)) {
					/* Fail on RPC failure to self */
					mde_perror(ep, gettext(
					    "Unable to get information for "
					    "set number %d"), setno);
					md_exit(local_sp, 1);
				}
				mde_perror(ep, gettext("Unable to get set "
				    "%s desc information"), sp->setname);
				mdclrerror(ep);
				continue;
			}

			/* Only check MN disksets */
			if (!(MD_MNSET_DESC(sd))) {
				continue;
			}

			/* All actions in step 2 are driven by master */
			if (!(sd->sd_mn_am_i_master)) {
				continue;
			}

			meta_mc_log(MC_LOG3, gettext("Step2 - begin record "
			    "synchronization for set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));

			/*
			 * Synchronize the USER records in the local mddbs
			 * for hosts that are members.  The USER records
			 * contain set, drive and host information.
			 */
			rval = meta_mnsync_user_records(sp, ep);
			if (rval != 0) {
				mde_perror(ep, gettext(
				    "Synchronization of user records "
				    "in set %s failed\n"), sp->setname);
				if (rval == 205) {
					/*
					 * NOTE: Should issue call to reboot
					 * remote host that is causing the RPC
					 * failure.  Clustering to provide
					 * interface in the future.  This
					 * should stop a never-ending set of
					 * 205 reconfig cycles.
					 * Remote host causing failure is
					 * stored in ep->host if ep is an
					 * RPC error.
					 * if (mdanyrpcerror(ep))
					 * 	reboot (ep->host);
					 */
					md_exit(local_sp, 205);
				} else {
					md_exit(local_sp, 1);
				}
			}

			/* Reget sd since sync_user_recs may have flushed it */
			if ((sd = metaget_setdesc(sp, ep)) == NULL) {
				mde_perror(ep, gettext("Unable to get set "
				    "%s desc information"), sp->setname);
				md_exit(local_sp, 1);
			}

			dd = metaget_drivedesc(sp,
			    (MD_BASICNAME_OK | PRINT_FAST), ep);
			if (! mdisok(ep)) {
				mde_perror(ep, gettext("Unable to get set "
				    "%s drive information"), sp->setname);
				md_exit(local_sp, 1);
			}

			/*
			 * No drives in set, continue to next set.
			 */
			if (dd == NULL) {
				/* Done with this set */
				continue;
			}

			meta_mc_log(MC_LOG3, gettext("Step2 - local set user "
			    "records completed for set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));

			/*
			 * Synchronize the diskset mddbs for hosts
			 * that are members.  This may involve
			 * playing the changelog and writing out
			 * to the diskset mddbs.
			 */
			rval = meta_mnsync_diskset_mddbs(sp, ep);
			if (rval != 0) {
				mde_perror(ep, gettext(
				    "Synchronization of diskset mddbs "
				    "in set %s failed\n"), sp->setname);
				meta_mc_log(MC_LOG3, gettext("Step2 - diskset "
				    "mddb synchronization failed for "
				    "set %s: %s"), sp->setname,
				    meta_print_hrtime(gethrtime() -
				    start_time));
				if (rval == 205) {
					/*
					 * NOTE: Should issue call to reboot
					 * remote host that is causing the RPC
					 * failure.  Clustering to provide
					 * interface in the future.  This
					 * should stop a never-ending set of
					 * 205 reconfig cycles.
					 * Remote host causing failure is
					 * stored in ep->host if ep is an
					 * RPC error.
					 * if (mdanyrpcerror(ep))
					 * 	reboot (ep->host);
					 */
					md_exit(local_sp, 205);
				} else if (rval == 1) {
					continue;
				} else {
					md_exit(local_sp, 1);
				}
			}

			meta_mc_log(MC_LOG3, gettext("Step2 - diskset mddb "
			    "synchronization completed for set %s: %s"),
			    sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));

			/* Join the starting nodes to the diskset */
			rval = meta_mnjoin_all(sp, ep);
			if (rval != 0) {
				mde_perror(ep, gettext(
				    "Join of non-owner (starting) nodes "
				    "in set %s failed\n"), sp->setname);
				meta_mc_log(MC_LOG3, gettext("Step2 - non owner"
				    "nodes joined for set %s: %s"),
				    sp->setname,
				    meta_print_hrtime(gethrtime() -
				    start_time));
				if (rval == 205) {
					/*
					 * NOTE: Should issue call to reboot
					 * remote host that is causing the RPC
					 * failure.  Clustering to provide
					 * interface in the future.  This
					 * should stop a never-ending set of
					 * 205 reconfig cycles.
					 * Remote host causing failure is
					 * stored in ep->host if ep is an
					 * RPC error.
					 * if (mdanyrpcerror(ep))
					 * 	reboot (ep->host);
					 */
					md_exit(local_sp, 205);
				} else {
					md_exit(local_sp, 1);
				}
			}

			meta_mc_log(MC_LOG3, gettext("Step2 - non owner nodes "
			    "joined for set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));

		}

		meta_mc_log(MC_LOG2, gettext("Step2 completed: %s"),
		    meta_print_hrtime(gethrtime() - start_time));

		break;

	case MC_STEP3:
		/*
		 * Step 3
		 *
		 * For all multinode sets do,
		 * - Reinitialise rpc.mdcommd
		 * - Reset mirror owners to null if the current owner is
		 *   no longer in the membership list
		 */

		/* expect the nodelist to follow the step name */
		if (argc < 1)
			usage(sp, 1);

		meta_mc_log(MC_LOG2, gettext("Starting Step3: %s"),
		    meta_print_hrtime(0));

		/*
		 * Does local set exist? If not, exit with 0
		 * since there's no reason to have this node panic if
		 * the local set cannot be started.
		 */
		if ((local_sp = load_local_set(ep)) == NULL) {
			md_exit(local_sp, 0);
		}

		/*
		 * walk through all sets on this node which could include:
		 *	- MN disksets
		 *	- traditional disksets
		 *	- non-existent disksets
		 * start mirror resync for all MN sets
		 */
		if ((max_sets = get_max_sets(ep)) == 0) {
			mde_perror(ep, "");
			md_exit(local_sp, 1);
		}

		/* start walking through all possible disksets */
		for (setno = 1; setno < max_sets; setno++) {
			if ((sp = metasetnosetname(setno, ep)) == NULL) {
				if (mdiserror(ep, MDE_NO_SET)) {
					/* No set for this setno - continue */
					mdclrerror(ep);
					continue;
				} else {
					mde_perror(ep, gettext("Unable to "
					    "get set %d information"), setno);
					md_exit(local_sp, 1);
				}
			}

			/* only check multi-node disksets */
			if (!meta_is_mn_set(sp, ep)) {
				mdclrerror(ep);
				continue;
			}

			if (meta_lock(sp, TRUE, ep) != 0) {
				mde_perror(ep, "");
				md_exit(local_sp, 1);
			}

			/* If this node isn't joined to set, do nothing */
			if (s_ownset(sp->setno, ep) != MD_SETOWNER_YES) {
				if (!mdisok(ep)) {
					mde_perror(ep, gettext("Could "
					    "not get set %s ownership"),
					    sp->setname);
					md_exit(sp, 1);
				}
				mdclrerror(ep);
				(void) meta_unlock(sp, ep);
				continue;
			}

			meta_mc_log(MC_LOG3, gettext("Step3 - begin "
			    "re-initialising rpc.mdcommd and resetting mirror "
			    "owners for set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));

			/* reinitialzse rpc.mdcommd with new nodelist */
			if (mdmn_reinit_set(setno, commd_timeout)) {
				md_eprintf(gettext(
				    "Could not re-initialise rpc.mdcommd for "
				    "set %s\n"), sp->setname);
				md_exit(sp, 1);
			}

			(void) memset(&cfg, 0, sizeof (cfg));
			cfg.c_id = 0;
			cfg.c_setno = sp->setno;
			if (metaioctl(MD_DB_GETDEV, &cfg, &cfg.c_mde,
			    NULL) != 0) {
				(void) mdstealerror(ep, &cfg.c_mde);
				mde_perror(ep, gettext("Could "
				    "not get set %s information"),
				    sp->setname);
				md_exit(sp, 1);
			}

			/* Don't do anything else if set is stale */
			if (cfg.c_flags & MDDB_C_STALE) {
				(void) meta_unlock(sp, ep);
				mdclrerror(ep);
				continue;
			}

			/* reset mirror owners */
			if (reset_state(RESET_OWNER, sp, MD_MIRROR, ep) == -1) {
				md_exit(sp, 1);
			}

			(void) meta_unlock(sp, ep);

			meta_mc_log(MC_LOG3, gettext("Step3 - rpc.mdcommd "
			    "re-initialised and mirror owners reset for "
			    "set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));
		}

		meta_mc_log(MC_LOG2, gettext("Step3 completed: %s"),
		    meta_print_hrtime(gethrtime() - start_time));

		break;

	case MC_STEP4:
		/*
		 * Step 4
		 *
		 * For all multinode sets do:
		 * - Resume the rpc.mdcommd messages.  Must resume all
		 *	sets before issuing I/O to any set since an error
		 * 	encountered in a commd suspended set could be
		 *	blocked waiting for commd in another set to resume.
		 *	(This happens since the daemon queues service
		 *	all sets).  An open of a soft partition causes
		 *	a read of the watermarks during the open.
		 * - If set is non-writable (not an owner or STALE), then
		 *	continue to next set.
		 *
		 * For all multinode sets do,
		 * - Reset ABR states for all mirrors, ie clear ABR if not
		 *	open on any node.
		 * - Reset ABR states for all soft partitions, ie clear ABR if
		 *	not open on any node.
		 * - For all slave nodes that have entered through the start
		 *	step, update the ABR state to that of the master and
		 *	get the submirror state from the master
		 * - meta_lock set
		 * - Resync all mirrors
		 * - unlock meta_lock for this set.
		 * - Choose a new owner for any orphaned resyncs
		 *
		 * There is one potential issue here. when concurrently
		 * resetting and updating the ABR state. If the master has ABR
		 * set, but should no longer have because the only node that
		 * had the metadevice open and had ABR set has paniced, the
		 * master will send a message to all nodes to clear the ABR
		 * state. Meanwhile any node that has come through the
		 * start step will get tstate from the master and will update
		 * ABR if it was set in tstate. So, we appear to have a problem
		 * if the following sequence occurs:-
		 * - The slave gets tstate with ABR set
		 * - The master sends a message to clear ABR
		 * - The slave updates ABR with the value it got from tstate.
		 * We now have the master with ABR clear and the slave with ABR
		 * set. Fortunately, having set ABR, the slave will close the
		 * metadevice after setting ABR and as there are no nodes with
		 * the device open, the close will send a message to clear ABR
		 * on all nodes. So, the nodes will all have ABR unset.
		 */

		/* expect the nodelist to follow the step name */
		if (argc < 1)
			usage(sp, 1);

		meta_mc_log(MC_LOG2, gettext("Starting Step4: %s"),
		    meta_print_hrtime(0));

		/*
		 * Does local set exist? If not, exit with 0
		 * since there's no reason to have this node panic if
		 * the local set cannot be started.
		 */
		if ((local_sp = load_local_set(ep)) == NULL) {
			md_exit(local_sp, 0);
		}

		/*
		 * walk through all sets on this node which could include:
		 *	- MN disksets
		 *	- traditional disksets
		 *	- non-existent disksets
		 * start mirror resync for all MN sets
		 */
		if ((max_sets = get_max_sets(ep)) == 0) {
			mde_perror(ep, "");
			md_exit(local_sp, 1);
		}

		/* Clear set_info structure */
		for (setno = 1; setno < max_sets; setno++) {
			set_info[setno] = 0;
		}

		/* start walking through all possible disksets */
		for (setno = 1; setno < max_sets; setno++) {
			if ((sp = metasetnosetname(setno, ep)) == NULL) {
				if (mdiserror(ep, MDE_NO_SET)) {
					/* No set for this setno - continue */
					mdclrerror(ep);
					continue;
				} else {
					mde_perror(ep, gettext("Unable to "
					    "get set %d information"), setno);
					md_exit(local_sp, 1);
				}
			}

			if ((sd = metaget_setdesc(sp, ep)) == NULL) {
				mde_perror(ep, gettext("Unable to get set "
				    "%s desc information"), sp->setname);
				mdclrerror(ep);
				continue;
			}

			/* only check multi-node disksets */
			if (!meta_is_mn_set(sp, ep)) {
				mdclrerror(ep);
				continue;
			}

			set_info[setno] |= SET_INFO_MN;

			/*
			 * If not an owner (all mddbs failed) or stale
			 * (< 50% mddbs operational), then set is
			 * non-writable so just resume commd and
			 * unblock mddb messages.
			 */
			mdclrerror(ep);
			if (s_ownset(sp->setno, ep) != MD_SETOWNER_YES) {
				set_info[setno] |= SET_INFO_NO_WR;
			}
			if (!mdisok(ep)) {
				mde_perror(ep, gettext("Could "
				    "not get set %s ownership"),
				    sp->setname);
				md_exit(local_sp, 1);
			}
			/* Set is owned - is it stale? */
			if (!set_info[setno] & SET_INFO_NO_WR) {
				(void) memset(&cfg, 0, sizeof (cfg));
				cfg.c_id = 0;
				cfg.c_setno = sp->setno;
				if (metaioctl(MD_DB_GETDEV, &cfg, &cfg.c_mde,
				    NULL) != 0) {
					(void) mdstealerror(ep, &cfg.c_mde);
					mde_perror(ep, gettext("Could "
					    "not get set %s information"),
					    sp->setname);
					md_exit(local_sp, 1);
				}
				if (cfg.c_flags & MDDB_C_STALE) {
					set_info[setno] |= SET_INFO_NO_WR;
				}
			}

			/* resume rpc.mdcommd */
			if (mdmn_resume(setno, MD_COMM_ALL_CLASSES, 0,
			    commd_timeout)) {
				md_eprintf(gettext("Unable to resume "
				    "rpc.mdcommd for set %s\n"), sp->setname);
				md_exit(local_sp, 1);
			}

			/* Unblock mddb parse messages */
			if (s_ownset(sp->setno, ep) == MD_SETOWNER_YES) {
				(void) memset(&mbp, 0, sizeof (mbp));
				mbp.c_setno = setno;
				mbp.c_blk_flags = MDDB_UNBLOCK_PARSE;
				if (metaioctl(MD_MN_MDDB_BLOCK, &mbp,
				    &mbp.c_mde, NULL)) {
					(void) mdstealerror(ep, &mbp.c_mde);
					mde_perror(ep, gettext("Could not "
					    "unblock set %s"), sp->setname);
					md_exit(local_sp, 1);
				}
			}
			meta_mc_log(MC_LOG3, gettext("Step4 - rpc.mdcommd "
			    "resumed and messages unblocked for set %s: %s"),
			    sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));
		}

		for (setno = 1; setno < max_sets; setno++) {
			int			start_step;

			/* Skip traditional disksets. */
			if ((set_info[setno] & SET_INFO_MN) == 0)
				continue;

			/*
			 * If already determined that this set is
			 * a non-writable set, then just continue
			 * to next set since there's nothing else
			 * to do for a non-writable set.
			 */
			if (set_info[setno] & SET_INFO_NO_WR)
				continue;

			if ((sp = metasetnosetname(setno, ep)) == NULL) {
				if (mdiserror(ep, MDE_NO_SET)) {
					/* No set for this setno - continue */
					mdclrerror(ep);
					continue;
				} else {
					mde_perror(ep, gettext("Unable to "
					    "get set %d information"), setno);
					md_exit(local_sp, 1);
				}
			}

			if ((sd = metaget_setdesc(sp, ep)) == NULL) {
				mde_perror(ep, gettext("Unable to get set "
				    "%s desc information"), sp->setname);
				mdclrerror(ep);
				continue;
			}

			/* See if this node came through the start step */
			(void) memset(&sf, 0, sizeof (sf));
			sf.sf_setno = sp->setno;
			sf.sf_flags = MDDB_NM_GET;
			/* Use magic to help protect ioctl against attack. */
			sf.sf_magic = MDDB_SETFLAGS_MAGIC;
			if (metaioctl(MD_MN_GET_SETFLAGS, &sf,
			    &sf.sf_mde, NULL)) {
				(void) mdstealerror(ep, &sf.sf_mde);
				mde_perror(ep, gettext("Could not get "
				    "start_step flag for set %s"), sp->setname);
				md_exit(local_sp, 1);
			}
			start_step =
			    (sf.sf_setflags & MD_SET_MN_START_RC)? 1: 0;

			/*
			 * We can now reset the start_step flag for the set
			 * if it was already set.
			 */
			if (start_step) {
				(void) memset(&sf, 0, sizeof (sf));
					sf.sf_setno = sp->setno;
				sf.sf_setflags = MD_SET_MN_START_RC;
				sf.sf_flags = MDDB_NM_RESET;
				/*
				 * Use magic to help protect ioctl
				 * against attack.
				 */
				sf.sf_magic = MDDB_SETFLAGS_MAGIC;
				if (metaioctl(MD_MN_SET_SETFLAGS, &sf,
				    &sf.sf_mde, NULL)) {
					(void) mdstealerror(ep, &sf.sf_mde);
					mde_perror(ep,
					    gettext("Could not reset "
					    "start_step flag for set %s"),
					    sp->setname);
				}
			}

			meta_mc_log(MC_LOG3, gettext("Step4 - begin setting "
			    "ABR state and restarting io's for "
			    "set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));


			/*
			 * If we are not the master and we have come through
			 * the start step, we must update the ABR states
			 * for mirrors and soft partitions. Also the submirror
			 * states need to be synchronised so that we see the
			 * same status as other previously joined members.
			 * This _must_ be done before starting the resync.
			 */
			if (!(sd->sd_mn_am_i_master) && start_step) {
				if (reset_state(GET_MIRROR_STATE, sp, MD_MIRROR,
				    ep) == -1) {
					md_exit(local_sp, 1);
				}
				if (reset_state(UPDATE_ABR, sp, MD_SP,
				    ep) == -1) {
					md_exit(local_sp, 1);
				}
				/*
				 * Mark the fact that we've got the mirror
				 * state. This allows the resync thread to
				 * determine if _it_ needs to issue this. This
				 * can happen if a node is added to a set after
				 * a reconfig cycle has completed.
				 */
				(void) memset(&sf, 0, sizeof (sf));
					sf.sf_setno = sp->setno;
				sf.sf_setflags = MD_SET_MN_MIR_STATE_RC;
				sf.sf_flags = MDDB_NM_SET;
				/*
				 * Use magic to help protect ioctl
				 * against attack.
				 */
				sf.sf_magic = MDDB_SETFLAGS_MAGIC;
				if (metaioctl(MD_MN_SET_SETFLAGS, &sf,
				    &sf.sf_mde, NULL)) {
					(void) mdstealerror(ep, &sf.sf_mde);
					mde_perror(ep,
					    gettext("Could not set "
					    "submirror state flag for set %s"),
					    sp->setname);
				}
			}

			/*
			 * All remaining actions are only performed by the
			 * master
			 */
			if (!(sd->sd_mn_am_i_master)) {
				if (meta_lock(sp, TRUE, ep) != 0) {
					mde_perror(ep, "");
					md_exit(local_sp, 1);
				}
				meta_mirror_resync_unblock(sp);
				(void) meta_unlock(sp, ep);
				continue;
			}

			/*
			 * If the master came through the start step, this
			 * implies that all of the nodes must have done the
			 * same and hence there can be no applications
			 * running. Hence no need to reset ABR
			 */
			if (!start_step) {
				/* Reset ABR state for mirrors */
				if (reset_state(RESET_ABR, sp, MD_MIRROR,
				    ep) == -1) {
					md_exit(local_sp, 1);
				}
				/* ...and now the same for soft partitions */
				if (reset_state(RESET_ABR, sp, MD_SP,
				    ep) == -1) {
					md_exit(local_sp, 1);
				}
			}

			/*
			 * choose owners for orphaned resyncs and reset
			 * non-orphaned resyncs so that an owner node that
			 * reboots will restart the resync if needed.
			 */
			if (reset_state(CHOOSE_OWNER, sp, MD_MIRROR, ep) == -1)
				md_exit(local_sp, 1);

			/*
			 * Must unlock set lock before meta_mirror_resync_all
			 * sends a message to run the metasync command
			 * which also grabs the meta_lock.
			 */
			if (meta_lock(sp, TRUE, ep) != 0) {
				mde_perror(ep, "");
				md_exit(local_sp, 1);
			}
			meta_mirror_resync_unblock(sp);
			(void) meta_unlock(sp, ep);

			/* resync all mirrors in set */
			if (meta_mirror_resync_all(sp, 0, ep) != 0) {
				mde_perror(ep, gettext("Mirror resyncs "
				    "failed for set %s"), sp->setname);
				md_exit(local_sp, 1);
			}

			meta_mc_log(MC_LOG3, gettext("Step4 - io's restarted "
			    "for set %s: %s"), sp->setname,
			    meta_print_hrtime(gethrtime() - start_time));
		}

		meta_mc_log(MC_LOG2, gettext("Step4 completed: %s"),
		    meta_print_hrtime(gethrtime() - start_time));

		break;

	default:
		usage(sp, 1);
		break;
	}

	md_exit(sp, 0);
	/* NOTREACHED */
	return (0);
}