xref: /titanic_50/usr/src/cmd/svc/startd/method.c (revision dfe5735016dd804901e28c9585549f5aa15bb63b)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
53ad28c1eSrm88369  * Common Development and Distribution License (the "License").
63ad28c1eSrm88369  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22*dfe57350Sjeanm  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate /*
297c478bd9Sstevel@tonic-gate  * method.c - method execution functions
307c478bd9Sstevel@tonic-gate  *
317c478bd9Sstevel@tonic-gate  * This file contains the routines needed to run a method:  a fork(2)-exec(2)
327c478bd9Sstevel@tonic-gate  * invocation monitored using either the contract filesystem or waitpid(2).
337c478bd9Sstevel@tonic-gate  * (Plain fork1(2) support is provided in fork.c.)
347c478bd9Sstevel@tonic-gate  *
357c478bd9Sstevel@tonic-gate  * Contract Transfer
367c478bd9Sstevel@tonic-gate  *   When we restart a service, we want to transfer any contracts that the old
377c478bd9Sstevel@tonic-gate  *   service's contract inherited.  This means that (a) we must not abandon the
387c478bd9Sstevel@tonic-gate  *   old contract when the service dies and (b) we must write the id of the old
397c478bd9Sstevel@tonic-gate  *   contract into the terms of the new contract.  There should be limits to
407c478bd9Sstevel@tonic-gate  *   (a), though, since we don't want to keep the contract around forever.  To
417c478bd9Sstevel@tonic-gate  *   this end we'll say that services in the offline state may have a contract
427c478bd9Sstevel@tonic-gate  *   to be transfered and services in the disabled or maintenance states cannot.
437c478bd9Sstevel@tonic-gate  *   This means that when a service transitions from online (or degraded) to
447c478bd9Sstevel@tonic-gate  *   offline, the contract should be preserved, and when the service transitions
457c478bd9Sstevel@tonic-gate  *   from offline to online (i.e., the start method), we'll transfer inherited
467c478bd9Sstevel@tonic-gate  *   contracts.
477c478bd9Sstevel@tonic-gate  */
487c478bd9Sstevel@tonic-gate 
497c478bd9Sstevel@tonic-gate #include <sys/contract/process.h>
507c478bd9Sstevel@tonic-gate #include <sys/ctfs.h>
517c478bd9Sstevel@tonic-gate #include <sys/stat.h>
527c478bd9Sstevel@tonic-gate #include <sys/time.h>
537c478bd9Sstevel@tonic-gate #include <sys/types.h>
547c478bd9Sstevel@tonic-gate #include <sys/uio.h>
557c478bd9Sstevel@tonic-gate #include <sys/wait.h>
567c478bd9Sstevel@tonic-gate #include <alloca.h>
577c478bd9Sstevel@tonic-gate #include <assert.h>
587c478bd9Sstevel@tonic-gate #include <errno.h>
597c478bd9Sstevel@tonic-gate #include <fcntl.h>
607c478bd9Sstevel@tonic-gate #include <libcontract.h>
617c478bd9Sstevel@tonic-gate #include <libcontract_priv.h>
627c478bd9Sstevel@tonic-gate #include <libgen.h>
637c478bd9Sstevel@tonic-gate #include <librestart.h>
647c478bd9Sstevel@tonic-gate #include <libscf.h>
657c478bd9Sstevel@tonic-gate #include <limits.h>
667c478bd9Sstevel@tonic-gate #include <port.h>
677c478bd9Sstevel@tonic-gate #include <sac.h>
687c478bd9Sstevel@tonic-gate #include <signal.h>
697c478bd9Sstevel@tonic-gate #include <stdlib.h>
707c478bd9Sstevel@tonic-gate #include <string.h>
717c478bd9Sstevel@tonic-gate #include <strings.h>
727c478bd9Sstevel@tonic-gate #include <unistd.h>
73*dfe57350Sjeanm #include <atomic.h>
74*dfe57350Sjeanm #include <poll.h>
757c478bd9Sstevel@tonic-gate 
767c478bd9Sstevel@tonic-gate #include "startd.h"
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate #define	SBIN_SH		"/sbin/sh"
797c478bd9Sstevel@tonic-gate 
807c478bd9Sstevel@tonic-gate /*
81*dfe57350Sjeanm  * Used to tell if contracts are in the process of being
82*dfe57350Sjeanm  * stored into the svc.startd internal hash table.
83*dfe57350Sjeanm  */
84*dfe57350Sjeanm volatile uint16_t	storing_contract = 0;
85*dfe57350Sjeanm 
86*dfe57350Sjeanm /*
877c478bd9Sstevel@tonic-gate  * Mapping from restart_on method-type to contract events.  Must correspond to
887c478bd9Sstevel@tonic-gate  * enum method_restart_t.
897c478bd9Sstevel@tonic-gate  */
907c478bd9Sstevel@tonic-gate static uint_t method_events[] = {
917c478bd9Sstevel@tonic-gate 	/* METHOD_RESTART_ALL */
927c478bd9Sstevel@tonic-gate 	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
937c478bd9Sstevel@tonic-gate 	/* METHOD_RESTART_EXTERNAL_FAULT */
947c478bd9Sstevel@tonic-gate 	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
957c478bd9Sstevel@tonic-gate 	/* METHOD_RESTART_ANY_FAULT */
967c478bd9Sstevel@tonic-gate 	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
977c478bd9Sstevel@tonic-gate };
987c478bd9Sstevel@tonic-gate 
997c478bd9Sstevel@tonic-gate /*
1007c478bd9Sstevel@tonic-gate  * method_record_start(restarter_inst_t *)
1017c478bd9Sstevel@tonic-gate  *   Record a service start for rate limiting.  Place the current time
1027c478bd9Sstevel@tonic-gate  *   in the circular array of instance starts.
1037c478bd9Sstevel@tonic-gate  */
1047c478bd9Sstevel@tonic-gate static void
1057c478bd9Sstevel@tonic-gate method_record_start(restarter_inst_t *inst)
1067c478bd9Sstevel@tonic-gate {
1077c478bd9Sstevel@tonic-gate 	int index = inst->ri_start_index++ % RINST_START_TIMES;
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate 	inst->ri_start_time[index] = gethrtime();
1107c478bd9Sstevel@tonic-gate }
1117c478bd9Sstevel@tonic-gate 
1127c478bd9Sstevel@tonic-gate /*
1137c478bd9Sstevel@tonic-gate  * method_rate_critical(restarter_inst_t *)
1147c478bd9Sstevel@tonic-gate  *    Return true if the average start interval is less than the permitted
1157c478bd9Sstevel@tonic-gate  *    interval.  Implicit success if insufficient measurements for an
1167c478bd9Sstevel@tonic-gate  *    average exist.
1177c478bd9Sstevel@tonic-gate  */
1187c478bd9Sstevel@tonic-gate static int
1197c478bd9Sstevel@tonic-gate method_rate_critical(restarter_inst_t *inst)
1207c478bd9Sstevel@tonic-gate {
1217c478bd9Sstevel@tonic-gate 	uint_t n = inst->ri_start_index;
1227c478bd9Sstevel@tonic-gate 	hrtime_t avg_ns = 0;
1237c478bd9Sstevel@tonic-gate 
1247c478bd9Sstevel@tonic-gate 	if (inst->ri_start_index < RINST_START_TIMES)
1257c478bd9Sstevel@tonic-gate 		return (0);
1267c478bd9Sstevel@tonic-gate 
1277c478bd9Sstevel@tonic-gate 	avg_ns =
1287c478bd9Sstevel@tonic-gate 	    (inst->ri_start_time[(n - 1) % RINST_START_TIMES] -
1297c478bd9Sstevel@tonic-gate 	    inst->ri_start_time[n % RINST_START_TIMES]) /
1307c478bd9Sstevel@tonic-gate 	    (RINST_START_TIMES - 1);
1317c478bd9Sstevel@tonic-gate 
1327c478bd9Sstevel@tonic-gate 	return (avg_ns < RINST_FAILURE_RATE_NS);
1337c478bd9Sstevel@tonic-gate }
1347c478bd9Sstevel@tonic-gate 
1357c478bd9Sstevel@tonic-gate /*
1367c478bd9Sstevel@tonic-gate  * int method_is_transient()
1377c478bd9Sstevel@tonic-gate  *   Determine if the method for the given instance is transient,
1387c478bd9Sstevel@tonic-gate  *   from a contract perspective. Return 1 if it is, and 0 if it isn't.
1397c478bd9Sstevel@tonic-gate  */
1407c478bd9Sstevel@tonic-gate static int
1417c478bd9Sstevel@tonic-gate method_is_transient(restarter_inst_t *inst, int type)
1427c478bd9Sstevel@tonic-gate {
1437c478bd9Sstevel@tonic-gate 	if (instance_is_transient_style(inst) || type != METHOD_START)
1447c478bd9Sstevel@tonic-gate 		return (1);
1457c478bd9Sstevel@tonic-gate 	else
1467c478bd9Sstevel@tonic-gate 		return (0);
1477c478bd9Sstevel@tonic-gate }
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate /*
1507c478bd9Sstevel@tonic-gate  * void method_store_contract()
1517c478bd9Sstevel@tonic-gate  *   Store the newly created contract id into local structures and
1527c478bd9Sstevel@tonic-gate  *   the repository.  If the repository connection is broken it is rebound.
1537c478bd9Sstevel@tonic-gate  */
1547c478bd9Sstevel@tonic-gate static void
1557c478bd9Sstevel@tonic-gate method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
1567c478bd9Sstevel@tonic-gate {
1577c478bd9Sstevel@tonic-gate 	int r;
1587c478bd9Sstevel@tonic-gate 	boolean_t primary;
1597c478bd9Sstevel@tonic-gate 
1607c478bd9Sstevel@tonic-gate 	if (errno = contract_latest(cid))
1617c478bd9Sstevel@tonic-gate 		uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
1627c478bd9Sstevel@tonic-gate 
1637c478bd9Sstevel@tonic-gate 	primary = !method_is_transient(inst, type);
1647c478bd9Sstevel@tonic-gate 
1657c478bd9Sstevel@tonic-gate 	if (!primary) {
1667c478bd9Sstevel@tonic-gate 		if (inst->ri_i.i_transient_ctid != 0) {
1677c478bd9Sstevel@tonic-gate 			log_framework(LOG_INFO,
1687c478bd9Sstevel@tonic-gate 			    "%s: transient ctid expected to be 0 but "
1697c478bd9Sstevel@tonic-gate 			    "was set to %ld\n", inst->ri_i.i_fmri,
1707c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_transient_ctid);
1717c478bd9Sstevel@tonic-gate 		}
1727c478bd9Sstevel@tonic-gate 
1737c478bd9Sstevel@tonic-gate 		inst->ri_i.i_transient_ctid = *cid;
1747c478bd9Sstevel@tonic-gate 	} else {
1757c478bd9Sstevel@tonic-gate 		if (inst->ri_i.i_primary_ctid != 0) {
1767c478bd9Sstevel@tonic-gate 			/*
1777c478bd9Sstevel@tonic-gate 			 * There was an old contract that we transferred.
1787c478bd9Sstevel@tonic-gate 			 * Remove it.
1797c478bd9Sstevel@tonic-gate 			 */
1807c478bd9Sstevel@tonic-gate 			method_remove_contract(inst, B_TRUE, B_FALSE);
1817c478bd9Sstevel@tonic-gate 		}
1827c478bd9Sstevel@tonic-gate 
1837c478bd9Sstevel@tonic-gate 		if (inst->ri_i.i_primary_ctid != 0) {
1847c478bd9Sstevel@tonic-gate 			log_framework(LOG_INFO,
1857c478bd9Sstevel@tonic-gate 			    "%s: primary ctid expected to be 0 but "
1867c478bd9Sstevel@tonic-gate 			    "was set to %ld\n", inst->ri_i.i_fmri,
1877c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_primary_ctid);
1887c478bd9Sstevel@tonic-gate 		}
1897c478bd9Sstevel@tonic-gate 
1907c478bd9Sstevel@tonic-gate 		inst->ri_i.i_primary_ctid = *cid;
1917c478bd9Sstevel@tonic-gate 		inst->ri_i.i_primary_ctid_stopped = 0;
1927c478bd9Sstevel@tonic-gate 
193*dfe57350Sjeanm 		log_framework(LOG_DEBUG, "Storing primary contract %ld for "
194*dfe57350Sjeanm 		    "%s.\n", *cid, inst->ri_i.i_fmri);
195*dfe57350Sjeanm 
1967c478bd9Sstevel@tonic-gate 		contract_hash_store(*cid, inst->ri_id);
1977c478bd9Sstevel@tonic-gate 	}
1987c478bd9Sstevel@tonic-gate 
1997c478bd9Sstevel@tonic-gate again:
2007c478bd9Sstevel@tonic-gate 	if (inst->ri_mi_deleted)
2017c478bd9Sstevel@tonic-gate 		return;
2027c478bd9Sstevel@tonic-gate 
2037c478bd9Sstevel@tonic-gate 	r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
2047c478bd9Sstevel@tonic-gate 	    RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
2057c478bd9Sstevel@tonic-gate 	switch (r) {
2067c478bd9Sstevel@tonic-gate 	case 0:
2077c478bd9Sstevel@tonic-gate 		break;
2087c478bd9Sstevel@tonic-gate 
2097c478bd9Sstevel@tonic-gate 	case ECANCELED:
2107c478bd9Sstevel@tonic-gate 		inst->ri_mi_deleted = B_TRUE;
2117c478bd9Sstevel@tonic-gate 		break;
2127c478bd9Sstevel@tonic-gate 
2137c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
2147c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
2157c478bd9Sstevel@tonic-gate 		/* FALLTHROUGH */
2167c478bd9Sstevel@tonic-gate 
2177c478bd9Sstevel@tonic-gate 	case EBADF:
2187c478bd9Sstevel@tonic-gate 		libscf_reget_instance(inst);
2197c478bd9Sstevel@tonic-gate 		goto again;
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate 	case ENOMEM:
2227c478bd9Sstevel@tonic-gate 	case EPERM:
2237c478bd9Sstevel@tonic-gate 	case EACCES:
2247c478bd9Sstevel@tonic-gate 	case EROFS:
2257c478bd9Sstevel@tonic-gate 		uu_die("%s: Couldn't store contract id %ld",
2267c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri, *cid);
2277c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
2287c478bd9Sstevel@tonic-gate 
2297c478bd9Sstevel@tonic-gate 	case EINVAL:
2307c478bd9Sstevel@tonic-gate 	default:
2317c478bd9Sstevel@tonic-gate 		bad_error("restarter_store_contract", r);
2327c478bd9Sstevel@tonic-gate 	}
2337c478bd9Sstevel@tonic-gate }
2347c478bd9Sstevel@tonic-gate 
2357c478bd9Sstevel@tonic-gate /*
2367c478bd9Sstevel@tonic-gate  * void method_remove_contract()
2377c478bd9Sstevel@tonic-gate  *   Remove any non-permanent contracts from internal structures and
2387c478bd9Sstevel@tonic-gate  *   the repository, then abandon them.
2397c478bd9Sstevel@tonic-gate  *   Returns
2407c478bd9Sstevel@tonic-gate  *     0 - success
2417c478bd9Sstevel@tonic-gate  *     ECANCELED - inst was deleted from the repository
2427c478bd9Sstevel@tonic-gate  *
2437c478bd9Sstevel@tonic-gate  *   If the repository connection was broken, it is rebound.
2447c478bd9Sstevel@tonic-gate  */
2457c478bd9Sstevel@tonic-gate void
2467c478bd9Sstevel@tonic-gate method_remove_contract(restarter_inst_t *inst, boolean_t primary,
2477c478bd9Sstevel@tonic-gate     boolean_t abandon)
2487c478bd9Sstevel@tonic-gate {
2497c478bd9Sstevel@tonic-gate 	ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
2507c478bd9Sstevel@tonic-gate 	    &inst->ri_i.i_transient_ctid;
2517c478bd9Sstevel@tonic-gate 
2527c478bd9Sstevel@tonic-gate 	int r;
2537c478bd9Sstevel@tonic-gate 
2547c478bd9Sstevel@tonic-gate 	assert(*ctidp != 0);
2557c478bd9Sstevel@tonic-gate 
2567c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
2577c478bd9Sstevel@tonic-gate 	    primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
2587c478bd9Sstevel@tonic-gate 
2597c478bd9Sstevel@tonic-gate 	if (abandon)
2607c478bd9Sstevel@tonic-gate 		contract_abandon(*ctidp);
2617c478bd9Sstevel@tonic-gate 
2627c478bd9Sstevel@tonic-gate again:
2637c478bd9Sstevel@tonic-gate 	if (inst->ri_mi_deleted) {
2647c478bd9Sstevel@tonic-gate 		r = ECANCELED;
2657c478bd9Sstevel@tonic-gate 		goto out;
2667c478bd9Sstevel@tonic-gate 	}
2677c478bd9Sstevel@tonic-gate 
2687c478bd9Sstevel@tonic-gate 	r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
2697c478bd9Sstevel@tonic-gate 	    RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
2707c478bd9Sstevel@tonic-gate 	switch (r) {
2717c478bd9Sstevel@tonic-gate 	case 0:
2727c478bd9Sstevel@tonic-gate 		break;
2737c478bd9Sstevel@tonic-gate 
2747c478bd9Sstevel@tonic-gate 	case ECANCELED:
2757c478bd9Sstevel@tonic-gate 		inst->ri_mi_deleted = B_TRUE;
2767c478bd9Sstevel@tonic-gate 		break;
2777c478bd9Sstevel@tonic-gate 
2787c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
2797c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
2807c478bd9Sstevel@tonic-gate 		/* FALLTHROUGH */
2817c478bd9Sstevel@tonic-gate 
2827c478bd9Sstevel@tonic-gate 	case EBADF:
2837c478bd9Sstevel@tonic-gate 		libscf_reget_instance(inst);
2847c478bd9Sstevel@tonic-gate 		goto again;
2857c478bd9Sstevel@tonic-gate 
2867c478bd9Sstevel@tonic-gate 	case ENOMEM:
2877c478bd9Sstevel@tonic-gate 	case EPERM:
2887c478bd9Sstevel@tonic-gate 	case EACCES:
2897c478bd9Sstevel@tonic-gate 	case EROFS:
2907c478bd9Sstevel@tonic-gate 		log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
2917c478bd9Sstevel@tonic-gate 		    "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
2927c478bd9Sstevel@tonic-gate 		break;
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate 	case EINVAL:
2957c478bd9Sstevel@tonic-gate 	default:
2967c478bd9Sstevel@tonic-gate 		bad_error("restarter_remove_contract", r);
2977c478bd9Sstevel@tonic-gate 	}
2987c478bd9Sstevel@tonic-gate 
2997c478bd9Sstevel@tonic-gate out:
3007c478bd9Sstevel@tonic-gate 	if (primary)
3017c478bd9Sstevel@tonic-gate 		contract_hash_remove(*ctidp);
3027c478bd9Sstevel@tonic-gate 
3037c478bd9Sstevel@tonic-gate 	*ctidp = 0;
3047c478bd9Sstevel@tonic-gate }
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate /*
3077c478bd9Sstevel@tonic-gate  * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
3087c478bd9Sstevel@tonic-gate  *
3097c478bd9Sstevel@tonic-gate  *   Activate a contract template for the type method of inst.  type,
3107c478bd9Sstevel@tonic-gate  *   restart_on, and cte_mask dictate the critical events term of the contract.
3117c478bd9Sstevel@tonic-gate  *   Returns
3127c478bd9Sstevel@tonic-gate  *     0 - success
3137c478bd9Sstevel@tonic-gate  *     ECANCELED - inst has been deleted from the repository
3147c478bd9Sstevel@tonic-gate  */
3157c478bd9Sstevel@tonic-gate static int
3167c478bd9Sstevel@tonic-gate method_ready_contract(restarter_inst_t *inst, int type,
3177c478bd9Sstevel@tonic-gate     method_restart_t restart_on, uint_t cte_mask)
3187c478bd9Sstevel@tonic-gate {
3197c478bd9Sstevel@tonic-gate 	int tmpl, err, istrans, iswait, ret;
3207c478bd9Sstevel@tonic-gate 	uint_t cevents, fevents;
3217c478bd9Sstevel@tonic-gate 
3227c478bd9Sstevel@tonic-gate 	/*
3237c478bd9Sstevel@tonic-gate 	 * Correctly supporting wait-style services is tricky without
3247c478bd9Sstevel@tonic-gate 	 * rearchitecting startd to cope with multiple event sources
3257c478bd9Sstevel@tonic-gate 	 * simultaneously trying to stop an instance.  Until a better
3267c478bd9Sstevel@tonic-gate 	 * solution is implemented, we avoid this problem for
3277c478bd9Sstevel@tonic-gate 	 * wait-style services by making contract events fatal and
3287c478bd9Sstevel@tonic-gate 	 * letting the wait code alone handle stopping the service.
3297c478bd9Sstevel@tonic-gate 	 */
3307c478bd9Sstevel@tonic-gate 	iswait = instance_is_wait_style(inst);
3317c478bd9Sstevel@tonic-gate 	istrans = method_is_transient(inst, type);
3327c478bd9Sstevel@tonic-gate 
3337c478bd9Sstevel@tonic-gate 	tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
3347c478bd9Sstevel@tonic-gate 	if (tmpl == -1)
3357c478bd9Sstevel@tonic-gate 		uu_die("Could not create contract template");
3367c478bd9Sstevel@tonic-gate 
3377c478bd9Sstevel@tonic-gate 	/*
3387c478bd9Sstevel@tonic-gate 	 * We assume non-login processes are unlikely to create
3397c478bd9Sstevel@tonic-gate 	 * multiple process groups, and set CT_PR_PGRPONLY for all
3407c478bd9Sstevel@tonic-gate 	 * wait-style services' contracts.
3417c478bd9Sstevel@tonic-gate 	 */
3427c478bd9Sstevel@tonic-gate 	err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
3437c478bd9Sstevel@tonic-gate 	    (iswait ? CT_PR_PGRPONLY : 0));
3447c478bd9Sstevel@tonic-gate 	assert(err == 0);
3457c478bd9Sstevel@tonic-gate 
3467c478bd9Sstevel@tonic-gate 	if (istrans) {
3477c478bd9Sstevel@tonic-gate 		cevents = 0;
3487c478bd9Sstevel@tonic-gate 		fevents = 0;
3497c478bd9Sstevel@tonic-gate 	} else {
3507c478bd9Sstevel@tonic-gate 		assert(restart_on >= 0);
3517c478bd9Sstevel@tonic-gate 		assert(restart_on <= METHOD_RESTART_ANY_FAULT);
3527c478bd9Sstevel@tonic-gate 		cevents = method_events[restart_on] & ~cte_mask;
3537c478bd9Sstevel@tonic-gate 		fevents = iswait ?
3547c478bd9Sstevel@tonic-gate 		    (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
3557c478bd9Sstevel@tonic-gate 		    0;
3567c478bd9Sstevel@tonic-gate 	}
3577c478bd9Sstevel@tonic-gate 
3587c478bd9Sstevel@tonic-gate 	err = ct_tmpl_set_critical(tmpl, cevents);
3597c478bd9Sstevel@tonic-gate 	assert(err == 0);
3607c478bd9Sstevel@tonic-gate 
3617c478bd9Sstevel@tonic-gate 	err = ct_tmpl_set_informative(tmpl, 0);
3627c478bd9Sstevel@tonic-gate 	assert(err == 0);
3637c478bd9Sstevel@tonic-gate 	err = ct_pr_tmpl_set_fatal(tmpl, fevents);
3647c478bd9Sstevel@tonic-gate 	assert(err == 0);
3657c478bd9Sstevel@tonic-gate 
3667c478bd9Sstevel@tonic-gate 	err = ct_tmpl_set_cookie(tmpl, istrans ?  METHOD_OTHER_COOKIE :
3677c478bd9Sstevel@tonic-gate 	    METHOD_START_COOKIE);
3687c478bd9Sstevel@tonic-gate 	assert(err == 0);
3697c478bd9Sstevel@tonic-gate 
3707c478bd9Sstevel@tonic-gate 	if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
3717c478bd9Sstevel@tonic-gate 		ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
3727c478bd9Sstevel@tonic-gate 		switch (ret) {
3737c478bd9Sstevel@tonic-gate 		case 0:
3747c478bd9Sstevel@tonic-gate 			break;
3757c478bd9Sstevel@tonic-gate 
3767c478bd9Sstevel@tonic-gate 		case ENOTEMPTY:
3777c478bd9Sstevel@tonic-gate 			/* No contracts for you! */
3787c478bd9Sstevel@tonic-gate 			method_remove_contract(inst, B_TRUE, B_TRUE);
3797c478bd9Sstevel@tonic-gate 			if (inst->ri_mi_deleted) {
3807c478bd9Sstevel@tonic-gate 				ret = ECANCELED;
3817c478bd9Sstevel@tonic-gate 				goto out;
3827c478bd9Sstevel@tonic-gate 			}
3837c478bd9Sstevel@tonic-gate 			break;
3847c478bd9Sstevel@tonic-gate 
3857c478bd9Sstevel@tonic-gate 		case EINVAL:
3867c478bd9Sstevel@tonic-gate 		case ESRCH:
3877c478bd9Sstevel@tonic-gate 		case EACCES:
3887c478bd9Sstevel@tonic-gate 		default:
3897c478bd9Sstevel@tonic-gate 			bad_error("ct_pr_tmpl_set_transfer", ret);
3907c478bd9Sstevel@tonic-gate 		}
3917c478bd9Sstevel@tonic-gate 	}
3927c478bd9Sstevel@tonic-gate 
3937c478bd9Sstevel@tonic-gate 	err = ct_tmpl_activate(tmpl);
3947c478bd9Sstevel@tonic-gate 	assert(err == 0);
3957c478bd9Sstevel@tonic-gate 
3967c478bd9Sstevel@tonic-gate 	ret = 0;
3977c478bd9Sstevel@tonic-gate 
3987c478bd9Sstevel@tonic-gate out:
3997c478bd9Sstevel@tonic-gate 	err = close(tmpl);
4007c478bd9Sstevel@tonic-gate 	assert(err == 0);
4017c478bd9Sstevel@tonic-gate 
4027c478bd9Sstevel@tonic-gate 	return (ret);
4037c478bd9Sstevel@tonic-gate }
4047c478bd9Sstevel@tonic-gate 
4057c478bd9Sstevel@tonic-gate static const char *method_names[] = { "start", "stop", "refresh" };
4067c478bd9Sstevel@tonic-gate 
4077c478bd9Sstevel@tonic-gate static void
4087c478bd9Sstevel@tonic-gate exec_method(const restarter_inst_t *inst, int type, const char *method,
4097c478bd9Sstevel@tonic-gate     struct method_context *mcp, uint8_t need_session)
4107c478bd9Sstevel@tonic-gate {
4117c478bd9Sstevel@tonic-gate 	char *cmd;
4127c478bd9Sstevel@tonic-gate 	const char *errf;
4137c478bd9Sstevel@tonic-gate 	char **nenv;
4147c478bd9Sstevel@tonic-gate 
4157c478bd9Sstevel@tonic-gate 	cmd = uu_msprintf("exec %s", method);
4167c478bd9Sstevel@tonic-gate 
4177c478bd9Sstevel@tonic-gate 	if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
4187c478bd9Sstevel@tonic-gate 		(void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
4197c478bd9Sstevel@tonic-gate 
4207c478bd9Sstevel@tonic-gate 	setlog(inst->ri_logstem);
4217c478bd9Sstevel@tonic-gate 	log_instance(inst, B_FALSE, "Executing %s method (\"%s\")",
4227c478bd9Sstevel@tonic-gate 	    method_names[type], method);
4237c478bd9Sstevel@tonic-gate 
4247c478bd9Sstevel@tonic-gate 	if (need_session)
4257c478bd9Sstevel@tonic-gate 		(void) setpgrp();
4267c478bd9Sstevel@tonic-gate 
4277c478bd9Sstevel@tonic-gate 	/* Set credentials. */
4287c478bd9Sstevel@tonic-gate 	errno = restarter_set_method_context(mcp, &errf);
4297c478bd9Sstevel@tonic-gate 	if (errno != 0) {
4307c478bd9Sstevel@tonic-gate 		(void) fputs("svc.startd could not set context for method: ",
4317c478bd9Sstevel@tonic-gate 		    stderr);
4327c478bd9Sstevel@tonic-gate 
4337c478bd9Sstevel@tonic-gate 		if (errno == -1) {
4347c478bd9Sstevel@tonic-gate 			if (strcmp(errf, "core_set_process_path") == 0) {
4357c478bd9Sstevel@tonic-gate 				(void) fputs("Could not set corefile path.\n",
4367c478bd9Sstevel@tonic-gate 				    stderr);
4377c478bd9Sstevel@tonic-gate 			} else if (strcmp(errf, "setproject") == 0) {
4387c478bd9Sstevel@tonic-gate 				(void) fprintf(stderr, "%s: a resource control "
4397c478bd9Sstevel@tonic-gate 				    "assignment failed\n", errf);
4407c478bd9Sstevel@tonic-gate 			} else if (strcmp(errf, "pool_set_binding") == 0) {
4417c478bd9Sstevel@tonic-gate 				(void) fprintf(stderr, "%s: a system error "
4427c478bd9Sstevel@tonic-gate 				    "occurred\n", errf);
4437c478bd9Sstevel@tonic-gate 			} else {
4447c478bd9Sstevel@tonic-gate #ifndef NDEBUG
4457c478bd9Sstevel@tonic-gate 				uu_warn("%s:%d: Bad function name \"%s\" for "
4467c478bd9Sstevel@tonic-gate 				    "error %d from "
4477c478bd9Sstevel@tonic-gate 				    "restarter_set_method_context().\n",
4487c478bd9Sstevel@tonic-gate 				    __FILE__, __LINE__, errf, errno);
4497c478bd9Sstevel@tonic-gate #endif
4507c478bd9Sstevel@tonic-gate 				abort();
4517c478bd9Sstevel@tonic-gate 			}
4527c478bd9Sstevel@tonic-gate 
4537c478bd9Sstevel@tonic-gate 			exit(1);
4547c478bd9Sstevel@tonic-gate 		}
4557c478bd9Sstevel@tonic-gate 
4567c478bd9Sstevel@tonic-gate 		if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
4577c478bd9Sstevel@tonic-gate 			switch (errno) {
4587c478bd9Sstevel@tonic-gate 			case ENOENT:
4597c478bd9Sstevel@tonic-gate 				(void) fprintf(stderr, "%s: the pool could not "
4607c478bd9Sstevel@tonic-gate 				    "be found\n", errf);
4617c478bd9Sstevel@tonic-gate 				break;
4627c478bd9Sstevel@tonic-gate 
4637c478bd9Sstevel@tonic-gate 			case EBADF:
4647c478bd9Sstevel@tonic-gate 				(void) fprintf(stderr, "%s: the configuration "
4657c478bd9Sstevel@tonic-gate 				    "is invalid\n", errf);
4667c478bd9Sstevel@tonic-gate 				break;
4677c478bd9Sstevel@tonic-gate 
4683ad28c1eSrm88369 			case EINVAL:
4693ad28c1eSrm88369 				(void) fprintf(stderr, "%s: pool name \"%s\" "
4703ad28c1eSrm88369 				    "is invalid\n", errf, mcp->resource_pool);
4713ad28c1eSrm88369 				break;
4723ad28c1eSrm88369 
4737c478bd9Sstevel@tonic-gate 			default:
4747c478bd9Sstevel@tonic-gate #ifndef NDEBUG
4757c478bd9Sstevel@tonic-gate 				uu_warn("%s:%d: Bad error %d for function %s "
4767c478bd9Sstevel@tonic-gate 				    "in restarter_set_method_context().\n",
4777c478bd9Sstevel@tonic-gate 				    __FILE__, __LINE__, errno, errf);
4787c478bd9Sstevel@tonic-gate #endif
4797c478bd9Sstevel@tonic-gate 				abort();
4807c478bd9Sstevel@tonic-gate 			}
4817c478bd9Sstevel@tonic-gate 
4827c478bd9Sstevel@tonic-gate 			exit(SMF_EXIT_ERR_CONFIG);
4837c478bd9Sstevel@tonic-gate 		}
4847c478bd9Sstevel@tonic-gate 
4857c478bd9Sstevel@tonic-gate 		if (errf != NULL) {
4867c478bd9Sstevel@tonic-gate 			perror(errf);
4877c478bd9Sstevel@tonic-gate 
4887c478bd9Sstevel@tonic-gate 			switch (errno) {
4897c478bd9Sstevel@tonic-gate 			case EINVAL:
4907c478bd9Sstevel@tonic-gate 			case EPERM:
4917c478bd9Sstevel@tonic-gate 			case ENOENT:
4927c478bd9Sstevel@tonic-gate 			case ENAMETOOLONG:
4937c478bd9Sstevel@tonic-gate 			case ERANGE:
4947c478bd9Sstevel@tonic-gate 			case ESRCH:
4957c478bd9Sstevel@tonic-gate 				exit(SMF_EXIT_ERR_CONFIG);
4967c478bd9Sstevel@tonic-gate 				/* NOTREACHED */
4977c478bd9Sstevel@tonic-gate 
4987c478bd9Sstevel@tonic-gate 			default:
4997c478bd9Sstevel@tonic-gate 				exit(1);
5007c478bd9Sstevel@tonic-gate 			}
5017c478bd9Sstevel@tonic-gate 		}
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 		switch (errno) {
5047c478bd9Sstevel@tonic-gate 		case ENOMEM:
5057c478bd9Sstevel@tonic-gate 			(void) fputs("Out of memory.\n", stderr);
5067c478bd9Sstevel@tonic-gate 			exit(1);
5077c478bd9Sstevel@tonic-gate 			/* NOTREACHED */
5087c478bd9Sstevel@tonic-gate 
5097c478bd9Sstevel@tonic-gate 		case ENOENT:
5107c478bd9Sstevel@tonic-gate 			(void) fputs("Missing passwd entry for user.\n",
5117c478bd9Sstevel@tonic-gate 			    stderr);
5127c478bd9Sstevel@tonic-gate 			exit(SMF_EXIT_ERR_CONFIG);
5137c478bd9Sstevel@tonic-gate 			/* NOTREACHED */
5147c478bd9Sstevel@tonic-gate 
5157c478bd9Sstevel@tonic-gate 		default:
5167c478bd9Sstevel@tonic-gate #ifndef NDEBUG
5177c478bd9Sstevel@tonic-gate 			uu_warn("%s:%d: Bad miscellaneous error %d from "
5187c478bd9Sstevel@tonic-gate 			    "restarter_set_method_context().\n", __FILE__,
5197c478bd9Sstevel@tonic-gate 			    __LINE__, errno);
5207c478bd9Sstevel@tonic-gate #endif
5217c478bd9Sstevel@tonic-gate 			abort();
5227c478bd9Sstevel@tonic-gate 		}
5237c478bd9Sstevel@tonic-gate 	}
5247c478bd9Sstevel@tonic-gate 
5257c478bd9Sstevel@tonic-gate 	nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst, method);
5267c478bd9Sstevel@tonic-gate 
5277c478bd9Sstevel@tonic-gate 	log_preexec();
5287c478bd9Sstevel@tonic-gate 
5297c478bd9Sstevel@tonic-gate 	(void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
5307c478bd9Sstevel@tonic-gate 
5317c478bd9Sstevel@tonic-gate 	exit(10);
5327c478bd9Sstevel@tonic-gate }
5337c478bd9Sstevel@tonic-gate 
5347c478bd9Sstevel@tonic-gate static void
5357c478bd9Sstevel@tonic-gate write_status(restarter_inst_t *inst, const char *mname, int stat)
5367c478bd9Sstevel@tonic-gate {
5377c478bd9Sstevel@tonic-gate 	int r;
5387c478bd9Sstevel@tonic-gate 
5397c478bd9Sstevel@tonic-gate again:
5407c478bd9Sstevel@tonic-gate 	if (inst->ri_mi_deleted)
5417c478bd9Sstevel@tonic-gate 		return;
5427c478bd9Sstevel@tonic-gate 
5437c478bd9Sstevel@tonic-gate 	r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
5447c478bd9Sstevel@tonic-gate 	switch (r) {
5457c478bd9Sstevel@tonic-gate 	case 0:
5467c478bd9Sstevel@tonic-gate 		break;
5477c478bd9Sstevel@tonic-gate 
5487c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
5497c478bd9Sstevel@tonic-gate 		libscf_reget_instance(inst);
5507c478bd9Sstevel@tonic-gate 		goto again;
5517c478bd9Sstevel@tonic-gate 
5527c478bd9Sstevel@tonic-gate 	case ECANCELED:
5537c478bd9Sstevel@tonic-gate 		inst->ri_mi_deleted = 1;
5547c478bd9Sstevel@tonic-gate 		break;
5557c478bd9Sstevel@tonic-gate 
5567c478bd9Sstevel@tonic-gate 	case EPERM:
5577c478bd9Sstevel@tonic-gate 	case EACCES:
5587c478bd9Sstevel@tonic-gate 	case EROFS:
5597c478bd9Sstevel@tonic-gate 		log_framework(LOG_INFO, "Could not write exit status "
5607c478bd9Sstevel@tonic-gate 		    "for %s method of %s: %s.\n", mname,
5617c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri, strerror(r));
5627c478bd9Sstevel@tonic-gate 		break;
5637c478bd9Sstevel@tonic-gate 
5647c478bd9Sstevel@tonic-gate 	case ENAMETOOLONG:
5657c478bd9Sstevel@tonic-gate 	default:
5667c478bd9Sstevel@tonic-gate 		bad_error("libscf_write_method_status", r);
5677c478bd9Sstevel@tonic-gate 	}
5687c478bd9Sstevel@tonic-gate }
5697c478bd9Sstevel@tonic-gate 
5707c478bd9Sstevel@tonic-gate /*
5717c478bd9Sstevel@tonic-gate  * int method_run()
5727c478bd9Sstevel@tonic-gate  *   Execute the type method of instp.  If it requires a fork(), wait for it
5737c478bd9Sstevel@tonic-gate  *   to return and return its exit code in *exit_code.  Otherwise set
5747c478bd9Sstevel@tonic-gate  *   *exit_code to 0 if the method succeeds & -1 if it fails.  If the
5757c478bd9Sstevel@tonic-gate  *   repository connection is broken, it is rebound, but inst may not be
5767c478bd9Sstevel@tonic-gate  *   reset.
5777c478bd9Sstevel@tonic-gate  *   Returns
5787c478bd9Sstevel@tonic-gate  *     0 - success
5797c478bd9Sstevel@tonic-gate  *     EINVAL - A correct method or method context couldn't be retrieved.
5807c478bd9Sstevel@tonic-gate  *     EIO - Contract kill failed.
5817c478bd9Sstevel@tonic-gate  *     EFAULT - Method couldn't be executed successfully.
5827c478bd9Sstevel@tonic-gate  *     ELOOP - Retry threshold exceeded.
5837c478bd9Sstevel@tonic-gate  *     ECANCELED - inst was deleted from the repository before method was run
5847c478bd9Sstevel@tonic-gate  *     ERANGE - Timeout retry threshold exceeded.
5857c478bd9Sstevel@tonic-gate  *     EAGAIN - Failed due to external cause, retry.
5867c478bd9Sstevel@tonic-gate  */
5877c478bd9Sstevel@tonic-gate int
5887c478bd9Sstevel@tonic-gate method_run(restarter_inst_t **instp, int type, int *exit_code)
5897c478bd9Sstevel@tonic-gate {
5907c478bd9Sstevel@tonic-gate 	char *method;
5917c478bd9Sstevel@tonic-gate 	int ret_status;
5927c478bd9Sstevel@tonic-gate 	pid_t pid;
5937c478bd9Sstevel@tonic-gate 	method_restart_t restart_on;
5947c478bd9Sstevel@tonic-gate 	uint_t cte_mask;
5957c478bd9Sstevel@tonic-gate 	uint8_t need_session;
5967c478bd9Sstevel@tonic-gate 	scf_handle_t *h;
5977c478bd9Sstevel@tonic-gate 	scf_snapshot_t *snap;
5987c478bd9Sstevel@tonic-gate 	const char *mname;
5997c478bd9Sstevel@tonic-gate 	const char *errstr;
6007c478bd9Sstevel@tonic-gate 	struct method_context *mcp;
6017c478bd9Sstevel@tonic-gate 	int result = 0, timeout_fired = 0;
6027c478bd9Sstevel@tonic-gate 	int sig, r;
6037c478bd9Sstevel@tonic-gate 	boolean_t transient;
6047c478bd9Sstevel@tonic-gate 	uint64_t timeout;
6057c478bd9Sstevel@tonic-gate 	uint8_t timeout_retry;
6067c478bd9Sstevel@tonic-gate 	ctid_t ctid;
6077c478bd9Sstevel@tonic-gate 	int ctfd = -1;
6087c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst = *instp;
6097c478bd9Sstevel@tonic-gate 	int id = inst->ri_id;
610ca948064Ssl108498 	int forkerr;
6117c478bd9Sstevel@tonic-gate 
6127c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
6137c478bd9Sstevel@tonic-gate 	assert(instance_in_transition(inst));
6147c478bd9Sstevel@tonic-gate 
6157c478bd9Sstevel@tonic-gate 	if (inst->ri_mi_deleted)
6167c478bd9Sstevel@tonic-gate 		return (ECANCELED);
6177c478bd9Sstevel@tonic-gate 
6187c478bd9Sstevel@tonic-gate 	*exit_code = 0;
6197c478bd9Sstevel@tonic-gate 
6207c478bd9Sstevel@tonic-gate 	assert(0 <= type && type <= 2);
6217c478bd9Sstevel@tonic-gate 	mname = method_names[type];
6227c478bd9Sstevel@tonic-gate 
6237c478bd9Sstevel@tonic-gate 	if (type == METHOD_START)
6247c478bd9Sstevel@tonic-gate 		inst->ri_pre_online_hook();
6257c478bd9Sstevel@tonic-gate 
6267c478bd9Sstevel@tonic-gate 	h = scf_instance_handle(inst->ri_m_inst);
6277c478bd9Sstevel@tonic-gate 
6287c478bd9Sstevel@tonic-gate 	snap = scf_snapshot_create(h);
6297c478bd9Sstevel@tonic-gate 	if (snap == NULL ||
6307c478bd9Sstevel@tonic-gate 	    scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
6317c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
6327c478bd9Sstevel@tonic-gate 		    "Could not get running snapshot for %s.  "
6337c478bd9Sstevel@tonic-gate 		    "Using editing version to run method %s.\n",
6347c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri, mname);
6357c478bd9Sstevel@tonic-gate 		scf_snapshot_destroy(snap);
6367c478bd9Sstevel@tonic-gate 		snap = NULL;
6377c478bd9Sstevel@tonic-gate 	}
6387c478bd9Sstevel@tonic-gate 
6397c478bd9Sstevel@tonic-gate 	/*
6407c478bd9Sstevel@tonic-gate 	 * After this point, we may be logging to the instance log.
6417c478bd9Sstevel@tonic-gate 	 * Make sure we've noted where that log is as a property of
6427c478bd9Sstevel@tonic-gate 	 * the instance.
6437c478bd9Sstevel@tonic-gate 	 */
6447c478bd9Sstevel@tonic-gate 	r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
6457c478bd9Sstevel@tonic-gate 	    inst->ri_logstem);
6467c478bd9Sstevel@tonic-gate 	if (r != 0) {
6477c478bd9Sstevel@tonic-gate 		log_framework(LOG_WARNING,
6487c478bd9Sstevel@tonic-gate 		    "%s: couldn't note log location: %s\n",
6497c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri, strerror(r));
6507c478bd9Sstevel@tonic-gate 	}
6517c478bd9Sstevel@tonic-gate 
6527c478bd9Sstevel@tonic-gate 	if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
6537c478bd9Sstevel@tonic-gate 	    &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
6547c478bd9Sstevel@tonic-gate 		if (errno == LIBSCF_PGROUP_ABSENT)  {
6557c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG,
6567c478bd9Sstevel@tonic-gate 			    "%s: instance has no method property group '%s'.\n",
6577c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri, mname);
6587c478bd9Sstevel@tonic-gate 			if (type == METHOD_REFRESH)
6597c478bd9Sstevel@tonic-gate 				log_instance(inst, B_TRUE, "No '%s' method "
6607c478bd9Sstevel@tonic-gate 				    "defined.  Treating as :true.", mname);
6617c478bd9Sstevel@tonic-gate 			else
6627c478bd9Sstevel@tonic-gate 				log_instance(inst, B_TRUE, "Method property "
6637c478bd9Sstevel@tonic-gate 				    "group '%s' is not present.", mname);
6647c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(snap);
6657c478bd9Sstevel@tonic-gate 			return (0);
6667c478bd9Sstevel@tonic-gate 		} else if (errno == LIBSCF_PROPERTY_ABSENT)  {
6677c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG,
6687c478bd9Sstevel@tonic-gate 			    "%s: instance has no '%s/exec' method property.\n",
6697c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri, mname);
6707c478bd9Sstevel@tonic-gate 			log_instance(inst, B_TRUE, "Method property '%s/exec "
6717c478bd9Sstevel@tonic-gate 			    "is not present.", mname);
6727c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(snap);
6737c478bd9Sstevel@tonic-gate 			return (0);
6747c478bd9Sstevel@tonic-gate 		} else {
6757c478bd9Sstevel@tonic-gate 			log_error(LOG_WARNING,
6767c478bd9Sstevel@tonic-gate 			    "%s: instance libscf_get_method failed\n",
6777c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri);
6787c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(snap);
6797c478bd9Sstevel@tonic-gate 			return (EINVAL);
6807c478bd9Sstevel@tonic-gate 		}
6817c478bd9Sstevel@tonic-gate 	}
6827c478bd9Sstevel@tonic-gate 
6837c478bd9Sstevel@tonic-gate 	/* open service contract if stopping a non-transient service */
6847c478bd9Sstevel@tonic-gate 	if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
6857c478bd9Sstevel@tonic-gate 		if (inst->ri_i.i_primary_ctid == 0) {
6867c478bd9Sstevel@tonic-gate 			/* service is not running, nothing to stop */
6877c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG, "%s: instance has no primary "
6887c478bd9Sstevel@tonic-gate 			    "contract, no service to stop.\n",
6897c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri);
6907c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(snap);
6917c478bd9Sstevel@tonic-gate 			return (0);
6927c478bd9Sstevel@tonic-gate 		}
6937c478bd9Sstevel@tonic-gate 		if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
6947c478bd9Sstevel@tonic-gate 		    "events", O_RDONLY)) < 0) {
6957c478bd9Sstevel@tonic-gate 			result = EFAULT;
6967c478bd9Sstevel@tonic-gate 			log_instance(inst, B_TRUE, "Could not open service "
6977c478bd9Sstevel@tonic-gate 			    "contract %ld.  Stop method not run.\n",
6987c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_primary_ctid);
6997c478bd9Sstevel@tonic-gate 			goto out;
7007c478bd9Sstevel@tonic-gate 		}
7017c478bd9Sstevel@tonic-gate 	}
7027c478bd9Sstevel@tonic-gate 
7037c478bd9Sstevel@tonic-gate 	if (restarter_is_null_method(method)) {
7047c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG, "%s: null method succeeds\n",
7057c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri);
7067c478bd9Sstevel@tonic-gate 
7077c478bd9Sstevel@tonic-gate 		log_instance(inst, B_TRUE, "Executing %s method (null)", mname);
7087c478bd9Sstevel@tonic-gate 
7097c478bd9Sstevel@tonic-gate 		if (type == METHOD_START)
7107c478bd9Sstevel@tonic-gate 			write_status(inst, mname, 0);
7117c478bd9Sstevel@tonic-gate 		goto out;
7127c478bd9Sstevel@tonic-gate 	}
7137c478bd9Sstevel@tonic-gate 
7147c478bd9Sstevel@tonic-gate 	sig = restarter_is_kill_method(method);
7157c478bd9Sstevel@tonic-gate 	if (sig >= 0) {
7167c478bd9Sstevel@tonic-gate 
7177c478bd9Sstevel@tonic-gate 		if (inst->ri_i.i_primary_ctid == 0) {
7187c478bd9Sstevel@tonic-gate 			log_error(LOG_ERR, "%s: :kill with no contract\n",
7197c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri);
7207c478bd9Sstevel@tonic-gate 			result = EINVAL;
7217c478bd9Sstevel@tonic-gate 			goto out;
7227c478bd9Sstevel@tonic-gate 		}
7237c478bd9Sstevel@tonic-gate 
7247c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
7257c478bd9Sstevel@tonic-gate 		    "%s: :killing contract with signal %d\n",
7267c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri, sig);
7277c478bd9Sstevel@tonic-gate 
7287c478bd9Sstevel@tonic-gate 		log_instance(inst, B_TRUE, "Executing %s method (:kill)",
7297c478bd9Sstevel@tonic-gate 		    mname);
7307c478bd9Sstevel@tonic-gate 
7317c478bd9Sstevel@tonic-gate 		if (contract_kill(inst->ri_i.i_primary_ctid, sig,
7327c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri) != 0) {
7337c478bd9Sstevel@tonic-gate 			result = EIO;
7347c478bd9Sstevel@tonic-gate 			goto out;
7357c478bd9Sstevel@tonic-gate 		} else
7367c478bd9Sstevel@tonic-gate 			goto assured_kill;
7377c478bd9Sstevel@tonic-gate 	}
7387c478bd9Sstevel@tonic-gate 
7397c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
7407c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_fmri, method);
7417c478bd9Sstevel@tonic-gate 
7427c478bd9Sstevel@tonic-gate 	errstr = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
7437c478bd9Sstevel@tonic-gate 	    inst->ri_m_inst, snap, mname, method, &mcp);
7447c478bd9Sstevel@tonic-gate 
7457c478bd9Sstevel@tonic-gate 	if (errstr != NULL) {
7467c478bd9Sstevel@tonic-gate 		log_error(LOG_WARNING, "%s: %s\n", inst->ri_i.i_fmri, errstr);
7477c478bd9Sstevel@tonic-gate 		result = EINVAL;
7487c478bd9Sstevel@tonic-gate 		goto out;
7497c478bd9Sstevel@tonic-gate 	}
7507c478bd9Sstevel@tonic-gate 
7517c478bd9Sstevel@tonic-gate 	r = method_ready_contract(inst, type, restart_on, cte_mask);
7527c478bd9Sstevel@tonic-gate 	if (r != 0) {
7537c478bd9Sstevel@tonic-gate 		assert(r == ECANCELED);
7547c478bd9Sstevel@tonic-gate 		assert(inst->ri_mi_deleted);
7557c478bd9Sstevel@tonic-gate 		restarter_free_method_context(mcp);
7567c478bd9Sstevel@tonic-gate 		result = ECANCELED;
7577c478bd9Sstevel@tonic-gate 		goto out;
7587c478bd9Sstevel@tonic-gate 	}
7597c478bd9Sstevel@tonic-gate 
7607c478bd9Sstevel@tonic-gate 	/*
7617c478bd9Sstevel@tonic-gate 	 * Validate safety of method contexts, to save children work.
7627c478bd9Sstevel@tonic-gate 	 */
7637c478bd9Sstevel@tonic-gate 	if (!restarter_rm_libs_loadable())
7647c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG, "%s: method contexts limited "
7657c478bd9Sstevel@tonic-gate 		    "to root-accessible libraries\n", inst->ri_i.i_fmri);
7667c478bd9Sstevel@tonic-gate 
7677c478bd9Sstevel@tonic-gate 	/*
7687c478bd9Sstevel@tonic-gate 	 * If the service is restarting too quickly, send it to
7697c478bd9Sstevel@tonic-gate 	 * maintenance.
7707c478bd9Sstevel@tonic-gate 	 */
7717c478bd9Sstevel@tonic-gate 	if (type == METHOD_START) {
7727c478bd9Sstevel@tonic-gate 		method_record_start(inst);
7737c478bd9Sstevel@tonic-gate 		if (method_rate_critical(inst)) {
7747c478bd9Sstevel@tonic-gate 			log_instance(inst, B_TRUE, "Restarting too quickly, "
7757c478bd9Sstevel@tonic-gate 			    "changing state to maintenance");
7767c478bd9Sstevel@tonic-gate 			result = ELOOP;
777d3186a0eSjeanm 			restarter_free_method_context(mcp);
7787c478bd9Sstevel@tonic-gate 			goto out;
7797c478bd9Sstevel@tonic-gate 		}
7807c478bd9Sstevel@tonic-gate 	}
7817c478bd9Sstevel@tonic-gate 
782*dfe57350Sjeanm 	atomic_add_16(&storing_contract, 1);
783ca948064Ssl108498 	pid = startd_fork1(&forkerr);
7847c478bd9Sstevel@tonic-gate 	if (pid == 0)
7857c478bd9Sstevel@tonic-gate 		exec_method(inst, type, method, mcp, need_session);
7867c478bd9Sstevel@tonic-gate 
7877c478bd9Sstevel@tonic-gate 	if (pid == -1) {
788*dfe57350Sjeanm 		atomic_add_16(&storing_contract, -1);
789ca948064Ssl108498 		if (forkerr == EAGAIN)
790ca948064Ssl108498 			result = EAGAIN;
791ca948064Ssl108498 		else
7927c478bd9Sstevel@tonic-gate 			result = EFAULT;
793ca948064Ssl108498 
794ca948064Ssl108498 		log_error(LOG_WARNING,
795ca948064Ssl108498 		    "%s: Couldn't fork to execute method %s: %s\n",
796ca948064Ssl108498 		    inst->ri_i.i_fmri, method, strerror(forkerr));
797ca948064Ssl108498 
798*dfe57350Sjeanm 		restarter_free_method_context(mcp);
7997c478bd9Sstevel@tonic-gate 		goto out;
8007c478bd9Sstevel@tonic-gate 	}
8017c478bd9Sstevel@tonic-gate 
8027c478bd9Sstevel@tonic-gate 
8037c478bd9Sstevel@tonic-gate 	/*
8047c478bd9Sstevel@tonic-gate 	 * Get the contract id, decide whether it is primary or transient, and
8057c478bd9Sstevel@tonic-gate 	 * stash it in inst & the repository.
8067c478bd9Sstevel@tonic-gate 	 */
8077c478bd9Sstevel@tonic-gate 	method_store_contract(inst, type, &ctid);
808*dfe57350Sjeanm 	atomic_add_16(&storing_contract, -1);
809*dfe57350Sjeanm 
810*dfe57350Sjeanm 	restarter_free_method_context(mcp);
8117c478bd9Sstevel@tonic-gate 
8127c478bd9Sstevel@tonic-gate 	/*
8137c478bd9Sstevel@tonic-gate 	 * Similarly for the start method PID.
8147c478bd9Sstevel@tonic-gate 	 */
8157c478bd9Sstevel@tonic-gate 	if (type == METHOD_START && !inst->ri_mi_deleted)
8167c478bd9Sstevel@tonic-gate 		(void) libscf_write_start_pid(inst->ri_m_inst, pid);
8177c478bd9Sstevel@tonic-gate 
8187c478bd9Sstevel@tonic-gate 	if (instance_is_wait_style(inst) && type == METHOD_START) {
8197c478bd9Sstevel@tonic-gate 		/* Wait style instances don't get timeouts on start methods. */
8207c478bd9Sstevel@tonic-gate 		if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
8217c478bd9Sstevel@tonic-gate 			log_error(LOG_WARNING,
8227c478bd9Sstevel@tonic-gate 			    "%s: couldn't register %ld for wait\n",
8237c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri, pid);
8247c478bd9Sstevel@tonic-gate 			result = EFAULT;
8257c478bd9Sstevel@tonic-gate 			goto contract_out;
8267c478bd9Sstevel@tonic-gate 		}
8277c478bd9Sstevel@tonic-gate 		write_status(inst, mname, 0);
8287c478bd9Sstevel@tonic-gate 
8297c478bd9Sstevel@tonic-gate 	} else {
8307c478bd9Sstevel@tonic-gate 		int r, err;
8317c478bd9Sstevel@tonic-gate 		time_t start_time;
8327c478bd9Sstevel@tonic-gate 		time_t end_time;
8337c478bd9Sstevel@tonic-gate 
8347c478bd9Sstevel@tonic-gate 		/*
8357c478bd9Sstevel@tonic-gate 		 * Because on upgrade/live-upgrade we may have no chance
8367c478bd9Sstevel@tonic-gate 		 * to override faulty timeout values on the way to
8377c478bd9Sstevel@tonic-gate 		 * manifest import, all services on the path to manifest
8387c478bd9Sstevel@tonic-gate 		 * import are treated the same as INFINITE timeout services.
8397c478bd9Sstevel@tonic-gate 		 */
8407c478bd9Sstevel@tonic-gate 
8417c478bd9Sstevel@tonic-gate 		start_time = time(NULL);
8427c478bd9Sstevel@tonic-gate 		if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
8437c478bd9Sstevel@tonic-gate 			timeout_insert(inst, ctid, timeout);
8447c478bd9Sstevel@tonic-gate 		else
8457c478bd9Sstevel@tonic-gate 			timeout = METHOD_TIMEOUT_INFINITE;
8467c478bd9Sstevel@tonic-gate 
8477c478bd9Sstevel@tonic-gate 		/* Unlock the instance while waiting for the method. */
8487c478bd9Sstevel@tonic-gate 		MUTEX_UNLOCK(&inst->ri_lock);
8497c478bd9Sstevel@tonic-gate 
8507c478bd9Sstevel@tonic-gate 		do
8517c478bd9Sstevel@tonic-gate 			r = waitpid(pid, &ret_status, NULL);
8527c478bd9Sstevel@tonic-gate 		while (r == -1 && errno == EINTR);
8537c478bd9Sstevel@tonic-gate 		if (r == -1)
8547c478bd9Sstevel@tonic-gate 			err = errno;
8557c478bd9Sstevel@tonic-gate 
8567c478bd9Sstevel@tonic-gate 		/* Re-grab the lock. */
8577c478bd9Sstevel@tonic-gate 		inst = inst_lookup_by_id(id);
8587c478bd9Sstevel@tonic-gate 
8597c478bd9Sstevel@tonic-gate 		/*
8607c478bd9Sstevel@tonic-gate 		 * inst can't be removed, as the removal thread waits
8617c478bd9Sstevel@tonic-gate 		 * for completion of this one.
8627c478bd9Sstevel@tonic-gate 		 */
8637c478bd9Sstevel@tonic-gate 		assert(inst != NULL);
8647c478bd9Sstevel@tonic-gate 		*instp = inst;
8657c478bd9Sstevel@tonic-gate 
8667c478bd9Sstevel@tonic-gate 		if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
8677c478bd9Sstevel@tonic-gate 			timeout_fired = 1;
8687c478bd9Sstevel@tonic-gate 
8697c478bd9Sstevel@tonic-gate 		timeout_remove(inst, ctid);
8707c478bd9Sstevel@tonic-gate 
8717c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
8727c478bd9Sstevel@tonic-gate 		    "%s method for %s exited with status %d.\n", mname,
8737c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
8747c478bd9Sstevel@tonic-gate 
8757c478bd9Sstevel@tonic-gate 		if (r == -1) {
8767c478bd9Sstevel@tonic-gate 			log_error(LOG_WARNING,
8777c478bd9Sstevel@tonic-gate 			    "Couldn't waitpid() for %s method of %s (%s).\n",
8787c478bd9Sstevel@tonic-gate 			    mname, inst->ri_i.i_fmri, strerror(err));
8797c478bd9Sstevel@tonic-gate 			result = EFAULT;
8807c478bd9Sstevel@tonic-gate 			goto contract_out;
8817c478bd9Sstevel@tonic-gate 		}
8827c478bd9Sstevel@tonic-gate 
8837c478bd9Sstevel@tonic-gate 		if (type == METHOD_START)
8847c478bd9Sstevel@tonic-gate 			write_status(inst, mname, ret_status);
8857c478bd9Sstevel@tonic-gate 
8867c478bd9Sstevel@tonic-gate 		/* return ERANGE if this service doesn't retry on timeout */
8877c478bd9Sstevel@tonic-gate 		if (timeout_fired == 1 && timeout_retry == 0) {
8887c478bd9Sstevel@tonic-gate 			result = ERANGE;
8897c478bd9Sstevel@tonic-gate 			goto contract_out;
8907c478bd9Sstevel@tonic-gate 		}
8917c478bd9Sstevel@tonic-gate 
8927c478bd9Sstevel@tonic-gate 		if (!WIFEXITED(ret_status)) {
8937c478bd9Sstevel@tonic-gate 			/*
8947c478bd9Sstevel@tonic-gate 			 * If method didn't exit itself (it was killed by an
8957c478bd9Sstevel@tonic-gate 			 * external entity, etc.), consider the entire
8967c478bd9Sstevel@tonic-gate 			 * method_run as failed.
8977c478bd9Sstevel@tonic-gate 			 */
8987c478bd9Sstevel@tonic-gate 			if (WIFSIGNALED(ret_status)) {
8997c478bd9Sstevel@tonic-gate 				char buf[SIG2STR_MAX];
9007c478bd9Sstevel@tonic-gate 				(void) sig2str(WTERMSIG(ret_status), buf);
9017c478bd9Sstevel@tonic-gate 
9027c478bd9Sstevel@tonic-gate 				log_error(LOG_WARNING, "%s: Method \"%s\" "
9037c478bd9Sstevel@tonic-gate 				    "failed due to signal %s.\n",
9047c478bd9Sstevel@tonic-gate 				    inst->ri_i.i_fmri, method, buf);
9057c478bd9Sstevel@tonic-gate 				log_instance(inst, B_TRUE, "Method \"%s\" "
9067c478bd9Sstevel@tonic-gate 				    "failed due to signal %s", mname, buf);
9077c478bd9Sstevel@tonic-gate 			} else {
9087c478bd9Sstevel@tonic-gate 				log_error(LOG_WARNING, "%s: Method \"%s\" "
9097c478bd9Sstevel@tonic-gate 				    "failed with exit status %d.\n",
9107c478bd9Sstevel@tonic-gate 				    inst->ri_i.i_fmri, method,
9117c478bd9Sstevel@tonic-gate 				    WEXITSTATUS(ret_status));
9127c478bd9Sstevel@tonic-gate 				log_instance(inst, B_TRUE, "Method \"%s\" "
9137c478bd9Sstevel@tonic-gate 				    "failed with exit status %d", mname,
9147c478bd9Sstevel@tonic-gate 				    WEXITSTATUS(ret_status));
9157c478bd9Sstevel@tonic-gate 			}
9167c478bd9Sstevel@tonic-gate 			result = EAGAIN;
9177c478bd9Sstevel@tonic-gate 			goto contract_out;
9187c478bd9Sstevel@tonic-gate 		}
9197c478bd9Sstevel@tonic-gate 
9207c478bd9Sstevel@tonic-gate 		*exit_code = WEXITSTATUS(ret_status);
9217c478bd9Sstevel@tonic-gate 		if (*exit_code != 0) {
9227c478bd9Sstevel@tonic-gate 			log_error(LOG_WARNING,
9237c478bd9Sstevel@tonic-gate 			    "%s: Method \"%s\" failed with exit status %d.\n",
9247c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
9257c478bd9Sstevel@tonic-gate 		}
9267c478bd9Sstevel@tonic-gate 
9277c478bd9Sstevel@tonic-gate 		log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
9287c478bd9Sstevel@tonic-gate 		    "%d", mname, *exit_code);
9297c478bd9Sstevel@tonic-gate 
9307c478bd9Sstevel@tonic-gate 		if (*exit_code != 0)
9317c478bd9Sstevel@tonic-gate 			goto contract_out;
9327c478bd9Sstevel@tonic-gate 
9337c478bd9Sstevel@tonic-gate 		end_time = time(NULL);
9347c478bd9Sstevel@tonic-gate 
9357c478bd9Sstevel@tonic-gate 		/* Give service contract remaining seconds to empty */
9367c478bd9Sstevel@tonic-gate 		if (timeout != METHOD_TIMEOUT_INFINITE)
9377c478bd9Sstevel@tonic-gate 			timeout -= (end_time - start_time);
9387c478bd9Sstevel@tonic-gate 	}
9397c478bd9Sstevel@tonic-gate 
9407c478bd9Sstevel@tonic-gate assured_kill:
9417c478bd9Sstevel@tonic-gate 	/*
9427c478bd9Sstevel@tonic-gate 	 * For stop methods, assure that the service contract has emptied
9437c478bd9Sstevel@tonic-gate 	 * before returning.
9447c478bd9Sstevel@tonic-gate 	 */
9457c478bd9Sstevel@tonic-gate 	if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
9467c478bd9Sstevel@tonic-gate 	    !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
9477c478bd9Sstevel@tonic-gate 
9487c478bd9Sstevel@tonic-gate 		if (timeout != METHOD_TIMEOUT_INFINITE)
9497c478bd9Sstevel@tonic-gate 			timeout_insert(inst, inst->ri_i.i_primary_ctid,
9507c478bd9Sstevel@tonic-gate 			    timeout);
9517c478bd9Sstevel@tonic-gate 
9527c478bd9Sstevel@tonic-gate 		for (;;) {
953*dfe57350Sjeanm 			(void) poll(NULL, 0, 100);
954*dfe57350Sjeanm 			if (contract_is_empty(inst->ri_i.i_primary_ctid))
9557c478bd9Sstevel@tonic-gate 				break;
9567c478bd9Sstevel@tonic-gate 		}
9577c478bd9Sstevel@tonic-gate 		if (r) {
9587c478bd9Sstevel@tonic-gate 			result = EFAULT;
9597c478bd9Sstevel@tonic-gate 			log_instance(inst, B_TRUE, "Error reading service "
9607c478bd9Sstevel@tonic-gate 			    "contract %ld.\n", inst->ri_i.i_primary_ctid);
9617c478bd9Sstevel@tonic-gate 		}
9627c478bd9Sstevel@tonic-gate 
9637c478bd9Sstevel@tonic-gate 		if (timeout != METHOD_TIMEOUT_INFINITE)
9647c478bd9Sstevel@tonic-gate 			if (inst->ri_timeout->te_fired)
9657c478bd9Sstevel@tonic-gate 				result = EFAULT;
9667c478bd9Sstevel@tonic-gate 
9677c478bd9Sstevel@tonic-gate 		timeout_remove(inst, inst->ri_i.i_primary_ctid);
9687c478bd9Sstevel@tonic-gate 	}
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate contract_out:
9717c478bd9Sstevel@tonic-gate 	/* Abandon contracts for transient methods & methods that fail. */
9727c478bd9Sstevel@tonic-gate 	transient = method_is_transient(inst, type);
9737c478bd9Sstevel@tonic-gate 	if ((transient || *exit_code != 0 || result != 0) &&
9747c478bd9Sstevel@tonic-gate 	    (restarter_is_kill_method(method) < 0))
9757c478bd9Sstevel@tonic-gate 		method_remove_contract(inst, !transient, B_TRUE);
9767c478bd9Sstevel@tonic-gate 
9777c478bd9Sstevel@tonic-gate out:
9787c478bd9Sstevel@tonic-gate 	if (ctfd >= 0)
9797c478bd9Sstevel@tonic-gate 		(void) close(ctfd);
9807c478bd9Sstevel@tonic-gate 	scf_snapshot_destroy(snap);
9817c478bd9Sstevel@tonic-gate 	free(method);
9827c478bd9Sstevel@tonic-gate 	return (result);
9837c478bd9Sstevel@tonic-gate }
9847c478bd9Sstevel@tonic-gate 
9857c478bd9Sstevel@tonic-gate /*
9867c478bd9Sstevel@tonic-gate  * The method thread executes a service method to effect a state transition.
9877c478bd9Sstevel@tonic-gate  * The next_state of info->sf_id should be non-_NONE on entrance, and it will
9887c478bd9Sstevel@tonic-gate  * be _NONE on exit (state will either be what next_state was (on success), or
9897c478bd9Sstevel@tonic-gate  * it will be _MAINT (on error)).
9907c478bd9Sstevel@tonic-gate  *
9917c478bd9Sstevel@tonic-gate  * There are six classes of methods to consider: start & other (stop, refresh)
9927c478bd9Sstevel@tonic-gate  * for each of "normal" services, wait services, and transient services.  For
9937c478bd9Sstevel@tonic-gate  * each, the method must be fetched from the repository & executed.  fork()ed
9947c478bd9Sstevel@tonic-gate  * methods must be waited on, except for the start method of wait services
9957c478bd9Sstevel@tonic-gate  * (which must be registered with the wait subsystem via wait_register()).  If
9967c478bd9Sstevel@tonic-gate  * the method succeeded (returned 0), then for start methods its contract
9977c478bd9Sstevel@tonic-gate  * should be recorded as the primary contract for the service.  For other
9987c478bd9Sstevel@tonic-gate  * methods, it should be abandoned.  If the method fails, then depending on
9997c478bd9Sstevel@tonic-gate  * the failure, either the method should be reexecuted or the service should
10007c478bd9Sstevel@tonic-gate  * be put into maintenance.  Either way the contract should be abandoned.
10017c478bd9Sstevel@tonic-gate  */
10027c478bd9Sstevel@tonic-gate void *
10037c478bd9Sstevel@tonic-gate method_thread(void *arg)
10047c478bd9Sstevel@tonic-gate {
10057c478bd9Sstevel@tonic-gate 	fork_info_t *info = arg;
10067c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst;
10077c478bd9Sstevel@tonic-gate 	scf_handle_t	*local_handle;
10087c478bd9Sstevel@tonic-gate 	scf_instance_t	*s_inst = NULL;
10097c478bd9Sstevel@tonic-gate 	int r, exit_code;
10107c478bd9Sstevel@tonic-gate 	boolean_t retryable;
10117c478bd9Sstevel@tonic-gate 	const char *aux;
10127c478bd9Sstevel@tonic-gate 
10137c478bd9Sstevel@tonic-gate 	assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
10147c478bd9Sstevel@tonic-gate 
10157c478bd9Sstevel@tonic-gate 	/* Get (and lock) the restarter_inst_t. */
10167c478bd9Sstevel@tonic-gate 	inst = inst_lookup_by_id(info->sf_id);
10177c478bd9Sstevel@tonic-gate 
10187c478bd9Sstevel@tonic-gate 	assert(inst->ri_method_thread != 0);
10197c478bd9Sstevel@tonic-gate 	assert(instance_in_transition(inst) == 1);
10207c478bd9Sstevel@tonic-gate 
10217c478bd9Sstevel@tonic-gate 	/*
10227c478bd9Sstevel@tonic-gate 	 * We cannot leave this function with inst in transition, because
10237c478bd9Sstevel@tonic-gate 	 * protocol.c withholds messages for inst otherwise.
10247c478bd9Sstevel@tonic-gate 	 */
10257c478bd9Sstevel@tonic-gate 
10267c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
10277c478bd9Sstevel@tonic-gate 	    method_names[info->sf_method_type], inst->ri_i.i_fmri);
10287c478bd9Sstevel@tonic-gate 
10297c478bd9Sstevel@tonic-gate 	local_handle = libscf_handle_create_bound_loop();
10307c478bd9Sstevel@tonic-gate 
10317c478bd9Sstevel@tonic-gate rebind_retry:
10327c478bd9Sstevel@tonic-gate 	/* get scf_instance_t */
10337c478bd9Sstevel@tonic-gate 	switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
10347c478bd9Sstevel@tonic-gate 	    &s_inst)) {
10357c478bd9Sstevel@tonic-gate 	case 0:
10367c478bd9Sstevel@tonic-gate 		break;
10377c478bd9Sstevel@tonic-gate 
10387c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
10397c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(local_handle);
10407c478bd9Sstevel@tonic-gate 		goto rebind_retry;
10417c478bd9Sstevel@tonic-gate 
10427c478bd9Sstevel@tonic-gate 	case ENOENT:
10437c478bd9Sstevel@tonic-gate 		/*
10447c478bd9Sstevel@tonic-gate 		 * It's not there, but we need to call this so protocol.c
10457c478bd9Sstevel@tonic-gate 		 * doesn't think it's in transition anymore.
10467c478bd9Sstevel@tonic-gate 		 */
10477c478bd9Sstevel@tonic-gate 		(void) restarter_instance_update_states(local_handle, inst,
10487c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
10497c478bd9Sstevel@tonic-gate 		    NULL);
10507c478bd9Sstevel@tonic-gate 		goto out;
10517c478bd9Sstevel@tonic-gate 
10527c478bd9Sstevel@tonic-gate 	case EINVAL:
10537c478bd9Sstevel@tonic-gate 	case ENOTSUP:
10547c478bd9Sstevel@tonic-gate 	default:
10557c478bd9Sstevel@tonic-gate 		bad_error("libscf_fmri_get_instance", r);
10567c478bd9Sstevel@tonic-gate 	}
10577c478bd9Sstevel@tonic-gate 
10587c478bd9Sstevel@tonic-gate 	inst->ri_m_inst = s_inst;
10597c478bd9Sstevel@tonic-gate 	inst->ri_mi_deleted = B_FALSE;
10607c478bd9Sstevel@tonic-gate 
10617c478bd9Sstevel@tonic-gate retry:
10627c478bd9Sstevel@tonic-gate 	if (info->sf_method_type == METHOD_START)
10637c478bd9Sstevel@tonic-gate 		log_transition(inst, START_REQUESTED);
10647c478bd9Sstevel@tonic-gate 
10657c478bd9Sstevel@tonic-gate 	r = method_run(&inst, info->sf_method_type, &exit_code);
10667c478bd9Sstevel@tonic-gate 
10677c478bd9Sstevel@tonic-gate 	if (r == 0 && exit_code == 0) {
10687c478bd9Sstevel@tonic-gate 		/* Success! */
10697c478bd9Sstevel@tonic-gate 		assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
10707c478bd9Sstevel@tonic-gate 
10717c478bd9Sstevel@tonic-gate 		/*
10727c478bd9Sstevel@tonic-gate 		 * When a stop method succeeds, remove the primary contract of
10737c478bd9Sstevel@tonic-gate 		 * the service, unless we're going to offline, in which case
10747c478bd9Sstevel@tonic-gate 		 * retain the contract so we can transfer inherited contracts to
10757c478bd9Sstevel@tonic-gate 		 * the replacement service.
10767c478bd9Sstevel@tonic-gate 		 */
10777c478bd9Sstevel@tonic-gate 
10787c478bd9Sstevel@tonic-gate 		if (info->sf_method_type == METHOD_STOP &&
10797c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_primary_ctid != 0) {
10807c478bd9Sstevel@tonic-gate 			if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
10817c478bd9Sstevel@tonic-gate 				inst->ri_i.i_primary_ctid_stopped = 1;
10827c478bd9Sstevel@tonic-gate 			else
10837c478bd9Sstevel@tonic-gate 				method_remove_contract(inst, B_TRUE, B_TRUE);
10847c478bd9Sstevel@tonic-gate 		}
10857c478bd9Sstevel@tonic-gate 		/*
10867c478bd9Sstevel@tonic-gate 		 * We don't care whether the handle was rebound because this is
10877c478bd9Sstevel@tonic-gate 		 * the last thing we do with it.
10887c478bd9Sstevel@tonic-gate 		 */
10897c478bd9Sstevel@tonic-gate 		(void) restarter_instance_update_states(local_handle, inst,
10907c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
10917c478bd9Sstevel@tonic-gate 		    info->sf_event_type, NULL);
10927c478bd9Sstevel@tonic-gate 
10937c478bd9Sstevel@tonic-gate 		(void) update_fault_count(inst, FAULT_COUNT_RESET);
10947c478bd9Sstevel@tonic-gate 
10957c478bd9Sstevel@tonic-gate 		goto out;
10967c478bd9Sstevel@tonic-gate 	}
10977c478bd9Sstevel@tonic-gate 
10987c478bd9Sstevel@tonic-gate 	/* Failure.  Retry or go to maintenance. */
10997c478bd9Sstevel@tonic-gate 
11007c478bd9Sstevel@tonic-gate 	if (r != 0 && r != EAGAIN) {
11017c478bd9Sstevel@tonic-gate 		retryable = B_FALSE;
11027c478bd9Sstevel@tonic-gate 	} else {
11037c478bd9Sstevel@tonic-gate 		switch (exit_code) {
11047c478bd9Sstevel@tonic-gate 		case SMF_EXIT_ERR_CONFIG:
11057c478bd9Sstevel@tonic-gate 		case SMF_EXIT_ERR_NOSMF:
11067c478bd9Sstevel@tonic-gate 		case SMF_EXIT_ERR_PERM:
11077c478bd9Sstevel@tonic-gate 		case SMF_EXIT_ERR_FATAL:
11087c478bd9Sstevel@tonic-gate 			retryable = B_FALSE;
11097c478bd9Sstevel@tonic-gate 			break;
11107c478bd9Sstevel@tonic-gate 
11117c478bd9Sstevel@tonic-gate 		default:
11127c478bd9Sstevel@tonic-gate 			retryable = B_TRUE;
11137c478bd9Sstevel@tonic-gate 		}
11147c478bd9Sstevel@tonic-gate 	}
11157c478bd9Sstevel@tonic-gate 
11167c478bd9Sstevel@tonic-gate 	if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
11177c478bd9Sstevel@tonic-gate 		goto retry;
11187c478bd9Sstevel@tonic-gate 
11197c478bd9Sstevel@tonic-gate 	/* maintenance */
11207c478bd9Sstevel@tonic-gate 	if (r == ELOOP)
11217c478bd9Sstevel@tonic-gate 		log_transition(inst, START_FAILED_REPEATEDLY);
11227c478bd9Sstevel@tonic-gate 	else if (r == ERANGE)
11237c478bd9Sstevel@tonic-gate 		log_transition(inst, START_FAILED_TIMEOUT_FATAL);
11247c478bd9Sstevel@tonic-gate 	else if (exit_code == SMF_EXIT_ERR_CONFIG)
11257c478bd9Sstevel@tonic-gate 		log_transition(inst, START_FAILED_CONFIGURATION);
11267c478bd9Sstevel@tonic-gate 	else if (exit_code == SMF_EXIT_ERR_FATAL)
11277c478bd9Sstevel@tonic-gate 		log_transition(inst, START_FAILED_FATAL);
11287c478bd9Sstevel@tonic-gate 	else
11297c478bd9Sstevel@tonic-gate 		log_transition(inst, START_FAILED_OTHER);
11307c478bd9Sstevel@tonic-gate 
11317c478bd9Sstevel@tonic-gate 	if (r == ELOOP)
11327c478bd9Sstevel@tonic-gate 		aux = "restarting_too_quickly";
11337c478bd9Sstevel@tonic-gate 	else if (retryable)
11347c478bd9Sstevel@tonic-gate 		aux = "fault_threshold_reached";
11357c478bd9Sstevel@tonic-gate 	else
11367c478bd9Sstevel@tonic-gate 		aux = "method_failed";
11377c478bd9Sstevel@tonic-gate 
11387c478bd9Sstevel@tonic-gate 	(void) restarter_instance_update_states(local_handle, inst,
11397c478bd9Sstevel@tonic-gate 	    RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
11407c478bd9Sstevel@tonic-gate 	    (char *)aux);
11417c478bd9Sstevel@tonic-gate 
11427c478bd9Sstevel@tonic-gate 	if (!method_is_transient(inst, info->sf_method_type) &&
11437c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_primary_ctid != 0)
11447c478bd9Sstevel@tonic-gate 		method_remove_contract(inst, B_TRUE, B_TRUE);
11457c478bd9Sstevel@tonic-gate 
11467c478bd9Sstevel@tonic-gate out:
11477c478bd9Sstevel@tonic-gate 	inst->ri_method_thread = 0;
11487c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&inst->ri_lock);
11497c478bd9Sstevel@tonic-gate 	(void) pthread_cond_broadcast(&inst->ri_method_cv);
11507c478bd9Sstevel@tonic-gate 
11517c478bd9Sstevel@tonic-gate 	scf_instance_destroy(s_inst);
11527c478bd9Sstevel@tonic-gate 	scf_handle_destroy(local_handle);
11537c478bd9Sstevel@tonic-gate 	startd_free(info, sizeof (fork_info_t));
11547c478bd9Sstevel@tonic-gate 	return (NULL);
11557c478bd9Sstevel@tonic-gate }
1156