xref: /titanic_52/usr/src/cmd/svc/startd/restarter.c (revision 7c478bd95313f5f23a4c958a745db2134aa03244)
1*7c478bd9Sstevel@tonic-gate /*
2*7c478bd9Sstevel@tonic-gate  * CDDL HEADER START
3*7c478bd9Sstevel@tonic-gate  *
4*7c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*7c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*7c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*7c478bd9Sstevel@tonic-gate  * with the License.
8*7c478bd9Sstevel@tonic-gate  *
9*7c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*7c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*7c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*7c478bd9Sstevel@tonic-gate  * and limitations under the License.
13*7c478bd9Sstevel@tonic-gate  *
14*7c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*7c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*7c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*7c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*7c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*7c478bd9Sstevel@tonic-gate  *
20*7c478bd9Sstevel@tonic-gate  * CDDL HEADER END
21*7c478bd9Sstevel@tonic-gate  */
22*7c478bd9Sstevel@tonic-gate /*
23*7c478bd9Sstevel@tonic-gate  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24*7c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
25*7c478bd9Sstevel@tonic-gate  */
26*7c478bd9Sstevel@tonic-gate 
27*7c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
28*7c478bd9Sstevel@tonic-gate 
29*7c478bd9Sstevel@tonic-gate /*
30*7c478bd9Sstevel@tonic-gate  * restarter.c - service manipulation
31*7c478bd9Sstevel@tonic-gate  *
32*7c478bd9Sstevel@tonic-gate  * This component manages services whose restarter is svc.startd, the standard
33*7c478bd9Sstevel@tonic-gate  * restarter.  It translates restarter protocol events from the graph engine
34*7c478bd9Sstevel@tonic-gate  * into actions on processes, as a delegated restarter would do.
35*7c478bd9Sstevel@tonic-gate  *
36*7c478bd9Sstevel@tonic-gate  * The master restarter manages a number of always-running threads:
37*7c478bd9Sstevel@tonic-gate  *   - restarter event thread: events from the graph engine
38*7c478bd9Sstevel@tonic-gate  *   - timeout thread: thread to fire queued timeouts
39*7c478bd9Sstevel@tonic-gate  *   - contract thread: thread to handle contract events
40*7c478bd9Sstevel@tonic-gate  *   - wait thread: thread to handle wait-based services
41*7c478bd9Sstevel@tonic-gate  *
42*7c478bd9Sstevel@tonic-gate  * The other threads are created as-needed:
43*7c478bd9Sstevel@tonic-gate  *   - per-instance method threads
44*7c478bd9Sstevel@tonic-gate  *   - per-instance event processing threads
45*7c478bd9Sstevel@tonic-gate  *
46*7c478bd9Sstevel@tonic-gate  * The interaction of all threads must result in the following conditions
47*7c478bd9Sstevel@tonic-gate  * being satisfied (on a per-instance basis):
48*7c478bd9Sstevel@tonic-gate  *   - restarter events must be processed in order
49*7c478bd9Sstevel@tonic-gate  *   - method execution must be serialized
50*7c478bd9Sstevel@tonic-gate  *   - instance delete must be held until outstanding methods are complete
51*7c478bd9Sstevel@tonic-gate  *   - contract events shouldn't be processed while a method is running
52*7c478bd9Sstevel@tonic-gate  *   - timeouts should fire even when a method is running
53*7c478bd9Sstevel@tonic-gate  *
54*7c478bd9Sstevel@tonic-gate  * Service instances are represented by restarter_inst_t's and are kept in the
55*7c478bd9Sstevel@tonic-gate  * instance_list list.
56*7c478bd9Sstevel@tonic-gate  *
57*7c478bd9Sstevel@tonic-gate  * Service States
58*7c478bd9Sstevel@tonic-gate  *   The current state of a service instance is kept in
59*7c478bd9Sstevel@tonic-gate  *   restarter_inst_t->ri_i.i_state.  If transition to a new state could take
60*7c478bd9Sstevel@tonic-gate  *   some time, then before we effect the transition we set
61*7c478bd9Sstevel@tonic-gate  *   restarter_inst_t->ri_i.i_next_state to the target state, and afterwards we
62*7c478bd9Sstevel@tonic-gate  *   rotate i_next_state to i_state and set i_next_state to
63*7c478bd9Sstevel@tonic-gate  *   RESTARTER_STATE_NONE.  So usually i_next_state is _NONE when ri_lock is not
64*7c478bd9Sstevel@tonic-gate  *   held.  The exception is when we launch methods, which are done with
65*7c478bd9Sstevel@tonic-gate  *   a separate thread.  To keep any other threads from grabbing ri_lock before
66*7c478bd9Sstevel@tonic-gate  *   method_thread() does, we set ri_method_thread to the thread id of the
67*7c478bd9Sstevel@tonic-gate  *   method thread, and when it is nonzero any thread with a different thread id
68*7c478bd9Sstevel@tonic-gate  *   waits on ri_method_cv.
69*7c478bd9Sstevel@tonic-gate  *
70*7c478bd9Sstevel@tonic-gate  * Method execution is serialized by blocking on ri_method_cv in
71*7c478bd9Sstevel@tonic-gate  * inst_lookup_by_id() and waiting for a 0 value of ri_method_thread.  This
72*7c478bd9Sstevel@tonic-gate  * also prevents the instance structure from being deleted until all
73*7c478bd9Sstevel@tonic-gate  * outstanding operations such as method_thread() have finished.
74*7c478bd9Sstevel@tonic-gate  *
75*7c478bd9Sstevel@tonic-gate  * Lock ordering:
76*7c478bd9Sstevel@tonic-gate  *
77*7c478bd9Sstevel@tonic-gate  * dgraph_lock [can be held when taking:]
78*7c478bd9Sstevel@tonic-gate  *   utmpx_lock
79*7c478bd9Sstevel@tonic-gate  *   dictionary->dict_lock
80*7c478bd9Sstevel@tonic-gate  *   st->st_load_lock
81*7c478bd9Sstevel@tonic-gate  *   wait_info_lock
82*7c478bd9Sstevel@tonic-gate  *   ru->restarter_update_lock
83*7c478bd9Sstevel@tonic-gate  *     restarter_queue->rpeq_lock
84*7c478bd9Sstevel@tonic-gate  *   instance_list.ril_lock
85*7c478bd9Sstevel@tonic-gate  *     inst->ri_lock
86*7c478bd9Sstevel@tonic-gate  *   st->st_configd_live_lock
87*7c478bd9Sstevel@tonic-gate  *
88*7c478bd9Sstevel@tonic-gate  * instance_list.ril_lock
89*7c478bd9Sstevel@tonic-gate  *   graph_queue->gpeq_lock
90*7c478bd9Sstevel@tonic-gate  *   gu->gu_lock
91*7c478bd9Sstevel@tonic-gate  *   st->st_configd_live_lock
92*7c478bd9Sstevel@tonic-gate  *   dictionary->dict_lock
93*7c478bd9Sstevel@tonic-gate  *   inst->ri_lock
94*7c478bd9Sstevel@tonic-gate  *     graph_queue->gpeq_lock
95*7c478bd9Sstevel@tonic-gate  *     gu->gu_lock
96*7c478bd9Sstevel@tonic-gate  *     tu->tu_lock
97*7c478bd9Sstevel@tonic-gate  *     tq->tq_lock
98*7c478bd9Sstevel@tonic-gate  *     inst->ri_queue_lock
99*7c478bd9Sstevel@tonic-gate  *       wait_info_lock
100*7c478bd9Sstevel@tonic-gate  *       bp->cb_lock
101*7c478bd9Sstevel@tonic-gate  *     utmpx_lock
102*7c478bd9Sstevel@tonic-gate  *
103*7c478bd9Sstevel@tonic-gate  * single_user_thread_lock
104*7c478bd9Sstevel@tonic-gate  *   wait_info_lock
105*7c478bd9Sstevel@tonic-gate  *   utmpx_lock
106*7c478bd9Sstevel@tonic-gate  *
107*7c478bd9Sstevel@tonic-gate  * gu_freeze_lock
108*7c478bd9Sstevel@tonic-gate  *
109*7c478bd9Sstevel@tonic-gate  * logbuf_mutex nests inside pretty much everything.
110*7c478bd9Sstevel@tonic-gate  */
111*7c478bd9Sstevel@tonic-gate 
112*7c478bd9Sstevel@tonic-gate #include <sys/contract/process.h>
113*7c478bd9Sstevel@tonic-gate #include <sys/ctfs.h>
114*7c478bd9Sstevel@tonic-gate #include <sys/stat.h>
115*7c478bd9Sstevel@tonic-gate #include <sys/time.h>
116*7c478bd9Sstevel@tonic-gate #include <sys/types.h>
117*7c478bd9Sstevel@tonic-gate #include <sys/uio.h>
118*7c478bd9Sstevel@tonic-gate #include <sys/wait.h>
119*7c478bd9Sstevel@tonic-gate #include <assert.h>
120*7c478bd9Sstevel@tonic-gate #include <errno.h>
121*7c478bd9Sstevel@tonic-gate #include <fcntl.h>
122*7c478bd9Sstevel@tonic-gate #include <libcontract.h>
123*7c478bd9Sstevel@tonic-gate #include <libcontract_priv.h>
124*7c478bd9Sstevel@tonic-gate #include <libintl.h>
125*7c478bd9Sstevel@tonic-gate #include <librestart.h>
126*7c478bd9Sstevel@tonic-gate #include <librestart_priv.h>
127*7c478bd9Sstevel@tonic-gate #include <libuutil.h>
128*7c478bd9Sstevel@tonic-gate #include <limits.h>
129*7c478bd9Sstevel@tonic-gate #include <poll.h>
130*7c478bd9Sstevel@tonic-gate #include <port.h>
131*7c478bd9Sstevel@tonic-gate #include <pthread.h>
132*7c478bd9Sstevel@tonic-gate #include <stdarg.h>
133*7c478bd9Sstevel@tonic-gate #include <stdio.h>
134*7c478bd9Sstevel@tonic-gate #include <strings.h>
135*7c478bd9Sstevel@tonic-gate #include <unistd.h>
136*7c478bd9Sstevel@tonic-gate 
137*7c478bd9Sstevel@tonic-gate #include "startd.h"
138*7c478bd9Sstevel@tonic-gate #include "protocol.h"
139*7c478bd9Sstevel@tonic-gate 
140*7c478bd9Sstevel@tonic-gate static uu_list_pool_t *restarter_instance_pool;
141*7c478bd9Sstevel@tonic-gate static restarter_instance_list_t instance_list;
142*7c478bd9Sstevel@tonic-gate 
143*7c478bd9Sstevel@tonic-gate static uu_list_pool_t *restarter_queue_pool;
144*7c478bd9Sstevel@tonic-gate 
145*7c478bd9Sstevel@tonic-gate /*ARGSUSED*/
146*7c478bd9Sstevel@tonic-gate static int
147*7c478bd9Sstevel@tonic-gate restarter_instance_compare(const void *lc_arg, const void *rc_arg,
148*7c478bd9Sstevel@tonic-gate     void *private)
149*7c478bd9Sstevel@tonic-gate {
150*7c478bd9Sstevel@tonic-gate 	int lc_id = ((const restarter_inst_t *)lc_arg)->ri_id;
151*7c478bd9Sstevel@tonic-gate 	int rc_id = *(int *)rc_arg;
152*7c478bd9Sstevel@tonic-gate 
153*7c478bd9Sstevel@tonic-gate 	if (lc_id > rc_id)
154*7c478bd9Sstevel@tonic-gate 		return (1);
155*7c478bd9Sstevel@tonic-gate 	if (lc_id < rc_id)
156*7c478bd9Sstevel@tonic-gate 		return (-1);
157*7c478bd9Sstevel@tonic-gate 	return (0);
158*7c478bd9Sstevel@tonic-gate }
159*7c478bd9Sstevel@tonic-gate 
160*7c478bd9Sstevel@tonic-gate static restarter_inst_t *
161*7c478bd9Sstevel@tonic-gate inst_lookup_by_name(const char *name)
162*7c478bd9Sstevel@tonic-gate {
163*7c478bd9Sstevel@tonic-gate 	int id;
164*7c478bd9Sstevel@tonic-gate 
165*7c478bd9Sstevel@tonic-gate 	id = dict_lookup_byname(name);
166*7c478bd9Sstevel@tonic-gate 	if (id == -1)
167*7c478bd9Sstevel@tonic-gate 		return (NULL);
168*7c478bd9Sstevel@tonic-gate 
169*7c478bd9Sstevel@tonic-gate 	return (inst_lookup_by_id(id));
170*7c478bd9Sstevel@tonic-gate }
171*7c478bd9Sstevel@tonic-gate 
172*7c478bd9Sstevel@tonic-gate restarter_inst_t *
173*7c478bd9Sstevel@tonic-gate inst_lookup_by_id(int id)
174*7c478bd9Sstevel@tonic-gate {
175*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst;
176*7c478bd9Sstevel@tonic-gate 
177*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&instance_list.ril_lock);
178*7c478bd9Sstevel@tonic-gate 	inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
179*7c478bd9Sstevel@tonic-gate 	if (inst != NULL)
180*7c478bd9Sstevel@tonic-gate 		MUTEX_LOCK(&inst->ri_lock);
181*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&instance_list.ril_lock);
182*7c478bd9Sstevel@tonic-gate 
183*7c478bd9Sstevel@tonic-gate 	if (inst != NULL) {
184*7c478bd9Sstevel@tonic-gate 		while (inst->ri_method_thread != 0 &&
185*7c478bd9Sstevel@tonic-gate 		    !pthread_equal(inst->ri_method_thread, pthread_self())) {
186*7c478bd9Sstevel@tonic-gate 			++inst->ri_method_waiters;
187*7c478bd9Sstevel@tonic-gate 			(void) pthread_cond_wait(&inst->ri_method_cv,
188*7c478bd9Sstevel@tonic-gate 			    &inst->ri_lock);
189*7c478bd9Sstevel@tonic-gate 			assert(inst->ri_method_waiters > 0);
190*7c478bd9Sstevel@tonic-gate 			--inst->ri_method_waiters;
191*7c478bd9Sstevel@tonic-gate 		}
192*7c478bd9Sstevel@tonic-gate 	}
193*7c478bd9Sstevel@tonic-gate 
194*7c478bd9Sstevel@tonic-gate 	return (inst);
195*7c478bd9Sstevel@tonic-gate }
196*7c478bd9Sstevel@tonic-gate 
197*7c478bd9Sstevel@tonic-gate static restarter_inst_t *
198*7c478bd9Sstevel@tonic-gate inst_lookup_queue(const char *name)
199*7c478bd9Sstevel@tonic-gate {
200*7c478bd9Sstevel@tonic-gate 	int id;
201*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst;
202*7c478bd9Sstevel@tonic-gate 
203*7c478bd9Sstevel@tonic-gate 	id = dict_lookup_byname(name);
204*7c478bd9Sstevel@tonic-gate 	if (id == -1)
205*7c478bd9Sstevel@tonic-gate 		return (NULL);
206*7c478bd9Sstevel@tonic-gate 
207*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&instance_list.ril_lock);
208*7c478bd9Sstevel@tonic-gate 	inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
209*7c478bd9Sstevel@tonic-gate 	if (inst != NULL)
210*7c478bd9Sstevel@tonic-gate 		MUTEX_LOCK(&inst->ri_queue_lock);
211*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&instance_list.ril_lock);
212*7c478bd9Sstevel@tonic-gate 
213*7c478bd9Sstevel@tonic-gate 	return (inst);
214*7c478bd9Sstevel@tonic-gate }
215*7c478bd9Sstevel@tonic-gate 
216*7c478bd9Sstevel@tonic-gate const char *
217*7c478bd9Sstevel@tonic-gate service_style(int flags)
218*7c478bd9Sstevel@tonic-gate {
219*7c478bd9Sstevel@tonic-gate 	switch (flags & RINST_STYLE_MASK) {
220*7c478bd9Sstevel@tonic-gate 	case RINST_CONTRACT:	return ("contract");
221*7c478bd9Sstevel@tonic-gate 	case RINST_TRANSIENT:	return ("transient");
222*7c478bd9Sstevel@tonic-gate 	case RINST_WAIT:	return ("wait");
223*7c478bd9Sstevel@tonic-gate 
224*7c478bd9Sstevel@tonic-gate 	default:
225*7c478bd9Sstevel@tonic-gate #ifndef NDEBUG
226*7c478bd9Sstevel@tonic-gate 		uu_warn("%s:%d: Bad flags 0x%x.\n", __FILE__, __LINE__, flags);
227*7c478bd9Sstevel@tonic-gate #endif
228*7c478bd9Sstevel@tonic-gate 		abort();
229*7c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
230*7c478bd9Sstevel@tonic-gate 	}
231*7c478bd9Sstevel@tonic-gate }
232*7c478bd9Sstevel@tonic-gate 
233*7c478bd9Sstevel@tonic-gate /*
234*7c478bd9Sstevel@tonic-gate  * Fails with ECONNABORTED or ECANCELED.
235*7c478bd9Sstevel@tonic-gate  */
236*7c478bd9Sstevel@tonic-gate static int
237*7c478bd9Sstevel@tonic-gate check_contract(restarter_inst_t *inst, boolean_t primary,
238*7c478bd9Sstevel@tonic-gate     scf_instance_t *scf_inst)
239*7c478bd9Sstevel@tonic-gate {
240*7c478bd9Sstevel@tonic-gate 	ctid_t *ctidp;
241*7c478bd9Sstevel@tonic-gate 	int fd, r;
242*7c478bd9Sstevel@tonic-gate 
243*7c478bd9Sstevel@tonic-gate 	ctidp = primary ? &inst->ri_i.i_primary_ctid :
244*7c478bd9Sstevel@tonic-gate 	    &inst->ri_i.i_transient_ctid;
245*7c478bd9Sstevel@tonic-gate 
246*7c478bd9Sstevel@tonic-gate 	assert(*ctidp >= 1);
247*7c478bd9Sstevel@tonic-gate 
248*7c478bd9Sstevel@tonic-gate 	fd = contract_open(*ctidp, NULL, "status", O_RDONLY);
249*7c478bd9Sstevel@tonic-gate 	if (fd >= 0) {
250*7c478bd9Sstevel@tonic-gate 		r = close(fd);
251*7c478bd9Sstevel@tonic-gate 		assert(r == 0);
252*7c478bd9Sstevel@tonic-gate 		return (0);
253*7c478bd9Sstevel@tonic-gate 	}
254*7c478bd9Sstevel@tonic-gate 
255*7c478bd9Sstevel@tonic-gate 	r = restarter_remove_contract(scf_inst, *ctidp, primary ?
256*7c478bd9Sstevel@tonic-gate 	    RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
257*7c478bd9Sstevel@tonic-gate 	switch (r) {
258*7c478bd9Sstevel@tonic-gate 	case 0:
259*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
260*7c478bd9Sstevel@tonic-gate 	case ECANCELED:
261*7c478bd9Sstevel@tonic-gate 		*ctidp = 0;
262*7c478bd9Sstevel@tonic-gate 		return (r);
263*7c478bd9Sstevel@tonic-gate 
264*7c478bd9Sstevel@tonic-gate 	case ENOMEM:
265*7c478bd9Sstevel@tonic-gate 		uu_die("Out of memory\n");
266*7c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
267*7c478bd9Sstevel@tonic-gate 
268*7c478bd9Sstevel@tonic-gate 	case EPERM:
269*7c478bd9Sstevel@tonic-gate 		uu_die("Insufficient privilege.\n");
270*7c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
271*7c478bd9Sstevel@tonic-gate 
272*7c478bd9Sstevel@tonic-gate 	case EACCES:
273*7c478bd9Sstevel@tonic-gate 		uu_die("Repository backend access denied.\n");
274*7c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
275*7c478bd9Sstevel@tonic-gate 
276*7c478bd9Sstevel@tonic-gate 	case EROFS:
277*7c478bd9Sstevel@tonic-gate 		log_error(LOG_INFO, "Could not remove unusable contract id %ld "
278*7c478bd9Sstevel@tonic-gate 		    "for %s from repository.\n", *ctidp, inst->ri_i.i_fmri);
279*7c478bd9Sstevel@tonic-gate 		return (0);
280*7c478bd9Sstevel@tonic-gate 
281*7c478bd9Sstevel@tonic-gate 	case EINVAL:
282*7c478bd9Sstevel@tonic-gate 	case EBADF:
283*7c478bd9Sstevel@tonic-gate 	default:
284*7c478bd9Sstevel@tonic-gate 		assert(0);
285*7c478bd9Sstevel@tonic-gate 		abort();
286*7c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
287*7c478bd9Sstevel@tonic-gate 	}
288*7c478bd9Sstevel@tonic-gate }
289*7c478bd9Sstevel@tonic-gate 
290*7c478bd9Sstevel@tonic-gate static int stop_instance(scf_handle_t *, restarter_inst_t *, stop_cause_t);
291*7c478bd9Sstevel@tonic-gate 
292*7c478bd9Sstevel@tonic-gate /*
293*7c478bd9Sstevel@tonic-gate  * int restarter_insert_inst(scf_handle_t *, char *)
294*7c478bd9Sstevel@tonic-gate  *   If the inst is already in the restarter list, return its id.  If the inst
295*7c478bd9Sstevel@tonic-gate  *   is not in the restarter list, initialize a restarter_inst_t, initialize its
296*7c478bd9Sstevel@tonic-gate  *   states, insert it into the list, and return 0.
297*7c478bd9Sstevel@tonic-gate  *
298*7c478bd9Sstevel@tonic-gate  *   Fails with
299*7c478bd9Sstevel@tonic-gate  *     ENOENT - name is not in the repository
300*7c478bd9Sstevel@tonic-gate  */
301*7c478bd9Sstevel@tonic-gate static int
302*7c478bd9Sstevel@tonic-gate restarter_insert_inst(scf_handle_t *h, const char *name)
303*7c478bd9Sstevel@tonic-gate {
304*7c478bd9Sstevel@tonic-gate 	int id, r;
305*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst;
306*7c478bd9Sstevel@tonic-gate 	uu_list_index_t idx;
307*7c478bd9Sstevel@tonic-gate 	scf_service_t *scf_svc;
308*7c478bd9Sstevel@tonic-gate 	scf_instance_t *scf_inst;
309*7c478bd9Sstevel@tonic-gate 	scf_snapshot_t *snap;
310*7c478bd9Sstevel@tonic-gate 	scf_propertygroup_t *pg;
311*7c478bd9Sstevel@tonic-gate 	char *svc_name, *inst_name;
312*7c478bd9Sstevel@tonic-gate 	char logfilebuf[PATH_MAX];
313*7c478bd9Sstevel@tonic-gate 	char *c;
314*7c478bd9Sstevel@tonic-gate 	boolean_t do_commit_states;
315*7c478bd9Sstevel@tonic-gate 	restarter_instance_state_t state, next_state;
316*7c478bd9Sstevel@tonic-gate 	protocol_states_t *ps;
317*7c478bd9Sstevel@tonic-gate 	pid_t start_pid;
318*7c478bd9Sstevel@tonic-gate 
319*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&instance_list.ril_lock);
320*7c478bd9Sstevel@tonic-gate 
321*7c478bd9Sstevel@tonic-gate 	/*
322*7c478bd9Sstevel@tonic-gate 	 * We don't use inst_lookup_by_name() here because we want the lookup
323*7c478bd9Sstevel@tonic-gate 	 * & insert to be atomic.
324*7c478bd9Sstevel@tonic-gate 	 */
325*7c478bd9Sstevel@tonic-gate 	id = dict_lookup_byname(name);
326*7c478bd9Sstevel@tonic-gate 	if (id != -1) {
327*7c478bd9Sstevel@tonic-gate 		inst = uu_list_find(instance_list.ril_instance_list, &id, NULL,
328*7c478bd9Sstevel@tonic-gate 		    &idx);
329*7c478bd9Sstevel@tonic-gate 		if (inst != NULL) {
330*7c478bd9Sstevel@tonic-gate 			MUTEX_UNLOCK(&instance_list.ril_lock);
331*7c478bd9Sstevel@tonic-gate 			return (0);
332*7c478bd9Sstevel@tonic-gate 		}
333*7c478bd9Sstevel@tonic-gate 	}
334*7c478bd9Sstevel@tonic-gate 
335*7c478bd9Sstevel@tonic-gate 	/* Allocate an instance */
336*7c478bd9Sstevel@tonic-gate 	inst = startd_zalloc(sizeof (restarter_inst_t));
337*7c478bd9Sstevel@tonic-gate 	inst->ri_utmpx_prefix = startd_alloc(max_scf_value_size);
338*7c478bd9Sstevel@tonic-gate 	inst->ri_utmpx_prefix[0] = '\0';
339*7c478bd9Sstevel@tonic-gate 
340*7c478bd9Sstevel@tonic-gate 	inst->ri_i.i_fmri = startd_alloc(strlen(name) + 1);
341*7c478bd9Sstevel@tonic-gate 	(void) strcpy((char *)inst->ri_i.i_fmri, name);
342*7c478bd9Sstevel@tonic-gate 
343*7c478bd9Sstevel@tonic-gate 	inst->ri_queue = startd_list_create(restarter_queue_pool, inst, 0);
344*7c478bd9Sstevel@tonic-gate 
345*7c478bd9Sstevel@tonic-gate 	/*
346*7c478bd9Sstevel@tonic-gate 	 * id shouldn't be -1 since we use the same dictionary as graph.c, but
347*7c478bd9Sstevel@tonic-gate 	 * just in case.
348*7c478bd9Sstevel@tonic-gate 	 */
349*7c478bd9Sstevel@tonic-gate 	inst->ri_id = (id != -1 ? id : dict_insert(name));
350*7c478bd9Sstevel@tonic-gate 
351*7c478bd9Sstevel@tonic-gate 	special_online_hooks_get(name, &inst->ri_pre_online_hook,
352*7c478bd9Sstevel@tonic-gate 	    &inst->ri_post_online_hook, &inst->ri_post_offline_hook);
353*7c478bd9Sstevel@tonic-gate 
354*7c478bd9Sstevel@tonic-gate 	scf_svc = safe_scf_service_create(h);
355*7c478bd9Sstevel@tonic-gate 	scf_inst = safe_scf_instance_create(h);
356*7c478bd9Sstevel@tonic-gate 	pg = safe_scf_pg_create(h);
357*7c478bd9Sstevel@tonic-gate 	svc_name = startd_alloc(max_scf_name_size);
358*7c478bd9Sstevel@tonic-gate 	inst_name = startd_alloc(max_scf_name_size);
359*7c478bd9Sstevel@tonic-gate 
360*7c478bd9Sstevel@tonic-gate rep_retry:
361*7c478bd9Sstevel@tonic-gate 	if (scf_handle_decode_fmri(h, name, NULL, scf_svc, scf_inst, NULL,
362*7c478bd9Sstevel@tonic-gate 	    NULL, SCF_DECODE_FMRI_EXACT) != 0) {
363*7c478bd9Sstevel@tonic-gate 		switch (scf_error()) {
364*7c478bd9Sstevel@tonic-gate 		case SCF_ERROR_CONNECTION_BROKEN:
365*7c478bd9Sstevel@tonic-gate 			libscf_handle_rebind(h);
366*7c478bd9Sstevel@tonic-gate 			goto rep_retry;
367*7c478bd9Sstevel@tonic-gate 
368*7c478bd9Sstevel@tonic-gate 		case SCF_ERROR_NOT_FOUND:
369*7c478bd9Sstevel@tonic-gate deleted:
370*7c478bd9Sstevel@tonic-gate 			MUTEX_UNLOCK(&instance_list.ril_lock);
371*7c478bd9Sstevel@tonic-gate 			startd_free(inst_name, max_scf_name_size);
372*7c478bd9Sstevel@tonic-gate 			startd_free(svc_name, max_scf_name_size);
373*7c478bd9Sstevel@tonic-gate 			scf_pg_destroy(pg);
374*7c478bd9Sstevel@tonic-gate 			scf_instance_destroy(scf_inst);
375*7c478bd9Sstevel@tonic-gate 			scf_service_destroy(scf_svc);
376*7c478bd9Sstevel@tonic-gate 			startd_free((void *)inst->ri_i.i_fmri,
377*7c478bd9Sstevel@tonic-gate 			    strlen(inst->ri_i.i_fmri) + 1);
378*7c478bd9Sstevel@tonic-gate 			startd_free(inst, sizeof (restarter_inst_t));
379*7c478bd9Sstevel@tonic-gate 			return (ENOENT);
380*7c478bd9Sstevel@tonic-gate 		}
381*7c478bd9Sstevel@tonic-gate 
382*7c478bd9Sstevel@tonic-gate 		uu_die("Can't decode FMRI %s: %s\n", name,
383*7c478bd9Sstevel@tonic-gate 		    scf_strerror(scf_error()));
384*7c478bd9Sstevel@tonic-gate 	}
385*7c478bd9Sstevel@tonic-gate 
386*7c478bd9Sstevel@tonic-gate 	/*
387*7c478bd9Sstevel@tonic-gate 	 * If there's no running snapshot, then we execute using the editing
388*7c478bd9Sstevel@tonic-gate 	 * snapshot.  Pending snapshots will be taken later.
389*7c478bd9Sstevel@tonic-gate 	 */
390*7c478bd9Sstevel@tonic-gate 	snap = libscf_get_running_snapshot(scf_inst);
391*7c478bd9Sstevel@tonic-gate 
392*7c478bd9Sstevel@tonic-gate 	if ((scf_service_get_name(scf_svc, svc_name, max_scf_name_size) < 0) ||
393*7c478bd9Sstevel@tonic-gate 	    (scf_instance_get_name(scf_inst, inst_name, max_scf_name_size) <
394*7c478bd9Sstevel@tonic-gate 	    0)) {
395*7c478bd9Sstevel@tonic-gate 		switch (scf_error()) {
396*7c478bd9Sstevel@tonic-gate 		case SCF_ERROR_NOT_SET:
397*7c478bd9Sstevel@tonic-gate 			break;
398*7c478bd9Sstevel@tonic-gate 
399*7c478bd9Sstevel@tonic-gate 		case SCF_ERROR_CONNECTION_BROKEN:
400*7c478bd9Sstevel@tonic-gate 			libscf_handle_rebind(h);
401*7c478bd9Sstevel@tonic-gate 			goto rep_retry;
402*7c478bd9Sstevel@tonic-gate 
403*7c478bd9Sstevel@tonic-gate 		default:
404*7c478bd9Sstevel@tonic-gate 			assert(0);
405*7c478bd9Sstevel@tonic-gate 			abort();
406*7c478bd9Sstevel@tonic-gate 		}
407*7c478bd9Sstevel@tonic-gate 
408*7c478bd9Sstevel@tonic-gate 		scf_snapshot_destroy(snap);
409*7c478bd9Sstevel@tonic-gate 		goto deleted;
410*7c478bd9Sstevel@tonic-gate 	}
411*7c478bd9Sstevel@tonic-gate 
412*7c478bd9Sstevel@tonic-gate 	/*
413*7c478bd9Sstevel@tonic-gate 	 * If the restarter group is missing, use uninit/none.  Otherwise,
414*7c478bd9Sstevel@tonic-gate 	 * we're probably being restarted & don't want to mess up the states
415*7c478bd9Sstevel@tonic-gate 	 * that are there.
416*7c478bd9Sstevel@tonic-gate 	 */
417*7c478bd9Sstevel@tonic-gate 	state = RESTARTER_STATE_UNINIT;
418*7c478bd9Sstevel@tonic-gate 	next_state = RESTARTER_STATE_NONE;
419*7c478bd9Sstevel@tonic-gate 
420*7c478bd9Sstevel@tonic-gate 	r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
421*7c478bd9Sstevel@tonic-gate 	if (r != 0) {
422*7c478bd9Sstevel@tonic-gate 		switch (scf_error()) {
423*7c478bd9Sstevel@tonic-gate 		case SCF_ERROR_CONNECTION_BROKEN:
424*7c478bd9Sstevel@tonic-gate 			libscf_handle_rebind(h);
425*7c478bd9Sstevel@tonic-gate 			goto rep_retry;
426*7c478bd9Sstevel@tonic-gate 
427*7c478bd9Sstevel@tonic-gate 		case SCF_ERROR_NOT_SET:
428*7c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(snap);
429*7c478bd9Sstevel@tonic-gate 			goto deleted;
430*7c478bd9Sstevel@tonic-gate 
431*7c478bd9Sstevel@tonic-gate 		case SCF_ERROR_NOT_FOUND:
432*7c478bd9Sstevel@tonic-gate 			/*
433*7c478bd9Sstevel@tonic-gate 			 * This shouldn't happen since the graph engine should
434*7c478bd9Sstevel@tonic-gate 			 * have initialized the state to uninitialized/none if
435*7c478bd9Sstevel@tonic-gate 			 * there was no restarter pg.  In case somebody
436*7c478bd9Sstevel@tonic-gate 			 * deleted it, though....
437*7c478bd9Sstevel@tonic-gate 			 */
438*7c478bd9Sstevel@tonic-gate 			do_commit_states = B_TRUE;
439*7c478bd9Sstevel@tonic-gate 			break;
440*7c478bd9Sstevel@tonic-gate 
441*7c478bd9Sstevel@tonic-gate 		default:
442*7c478bd9Sstevel@tonic-gate 			assert(0);
443*7c478bd9Sstevel@tonic-gate 			abort();
444*7c478bd9Sstevel@tonic-gate 		}
445*7c478bd9Sstevel@tonic-gate 	} else {
446*7c478bd9Sstevel@tonic-gate 		r = libscf_read_states(pg, &state, &next_state);
447*7c478bd9Sstevel@tonic-gate 		if (r != 0) {
448*7c478bd9Sstevel@tonic-gate 			do_commit_states = B_TRUE;
449*7c478bd9Sstevel@tonic-gate 		} else {
450*7c478bd9Sstevel@tonic-gate 			if (next_state != RESTARTER_STATE_NONE) {
451*7c478bd9Sstevel@tonic-gate 				/*
452*7c478bd9Sstevel@tonic-gate 				 * Force next_state to _NONE since we
453*7c478bd9Sstevel@tonic-gate 				 * don't look for method processes.
454*7c478bd9Sstevel@tonic-gate 				 */
455*7c478bd9Sstevel@tonic-gate 				next_state = RESTARTER_STATE_NONE;
456*7c478bd9Sstevel@tonic-gate 				do_commit_states = B_TRUE;
457*7c478bd9Sstevel@tonic-gate 			} else {
458*7c478bd9Sstevel@tonic-gate 				/*
459*7c478bd9Sstevel@tonic-gate 				 * Inform the restarter of our state without
460*7c478bd9Sstevel@tonic-gate 				 * changing the STIME in the repository.
461*7c478bd9Sstevel@tonic-gate 				 */
462*7c478bd9Sstevel@tonic-gate 				ps = startd_alloc(sizeof (*ps));
463*7c478bd9Sstevel@tonic-gate 				inst->ri_i.i_state = ps->ps_state = state;
464*7c478bd9Sstevel@tonic-gate 				inst->ri_i.i_next_state = ps->ps_state_next =
465*7c478bd9Sstevel@tonic-gate 				    next_state;
466*7c478bd9Sstevel@tonic-gate 
467*7c478bd9Sstevel@tonic-gate 				graph_protocol_send_event(inst->ri_i.i_fmri,
468*7c478bd9Sstevel@tonic-gate 				    GRAPH_UPDATE_STATE_CHANGE, ps);
469*7c478bd9Sstevel@tonic-gate 
470*7c478bd9Sstevel@tonic-gate 				do_commit_states = B_FALSE;
471*7c478bd9Sstevel@tonic-gate 			}
472*7c478bd9Sstevel@tonic-gate 		}
473*7c478bd9Sstevel@tonic-gate 	}
474*7c478bd9Sstevel@tonic-gate 
475*7c478bd9Sstevel@tonic-gate 	switch (libscf_get_startd_properties(scf_inst, snap, &inst->ri_flags,
476*7c478bd9Sstevel@tonic-gate 	    &inst->ri_utmpx_prefix)) {
477*7c478bd9Sstevel@tonic-gate 	case 0:
478*7c478bd9Sstevel@tonic-gate 		break;
479*7c478bd9Sstevel@tonic-gate 
480*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
481*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
482*7c478bd9Sstevel@tonic-gate 		goto rep_retry;
483*7c478bd9Sstevel@tonic-gate 
484*7c478bd9Sstevel@tonic-gate 	case ECANCELED:
485*7c478bd9Sstevel@tonic-gate 		scf_snapshot_destroy(snap);
486*7c478bd9Sstevel@tonic-gate 		startd_free(inst->ri_utmpx_prefix, max_scf_value_size);
487*7c478bd9Sstevel@tonic-gate 		goto deleted;
488*7c478bd9Sstevel@tonic-gate 
489*7c478bd9Sstevel@tonic-gate 	case ENOENT:
490*7c478bd9Sstevel@tonic-gate 		/*
491*7c478bd9Sstevel@tonic-gate 		 * This is odd, because the graph engine should have required
492*7c478bd9Sstevel@tonic-gate 		 * the general property group.  So we'll just use default
493*7c478bd9Sstevel@tonic-gate 		 * flags in anticipation of the graph engine sending us
494*7c478bd9Sstevel@tonic-gate 		 * REMOVE_INSTANCE when it finds out that the general property
495*7c478bd9Sstevel@tonic-gate 		 * group has been deleted.
496*7c478bd9Sstevel@tonic-gate 		 */
497*7c478bd9Sstevel@tonic-gate 		inst->ri_flags = RINST_CONTRACT;
498*7c478bd9Sstevel@tonic-gate 		break;
499*7c478bd9Sstevel@tonic-gate 
500*7c478bd9Sstevel@tonic-gate 	default:
501*7c478bd9Sstevel@tonic-gate 		assert(0);
502*7c478bd9Sstevel@tonic-gate 		abort();
503*7c478bd9Sstevel@tonic-gate 	}
504*7c478bd9Sstevel@tonic-gate 
505*7c478bd9Sstevel@tonic-gate 	switch (libscf_get_template_values(scf_inst, snap,
506*7c478bd9Sstevel@tonic-gate 	    &inst->ri_common_name, &inst->ri_C_common_name)) {
507*7c478bd9Sstevel@tonic-gate 	case 0:
508*7c478bd9Sstevel@tonic-gate 		break;
509*7c478bd9Sstevel@tonic-gate 
510*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
511*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
512*7c478bd9Sstevel@tonic-gate 		goto rep_retry;
513*7c478bd9Sstevel@tonic-gate 
514*7c478bd9Sstevel@tonic-gate 	case ECANCELED:
515*7c478bd9Sstevel@tonic-gate 		scf_snapshot_destroy(snap);
516*7c478bd9Sstevel@tonic-gate 		startd_free(inst->ri_common_name, max_scf_value_size);
517*7c478bd9Sstevel@tonic-gate 		inst->ri_common_name = NULL;
518*7c478bd9Sstevel@tonic-gate 		goto deleted;
519*7c478bd9Sstevel@tonic-gate 
520*7c478bd9Sstevel@tonic-gate 	case ECHILD:
521*7c478bd9Sstevel@tonic-gate 	case ENOENT:
522*7c478bd9Sstevel@tonic-gate 		break;
523*7c478bd9Sstevel@tonic-gate 
524*7c478bd9Sstevel@tonic-gate 	default:
525*7c478bd9Sstevel@tonic-gate 		assert(0);
526*7c478bd9Sstevel@tonic-gate 		abort();
527*7c478bd9Sstevel@tonic-gate 	}
528*7c478bd9Sstevel@tonic-gate 
529*7c478bd9Sstevel@tonic-gate 	switch (libscf_read_method_ids(h, scf_inst, inst->ri_i.i_fmri,
530*7c478bd9Sstevel@tonic-gate 	    &inst->ri_i.i_primary_ctid, &inst->ri_i.i_transient_ctid,
531*7c478bd9Sstevel@tonic-gate 	    &start_pid)) {
532*7c478bd9Sstevel@tonic-gate 	case 0:
533*7c478bd9Sstevel@tonic-gate 		break;
534*7c478bd9Sstevel@tonic-gate 
535*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
536*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
537*7c478bd9Sstevel@tonic-gate 		goto rep_retry;
538*7c478bd9Sstevel@tonic-gate 
539*7c478bd9Sstevel@tonic-gate 	case ECANCELED:
540*7c478bd9Sstevel@tonic-gate 		scf_snapshot_destroy(snap);
541*7c478bd9Sstevel@tonic-gate 		goto deleted;
542*7c478bd9Sstevel@tonic-gate 
543*7c478bd9Sstevel@tonic-gate 	default:
544*7c478bd9Sstevel@tonic-gate 		assert(0);
545*7c478bd9Sstevel@tonic-gate 		abort();
546*7c478bd9Sstevel@tonic-gate 	}
547*7c478bd9Sstevel@tonic-gate 
548*7c478bd9Sstevel@tonic-gate 	if (inst->ri_i.i_primary_ctid >= 1) {
549*7c478bd9Sstevel@tonic-gate 		contract_hash_store(inst->ri_i.i_primary_ctid, inst->ri_id);
550*7c478bd9Sstevel@tonic-gate 
551*7c478bd9Sstevel@tonic-gate 		switch (check_contract(inst, B_TRUE, scf_inst)) {
552*7c478bd9Sstevel@tonic-gate 		case 0:
553*7c478bd9Sstevel@tonic-gate 			break;
554*7c478bd9Sstevel@tonic-gate 
555*7c478bd9Sstevel@tonic-gate 		case ECONNABORTED:
556*7c478bd9Sstevel@tonic-gate 			libscf_handle_rebind(h);
557*7c478bd9Sstevel@tonic-gate 			goto rep_retry;
558*7c478bd9Sstevel@tonic-gate 
559*7c478bd9Sstevel@tonic-gate 		case ECANCELED:
560*7c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(snap);
561*7c478bd9Sstevel@tonic-gate 			goto deleted;
562*7c478bd9Sstevel@tonic-gate 
563*7c478bd9Sstevel@tonic-gate 		default:
564*7c478bd9Sstevel@tonic-gate 			assert(0);
565*7c478bd9Sstevel@tonic-gate 			abort();
566*7c478bd9Sstevel@tonic-gate 		}
567*7c478bd9Sstevel@tonic-gate 	}
568*7c478bd9Sstevel@tonic-gate 
569*7c478bd9Sstevel@tonic-gate 	if (inst->ri_i.i_transient_ctid >= 1) {
570*7c478bd9Sstevel@tonic-gate 		switch (check_contract(inst, B_FALSE, scf_inst)) {
571*7c478bd9Sstevel@tonic-gate 		case 0:
572*7c478bd9Sstevel@tonic-gate 			break;
573*7c478bd9Sstevel@tonic-gate 
574*7c478bd9Sstevel@tonic-gate 		case ECONNABORTED:
575*7c478bd9Sstevel@tonic-gate 			libscf_handle_rebind(h);
576*7c478bd9Sstevel@tonic-gate 			goto rep_retry;
577*7c478bd9Sstevel@tonic-gate 
578*7c478bd9Sstevel@tonic-gate 		case ECANCELED:
579*7c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(snap);
580*7c478bd9Sstevel@tonic-gate 			goto deleted;
581*7c478bd9Sstevel@tonic-gate 
582*7c478bd9Sstevel@tonic-gate 		default:
583*7c478bd9Sstevel@tonic-gate 			assert(0);
584*7c478bd9Sstevel@tonic-gate 			abort();
585*7c478bd9Sstevel@tonic-gate 		}
586*7c478bd9Sstevel@tonic-gate 	}
587*7c478bd9Sstevel@tonic-gate 
588*7c478bd9Sstevel@tonic-gate 	/* No more failures we live through, so add it to the list. */
589*7c478bd9Sstevel@tonic-gate 	(void) pthread_mutex_init(&inst->ri_lock, &mutex_attrs);
590*7c478bd9Sstevel@tonic-gate 	(void) pthread_mutex_init(&inst->ri_queue_lock, &mutex_attrs);
591*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&inst->ri_lock);
592*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&inst->ri_queue_lock);
593*7c478bd9Sstevel@tonic-gate 
594*7c478bd9Sstevel@tonic-gate 	(void) pthread_cond_init(&inst->ri_method_cv, NULL);
595*7c478bd9Sstevel@tonic-gate 
596*7c478bd9Sstevel@tonic-gate 	uu_list_node_init(inst, &inst->ri_link, restarter_instance_pool);
597*7c478bd9Sstevel@tonic-gate 	uu_list_insert(instance_list.ril_instance_list, inst, idx);
598*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&instance_list.ril_lock);
599*7c478bd9Sstevel@tonic-gate 
600*7c478bd9Sstevel@tonic-gate 	if (start_pid != -1 &&
601*7c478bd9Sstevel@tonic-gate 	    (inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT) {
602*7c478bd9Sstevel@tonic-gate 		int ret;
603*7c478bd9Sstevel@tonic-gate 		ret = wait_register(start_pid, inst->ri_i.i_fmri, 0, 1);
604*7c478bd9Sstevel@tonic-gate 		if (ret == -1) {
605*7c478bd9Sstevel@tonic-gate 			/*
606*7c478bd9Sstevel@tonic-gate 			 * Implication:  if we can't reregister the
607*7c478bd9Sstevel@tonic-gate 			 * instance, we will start another one.  Two
608*7c478bd9Sstevel@tonic-gate 			 * instances may or may not result in a resource
609*7c478bd9Sstevel@tonic-gate 			 * conflict.
610*7c478bd9Sstevel@tonic-gate 			 */
611*7c478bd9Sstevel@tonic-gate 			log_error(LOG_WARNING,
612*7c478bd9Sstevel@tonic-gate 			    "%s: couldn't reregister %ld for wait\n",
613*7c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri, start_pid);
614*7c478bd9Sstevel@tonic-gate 		} else if (ret == 1) {
615*7c478bd9Sstevel@tonic-gate 			/*
616*7c478bd9Sstevel@tonic-gate 			 * Leading PID has exited.
617*7c478bd9Sstevel@tonic-gate 			 */
618*7c478bd9Sstevel@tonic-gate 			(void) stop_instance(h, inst, RSTOP_EXIT);
619*7c478bd9Sstevel@tonic-gate 		}
620*7c478bd9Sstevel@tonic-gate 	}
621*7c478bd9Sstevel@tonic-gate 
622*7c478bd9Sstevel@tonic-gate 
623*7c478bd9Sstevel@tonic-gate 	scf_pg_destroy(pg);
624*7c478bd9Sstevel@tonic-gate 
625*7c478bd9Sstevel@tonic-gate 	if (do_commit_states)
626*7c478bd9Sstevel@tonic-gate 		(void) restarter_instance_update_states(h, inst, state,
627*7c478bd9Sstevel@tonic-gate 		    next_state, RERR_NONE, NULL);
628*7c478bd9Sstevel@tonic-gate 
629*7c478bd9Sstevel@tonic-gate 	(void) snprintf(logfilebuf, PATH_MAX, "%s:%s", svc_name, inst_name);
630*7c478bd9Sstevel@tonic-gate 	for (c = logfilebuf; *c != '\0'; c++)
631*7c478bd9Sstevel@tonic-gate 		if (*c == '/')
632*7c478bd9Sstevel@tonic-gate 			*c = '-';
633*7c478bd9Sstevel@tonic-gate 
634*7c478bd9Sstevel@tonic-gate 	if ((inst->ri_logstem = uu_msprintf("%s%s", logfilebuf, LOG_SUFFIX)) ==
635*7c478bd9Sstevel@tonic-gate 	    NULL)
636*7c478bd9Sstevel@tonic-gate 		uu_die("Allocation failure\n");
637*7c478bd9Sstevel@tonic-gate 
638*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s is a %s-style service\n", name,
639*7c478bd9Sstevel@tonic-gate 	    service_style(inst->ri_flags));
640*7c478bd9Sstevel@tonic-gate 
641*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&inst->ri_queue_lock);
642*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&inst->ri_lock);
643*7c478bd9Sstevel@tonic-gate 
644*7c478bd9Sstevel@tonic-gate 	startd_free(svc_name, max_scf_name_size);
645*7c478bd9Sstevel@tonic-gate 	startd_free(inst_name, max_scf_name_size);
646*7c478bd9Sstevel@tonic-gate 	scf_snapshot_destroy(snap);
647*7c478bd9Sstevel@tonic-gate 	scf_instance_destroy(scf_inst);
648*7c478bd9Sstevel@tonic-gate 	scf_service_destroy(scf_svc);
649*7c478bd9Sstevel@tonic-gate 
650*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: inserted instance into restarter list\n",
651*7c478bd9Sstevel@tonic-gate 	    name);
652*7c478bd9Sstevel@tonic-gate 
653*7c478bd9Sstevel@tonic-gate 	return (0);
654*7c478bd9Sstevel@tonic-gate }
655*7c478bd9Sstevel@tonic-gate 
656*7c478bd9Sstevel@tonic-gate static void
657*7c478bd9Sstevel@tonic-gate restarter_delete_inst(restarter_inst_t *ri)
658*7c478bd9Sstevel@tonic-gate {
659*7c478bd9Sstevel@tonic-gate 	int id;
660*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *rip;
661*7c478bd9Sstevel@tonic-gate 	void *cookie = NULL;
662*7c478bd9Sstevel@tonic-gate 	restarter_instance_qentry_t *e;
663*7c478bd9Sstevel@tonic-gate 
664*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&ri->ri_lock));
665*7c478bd9Sstevel@tonic-gate 
666*7c478bd9Sstevel@tonic-gate 	/*
667*7c478bd9Sstevel@tonic-gate 	 * Must drop the instance lock so we can pick up the instance_list
668*7c478bd9Sstevel@tonic-gate 	 * lock & remove the instance.
669*7c478bd9Sstevel@tonic-gate 	 */
670*7c478bd9Sstevel@tonic-gate 	id = ri->ri_id;
671*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&ri->ri_lock);
672*7c478bd9Sstevel@tonic-gate 
673*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&instance_list.ril_lock);
674*7c478bd9Sstevel@tonic-gate 
675*7c478bd9Sstevel@tonic-gate 	rip = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
676*7c478bd9Sstevel@tonic-gate 	if (rip == NULL) {
677*7c478bd9Sstevel@tonic-gate 		MUTEX_UNLOCK(&instance_list.ril_lock);
678*7c478bd9Sstevel@tonic-gate 		return;
679*7c478bd9Sstevel@tonic-gate 	}
680*7c478bd9Sstevel@tonic-gate 
681*7c478bd9Sstevel@tonic-gate 	assert(ri == rip);
682*7c478bd9Sstevel@tonic-gate 
683*7c478bd9Sstevel@tonic-gate 	uu_list_remove(instance_list.ril_instance_list, ri);
684*7c478bd9Sstevel@tonic-gate 
685*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: deleted instance from restarter list\n",
686*7c478bd9Sstevel@tonic-gate 	    ri->ri_i.i_fmri);
687*7c478bd9Sstevel@tonic-gate 
688*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&instance_list.ril_lock);
689*7c478bd9Sstevel@tonic-gate 
690*7c478bd9Sstevel@tonic-gate 	/*
691*7c478bd9Sstevel@tonic-gate 	 * We can lock the instance without holding the instance_list lock
692*7c478bd9Sstevel@tonic-gate 	 * since we removed the instance from the list.
693*7c478bd9Sstevel@tonic-gate 	 */
694*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&ri->ri_lock);
695*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&ri->ri_queue_lock);
696*7c478bd9Sstevel@tonic-gate 
697*7c478bd9Sstevel@tonic-gate 	if (ri->ri_i.i_primary_ctid >= 1)
698*7c478bd9Sstevel@tonic-gate 		contract_hash_remove(ri->ri_i.i_primary_ctid);
699*7c478bd9Sstevel@tonic-gate 
700*7c478bd9Sstevel@tonic-gate 	while (ri->ri_method_thread != 0 || ri->ri_method_waiters > 0)
701*7c478bd9Sstevel@tonic-gate 		(void) pthread_cond_wait(&ri->ri_method_cv, &ri->ri_lock);
702*7c478bd9Sstevel@tonic-gate 
703*7c478bd9Sstevel@tonic-gate 	while ((e = uu_list_teardown(ri->ri_queue, &cookie)) != NULL)
704*7c478bd9Sstevel@tonic-gate 		startd_free(e, sizeof (*e));
705*7c478bd9Sstevel@tonic-gate 	uu_list_destroy(ri->ri_queue);
706*7c478bd9Sstevel@tonic-gate 
707*7c478bd9Sstevel@tonic-gate 	startd_free((void *)ri->ri_i.i_fmri, strlen(ri->ri_i.i_fmri) + 1);
708*7c478bd9Sstevel@tonic-gate 	free(ri->ri_logstem);
709*7c478bd9Sstevel@tonic-gate 	startd_free(ri->ri_utmpx_prefix, max_scf_value_size);
710*7c478bd9Sstevel@tonic-gate 	(void) pthread_mutex_destroy(&ri->ri_lock);
711*7c478bd9Sstevel@tonic-gate 	(void) pthread_mutex_destroy(&ri->ri_queue_lock);
712*7c478bd9Sstevel@tonic-gate 	startd_free(ri, sizeof (restarter_inst_t));
713*7c478bd9Sstevel@tonic-gate }
714*7c478bd9Sstevel@tonic-gate 
715*7c478bd9Sstevel@tonic-gate /*
716*7c478bd9Sstevel@tonic-gate  * instance_is_wait_style()
717*7c478bd9Sstevel@tonic-gate  *
718*7c478bd9Sstevel@tonic-gate  *   Returns 1 if the given instance is a "wait-style" service instance.
719*7c478bd9Sstevel@tonic-gate  */
720*7c478bd9Sstevel@tonic-gate int
721*7c478bd9Sstevel@tonic-gate instance_is_wait_style(restarter_inst_t *inst)
722*7c478bd9Sstevel@tonic-gate {
723*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
724*7c478bd9Sstevel@tonic-gate 	return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT);
725*7c478bd9Sstevel@tonic-gate }
726*7c478bd9Sstevel@tonic-gate 
727*7c478bd9Sstevel@tonic-gate /*
728*7c478bd9Sstevel@tonic-gate  * instance_is_transient_style()
729*7c478bd9Sstevel@tonic-gate  *
730*7c478bd9Sstevel@tonic-gate  *   Returns 1 if the given instance is a transient service instance.
731*7c478bd9Sstevel@tonic-gate  */
732*7c478bd9Sstevel@tonic-gate int
733*7c478bd9Sstevel@tonic-gate instance_is_transient_style(restarter_inst_t *inst)
734*7c478bd9Sstevel@tonic-gate {
735*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
736*7c478bd9Sstevel@tonic-gate 	return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_TRANSIENT);
737*7c478bd9Sstevel@tonic-gate }
738*7c478bd9Sstevel@tonic-gate 
739*7c478bd9Sstevel@tonic-gate /*
740*7c478bd9Sstevel@tonic-gate  * instance_in_transition()
741*7c478bd9Sstevel@tonic-gate  * Returns 1 if instance is in transition, 0 if not
742*7c478bd9Sstevel@tonic-gate  */
743*7c478bd9Sstevel@tonic-gate int
744*7c478bd9Sstevel@tonic-gate instance_in_transition(restarter_inst_t *inst)
745*7c478bd9Sstevel@tonic-gate {
746*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
747*7c478bd9Sstevel@tonic-gate 	if (inst->ri_i.i_next_state == RESTARTER_STATE_NONE)
748*7c478bd9Sstevel@tonic-gate 		return (0);
749*7c478bd9Sstevel@tonic-gate 	return (1);
750*7c478bd9Sstevel@tonic-gate }
751*7c478bd9Sstevel@tonic-gate 
752*7c478bd9Sstevel@tonic-gate /*
753*7c478bd9Sstevel@tonic-gate  * Returns
754*7c478bd9Sstevel@tonic-gate  *   0 - success
755*7c478bd9Sstevel@tonic-gate  *   ECONNRESET - success, but h was rebound
756*7c478bd9Sstevel@tonic-gate  */
757*7c478bd9Sstevel@tonic-gate int
758*7c478bd9Sstevel@tonic-gate restarter_instance_update_states(scf_handle_t *h, restarter_inst_t *ri,
759*7c478bd9Sstevel@tonic-gate     restarter_instance_state_t new_state,
760*7c478bd9Sstevel@tonic-gate     restarter_instance_state_t new_state_next, restarter_error_t err, char *aux)
761*7c478bd9Sstevel@tonic-gate {
762*7c478bd9Sstevel@tonic-gate 	protocol_states_t *states;
763*7c478bd9Sstevel@tonic-gate 	int e;
764*7c478bd9Sstevel@tonic-gate 	uint_t retry_count = 0, msecs = ALLOC_DELAY;
765*7c478bd9Sstevel@tonic-gate 	boolean_t rebound = B_FALSE;
766*7c478bd9Sstevel@tonic-gate 
767*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&ri->ri_lock));
768*7c478bd9Sstevel@tonic-gate 
769*7c478bd9Sstevel@tonic-gate retry:
770*7c478bd9Sstevel@tonic-gate 	e = _restarter_commit_states(h, &ri->ri_i, new_state, new_state_next,
771*7c478bd9Sstevel@tonic-gate 	    aux);
772*7c478bd9Sstevel@tonic-gate 	switch (e) {
773*7c478bd9Sstevel@tonic-gate 	case 0:
774*7c478bd9Sstevel@tonic-gate 		break;
775*7c478bd9Sstevel@tonic-gate 
776*7c478bd9Sstevel@tonic-gate 	case ENOMEM:
777*7c478bd9Sstevel@tonic-gate 		++retry_count;
778*7c478bd9Sstevel@tonic-gate 		if (retry_count < ALLOC_RETRY) {
779*7c478bd9Sstevel@tonic-gate 			(void) poll(NULL, 0, msecs);
780*7c478bd9Sstevel@tonic-gate 			msecs *= ALLOC_DELAY_MULT;
781*7c478bd9Sstevel@tonic-gate 			goto retry;
782*7c478bd9Sstevel@tonic-gate 		}
783*7c478bd9Sstevel@tonic-gate 
784*7c478bd9Sstevel@tonic-gate 		/* Like startd_alloc(). */
785*7c478bd9Sstevel@tonic-gate 		uu_die("Insufficient memory.\n");
786*7c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
787*7c478bd9Sstevel@tonic-gate 
788*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
789*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
790*7c478bd9Sstevel@tonic-gate 		rebound = B_TRUE;
791*7c478bd9Sstevel@tonic-gate 		goto retry;
792*7c478bd9Sstevel@tonic-gate 
793*7c478bd9Sstevel@tonic-gate 	case EPERM:
794*7c478bd9Sstevel@tonic-gate 	case EACCES:
795*7c478bd9Sstevel@tonic-gate 	case EROFS:
796*7c478bd9Sstevel@tonic-gate 		log_error(LOG_NOTICE, "Could not commit state change for %s "
797*7c478bd9Sstevel@tonic-gate 		    "to repository: %s.\n", ri->ri_i.i_fmri, strerror(e));
798*7c478bd9Sstevel@tonic-gate 		/* FALLTHROUGH */
799*7c478bd9Sstevel@tonic-gate 
800*7c478bd9Sstevel@tonic-gate 	case ENOENT:
801*7c478bd9Sstevel@tonic-gate 		ri->ri_i.i_state = new_state;
802*7c478bd9Sstevel@tonic-gate 		ri->ri_i.i_next_state = new_state_next;
803*7c478bd9Sstevel@tonic-gate 		break;
804*7c478bd9Sstevel@tonic-gate 
805*7c478bd9Sstevel@tonic-gate 	case EINVAL:
806*7c478bd9Sstevel@tonic-gate 	default:
807*7c478bd9Sstevel@tonic-gate 		bad_error("_restarter_commit_states", e);
808*7c478bd9Sstevel@tonic-gate 	}
809*7c478bd9Sstevel@tonic-gate 
810*7c478bd9Sstevel@tonic-gate 	states = startd_alloc(sizeof (protocol_states_t));
811*7c478bd9Sstevel@tonic-gate 	states->ps_state = new_state;
812*7c478bd9Sstevel@tonic-gate 	states->ps_state_next = new_state_next;
813*7c478bd9Sstevel@tonic-gate 	states->ps_err = err;
814*7c478bd9Sstevel@tonic-gate 	graph_protocol_send_event(ri->ri_i.i_fmri, GRAPH_UPDATE_STATE_CHANGE,
815*7c478bd9Sstevel@tonic-gate 	    (void *)states);
816*7c478bd9Sstevel@tonic-gate 
817*7c478bd9Sstevel@tonic-gate 	if (new_state == RESTARTER_STATE_ONLINE)
818*7c478bd9Sstevel@tonic-gate 		ri->ri_post_online_hook();
819*7c478bd9Sstevel@tonic-gate 
820*7c478bd9Sstevel@tonic-gate 	return (rebound ? ECONNRESET : 0);
821*7c478bd9Sstevel@tonic-gate }
822*7c478bd9Sstevel@tonic-gate 
823*7c478bd9Sstevel@tonic-gate void
824*7c478bd9Sstevel@tonic-gate restarter_mark_pending_snapshot(const char *fmri, uint_t flag)
825*7c478bd9Sstevel@tonic-gate {
826*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst;
827*7c478bd9Sstevel@tonic-gate 
828*7c478bd9Sstevel@tonic-gate 	assert(flag == RINST_RETAKE_RUNNING || flag == RINST_RETAKE_START);
829*7c478bd9Sstevel@tonic-gate 
830*7c478bd9Sstevel@tonic-gate 	inst = inst_lookup_by_name(fmri);
831*7c478bd9Sstevel@tonic-gate 	if (inst == NULL)
832*7c478bd9Sstevel@tonic-gate 		return;
833*7c478bd9Sstevel@tonic-gate 
834*7c478bd9Sstevel@tonic-gate 	inst->ri_flags |= flag;
835*7c478bd9Sstevel@tonic-gate 
836*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&inst->ri_lock);
837*7c478bd9Sstevel@tonic-gate }
838*7c478bd9Sstevel@tonic-gate 
839*7c478bd9Sstevel@tonic-gate static void
840*7c478bd9Sstevel@tonic-gate restarter_take_pending_snapshots(scf_handle_t *h)
841*7c478bd9Sstevel@tonic-gate {
842*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst;
843*7c478bd9Sstevel@tonic-gate 	int r;
844*7c478bd9Sstevel@tonic-gate 
845*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&instance_list.ril_lock);
846*7c478bd9Sstevel@tonic-gate 
847*7c478bd9Sstevel@tonic-gate 	for (inst = uu_list_first(instance_list.ril_instance_list);
848*7c478bd9Sstevel@tonic-gate 	    inst != NULL;
849*7c478bd9Sstevel@tonic-gate 	    inst = uu_list_next(instance_list.ril_instance_list, inst)) {
850*7c478bd9Sstevel@tonic-gate 		const char *fmri;
851*7c478bd9Sstevel@tonic-gate 		scf_instance_t *sinst = NULL;
852*7c478bd9Sstevel@tonic-gate 
853*7c478bd9Sstevel@tonic-gate 		MUTEX_LOCK(&inst->ri_lock);
854*7c478bd9Sstevel@tonic-gate 
855*7c478bd9Sstevel@tonic-gate 		/*
856*7c478bd9Sstevel@tonic-gate 		 * This is where we'd check inst->ri_method_thread and if it
857*7c478bd9Sstevel@tonic-gate 		 * were nonzero we'd wait in anticipation of another thread
858*7c478bd9Sstevel@tonic-gate 		 * executing a method for inst.  Doing so with the instance_list
859*7c478bd9Sstevel@tonic-gate 		 * locked, though, leads to deadlock.  Since taking a snapshot
860*7c478bd9Sstevel@tonic-gate 		 * during that window won't hurt anything, we'll just continue.
861*7c478bd9Sstevel@tonic-gate 		 */
862*7c478bd9Sstevel@tonic-gate 
863*7c478bd9Sstevel@tonic-gate 		fmri = inst->ri_i.i_fmri;
864*7c478bd9Sstevel@tonic-gate 
865*7c478bd9Sstevel@tonic-gate 		if (inst->ri_flags & RINST_RETAKE_RUNNING) {
866*7c478bd9Sstevel@tonic-gate 			scf_snapshot_t *rsnap;
867*7c478bd9Sstevel@tonic-gate 
868*7c478bd9Sstevel@tonic-gate 			(void) libscf_fmri_get_instance(h, fmri, &sinst);
869*7c478bd9Sstevel@tonic-gate 
870*7c478bd9Sstevel@tonic-gate 			rsnap = libscf_get_or_make_running_snapshot(sinst,
871*7c478bd9Sstevel@tonic-gate 			    fmri, B_FALSE);
872*7c478bd9Sstevel@tonic-gate 
873*7c478bd9Sstevel@tonic-gate 			scf_instance_destroy(sinst);
874*7c478bd9Sstevel@tonic-gate 
875*7c478bd9Sstevel@tonic-gate 			if (rsnap != NULL)
876*7c478bd9Sstevel@tonic-gate 				inst->ri_flags &= ~RINST_RETAKE_RUNNING;
877*7c478bd9Sstevel@tonic-gate 
878*7c478bd9Sstevel@tonic-gate 			scf_snapshot_destroy(rsnap);
879*7c478bd9Sstevel@tonic-gate 		}
880*7c478bd9Sstevel@tonic-gate 
881*7c478bd9Sstevel@tonic-gate 		if (inst->ri_flags & RINST_RETAKE_START) {
882*7c478bd9Sstevel@tonic-gate 			switch (r = libscf_snapshots_poststart(h, fmri,
883*7c478bd9Sstevel@tonic-gate 			    B_FALSE)) {
884*7c478bd9Sstevel@tonic-gate 			case 0:
885*7c478bd9Sstevel@tonic-gate 			case ENOENT:
886*7c478bd9Sstevel@tonic-gate 				inst->ri_flags &= ~RINST_RETAKE_START;
887*7c478bd9Sstevel@tonic-gate 				break;
888*7c478bd9Sstevel@tonic-gate 
889*7c478bd9Sstevel@tonic-gate 			case ECONNABORTED:
890*7c478bd9Sstevel@tonic-gate 				break;
891*7c478bd9Sstevel@tonic-gate 
892*7c478bd9Sstevel@tonic-gate 			case EACCES:
893*7c478bd9Sstevel@tonic-gate 			default:
894*7c478bd9Sstevel@tonic-gate 				bad_error("libscf_snapshots_poststart", r);
895*7c478bd9Sstevel@tonic-gate 			}
896*7c478bd9Sstevel@tonic-gate 		}
897*7c478bd9Sstevel@tonic-gate 
898*7c478bd9Sstevel@tonic-gate 		MUTEX_UNLOCK(&inst->ri_lock);
899*7c478bd9Sstevel@tonic-gate 	}
900*7c478bd9Sstevel@tonic-gate 
901*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&instance_list.ril_lock);
902*7c478bd9Sstevel@tonic-gate }
903*7c478bd9Sstevel@tonic-gate 
904*7c478bd9Sstevel@tonic-gate /* ARGSUSED */
905*7c478bd9Sstevel@tonic-gate void *
906*7c478bd9Sstevel@tonic-gate restarter_post_fsminimal_thread(void *unused)
907*7c478bd9Sstevel@tonic-gate {
908*7c478bd9Sstevel@tonic-gate 	scf_handle_t *h;
909*7c478bd9Sstevel@tonic-gate 	int r;
910*7c478bd9Sstevel@tonic-gate 
911*7c478bd9Sstevel@tonic-gate 	h = libscf_handle_create_bound_loop();
912*7c478bd9Sstevel@tonic-gate 
913*7c478bd9Sstevel@tonic-gate 	for (;;) {
914*7c478bd9Sstevel@tonic-gate 		r = libscf_create_self(h);
915*7c478bd9Sstevel@tonic-gate 		if (r == 0)
916*7c478bd9Sstevel@tonic-gate 			break;
917*7c478bd9Sstevel@tonic-gate 
918*7c478bd9Sstevel@tonic-gate 		assert(r == ECONNABORTED);
919*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
920*7c478bd9Sstevel@tonic-gate 	}
921*7c478bd9Sstevel@tonic-gate 
922*7c478bd9Sstevel@tonic-gate 	restarter_take_pending_snapshots(h);
923*7c478bd9Sstevel@tonic-gate 
924*7c478bd9Sstevel@tonic-gate 	(void) scf_handle_unbind(h);
925*7c478bd9Sstevel@tonic-gate 	scf_handle_destroy(h);
926*7c478bd9Sstevel@tonic-gate 
927*7c478bd9Sstevel@tonic-gate 	return (NULL);
928*7c478bd9Sstevel@tonic-gate }
929*7c478bd9Sstevel@tonic-gate 
930*7c478bd9Sstevel@tonic-gate /*
931*7c478bd9Sstevel@tonic-gate  * returns 1 if instance is already started, 0 if not
932*7c478bd9Sstevel@tonic-gate  */
933*7c478bd9Sstevel@tonic-gate static int
934*7c478bd9Sstevel@tonic-gate instance_started(restarter_inst_t *inst)
935*7c478bd9Sstevel@tonic-gate {
936*7c478bd9Sstevel@tonic-gate 	int ret;
937*7c478bd9Sstevel@tonic-gate 
938*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
939*7c478bd9Sstevel@tonic-gate 
940*7c478bd9Sstevel@tonic-gate 	if (inst->ri_i.i_state == RESTARTER_STATE_ONLINE ||
941*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_state == RESTARTER_STATE_DEGRADED)
942*7c478bd9Sstevel@tonic-gate 		ret = 1;
943*7c478bd9Sstevel@tonic-gate 	else
944*7c478bd9Sstevel@tonic-gate 		ret = 0;
945*7c478bd9Sstevel@tonic-gate 
946*7c478bd9Sstevel@tonic-gate 	return (ret);
947*7c478bd9Sstevel@tonic-gate }
948*7c478bd9Sstevel@tonic-gate 
949*7c478bd9Sstevel@tonic-gate /*
950*7c478bd9Sstevel@tonic-gate  * int stop_instance()
951*7c478bd9Sstevel@tonic-gate  *
952*7c478bd9Sstevel@tonic-gate  *   Stop the instance identified by the instance given as the second argument,
953*7c478bd9Sstevel@tonic-gate  *   for the cause stated.
954*7c478bd9Sstevel@tonic-gate  *
955*7c478bd9Sstevel@tonic-gate  *   Returns
956*7c478bd9Sstevel@tonic-gate  *     0 - success
957*7c478bd9Sstevel@tonic-gate  *     -1 - inst is in transition
958*7c478bd9Sstevel@tonic-gate  */
959*7c478bd9Sstevel@tonic-gate static int
960*7c478bd9Sstevel@tonic-gate stop_instance(scf_handle_t *local_handle, restarter_inst_t *inst,
961*7c478bd9Sstevel@tonic-gate     stop_cause_t cause)
962*7c478bd9Sstevel@tonic-gate {
963*7c478bd9Sstevel@tonic-gate 	fork_info_t *info;
964*7c478bd9Sstevel@tonic-gate 	const char *cp;
965*7c478bd9Sstevel@tonic-gate 	int err;
966*7c478bd9Sstevel@tonic-gate 	restarter_error_t re;
967*7c478bd9Sstevel@tonic-gate 
968*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
969*7c478bd9Sstevel@tonic-gate 	assert(inst->ri_method_thread == 0);
970*7c478bd9Sstevel@tonic-gate 
971*7c478bd9Sstevel@tonic-gate 	switch (cause) {
972*7c478bd9Sstevel@tonic-gate 	case RSTOP_EXIT:
973*7c478bd9Sstevel@tonic-gate 		re = RERR_RESTART;
974*7c478bd9Sstevel@tonic-gate 		cp = "all processes in service exited";
975*7c478bd9Sstevel@tonic-gate 		break;
976*7c478bd9Sstevel@tonic-gate 	case RSTOP_CORE:
977*7c478bd9Sstevel@tonic-gate 		re = RERR_FAULT;
978*7c478bd9Sstevel@tonic-gate 		cp = "process dumped core";
979*7c478bd9Sstevel@tonic-gate 		break;
980*7c478bd9Sstevel@tonic-gate 	case RSTOP_SIGNAL:
981*7c478bd9Sstevel@tonic-gate 		re = RERR_FAULT;
982*7c478bd9Sstevel@tonic-gate 		cp = "process received fatal signal from outside the service";
983*7c478bd9Sstevel@tonic-gate 		break;
984*7c478bd9Sstevel@tonic-gate 	case RSTOP_HWERR:
985*7c478bd9Sstevel@tonic-gate 		re = RERR_FAULT;
986*7c478bd9Sstevel@tonic-gate 		cp = "process killed due to uncorrectable hardware error";
987*7c478bd9Sstevel@tonic-gate 		break;
988*7c478bd9Sstevel@tonic-gate 	case RSTOP_DEPENDENCY:
989*7c478bd9Sstevel@tonic-gate 		re = RERR_RESTART;
990*7c478bd9Sstevel@tonic-gate 		cp = "dependency activity requires stop";
991*7c478bd9Sstevel@tonic-gate 		break;
992*7c478bd9Sstevel@tonic-gate 	case RSTOP_DISABLE:
993*7c478bd9Sstevel@tonic-gate 		re = RERR_RESTART;
994*7c478bd9Sstevel@tonic-gate 		cp = "service disabled";
995*7c478bd9Sstevel@tonic-gate 		break;
996*7c478bd9Sstevel@tonic-gate 	case RSTOP_RESTART:
997*7c478bd9Sstevel@tonic-gate 		re = RERR_RESTART;
998*7c478bd9Sstevel@tonic-gate 		cp = "service restarting";
999*7c478bd9Sstevel@tonic-gate 		break;
1000*7c478bd9Sstevel@tonic-gate 	default:
1001*7c478bd9Sstevel@tonic-gate #ifndef NDEBUG
1002*7c478bd9Sstevel@tonic-gate 		(void) fprintf(stderr, "Unknown cause %d at %s:%d.\n",
1003*7c478bd9Sstevel@tonic-gate 		    cause, __FILE__, __LINE__);
1004*7c478bd9Sstevel@tonic-gate #endif
1005*7c478bd9Sstevel@tonic-gate 		abort();
1006*7c478bd9Sstevel@tonic-gate 	}
1007*7c478bd9Sstevel@tonic-gate 
1008*7c478bd9Sstevel@tonic-gate 	/* Services in the disabled and maintenance state are ignored */
1009*7c478bd9Sstevel@tonic-gate 	if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1010*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_state == RESTARTER_STATE_DISABLED) {
1011*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
1012*7c478bd9Sstevel@tonic-gate 		    "%s: stop_instance -> is maint/disabled\n",
1013*7c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri);
1014*7c478bd9Sstevel@tonic-gate 		return (0);
1015*7c478bd9Sstevel@tonic-gate 	}
1016*7c478bd9Sstevel@tonic-gate 
1017*7c478bd9Sstevel@tonic-gate 	/* Already stopped instances are left alone */
1018*7c478bd9Sstevel@tonic-gate 	if (instance_started(inst) == 0) {
1019*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG, "Restarter: %s is already stopped.\n",
1020*7c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri);
1021*7c478bd9Sstevel@tonic-gate 		return (0);
1022*7c478bd9Sstevel@tonic-gate 	}
1023*7c478bd9Sstevel@tonic-gate 
1024*7c478bd9Sstevel@tonic-gate 	if (instance_in_transition(inst)) {
1025*7c478bd9Sstevel@tonic-gate 		/* requeue event by returning -1 */
1026*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
1027*7c478bd9Sstevel@tonic-gate 		    "Restarter: Not stopping %s, in transition.\n",
1028*7c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri);
1029*7c478bd9Sstevel@tonic-gate 		return (-1);
1030*7c478bd9Sstevel@tonic-gate 	}
1031*7c478bd9Sstevel@tonic-gate 
1032*7c478bd9Sstevel@tonic-gate 	log_instance(inst, B_TRUE, "Stopping because %s.", cp);
1033*7c478bd9Sstevel@tonic-gate 
1034*7c478bd9Sstevel@tonic-gate 	log_framework(re == RERR_FAULT ? LOG_INFO : LOG_DEBUG,
1035*7c478bd9Sstevel@tonic-gate 	    "%s: Instance stopping because %s.\n", inst->ri_i.i_fmri, cp);
1036*7c478bd9Sstevel@tonic-gate 
1037*7c478bd9Sstevel@tonic-gate 	if (instance_is_wait_style(inst) && cause == RSTOP_EXIT) {
1038*7c478bd9Sstevel@tonic-gate 		/*
1039*7c478bd9Sstevel@tonic-gate 		 * No need to stop instance, as child has exited; remove
1040*7c478bd9Sstevel@tonic-gate 		 * contract and move the instance to the offline state.
1041*7c478bd9Sstevel@tonic-gate 		 */
1042*7c478bd9Sstevel@tonic-gate 		switch (err = restarter_instance_update_states(local_handle,
1043*7c478bd9Sstevel@tonic-gate 		    inst, inst->ri_i.i_state, RESTARTER_STATE_OFFLINE, re,
1044*7c478bd9Sstevel@tonic-gate 		    NULL)) {
1045*7c478bd9Sstevel@tonic-gate 		case 0:
1046*7c478bd9Sstevel@tonic-gate 		case ECONNRESET:
1047*7c478bd9Sstevel@tonic-gate 			break;
1048*7c478bd9Sstevel@tonic-gate 
1049*7c478bd9Sstevel@tonic-gate 		default:
1050*7c478bd9Sstevel@tonic-gate 			bad_error("restarter_instance_update_states", err);
1051*7c478bd9Sstevel@tonic-gate 		}
1052*7c478bd9Sstevel@tonic-gate 
1053*7c478bd9Sstevel@tonic-gate 		(void) update_fault_count(inst, FAULT_COUNT_RESET);
1054*7c478bd9Sstevel@tonic-gate 
1055*7c478bd9Sstevel@tonic-gate 		if (inst->ri_i.i_primary_ctid != 0) {
1056*7c478bd9Sstevel@tonic-gate 			inst->ri_m_inst =
1057*7c478bd9Sstevel@tonic-gate 			    safe_scf_instance_create(local_handle);
1058*7c478bd9Sstevel@tonic-gate 			inst->ri_mi_deleted = B_FALSE;
1059*7c478bd9Sstevel@tonic-gate 
1060*7c478bd9Sstevel@tonic-gate 			libscf_reget_instance(inst);
1061*7c478bd9Sstevel@tonic-gate 			method_remove_contract(inst, B_TRUE, B_TRUE);
1062*7c478bd9Sstevel@tonic-gate 
1063*7c478bd9Sstevel@tonic-gate 			scf_instance_destroy(inst->ri_m_inst);
1064*7c478bd9Sstevel@tonic-gate 			inst->ri_m_inst = NULL;
1065*7c478bd9Sstevel@tonic-gate 		}
1066*7c478bd9Sstevel@tonic-gate 
1067*7c478bd9Sstevel@tonic-gate 		switch (err = restarter_instance_update_states(local_handle,
1068*7c478bd9Sstevel@tonic-gate 		    inst, inst->ri_i.i_next_state, RESTARTER_STATE_NONE, re,
1069*7c478bd9Sstevel@tonic-gate 		    NULL)) {
1070*7c478bd9Sstevel@tonic-gate 		case 0:
1071*7c478bd9Sstevel@tonic-gate 		case ECONNRESET:
1072*7c478bd9Sstevel@tonic-gate 			break;
1073*7c478bd9Sstevel@tonic-gate 
1074*7c478bd9Sstevel@tonic-gate 		default:
1075*7c478bd9Sstevel@tonic-gate 			bad_error("restarter_instance_update_states", err);
1076*7c478bd9Sstevel@tonic-gate 		}
1077*7c478bd9Sstevel@tonic-gate 
1078*7c478bd9Sstevel@tonic-gate 		return (0);
1079*7c478bd9Sstevel@tonic-gate 	}
1080*7c478bd9Sstevel@tonic-gate 
1081*7c478bd9Sstevel@tonic-gate 	switch (err = restarter_instance_update_states(local_handle, inst,
1082*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_state, inst->ri_i.i_enabled ? RESTARTER_STATE_OFFLINE :
1083*7c478bd9Sstevel@tonic-gate 	    RESTARTER_STATE_DISABLED, RERR_NONE, NULL)) {
1084*7c478bd9Sstevel@tonic-gate 	case 0:
1085*7c478bd9Sstevel@tonic-gate 	case ECONNRESET:
1086*7c478bd9Sstevel@tonic-gate 		break;
1087*7c478bd9Sstevel@tonic-gate 
1088*7c478bd9Sstevel@tonic-gate 	default:
1089*7c478bd9Sstevel@tonic-gate 		bad_error("restarter_instance_update_states", err);
1090*7c478bd9Sstevel@tonic-gate 	}
1091*7c478bd9Sstevel@tonic-gate 
1092*7c478bd9Sstevel@tonic-gate 	info = startd_zalloc(sizeof (fork_info_t));
1093*7c478bd9Sstevel@tonic-gate 
1094*7c478bd9Sstevel@tonic-gate 	info->sf_id = inst->ri_id;
1095*7c478bd9Sstevel@tonic-gate 	info->sf_method_type = METHOD_STOP;
1096*7c478bd9Sstevel@tonic-gate 	info->sf_event_type = re;
1097*7c478bd9Sstevel@tonic-gate 	inst->ri_method_thread = startd_thread_create(method_thread, info);
1098*7c478bd9Sstevel@tonic-gate 
1099*7c478bd9Sstevel@tonic-gate 	return (0);
1100*7c478bd9Sstevel@tonic-gate }
1101*7c478bd9Sstevel@tonic-gate 
1102*7c478bd9Sstevel@tonic-gate /*
1103*7c478bd9Sstevel@tonic-gate  * Returns
1104*7c478bd9Sstevel@tonic-gate  *   ENOENT - fmri is not in instance_list
1105*7c478bd9Sstevel@tonic-gate  *   0 - success
1106*7c478bd9Sstevel@tonic-gate  *   ECONNRESET - success, though handle was rebound
1107*7c478bd9Sstevel@tonic-gate  *   -1 - instance is in transition
1108*7c478bd9Sstevel@tonic-gate  */
1109*7c478bd9Sstevel@tonic-gate int
1110*7c478bd9Sstevel@tonic-gate stop_instance_fmri(scf_handle_t *h, const char *fmri, uint_t flags)
1111*7c478bd9Sstevel@tonic-gate {
1112*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *rip;
1113*7c478bd9Sstevel@tonic-gate 	int r;
1114*7c478bd9Sstevel@tonic-gate 
1115*7c478bd9Sstevel@tonic-gate 	rip = inst_lookup_by_name(fmri);
1116*7c478bd9Sstevel@tonic-gate 	if (rip == NULL)
1117*7c478bd9Sstevel@tonic-gate 		return (ENOENT);
1118*7c478bd9Sstevel@tonic-gate 
1119*7c478bd9Sstevel@tonic-gate 	r = stop_instance(h, rip, flags);
1120*7c478bd9Sstevel@tonic-gate 
1121*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&rip->ri_lock);
1122*7c478bd9Sstevel@tonic-gate 
1123*7c478bd9Sstevel@tonic-gate 	return (r);
1124*7c478bd9Sstevel@tonic-gate }
1125*7c478bd9Sstevel@tonic-gate 
1126*7c478bd9Sstevel@tonic-gate static void
1127*7c478bd9Sstevel@tonic-gate unmaintain_instance(scf_handle_t *h, restarter_inst_t *rip,
1128*7c478bd9Sstevel@tonic-gate     unmaint_cause_t cause)
1129*7c478bd9Sstevel@tonic-gate {
1130*7c478bd9Sstevel@tonic-gate 	ctid_t ctid;
1131*7c478bd9Sstevel@tonic-gate 	scf_instance_t *inst;
1132*7c478bd9Sstevel@tonic-gate 	int r;
1133*7c478bd9Sstevel@tonic-gate 	uint_t tries = 0, msecs = ALLOC_DELAY;
1134*7c478bd9Sstevel@tonic-gate 	const char *cp;
1135*7c478bd9Sstevel@tonic-gate 
1136*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&rip->ri_lock));
1137*7c478bd9Sstevel@tonic-gate 
1138*7c478bd9Sstevel@tonic-gate 	if (rip->ri_i.i_state != RESTARTER_STATE_MAINT) {
1139*7c478bd9Sstevel@tonic-gate 		log_error(LOG_DEBUG, "Restarter: "
1140*7c478bd9Sstevel@tonic-gate 		    "Ignoring maintenance off command because %s is not in the "
1141*7c478bd9Sstevel@tonic-gate 		    "maintenance state.\n", rip->ri_i.i_fmri);
1142*7c478bd9Sstevel@tonic-gate 		return;
1143*7c478bd9Sstevel@tonic-gate 	}
1144*7c478bd9Sstevel@tonic-gate 
1145*7c478bd9Sstevel@tonic-gate 	switch (cause) {
1146*7c478bd9Sstevel@tonic-gate 	case RUNMAINT_CLEAR:
1147*7c478bd9Sstevel@tonic-gate 		cp = "clear requested";
1148*7c478bd9Sstevel@tonic-gate 		break;
1149*7c478bd9Sstevel@tonic-gate 	case RUNMAINT_DISABLE:
1150*7c478bd9Sstevel@tonic-gate 		cp = "disable requested";
1151*7c478bd9Sstevel@tonic-gate 		break;
1152*7c478bd9Sstevel@tonic-gate 	default:
1153*7c478bd9Sstevel@tonic-gate #ifndef NDEBUG
1154*7c478bd9Sstevel@tonic-gate 		(void) fprintf(stderr, "Uncaught case for %d at %s:%d.\n",
1155*7c478bd9Sstevel@tonic-gate 		    cause, __FILE__, __LINE__);
1156*7c478bd9Sstevel@tonic-gate #endif
1157*7c478bd9Sstevel@tonic-gate 		abort();
1158*7c478bd9Sstevel@tonic-gate 	}
1159*7c478bd9Sstevel@tonic-gate 
1160*7c478bd9Sstevel@tonic-gate 	log_instance(rip, B_TRUE, "Leaving maintenance because %s.",
1161*7c478bd9Sstevel@tonic-gate 	    cp);
1162*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: Instance leaving maintenance because "
1163*7c478bd9Sstevel@tonic-gate 	    "%s.\n", rip->ri_i.i_fmri, cp);
1164*7c478bd9Sstevel@tonic-gate 
1165*7c478bd9Sstevel@tonic-gate 	(void) restarter_instance_update_states(h, rip, RESTARTER_STATE_UNINIT,
1166*7c478bd9Sstevel@tonic-gate 	    RESTARTER_STATE_NONE, RERR_RESTART, NULL);
1167*7c478bd9Sstevel@tonic-gate 
1168*7c478bd9Sstevel@tonic-gate 	/*
1169*7c478bd9Sstevel@tonic-gate 	 * If we did ADMIN_MAINT_ON_IMMEDIATE, then there might still be
1170*7c478bd9Sstevel@tonic-gate 	 * a primary contract.
1171*7c478bd9Sstevel@tonic-gate 	 */
1172*7c478bd9Sstevel@tonic-gate 	if (rip->ri_i.i_primary_ctid == 0)
1173*7c478bd9Sstevel@tonic-gate 		return;
1174*7c478bd9Sstevel@tonic-gate 
1175*7c478bd9Sstevel@tonic-gate 	ctid = rip->ri_i.i_primary_ctid;
1176*7c478bd9Sstevel@tonic-gate 	contract_abandon(ctid);
1177*7c478bd9Sstevel@tonic-gate 	rip->ri_i.i_primary_ctid = 0;
1178*7c478bd9Sstevel@tonic-gate 
1179*7c478bd9Sstevel@tonic-gate rep_retry:
1180*7c478bd9Sstevel@tonic-gate 	switch (r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst)) {
1181*7c478bd9Sstevel@tonic-gate 	case 0:
1182*7c478bd9Sstevel@tonic-gate 		break;
1183*7c478bd9Sstevel@tonic-gate 
1184*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
1185*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
1186*7c478bd9Sstevel@tonic-gate 		goto rep_retry;
1187*7c478bd9Sstevel@tonic-gate 
1188*7c478bd9Sstevel@tonic-gate 	case ENOENT:
1189*7c478bd9Sstevel@tonic-gate 		/* Must have been deleted. */
1190*7c478bd9Sstevel@tonic-gate 		return;
1191*7c478bd9Sstevel@tonic-gate 
1192*7c478bd9Sstevel@tonic-gate 	case EINVAL:
1193*7c478bd9Sstevel@tonic-gate 	case ENOTSUP:
1194*7c478bd9Sstevel@tonic-gate 	default:
1195*7c478bd9Sstevel@tonic-gate 		bad_error("libscf_handle_rebind", r);
1196*7c478bd9Sstevel@tonic-gate 	}
1197*7c478bd9Sstevel@tonic-gate 
1198*7c478bd9Sstevel@tonic-gate again:
1199*7c478bd9Sstevel@tonic-gate 	r = restarter_remove_contract(inst, ctid, RESTARTER_CONTRACT_PRIMARY);
1200*7c478bd9Sstevel@tonic-gate 	switch (r) {
1201*7c478bd9Sstevel@tonic-gate 	case 0:
1202*7c478bd9Sstevel@tonic-gate 		break;
1203*7c478bd9Sstevel@tonic-gate 
1204*7c478bd9Sstevel@tonic-gate 	case ENOMEM:
1205*7c478bd9Sstevel@tonic-gate 		++tries;
1206*7c478bd9Sstevel@tonic-gate 		if (tries < ALLOC_RETRY) {
1207*7c478bd9Sstevel@tonic-gate 			(void) poll(NULL, 0, msecs);
1208*7c478bd9Sstevel@tonic-gate 			msecs *= ALLOC_DELAY_MULT;
1209*7c478bd9Sstevel@tonic-gate 			goto again;
1210*7c478bd9Sstevel@tonic-gate 		}
1211*7c478bd9Sstevel@tonic-gate 
1212*7c478bd9Sstevel@tonic-gate 		uu_die("Insufficient memory.\n");
1213*7c478bd9Sstevel@tonic-gate 		/* NOTREACHED */
1214*7c478bd9Sstevel@tonic-gate 
1215*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
1216*7c478bd9Sstevel@tonic-gate 		scf_instance_destroy(inst);
1217*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
1218*7c478bd9Sstevel@tonic-gate 		goto rep_retry;
1219*7c478bd9Sstevel@tonic-gate 
1220*7c478bd9Sstevel@tonic-gate 	case ECANCELED:
1221*7c478bd9Sstevel@tonic-gate 		break;
1222*7c478bd9Sstevel@tonic-gate 
1223*7c478bd9Sstevel@tonic-gate 	case EPERM:
1224*7c478bd9Sstevel@tonic-gate 	case EACCES:
1225*7c478bd9Sstevel@tonic-gate 	case EROFS:
1226*7c478bd9Sstevel@tonic-gate 		log_error(LOG_INFO,
1227*7c478bd9Sstevel@tonic-gate 		    "Could not remove contract id %lu for %s (%s).\n", ctid,
1228*7c478bd9Sstevel@tonic-gate 		    rip->ri_i.i_fmri, strerror(r));
1229*7c478bd9Sstevel@tonic-gate 		break;
1230*7c478bd9Sstevel@tonic-gate 
1231*7c478bd9Sstevel@tonic-gate 	case EINVAL:
1232*7c478bd9Sstevel@tonic-gate 	case EBADF:
1233*7c478bd9Sstevel@tonic-gate 	default:
1234*7c478bd9Sstevel@tonic-gate 		bad_error("restarter_remove_contract", r);
1235*7c478bd9Sstevel@tonic-gate 	}
1236*7c478bd9Sstevel@tonic-gate 
1237*7c478bd9Sstevel@tonic-gate 	scf_instance_destroy(inst);
1238*7c478bd9Sstevel@tonic-gate }
1239*7c478bd9Sstevel@tonic-gate 
1240*7c478bd9Sstevel@tonic-gate /*
1241*7c478bd9Sstevel@tonic-gate  * enable_inst()
1242*7c478bd9Sstevel@tonic-gate  *   Set inst->ri_i.i_enabled.  Expects 'e' to be _ENABLE, _DISABLE, or
1243*7c478bd9Sstevel@tonic-gate  *   _ADMIN_DISABLE.  If the event is _ENABLE and inst is uninitialized or
1244*7c478bd9Sstevel@tonic-gate  *   disabled, move it to offline.  If the event is _DISABLE or
1245*7c478bd9Sstevel@tonic-gate  *   _ADMIN_DISABLE, make sure inst will move to disabled.
1246*7c478bd9Sstevel@tonic-gate  *
1247*7c478bd9Sstevel@tonic-gate  *   Returns
1248*7c478bd9Sstevel@tonic-gate  *     0 - success
1249*7c478bd9Sstevel@tonic-gate  *     ECONNRESET - h was rebound
1250*7c478bd9Sstevel@tonic-gate  */
1251*7c478bd9Sstevel@tonic-gate static int
1252*7c478bd9Sstevel@tonic-gate enable_inst(scf_handle_t *h, restarter_inst_t *inst, restarter_event_type_t e)
1253*7c478bd9Sstevel@tonic-gate {
1254*7c478bd9Sstevel@tonic-gate 	restarter_instance_state_t state;
1255*7c478bd9Sstevel@tonic-gate 	int r;
1256*7c478bd9Sstevel@tonic-gate 
1257*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
1258*7c478bd9Sstevel@tonic-gate 	assert(e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE ||
1259*7c478bd9Sstevel@tonic-gate 	    e == RESTARTER_EVENT_TYPE_DISABLE ||
1260*7c478bd9Sstevel@tonic-gate 	    e == RESTARTER_EVENT_TYPE_ENABLE);
1261*7c478bd9Sstevel@tonic-gate 	assert(instance_in_transition(inst) == 0);
1262*7c478bd9Sstevel@tonic-gate 
1263*7c478bd9Sstevel@tonic-gate 	state = inst->ri_i.i_state;
1264*7c478bd9Sstevel@tonic-gate 
1265*7c478bd9Sstevel@tonic-gate 	if (e == RESTARTER_EVENT_TYPE_ENABLE) {
1266*7c478bd9Sstevel@tonic-gate 		inst->ri_i.i_enabled = 1;
1267*7c478bd9Sstevel@tonic-gate 
1268*7c478bd9Sstevel@tonic-gate 		if (state == RESTARTER_STATE_UNINIT ||
1269*7c478bd9Sstevel@tonic-gate 		    state == RESTARTER_STATE_DISABLED) {
1270*7c478bd9Sstevel@tonic-gate 			/*
1271*7c478bd9Sstevel@tonic-gate 			 * B_FALSE: Don't log an error if the log_instance()
1272*7c478bd9Sstevel@tonic-gate 			 * fails because it will fail on the miniroot before
1273*7c478bd9Sstevel@tonic-gate 			 * install-discovery runs.
1274*7c478bd9Sstevel@tonic-gate 			 */
1275*7c478bd9Sstevel@tonic-gate 			log_instance(inst, B_FALSE, "Enabled.");
1276*7c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG, "%s: Instance enabled.\n",
1277*7c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri);
1278*7c478bd9Sstevel@tonic-gate 			(void) restarter_instance_update_states(h, inst,
1279*7c478bd9Sstevel@tonic-gate 			    RESTARTER_STATE_OFFLINE, RESTARTER_STATE_NONE,
1280*7c478bd9Sstevel@tonic-gate 			    RERR_NONE, NULL);
1281*7c478bd9Sstevel@tonic-gate 		} else {
1282*7c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG, "Restarter: "
1283*7c478bd9Sstevel@tonic-gate 			    "Not changing state of %s for enable command.\n",
1284*7c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri);
1285*7c478bd9Sstevel@tonic-gate 		}
1286*7c478bd9Sstevel@tonic-gate 	} else {
1287*7c478bd9Sstevel@tonic-gate 		inst->ri_i.i_enabled = 0;
1288*7c478bd9Sstevel@tonic-gate 
1289*7c478bd9Sstevel@tonic-gate 		switch (state) {
1290*7c478bd9Sstevel@tonic-gate 		case RESTARTER_STATE_ONLINE:
1291*7c478bd9Sstevel@tonic-gate 		case RESTARTER_STATE_DEGRADED:
1292*7c478bd9Sstevel@tonic-gate 			r = stop_instance(h, inst, RSTOP_DISABLE);
1293*7c478bd9Sstevel@tonic-gate 			return (r == ECONNRESET ? 0 : r);
1294*7c478bd9Sstevel@tonic-gate 
1295*7c478bd9Sstevel@tonic-gate 		case RESTARTER_STATE_OFFLINE:
1296*7c478bd9Sstevel@tonic-gate 		case RESTARTER_STATE_UNINIT:
1297*7c478bd9Sstevel@tonic-gate 			if (inst->ri_i.i_primary_ctid != 0) {
1298*7c478bd9Sstevel@tonic-gate 				inst->ri_m_inst = safe_scf_instance_create(h);
1299*7c478bd9Sstevel@tonic-gate 				inst->ri_mi_deleted = B_FALSE;
1300*7c478bd9Sstevel@tonic-gate 
1301*7c478bd9Sstevel@tonic-gate 				libscf_reget_instance(inst);
1302*7c478bd9Sstevel@tonic-gate 				method_remove_contract(inst, B_TRUE, B_TRUE);
1303*7c478bd9Sstevel@tonic-gate 
1304*7c478bd9Sstevel@tonic-gate 				scf_instance_destroy(inst->ri_m_inst);
1305*7c478bd9Sstevel@tonic-gate 			}
1306*7c478bd9Sstevel@tonic-gate 			/* B_FALSE: See log_instance(..., "Enabled."); above */
1307*7c478bd9Sstevel@tonic-gate 			log_instance(inst, B_FALSE, "Disabled.");
1308*7c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG, "%s: Instance disabled.\n",
1309*7c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri);
1310*7c478bd9Sstevel@tonic-gate 			(void) restarter_instance_update_states(h, inst,
1311*7c478bd9Sstevel@tonic-gate 			    RESTARTER_STATE_DISABLED, RESTARTER_STATE_NONE,
1312*7c478bd9Sstevel@tonic-gate 			    RERR_RESTART, NULL);
1313*7c478bd9Sstevel@tonic-gate 			return (0);
1314*7c478bd9Sstevel@tonic-gate 
1315*7c478bd9Sstevel@tonic-gate 		case RESTARTER_STATE_DISABLED:
1316*7c478bd9Sstevel@tonic-gate 			break;
1317*7c478bd9Sstevel@tonic-gate 
1318*7c478bd9Sstevel@tonic-gate 		case RESTARTER_STATE_MAINT:
1319*7c478bd9Sstevel@tonic-gate 			/*
1320*7c478bd9Sstevel@tonic-gate 			 * We only want to pull the instance out of maintenance
1321*7c478bd9Sstevel@tonic-gate 			 * if the disable is on adminstrative request.  The
1322*7c478bd9Sstevel@tonic-gate 			 * graph engine sends _DISABLE events whenever a
1323*7c478bd9Sstevel@tonic-gate 			 * service isn't in the disabled state, and we don't
1324*7c478bd9Sstevel@tonic-gate 			 * want to pull the service out of maintenance if,
1325*7c478bd9Sstevel@tonic-gate 			 * for example, it is there due to a dependency cycle.
1326*7c478bd9Sstevel@tonic-gate 			 */
1327*7c478bd9Sstevel@tonic-gate 			if (e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE)
1328*7c478bd9Sstevel@tonic-gate 				unmaintain_instance(h, inst, RUNMAINT_DISABLE);
1329*7c478bd9Sstevel@tonic-gate 			break;
1330*7c478bd9Sstevel@tonic-gate 
1331*7c478bd9Sstevel@tonic-gate 		default:
1332*7c478bd9Sstevel@tonic-gate #ifndef NDEBUG
1333*7c478bd9Sstevel@tonic-gate 			(void) fprintf(stderr, "Restarter instance %s has "
1334*7c478bd9Sstevel@tonic-gate 			    "unknown state %d.\n", inst->ri_i.i_fmri, state);
1335*7c478bd9Sstevel@tonic-gate #endif
1336*7c478bd9Sstevel@tonic-gate 			abort();
1337*7c478bd9Sstevel@tonic-gate 		}
1338*7c478bd9Sstevel@tonic-gate 	}
1339*7c478bd9Sstevel@tonic-gate 
1340*7c478bd9Sstevel@tonic-gate 	return (0);
1341*7c478bd9Sstevel@tonic-gate }
1342*7c478bd9Sstevel@tonic-gate 
1343*7c478bd9Sstevel@tonic-gate static void
1344*7c478bd9Sstevel@tonic-gate start_instance(scf_handle_t *local_handle, restarter_inst_t *inst)
1345*7c478bd9Sstevel@tonic-gate {
1346*7c478bd9Sstevel@tonic-gate 	fork_info_t *info;
1347*7c478bd9Sstevel@tonic-gate 
1348*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
1349*7c478bd9Sstevel@tonic-gate 	assert(instance_in_transition(inst) == 0);
1350*7c478bd9Sstevel@tonic-gate 	assert(inst->ri_method_thread == 0);
1351*7c478bd9Sstevel@tonic-gate 
1352*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: trying to start instance\n",
1353*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_fmri);
1354*7c478bd9Sstevel@tonic-gate 
1355*7c478bd9Sstevel@tonic-gate 	/* Services in the disabled and maintenance state are ignored */
1356*7c478bd9Sstevel@tonic-gate 	if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1357*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_state == RESTARTER_STATE_DISABLED ||
1358*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_enabled == 0) {
1359*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
1360*7c478bd9Sstevel@tonic-gate 		    "%s: start_instance -> is maint/disabled\n",
1361*7c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri);
1362*7c478bd9Sstevel@tonic-gate 		return;
1363*7c478bd9Sstevel@tonic-gate 	}
1364*7c478bd9Sstevel@tonic-gate 
1365*7c478bd9Sstevel@tonic-gate 	/* Already started instances are left alone */
1366*7c478bd9Sstevel@tonic-gate 	if (instance_started(inst) == 1) {
1367*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
1368*7c478bd9Sstevel@tonic-gate 		    "%s: start_instance -> is already started\n",
1369*7c478bd9Sstevel@tonic-gate 		    inst->ri_i.i_fmri);
1370*7c478bd9Sstevel@tonic-gate 		return;
1371*7c478bd9Sstevel@tonic-gate 	}
1372*7c478bd9Sstevel@tonic-gate 
1373*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: starting instance.\n", inst->ri_i.i_fmri);
1374*7c478bd9Sstevel@tonic-gate 
1375*7c478bd9Sstevel@tonic-gate 	(void) restarter_instance_update_states(local_handle, inst,
1376*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_state, RESTARTER_STATE_ONLINE, RERR_NONE, NULL);
1377*7c478bd9Sstevel@tonic-gate 
1378*7c478bd9Sstevel@tonic-gate 	info = startd_zalloc(sizeof (fork_info_t));
1379*7c478bd9Sstevel@tonic-gate 
1380*7c478bd9Sstevel@tonic-gate 	info->sf_id = inst->ri_id;
1381*7c478bd9Sstevel@tonic-gate 	info->sf_method_type = METHOD_START;
1382*7c478bd9Sstevel@tonic-gate 	info->sf_event_type = RERR_NONE;
1383*7c478bd9Sstevel@tonic-gate 	inst->ri_method_thread = startd_thread_create(method_thread, info);
1384*7c478bd9Sstevel@tonic-gate }
1385*7c478bd9Sstevel@tonic-gate 
1386*7c478bd9Sstevel@tonic-gate static void
1387*7c478bd9Sstevel@tonic-gate maintain_instance(scf_handle_t *h, restarter_inst_t *rip, int immediate,
1388*7c478bd9Sstevel@tonic-gate     const char *aux)
1389*7c478bd9Sstevel@tonic-gate {
1390*7c478bd9Sstevel@tonic-gate 	fork_info_t *info;
1391*7c478bd9Sstevel@tonic-gate 
1392*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&rip->ri_lock));
1393*7c478bd9Sstevel@tonic-gate 	assert(aux != NULL);
1394*7c478bd9Sstevel@tonic-gate 	assert(rip->ri_method_thread == 0);
1395*7c478bd9Sstevel@tonic-gate 
1396*7c478bd9Sstevel@tonic-gate 	log_instance(rip, B_TRUE, "Stopping for maintenance due to %s.", aux);
1397*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: stopping for maintenance due to %s.\n",
1398*7c478bd9Sstevel@tonic-gate 	    rip->ri_i.i_fmri, aux);
1399*7c478bd9Sstevel@tonic-gate 
1400*7c478bd9Sstevel@tonic-gate 	/* Services in the maintenance state are ignored */
1401*7c478bd9Sstevel@tonic-gate 	if (rip->ri_i.i_state == RESTARTER_STATE_MAINT) {
1402*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
1403*7c478bd9Sstevel@tonic-gate 		    "%s: maintain_instance -> is already in maintenance\n",
1404*7c478bd9Sstevel@tonic-gate 		    rip->ri_i.i_fmri);
1405*7c478bd9Sstevel@tonic-gate 		return;
1406*7c478bd9Sstevel@tonic-gate 	}
1407*7c478bd9Sstevel@tonic-gate 
1408*7c478bd9Sstevel@tonic-gate 	if (immediate || !instance_started(rip)) {
1409*7c478bd9Sstevel@tonic-gate 		if (rip->ri_i.i_primary_ctid != 0) {
1410*7c478bd9Sstevel@tonic-gate 			rip->ri_m_inst = safe_scf_instance_create(h);
1411*7c478bd9Sstevel@tonic-gate 			rip->ri_mi_deleted = B_FALSE;
1412*7c478bd9Sstevel@tonic-gate 
1413*7c478bd9Sstevel@tonic-gate 			libscf_reget_instance(rip);
1414*7c478bd9Sstevel@tonic-gate 			method_remove_contract(rip, B_TRUE, B_TRUE);
1415*7c478bd9Sstevel@tonic-gate 
1416*7c478bd9Sstevel@tonic-gate 			scf_instance_destroy(rip->ri_m_inst);
1417*7c478bd9Sstevel@tonic-gate 		}
1418*7c478bd9Sstevel@tonic-gate 
1419*7c478bd9Sstevel@tonic-gate 		(void) restarter_instance_update_states(h, rip,
1420*7c478bd9Sstevel@tonic-gate 		    RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_RESTART,
1421*7c478bd9Sstevel@tonic-gate 		    (char *)aux);
1422*7c478bd9Sstevel@tonic-gate 		return;
1423*7c478bd9Sstevel@tonic-gate 	}
1424*7c478bd9Sstevel@tonic-gate 
1425*7c478bd9Sstevel@tonic-gate 	(void) restarter_instance_update_states(h, rip, rip->ri_i.i_state,
1426*7c478bd9Sstevel@tonic-gate 	    RESTARTER_STATE_MAINT, RERR_NONE, (char *)aux);
1427*7c478bd9Sstevel@tonic-gate 
1428*7c478bd9Sstevel@tonic-gate 	info = startd_zalloc(sizeof (*info));
1429*7c478bd9Sstevel@tonic-gate 	info->sf_id = rip->ri_id;
1430*7c478bd9Sstevel@tonic-gate 	info->sf_method_type = METHOD_STOP;
1431*7c478bd9Sstevel@tonic-gate 	info->sf_event_type = RERR_RESTART;
1432*7c478bd9Sstevel@tonic-gate 	rip->ri_method_thread = startd_thread_create(method_thread, info);
1433*7c478bd9Sstevel@tonic-gate }
1434*7c478bd9Sstevel@tonic-gate 
1435*7c478bd9Sstevel@tonic-gate static void
1436*7c478bd9Sstevel@tonic-gate refresh_instance(scf_handle_t *h, restarter_inst_t *rip)
1437*7c478bd9Sstevel@tonic-gate {
1438*7c478bd9Sstevel@tonic-gate 	scf_instance_t *inst;
1439*7c478bd9Sstevel@tonic-gate 	scf_snapshot_t *snap;
1440*7c478bd9Sstevel@tonic-gate 	fork_info_t *info;
1441*7c478bd9Sstevel@tonic-gate 	int r;
1442*7c478bd9Sstevel@tonic-gate 
1443*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&rip->ri_lock));
1444*7c478bd9Sstevel@tonic-gate 
1445*7c478bd9Sstevel@tonic-gate 	log_instance(rip, B_TRUE, "Rereading configuration.");
1446*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "%s: rereading configuration.\n",
1447*7c478bd9Sstevel@tonic-gate 	    rip->ri_i.i_fmri);
1448*7c478bd9Sstevel@tonic-gate 
1449*7c478bd9Sstevel@tonic-gate rep_retry:
1450*7c478bd9Sstevel@tonic-gate 	r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst);
1451*7c478bd9Sstevel@tonic-gate 	switch (r) {
1452*7c478bd9Sstevel@tonic-gate 	case 0:
1453*7c478bd9Sstevel@tonic-gate 		break;
1454*7c478bd9Sstevel@tonic-gate 
1455*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
1456*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
1457*7c478bd9Sstevel@tonic-gate 		goto rep_retry;
1458*7c478bd9Sstevel@tonic-gate 
1459*7c478bd9Sstevel@tonic-gate 	case ENOENT:
1460*7c478bd9Sstevel@tonic-gate 		/* Must have been deleted. */
1461*7c478bd9Sstevel@tonic-gate 		return;
1462*7c478bd9Sstevel@tonic-gate 
1463*7c478bd9Sstevel@tonic-gate 	case EINVAL:
1464*7c478bd9Sstevel@tonic-gate 	case ENOTSUP:
1465*7c478bd9Sstevel@tonic-gate 	default:
1466*7c478bd9Sstevel@tonic-gate 		bad_error("libscf_fmri_get_instance", r);
1467*7c478bd9Sstevel@tonic-gate 	}
1468*7c478bd9Sstevel@tonic-gate 
1469*7c478bd9Sstevel@tonic-gate 	snap = libscf_get_running_snapshot(inst);
1470*7c478bd9Sstevel@tonic-gate 
1471*7c478bd9Sstevel@tonic-gate 	r = libscf_get_startd_properties(inst, snap, &rip->ri_flags,
1472*7c478bd9Sstevel@tonic-gate 	    &rip->ri_utmpx_prefix);
1473*7c478bd9Sstevel@tonic-gate 	switch (r) {
1474*7c478bd9Sstevel@tonic-gate 	case 0:
1475*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG, "%s is a %s-style service\n",
1476*7c478bd9Sstevel@tonic-gate 		    rip->ri_i.i_fmri, service_style(rip->ri_flags));
1477*7c478bd9Sstevel@tonic-gate 		break;
1478*7c478bd9Sstevel@tonic-gate 
1479*7c478bd9Sstevel@tonic-gate 	case ECONNABORTED:
1480*7c478bd9Sstevel@tonic-gate 		scf_instance_destroy(inst);
1481*7c478bd9Sstevel@tonic-gate 		scf_snapshot_destroy(snap);
1482*7c478bd9Sstevel@tonic-gate 		libscf_handle_rebind(h);
1483*7c478bd9Sstevel@tonic-gate 		goto rep_retry;
1484*7c478bd9Sstevel@tonic-gate 
1485*7c478bd9Sstevel@tonic-gate 	case ECANCELED:
1486*7c478bd9Sstevel@tonic-gate 	case ENOENT:
1487*7c478bd9Sstevel@tonic-gate 		/* Succeed in anticipation of REMOVE_INSTANCE. */
1488*7c478bd9Sstevel@tonic-gate 		break;
1489*7c478bd9Sstevel@tonic-gate 
1490*7c478bd9Sstevel@tonic-gate 	default:
1491*7c478bd9Sstevel@tonic-gate 		bad_error("libscf_get_startd_properties", r);
1492*7c478bd9Sstevel@tonic-gate 	}
1493*7c478bd9Sstevel@tonic-gate 
1494*7c478bd9Sstevel@tonic-gate 	if (instance_started(rip)) {
1495*7c478bd9Sstevel@tonic-gate 		/* Refresh does not change the state. */
1496*7c478bd9Sstevel@tonic-gate 		(void) restarter_instance_update_states(h, rip,
1497*7c478bd9Sstevel@tonic-gate 		    rip->ri_i.i_state, rip->ri_i.i_state, RERR_NONE, NULL);
1498*7c478bd9Sstevel@tonic-gate 
1499*7c478bd9Sstevel@tonic-gate 		info = startd_zalloc(sizeof (*info));
1500*7c478bd9Sstevel@tonic-gate 		info->sf_id = rip->ri_id;
1501*7c478bd9Sstevel@tonic-gate 		info->sf_method_type = METHOD_REFRESH;
1502*7c478bd9Sstevel@tonic-gate 		info->sf_event_type = RERR_REFRESH;
1503*7c478bd9Sstevel@tonic-gate 
1504*7c478bd9Sstevel@tonic-gate 		assert(rip->ri_method_thread == 0);
1505*7c478bd9Sstevel@tonic-gate 		rip->ri_method_thread =
1506*7c478bd9Sstevel@tonic-gate 		    startd_thread_create(method_thread, info);
1507*7c478bd9Sstevel@tonic-gate 	}
1508*7c478bd9Sstevel@tonic-gate 
1509*7c478bd9Sstevel@tonic-gate 	scf_snapshot_destroy(snap);
1510*7c478bd9Sstevel@tonic-gate 	scf_instance_destroy(inst);
1511*7c478bd9Sstevel@tonic-gate }
1512*7c478bd9Sstevel@tonic-gate 
1513*7c478bd9Sstevel@tonic-gate const char *event_names[] = { "INVALID", "ADD_INSTANCE", "REMOVE_INSTANCE",
1514*7c478bd9Sstevel@tonic-gate 	"ENABLE", "DISABLE", "ADMIN_DEGRADED", "ADMIN_REFRESH",
1515*7c478bd9Sstevel@tonic-gate 	"ADMIN_RESTART", "ADMIN_MAINT_OFF", "ADMIN_MAINT_ON",
1516*7c478bd9Sstevel@tonic-gate 	"ADMIN_MAINT_ON_IMMEDIATE", "STOP", "START", "DEPENDENCY_CYCLE",
1517*7c478bd9Sstevel@tonic-gate 	"INVALID_DEPENDENCY", "ADMIN_DISABLE"
1518*7c478bd9Sstevel@tonic-gate };
1519*7c478bd9Sstevel@tonic-gate 
1520*7c478bd9Sstevel@tonic-gate /*
1521*7c478bd9Sstevel@tonic-gate  * void *restarter_process_events()
1522*7c478bd9Sstevel@tonic-gate  *
1523*7c478bd9Sstevel@tonic-gate  *   Called in a separate thread to process the events on an instance's
1524*7c478bd9Sstevel@tonic-gate  *   queue.  Empties the queue completely, and tries to keep the thread
1525*7c478bd9Sstevel@tonic-gate  *   around for a little while after the queue is empty to save on
1526*7c478bd9Sstevel@tonic-gate  *   startup costs.
1527*7c478bd9Sstevel@tonic-gate  */
1528*7c478bd9Sstevel@tonic-gate static void *
1529*7c478bd9Sstevel@tonic-gate restarter_process_events(void *arg)
1530*7c478bd9Sstevel@tonic-gate {
1531*7c478bd9Sstevel@tonic-gate 	scf_handle_t *h;
1532*7c478bd9Sstevel@tonic-gate 	restarter_instance_qentry_t *event;
1533*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *rip;
1534*7c478bd9Sstevel@tonic-gate 	char *fmri = (char *)arg;
1535*7c478bd9Sstevel@tonic-gate 	struct timespec to;
1536*7c478bd9Sstevel@tonic-gate 
1537*7c478bd9Sstevel@tonic-gate 	assert(fmri != NULL);
1538*7c478bd9Sstevel@tonic-gate 
1539*7c478bd9Sstevel@tonic-gate 	h = libscf_handle_create_bound_loop();
1540*7c478bd9Sstevel@tonic-gate 
1541*7c478bd9Sstevel@tonic-gate 	/* grab the queue lock */
1542*7c478bd9Sstevel@tonic-gate 	rip = inst_lookup_queue(fmri);
1543*7c478bd9Sstevel@tonic-gate 	if (rip == NULL)
1544*7c478bd9Sstevel@tonic-gate 		goto out;
1545*7c478bd9Sstevel@tonic-gate 
1546*7c478bd9Sstevel@tonic-gate again:
1547*7c478bd9Sstevel@tonic-gate 
1548*7c478bd9Sstevel@tonic-gate 	while ((event = uu_list_first(rip->ri_queue)) != NULL) {
1549*7c478bd9Sstevel@tonic-gate 		restarter_inst_t *inst;
1550*7c478bd9Sstevel@tonic-gate 
1551*7c478bd9Sstevel@tonic-gate 		/* drop the queue lock */
1552*7c478bd9Sstevel@tonic-gate 		MUTEX_UNLOCK(&rip->ri_queue_lock);
1553*7c478bd9Sstevel@tonic-gate 
1554*7c478bd9Sstevel@tonic-gate 		/*
1555*7c478bd9Sstevel@tonic-gate 		 * Grab the inst lock -- this waits until any outstanding
1556*7c478bd9Sstevel@tonic-gate 		 * method finishes running.
1557*7c478bd9Sstevel@tonic-gate 		 */
1558*7c478bd9Sstevel@tonic-gate 		inst = inst_lookup_by_name(fmri);
1559*7c478bd9Sstevel@tonic-gate 		if (inst == NULL) {
1560*7c478bd9Sstevel@tonic-gate 			/* Getting deleted in the middle isn't an error. */
1561*7c478bd9Sstevel@tonic-gate 			goto cont;
1562*7c478bd9Sstevel@tonic-gate 		}
1563*7c478bd9Sstevel@tonic-gate 
1564*7c478bd9Sstevel@tonic-gate 		assert(instance_in_transition(inst) == 0);
1565*7c478bd9Sstevel@tonic-gate 
1566*7c478bd9Sstevel@tonic-gate 		/* process the event */
1567*7c478bd9Sstevel@tonic-gate 		switch (event->riq_type) {
1568*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ENABLE:
1569*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_DISABLE:
1570*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
1571*7c478bd9Sstevel@tonic-gate 			(void) enable_inst(h, inst, event->riq_type);
1572*7c478bd9Sstevel@tonic-gate 			break;
1573*7c478bd9Sstevel@tonic-gate 
1574*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
1575*7c478bd9Sstevel@tonic-gate 			restarter_delete_inst(inst);
1576*7c478bd9Sstevel@tonic-gate 			inst = NULL;
1577*7c478bd9Sstevel@tonic-gate 			goto cont;
1578*7c478bd9Sstevel@tonic-gate 
1579*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_STOP:
1580*7c478bd9Sstevel@tonic-gate 			(void) stop_instance(h, inst, RSTOP_DEPENDENCY);
1581*7c478bd9Sstevel@tonic-gate 			break;
1582*7c478bd9Sstevel@tonic-gate 
1583*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_START:
1584*7c478bd9Sstevel@tonic-gate 			start_instance(h, inst);
1585*7c478bd9Sstevel@tonic-gate 			break;
1586*7c478bd9Sstevel@tonic-gate 
1587*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
1588*7c478bd9Sstevel@tonic-gate 			maintain_instance(h, inst, 0, "dependency_cycle");
1589*7c478bd9Sstevel@tonic-gate 			break;
1590*7c478bd9Sstevel@tonic-gate 
1591*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
1592*7c478bd9Sstevel@tonic-gate 			maintain_instance(h, inst, 0, "invalid_dependency");
1593*7c478bd9Sstevel@tonic-gate 			break;
1594*7c478bd9Sstevel@tonic-gate 
1595*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1596*7c478bd9Sstevel@tonic-gate 			maintain_instance(h, inst, 0, "administrative_request");
1597*7c478bd9Sstevel@tonic-gate 			break;
1598*7c478bd9Sstevel@tonic-gate 
1599*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1600*7c478bd9Sstevel@tonic-gate 			maintain_instance(h, inst, 1, "administrative_request");
1601*7c478bd9Sstevel@tonic-gate 			break;
1602*7c478bd9Sstevel@tonic-gate 
1603*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1604*7c478bd9Sstevel@tonic-gate 			unmaintain_instance(h, inst, RUNMAINT_CLEAR);
1605*7c478bd9Sstevel@tonic-gate 			break;
1606*7c478bd9Sstevel@tonic-gate 
1607*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1608*7c478bd9Sstevel@tonic-gate 			refresh_instance(h, inst);
1609*7c478bd9Sstevel@tonic-gate 			break;
1610*7c478bd9Sstevel@tonic-gate 
1611*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1612*7c478bd9Sstevel@tonic-gate 			log_framework(LOG_WARNING, "Restarter: "
1613*7c478bd9Sstevel@tonic-gate 			    "%s command (for %s) unimplemented.\n",
1614*7c478bd9Sstevel@tonic-gate 			    event_names[event->riq_type], inst->ri_i.i_fmri);
1615*7c478bd9Sstevel@tonic-gate 			break;
1616*7c478bd9Sstevel@tonic-gate 
1617*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1618*7c478bd9Sstevel@tonic-gate 			if (!instance_started(inst)) {
1619*7c478bd9Sstevel@tonic-gate 				log_framework(LOG_DEBUG, "Restarter: "
1620*7c478bd9Sstevel@tonic-gate 				    "Not restarting %s; not running.\n",
1621*7c478bd9Sstevel@tonic-gate 				    inst->ri_i.i_fmri);
1622*7c478bd9Sstevel@tonic-gate 			} else {
1623*7c478bd9Sstevel@tonic-gate 				/*
1624*7c478bd9Sstevel@tonic-gate 				 * Stop the instance.  If it can be restarted,
1625*7c478bd9Sstevel@tonic-gate 				 * the graph engine will send a new event.
1626*7c478bd9Sstevel@tonic-gate 				 */
1627*7c478bd9Sstevel@tonic-gate 				(void) stop_instance(h, inst, RSTOP_RESTART);
1628*7c478bd9Sstevel@tonic-gate 			}
1629*7c478bd9Sstevel@tonic-gate 			break;
1630*7c478bd9Sstevel@tonic-gate 
1631*7c478bd9Sstevel@tonic-gate 		case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
1632*7c478bd9Sstevel@tonic-gate 		default:
1633*7c478bd9Sstevel@tonic-gate #ifndef NDEBUG
1634*7c478bd9Sstevel@tonic-gate 			uu_warn("%s:%d: Bad restarter event %d.  "
1635*7c478bd9Sstevel@tonic-gate 			    "Aborting.\n", __FILE__, __LINE__, event->riq_type);
1636*7c478bd9Sstevel@tonic-gate #endif
1637*7c478bd9Sstevel@tonic-gate 			abort();
1638*7c478bd9Sstevel@tonic-gate 		}
1639*7c478bd9Sstevel@tonic-gate 
1640*7c478bd9Sstevel@tonic-gate 		assert(inst != NULL);
1641*7c478bd9Sstevel@tonic-gate 		MUTEX_UNLOCK(&inst->ri_lock);
1642*7c478bd9Sstevel@tonic-gate 
1643*7c478bd9Sstevel@tonic-gate cont:
1644*7c478bd9Sstevel@tonic-gate 		/* grab the queue lock */
1645*7c478bd9Sstevel@tonic-gate 		rip = inst_lookup_queue(fmri);
1646*7c478bd9Sstevel@tonic-gate 		if (rip == NULL)
1647*7c478bd9Sstevel@tonic-gate 			goto out;
1648*7c478bd9Sstevel@tonic-gate 
1649*7c478bd9Sstevel@tonic-gate 		/* delete the event */
1650*7c478bd9Sstevel@tonic-gate 		uu_list_remove(rip->ri_queue, event);
1651*7c478bd9Sstevel@tonic-gate 		startd_free(event, sizeof (restarter_instance_qentry_t));
1652*7c478bd9Sstevel@tonic-gate 	}
1653*7c478bd9Sstevel@tonic-gate 
1654*7c478bd9Sstevel@tonic-gate 	assert(rip != NULL);
1655*7c478bd9Sstevel@tonic-gate 
1656*7c478bd9Sstevel@tonic-gate 	/*
1657*7c478bd9Sstevel@tonic-gate 	 * Try to preserve the thread for a little while for future use.
1658*7c478bd9Sstevel@tonic-gate 	 */
1659*7c478bd9Sstevel@tonic-gate 	to.tv_sec = 3;
1660*7c478bd9Sstevel@tonic-gate 	to.tv_nsec = 0;
1661*7c478bd9Sstevel@tonic-gate 	(void) pthread_cond_reltimedwait_np(&rip->ri_queue_cv,
1662*7c478bd9Sstevel@tonic-gate 	    &rip->ri_queue_lock, &to);
1663*7c478bd9Sstevel@tonic-gate 
1664*7c478bd9Sstevel@tonic-gate 	if (uu_list_first(rip->ri_queue) != NULL)
1665*7c478bd9Sstevel@tonic-gate 		goto again;
1666*7c478bd9Sstevel@tonic-gate 
1667*7c478bd9Sstevel@tonic-gate 	rip->ri_queue_thread = 0;
1668*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&rip->ri_queue_lock);
1669*7c478bd9Sstevel@tonic-gate out:
1670*7c478bd9Sstevel@tonic-gate 	(void) scf_handle_unbind(h);
1671*7c478bd9Sstevel@tonic-gate 	scf_handle_destroy(h);
1672*7c478bd9Sstevel@tonic-gate 	free(fmri);
1673*7c478bd9Sstevel@tonic-gate 	return (NULL);
1674*7c478bd9Sstevel@tonic-gate }
1675*7c478bd9Sstevel@tonic-gate 
1676*7c478bd9Sstevel@tonic-gate static int
1677*7c478bd9Sstevel@tonic-gate is_admin_event(restarter_event_type_t t) {
1678*7c478bd9Sstevel@tonic-gate 
1679*7c478bd9Sstevel@tonic-gate 	switch (t) {
1680*7c478bd9Sstevel@tonic-gate 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1681*7c478bd9Sstevel@tonic-gate 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1682*7c478bd9Sstevel@tonic-gate 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1683*7c478bd9Sstevel@tonic-gate 	case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1684*7c478bd9Sstevel@tonic-gate 	case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1685*7c478bd9Sstevel@tonic-gate 	case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1686*7c478bd9Sstevel@tonic-gate 		return (1);
1687*7c478bd9Sstevel@tonic-gate 	default:
1688*7c478bd9Sstevel@tonic-gate 		return (0);
1689*7c478bd9Sstevel@tonic-gate 	}
1690*7c478bd9Sstevel@tonic-gate }
1691*7c478bd9Sstevel@tonic-gate 
1692*7c478bd9Sstevel@tonic-gate static void
1693*7c478bd9Sstevel@tonic-gate restarter_queue_event(restarter_inst_t *ri, restarter_protocol_event_t *e)
1694*7c478bd9Sstevel@tonic-gate {
1695*7c478bd9Sstevel@tonic-gate 	restarter_instance_qentry_t *qe;
1696*7c478bd9Sstevel@tonic-gate 	int r;
1697*7c478bd9Sstevel@tonic-gate 
1698*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&ri->ri_queue_lock));
1699*7c478bd9Sstevel@tonic-gate 	assert(!PTHREAD_MUTEX_HELD(&ri->ri_lock));
1700*7c478bd9Sstevel@tonic-gate 
1701*7c478bd9Sstevel@tonic-gate 	qe = startd_zalloc(sizeof (restarter_instance_qentry_t));
1702*7c478bd9Sstevel@tonic-gate 	qe->riq_type = e->rpe_type;
1703*7c478bd9Sstevel@tonic-gate 
1704*7c478bd9Sstevel@tonic-gate 	uu_list_node_init(qe, &qe->riq_link, restarter_queue_pool);
1705*7c478bd9Sstevel@tonic-gate 	r = uu_list_insert_before(ri->ri_queue, NULL, qe);
1706*7c478bd9Sstevel@tonic-gate 	assert(r == 0);
1707*7c478bd9Sstevel@tonic-gate }
1708*7c478bd9Sstevel@tonic-gate 
1709*7c478bd9Sstevel@tonic-gate /*
1710*7c478bd9Sstevel@tonic-gate  * void *restarter_event_thread()
1711*7c478bd9Sstevel@tonic-gate  *
1712*7c478bd9Sstevel@tonic-gate  *  Handle incoming graph events by placing them on a per-instance
1713*7c478bd9Sstevel@tonic-gate  *  queue.  We can't lock the main part of the instance structure, so
1714*7c478bd9Sstevel@tonic-gate  *  just modify the seprarately locked event queue portion.
1715*7c478bd9Sstevel@tonic-gate  */
1716*7c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1717*7c478bd9Sstevel@tonic-gate static void *
1718*7c478bd9Sstevel@tonic-gate restarter_event_thread(void *unused)
1719*7c478bd9Sstevel@tonic-gate {
1720*7c478bd9Sstevel@tonic-gate 	scf_handle_t *h;
1721*7c478bd9Sstevel@tonic-gate 
1722*7c478bd9Sstevel@tonic-gate 	/*
1723*7c478bd9Sstevel@tonic-gate 	 * This is a new thread, and thus, gets its own handle
1724*7c478bd9Sstevel@tonic-gate 	 * to the repository.
1725*7c478bd9Sstevel@tonic-gate 	 */
1726*7c478bd9Sstevel@tonic-gate 	h = libscf_handle_create_bound_loop();
1727*7c478bd9Sstevel@tonic-gate 
1728*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&ru->restarter_update_lock);
1729*7c478bd9Sstevel@tonic-gate 
1730*7c478bd9Sstevel@tonic-gate 	/*CONSTCOND*/
1731*7c478bd9Sstevel@tonic-gate 	while (1) {
1732*7c478bd9Sstevel@tonic-gate 		restarter_protocol_event_t *e;
1733*7c478bd9Sstevel@tonic-gate 
1734*7c478bd9Sstevel@tonic-gate 		while (ru->restarter_update_wakeup == 0)
1735*7c478bd9Sstevel@tonic-gate 			(void) pthread_cond_wait(&ru->restarter_update_cv,
1736*7c478bd9Sstevel@tonic-gate 			    &ru->restarter_update_lock);
1737*7c478bd9Sstevel@tonic-gate 
1738*7c478bd9Sstevel@tonic-gate 		ru->restarter_update_wakeup = 0;
1739*7c478bd9Sstevel@tonic-gate 
1740*7c478bd9Sstevel@tonic-gate 		while ((e = restarter_event_dequeue()) != NULL) {
1741*7c478bd9Sstevel@tonic-gate 			restarter_inst_t *rip;
1742*7c478bd9Sstevel@tonic-gate 			char *fmri;
1743*7c478bd9Sstevel@tonic-gate 
1744*7c478bd9Sstevel@tonic-gate 			MUTEX_UNLOCK(&ru->restarter_update_lock);
1745*7c478bd9Sstevel@tonic-gate 
1746*7c478bd9Sstevel@tonic-gate 			/*
1747*7c478bd9Sstevel@tonic-gate 			 * ADD_INSTANCE is special: there's likely no
1748*7c478bd9Sstevel@tonic-gate 			 * instance structure yet, so we need to handle the
1749*7c478bd9Sstevel@tonic-gate 			 * addition synchronously.
1750*7c478bd9Sstevel@tonic-gate 			 */
1751*7c478bd9Sstevel@tonic-gate 			switch (e->rpe_type) {
1752*7c478bd9Sstevel@tonic-gate 			case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
1753*7c478bd9Sstevel@tonic-gate 				if (restarter_insert_inst(h, e->rpe_inst) != 0)
1754*7c478bd9Sstevel@tonic-gate 					log_error(LOG_INFO, "Restarter: "
1755*7c478bd9Sstevel@tonic-gate 					    "Could not add %s.\n", e->rpe_inst);
1756*7c478bd9Sstevel@tonic-gate 
1757*7c478bd9Sstevel@tonic-gate 				MUTEX_LOCK(&st->st_load_lock);
1758*7c478bd9Sstevel@tonic-gate 				if (--st->st_load_instances == 0)
1759*7c478bd9Sstevel@tonic-gate 					(void) pthread_cond_broadcast(
1760*7c478bd9Sstevel@tonic-gate 					    &st->st_load_cv);
1761*7c478bd9Sstevel@tonic-gate 				MUTEX_UNLOCK(&st->st_load_lock);
1762*7c478bd9Sstevel@tonic-gate 
1763*7c478bd9Sstevel@tonic-gate 				goto nolookup;
1764*7c478bd9Sstevel@tonic-gate 			}
1765*7c478bd9Sstevel@tonic-gate 
1766*7c478bd9Sstevel@tonic-gate 			/*
1767*7c478bd9Sstevel@tonic-gate 			 * Lookup the instance, locking only the event queue.
1768*7c478bd9Sstevel@tonic-gate 			 * Can't grab ri_lock here because it might be held
1769*7c478bd9Sstevel@tonic-gate 			 * by a long-running method.
1770*7c478bd9Sstevel@tonic-gate 			 */
1771*7c478bd9Sstevel@tonic-gate 			rip = inst_lookup_queue(e->rpe_inst);
1772*7c478bd9Sstevel@tonic-gate 			if (rip == NULL) {
1773*7c478bd9Sstevel@tonic-gate 				log_error(LOG_INFO, "Restarter: "
1774*7c478bd9Sstevel@tonic-gate 				    "Ignoring %s command for unknown service "
1775*7c478bd9Sstevel@tonic-gate 				    "%s.\n", event_names[e->rpe_type],
1776*7c478bd9Sstevel@tonic-gate 				    e->rpe_inst);
1777*7c478bd9Sstevel@tonic-gate 				goto nolookup;
1778*7c478bd9Sstevel@tonic-gate 			}
1779*7c478bd9Sstevel@tonic-gate 
1780*7c478bd9Sstevel@tonic-gate 			/* Keep ADMIN events from filling up the queue. */
1781*7c478bd9Sstevel@tonic-gate 			if (is_admin_event(e->rpe_type) &&
1782*7c478bd9Sstevel@tonic-gate 			    uu_list_numnodes(rip->ri_queue) >
1783*7c478bd9Sstevel@tonic-gate 			    RINST_QUEUE_THRESHOLD) {
1784*7c478bd9Sstevel@tonic-gate 				MUTEX_UNLOCK(&rip->ri_queue_lock);
1785*7c478bd9Sstevel@tonic-gate 				log_instance(rip, B_TRUE, "Instance event "
1786*7c478bd9Sstevel@tonic-gate 				    "queue overflow.  Dropping administrative "
1787*7c478bd9Sstevel@tonic-gate 				    "request.");
1788*7c478bd9Sstevel@tonic-gate 				log_framework(LOG_DEBUG, "%s: Instance event "
1789*7c478bd9Sstevel@tonic-gate 				    "queue overflow.  Dropping administrative "
1790*7c478bd9Sstevel@tonic-gate 				    "request.\n", rip->ri_i.i_fmri);
1791*7c478bd9Sstevel@tonic-gate 				goto nolookup;
1792*7c478bd9Sstevel@tonic-gate 			}
1793*7c478bd9Sstevel@tonic-gate 
1794*7c478bd9Sstevel@tonic-gate 			/* Now add the event to the instance queue. */
1795*7c478bd9Sstevel@tonic-gate 			restarter_queue_event(rip, e);
1796*7c478bd9Sstevel@tonic-gate 
1797*7c478bd9Sstevel@tonic-gate 			if (rip->ri_queue_thread == 0) {
1798*7c478bd9Sstevel@tonic-gate 				/*
1799*7c478bd9Sstevel@tonic-gate 				 * Start a thread if one isn't already
1800*7c478bd9Sstevel@tonic-gate 				 * running.
1801*7c478bd9Sstevel@tonic-gate 				 */
1802*7c478bd9Sstevel@tonic-gate 				fmri = safe_strdup(e->rpe_inst);
1803*7c478bd9Sstevel@tonic-gate 				rip->ri_queue_thread =  startd_thread_create(
1804*7c478bd9Sstevel@tonic-gate 				    restarter_process_events, (void *)fmri);
1805*7c478bd9Sstevel@tonic-gate 			} else {
1806*7c478bd9Sstevel@tonic-gate 				/*
1807*7c478bd9Sstevel@tonic-gate 				 * Signal the existing thread that there's
1808*7c478bd9Sstevel@tonic-gate 				 * a new event.
1809*7c478bd9Sstevel@tonic-gate 				 */
1810*7c478bd9Sstevel@tonic-gate 				(void) pthread_cond_broadcast(
1811*7c478bd9Sstevel@tonic-gate 				    &rip->ri_queue_cv);
1812*7c478bd9Sstevel@tonic-gate 			}
1813*7c478bd9Sstevel@tonic-gate 
1814*7c478bd9Sstevel@tonic-gate 			MUTEX_UNLOCK(&rip->ri_queue_lock);
1815*7c478bd9Sstevel@tonic-gate nolookup:
1816*7c478bd9Sstevel@tonic-gate 			restarter_event_release(e);
1817*7c478bd9Sstevel@tonic-gate 
1818*7c478bd9Sstevel@tonic-gate 			MUTEX_LOCK(&ru->restarter_update_lock);
1819*7c478bd9Sstevel@tonic-gate 		}
1820*7c478bd9Sstevel@tonic-gate 	}
1821*7c478bd9Sstevel@tonic-gate 
1822*7c478bd9Sstevel@tonic-gate 	/*
1823*7c478bd9Sstevel@tonic-gate 	 * Unreachable for now -- there's currently no graceful cleanup
1824*7c478bd9Sstevel@tonic-gate 	 * called on exit().
1825*7c478bd9Sstevel@tonic-gate 	 */
1826*7c478bd9Sstevel@tonic-gate 	(void) scf_handle_unbind(h);
1827*7c478bd9Sstevel@tonic-gate 	scf_handle_destroy(h);
1828*7c478bd9Sstevel@tonic-gate 	return (NULL);
1829*7c478bd9Sstevel@tonic-gate }
1830*7c478bd9Sstevel@tonic-gate 
1831*7c478bd9Sstevel@tonic-gate static restarter_inst_t *
1832*7c478bd9Sstevel@tonic-gate contract_to_inst(ctid_t ctid)
1833*7c478bd9Sstevel@tonic-gate {
1834*7c478bd9Sstevel@tonic-gate 	restarter_inst_t *inst;
1835*7c478bd9Sstevel@tonic-gate 	int id;
1836*7c478bd9Sstevel@tonic-gate 
1837*7c478bd9Sstevel@tonic-gate 	id = lookup_inst_by_contract(ctid);
1838*7c478bd9Sstevel@tonic-gate 	if (id == -1)
1839*7c478bd9Sstevel@tonic-gate 		return (NULL);
1840*7c478bd9Sstevel@tonic-gate 
1841*7c478bd9Sstevel@tonic-gate 	inst = inst_lookup_by_id(id);
1842*7c478bd9Sstevel@tonic-gate 	if (inst != NULL) {
1843*7c478bd9Sstevel@tonic-gate 		/*
1844*7c478bd9Sstevel@tonic-gate 		 * Since ri_lock isn't held by the contract id lookup, this
1845*7c478bd9Sstevel@tonic-gate 		 * instance may have been restarted and now be in a new
1846*7c478bd9Sstevel@tonic-gate 		 * contract, making the old contract no longer valid for this
1847*7c478bd9Sstevel@tonic-gate 		 * instance.
1848*7c478bd9Sstevel@tonic-gate 		 */
1849*7c478bd9Sstevel@tonic-gate 		if (ctid != inst->ri_i.i_primary_ctid) {
1850*7c478bd9Sstevel@tonic-gate 			MUTEX_UNLOCK(&inst->ri_lock);
1851*7c478bd9Sstevel@tonic-gate 			inst = NULL;
1852*7c478bd9Sstevel@tonic-gate 		}
1853*7c478bd9Sstevel@tonic-gate 	}
1854*7c478bd9Sstevel@tonic-gate 	return (inst);
1855*7c478bd9Sstevel@tonic-gate }
1856*7c478bd9Sstevel@tonic-gate 
1857*7c478bd9Sstevel@tonic-gate /*
1858*7c478bd9Sstevel@tonic-gate  * void contract_action()
1859*7c478bd9Sstevel@tonic-gate  *   Take action on contract events.
1860*7c478bd9Sstevel@tonic-gate  */
1861*7c478bd9Sstevel@tonic-gate static void
1862*7c478bd9Sstevel@tonic-gate contract_action(scf_handle_t *h, restarter_inst_t *inst, ctid_t id,
1863*7c478bd9Sstevel@tonic-gate     uint32_t type)
1864*7c478bd9Sstevel@tonic-gate {
1865*7c478bd9Sstevel@tonic-gate 	const char *fmri = inst->ri_i.i_fmri;
1866*7c478bd9Sstevel@tonic-gate 
1867*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
1868*7c478bd9Sstevel@tonic-gate 
1869*7c478bd9Sstevel@tonic-gate 	/*
1870*7c478bd9Sstevel@tonic-gate 	 * If startd has stopped this contract, there is no need to
1871*7c478bd9Sstevel@tonic-gate 	 * stop it again.
1872*7c478bd9Sstevel@tonic-gate 	 */
1873*7c478bd9Sstevel@tonic-gate 	if (inst->ri_i.i_primary_ctid > 0 &&
1874*7c478bd9Sstevel@tonic-gate 	    inst->ri_i.i_primary_ctid_stopped)
1875*7c478bd9Sstevel@tonic-gate 		return;
1876*7c478bd9Sstevel@tonic-gate 
1877*7c478bd9Sstevel@tonic-gate 	if ((type & (CT_PR_EV_EMPTY | CT_PR_EV_CORE | CT_PR_EV_SIGNAL
1878*7c478bd9Sstevel@tonic-gate 	    | CT_PR_EV_HWERR)) == 0) {
1879*7c478bd9Sstevel@tonic-gate 		/*
1880*7c478bd9Sstevel@tonic-gate 		 * There shouldn't be other events, since that's not how we set
1881*7c478bd9Sstevel@tonic-gate 		 * the terms. Thus, just log an error and drive on.
1882*7c478bd9Sstevel@tonic-gate 		 */
1883*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_NOTICE,
1884*7c478bd9Sstevel@tonic-gate 		    "%s: contract %ld received unexpected critical event "
1885*7c478bd9Sstevel@tonic-gate 		    "(%d)\n", fmri, id, type);
1886*7c478bd9Sstevel@tonic-gate 		    return;
1887*7c478bd9Sstevel@tonic-gate 	}
1888*7c478bd9Sstevel@tonic-gate 
1889*7c478bd9Sstevel@tonic-gate 	assert(instance_in_transition(inst) == 0);
1890*7c478bd9Sstevel@tonic-gate 
1891*7c478bd9Sstevel@tonic-gate 	if (instance_is_wait_style(inst)) {
1892*7c478bd9Sstevel@tonic-gate 		/*
1893*7c478bd9Sstevel@tonic-gate 		 * We ignore all events; if they impact the
1894*7c478bd9Sstevel@tonic-gate 		 * process we're monitoring, then the
1895*7c478bd9Sstevel@tonic-gate 		 * wait_thread will stop the instance.
1896*7c478bd9Sstevel@tonic-gate 		 */
1897*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_DEBUG,
1898*7c478bd9Sstevel@tonic-gate 		    "%s: ignoring contract event on wait-style service\n",
1899*7c478bd9Sstevel@tonic-gate 		    fmri);
1900*7c478bd9Sstevel@tonic-gate 	} else {
1901*7c478bd9Sstevel@tonic-gate 		/*
1902*7c478bd9Sstevel@tonic-gate 		 * A CT_PR_EV_EMPTY event is an RSTOP_EXIT request.
1903*7c478bd9Sstevel@tonic-gate 		 */
1904*7c478bd9Sstevel@tonic-gate 		switch (type) {
1905*7c478bd9Sstevel@tonic-gate 		case CT_PR_EV_EMPTY:
1906*7c478bd9Sstevel@tonic-gate 			(void) stop_instance(h, inst, RSTOP_EXIT);
1907*7c478bd9Sstevel@tonic-gate 			break;
1908*7c478bd9Sstevel@tonic-gate 		case CT_PR_EV_CORE:
1909*7c478bd9Sstevel@tonic-gate 			(void) stop_instance(h, inst, RSTOP_CORE);
1910*7c478bd9Sstevel@tonic-gate 			break;
1911*7c478bd9Sstevel@tonic-gate 		case CT_PR_EV_SIGNAL:
1912*7c478bd9Sstevel@tonic-gate 			(void) stop_instance(h, inst, RSTOP_SIGNAL);
1913*7c478bd9Sstevel@tonic-gate 			break;
1914*7c478bd9Sstevel@tonic-gate 		case CT_PR_EV_HWERR:
1915*7c478bd9Sstevel@tonic-gate 			(void) stop_instance(h, inst, RSTOP_HWERR);
1916*7c478bd9Sstevel@tonic-gate 			break;
1917*7c478bd9Sstevel@tonic-gate 		}
1918*7c478bd9Sstevel@tonic-gate 	}
1919*7c478bd9Sstevel@tonic-gate }
1920*7c478bd9Sstevel@tonic-gate 
1921*7c478bd9Sstevel@tonic-gate /*
1922*7c478bd9Sstevel@tonic-gate  * void *restarter_contract_event_thread(void *)
1923*7c478bd9Sstevel@tonic-gate  *   Listens to the process contract bundle for critical events, taking action
1924*7c478bd9Sstevel@tonic-gate  *   on events from contracts we know we are responsible for.
1925*7c478bd9Sstevel@tonic-gate  */
1926*7c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1927*7c478bd9Sstevel@tonic-gate static void *
1928*7c478bd9Sstevel@tonic-gate restarter_contracts_event_thread(void *unused)
1929*7c478bd9Sstevel@tonic-gate {
1930*7c478bd9Sstevel@tonic-gate 	int fd, err;
1931*7c478bd9Sstevel@tonic-gate 	scf_handle_t *local_handle;
1932*7c478bd9Sstevel@tonic-gate 
1933*7c478bd9Sstevel@tonic-gate 	/*
1934*7c478bd9Sstevel@tonic-gate 	 * Await graph load completion.  That is, stop here, until we've scanned
1935*7c478bd9Sstevel@tonic-gate 	 * the repository for contract - instance associations.
1936*7c478bd9Sstevel@tonic-gate 	 */
1937*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&st->st_load_lock);
1938*7c478bd9Sstevel@tonic-gate 	while (!(st->st_load_complete && st->st_load_instances == 0))
1939*7c478bd9Sstevel@tonic-gate 		(void) pthread_cond_wait(&st->st_load_cv, &st->st_load_lock);
1940*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&st->st_load_lock);
1941*7c478bd9Sstevel@tonic-gate 
1942*7c478bd9Sstevel@tonic-gate 	/*
1943*7c478bd9Sstevel@tonic-gate 	 * This is a new thread, and thus, gets its own handle
1944*7c478bd9Sstevel@tonic-gate 	 * to the repository.
1945*7c478bd9Sstevel@tonic-gate 	 */
1946*7c478bd9Sstevel@tonic-gate 	if ((local_handle = libscf_handle_create_bound(SCF_VERSION)) == NULL)
1947*7c478bd9Sstevel@tonic-gate 		uu_die("Unable to bind a new repository handle: %s\n",
1948*7c478bd9Sstevel@tonic-gate 		    scf_strerror(scf_error()));
1949*7c478bd9Sstevel@tonic-gate 
1950*7c478bd9Sstevel@tonic-gate 	fd = open64(CTFS_ROOT "/process/pbundle", O_RDONLY);
1951*7c478bd9Sstevel@tonic-gate 	if (fd == -1)
1952*7c478bd9Sstevel@tonic-gate 		uu_die("process bundle open failed");
1953*7c478bd9Sstevel@tonic-gate 
1954*7c478bd9Sstevel@tonic-gate 	/*
1955*7c478bd9Sstevel@tonic-gate 	 * Make sure we get all events (including those generated by configd
1956*7c478bd9Sstevel@tonic-gate 	 * before this thread was started).
1957*7c478bd9Sstevel@tonic-gate 	 */
1958*7c478bd9Sstevel@tonic-gate 	err = ct_event_reset(fd);
1959*7c478bd9Sstevel@tonic-gate 	assert(err == 0);
1960*7c478bd9Sstevel@tonic-gate 
1961*7c478bd9Sstevel@tonic-gate 	for (;;) {
1962*7c478bd9Sstevel@tonic-gate 		int efd, sfd;
1963*7c478bd9Sstevel@tonic-gate 		ct_evthdl_t ev;
1964*7c478bd9Sstevel@tonic-gate 		uint32_t type;
1965*7c478bd9Sstevel@tonic-gate 		ctevid_t evid;
1966*7c478bd9Sstevel@tonic-gate 		ct_stathdl_t status;
1967*7c478bd9Sstevel@tonic-gate 		ctid_t ctid;
1968*7c478bd9Sstevel@tonic-gate 		restarter_inst_t *inst;
1969*7c478bd9Sstevel@tonic-gate 		uint64_t cookie;
1970*7c478bd9Sstevel@tonic-gate 
1971*7c478bd9Sstevel@tonic-gate 		if (err = ct_event_read_critical(fd, &ev)) {
1972*7c478bd9Sstevel@tonic-gate 			log_error(LOG_WARNING,
1973*7c478bd9Sstevel@tonic-gate 			    "Error reading next contract event: %s",
1974*7c478bd9Sstevel@tonic-gate 			    strerror(err));
1975*7c478bd9Sstevel@tonic-gate 			continue;
1976*7c478bd9Sstevel@tonic-gate 		}
1977*7c478bd9Sstevel@tonic-gate 
1978*7c478bd9Sstevel@tonic-gate 		evid = ct_event_get_evid(ev);
1979*7c478bd9Sstevel@tonic-gate 		ctid = ct_event_get_ctid(ev);
1980*7c478bd9Sstevel@tonic-gate 		type = ct_event_get_type(ev);
1981*7c478bd9Sstevel@tonic-gate 
1982*7c478bd9Sstevel@tonic-gate 		/* Fetch cookie. */
1983*7c478bd9Sstevel@tonic-gate 		if ((sfd = contract_open(ctid, "process", "status", O_RDONLY))
1984*7c478bd9Sstevel@tonic-gate 		    < 0) {
1985*7c478bd9Sstevel@tonic-gate 			ct_event_free(ev);
1986*7c478bd9Sstevel@tonic-gate 			continue;
1987*7c478bd9Sstevel@tonic-gate 		}
1988*7c478bd9Sstevel@tonic-gate 
1989*7c478bd9Sstevel@tonic-gate 		if (err = ct_status_read(sfd, CTD_COMMON, &status)) {
1990*7c478bd9Sstevel@tonic-gate 			log_framework(LOG_WARNING, "Could not get status for "
1991*7c478bd9Sstevel@tonic-gate 			    "contract %ld: %s\n", ctid, strerror(err));
1992*7c478bd9Sstevel@tonic-gate 
1993*7c478bd9Sstevel@tonic-gate 			startd_close(sfd);
1994*7c478bd9Sstevel@tonic-gate 			ct_event_free(ev);
1995*7c478bd9Sstevel@tonic-gate 			continue;
1996*7c478bd9Sstevel@tonic-gate 		}
1997*7c478bd9Sstevel@tonic-gate 
1998*7c478bd9Sstevel@tonic-gate 		cookie = ct_status_get_cookie(status);
1999*7c478bd9Sstevel@tonic-gate 
2000*7c478bd9Sstevel@tonic-gate 		ct_status_free(status);
2001*7c478bd9Sstevel@tonic-gate 
2002*7c478bd9Sstevel@tonic-gate 		startd_close(sfd);
2003*7c478bd9Sstevel@tonic-gate 
2004*7c478bd9Sstevel@tonic-gate 		/*
2005*7c478bd9Sstevel@tonic-gate 		 * svc.configd(1M) restart handling performed by the
2006*7c478bd9Sstevel@tonic-gate 		 * fork_configd_thread.  We don't acknowledge, as that thread
2007*7c478bd9Sstevel@tonic-gate 		 * will do so.
2008*7c478bd9Sstevel@tonic-gate 		 */
2009*7c478bd9Sstevel@tonic-gate 		if (cookie == CONFIGD_COOKIE) {
2010*7c478bd9Sstevel@tonic-gate 			ct_event_free(ev);
2011*7c478bd9Sstevel@tonic-gate 			continue;
2012*7c478bd9Sstevel@tonic-gate 		}
2013*7c478bd9Sstevel@tonic-gate 
2014*7c478bd9Sstevel@tonic-gate 		inst = contract_to_inst(ctid);
2015*7c478bd9Sstevel@tonic-gate 		if (inst == NULL) {
2016*7c478bd9Sstevel@tonic-gate 			/*
2017*7c478bd9Sstevel@tonic-gate 			 * This can happen if we receive an EMPTY
2018*7c478bd9Sstevel@tonic-gate 			 * event for an abandoned contract.
2019*7c478bd9Sstevel@tonic-gate 			 */
2020*7c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG,
2021*7c478bd9Sstevel@tonic-gate 			    "Received event %d for unknown contract id "
2022*7c478bd9Sstevel@tonic-gate 			    "%ld\n", type, ctid);
2023*7c478bd9Sstevel@tonic-gate 		} else {
2024*7c478bd9Sstevel@tonic-gate 			log_framework(LOG_DEBUG,
2025*7c478bd9Sstevel@tonic-gate 			    "Received event %d for contract id "
2026*7c478bd9Sstevel@tonic-gate 			    "%ld (%s)\n", type, ctid,
2027*7c478bd9Sstevel@tonic-gate 			    inst->ri_i.i_fmri);
2028*7c478bd9Sstevel@tonic-gate 
2029*7c478bd9Sstevel@tonic-gate 			contract_action(local_handle, inst, ctid, type);
2030*7c478bd9Sstevel@tonic-gate 
2031*7c478bd9Sstevel@tonic-gate 			MUTEX_UNLOCK(&inst->ri_lock);
2032*7c478bd9Sstevel@tonic-gate 		}
2033*7c478bd9Sstevel@tonic-gate 
2034*7c478bd9Sstevel@tonic-gate 		efd = contract_open(ct_event_get_ctid(ev), "process", "ctl",
2035*7c478bd9Sstevel@tonic-gate 		    O_WRONLY);
2036*7c478bd9Sstevel@tonic-gate 		if (efd != -1) {
2037*7c478bd9Sstevel@tonic-gate 			(void) ct_ctl_ack(efd, evid);
2038*7c478bd9Sstevel@tonic-gate 			startd_close(efd);
2039*7c478bd9Sstevel@tonic-gate 		}
2040*7c478bd9Sstevel@tonic-gate 
2041*7c478bd9Sstevel@tonic-gate 		ct_event_free(ev);
2042*7c478bd9Sstevel@tonic-gate 
2043*7c478bd9Sstevel@tonic-gate 	}
2044*7c478bd9Sstevel@tonic-gate 
2045*7c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
2046*7c478bd9Sstevel@tonic-gate 	return (NULL);
2047*7c478bd9Sstevel@tonic-gate }
2048*7c478bd9Sstevel@tonic-gate 
2049*7c478bd9Sstevel@tonic-gate /*
2050*7c478bd9Sstevel@tonic-gate  * Timeout queue, processed by restarter_timeouts_event_thread().
2051*7c478bd9Sstevel@tonic-gate  */
2052*7c478bd9Sstevel@tonic-gate timeout_queue_t *timeouts;
2053*7c478bd9Sstevel@tonic-gate static uu_list_pool_t *timeout_pool;
2054*7c478bd9Sstevel@tonic-gate 
2055*7c478bd9Sstevel@tonic-gate typedef struct timeout_update {
2056*7c478bd9Sstevel@tonic-gate 	pthread_mutex_t		tu_lock;
2057*7c478bd9Sstevel@tonic-gate 	pthread_cond_t		tu_cv;
2058*7c478bd9Sstevel@tonic-gate 	int			tu_wakeup;
2059*7c478bd9Sstevel@tonic-gate } timeout_update_t;
2060*7c478bd9Sstevel@tonic-gate 
2061*7c478bd9Sstevel@tonic-gate timeout_update_t *tu;
2062*7c478bd9Sstevel@tonic-gate 
2063*7c478bd9Sstevel@tonic-gate static const char *timeout_ovr_svcs[] = {
2064*7c478bd9Sstevel@tonic-gate 	"svc:/system/manifest-import:default",
2065*7c478bd9Sstevel@tonic-gate 	"svc:/network/initial:default",
2066*7c478bd9Sstevel@tonic-gate 	"svc:/network/service:default",
2067*7c478bd9Sstevel@tonic-gate 	"svc:/system/rmtmpfiles:default",
2068*7c478bd9Sstevel@tonic-gate 	"svc:/network/loopback:default",
2069*7c478bd9Sstevel@tonic-gate 	"svc:/network/physical:default",
2070*7c478bd9Sstevel@tonic-gate 	"svc:/system/device/local:default",
2071*7c478bd9Sstevel@tonic-gate 	"svc:/system/metainit:default",
2072*7c478bd9Sstevel@tonic-gate 	"svc:/system/filesystem/usr:default",
2073*7c478bd9Sstevel@tonic-gate 	"svc:/system/filesystem/minimal:default",
2074*7c478bd9Sstevel@tonic-gate 	"svc:/system/filesystem/local:default",
2075*7c478bd9Sstevel@tonic-gate 	NULL
2076*7c478bd9Sstevel@tonic-gate };
2077*7c478bd9Sstevel@tonic-gate 
2078*7c478bd9Sstevel@tonic-gate int
2079*7c478bd9Sstevel@tonic-gate is_timeout_ovr(restarter_inst_t *inst)
2080*7c478bd9Sstevel@tonic-gate {
2081*7c478bd9Sstevel@tonic-gate 	int i;
2082*7c478bd9Sstevel@tonic-gate 
2083*7c478bd9Sstevel@tonic-gate 	for (i = 0; timeout_ovr_svcs[i] != NULL; ++i) {
2084*7c478bd9Sstevel@tonic-gate 		if (strcmp(inst->ri_i.i_fmri, timeout_ovr_svcs[i]) == 0) {
2085*7c478bd9Sstevel@tonic-gate 			log_instance(inst, B_TRUE, "Timeout override by "
2086*7c478bd9Sstevel@tonic-gate 			    "svc.startd.  Using infinite timeout");
2087*7c478bd9Sstevel@tonic-gate 			return (1);
2088*7c478bd9Sstevel@tonic-gate 		}
2089*7c478bd9Sstevel@tonic-gate 	}
2090*7c478bd9Sstevel@tonic-gate 
2091*7c478bd9Sstevel@tonic-gate 	return (0);
2092*7c478bd9Sstevel@tonic-gate }
2093*7c478bd9Sstevel@tonic-gate 
2094*7c478bd9Sstevel@tonic-gate /*ARGSUSED*/
2095*7c478bd9Sstevel@tonic-gate static int
2096*7c478bd9Sstevel@tonic-gate timeout_compare(const void *lc_arg, const void *rc_arg, void *private)
2097*7c478bd9Sstevel@tonic-gate {
2098*7c478bd9Sstevel@tonic-gate 	hrtime_t t1 = ((const timeout_entry_t *)lc_arg)->te_timeout;
2099*7c478bd9Sstevel@tonic-gate 	hrtime_t t2 = ((const timeout_entry_t *)rc_arg)->te_timeout;
2100*7c478bd9Sstevel@tonic-gate 
2101*7c478bd9Sstevel@tonic-gate 	if (t1 > t2)
2102*7c478bd9Sstevel@tonic-gate 		return (1);
2103*7c478bd9Sstevel@tonic-gate 	else if (t1 < t2)
2104*7c478bd9Sstevel@tonic-gate 		return (-1);
2105*7c478bd9Sstevel@tonic-gate 	return (0);
2106*7c478bd9Sstevel@tonic-gate }
2107*7c478bd9Sstevel@tonic-gate 
2108*7c478bd9Sstevel@tonic-gate void
2109*7c478bd9Sstevel@tonic-gate timeout_init()
2110*7c478bd9Sstevel@tonic-gate {
2111*7c478bd9Sstevel@tonic-gate 	timeouts = startd_zalloc(sizeof (timeout_queue_t));
2112*7c478bd9Sstevel@tonic-gate 
2113*7c478bd9Sstevel@tonic-gate 	(void) pthread_mutex_init(&timeouts->tq_lock, &mutex_attrs);
2114*7c478bd9Sstevel@tonic-gate 
2115*7c478bd9Sstevel@tonic-gate 	timeout_pool = startd_list_pool_create("timeouts",
2116*7c478bd9Sstevel@tonic-gate 	    sizeof (timeout_entry_t), offsetof(timeout_entry_t, te_link),
2117*7c478bd9Sstevel@tonic-gate 	    timeout_compare, UU_LIST_POOL_DEBUG);
2118*7c478bd9Sstevel@tonic-gate 	assert(timeout_pool != NULL);
2119*7c478bd9Sstevel@tonic-gate 
2120*7c478bd9Sstevel@tonic-gate 	timeouts->tq_list = startd_list_create(timeout_pool,
2121*7c478bd9Sstevel@tonic-gate 	    timeouts, UU_LIST_SORTED);
2122*7c478bd9Sstevel@tonic-gate 	assert(timeouts->tq_list != NULL);
2123*7c478bd9Sstevel@tonic-gate 
2124*7c478bd9Sstevel@tonic-gate 	tu = startd_zalloc(sizeof (timeout_update_t));
2125*7c478bd9Sstevel@tonic-gate 	(void) pthread_cond_init(&tu->tu_cv, NULL);
2126*7c478bd9Sstevel@tonic-gate 	(void) pthread_mutex_init(&tu->tu_lock, &mutex_attrs);
2127*7c478bd9Sstevel@tonic-gate }
2128*7c478bd9Sstevel@tonic-gate 
2129*7c478bd9Sstevel@tonic-gate void
2130*7c478bd9Sstevel@tonic-gate timeout_insert(restarter_inst_t *inst, ctid_t cid, uint64_t timeout_sec)
2131*7c478bd9Sstevel@tonic-gate {
2132*7c478bd9Sstevel@tonic-gate 	hrtime_t now, timeout;
2133*7c478bd9Sstevel@tonic-gate 	timeout_entry_t *entry;
2134*7c478bd9Sstevel@tonic-gate 	uu_list_index_t idx;
2135*7c478bd9Sstevel@tonic-gate 
2136*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
2137*7c478bd9Sstevel@tonic-gate 
2138*7c478bd9Sstevel@tonic-gate 	now = gethrtime();
2139*7c478bd9Sstevel@tonic-gate 
2140*7c478bd9Sstevel@tonic-gate 	/*
2141*7c478bd9Sstevel@tonic-gate 	 * If we overflow LLONG_MAX, we're never timing out anyways, so
2142*7c478bd9Sstevel@tonic-gate 	 * just return.
2143*7c478bd9Sstevel@tonic-gate 	 */
2144*7c478bd9Sstevel@tonic-gate 	if (timeout_sec >= (LLONG_MAX - now) / 1000000000LL) {
2145*7c478bd9Sstevel@tonic-gate 		log_instance(inst, B_TRUE, "timeout_seconds too large, "
2146*7c478bd9Sstevel@tonic-gate 		    "treating as infinite.");
2147*7c478bd9Sstevel@tonic-gate 		return;
2148*7c478bd9Sstevel@tonic-gate 	}
2149*7c478bd9Sstevel@tonic-gate 
2150*7c478bd9Sstevel@tonic-gate 	/* hrtime is in nanoseconds. Convert timeout_sec. */
2151*7c478bd9Sstevel@tonic-gate 	timeout = now + (timeout_sec * 1000000000LL);
2152*7c478bd9Sstevel@tonic-gate 
2153*7c478bd9Sstevel@tonic-gate 	entry = startd_alloc(sizeof (timeout_entry_t));
2154*7c478bd9Sstevel@tonic-gate 	entry->te_timeout = timeout;
2155*7c478bd9Sstevel@tonic-gate 	entry->te_ctid = cid;
2156*7c478bd9Sstevel@tonic-gate 	entry->te_fmri = safe_strdup(inst->ri_i.i_fmri);
2157*7c478bd9Sstevel@tonic-gate 	entry->te_logstem = safe_strdup(inst->ri_logstem);
2158*7c478bd9Sstevel@tonic-gate 	entry->te_fired = 0;
2159*7c478bd9Sstevel@tonic-gate 	/* Insert the calculated timeout time onto the queue. */
2160*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&timeouts->tq_lock);
2161*7c478bd9Sstevel@tonic-gate 	(void) uu_list_find(timeouts->tq_list, entry, NULL, &idx);
2162*7c478bd9Sstevel@tonic-gate 	uu_list_node_init(entry, &entry->te_link, timeout_pool);
2163*7c478bd9Sstevel@tonic-gate 	uu_list_insert(timeouts->tq_list, entry, idx);
2164*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&timeouts->tq_lock);
2165*7c478bd9Sstevel@tonic-gate 
2166*7c478bd9Sstevel@tonic-gate 	assert(inst->ri_timeout == NULL);
2167*7c478bd9Sstevel@tonic-gate 	inst->ri_timeout = entry;
2168*7c478bd9Sstevel@tonic-gate 
2169*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&tu->tu_lock);
2170*7c478bd9Sstevel@tonic-gate 	tu->tu_wakeup = 1;
2171*7c478bd9Sstevel@tonic-gate 	(void) pthread_cond_broadcast(&tu->tu_cv);
2172*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&tu->tu_lock);
2173*7c478bd9Sstevel@tonic-gate }
2174*7c478bd9Sstevel@tonic-gate 
2175*7c478bd9Sstevel@tonic-gate 
2176*7c478bd9Sstevel@tonic-gate void
2177*7c478bd9Sstevel@tonic-gate timeout_remove(restarter_inst_t *inst, ctid_t cid)
2178*7c478bd9Sstevel@tonic-gate {
2179*7c478bd9Sstevel@tonic-gate 	assert(PTHREAD_MUTEX_HELD(&inst->ri_lock));
2180*7c478bd9Sstevel@tonic-gate 
2181*7c478bd9Sstevel@tonic-gate 	if (inst->ri_timeout == NULL)
2182*7c478bd9Sstevel@tonic-gate 		return;
2183*7c478bd9Sstevel@tonic-gate 
2184*7c478bd9Sstevel@tonic-gate 	assert(inst->ri_timeout->te_ctid == cid);
2185*7c478bd9Sstevel@tonic-gate 
2186*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&timeouts->tq_lock);
2187*7c478bd9Sstevel@tonic-gate 	uu_list_remove(timeouts->tq_list, inst->ri_timeout);
2188*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&timeouts->tq_lock);
2189*7c478bd9Sstevel@tonic-gate 
2190*7c478bd9Sstevel@tonic-gate 	free(inst->ri_timeout->te_fmri);
2191*7c478bd9Sstevel@tonic-gate 	free(inst->ri_timeout->te_logstem);
2192*7c478bd9Sstevel@tonic-gate 	startd_free(inst->ri_timeout, sizeof (timeout_entry_t));
2193*7c478bd9Sstevel@tonic-gate 	inst->ri_timeout = NULL;
2194*7c478bd9Sstevel@tonic-gate }
2195*7c478bd9Sstevel@tonic-gate 
2196*7c478bd9Sstevel@tonic-gate static int
2197*7c478bd9Sstevel@tonic-gate timeout_now()
2198*7c478bd9Sstevel@tonic-gate {
2199*7c478bd9Sstevel@tonic-gate 	timeout_entry_t *e;
2200*7c478bd9Sstevel@tonic-gate 	hrtime_t now;
2201*7c478bd9Sstevel@tonic-gate 	int ret;
2202*7c478bd9Sstevel@tonic-gate 
2203*7c478bd9Sstevel@tonic-gate 	now = gethrtime();
2204*7c478bd9Sstevel@tonic-gate 
2205*7c478bd9Sstevel@tonic-gate 	/*
2206*7c478bd9Sstevel@tonic-gate 	 * Walk through the (sorted) timeouts list.  While the timeout
2207*7c478bd9Sstevel@tonic-gate 	 * at the head of the list is <= the current time, kill the
2208*7c478bd9Sstevel@tonic-gate 	 * method.
2209*7c478bd9Sstevel@tonic-gate 	 */
2210*7c478bd9Sstevel@tonic-gate 	MUTEX_LOCK(&timeouts->tq_lock);
2211*7c478bd9Sstevel@tonic-gate 
2212*7c478bd9Sstevel@tonic-gate 	for (e = uu_list_first(timeouts->tq_list);
2213*7c478bd9Sstevel@tonic-gate 	    e != NULL && e->te_timeout <= now;
2214*7c478bd9Sstevel@tonic-gate 	    e = uu_list_next(timeouts->tq_list, e)) {
2215*7c478bd9Sstevel@tonic-gate 		log_framework(LOG_WARNING, "%s: Method or service exit timed "
2216*7c478bd9Sstevel@tonic-gate 		    "out.  Killing contract %ld.\n", e->te_fmri, e->te_ctid);
2217*7c478bd9Sstevel@tonic-gate 		log_instance_fmri(e->te_fmri, e->te_logstem, B_TRUE,
2218*7c478bd9Sstevel@tonic-gate 		    "Method or service exit timed out.  Killing contract %ld",
2219*7c478bd9Sstevel@tonic-gate 		    e->te_ctid);
2220*7c478bd9Sstevel@tonic-gate 		e->te_fired = 1;
2221*7c478bd9Sstevel@tonic-gate 		(void) contract_kill(e->te_ctid, SIGKILL, e->te_fmri);
2222*7c478bd9Sstevel@tonic-gate 	}
2223*7c478bd9Sstevel@tonic-gate 
2224*7c478bd9Sstevel@tonic-gate 	if (uu_list_numnodes(timeouts->tq_list) > 0)
2225*7c478bd9Sstevel@tonic-gate 		ret = 0;
2226*7c478bd9Sstevel@tonic-gate 	else
2227*7c478bd9Sstevel@tonic-gate 		ret = -1;
2228*7c478bd9Sstevel@tonic-gate 
2229*7c478bd9Sstevel@tonic-gate 	MUTEX_UNLOCK(&timeouts->tq_lock);
2230*7c478bd9Sstevel@tonic-gate 
2231*7c478bd9Sstevel@tonic-gate 	return (ret);
2232*7c478bd9Sstevel@tonic-gate }
2233*7c478bd9Sstevel@tonic-gate 
2234*7c478bd9Sstevel@tonic-gate /*
2235*7c478bd9Sstevel@tonic-gate  * void *restarter_timeouts_event_thread(void *)
2236*7c478bd9Sstevel@tonic-gate  *   Responsible for monitoring the method timeouts.  This thread must
2237*7c478bd9Sstevel@tonic-gate  *   be started before any methods are called.
2238*7c478bd9Sstevel@tonic-gate  */
2239*7c478bd9Sstevel@tonic-gate /*ARGSUSED*/
2240*7c478bd9Sstevel@tonic-gate static void *
2241*7c478bd9Sstevel@tonic-gate restarter_timeouts_event_thread(void *unused)
2242*7c478bd9Sstevel@tonic-gate {
2243*7c478bd9Sstevel@tonic-gate 	/*
2244*7c478bd9Sstevel@tonic-gate 	 * Timeouts are entered on a priority queue, which is processed by
2245*7c478bd9Sstevel@tonic-gate 	 * this thread.  As timeouts are specified in seconds, we'll do
2246*7c478bd9Sstevel@tonic-gate 	 * the necessary processing every second, as long as the queue
2247*7c478bd9Sstevel@tonic-gate 	 * is not empty.
2248*7c478bd9Sstevel@tonic-gate 	 */
2249*7c478bd9Sstevel@tonic-gate 
2250*7c478bd9Sstevel@tonic-gate 	/*CONSTCOND*/
2251*7c478bd9Sstevel@tonic-gate 	while (1) {
2252*7c478bd9Sstevel@tonic-gate 		/*
2253*7c478bd9Sstevel@tonic-gate 		 * As long as the timeout list isn't empty, process it
2254*7c478bd9Sstevel@tonic-gate 		 * every second.
2255*7c478bd9Sstevel@tonic-gate 		 */
2256*7c478bd9Sstevel@tonic-gate 		if (timeout_now() == 0) {
2257*7c478bd9Sstevel@tonic-gate 			(void) sleep(1);
2258*7c478bd9Sstevel@tonic-gate 			continue;
2259*7c478bd9Sstevel@tonic-gate 		}
2260*7c478bd9Sstevel@tonic-gate 
2261*7c478bd9Sstevel@tonic-gate 		/* The list is empty, wait until we have more timeouts. */
2262*7c478bd9Sstevel@tonic-gate 		MUTEX_LOCK(&tu->tu_lock);
2263*7c478bd9Sstevel@tonic-gate 
2264*7c478bd9Sstevel@tonic-gate 		while (tu->tu_wakeup == 0)
2265*7c478bd9Sstevel@tonic-gate 			(void) pthread_cond_wait(&tu->tu_cv, &tu->tu_lock);
2266*7c478bd9Sstevel@tonic-gate 
2267*7c478bd9Sstevel@tonic-gate 		tu->tu_wakeup = 0;
2268*7c478bd9Sstevel@tonic-gate 		MUTEX_UNLOCK(&tu->tu_lock);
2269*7c478bd9Sstevel@tonic-gate 	}
2270*7c478bd9Sstevel@tonic-gate 
2271*7c478bd9Sstevel@tonic-gate 	return (NULL);
2272*7c478bd9Sstevel@tonic-gate }
2273*7c478bd9Sstevel@tonic-gate 
2274*7c478bd9Sstevel@tonic-gate void
2275*7c478bd9Sstevel@tonic-gate restarter_start()
2276*7c478bd9Sstevel@tonic-gate {
2277*7c478bd9Sstevel@tonic-gate 	(void) startd_thread_create(restarter_timeouts_event_thread, NULL);
2278*7c478bd9Sstevel@tonic-gate 	(void) startd_thread_create(restarter_event_thread, NULL);
2279*7c478bd9Sstevel@tonic-gate 	(void) startd_thread_create(restarter_contracts_event_thread, NULL);
2280*7c478bd9Sstevel@tonic-gate 	(void) startd_thread_create(wait_thread, NULL);
2281*7c478bd9Sstevel@tonic-gate }
2282*7c478bd9Sstevel@tonic-gate 
2283*7c478bd9Sstevel@tonic-gate 
2284*7c478bd9Sstevel@tonic-gate void
2285*7c478bd9Sstevel@tonic-gate restarter_init()
2286*7c478bd9Sstevel@tonic-gate {
2287*7c478bd9Sstevel@tonic-gate 	restarter_instance_pool = startd_list_pool_create("restarter_instances",
2288*7c478bd9Sstevel@tonic-gate 	    sizeof (restarter_inst_t), offsetof(restarter_inst_t,
2289*7c478bd9Sstevel@tonic-gate 		ri_link), restarter_instance_compare, UU_LIST_POOL_DEBUG);
2290*7c478bd9Sstevel@tonic-gate 	(void) memset(&instance_list, 0, sizeof (instance_list));
2291*7c478bd9Sstevel@tonic-gate 
2292*7c478bd9Sstevel@tonic-gate 	(void) pthread_mutex_init(&instance_list.ril_lock, &mutex_attrs);
2293*7c478bd9Sstevel@tonic-gate 	instance_list.ril_instance_list = startd_list_create(
2294*7c478bd9Sstevel@tonic-gate 	    restarter_instance_pool, &instance_list, UU_LIST_SORTED);
2295*7c478bd9Sstevel@tonic-gate 
2296*7c478bd9Sstevel@tonic-gate 	restarter_queue_pool = startd_list_pool_create(
2297*7c478bd9Sstevel@tonic-gate 	    "restarter_instance_queue", sizeof (restarter_instance_qentry_t),
2298*7c478bd9Sstevel@tonic-gate 	    offsetof(restarter_instance_qentry_t,  riq_link), NULL,
2299*7c478bd9Sstevel@tonic-gate 	    UU_LIST_POOL_DEBUG);
2300*7c478bd9Sstevel@tonic-gate 
2301*7c478bd9Sstevel@tonic-gate 	contract_list_pool = startd_list_pool_create(
2302*7c478bd9Sstevel@tonic-gate 	    "contract_list", sizeof (contract_entry_t),
2303*7c478bd9Sstevel@tonic-gate 	    offsetof(contract_entry_t,  ce_link), NULL,
2304*7c478bd9Sstevel@tonic-gate 	    UU_LIST_POOL_DEBUG);
2305*7c478bd9Sstevel@tonic-gate 	contract_hash_init();
2306*7c478bd9Sstevel@tonic-gate 
2307*7c478bd9Sstevel@tonic-gate 	log_framework(LOG_DEBUG, "Initialized restarter\n");
2308*7c478bd9Sstevel@tonic-gate }
2309