xref: /titanic_52/usr/src/cmd/svc/startd/graph.c (revision b9e93c10c0a2a4bb069d38bb311021a9478c4711)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * graph.c - master restarter graph engine
28  *
29  *   The graph engine keeps a dependency graph of all service instances on the
30  *   system, as recorded in the repository.  It decides when services should
31  *   be brought up or down based on service states and dependencies and sends
32  *   commands to restarters to effect any changes.  It also executes
33  *   administrator commands sent by svcadm via the repository.
34  *
35  *   The graph is stored in uu_list_t *dgraph and its vertices are
36  *   graph_vertex_t's, each of which has a name and an integer id unique to
37  *   its name (see dict.c).  A vertex's type attribute designates the type
38  *   of object it represents: GVT_INST for service instances, GVT_SVC for
39  *   service objects (since service instances may depend on another service,
40  *   rather than service instance), GVT_FILE for files (which services may
41  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
42  *   vertices are necessary because dependency lists may have particular
43  *   grouping types (require any, require all, optional, or exclude) and
44  *   event-propagation characteristics.
45  *
46  *   The initial graph is built by libscf_populate_graph() invoking
47  *   dgraph_add_instance() for each instance in the repository.  The function
48  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
49  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
50  *   The resulting web of vertices & edges associated with an instance's vertex
51  *   includes
52  *
53  *     - an edge from the GVT_SVC vertex for the instance's service
54  *
55  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
56  *       restarter is not svc.startd
57  *
58  *     - edges from other GVT_INST vertices if the instance is a restarter
59  *
60  *     - for each dependency property group in the instance's "running"
61  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
62  *       instance and the name of the property group
63  *
64  *     - for each value of the "entities" property in each dependency property
65  *       group, an edge from the corresponding GVT_GROUP vertex to a
66  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
67  *
68  *     - edges from GVT_GROUP vertices for each dependent instance
69  *
70  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
71  *   there are problems, or if a service is mentioned in a dependency but does
72  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
73  *
74  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
75  *   See restarter.c for more information.
76  *
77  *   The properties of an instance fall into two classes: immediate and
78  *   snapshotted.  Immediate properties should have an immediate effect when
79  *   changed.  Snapshotted properties should be read from a snapshot, so they
80  *   only change when the snapshot changes.  The immediate properties used by
81  *   the graph engine are general/enabled, general/restarter, and the properties
82  *   in the restarter_actions property group.  Since they are immediate, they
83  *   are not read out of a snapshot.  The snapshotted properties used by the
84  *   graph engine are those in the property groups with type "dependency" and
85  *   are read out of the "running" snapshot.  The "running" snapshot is created
86  *   by the the graph engine as soon as possible, and it is updated, along with
87  *   in-core copies of the data (dependency information for the graph engine) on
88  *   receipt of the refresh command from svcadm.  In addition, the graph engine
89  *   updates the "start" snapshot from the "running" snapshot whenever a service
90  *   comes online.
91  *
92  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
93  *   the dependents first before shutting down the requested service.
94  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
95  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
96  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
97  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
98  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
99  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
100  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
101  *   exposed leaves. We do the same until we reach the last leaf (the one with
102  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
103  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
104  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
105  *   continue to propagate the offline event to the vertex's dependencies.
106  */
107 
108 #include <sys/uadmin.h>
109 #include <sys/wait.h>
110 
111 #include <assert.h>
112 #include <errno.h>
113 #include <fcntl.h>
114 #include <libscf.h>
115 #include <libscf_priv.h>
116 #include <libuutil.h>
117 #include <locale.h>
118 #include <poll.h>
119 #include <pthread.h>
120 #include <signal.h>
121 #include <stddef.h>
122 #include <stdio.h>
123 #include <stdlib.h>
124 #include <string.h>
125 #include <strings.h>
126 #include <sys/statvfs.h>
127 #include <sys/uadmin.h>
128 #include <zone.h>
129 
130 #include "startd.h"
131 #include "protocol.h"
132 
133 
134 #define	MILESTONE_NONE	((graph_vertex_t *)1)
135 
136 #define	CONSOLE_LOGIN_FMRI	"svc:/system/console-login:default"
137 #define	FS_MINIMAL_FMRI		"svc:/system/filesystem/minimal:default"
138 
139 #define	VERTEX_REMOVED	0	/* vertex has been freed  */
140 #define	VERTEX_INUSE	1	/* vertex is still in use */
141 
142 /*
143  * Services in these states are not considered 'down' by the
144  * milestone/shutdown code.
145  */
146 #define	up_state(state)	((state) == RESTARTER_STATE_ONLINE || \
147 	(state) == RESTARTER_STATE_DEGRADED || \
148 	(state) == RESTARTER_STATE_OFFLINE)
149 
150 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
151 static uu_list_t *dgraph;
152 static pthread_mutex_t dgraph_lock;
153 
154 /*
155  * milestone indicates the current subgraph.  When NULL, it is the entire
156  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
157  * services on which the target vertex depends.
158  */
159 static graph_vertex_t *milestone = NULL;
160 static boolean_t initial_milestone_set = B_FALSE;
161 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
162 
163 /* protected by dgraph_lock */
164 static boolean_t sulogin_thread_running = B_FALSE;
165 static boolean_t sulogin_running = B_FALSE;
166 static boolean_t console_login_ready = B_FALSE;
167 
168 /* Number of services to come down to complete milestone transition. */
169 static uint_t non_subgraph_svcs;
170 
171 /*
172  * These variables indicate what should be done when we reach the milestone
173  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
174  * dgraph_set_instance_state().
175  */
176 static int halting = -1;
177 static boolean_t go_single_user_mode = B_FALSE;
178 static boolean_t go_to_level1 = B_FALSE;
179 
180 /*
181  * This tracks the legacy runlevel to ensure we signal init and manage
182  * utmpx entries correctly.
183  */
184 static char current_runlevel = '\0';
185 
186 /* Number of single user threads currently running */
187 static pthread_mutex_t single_user_thread_lock;
188 static int single_user_thread_count = 0;
189 
190 /* Statistics for dependency cycle-checking */
191 static u_longlong_t dep_inserts = 0;
192 static u_longlong_t dep_cycle_ns = 0;
193 static u_longlong_t dep_insert_ns = 0;
194 
195 
196 static const char * const emsg_invalid_restarter =
197 	"Transitioning %s to maintenance, restarter FMRI %s is invalid "
198 	"(see 'svcs -xv' for details).\n";
199 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
200 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
201 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
202 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
203 
204 
205 /*
206  * These services define the system being "up".  If none of them can come
207  * online, then we will run sulogin on the console.  Note that the install ones
208  * are for the miniroot and when installing CDs after the first.  can_come_up()
209  * does the decision making, and an sulogin_thread() runs sulogin, which can be
210  * started by dgraph_set_instance_state() or single_user_thread().
211  *
212  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
213  * entry, which is only used when booting_to_single_user (boot -s) is set.
214  * This is because when doing a "boot -s", sulogin is started from specials.c
215  * after milestone/single-user comes online, for backwards compatibility.
216  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
217  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
218  */
219 static const char * const up_svcs[] = {
220 	SCF_MILESTONE_SINGLE_USER,
221 	CONSOLE_LOGIN_FMRI,
222 	"svc:/system/install-setup:default",
223 	"svc:/system/install:default",
224 	NULL
225 };
226 
227 /* This array must have an element for each non-NULL element of up_svcs[]. */
228 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
229 
230 /* These are for seed repository magic.  See can_come_up(). */
231 static const char * const manifest_import =
232 	"svc:/system/manifest-import:default";
233 static graph_vertex_t *manifest_import_p = NULL;
234 
235 
236 static char target_milestone_as_runlevel(void);
237 static void graph_runlevel_changed(char rl, int online);
238 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
239 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
240 static int mark_subtree(graph_edge_t *, void *);
241 static boolean_t insubtree_dependents_down(graph_vertex_t *);
242 
243 /*
244  * graph_vertex_compare()
245  *	This function can compare either int *id or * graph_vertex_t *gv
246  *	values, as the vertex id is always the first element of a
247  *	graph_vertex structure.
248  */
249 /* ARGSUSED */
250 static int
251 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
252 {
253 	int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
254 	int rc_id = *(int *)rc_arg;
255 
256 	if (lc_id > rc_id)
257 		return (1);
258 	if (lc_id < rc_id)
259 		return (-1);
260 	return (0);
261 }
262 
263 void
264 graph_init()
265 {
266 	graph_edge_pool = startd_list_pool_create("graph_edges",
267 	    sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
268 	    UU_LIST_POOL_DEBUG);
269 	assert(graph_edge_pool != NULL);
270 
271 	graph_vertex_pool = startd_list_pool_create("graph_vertices",
272 	    sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
273 	    graph_vertex_compare, UU_LIST_POOL_DEBUG);
274 	assert(graph_vertex_pool != NULL);
275 
276 	(void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
277 	(void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
278 	dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
279 	assert(dgraph != NULL);
280 
281 	if (!st->st_initial)
282 		current_runlevel = utmpx_get_runlevel();
283 
284 	log_framework(LOG_DEBUG, "Initialized graph\n");
285 }
286 
287 static graph_vertex_t *
288 vertex_get_by_name(const char *name)
289 {
290 	int id;
291 
292 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
293 
294 	id = dict_lookup_byname(name);
295 	if (id == -1)
296 		return (NULL);
297 
298 	return (uu_list_find(dgraph, &id, NULL, NULL));
299 }
300 
301 static graph_vertex_t *
302 vertex_get_by_id(int id)
303 {
304 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
305 
306 	if (id == -1)
307 		return (NULL);
308 
309 	return (uu_list_find(dgraph, &id, NULL, NULL));
310 }
311 
312 /*
313  * Creates a new vertex with the given name, adds it to the graph, and returns
314  * a pointer to it.  The graph lock must be held by this thread on entry.
315  */
316 static graph_vertex_t *
317 graph_add_vertex(const char *name)
318 {
319 	int id;
320 	graph_vertex_t *v;
321 	void *p;
322 	uu_list_index_t idx;
323 
324 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
325 
326 	id = dict_insert(name);
327 
328 	v = startd_zalloc(sizeof (*v));
329 
330 	v->gv_id = id;
331 
332 	v->gv_name = startd_alloc(strlen(name) + 1);
333 	(void) strcpy(v->gv_name, name);
334 
335 	v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
336 	v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
337 
338 	p = uu_list_find(dgraph, &id, NULL, &idx);
339 	assert(p == NULL);
340 
341 	uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
342 	uu_list_insert(dgraph, v, idx);
343 
344 	return (v);
345 }
346 
347 /*
348  * Removes v from the graph and frees it.  The graph should be locked by this
349  * thread, and v should have no edges associated with it.
350  */
351 static void
352 graph_remove_vertex(graph_vertex_t *v)
353 {
354 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
355 
356 	assert(uu_list_numnodes(v->gv_dependencies) == 0);
357 	assert(uu_list_numnodes(v->gv_dependents) == 0);
358 	assert(v->gv_refs == 0);
359 
360 	startd_free(v->gv_name, strlen(v->gv_name) + 1);
361 	uu_list_destroy(v->gv_dependencies);
362 	uu_list_destroy(v->gv_dependents);
363 	uu_list_remove(dgraph, v);
364 
365 	startd_free(v, sizeof (graph_vertex_t));
366 }
367 
368 static void
369 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
370 {
371 	graph_edge_t *e, *re;
372 	int r;
373 
374 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
375 
376 	e = startd_alloc(sizeof (graph_edge_t));
377 	re = startd_alloc(sizeof (graph_edge_t));
378 
379 	e->ge_parent = fv;
380 	e->ge_vertex = tv;
381 
382 	re->ge_parent = tv;
383 	re->ge_vertex = fv;
384 
385 	uu_list_node_init(e, &e->ge_link, graph_edge_pool);
386 	r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
387 	assert(r == 0);
388 
389 	uu_list_node_init(re, &re->ge_link, graph_edge_pool);
390 	r = uu_list_insert_before(tv->gv_dependents, NULL, re);
391 	assert(r == 0);
392 }
393 
394 static void
395 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
396 {
397 	graph_edge_t *e;
398 
399 	for (e = uu_list_first(v->gv_dependencies);
400 	    e != NULL;
401 	    e = uu_list_next(v->gv_dependencies, e)) {
402 		if (e->ge_vertex == dv) {
403 			uu_list_remove(v->gv_dependencies, e);
404 			startd_free(e, sizeof (graph_edge_t));
405 			break;
406 		}
407 	}
408 
409 	for (e = uu_list_first(dv->gv_dependents);
410 	    e != NULL;
411 	    e = uu_list_next(dv->gv_dependents, e)) {
412 		if (e->ge_vertex == v) {
413 			uu_list_remove(dv->gv_dependents, e);
414 			startd_free(e, sizeof (graph_edge_t));
415 			break;
416 		}
417 	}
418 }
419 
420 static void
421 remove_inst_vertex(graph_vertex_t *v)
422 {
423 	graph_edge_t *e;
424 	graph_vertex_t *sv;
425 	int i;
426 
427 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
428 	assert(uu_list_numnodes(v->gv_dependents) == 1);
429 	assert(uu_list_numnodes(v->gv_dependencies) == 0);
430 	assert(v->gv_refs == 0);
431 	assert((v->gv_flags & GV_CONFIGURED) == 0);
432 
433 	e = uu_list_first(v->gv_dependents);
434 	sv = e->ge_vertex;
435 	graph_remove_edge(sv, v);
436 
437 	for (i = 0; up_svcs[i] != NULL; ++i) {
438 		if (up_svcs_p[i] == v)
439 			up_svcs_p[i] = NULL;
440 	}
441 
442 	if (manifest_import_p == v)
443 		manifest_import_p = NULL;
444 
445 	graph_remove_vertex(v);
446 
447 	if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
448 	    uu_list_numnodes(sv->gv_dependents) == 0 &&
449 	    sv->gv_refs == 0)
450 		graph_remove_vertex(sv);
451 }
452 
453 static void
454 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
455     void *arg)
456 {
457 	graph_edge_t *e;
458 
459 	for (e = uu_list_first(v->gv_dependents);
460 	    e != NULL;
461 	    e = uu_list_next(v->gv_dependents, e))
462 		func(e->ge_vertex, arg);
463 }
464 
465 static void
466 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
467 	void *), void *arg)
468 {
469 	graph_edge_t *e;
470 
471 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
472 
473 	for (e = uu_list_first(v->gv_dependencies);
474 	    e != NULL;
475 	    e = uu_list_next(v->gv_dependencies, e)) {
476 
477 		func(e->ge_vertex, arg);
478 	}
479 }
480 
481 /*
482  * Generic graph walking function.
483  *
484  * Given a vertex, this function will walk either dependencies
485  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
486  * for the entire graph.  It will avoid cycles and never visit the same vertex
487  * twice.
488  *
489  * We avoid traversing exclusion dependencies, because they are allowed to
490  * create cycles in the graph.  When propagating satisfiability, there is no
491  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
492  * test for satisfiability.
493  *
494  * The walker takes two callbacks.  The first is called before examining the
495  * dependents of each vertex.  The second is called on each vertex after
496  * examining its dependents.  This allows is_path_to() to construct a path only
497  * after the target vertex has been found.
498  */
499 typedef enum {
500 	WALK_DEPENDENTS,
501 	WALK_DEPENDENCIES
502 } graph_walk_dir_t;
503 
504 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
505 
506 typedef struct graph_walk_info {
507 	graph_walk_dir_t 	gi_dir;
508 	uchar_t			*gi_visited;	/* vertex bitmap */
509 	int			(*gi_pre)(graph_vertex_t *, void *);
510 	void			(*gi_post)(graph_vertex_t *, void *);
511 	void			*gi_arg;	/* callback arg */
512 	int			gi_ret;		/* return value */
513 } graph_walk_info_t;
514 
515 static int
516 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
517 {
518 	uu_list_t *list;
519 	int r;
520 	graph_vertex_t *v = e->ge_vertex;
521 	int i;
522 	uint_t b;
523 
524 	i = v->gv_id / 8;
525 	b = 1 << (v->gv_id % 8);
526 
527 	/*
528 	 * Check to see if we've visited this vertex already.
529 	 */
530 	if (gip->gi_visited[i] & b)
531 		return (UU_WALK_NEXT);
532 
533 	gip->gi_visited[i] |= b;
534 
535 	/*
536 	 * Don't follow exclusions.
537 	 */
538 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
539 		return (UU_WALK_NEXT);
540 
541 	/*
542 	 * Call pre-visit callback.  If this doesn't terminate the walk,
543 	 * continue search.
544 	 */
545 	if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
546 		/*
547 		 * Recurse using appropriate list.
548 		 */
549 		if (gip->gi_dir == WALK_DEPENDENTS)
550 			list = v->gv_dependents;
551 		else
552 			list = v->gv_dependencies;
553 
554 		r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
555 		    gip, 0);
556 		assert(r == 0);
557 	}
558 
559 	/*
560 	 * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
561 	 */
562 	assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
563 
564 	/*
565 	 * If given a post-callback, call the function for every vertex.
566 	 */
567 	if (gip->gi_post != NULL)
568 		(void) gip->gi_post(v, gip->gi_arg);
569 
570 	/*
571 	 * Preserve the callback's return value.  If the callback returns
572 	 * UU_WALK_DONE, then we propagate that to the caller in order to
573 	 * terminate the walk.
574 	 */
575 	return (gip->gi_ret);
576 }
577 
578 static void
579 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
580     int (*pre)(graph_vertex_t *, void *),
581     void (*post)(graph_vertex_t *, void *), void *arg)
582 {
583 	graph_walk_info_t gi;
584 	graph_edge_t fake;
585 	size_t sz = dictionary->dict_new_id / 8 + 1;
586 
587 	gi.gi_visited = startd_zalloc(sz);
588 	gi.gi_pre = pre;
589 	gi.gi_post = post;
590 	gi.gi_arg = arg;
591 	gi.gi_dir = dir;
592 	gi.gi_ret = 0;
593 
594 	/*
595 	 * Fake up an edge for the first iteration
596 	 */
597 	fake.ge_vertex = v;
598 	(void) graph_walk_recurse(&fake, &gi);
599 
600 	startd_free(gi.gi_visited, sz);
601 }
602 
603 typedef struct child_search {
604 	int	id;		/* id of vertex to look for */
605 	uint_t	depth;		/* recursion depth */
606 	/*
607 	 * While the vertex is not found, path is NULL.  After the search, if
608 	 * the vertex was found then path should point to a -1-terminated
609 	 * array of vertex id's which constitute the path to the vertex.
610 	 */
611 	int	*path;
612 } child_search_t;
613 
614 static int
615 child_pre(graph_vertex_t *v, void *arg)
616 {
617 	child_search_t *cs = arg;
618 
619 	cs->depth++;
620 
621 	if (v->gv_id == cs->id) {
622 		cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
623 		cs->path[cs->depth] = -1;
624 		return (UU_WALK_DONE);
625 	}
626 
627 	return (UU_WALK_NEXT);
628 }
629 
630 static void
631 child_post(graph_vertex_t *v, void *arg)
632 {
633 	child_search_t *cs = arg;
634 
635 	cs->depth--;
636 
637 	if (cs->path != NULL)
638 		cs->path[cs->depth] = v->gv_id;
639 }
640 
641 /*
642  * Look for a path from from to to.  If one exists, returns a pointer to
643  * a NULL-terminated array of pointers to the vertices along the path.  If
644  * there is no path, returns NULL.
645  */
646 static int *
647 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
648 {
649 	child_search_t cs;
650 
651 	cs.id = to->gv_id;
652 	cs.depth = 0;
653 	cs.path = NULL;
654 
655 	graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
656 
657 	return (cs.path);
658 }
659 
660 /*
661  * Given an array of int's as returned by is_path_to, allocates a string of
662  * their names joined by newlines.  Returns the size of the allocated buffer
663  * in *sz and frees path.
664  */
665 static void
666 path_to_str(int *path, char **cpp, size_t *sz)
667 {
668 	int i;
669 	graph_vertex_t *v;
670 	size_t allocd, new_allocd;
671 	char *new, *name;
672 
673 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
674 	assert(path[0] != -1);
675 
676 	allocd = 1;
677 	*cpp = startd_alloc(1);
678 	(*cpp)[0] = '\0';
679 
680 	for (i = 0; path[i] != -1; ++i) {
681 		name = NULL;
682 
683 		v = vertex_get_by_id(path[i]);
684 
685 		if (v == NULL)
686 			name = "<deleted>";
687 		else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
688 			name = v->gv_name;
689 
690 		if (name != NULL) {
691 			new_allocd = allocd + strlen(name) + 1;
692 			new = startd_alloc(new_allocd);
693 			(void) strcpy(new, *cpp);
694 			(void) strcat(new, name);
695 			(void) strcat(new, "\n");
696 
697 			startd_free(*cpp, allocd);
698 
699 			*cpp = new;
700 			allocd = new_allocd;
701 		}
702 	}
703 
704 	startd_free(path, sizeof (int) * (i + 1));
705 
706 	*sz = allocd;
707 }
708 
709 
710 /*
711  * This function along with run_sulogin() implements an exclusion relationship
712  * between system/console-login and sulogin.  run_sulogin() will fail if
713  * system/console-login is online, and the graph engine should call
714  * graph_clogin_start() to bring system/console-login online, which defers the
715  * start if sulogin is running.
716  */
717 static void
718 graph_clogin_start(graph_vertex_t *v)
719 {
720 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
721 
722 	if (sulogin_running)
723 		console_login_ready = B_TRUE;
724 	else
725 		vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
726 }
727 
728 static void
729 graph_su_start(graph_vertex_t *v)
730 {
731 	/*
732 	 * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
733 	 * entry with a runlevel of 'S', before jumping to the final
734 	 * target runlevel (as set in initdefault).  We mimic that legacy
735 	 * behavior here.
736 	 */
737 	utmpx_set_runlevel('S', '0', B_FALSE);
738 	vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
739 }
740 
741 static void
742 graph_post_su_online(void)
743 {
744 	graph_runlevel_changed('S', 1);
745 }
746 
747 static void
748 graph_post_su_disable(void)
749 {
750 	graph_runlevel_changed('S', 0);
751 }
752 
753 static void
754 graph_post_mu_online(void)
755 {
756 	graph_runlevel_changed('2', 1);
757 }
758 
759 static void
760 graph_post_mu_disable(void)
761 {
762 	graph_runlevel_changed('2', 0);
763 }
764 
765 static void
766 graph_post_mus_online(void)
767 {
768 	graph_runlevel_changed('3', 1);
769 }
770 
771 static void
772 graph_post_mus_disable(void)
773 {
774 	graph_runlevel_changed('3', 0);
775 }
776 
777 static struct special_vertex_info {
778 	const char	*name;
779 	void		(*start_f)(graph_vertex_t *);
780 	void		(*post_online_f)(void);
781 	void		(*post_disable_f)(void);
782 } special_vertices[] = {
783 	{ CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
784 	{ SCF_MILESTONE_SINGLE_USER, graph_su_start,
785 	    graph_post_su_online, graph_post_su_disable },
786 	{ SCF_MILESTONE_MULTI_USER, NULL,
787 	    graph_post_mu_online, graph_post_mu_disable },
788 	{ SCF_MILESTONE_MULTI_USER_SERVER, NULL,
789 	    graph_post_mus_online, graph_post_mus_disable },
790 	{ NULL },
791 };
792 
793 
794 void
795 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
796 {
797 	switch (e) {
798 	case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
799 		assert(v->gv_state == RESTARTER_STATE_UNINIT);
800 
801 		MUTEX_LOCK(&st->st_load_lock);
802 		st->st_load_instances++;
803 		MUTEX_UNLOCK(&st->st_load_lock);
804 		break;
805 
806 	case RESTARTER_EVENT_TYPE_ENABLE:
807 		log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
808 		assert(v->gv_state == RESTARTER_STATE_UNINIT ||
809 		    v->gv_state == RESTARTER_STATE_DISABLED ||
810 		    v->gv_state == RESTARTER_STATE_MAINT);
811 		break;
812 
813 	case RESTARTER_EVENT_TYPE_DISABLE:
814 	case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
815 		log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
816 		assert(v->gv_state != RESTARTER_STATE_DISABLED);
817 		break;
818 
819 	case RESTARTER_EVENT_TYPE_STOP:
820 		log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
821 		assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
822 		    v->gv_state == RESTARTER_STATE_ONLINE);
823 		break;
824 
825 	case RESTARTER_EVENT_TYPE_START:
826 		log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
827 		assert(v->gv_state == RESTARTER_STATE_OFFLINE);
828 		break;
829 
830 	case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
831 	case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
832 	case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
833 	case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
834 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
835 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
836 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
837 	case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
838 	case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
839 		break;
840 
841 	default:
842 #ifndef NDEBUG
843 		uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
844 #endif
845 		abort();
846 	}
847 
848 	restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e);
849 }
850 
851 static void
852 graph_unset_restarter(graph_vertex_t *v)
853 {
854 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
855 	assert(v->gv_flags & GV_CONFIGURED);
856 
857 	vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
858 
859 	if (v->gv_restarter_id != -1) {
860 		graph_vertex_t *rv;
861 
862 		rv = vertex_get_by_id(v->gv_restarter_id);
863 		graph_remove_edge(v, rv);
864 	}
865 
866 	v->gv_restarter_id = -1;
867 	v->gv_restarter_channel = NULL;
868 }
869 
870 /*
871  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
872  * dgraph otherwise return VERTEX_INUSE.
873  */
874 static int
875 free_if_unrefed(graph_vertex_t *v)
876 {
877 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
878 
879 	if (v->gv_refs > 0)
880 		return (VERTEX_INUSE);
881 
882 	if (v->gv_type == GVT_SVC &&
883 	    uu_list_numnodes(v->gv_dependents) == 0 &&
884 	    uu_list_numnodes(v->gv_dependencies) == 0) {
885 		graph_remove_vertex(v);
886 		return (VERTEX_REMOVED);
887 	} else if (v->gv_type == GVT_INST &&
888 	    (v->gv_flags & GV_CONFIGURED) == 0 &&
889 	    uu_list_numnodes(v->gv_dependents) == 1 &&
890 	    uu_list_numnodes(v->gv_dependencies) == 0) {
891 		remove_inst_vertex(v);
892 		return (VERTEX_REMOVED);
893 	}
894 
895 	return (VERTEX_INUSE);
896 }
897 
898 static void
899 delete_depgroup(graph_vertex_t *v)
900 {
901 	graph_edge_t *e;
902 	graph_vertex_t *dv;
903 
904 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
905 	assert(v->gv_type == GVT_GROUP);
906 	assert(uu_list_numnodes(v->gv_dependents) == 0);
907 
908 	while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
909 		dv = e->ge_vertex;
910 
911 		graph_remove_edge(v, dv);
912 
913 		switch (dv->gv_type) {
914 		case GVT_INST:		/* instance dependency */
915 		case GVT_SVC:		/* service dependency */
916 			(void) free_if_unrefed(dv);
917 			break;
918 
919 		case GVT_FILE:		/* file dependency */
920 			assert(uu_list_numnodes(dv->gv_dependencies) == 0);
921 			if (uu_list_numnodes(dv->gv_dependents) == 0)
922 				graph_remove_vertex(dv);
923 			break;
924 
925 		default:
926 #ifndef NDEBUG
927 			uu_warn("%s:%d: Unexpected node type %d", __FILE__,
928 			    __LINE__, dv->gv_type);
929 #endif
930 			abort();
931 		}
932 	}
933 
934 	graph_remove_vertex(v);
935 }
936 
937 static int
938 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
939 {
940 	graph_vertex_t *v = ptrs[0];
941 	boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
942 	graph_vertex_t *dv;
943 
944 	dv = e->ge_vertex;
945 
946 	/*
947 	 * We have four possibilities here:
948 	 *   - GVT_INST: restarter
949 	 *   - GVT_GROUP - GVT_INST: instance dependency
950 	 *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
951 	 *   - GVT_GROUP - GVT_FILE: file dependency
952 	 */
953 	switch (dv->gv_type) {
954 	case GVT_INST:	/* restarter */
955 		assert(dv->gv_id == v->gv_restarter_id);
956 		if (delete_restarter_dep)
957 			graph_remove_edge(v, dv);
958 		break;
959 
960 	case GVT_GROUP:	/* pg dependency */
961 		graph_remove_edge(v, dv);
962 		delete_depgroup(dv);
963 		break;
964 
965 	case GVT_FILE:
966 		/* These are currently not direct dependencies */
967 
968 	default:
969 #ifndef NDEBUG
970 		uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
971 		    dv->gv_type);
972 #endif
973 		abort();
974 	}
975 
976 	return (UU_WALK_NEXT);
977 }
978 
979 static void
980 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
981 {
982 	void *ptrs[2];
983 	int r;
984 
985 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
986 	assert(v->gv_type == GVT_INST);
987 
988 	ptrs[0] = v;
989 	ptrs[1] = (void *)delete_restarter_dep;
990 
991 	r = uu_list_walk(v->gv_dependencies,
992 	    (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
993 	assert(r == 0);
994 }
995 
996 /*
997  * int graph_insert_vertex_unconfigured()
998  *   Insert a vertex without sending any restarter events. If the vertex
999  *   already exists or creation is successful, return a pointer to it in *vp.
1000  *
1001  *   If type is not GVT_GROUP, dt can remain unset.
1002  *
1003  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1004  *   doesn't agree with type, or type doesn't agree with dt).
1005  */
1006 static int
1007 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1008     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1009 {
1010 	int r;
1011 	int i;
1012 
1013 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1014 
1015 	switch (type) {
1016 	case GVT_SVC:
1017 	case GVT_INST:
1018 		if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1019 			return (EINVAL);
1020 		break;
1021 
1022 	case GVT_FILE:
1023 		if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1024 			return (EINVAL);
1025 		break;
1026 
1027 	case GVT_GROUP:
1028 		if (dt <= 0 || rt < 0)
1029 			return (EINVAL);
1030 		break;
1031 
1032 	default:
1033 #ifndef NDEBUG
1034 		uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1035 #endif
1036 		abort();
1037 	}
1038 
1039 	*vp = vertex_get_by_name(fmri);
1040 	if (*vp != NULL)
1041 		return (EEXIST);
1042 
1043 	*vp = graph_add_vertex(fmri);
1044 
1045 	(*vp)->gv_type = type;
1046 	(*vp)->gv_depgroup = dt;
1047 	(*vp)->gv_restart = rt;
1048 
1049 	(*vp)->gv_flags = 0;
1050 	(*vp)->gv_state = RESTARTER_STATE_NONE;
1051 
1052 	for (i = 0; special_vertices[i].name != NULL; ++i) {
1053 		if (strcmp(fmri, special_vertices[i].name) == 0) {
1054 			(*vp)->gv_start_f = special_vertices[i].start_f;
1055 			(*vp)->gv_post_online_f =
1056 			    special_vertices[i].post_online_f;
1057 			(*vp)->gv_post_disable_f =
1058 			    special_vertices[i].post_disable_f;
1059 			break;
1060 		}
1061 	}
1062 
1063 	(*vp)->gv_restarter_id = -1;
1064 	(*vp)->gv_restarter_channel = 0;
1065 
1066 	if (type == GVT_INST) {
1067 		char *sfmri;
1068 		graph_vertex_t *sv;
1069 
1070 		sfmri = inst_fmri_to_svc_fmri(fmri);
1071 		sv = vertex_get_by_name(sfmri);
1072 		if (sv == NULL) {
1073 			r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1074 			    0, &sv);
1075 			assert(r == 0);
1076 		}
1077 		startd_free(sfmri, max_scf_fmri_size);
1078 
1079 		graph_add_edge(sv, *vp);
1080 	}
1081 
1082 	/*
1083 	 * If this vertex is in the subgraph, mark it as so, for both
1084 	 * GVT_INST and GVT_SERVICE verteces.
1085 	 * A GVT_SERVICE vertex can only be in the subgraph if another instance
1086 	 * depends on it, in which case it's already been added to the graph
1087 	 * and marked as in the subgraph (by refresh_vertex()).  If a
1088 	 * GVT_SERVICE vertex was freshly added (by the code above), it means
1089 	 * that it has no dependents, and cannot be in the subgraph.
1090 	 * Regardless of this, we still check that gv_flags includes
1091 	 * GV_INSUBGRAPH in the event that future behavior causes the above
1092 	 * code to add a GVT_SERVICE vertex which should be in the subgraph.
1093 	 */
1094 
1095 	(*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1096 
1097 	return (0);
1098 }
1099 
1100 /*
1101  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1102  */
1103 static int
1104 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1105 {
1106 	hrtime_t now;
1107 
1108 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1109 
1110 	/* cycle detection */
1111 	now = gethrtime();
1112 
1113 	/* Don't follow exclusions. */
1114 	if (!(fv->gv_type == GVT_GROUP &&
1115 	    fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1116 		*pathp = is_path_to(tv, fv);
1117 		if (*pathp)
1118 			return (ELOOP);
1119 	}
1120 
1121 	dep_cycle_ns += gethrtime() - now;
1122 	++dep_inserts;
1123 	now = gethrtime();
1124 
1125 	graph_add_edge(fv, tv);
1126 
1127 	dep_insert_ns += gethrtime() - now;
1128 
1129 	/* Check if the dependency adds the "to" vertex to the subgraph */
1130 	tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1131 
1132 	return (0);
1133 }
1134 
1135 static int
1136 inst_running(graph_vertex_t *v)
1137 {
1138 	assert(v->gv_type == GVT_INST);
1139 
1140 	if (v->gv_state == RESTARTER_STATE_ONLINE ||
1141 	    v->gv_state == RESTARTER_STATE_DEGRADED)
1142 		return (1);
1143 
1144 	return (0);
1145 }
1146 
1147 /*
1148  * The dependency evaluation functions return
1149  *   1 - dependency satisfied
1150  *   0 - dependency unsatisfied
1151  *   -1 - dependency unsatisfiable (without administrator intervention)
1152  *
1153  * The functions also take a boolean satbility argument.  When true, the
1154  * functions may recurse in order to determine satisfiability.
1155  */
1156 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1157 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1158 
1159 /*
1160  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1161  * is unsatisfiable if any elements are unsatisfiable.
1162  */
1163 static int
1164 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1165 {
1166 	graph_edge_t *edge;
1167 	int i;
1168 	boolean_t any_unsatisfied;
1169 
1170 	if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1171 		return (1);
1172 
1173 	any_unsatisfied = B_FALSE;
1174 
1175 	for (edge = uu_list_first(groupv->gv_dependencies);
1176 	    edge != NULL;
1177 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1178 		i = dependency_satisfied(edge->ge_vertex, satbility);
1179 		if (i == 1)
1180 			continue;
1181 
1182 		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1183 		    "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1184 		    edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1185 
1186 		if (!satbility)
1187 			return (0);
1188 
1189 		if (i == -1)
1190 			return (-1);
1191 
1192 		any_unsatisfied = B_TRUE;
1193 	}
1194 
1195 	return (any_unsatisfied ? 0 : 1);
1196 }
1197 
1198 /*
1199  * A require_any dependency is satisfied if any element is satisfied.  It is
1200  * satisfiable if any element is satisfiable.
1201  */
1202 static int
1203 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1204 {
1205 	graph_edge_t *edge;
1206 	int s;
1207 	boolean_t satisfiable;
1208 
1209 	if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1210 		return (1);
1211 
1212 	satisfiable = B_FALSE;
1213 
1214 	for (edge = uu_list_first(groupv->gv_dependencies);
1215 	    edge != NULL;
1216 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1217 		s = dependency_satisfied(edge->ge_vertex, satbility);
1218 
1219 		if (s == 1)
1220 			return (1);
1221 
1222 		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1223 		    "require_any(%s): %s is unsatisfi%s.\n",
1224 		    groupv->gv_name, edge->ge_vertex->gv_name,
1225 		    s == 0 ? "ed" : "able");
1226 
1227 		if (satbility && s == 0)
1228 			satisfiable = B_TRUE;
1229 	}
1230 
1231 	return (!satbility || satisfiable ? 0 : -1);
1232 }
1233 
1234 /*
1235  * An optional_all dependency only considers elements which are configured,
1236  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1237  * is unsatisfied.
1238  *
1239  * Offline dependencies which are waiting for a dependency to come online are
1240  * unsatisfied.  Offline dependences which cannot possibly come online
1241  * (unsatisfiable) are always considered satisfied.
1242  */
1243 static int
1244 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1245 {
1246 	graph_edge_t *edge;
1247 	graph_vertex_t *v;
1248 	boolean_t any_qualified;
1249 	boolean_t any_unsatisfied;
1250 	int i;
1251 
1252 	any_qualified = B_FALSE;
1253 	any_unsatisfied = B_FALSE;
1254 
1255 	for (edge = uu_list_first(groupv->gv_dependencies);
1256 	    edge != NULL;
1257 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1258 		v = edge->ge_vertex;
1259 
1260 		switch (v->gv_type) {
1261 		case GVT_INST:
1262 			/* Skip missing or disabled instances */
1263 			if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1264 			    (GV_CONFIGURED | GV_ENABLED))
1265 				continue;
1266 
1267 			if (v->gv_state == RESTARTER_STATE_MAINT)
1268 				continue;
1269 
1270 			if (v->gv_flags & GV_TOOFFLINE)
1271 				continue;
1272 
1273 			any_qualified = B_TRUE;
1274 			if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1275 				/*
1276 				 * For offline dependencies, treat unsatisfiable
1277 				 * as satisfied.
1278 				 */
1279 				i = dependency_satisfied(v, B_TRUE);
1280 				if (i == -1)
1281 					i = 1;
1282 			} else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1283 				/*
1284 				 * The service is enabled, but hasn't
1285 				 * transitioned out of disabled yet.  Treat it
1286 				 * as unsatisfied (not unsatisfiable).
1287 				 */
1288 				i = 0;
1289 			} else {
1290 				i = dependency_satisfied(v, satbility);
1291 			}
1292 			break;
1293 
1294 		case GVT_FILE:
1295 			any_qualified = B_TRUE;
1296 			i = dependency_satisfied(v, satbility);
1297 
1298 			break;
1299 
1300 		case GVT_SVC: {
1301 			boolean_t svc_any_qualified;
1302 			boolean_t svc_satisfied;
1303 			boolean_t svc_satisfiable;
1304 			graph_vertex_t *v2;
1305 			graph_edge_t *e2;
1306 
1307 			svc_any_qualified = B_FALSE;
1308 			svc_satisfied = B_FALSE;
1309 			svc_satisfiable = B_FALSE;
1310 
1311 			for (e2 = uu_list_first(v->gv_dependencies);
1312 			    e2 != NULL;
1313 			    e2 = uu_list_next(v->gv_dependencies, e2)) {
1314 				v2 = e2->ge_vertex;
1315 				assert(v2->gv_type == GVT_INST);
1316 
1317 				if ((v2->gv_flags &
1318 				    (GV_CONFIGURED | GV_ENABLED)) !=
1319 				    (GV_CONFIGURED | GV_ENABLED))
1320 					continue;
1321 
1322 				if (v2->gv_state == RESTARTER_STATE_MAINT)
1323 					continue;
1324 
1325 				if (v2->gv_flags & GV_TOOFFLINE)
1326 					continue;
1327 
1328 				svc_any_qualified = B_TRUE;
1329 
1330 				if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1331 					/*
1332 					 * For offline dependencies, treat
1333 					 * unsatisfiable as satisfied.
1334 					 */
1335 					i = dependency_satisfied(v2, B_TRUE);
1336 					if (i == -1)
1337 						i = 1;
1338 				} else if (v2->gv_state ==
1339 				    RESTARTER_STATE_DISABLED) {
1340 					i = 0;
1341 				} else {
1342 					i = dependency_satisfied(v2, satbility);
1343 				}
1344 
1345 				if (i == 1) {
1346 					svc_satisfied = B_TRUE;
1347 					break;
1348 				}
1349 				if (i == 0)
1350 					svc_satisfiable = B_TRUE;
1351 			}
1352 
1353 			if (!svc_any_qualified)
1354 				continue;
1355 			any_qualified = B_TRUE;
1356 			if (svc_satisfied) {
1357 				i = 1;
1358 			} else if (svc_satisfiable) {
1359 				i = 0;
1360 			} else {
1361 				i = -1;
1362 			}
1363 			break;
1364 		}
1365 
1366 		case GVT_GROUP:
1367 		default:
1368 #ifndef NDEBUG
1369 			uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1370 			    __LINE__, v->gv_type);
1371 #endif
1372 			abort();
1373 		}
1374 
1375 		if (i == 1)
1376 			continue;
1377 
1378 		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1379 		    "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1380 		    v->gv_name, i == 0 ? "ed" : "able");
1381 
1382 		if (!satbility)
1383 			return (0);
1384 		if (i == -1)
1385 			return (-1);
1386 		any_unsatisfied = B_TRUE;
1387 	}
1388 
1389 	if (!any_qualified)
1390 		return (1);
1391 
1392 	return (any_unsatisfied ? 0 : 1);
1393 }
1394 
1395 /*
1396  * An exclude_all dependency is unsatisfied if any non-service element is
1397  * satisfied or any service instance which is configured, enabled, and not in
1398  * maintenance is satisfied.  Usually when unsatisfied, it is also
1399  * unsatisfiable.
1400  */
1401 #define	LOG_EXCLUDE(u, v)						\
1402 	log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,			\
1403 	    "exclude_all(%s): %s is satisfied.\n",			\
1404 	    (u)->gv_name, (v)->gv_name)
1405 
1406 /* ARGSUSED */
1407 static int
1408 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1409 {
1410 	graph_edge_t *edge, *e2;
1411 	graph_vertex_t *v, *v2;
1412 
1413 	for (edge = uu_list_first(groupv->gv_dependencies);
1414 	    edge != NULL;
1415 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1416 		v = edge->ge_vertex;
1417 
1418 		switch (v->gv_type) {
1419 		case GVT_INST:
1420 			if ((v->gv_flags & GV_CONFIGURED) == 0)
1421 				continue;
1422 
1423 			switch (v->gv_state) {
1424 			case RESTARTER_STATE_ONLINE:
1425 			case RESTARTER_STATE_DEGRADED:
1426 				LOG_EXCLUDE(groupv, v);
1427 				return (v->gv_flags & GV_ENABLED ? -1 : 0);
1428 
1429 			case RESTARTER_STATE_OFFLINE:
1430 			case RESTARTER_STATE_UNINIT:
1431 				LOG_EXCLUDE(groupv, v);
1432 				return (0);
1433 
1434 			case RESTARTER_STATE_DISABLED:
1435 			case RESTARTER_STATE_MAINT:
1436 				continue;
1437 
1438 			default:
1439 #ifndef NDEBUG
1440 				uu_warn("%s:%d: Unexpected vertex state %d.\n",
1441 				    __FILE__, __LINE__, v->gv_state);
1442 #endif
1443 				abort();
1444 			}
1445 			/* NOTREACHED */
1446 
1447 		case GVT_SVC:
1448 			break;
1449 
1450 		case GVT_FILE:
1451 			if (!file_ready(v))
1452 				continue;
1453 			LOG_EXCLUDE(groupv, v);
1454 			return (-1);
1455 
1456 		case GVT_GROUP:
1457 		default:
1458 #ifndef NDEBUG
1459 			uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1460 			    __LINE__, v->gv_type);
1461 #endif
1462 			abort();
1463 		}
1464 
1465 		/* v represents a service */
1466 		if (uu_list_numnodes(v->gv_dependencies) == 0)
1467 			continue;
1468 
1469 		for (e2 = uu_list_first(v->gv_dependencies);
1470 		    e2 != NULL;
1471 		    e2 = uu_list_next(v->gv_dependencies, e2)) {
1472 			v2 = e2->ge_vertex;
1473 			assert(v2->gv_type == GVT_INST);
1474 
1475 			if ((v2->gv_flags & GV_CONFIGURED) == 0)
1476 				continue;
1477 
1478 			switch (v2->gv_state) {
1479 			case RESTARTER_STATE_ONLINE:
1480 			case RESTARTER_STATE_DEGRADED:
1481 				LOG_EXCLUDE(groupv, v2);
1482 				return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1483 
1484 			case RESTARTER_STATE_OFFLINE:
1485 			case RESTARTER_STATE_UNINIT:
1486 				LOG_EXCLUDE(groupv, v2);
1487 				return (0);
1488 
1489 			case RESTARTER_STATE_DISABLED:
1490 			case RESTARTER_STATE_MAINT:
1491 				continue;
1492 
1493 			default:
1494 #ifndef NDEBUG
1495 				uu_warn("%s:%d: Unexpected vertex type %d.\n",
1496 				    __FILE__, __LINE__, v2->gv_type);
1497 #endif
1498 				abort();
1499 			}
1500 		}
1501 	}
1502 
1503 	return (1);
1504 }
1505 
1506 /*
1507  * int instance_satisfied()
1508  *   Determine if all the dependencies are satisfied for the supplied instance
1509  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1510  *   without administrator intervention.
1511  */
1512 static int
1513 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1514 {
1515 	assert(v->gv_type == GVT_INST);
1516 	assert(!inst_running(v));
1517 
1518 	return (require_all_satisfied(v, satbility));
1519 }
1520 
1521 /*
1522  * Decide whether v can satisfy a dependency.  v can either be a child of
1523  * a group vertex, or of an instance vertex.
1524  */
1525 static int
1526 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1527 {
1528 	switch (v->gv_type) {
1529 	case GVT_INST:
1530 		if ((v->gv_flags & GV_CONFIGURED) == 0) {
1531 			if (v->gv_flags & GV_DEATHROW) {
1532 				/*
1533 				 * A dependency on an instance with GV_DEATHROW
1534 				 * flag is always considered as satisfied.
1535 				 */
1536 				return (1);
1537 			}
1538 			return (-1);
1539 		}
1540 
1541 		/*
1542 		 * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1543 		 * to have its dependencies unsatisfiable.
1544 		 */
1545 		if (v->gv_flags & GV_TOOFFLINE)
1546 			return (-1);
1547 
1548 		switch (v->gv_state) {
1549 		case RESTARTER_STATE_ONLINE:
1550 		case RESTARTER_STATE_DEGRADED:
1551 			return (1);
1552 
1553 		case RESTARTER_STATE_OFFLINE:
1554 			if (!satbility)
1555 				return (0);
1556 			return (instance_satisfied(v, satbility) != -1 ?
1557 			    0 : -1);
1558 
1559 		case RESTARTER_STATE_DISABLED:
1560 		case RESTARTER_STATE_MAINT:
1561 			return (-1);
1562 
1563 		case RESTARTER_STATE_UNINIT:
1564 			return (0);
1565 
1566 		default:
1567 #ifndef NDEBUG
1568 			uu_warn("%s:%d: Unexpected vertex state %d.\n",
1569 			    __FILE__, __LINE__, v->gv_state);
1570 #endif
1571 			abort();
1572 			/* NOTREACHED */
1573 		}
1574 
1575 	case GVT_SVC:
1576 		if (uu_list_numnodes(v->gv_dependencies) == 0)
1577 			return (-1);
1578 		return (require_any_satisfied(v, satbility));
1579 
1580 	case GVT_FILE:
1581 		/* i.e., we assume files will not be automatically generated */
1582 		return (file_ready(v) ? 1 : -1);
1583 
1584 	case GVT_GROUP:
1585 		break;
1586 
1587 	default:
1588 #ifndef NDEBUG
1589 		uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1590 		    v->gv_type);
1591 #endif
1592 		abort();
1593 		/* NOTREACHED */
1594 	}
1595 
1596 	switch (v->gv_depgroup) {
1597 	case DEPGRP_REQUIRE_ANY:
1598 		return (require_any_satisfied(v, satbility));
1599 
1600 	case DEPGRP_REQUIRE_ALL:
1601 		return (require_all_satisfied(v, satbility));
1602 
1603 	case DEPGRP_OPTIONAL_ALL:
1604 		return (optional_all_satisfied(v, satbility));
1605 
1606 	case DEPGRP_EXCLUDE_ALL:
1607 		return (exclude_all_satisfied(v, satbility));
1608 
1609 	default:
1610 #ifndef NDEBUG
1611 		uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1612 		    __LINE__, v->gv_depgroup);
1613 #endif
1614 		abort();
1615 	}
1616 }
1617 
1618 void
1619 graph_start_if_satisfied(graph_vertex_t *v)
1620 {
1621 	if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1622 	    instance_satisfied(v, B_FALSE) == 1) {
1623 		if (v->gv_start_f == NULL)
1624 			vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1625 		else
1626 			v->gv_start_f(v);
1627 	}
1628 }
1629 
1630 /*
1631  * propagate_satbility()
1632  *
1633  * This function is used when the given vertex changes state in such a way that
1634  * one of its dependents may become unsatisfiable.  This happens when an
1635  * instance transitions between offline -> online, or from !running ->
1636  * maintenance, as well as when an instance is removed from the graph.
1637  *
1638  * We have to walk all the dependents, since optional_all dependencies several
1639  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1640  *
1641  *	+-----+  optional_all  +-----+  require_all  +-----+
1642  *	|  A  |--------------->|  B  |-------------->|  C  |
1643  *	+-----+                +-----+               +-----+
1644  *
1645  *	                                        offline -> maintenance
1646  *
1647  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1648  * an optional dependency, what was previously an unsatisfiable situation is now
1649  * satisfied (B will never come online, even though its state hasn't changed).
1650  *
1651  * Note that it's not necessary to continue examining dependents after reaching
1652  * an optional_all dependency.  It's not possible for an optional_all dependency
1653  * to change satisfiability without also coming online, in which case we get a
1654  * start event and propagation continues naturally.  However, it does no harm to
1655  * continue propagating satisfiability (as it is a relatively rare event), and
1656  * keeps the walker code simple and generic.
1657  */
1658 /*ARGSUSED*/
1659 static int
1660 satbility_cb(graph_vertex_t *v, void *arg)
1661 {
1662 	if (v->gv_type == GVT_INST)
1663 		graph_start_if_satisfied(v);
1664 
1665 	return (UU_WALK_NEXT);
1666 }
1667 
1668 static void
1669 propagate_satbility(graph_vertex_t *v)
1670 {
1671 	graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1672 }
1673 
1674 static void propagate_stop(graph_vertex_t *, void *);
1675 
1676 /* ARGSUSED */
1677 static void
1678 propagate_start(graph_vertex_t *v, void *arg)
1679 {
1680 	switch (v->gv_type) {
1681 	case GVT_INST:
1682 		graph_start_if_satisfied(v);
1683 		break;
1684 
1685 	case GVT_GROUP:
1686 		if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1687 			graph_walk_dependents(v, propagate_stop,
1688 			    (void *)RERR_RESTART);
1689 			break;
1690 		}
1691 		/* FALLTHROUGH */
1692 
1693 	case GVT_SVC:
1694 		graph_walk_dependents(v, propagate_start, NULL);
1695 		break;
1696 
1697 	case GVT_FILE:
1698 #ifndef NDEBUG
1699 		uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1700 		    __FILE__, __LINE__);
1701 #endif
1702 		abort();
1703 		/* NOTREACHED */
1704 
1705 	default:
1706 #ifndef NDEBUG
1707 		uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1708 		    v->gv_type);
1709 #endif
1710 		abort();
1711 	}
1712 }
1713 
1714 static void
1715 propagate_stop(graph_vertex_t *v, void *arg)
1716 {
1717 	graph_edge_t *e;
1718 	graph_vertex_t *svc;
1719 	restarter_error_t err = (restarter_error_t)arg;
1720 
1721 	switch (v->gv_type) {
1722 	case GVT_INST:
1723 		/* Restarter */
1724 		if (err > RERR_NONE && inst_running(v))
1725 			vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1726 		break;
1727 
1728 	case GVT_SVC:
1729 		graph_walk_dependents(v, propagate_stop, arg);
1730 		break;
1731 
1732 	case GVT_FILE:
1733 #ifndef NDEBUG
1734 		uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1735 		    __FILE__, __LINE__);
1736 #endif
1737 		abort();
1738 		/* NOTREACHED */
1739 
1740 	case GVT_GROUP:
1741 		if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1742 			graph_walk_dependents(v, propagate_start, NULL);
1743 			break;
1744 		}
1745 
1746 		if (err == RERR_NONE || err > v->gv_restart)
1747 			break;
1748 
1749 		assert(uu_list_numnodes(v->gv_dependents) == 1);
1750 		e = uu_list_first(v->gv_dependents);
1751 		svc = e->ge_vertex;
1752 
1753 		if (inst_running(svc))
1754 			vertex_send_event(svc, RESTARTER_EVENT_TYPE_STOP);
1755 		break;
1756 
1757 	default:
1758 #ifndef NDEBUG
1759 		uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1760 		    v->gv_type);
1761 #endif
1762 		abort();
1763 	}
1764 }
1765 
1766 static void
1767 offline_vertex(graph_vertex_t *v)
1768 {
1769 	scf_handle_t *h = libscf_handle_create_bound_loop();
1770 	scf_instance_t *scf_inst = safe_scf_instance_create(h);
1771 	scf_propertygroup_t *pg = safe_scf_pg_create(h);
1772 	restarter_instance_state_t state, next_state;
1773 	int r;
1774 
1775 	assert(v->gv_type == GVT_INST);
1776 
1777 	if (scf_inst == NULL)
1778 		bad_error("safe_scf_instance_create", scf_error());
1779 	if (pg == NULL)
1780 		bad_error("safe_scf_pg_create", scf_error());
1781 
1782 	/* if the vertex is already going offline, return */
1783 rep_retry:
1784 	if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1785 	    NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1786 		switch (scf_error()) {
1787 		case SCF_ERROR_CONNECTION_BROKEN:
1788 			libscf_handle_rebind(h);
1789 			goto rep_retry;
1790 
1791 		case SCF_ERROR_NOT_FOUND:
1792 			scf_pg_destroy(pg);
1793 			scf_instance_destroy(scf_inst);
1794 			(void) scf_handle_unbind(h);
1795 			scf_handle_destroy(h);
1796 			return;
1797 		}
1798 		uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1799 		    scf_strerror(scf_error()));
1800 	}
1801 
1802 	r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1803 	if (r != 0) {
1804 		switch (scf_error()) {
1805 		case SCF_ERROR_CONNECTION_BROKEN:
1806 			libscf_handle_rebind(h);
1807 			goto rep_retry;
1808 
1809 		case SCF_ERROR_NOT_SET:
1810 		case SCF_ERROR_NOT_FOUND:
1811 			scf_pg_destroy(pg);
1812 			scf_instance_destroy(scf_inst);
1813 			(void) scf_handle_unbind(h);
1814 			scf_handle_destroy(h);
1815 			return;
1816 
1817 		default:
1818 			bad_error("scf_instance_get_pg", scf_error());
1819 		}
1820 	} else {
1821 		r = libscf_read_states(pg, &state, &next_state);
1822 		if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1823 		    next_state == RESTARTER_STATE_DISABLED)) {
1824 			log_framework(LOG_DEBUG,
1825 			    "%s: instance is already going down.\n",
1826 			    v->gv_name);
1827 			scf_pg_destroy(pg);
1828 			scf_instance_destroy(scf_inst);
1829 			(void) scf_handle_unbind(h);
1830 			scf_handle_destroy(h);
1831 			return;
1832 		}
1833 	}
1834 
1835 	scf_pg_destroy(pg);
1836 	scf_instance_destroy(scf_inst);
1837 	(void) scf_handle_unbind(h);
1838 	scf_handle_destroy(h);
1839 
1840 	vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1841 }
1842 
1843 /*
1844  * void graph_enable_by_vertex()
1845  *   If admin is non-zero, this is an administrative request for change
1846  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1847  *   a plain DISABLE restarter event.
1848  */
1849 void
1850 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1851 {
1852 	graph_vertex_t *v;
1853 	int r;
1854 
1855 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1856 	assert((vertex->gv_flags & GV_CONFIGURED));
1857 
1858 	vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1859 	    (enable ? GV_ENABLED : 0);
1860 
1861 	if (enable) {
1862 		if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1863 		    vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1864 		    vertex->gv_state != RESTARTER_STATE_ONLINE) {
1865 			/*
1866 			 * In case the vertex was notified to go down,
1867 			 * but now can return online, clear the _TOOFFLINE
1868 			 * and _TODISABLE flags.
1869 			 */
1870 			vertex->gv_flags &= ~GV_TOOFFLINE;
1871 			vertex->gv_flags &= ~GV_TODISABLE;
1872 
1873 			vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1874 		}
1875 
1876 		/*
1877 		 * Wait for state update from restarter before sending _START or
1878 		 * _STOP.
1879 		 */
1880 
1881 		return;
1882 	}
1883 
1884 	if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1885 		return;
1886 
1887 	if (!admin) {
1888 		vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1889 
1890 		/*
1891 		 * Wait for state update from restarter before sending _START or
1892 		 * _STOP.
1893 		 */
1894 
1895 		return;
1896 	}
1897 
1898 	/*
1899 	 * If it is a DISABLE event requested by the administrator then we are
1900 	 * offlining the dependents first.
1901 	 */
1902 
1903 	/*
1904 	 * Set GV_TOOFFLINE for the services we are offlining. We cannot
1905 	 * clear the GV_TOOFFLINE bits from all the services because
1906 	 * other DISABLE events might be handled at the same time.
1907 	 */
1908 	vertex->gv_flags |= GV_TOOFFLINE;
1909 
1910 	/* remember which vertex to disable... */
1911 	vertex->gv_flags |= GV_TODISABLE;
1912 
1913 	log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1914 	    "disabling %s.\n", vertex->gv_name);
1915 
1916 	/* set GV_TOOFFLINE for its dependents */
1917 	r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1918 	    NULL, 0);
1919 	assert(r == 0);
1920 
1921 	/* disable the instance now if there is nothing else to offline */
1922 	if (insubtree_dependents_down(vertex) == B_TRUE) {
1923 		vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1924 		return;
1925 	}
1926 
1927 	/*
1928 	 * This loop is similar to the one used for the graph reversal shutdown
1929 	 * and could be improved in term of performance for the subtree reversal
1930 	 * disable case.
1931 	 */
1932 	for (v = uu_list_first(dgraph); v != NULL;
1933 	    v = uu_list_next(dgraph, v)) {
1934 		/* skip the vertex we are disabling for now */
1935 		if (v == vertex)
1936 			continue;
1937 
1938 		if (v->gv_type != GVT_INST ||
1939 		    (v->gv_flags & GV_CONFIGURED) == 0 ||
1940 		    (v->gv_flags & GV_ENABLED) == 0 ||
1941 		    (v->gv_flags & GV_TOOFFLINE) == 0)
1942 			continue;
1943 
1944 		if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1945 		    (v->gv_state != RESTARTER_STATE_DEGRADED)) {
1946 			/* continue if there is nothing to offline */
1947 			continue;
1948 		}
1949 
1950 		/*
1951 		 * Instances which are up need to come down before we're
1952 		 * done, but we can only offline the leaves here. An
1953 		 * instance is a leaf when all its dependents are down.
1954 		 */
1955 		if (insubtree_dependents_down(v) == B_TRUE) {
1956 			log_framework(LOG_DEBUG, "Offlining in-subtree "
1957 			    "instance %s for %s.\n",
1958 			    v->gv_name, vertex->gv_name);
1959 			offline_vertex(v);
1960 		}
1961 	}
1962 }
1963 
1964 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
1965 
1966 /*
1967  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
1968  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
1969  * v is already configured and fmri_arg indicates the current restarter, do
1970  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
1971  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
1972  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
1973  * ECONNABORTED if the repository connection is broken, and ELOOP
1974  * if the dependency would create a cycle.  In the last case, *pathp will
1975  * point to a -1-terminated array of ids which compose the path from v to
1976  * restarter_fmri.
1977  */
1978 int
1979 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
1980     int **pathp)
1981 {
1982 	char *restarter_fmri = NULL;
1983 	graph_vertex_t *rv;
1984 	int err;
1985 	int id;
1986 
1987 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
1988 
1989 	if (fmri_arg[0] != '\0') {
1990 		err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
1991 		if (err != 0) {
1992 			assert(err == EINVAL);
1993 			return (err);
1994 		}
1995 	}
1996 
1997 	if (restarter_fmri == NULL ||
1998 	    strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
1999 		if (v->gv_flags & GV_CONFIGURED) {
2000 			if (v->gv_restarter_id == -1) {
2001 				if (restarter_fmri != NULL)
2002 					startd_free(restarter_fmri,
2003 					    max_scf_fmri_size);
2004 				return (0);
2005 			}
2006 
2007 			graph_unset_restarter(v);
2008 		}
2009 
2010 		/* Master restarter, nothing to do. */
2011 		v->gv_restarter_id = -1;
2012 		v->gv_restarter_channel = NULL;
2013 		vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2014 		return (0);
2015 	}
2016 
2017 	if (v->gv_flags & GV_CONFIGURED) {
2018 		id = dict_lookup_byname(restarter_fmri);
2019 		if (id != -1 && v->gv_restarter_id == id) {
2020 			startd_free(restarter_fmri, max_scf_fmri_size);
2021 			return (0);
2022 		}
2023 
2024 		graph_unset_restarter(v);
2025 	}
2026 
2027 	err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2028 	    RERR_NONE, &rv);
2029 	startd_free(restarter_fmri, max_scf_fmri_size);
2030 	assert(err == 0 || err == EEXIST);
2031 
2032 	if (rv->gv_delegate_initialized == 0) {
2033 		rv->gv_delegate_channel = restarter_protocol_init_delegate(
2034 		    rv->gv_name);
2035 		rv->gv_delegate_initialized = 1;
2036 	}
2037 	v->gv_restarter_id = rv->gv_id;
2038 	v->gv_restarter_channel = rv->gv_delegate_channel;
2039 
2040 	err = graph_insert_dependency(v, rv, pathp);
2041 	if (err != 0) {
2042 		assert(err == ELOOP);
2043 		return (ELOOP);
2044 	}
2045 
2046 	vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2047 
2048 	if (!(rv->gv_flags & GV_CONFIGURED)) {
2049 		scf_instance_t *inst;
2050 
2051 		err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2052 		switch (err) {
2053 		case 0:
2054 			err = configure_vertex(rv, inst);
2055 			scf_instance_destroy(inst);
2056 			switch (err) {
2057 			case 0:
2058 			case ECANCELED:
2059 				break;
2060 
2061 			case ECONNABORTED:
2062 				return (ECONNABORTED);
2063 
2064 			default:
2065 				bad_error("configure_vertex", err);
2066 			}
2067 			break;
2068 
2069 		case ECONNABORTED:
2070 			return (ECONNABORTED);
2071 
2072 		case ENOENT:
2073 			break;
2074 
2075 		case ENOTSUP:
2076 			/*
2077 			 * The fmri doesn't specify an instance - translate
2078 			 * to EINVAL.
2079 			 */
2080 			return (EINVAL);
2081 
2082 		case EINVAL:
2083 		default:
2084 			bad_error("libscf_fmri_get_instance", err);
2085 		}
2086 	}
2087 
2088 	return (0);
2089 }
2090 
2091 
2092 /*
2093  * Add all of the instances of the service named by fmri to the graph.
2094  * Returns
2095  *   0 - success
2096  *   ENOENT - service indicated by fmri does not exist
2097  *
2098  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2099  * otherwise.
2100  */
2101 static int
2102 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2103 {
2104 	scf_service_t *svc;
2105 	scf_instance_t *inst;
2106 	scf_iter_t *iter;
2107 	char *inst_fmri;
2108 	int ret, r;
2109 
2110 	*reboundp = B_FALSE;
2111 
2112 	svc = safe_scf_service_create(h);
2113 	inst = safe_scf_instance_create(h);
2114 	iter = safe_scf_iter_create(h);
2115 	inst_fmri = startd_alloc(max_scf_fmri_size);
2116 
2117 rebound:
2118 	if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2119 	    SCF_DECODE_FMRI_EXACT) != 0) {
2120 		switch (scf_error()) {
2121 		case SCF_ERROR_CONNECTION_BROKEN:
2122 		default:
2123 			libscf_handle_rebind(h);
2124 			*reboundp = B_TRUE;
2125 			goto rebound;
2126 
2127 		case SCF_ERROR_NOT_FOUND:
2128 			ret = ENOENT;
2129 			goto out;
2130 
2131 		case SCF_ERROR_INVALID_ARGUMENT:
2132 		case SCF_ERROR_CONSTRAINT_VIOLATED:
2133 		case SCF_ERROR_NOT_BOUND:
2134 		case SCF_ERROR_HANDLE_MISMATCH:
2135 			bad_error("scf_handle_decode_fmri", scf_error());
2136 		}
2137 	}
2138 
2139 	if (scf_iter_service_instances(iter, svc) != 0) {
2140 		switch (scf_error()) {
2141 		case SCF_ERROR_CONNECTION_BROKEN:
2142 		default:
2143 			libscf_handle_rebind(h);
2144 			*reboundp = B_TRUE;
2145 			goto rebound;
2146 
2147 		case SCF_ERROR_DELETED:
2148 			ret = ENOENT;
2149 			goto out;
2150 
2151 		case SCF_ERROR_HANDLE_MISMATCH:
2152 		case SCF_ERROR_NOT_BOUND:
2153 		case SCF_ERROR_NOT_SET:
2154 			bad_error("scf_iter_service_instances", scf_error());
2155 		}
2156 	}
2157 
2158 	for (;;) {
2159 		r = scf_iter_next_instance(iter, inst);
2160 		if (r == 0)
2161 			break;
2162 		if (r != 1) {
2163 			switch (scf_error()) {
2164 			case SCF_ERROR_CONNECTION_BROKEN:
2165 			default:
2166 				libscf_handle_rebind(h);
2167 				*reboundp = B_TRUE;
2168 				goto rebound;
2169 
2170 			case SCF_ERROR_DELETED:
2171 				ret = ENOENT;
2172 				goto out;
2173 
2174 			case SCF_ERROR_HANDLE_MISMATCH:
2175 			case SCF_ERROR_NOT_BOUND:
2176 			case SCF_ERROR_NOT_SET:
2177 			case SCF_ERROR_INVALID_ARGUMENT:
2178 				bad_error("scf_iter_next_instance",
2179 				    scf_error());
2180 			}
2181 		}
2182 
2183 		if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2184 		    0) {
2185 			switch (scf_error()) {
2186 			case SCF_ERROR_CONNECTION_BROKEN:
2187 				libscf_handle_rebind(h);
2188 				*reboundp = B_TRUE;
2189 				goto rebound;
2190 
2191 			case SCF_ERROR_DELETED:
2192 				continue;
2193 
2194 			case SCF_ERROR_NOT_BOUND:
2195 			case SCF_ERROR_NOT_SET:
2196 				bad_error("scf_instance_to_fmri", scf_error());
2197 			}
2198 		}
2199 
2200 		r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2201 		switch (r) {
2202 		case 0:
2203 		case ECANCELED:
2204 			break;
2205 
2206 		case EEXIST:
2207 			continue;
2208 
2209 		case ECONNABORTED:
2210 			libscf_handle_rebind(h);
2211 			*reboundp = B_TRUE;
2212 			goto rebound;
2213 
2214 		case EINVAL:
2215 		default:
2216 			bad_error("dgraph_add_instance", r);
2217 		}
2218 	}
2219 
2220 	ret = 0;
2221 
2222 out:
2223 	startd_free(inst_fmri, max_scf_fmri_size);
2224 	scf_iter_destroy(iter);
2225 	scf_instance_destroy(inst);
2226 	scf_service_destroy(svc);
2227 	return (ret);
2228 }
2229 
2230 struct depfmri_info {
2231 	graph_vertex_t	*v;		/* GVT_GROUP vertex */
2232 	gv_type_t	type;		/* type of dependency */
2233 	const char	*inst_fmri;	/* FMRI of parental GVT_INST vert. */
2234 	const char	*pg_name;	/* Name of dependency pg */
2235 	scf_handle_t	*h;
2236 	int		err;		/* return error code */
2237 	int		**pathp;	/* return circular dependency path */
2238 };
2239 
2240 /*
2241  * Find or create a vertex for fmri and make info->v depend on it.
2242  * Returns
2243  *   0 - success
2244  *   nonzero - failure
2245  *
2246  * On failure, sets info->err to
2247  *   EINVAL - fmri is invalid
2248  *	      fmri does not match info->type
2249  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2250  *	     will point to an array of the ids of the members of the cycle.
2251  *   ECONNABORTED - repository connection was broken
2252  *   ECONNRESET - succeeded, but repository connection was reset
2253  */
2254 static int
2255 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2256 {
2257 	int err;
2258 	graph_vertex_t *depgroup_v, *v;
2259 	char *fmri_copy, *cfmri;
2260 	size_t fmri_copy_sz;
2261 	const char *scope, *service, *instance, *pg;
2262 	scf_instance_t *inst;
2263 	boolean_t rebound;
2264 
2265 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2266 
2267 	/* Get or create vertex for FMRI */
2268 	depgroup_v = info->v;
2269 
2270 	if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2271 		if (info->type != GVT_FILE) {
2272 			log_framework(LOG_NOTICE,
2273 			    "FMRI \"%s\" is not allowed for the \"%s\" "
2274 			    "dependency's type of instance %s.\n", fmri,
2275 			    info->pg_name, info->inst_fmri);
2276 			return (info->err = EINVAL);
2277 		}
2278 
2279 		err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2280 		    RERR_NONE, &v);
2281 		switch (err) {
2282 		case 0:
2283 			break;
2284 
2285 		case EEXIST:
2286 			assert(v->gv_type == GVT_FILE);
2287 			break;
2288 
2289 		case EINVAL:		/* prevented above */
2290 		default:
2291 			bad_error("graph_insert_vertex_unconfigured", err);
2292 		}
2293 	} else {
2294 		if (info->type != GVT_INST) {
2295 			log_framework(LOG_NOTICE,
2296 			    "FMRI \"%s\" is not allowed for the \"%s\" "
2297 			    "dependency's type of instance %s.\n", fmri,
2298 			    info->pg_name, info->inst_fmri);
2299 			return (info->err = EINVAL);
2300 		}
2301 
2302 		/*
2303 		 * We must canonify fmri & add a vertex for it.
2304 		 */
2305 		fmri_copy_sz = strlen(fmri) + 1;
2306 		fmri_copy = startd_alloc(fmri_copy_sz);
2307 		(void) strcpy(fmri_copy, fmri);
2308 
2309 		/* Determine if the FMRI is a property group or instance */
2310 		if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2311 		    &instance, &pg, NULL) != 0) {
2312 			startd_free(fmri_copy, fmri_copy_sz);
2313 			log_framework(LOG_NOTICE,
2314 			    "Dependency \"%s\" of %s has invalid FMRI "
2315 			    "\"%s\".\n", info->pg_name, info->inst_fmri,
2316 			    fmri);
2317 			return (info->err = EINVAL);
2318 		}
2319 
2320 		if (service == NULL || pg != NULL) {
2321 			startd_free(fmri_copy, fmri_copy_sz);
2322 			log_framework(LOG_NOTICE,
2323 			    "Dependency \"%s\" of %s does not designate a "
2324 			    "service or instance.\n", info->pg_name,
2325 			    info->inst_fmri);
2326 			return (info->err = EINVAL);
2327 		}
2328 
2329 		if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2330 			cfmri = uu_msprintf("svc:/%s%s%s",
2331 			    service, instance ? ":" : "", instance ? instance :
2332 			    "");
2333 		} else {
2334 			cfmri = uu_msprintf("svc://%s/%s%s%s",
2335 			    scope, service, instance ? ":" : "", instance ?
2336 			    instance : "");
2337 		}
2338 
2339 		startd_free(fmri_copy, fmri_copy_sz);
2340 
2341 		err = graph_insert_vertex_unconfigured(cfmri, instance ?
2342 		    GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2343 		    RERR_NONE, &v);
2344 		uu_free(cfmri);
2345 		switch (err) {
2346 		case 0:
2347 			break;
2348 
2349 		case EEXIST:
2350 			/* Verify v. */
2351 			if (instance != NULL)
2352 				assert(v->gv_type == GVT_INST);
2353 			else
2354 				assert(v->gv_type == GVT_SVC);
2355 			break;
2356 
2357 		default:
2358 			bad_error("graph_insert_vertex_unconfigured", err);
2359 		}
2360 	}
2361 
2362 	/* Add dependency from depgroup_v to new vertex */
2363 	info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2364 	switch (info->err) {
2365 	case 0:
2366 		break;
2367 
2368 	case ELOOP:
2369 		return (ELOOP);
2370 
2371 	default:
2372 		bad_error("graph_insert_dependency", info->err);
2373 	}
2374 
2375 	/* This must be after we insert the dependency, to avoid looping. */
2376 	switch (v->gv_type) {
2377 	case GVT_INST:
2378 		if ((v->gv_flags & GV_CONFIGURED) != 0)
2379 			break;
2380 
2381 		inst = safe_scf_instance_create(info->h);
2382 
2383 		rebound = B_FALSE;
2384 
2385 rebound:
2386 		err = libscf_lookup_instance(v->gv_name, inst);
2387 		switch (err) {
2388 		case 0:
2389 			err = configure_vertex(v, inst);
2390 			switch (err) {
2391 			case 0:
2392 			case ECANCELED:
2393 				break;
2394 
2395 			case ECONNABORTED:
2396 				libscf_handle_rebind(info->h);
2397 				rebound = B_TRUE;
2398 				goto rebound;
2399 
2400 			default:
2401 				bad_error("configure_vertex", err);
2402 			}
2403 			break;
2404 
2405 		case ENOENT:
2406 			break;
2407 
2408 		case ECONNABORTED:
2409 			libscf_handle_rebind(info->h);
2410 			rebound = B_TRUE;
2411 			goto rebound;
2412 
2413 		case EINVAL:
2414 		case ENOTSUP:
2415 		default:
2416 			bad_error("libscf_fmri_get_instance", err);
2417 		}
2418 
2419 		scf_instance_destroy(inst);
2420 
2421 		if (rebound)
2422 			return (info->err = ECONNRESET);
2423 		break;
2424 
2425 	case GVT_SVC:
2426 		(void) add_service(v->gv_name, info->h, &rebound);
2427 		if (rebound)
2428 			return (info->err = ECONNRESET);
2429 	}
2430 
2431 	return (0);
2432 }
2433 
2434 struct deppg_info {
2435 	graph_vertex_t	*v;		/* GVT_INST vertex */
2436 	int		err;		/* return error */
2437 	int		**pathp;	/* return circular dependency path */
2438 };
2439 
2440 /*
2441  * Make info->v depend on a new GVT_GROUP node for this property group,
2442  * and then call process_dependency_fmri() for the values of the entity
2443  * property.  Return 0 on success, or if something goes wrong return nonzero
2444  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2445  * process_dependency_fmri().
2446  */
2447 static int
2448 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2449 {
2450 	scf_handle_t *h;
2451 	depgroup_type_t deptype;
2452 	restarter_error_t rerr;
2453 	struct depfmri_info linfo;
2454 	char *fmri, *pg_name;
2455 	size_t fmri_sz;
2456 	graph_vertex_t *depgrp;
2457 	scf_property_t *prop;
2458 	int err;
2459 	int empty;
2460 	scf_error_t scferr;
2461 	ssize_t len;
2462 
2463 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2464 
2465 	h = scf_pg_handle(pg);
2466 
2467 	pg_name = startd_alloc(max_scf_name_size);
2468 
2469 	len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2470 	if (len < 0) {
2471 		startd_free(pg_name, max_scf_name_size);
2472 		switch (scf_error()) {
2473 		case SCF_ERROR_CONNECTION_BROKEN:
2474 		default:
2475 			return (info->err = ECONNABORTED);
2476 
2477 		case SCF_ERROR_DELETED:
2478 			return (info->err = 0);
2479 
2480 		case SCF_ERROR_NOT_SET:
2481 			bad_error("scf_pg_get_name", scf_error());
2482 		}
2483 	}
2484 
2485 	/*
2486 	 * Skip over empty dependency groups.  Since dependency property
2487 	 * groups are updated atomically, they are either empty or
2488 	 * fully populated.
2489 	 */
2490 	empty = depgroup_empty(h, pg);
2491 	if (empty < 0) {
2492 		log_error(LOG_INFO,
2493 		    "Error reading dependency group \"%s\" of %s: %s\n",
2494 		    pg_name, info->v->gv_name, scf_strerror(scf_error()));
2495 		startd_free(pg_name, max_scf_name_size);
2496 		return (info->err = EINVAL);
2497 
2498 	} else if (empty == 1) {
2499 		log_framework(LOG_DEBUG,
2500 		    "Ignoring empty dependency group \"%s\" of %s\n",
2501 		    pg_name, info->v->gv_name);
2502 		startd_free(pg_name, max_scf_name_size);
2503 		return (info->err = 0);
2504 	}
2505 
2506 	fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2507 	fmri = startd_alloc(fmri_sz);
2508 
2509 	(void) snprintf(fmri, max_scf_name_size, "%s>%s", info->v->gv_name,
2510 	    pg_name);
2511 
2512 	/* Validate the pg before modifying the graph */
2513 	deptype = depgroup_read_grouping(h, pg);
2514 	if (deptype == DEPGRP_UNSUPPORTED) {
2515 		log_error(LOG_INFO,
2516 		    "Dependency \"%s\" of %s has an unknown grouping value.\n",
2517 		    pg_name, info->v->gv_name);
2518 		startd_free(fmri, fmri_sz);
2519 		startd_free(pg_name, max_scf_name_size);
2520 		return (info->err = EINVAL);
2521 	}
2522 
2523 	rerr = depgroup_read_restart(h, pg);
2524 	if (rerr == RERR_UNSUPPORTED) {
2525 		log_error(LOG_INFO,
2526 		    "Dependency \"%s\" of %s has an unknown restart_on value."
2527 		    "\n", pg_name, info->v->gv_name);
2528 		startd_free(fmri, fmri_sz);
2529 		startd_free(pg_name, max_scf_name_size);
2530 		return (info->err = EINVAL);
2531 	}
2532 
2533 	prop = safe_scf_property_create(h);
2534 
2535 	if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2536 		scferr = scf_error();
2537 		scf_property_destroy(prop);
2538 		if (scferr == SCF_ERROR_DELETED) {
2539 			startd_free(fmri, fmri_sz);
2540 			startd_free(pg_name, max_scf_name_size);
2541 			return (info->err = 0);
2542 		} else if (scferr != SCF_ERROR_NOT_FOUND) {
2543 			startd_free(fmri, fmri_sz);
2544 			startd_free(pg_name, max_scf_name_size);
2545 			return (info->err = ECONNABORTED);
2546 		}
2547 
2548 		log_error(LOG_INFO,
2549 		    "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2550 		    pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2551 
2552 		startd_free(fmri, fmri_sz);
2553 		startd_free(pg_name, max_scf_name_size);
2554 
2555 		return (info->err = EINVAL);
2556 	}
2557 
2558 	/* Create depgroup vertex for pg */
2559 	err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2560 	    rerr, &depgrp);
2561 	assert(err == 0);
2562 	startd_free(fmri, fmri_sz);
2563 
2564 	/* Add dependency from inst vertex to new vertex */
2565 	err = graph_insert_dependency(info->v, depgrp, info->pathp);
2566 	/* ELOOP can't happen because this should be a new vertex */
2567 	assert(err == 0);
2568 
2569 	linfo.v = depgrp;
2570 	linfo.type = depgroup_read_scheme(h, pg);
2571 	linfo.inst_fmri = info->v->gv_name;
2572 	linfo.pg_name = pg_name;
2573 	linfo.h = h;
2574 	linfo.err = 0;
2575 	linfo.pathp = info->pathp;
2576 	err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2577 	    &linfo);
2578 
2579 	scf_property_destroy(prop);
2580 	startd_free(pg_name, max_scf_name_size);
2581 
2582 	switch (err) {
2583 	case 0:
2584 	case EINTR:
2585 		return (info->err = linfo.err);
2586 
2587 	case ECONNABORTED:
2588 	case EINVAL:
2589 		return (info->err = err);
2590 
2591 	case ECANCELED:
2592 		return (info->err = 0);
2593 
2594 	case ECONNRESET:
2595 		return (info->err = ECONNABORTED);
2596 
2597 	default:
2598 		bad_error("walk_property_astrings", err);
2599 		/* NOTREACHED */
2600 	}
2601 }
2602 
2603 /*
2604  * Build the dependency info for v from the repository.  Returns 0 on success,
2605  * ECONNABORTED on repository disconnection, EINVAL if the repository
2606  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2607  * In the last case, *pathp will point to a -1-terminated array of ids which
2608  * constitute the rest of the dependency cycle.
2609  */
2610 static int
2611 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2612 {
2613 	struct deppg_info info;
2614 	int err;
2615 	uint_t old_configured;
2616 
2617 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2618 
2619 	/*
2620 	 * Mark the vertex as configured during dependency insertion to avoid
2621 	 * dependency cycles (which can appear in the graph if one of the
2622 	 * vertices is an exclusion-group).
2623 	 */
2624 	old_configured = v->gv_flags & GV_CONFIGURED;
2625 	v->gv_flags |= GV_CONFIGURED;
2626 
2627 	info.err = 0;
2628 	info.v = v;
2629 	info.pathp = pathp;
2630 
2631 	err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2632 	    &info);
2633 
2634 	if (!old_configured)
2635 		v->gv_flags &= ~GV_CONFIGURED;
2636 
2637 	switch (err) {
2638 	case 0:
2639 	case EINTR:
2640 		return (info.err);
2641 
2642 	case ECONNABORTED:
2643 		return (ECONNABORTED);
2644 
2645 	case ECANCELED:
2646 		/* Should get delete event, so return 0. */
2647 		return (0);
2648 
2649 	default:
2650 		bad_error("walk_dependency_pgs", err);
2651 		/* NOTREACHED */
2652 	}
2653 }
2654 
2655 
2656 static void
2657 handle_cycle(const char *fmri, int *path)
2658 {
2659 	const char *cp;
2660 	size_t sz;
2661 
2662 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2663 
2664 	path_to_str(path, (char **)&cp, &sz);
2665 
2666 	log_error(LOG_ERR, "Transitioning %s to maintenance "
2667 	    "because it completes a dependency cycle (see svcs -xv for "
2668 	    "details):\n%s", fmri ? fmri : "?", cp);
2669 
2670 	startd_free((void *)cp, sz);
2671 }
2672 
2673 /*
2674  * Increment the vertex's reference count to prevent the vertex removal
2675  * from the dgraph.
2676  */
2677 static void
2678 vertex_ref(graph_vertex_t *v)
2679 {
2680 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2681 
2682 	v->gv_refs++;
2683 }
2684 
2685 /*
2686  * Decrement the vertex's reference count and remove the vertex from
2687  * the dgraph when possible.
2688  *
2689  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2690  * return VERTEX_INUSE.
2691  */
2692 static int
2693 vertex_unref(graph_vertex_t *v)
2694 {
2695 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2696 	assert(v->gv_refs > 0);
2697 
2698 	v->gv_refs--;
2699 
2700 	return (free_if_unrefed(v));
2701 }
2702 
2703 /*
2704  * When run on the dependencies of a vertex, populates list with
2705  * graph_edge_t's which point to the service vertices or the instance
2706  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2707  *
2708  * Increment the vertex's reference count once the vertex is inserted
2709  * in the list. The vertex won't be able to be deleted from the dgraph
2710  * while it is referenced.
2711  */
2712 static int
2713 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2714 {
2715 	graph_vertex_t *v = e->ge_vertex;
2716 	graph_edge_t *new;
2717 	int r;
2718 
2719 	switch (v->gv_type) {
2720 	case GVT_INST:
2721 	case GVT_SVC:
2722 		break;
2723 
2724 	case GVT_GROUP:
2725 		r = uu_list_walk(v->gv_dependencies,
2726 		    (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2727 		assert(r == 0);
2728 		return (UU_WALK_NEXT);
2729 
2730 	case GVT_FILE:
2731 		return (UU_WALK_NEXT);
2732 
2733 	default:
2734 #ifndef NDEBUG
2735 		uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2736 		    __LINE__, v->gv_type);
2737 #endif
2738 		abort();
2739 	}
2740 
2741 	new = startd_alloc(sizeof (*new));
2742 	new->ge_vertex = v;
2743 	uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2744 	r = uu_list_insert_before(list, NULL, new);
2745 	assert(r == 0);
2746 
2747 	/*
2748 	 * Because we are inserting the vertex in a list, we don't want
2749 	 * the vertex to be freed while the list is in use. In order to
2750 	 * achieve that, increment the vertex's reference count.
2751 	 */
2752 	vertex_ref(v);
2753 
2754 	return (UU_WALK_NEXT);
2755 }
2756 
2757 static boolean_t
2758 should_be_in_subgraph(graph_vertex_t *v)
2759 {
2760 	graph_edge_t *e;
2761 
2762 	if (v == milestone)
2763 		return (B_TRUE);
2764 
2765 	/*
2766 	 * v is in the subgraph if any of its dependents are in the subgraph.
2767 	 * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2768 	 * count if we're enabled.
2769 	 */
2770 	for (e = uu_list_first(v->gv_dependents);
2771 	    e != NULL;
2772 	    e = uu_list_next(v->gv_dependents, e)) {
2773 		graph_vertex_t *dv = e->ge_vertex;
2774 
2775 		if (!(dv->gv_flags & GV_INSUBGRAPH))
2776 			continue;
2777 
2778 		/*
2779 		 * Don't include instances that are optional and disabled.
2780 		 */
2781 		if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2782 
2783 			int in = 0;
2784 			graph_edge_t *ee;
2785 
2786 			for (ee = uu_list_first(dv->gv_dependents);
2787 			    ee != NULL;
2788 			    ee = uu_list_next(dv->gv_dependents, ee)) {
2789 
2790 				graph_vertex_t *ddv = e->ge_vertex;
2791 
2792 				if (ddv->gv_type == GVT_GROUP &&
2793 				    ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2794 					continue;
2795 
2796 				if (ddv->gv_type == GVT_GROUP &&
2797 				    ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2798 				    !(v->gv_flags & GV_ENBLD_NOOVR))
2799 					continue;
2800 
2801 				in = 1;
2802 			}
2803 			if (!in)
2804 				continue;
2805 		}
2806 		if (v->gv_type == GVT_INST &&
2807 		    dv->gv_type == GVT_GROUP &&
2808 		    dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2809 		    !(v->gv_flags & GV_ENBLD_NOOVR))
2810 			continue;
2811 
2812 		/* Don't include excluded services and instances */
2813 		if (dv->gv_type == GVT_GROUP &&
2814 		    dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2815 			continue;
2816 
2817 		return (B_TRUE);
2818 	}
2819 
2820 	return (B_FALSE);
2821 }
2822 
2823 /*
2824  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2825  * any bits change, manipulate the repository appropriately.  Returns 0 or
2826  * ECONNABORTED.
2827  */
2828 static int
2829 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2830 {
2831 	boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2832 	boolean_t new;
2833 	graph_edge_t *e;
2834 	scf_instance_t *inst;
2835 	int ret = 0, r;
2836 
2837 	assert(milestone != NULL && milestone != MILESTONE_NONE);
2838 
2839 	new = should_be_in_subgraph(v);
2840 
2841 	if (new == old)
2842 		return (0);
2843 
2844 	log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2845 	    "Removing %s from the subgraph.\n", v->gv_name);
2846 
2847 	v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2848 	    (new ? GV_INSUBGRAPH : 0);
2849 
2850 	if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2851 		int err;
2852 
2853 get_inst:
2854 		err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2855 		if (err != 0) {
2856 			switch (err) {
2857 			case ECONNABORTED:
2858 				libscf_handle_rebind(h);
2859 				ret = ECONNABORTED;
2860 				goto get_inst;
2861 
2862 			case ENOENT:
2863 				break;
2864 
2865 			case EINVAL:
2866 			case ENOTSUP:
2867 			default:
2868 				bad_error("libscf_fmri_get_instance", err);
2869 			}
2870 		} else {
2871 			const char *f;
2872 
2873 			if (new) {
2874 				err = libscf_delete_enable_ovr(inst);
2875 				f = "libscf_delete_enable_ovr";
2876 			} else {
2877 				err = libscf_set_enable_ovr(inst, 0);
2878 				f = "libscf_set_enable_ovr";
2879 			}
2880 			scf_instance_destroy(inst);
2881 			switch (err) {
2882 			case 0:
2883 			case ECANCELED:
2884 				break;
2885 
2886 			case ECONNABORTED:
2887 				libscf_handle_rebind(h);
2888 				/*
2889 				 * We must continue so the graph is updated,
2890 				 * but we must return ECONNABORTED so any
2891 				 * libscf state held by any callers is reset.
2892 				 */
2893 				ret = ECONNABORTED;
2894 				goto get_inst;
2895 
2896 			case EROFS:
2897 			case EPERM:
2898 				log_error(LOG_WARNING,
2899 				    "Could not set %s/%s for %s: %s.\n",
2900 				    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2901 				    v->gv_name, strerror(err));
2902 				break;
2903 
2904 			default:
2905 				bad_error(f, err);
2906 			}
2907 		}
2908 	}
2909 
2910 	for (e = uu_list_first(v->gv_dependencies);
2911 	    e != NULL;
2912 	    e = uu_list_next(v->gv_dependencies, e)) {
2913 		r = eval_subgraph(e->ge_vertex, h);
2914 		if (r != 0) {
2915 			assert(r == ECONNABORTED);
2916 			ret = ECONNABORTED;
2917 		}
2918 	}
2919 
2920 	return (ret);
2921 }
2922 
2923 /*
2924  * Delete the (property group) dependencies of v & create new ones based on
2925  * inst.  If doing so would create a cycle, log a message and put the instance
2926  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2927  * ECONNABORTED.
2928  */
2929 int
2930 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2931 {
2932 	int err;
2933 	int *path;
2934 	char *fmri;
2935 	int r;
2936 	scf_handle_t *h = scf_instance_handle(inst);
2937 	uu_list_t *old_deps;
2938 	int ret = 0;
2939 	graph_edge_t *e;
2940 	graph_vertex_t *vv;
2941 
2942 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
2943 	assert(v->gv_type == GVT_INST);
2944 
2945 	log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
2946 
2947 	if (milestone > MILESTONE_NONE) {
2948 		/*
2949 		 * In case some of v's dependencies are being deleted we must
2950 		 * make a list of them now for GV_INSUBGRAPH-flag evaluation
2951 		 * after the new dependencies are in place.
2952 		 */
2953 		old_deps = startd_list_create(graph_edge_pool, NULL, 0);
2954 
2955 		err = uu_list_walk(v->gv_dependencies,
2956 		    (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
2957 		assert(err == 0);
2958 	}
2959 
2960 	delete_instance_dependencies(v, B_FALSE);
2961 
2962 	err = set_dependencies(v, inst, &path);
2963 	switch (err) {
2964 	case 0:
2965 		break;
2966 
2967 	case ECONNABORTED:
2968 		ret = err;
2969 		goto out;
2970 
2971 	case EINVAL:
2972 	case ELOOP:
2973 		r = libscf_instance_get_fmri(inst, &fmri);
2974 		switch (r) {
2975 		case 0:
2976 			break;
2977 
2978 		case ECONNABORTED:
2979 			ret = ECONNABORTED;
2980 			goto out;
2981 
2982 		case ECANCELED:
2983 			ret = 0;
2984 			goto out;
2985 
2986 		default:
2987 			bad_error("libscf_instance_get_fmri", r);
2988 		}
2989 
2990 		if (err == EINVAL) {
2991 			log_error(LOG_ERR, "Transitioning %s "
2992 			    "to maintenance due to misconfiguration.\n",
2993 			    fmri ? fmri : "?");
2994 			vertex_send_event(v,
2995 			    RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
2996 		} else {
2997 			handle_cycle(fmri, path);
2998 			vertex_send_event(v,
2999 			    RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3000 		}
3001 		startd_free(fmri, max_scf_fmri_size);
3002 		ret = 0;
3003 		goto out;
3004 
3005 	default:
3006 		bad_error("set_dependencies", err);
3007 	}
3008 
3009 	if (milestone > MILESTONE_NONE) {
3010 		boolean_t aborted = B_FALSE;
3011 
3012 		for (e = uu_list_first(old_deps);
3013 		    e != NULL;
3014 		    e = uu_list_next(old_deps, e)) {
3015 			vv = e->ge_vertex;
3016 
3017 			if (vertex_unref(vv) == VERTEX_INUSE &&
3018 			    eval_subgraph(vv, h) == ECONNABORTED)
3019 				aborted = B_TRUE;
3020 		}
3021 
3022 		for (e = uu_list_first(v->gv_dependencies);
3023 		    e != NULL;
3024 		    e = uu_list_next(v->gv_dependencies, e)) {
3025 			if (eval_subgraph(e->ge_vertex, h) ==
3026 			    ECONNABORTED)
3027 				aborted = B_TRUE;
3028 		}
3029 
3030 		if (aborted) {
3031 			ret = ECONNABORTED;
3032 			goto out;
3033 		}
3034 	}
3035 
3036 	graph_start_if_satisfied(v);
3037 
3038 	ret = 0;
3039 
3040 out:
3041 	if (milestone > MILESTONE_NONE) {
3042 		void *cookie = NULL;
3043 
3044 		while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3045 			startd_free(e, sizeof (*e));
3046 
3047 		uu_list_destroy(old_deps);
3048 	}
3049 
3050 	return (ret);
3051 }
3052 
3053 /*
3054  * Set up v according to inst.  That is, make sure it depends on its
3055  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3056  * the restarter, and send ENABLE or DISABLE as appropriate.
3057  *
3058  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3059  * ECANCELED if inst is deleted.
3060  */
3061 static int
3062 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3063 {
3064 	scf_handle_t *h;
3065 	scf_propertygroup_t *pg;
3066 	scf_snapshot_t *snap;
3067 	char *restarter_fmri = startd_alloc(max_scf_value_size);
3068 	int enabled, enabled_ovr;
3069 	int err;
3070 	int *path;
3071 	int deathrow;
3072 
3073 	restarter_fmri[0] = '\0';
3074 
3075 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3076 	assert(v->gv_type == GVT_INST);
3077 	assert((v->gv_flags & GV_CONFIGURED) == 0);
3078 
3079 	/* GV_INSUBGRAPH should already be set properly. */
3080 	assert(should_be_in_subgraph(v) ==
3081 	    ((v->gv_flags & GV_INSUBGRAPH) != 0));
3082 
3083 	/*
3084 	 * If the instance fmri is in the deathrow list then set the
3085 	 * GV_DEATHROW flag on the vertex and create and set to true the
3086 	 * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3087 	 * repository for this instance fmri.
3088 	 */
3089 	if ((v->gv_flags & GV_DEATHROW) ||
3090 	    (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3091 		if ((v->gv_flags & GV_DEATHROW) == 0) {
3092 			/*
3093 			 * Set flag GV_DEATHROW, create and set to true
3094 			 * the SCF_PROPERTY_DEATHROW property in the
3095 			 * non-persistent repository for this instance fmri.
3096 			 */
3097 			v->gv_flags |= GV_DEATHROW;
3098 
3099 			switch (err = libscf_set_deathrow(inst, 1)) {
3100 			case 0:
3101 				break;
3102 
3103 			case ECONNABORTED:
3104 			case ECANCELED:
3105 				startd_free(restarter_fmri, max_scf_value_size);
3106 				return (err);
3107 
3108 			case EROFS:
3109 				log_error(LOG_WARNING, "Could not set %s/%s "
3110 				    "for deathrow %s: %s.\n",
3111 				    SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3112 				    v->gv_name, strerror(err));
3113 				break;
3114 
3115 			case EPERM:
3116 				uu_die("Permission denied.\n");
3117 				/* NOTREACHED */
3118 
3119 			default:
3120 				bad_error("libscf_set_deathrow", err);
3121 			}
3122 			log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3123 			    v->gv_name);
3124 		}
3125 		startd_free(restarter_fmri, max_scf_value_size);
3126 		return (0);
3127 	}
3128 
3129 	h = scf_instance_handle(inst);
3130 
3131 	/*
3132 	 * Using a temporary deathrow boolean property, set through
3133 	 * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3134 	 * because deathrow_fini() may already have been called, and in case
3135 	 * of a refresh, GV_DEATHROW may need to be set again.
3136 	 * libscf_get_deathrow() sets deathrow to 1 only if this instance
3137 	 * has a temporary boolean property named 'deathrow' valued true
3138 	 * in a property group 'deathrow', -1 or 0 in all other cases.
3139 	 */
3140 	err = libscf_get_deathrow(h, inst, &deathrow);
3141 	switch (err) {
3142 	case 0:
3143 		break;
3144 
3145 	case ECONNABORTED:
3146 	case ECANCELED:
3147 		startd_free(restarter_fmri, max_scf_value_size);
3148 		return (err);
3149 
3150 	default:
3151 		bad_error("libscf_get_deathrow", err);
3152 	}
3153 
3154 	if (deathrow == 1) {
3155 		v->gv_flags |= GV_DEATHROW;
3156 		startd_free(restarter_fmri, max_scf_value_size);
3157 		return (0);
3158 	}
3159 
3160 	log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3161 
3162 	/*
3163 	 * If the instance does not have a restarter property group,
3164 	 * initialize its state to uninitialized/none, in case the restarter
3165 	 * is not enabled.
3166 	 */
3167 	pg = safe_scf_pg_create(h);
3168 
3169 	if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3170 		instance_data_t idata;
3171 		uint_t count = 0, msecs = ALLOC_DELAY;
3172 
3173 		switch (scf_error()) {
3174 		case SCF_ERROR_NOT_FOUND:
3175 			break;
3176 
3177 		case SCF_ERROR_CONNECTION_BROKEN:
3178 		default:
3179 			scf_pg_destroy(pg);
3180 			return (ECONNABORTED);
3181 
3182 		case SCF_ERROR_DELETED:
3183 			scf_pg_destroy(pg);
3184 			return (ECANCELED);
3185 
3186 		case SCF_ERROR_NOT_SET:
3187 			bad_error("scf_instance_get_pg", scf_error());
3188 		}
3189 
3190 		switch (err = libscf_instance_get_fmri(inst,
3191 		    (char **)&idata.i_fmri)) {
3192 		case 0:
3193 			break;
3194 
3195 		case ECONNABORTED:
3196 		case ECANCELED:
3197 			scf_pg_destroy(pg);
3198 			return (err);
3199 
3200 		default:
3201 			bad_error("libscf_instance_get_fmri", err);
3202 		}
3203 
3204 		idata.i_state = RESTARTER_STATE_NONE;
3205 		idata.i_next_state = RESTARTER_STATE_NONE;
3206 
3207 init_state:
3208 		switch (err = _restarter_commit_states(h, &idata,
3209 		    RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE, NULL)) {
3210 		case 0:
3211 			break;
3212 
3213 		case ENOMEM:
3214 			++count;
3215 			if (count < ALLOC_RETRY) {
3216 				(void) poll(NULL, 0, msecs);
3217 				msecs *= ALLOC_DELAY_MULT;
3218 				goto init_state;
3219 			}
3220 
3221 			uu_die("Insufficient memory.\n");
3222 			/* NOTREACHED */
3223 
3224 		case ECONNABORTED:
3225 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3226 			scf_pg_destroy(pg);
3227 			return (ECONNABORTED);
3228 
3229 		case ENOENT:
3230 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3231 			scf_pg_destroy(pg);
3232 			return (ECANCELED);
3233 
3234 		case EPERM:
3235 		case EACCES:
3236 		case EROFS:
3237 			log_error(LOG_NOTICE, "Could not initialize state for "
3238 			    "%s: %s.\n", idata.i_fmri, strerror(err));
3239 			break;
3240 
3241 		case EINVAL:
3242 		default:
3243 			bad_error("_restarter_commit_states", err);
3244 		}
3245 
3246 		startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3247 	}
3248 
3249 	scf_pg_destroy(pg);
3250 
3251 	if (milestone != NULL) {
3252 		/*
3253 		 * Make sure the enable-override is set properly before we
3254 		 * read whether we should be enabled.
3255 		 */
3256 		if (milestone == MILESTONE_NONE ||
3257 		    !(v->gv_flags & GV_INSUBGRAPH)) {
3258 			/*
3259 			 * This might seem unjustified after the milestone
3260 			 * transition has completed (non_subgraph_svcs == 0),
3261 			 * but it's important because when we boot to
3262 			 * a milestone, we set the milestone before populating
3263 			 * the graph, and all of the new non-subgraph services
3264 			 * need to be disabled here.
3265 			 */
3266 			switch (err = libscf_set_enable_ovr(inst, 0)) {
3267 			case 0:
3268 				break;
3269 
3270 			case ECONNABORTED:
3271 			case ECANCELED:
3272 				return (err);
3273 
3274 			case EROFS:
3275 				log_error(LOG_WARNING,
3276 				    "Could not set %s/%s for %s: %s.\n",
3277 				    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3278 				    v->gv_name, strerror(err));
3279 				break;
3280 
3281 			case EPERM:
3282 				uu_die("Permission denied.\n");
3283 				/* NOTREACHED */
3284 
3285 			default:
3286 				bad_error("libscf_set_enable_ovr", err);
3287 			}
3288 		} else {
3289 			assert(v->gv_flags & GV_INSUBGRAPH);
3290 			switch (err = libscf_delete_enable_ovr(inst)) {
3291 			case 0:
3292 				break;
3293 
3294 			case ECONNABORTED:
3295 			case ECANCELED:
3296 				return (err);
3297 
3298 			case EPERM:
3299 				uu_die("Permission denied.\n");
3300 				/* NOTREACHED */
3301 
3302 			default:
3303 				bad_error("libscf_delete_enable_ovr", err);
3304 			}
3305 		}
3306 	}
3307 
3308 	err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3309 	    &enabled_ovr, &restarter_fmri);
3310 	switch (err) {
3311 	case 0:
3312 		break;
3313 
3314 	case ECONNABORTED:
3315 	case ECANCELED:
3316 		startd_free(restarter_fmri, max_scf_value_size);
3317 		return (err);
3318 
3319 	case ENOENT:
3320 		log_framework(LOG_DEBUG,
3321 		    "Ignoring %s because it has no general property group.\n",
3322 		    v->gv_name);
3323 		startd_free(restarter_fmri, max_scf_value_size);
3324 		return (0);
3325 
3326 	default:
3327 		bad_error("libscf_get_basic_instance_data", err);
3328 	}
3329 
3330 	if (enabled == -1) {
3331 		startd_free(restarter_fmri, max_scf_value_size);
3332 		return (0);
3333 	}
3334 
3335 	v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3336 	    (enabled ? GV_ENBLD_NOOVR : 0);
3337 
3338 	if (enabled_ovr != -1)
3339 		enabled = enabled_ovr;
3340 
3341 	v->gv_state = RESTARTER_STATE_UNINIT;
3342 
3343 	snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3344 	scf_snapshot_destroy(snap);
3345 
3346 	/* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3347 	err = graph_change_restarter(v, restarter_fmri, h, &path);
3348 	if (err != 0) {
3349 		instance_data_t idata;
3350 		uint_t count = 0, msecs = ALLOC_DELAY;
3351 		const char *reason;
3352 
3353 		if (err == ECONNABORTED) {
3354 			startd_free(restarter_fmri, max_scf_value_size);
3355 			return (err);
3356 		}
3357 
3358 		assert(err == EINVAL || err == ELOOP);
3359 
3360 		if (err == EINVAL) {
3361 			log_framework(LOG_ERR, emsg_invalid_restarter,
3362 			    v->gv_name);
3363 			reason = "invalid_restarter";
3364 		} else {
3365 			handle_cycle(v->gv_name, path);
3366 			reason = "dependency_cycle";
3367 		}
3368 
3369 		startd_free(restarter_fmri, max_scf_value_size);
3370 
3371 		/*
3372 		 * We didn't register the instance with the restarter, so we
3373 		 * must set maintenance mode ourselves.
3374 		 */
3375 		err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3376 		if (err != 0) {
3377 			assert(err == ECONNABORTED || err == ECANCELED);
3378 			return (err);
3379 		}
3380 
3381 		idata.i_state = RESTARTER_STATE_NONE;
3382 		idata.i_next_state = RESTARTER_STATE_NONE;
3383 
3384 set_maint:
3385 		switch (err = _restarter_commit_states(h, &idata,
3386 		    RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, reason)) {
3387 		case 0:
3388 			break;
3389 
3390 		case ENOMEM:
3391 			++count;
3392 			if (count < ALLOC_RETRY) {
3393 				(void) poll(NULL, 0, msecs);
3394 				msecs *= ALLOC_DELAY_MULT;
3395 				goto set_maint;
3396 			}
3397 
3398 			uu_die("Insufficient memory.\n");
3399 			/* NOTREACHED */
3400 
3401 		case ECONNABORTED:
3402 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3403 			return (ECONNABORTED);
3404 
3405 		case ENOENT:
3406 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3407 			return (ECANCELED);
3408 
3409 		case EPERM:
3410 		case EACCES:
3411 		case EROFS:
3412 			log_error(LOG_NOTICE, "Could not initialize state for "
3413 			    "%s: %s.\n", idata.i_fmri, strerror(err));
3414 			break;
3415 
3416 		case EINVAL:
3417 		default:
3418 			bad_error("_restarter_commit_states", err);
3419 		}
3420 
3421 		startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3422 
3423 		v->gv_state = RESTARTER_STATE_MAINT;
3424 
3425 		goto out;
3426 	}
3427 	startd_free(restarter_fmri, max_scf_value_size);
3428 
3429 	/* Add all the other dependencies. */
3430 	err = refresh_vertex(v, inst);
3431 	if (err != 0) {
3432 		assert(err == ECONNABORTED);
3433 		return (err);
3434 	}
3435 
3436 out:
3437 	v->gv_flags |= GV_CONFIGURED;
3438 
3439 	graph_enable_by_vertex(v, enabled, 0);
3440 
3441 	return (0);
3442 }
3443 
3444 static void
3445 do_uadmin(void)
3446 {
3447 	int fd, left;
3448 	struct statvfs vfs;
3449 
3450 	const char * const resetting = "/etc/svc/volatile/resetting";
3451 
3452 	fd = creat(resetting, 0777);
3453 	if (fd >= 0)
3454 		startd_close(fd);
3455 	else
3456 		uu_warn("Could not create \"%s\"", resetting);
3457 
3458 	/* Kill dhcpagent if we're not using nfs for root */
3459 	if ((statvfs("/", &vfs) == 0) &&
3460 	    (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3461 		(void) system("/usr/bin/pkill -x -u 0 dhcpagent");
3462 
3463 	(void) system("/usr/sbin/killall");
3464 	left = 5;
3465 	while (left > 0)
3466 		left = sleep(left);
3467 
3468 	(void) system("/usr/sbin/killall 9");
3469 	left = 10;
3470 	while (left > 0)
3471 		left = sleep(left);
3472 
3473 	sync();
3474 	sync();
3475 	sync();
3476 
3477 	(void) system("/sbin/umountall -l");
3478 	(void) system("/sbin/umount /tmp >/dev/null 2>&1");
3479 	(void) system("/sbin/umount /var/adm >/dev/null 2>&1");
3480 	(void) system("/sbin/umount /var/run >/dev/null 2>&1");
3481 	(void) system("/sbin/umount /var >/dev/null 2>&1");
3482 	(void) system("/sbin/umount /usr >/dev/null 2>&1");
3483 
3484 	uu_warn("The system is down.\n");
3485 
3486 	(void) uadmin(A_SHUTDOWN, halting, NULL);
3487 	uu_warn("uadmin() failed");
3488 
3489 	if (remove(resetting) != 0 && errno != ENOENT)
3490 		uu_warn("Could not remove \"%s\"", resetting);
3491 }
3492 
3493 /*
3494  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3495  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3496  */
3497 boolean_t
3498 can_come_up(void)
3499 {
3500 	int i;
3501 
3502 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3503 
3504 	/*
3505 	 * If we are booting to single user (boot -s),
3506 	 * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3507 	 * spawns sulogin after single-user is online (see specials.c).
3508 	 */
3509 	i = (booting_to_single_user ? 0 : 1);
3510 
3511 	for (; up_svcs[i] != NULL; ++i) {
3512 		if (up_svcs_p[i] == NULL) {
3513 			up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3514 
3515 			if (up_svcs_p[i] == NULL)
3516 				continue;
3517 		}
3518 
3519 		/*
3520 		 * Ignore unconfigured services (the ones that have been
3521 		 * mentioned in a dependency from other services, but do
3522 		 * not exist in the repository).  Services which exist
3523 		 * in the repository but don't have general/enabled
3524 		 * property will be also ignored.
3525 		 */
3526 		if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3527 			continue;
3528 
3529 		switch (up_svcs_p[i]->gv_state) {
3530 		case RESTARTER_STATE_ONLINE:
3531 		case RESTARTER_STATE_DEGRADED:
3532 			/*
3533 			 * Deactivate verbose boot once a login service has been
3534 			 * reached.
3535 			 */
3536 			st->st_log_login_reached = 1;
3537 			/*FALLTHROUGH*/
3538 		case RESTARTER_STATE_UNINIT:
3539 			return (B_TRUE);
3540 
3541 		case RESTARTER_STATE_OFFLINE:
3542 			if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3543 				return (B_TRUE);
3544 			log_framework(LOG_DEBUG,
3545 			    "can_come_up(): %s is unsatisfiable.\n",
3546 			    up_svcs_p[i]->gv_name);
3547 			continue;
3548 
3549 		case RESTARTER_STATE_DISABLED:
3550 		case RESTARTER_STATE_MAINT:
3551 			log_framework(LOG_DEBUG,
3552 			    "can_come_up(): %s is in state %s.\n",
3553 			    up_svcs_p[i]->gv_name,
3554 			    instance_state_str[up_svcs_p[i]->gv_state]);
3555 			continue;
3556 
3557 		default:
3558 #ifndef NDEBUG
3559 			uu_warn("%s:%d: Unexpected vertex state %d.\n",
3560 			    __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3561 #endif
3562 			abort();
3563 		}
3564 	}
3565 
3566 	/*
3567 	 * In the seed repository, console-login is unsatisfiable because
3568 	 * services are missing.  To behave correctly in that case we don't want
3569 	 * to return false until manifest-import is online.
3570 	 */
3571 
3572 	if (manifest_import_p == NULL) {
3573 		manifest_import_p = vertex_get_by_name(manifest_import);
3574 
3575 		if (manifest_import_p == NULL)
3576 			return (B_FALSE);
3577 	}
3578 
3579 	switch (manifest_import_p->gv_state) {
3580 	case RESTARTER_STATE_ONLINE:
3581 	case RESTARTER_STATE_DEGRADED:
3582 	case RESTARTER_STATE_DISABLED:
3583 	case RESTARTER_STATE_MAINT:
3584 		break;
3585 
3586 	case RESTARTER_STATE_OFFLINE:
3587 		if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3588 			break;
3589 		/* FALLTHROUGH */
3590 
3591 	case RESTARTER_STATE_UNINIT:
3592 		return (B_TRUE);
3593 	}
3594 
3595 	return (B_FALSE);
3596 }
3597 
3598 /*
3599  * Runs sulogin.  Returns
3600  *   0 - success
3601  *   EALREADY - sulogin is already running
3602  *   EBUSY - console-login is running
3603  */
3604 static int
3605 run_sulogin(const char *msg)
3606 {
3607 	graph_vertex_t *v;
3608 
3609 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3610 
3611 	if (sulogin_running)
3612 		return (EALREADY);
3613 
3614 	v = vertex_get_by_name(console_login_fmri);
3615 	if (v != NULL && inst_running(v))
3616 		return (EBUSY);
3617 
3618 	sulogin_running = B_TRUE;
3619 
3620 	MUTEX_UNLOCK(&dgraph_lock);
3621 
3622 	fork_sulogin(B_FALSE, msg);
3623 
3624 	MUTEX_LOCK(&dgraph_lock);
3625 
3626 	sulogin_running = B_FALSE;
3627 
3628 	if (console_login_ready) {
3629 		v = vertex_get_by_name(console_login_fmri);
3630 
3631 		if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE &&
3632 		    !inst_running(v)) {
3633 			if (v->gv_start_f == NULL)
3634 				vertex_send_event(v,
3635 				    RESTARTER_EVENT_TYPE_START);
3636 			else
3637 				v->gv_start_f(v);
3638 		}
3639 
3640 		console_login_ready = B_FALSE;
3641 	}
3642 
3643 	return (0);
3644 }
3645 
3646 /*
3647  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3648  * keeps sulogin from stepping on console-login's toes.
3649  */
3650 /* ARGSUSED */
3651 static void *
3652 sulogin_thread(void *unused)
3653 {
3654 	MUTEX_LOCK(&dgraph_lock);
3655 
3656 	assert(sulogin_thread_running);
3657 
3658 	do {
3659 		(void) run_sulogin("Console login service(s) cannot run\n");
3660 	} while (!can_come_up());
3661 
3662 	sulogin_thread_running = B_FALSE;
3663 	MUTEX_UNLOCK(&dgraph_lock);
3664 
3665 	return (NULL);
3666 }
3667 
3668 /* ARGSUSED */
3669 void *
3670 single_user_thread(void *unused)
3671 {
3672 	uint_t left;
3673 	scf_handle_t *h;
3674 	scf_instance_t *inst;
3675 	scf_property_t *prop;
3676 	scf_value_t *val;
3677 	const char *msg;
3678 	char *buf;
3679 	int r;
3680 
3681 	MUTEX_LOCK(&single_user_thread_lock);
3682 	single_user_thread_count++;
3683 
3684 	if (!booting_to_single_user) {
3685 		/*
3686 		 * From rcS.sh: Look for ttymon, in.telnetd, in.rlogind and
3687 		 * processes in their process groups so they can be terminated.
3688 		 */
3689 		(void) fputs("svc.startd: Killing user processes: ", stdout);
3690 		(void) system("/usr/sbin/killall");
3691 		(void) system("/usr/sbin/killall 9");
3692 		(void) system("/usr/bin/pkill -TERM -v -u 0,1");
3693 
3694 		left = 5;
3695 		while (left > 0)
3696 			left = sleep(left);
3697 
3698 		(void) system("/usr/bin/pkill -KILL -v -u 0,1");
3699 		(void) puts("done.");
3700 	}
3701 
3702 	if (go_single_user_mode || booting_to_single_user) {
3703 		msg = "SINGLE USER MODE\n";
3704 	} else {
3705 		assert(go_to_level1);
3706 
3707 		fork_rc_script('1', "start", B_TRUE);
3708 
3709 		uu_warn("The system is ready for administration.\n");
3710 
3711 		msg = "";
3712 	}
3713 
3714 	MUTEX_UNLOCK(&single_user_thread_lock);
3715 
3716 	for (;;) {
3717 		MUTEX_LOCK(&dgraph_lock);
3718 		r = run_sulogin(msg);
3719 		MUTEX_UNLOCK(&dgraph_lock);
3720 		if (r == 0)
3721 			break;
3722 
3723 		assert(r == EALREADY || r == EBUSY);
3724 
3725 		left = 3;
3726 		while (left > 0)
3727 			left = sleep(left);
3728 	}
3729 
3730 	MUTEX_LOCK(&single_user_thread_lock);
3731 
3732 	/*
3733 	 * If another single user thread has started, let it finish changing
3734 	 * the run level.
3735 	 */
3736 	if (single_user_thread_count > 1) {
3737 		single_user_thread_count--;
3738 		MUTEX_UNLOCK(&single_user_thread_lock);
3739 		return (NULL);
3740 	}
3741 
3742 	h = libscf_handle_create_bound_loop();
3743 	inst = scf_instance_create(h);
3744 	prop = safe_scf_property_create(h);
3745 	val = safe_scf_value_create(h);
3746 	buf = startd_alloc(max_scf_fmri_size);
3747 
3748 lookup:
3749 	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3750 	    NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3751 		switch (scf_error()) {
3752 		case SCF_ERROR_NOT_FOUND:
3753 			r = libscf_create_self(h);
3754 			if (r == 0)
3755 				goto lookup;
3756 			assert(r == ECONNABORTED);
3757 			/* FALLTHROUGH */
3758 
3759 		case SCF_ERROR_CONNECTION_BROKEN:
3760 			libscf_handle_rebind(h);
3761 			goto lookup;
3762 
3763 		case SCF_ERROR_INVALID_ARGUMENT:
3764 		case SCF_ERROR_CONSTRAINT_VIOLATED:
3765 		case SCF_ERROR_NOT_BOUND:
3766 		case SCF_ERROR_HANDLE_MISMATCH:
3767 		default:
3768 			bad_error("scf_handle_decode_fmri", scf_error());
3769 		}
3770 	}
3771 
3772 	MUTEX_LOCK(&dgraph_lock);
3773 
3774 	r = libscf_inst_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3775 	    SCF_PROPERTY_MILESTONE);
3776 	switch (r) {
3777 	case 0:
3778 	case ECANCELED:
3779 		break;
3780 
3781 	case ECONNABORTED:
3782 		MUTEX_UNLOCK(&dgraph_lock);
3783 		libscf_handle_rebind(h);
3784 		goto lookup;
3785 
3786 	case EPERM:
3787 	case EACCES:
3788 	case EROFS:
3789 		log_error(LOG_WARNING, "Could not clear temporary milestone: "
3790 		    "%s.\n", strerror(r));
3791 		break;
3792 
3793 	default:
3794 		bad_error("libscf_inst_delete_prop", r);
3795 	}
3796 
3797 	MUTEX_UNLOCK(&dgraph_lock);
3798 
3799 	r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3800 	switch (r) {
3801 	case ECANCELED:
3802 	case ENOENT:
3803 	case EINVAL:
3804 		(void) strcpy(buf, "all");
3805 		/* FALLTHROUGH */
3806 
3807 	case 0:
3808 		uu_warn("Returning to milestone %s.\n", buf);
3809 		break;
3810 
3811 	case ECONNABORTED:
3812 		libscf_handle_rebind(h);
3813 		goto lookup;
3814 
3815 	default:
3816 		bad_error("libscf_get_milestone", r);
3817 	}
3818 
3819 	r = dgraph_set_milestone(buf, h, B_FALSE);
3820 	switch (r) {
3821 	case 0:
3822 	case ECONNRESET:
3823 	case EALREADY:
3824 	case EINVAL:
3825 	case ENOENT:
3826 		break;
3827 
3828 	default:
3829 		bad_error("dgraph_set_milestone", r);
3830 	}
3831 
3832 	/*
3833 	 * See graph_runlevel_changed().
3834 	 */
3835 	MUTEX_LOCK(&dgraph_lock);
3836 	utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
3837 	MUTEX_UNLOCK(&dgraph_lock);
3838 
3839 	startd_free(buf, max_scf_fmri_size);
3840 	scf_value_destroy(val);
3841 	scf_property_destroy(prop);
3842 	scf_instance_destroy(inst);
3843 	scf_handle_destroy(h);
3844 
3845 	/*
3846 	 * We'll give ourselves 3 seconds to respond to all of the enablings
3847 	 * that setting the milestone should have created before checking
3848 	 * whether to run sulogin.
3849 	 */
3850 	left = 3;
3851 	while (left > 0)
3852 		left = sleep(left);
3853 
3854 	MUTEX_LOCK(&dgraph_lock);
3855 	/*
3856 	 * Clearing these variables will allow the sulogin thread to run.  We
3857 	 * check here in case there aren't any more state updates anytime soon.
3858 	 */
3859 	go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
3860 	if (!sulogin_thread_running && !can_come_up()) {
3861 		(void) startd_thread_create(sulogin_thread, NULL);
3862 		sulogin_thread_running = B_TRUE;
3863 	}
3864 	MUTEX_UNLOCK(&dgraph_lock);
3865 	single_user_thread_count--;
3866 	MUTEX_UNLOCK(&single_user_thread_lock);
3867 	return (NULL);
3868 }
3869 
3870 
3871 /*
3872  * Dependency graph operations API.  These are handle-independent thread-safe
3873  * graph manipulation functions which are the entry points for the event
3874  * threads below.
3875  */
3876 
3877 /*
3878  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
3879  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
3880  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
3881  * Fetch whether the instance should be enabled from inst and send _ENABLE or
3882  * _DISABLE as appropriate.  Finally rummage through inst's dependency
3883  * property groups and add vertices and edges as appropriate.  If anything
3884  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
3885  * instance in maintenance.  Don't send _START or _STOP until we get a state
3886  * update in case we're being restarted and the service is already running.
3887  *
3888  * To support booting to a milestone, we must also make sure all dependencies
3889  * encountered are configured, if they exist in the repository.
3890  *
3891  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
3892  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
3893  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
3894  */
3895 int
3896 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
3897     boolean_t lock_graph)
3898 {
3899 	graph_vertex_t *v;
3900 	int err;
3901 
3902 	if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
3903 		return (0);
3904 
3905 	/* Check for a vertex for inst_fmri. */
3906 	if (lock_graph) {
3907 		MUTEX_LOCK(&dgraph_lock);
3908 	} else {
3909 		assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
3910 	}
3911 
3912 	v = vertex_get_by_name(inst_fmri);
3913 
3914 	if (v != NULL) {
3915 		assert(v->gv_type == GVT_INST);
3916 
3917 		if (v->gv_flags & GV_CONFIGURED) {
3918 			if (lock_graph)
3919 				MUTEX_UNLOCK(&dgraph_lock);
3920 			return (EEXIST);
3921 		}
3922 	} else {
3923 		/* Add the vertex. */
3924 		err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
3925 		    RERR_NONE, &v);
3926 		if (err != 0) {
3927 			assert(err == EINVAL);
3928 			if (lock_graph)
3929 				MUTEX_UNLOCK(&dgraph_lock);
3930 			return (EINVAL);
3931 		}
3932 	}
3933 
3934 	err = configure_vertex(v, inst);
3935 
3936 	if (lock_graph)
3937 		MUTEX_UNLOCK(&dgraph_lock);
3938 
3939 	return (err);
3940 }
3941 
3942 /*
3943  * Locate the vertex for this property group's instance.  If it doesn't exist
3944  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
3945  * the restarter for the instance, and if it has changed, send
3946  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
3947  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
3948  * the new restarter.  Then fetch whether the instance should be enabled, and
3949  * if it is different from what we had, or if we changed the restarter, send
3950  * the appropriate _ENABLE or _DISABLE command.
3951  *
3952  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
3953  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
3954  * deleted, or -1 if the instance's general property group is deleted or if
3955  * its enabled property is misconfigured.
3956  */
3957 static int
3958 dgraph_update_general(scf_propertygroup_t *pg)
3959 {
3960 	scf_handle_t *h;
3961 	scf_instance_t *inst;
3962 	char *fmri;
3963 	char *restarter_fmri;
3964 	graph_vertex_t *v;
3965 	int err;
3966 	int enabled, enabled_ovr;
3967 	int oldflags;
3968 
3969 	/* Find the vertex for this service */
3970 	h = scf_pg_handle(pg);
3971 
3972 	inst = safe_scf_instance_create(h);
3973 
3974 	if (scf_pg_get_parent_instance(pg, inst) != 0) {
3975 		switch (scf_error()) {
3976 		case SCF_ERROR_CONSTRAINT_VIOLATED:
3977 			return (ENOTSUP);
3978 
3979 		case SCF_ERROR_CONNECTION_BROKEN:
3980 		default:
3981 			return (ECONNABORTED);
3982 
3983 		case SCF_ERROR_DELETED:
3984 			return (0);
3985 
3986 		case SCF_ERROR_NOT_SET:
3987 			bad_error("scf_pg_get_parent_instance", scf_error());
3988 		}
3989 	}
3990 
3991 	err = libscf_instance_get_fmri(inst, &fmri);
3992 	switch (err) {
3993 	case 0:
3994 		break;
3995 
3996 	case ECONNABORTED:
3997 		scf_instance_destroy(inst);
3998 		return (ECONNABORTED);
3999 
4000 	case ECANCELED:
4001 		scf_instance_destroy(inst);
4002 		return (0);
4003 
4004 	default:
4005 		bad_error("libscf_instance_get_fmri", err);
4006 	}
4007 
4008 	log_framework(LOG_DEBUG,
4009 	    "Graph engine: Reloading general properties for %s.\n", fmri);
4010 
4011 	MUTEX_LOCK(&dgraph_lock);
4012 
4013 	v = vertex_get_by_name(fmri);
4014 	if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4015 		/* Will get the up-to-date properties. */
4016 		MUTEX_UNLOCK(&dgraph_lock);
4017 		err = dgraph_add_instance(fmri, inst, B_TRUE);
4018 		startd_free(fmri, max_scf_fmri_size);
4019 		scf_instance_destroy(inst);
4020 		return (err == ECANCELED ? 0 : err);
4021 	}
4022 
4023 	/* Read enabled & restarter from repository. */
4024 	restarter_fmri = startd_alloc(max_scf_value_size);
4025 	err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4026 	    &enabled_ovr, &restarter_fmri);
4027 	if (err != 0 || enabled == -1) {
4028 		MUTEX_UNLOCK(&dgraph_lock);
4029 		scf_instance_destroy(inst);
4030 		startd_free(fmri, max_scf_fmri_size);
4031 
4032 		switch (err) {
4033 		case ENOENT:
4034 		case 0:
4035 			startd_free(restarter_fmri, max_scf_value_size);
4036 			return (-1);
4037 
4038 		case ECONNABORTED:
4039 		case ECANCELED:
4040 			startd_free(restarter_fmri, max_scf_value_size);
4041 			return (err);
4042 
4043 		default:
4044 			bad_error("libscf_get_basic_instance_data", err);
4045 		}
4046 	}
4047 
4048 	oldflags = v->gv_flags;
4049 	v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4050 	    (enabled ? GV_ENBLD_NOOVR : 0);
4051 
4052 	if (enabled_ovr != -1)
4053 		enabled = enabled_ovr;
4054 
4055 	/*
4056 	 * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4057 	 * subgraph.
4058 	 */
4059 	if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4060 		(void) eval_subgraph(v, h);
4061 
4062 	scf_instance_destroy(inst);
4063 
4064 	/* Ignore restarter change for now. */
4065 
4066 	startd_free(restarter_fmri, max_scf_value_size);
4067 	startd_free(fmri, max_scf_fmri_size);
4068 
4069 	/*
4070 	 * Always send _ENABLE or _DISABLE.  We could avoid this if the
4071 	 * restarter didn't change and the enabled value didn't change, but
4072 	 * that's not easy to check and improbable anyway, so we'll just do
4073 	 * this.
4074 	 */
4075 	graph_enable_by_vertex(v, enabled, 1);
4076 
4077 	MUTEX_UNLOCK(&dgraph_lock);
4078 
4079 	return (0);
4080 }
4081 
4082 /*
4083  * Delete all of the property group dependencies of v, update inst's running
4084  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4085  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4086  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4087  * the same for v's dependents.
4088  *
4089  * Returns
4090  *   0 - success
4091  *   ECONNABORTED - repository connection broken
4092  *   ECANCELED - inst was deleted
4093  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4094  *   -1 - libscf_snapshots_refresh() failed
4095  */
4096 static int
4097 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4098 {
4099 	int r;
4100 	int enabled;
4101 
4102 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4103 	assert(v->gv_type == GVT_INST);
4104 
4105 	/* Only refresh services with valid general/enabled properties. */
4106 	r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4107 	    v->gv_name, &enabled, NULL, NULL);
4108 	switch (r) {
4109 	case 0:
4110 		break;
4111 
4112 	case ECONNABORTED:
4113 	case ECANCELED:
4114 		return (r);
4115 
4116 	case ENOENT:
4117 		log_framework(LOG_DEBUG,
4118 		    "Ignoring %s because it has no general property group.\n",
4119 		    v->gv_name);
4120 		return (EINVAL);
4121 
4122 	default:
4123 		bad_error("libscf_get_basic_instance_data", r);
4124 	}
4125 
4126 	if (enabled == -1)
4127 		return (EINVAL);
4128 
4129 	r = libscf_snapshots_refresh(inst, v->gv_name);
4130 	if (r != 0) {
4131 		if (r != -1)
4132 			bad_error("libscf_snapshots_refresh", r);
4133 
4134 		/* error logged */
4135 		return (r);
4136 	}
4137 
4138 	r = refresh_vertex(v, inst);
4139 	if (r != 0 && r != ECONNABORTED)
4140 		bad_error("refresh_vertex", r);
4141 	return (r);
4142 }
4143 
4144 /*
4145  * Returns true only if none of this service's dependents are 'up' -- online
4146  * or degraded (offline is considered down in this situation). This function
4147  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4148  */
4149 static boolean_t
4150 insubtree_dependents_down(graph_vertex_t *v)
4151 {
4152 	graph_vertex_t *vv;
4153 	graph_edge_t *e;
4154 
4155 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4156 
4157 	for (e = uu_list_first(v->gv_dependents); e != NULL;
4158 	    e = uu_list_next(v->gv_dependents, e)) {
4159 		vv = e->ge_vertex;
4160 		if (vv->gv_type == GVT_INST) {
4161 			if ((vv->gv_flags & GV_CONFIGURED) == 0)
4162 				continue;
4163 
4164 			if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4165 				continue;
4166 
4167 			if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4168 			    (vv->gv_state == RESTARTER_STATE_DEGRADED))
4169 				return (B_FALSE);
4170 		} else {
4171 			/*
4172 			 * For dependency groups or service vertices, keep
4173 			 * traversing to see if instances are running.
4174 			 */
4175 			if (insubtree_dependents_down(vv) == B_FALSE)
4176 				return (B_FALSE);
4177 		}
4178 	}
4179 
4180 	return (B_TRUE);
4181 }
4182 
4183 /*
4184  * Returns true only if none of this service's dependents are 'up' -- online,
4185  * degraded, or offline.
4186  */
4187 static int
4188 is_nonsubgraph_leaf(graph_vertex_t *v)
4189 {
4190 	graph_vertex_t *vv;
4191 	graph_edge_t *e;
4192 
4193 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4194 
4195 	for (e = uu_list_first(v->gv_dependents);
4196 	    e != NULL;
4197 	    e = uu_list_next(v->gv_dependents, e)) {
4198 
4199 		vv = e->ge_vertex;
4200 		if (vv->gv_type == GVT_INST) {
4201 			if ((vv->gv_flags & GV_CONFIGURED) == 0)
4202 				continue;
4203 
4204 			if (vv->gv_flags & GV_INSUBGRAPH)
4205 				continue;
4206 
4207 			if (up_state(vv->gv_state))
4208 				return (0);
4209 		} else {
4210 			/*
4211 			 * For dependency group or service vertices, keep
4212 			 * traversing to see if instances are running.
4213 			 *
4214 			 * We should skip exclude_all dependencies otherwise
4215 			 * the vertex will never be considered as a leaf
4216 			 * if the dependent is offline. The main reason for
4217 			 * this is that disable_nonsubgraph_leaves() skips
4218 			 * exclusion dependencies.
4219 			 */
4220 			if (vv->gv_type == GVT_GROUP &&
4221 			    vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4222 				continue;
4223 
4224 			if (!is_nonsubgraph_leaf(vv))
4225 				return (0);
4226 		}
4227 	}
4228 
4229 	return (1);
4230 }
4231 
4232 /*
4233  * Disable v temporarily.  Attempt to do this by setting its enabled override
4234  * property in the repository.  If that fails, send a _DISABLE command.
4235  * Returns 0 on success and ECONNABORTED if the repository connection is
4236  * broken.
4237  */
4238 static int
4239 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4240 {
4241 	const char * const emsg = "Could not temporarily disable %s because "
4242 	    "%s.  Will stop service anyways.  Repository status for the "
4243 	    "service may be inaccurate.\n";
4244 	const char * const emsg_cbroken =
4245 	    "the repository connection was broken";
4246 
4247 	scf_instance_t *inst;
4248 	int r;
4249 
4250 	inst = scf_instance_create(h);
4251 	if (inst == NULL) {
4252 		char buf[100];
4253 
4254 		(void) snprintf(buf, sizeof (buf),
4255 		    "scf_instance_create() failed (%s)",
4256 		    scf_strerror(scf_error()));
4257 		log_error(LOG_WARNING, emsg, v->gv_name, buf);
4258 
4259 		graph_enable_by_vertex(v, 0, 0);
4260 		return (0);
4261 	}
4262 
4263 	r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4264 	    NULL, NULL, SCF_DECODE_FMRI_EXACT);
4265 	if (r != 0) {
4266 		switch (scf_error()) {
4267 		case SCF_ERROR_CONNECTION_BROKEN:
4268 			log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4269 			graph_enable_by_vertex(v, 0, 0);
4270 			return (ECONNABORTED);
4271 
4272 		case SCF_ERROR_NOT_FOUND:
4273 			return (0);
4274 
4275 		case SCF_ERROR_HANDLE_MISMATCH:
4276 		case SCF_ERROR_INVALID_ARGUMENT:
4277 		case SCF_ERROR_CONSTRAINT_VIOLATED:
4278 		case SCF_ERROR_NOT_BOUND:
4279 		default:
4280 			bad_error("scf_handle_decode_fmri",
4281 			    scf_error());
4282 		}
4283 	}
4284 
4285 	r = libscf_set_enable_ovr(inst, 0);
4286 	switch (r) {
4287 	case 0:
4288 		scf_instance_destroy(inst);
4289 		return (0);
4290 
4291 	case ECANCELED:
4292 		scf_instance_destroy(inst);
4293 		return (0);
4294 
4295 	case ECONNABORTED:
4296 		log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4297 		graph_enable_by_vertex(v, 0, 0);
4298 		return (ECONNABORTED);
4299 
4300 	case EPERM:
4301 		log_error(LOG_WARNING, emsg, v->gv_name,
4302 		    "the repository denied permission");
4303 		graph_enable_by_vertex(v, 0, 0);
4304 		return (0);
4305 
4306 	case EROFS:
4307 		log_error(LOG_WARNING, emsg, v->gv_name,
4308 		    "the repository is read-only");
4309 		graph_enable_by_vertex(v, 0, 0);
4310 		return (0);
4311 
4312 	default:
4313 		bad_error("libscf_set_enable_ovr", r);
4314 		/* NOTREACHED */
4315 	}
4316 }
4317 
4318 /*
4319  * Of the transitive instance dependencies of v, offline those which are
4320  * in the subtree and which are leaves (i.e., have no dependents which are
4321  * "up").
4322  */
4323 void
4324 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4325 {
4326 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4327 
4328 	/* If v isn't an instance, recurse on its dependencies. */
4329 	if (v->gv_type != GVT_INST) {
4330 		graph_walk_dependencies(v, offline_subtree_leaves, arg);
4331 		return;
4332 	}
4333 
4334 	/*
4335 	 * If v is not in the subtree, so should all of its dependencies,
4336 	 * so do nothing.
4337 	 */
4338 	if ((v->gv_flags & GV_TOOFFLINE) == 0)
4339 		return;
4340 
4341 	/* If v isn't a leaf because it's already down, recurse. */
4342 	if (!up_state(v->gv_state)) {
4343 		graph_walk_dependencies(v, offline_subtree_leaves, arg);
4344 		return;
4345 	}
4346 
4347 	/* if v is a leaf, offline it or disable it if it's the last one */
4348 	if (insubtree_dependents_down(v) == B_TRUE) {
4349 		if (v->gv_flags & GV_TODISABLE)
4350 			vertex_send_event(v,
4351 			    RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4352 		else
4353 			offline_vertex(v);
4354 	}
4355 }
4356 
4357 void
4358 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4359 {
4360 	graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4361 }
4362 
4363 
4364 /*
4365  * Of the transitive instance dependencies of v, disable those which are not
4366  * in the subgraph and which are leaves (i.e., have no dependents which are
4367  * "up").
4368  */
4369 static void
4370 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4371 {
4372 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4373 
4374 	/*
4375 	 * We must skip exclusion dependencies because they are allowed to
4376 	 * complete dependency cycles.  This is correct because A's exclusion
4377 	 * dependency on B doesn't bear on the order in which they should be
4378 	 * stopped.  Indeed, the exclusion dependency should guarantee that
4379 	 * they are never online at the same time.
4380 	 */
4381 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4382 		return;
4383 
4384 	/* If v isn't an instance, recurse on its dependencies. */
4385 	if (v->gv_type != GVT_INST)
4386 		goto recurse;
4387 
4388 	if ((v->gv_flags & GV_CONFIGURED) == 0)
4389 		/*
4390 		 * Unconfigured instances should have no dependencies, but in
4391 		 * case they ever get them,
4392 		 */
4393 		goto recurse;
4394 
4395 	/*
4396 	 * If v is in the subgraph, so should all of its dependencies, so do
4397 	 * nothing.
4398 	 */
4399 	if (v->gv_flags & GV_INSUBGRAPH)
4400 		return;
4401 
4402 	/* If v isn't a leaf because it's already down, recurse. */
4403 	if (!up_state(v->gv_state))
4404 		goto recurse;
4405 
4406 	/* If v is disabled but not down yet, be patient. */
4407 	if ((v->gv_flags & GV_ENABLED) == 0)
4408 		return;
4409 
4410 	/* If v is a leaf, disable it. */
4411 	if (is_nonsubgraph_leaf(v))
4412 		(void) disable_service_temporarily(v, (scf_handle_t *)arg);
4413 
4414 	return;
4415 
4416 recurse:
4417 	graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4418 }
4419 
4420 /*
4421  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4422  * Otherwise set its state to state.  If the instance has entered a state
4423  * which requires automatic action, take it (Uninitialized: do
4424  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4425  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4426  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4427  * _START.  Online, Degraded: if the instance wasn't running, update its start
4428  * snapshot.  Maintenance: no action.)
4429  *
4430  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4431  */
4432 static int
4433 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4434     restarter_instance_state_t state, restarter_error_t serr)
4435 {
4436 	graph_vertex_t *v;
4437 	int err = 0;
4438 	restarter_instance_state_t old_state;
4439 
4440 	MUTEX_LOCK(&dgraph_lock);
4441 
4442 	v = vertex_get_by_name(inst_name);
4443 	if (v == NULL) {
4444 		MUTEX_UNLOCK(&dgraph_lock);
4445 		return (ENOENT);
4446 	}
4447 
4448 	assert(v->gv_type == GVT_INST);
4449 
4450 	switch (state) {
4451 	case RESTARTER_STATE_UNINIT:
4452 	case RESTARTER_STATE_DISABLED:
4453 	case RESTARTER_STATE_OFFLINE:
4454 	case RESTARTER_STATE_ONLINE:
4455 	case RESTARTER_STATE_DEGRADED:
4456 	case RESTARTER_STATE_MAINT:
4457 		break;
4458 
4459 	default:
4460 		MUTEX_UNLOCK(&dgraph_lock);
4461 		return (EINVAL);
4462 	}
4463 
4464 	log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4465 	    instance_state_str[v->gv_state], instance_state_str[state]);
4466 
4467 	old_state = v->gv_state;
4468 	v->gv_state = state;
4469 
4470 	err = gt_transition(h, v, serr, old_state);
4471 
4472 	MUTEX_UNLOCK(&dgraph_lock);
4473 	return (err);
4474 }
4475 
4476 /*
4477  * Handle state changes during milestone shutdown.  See
4478  * dgraph_set_milestone().  If the repository connection is broken,
4479  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4480  * the vertex anyway.
4481  */
4482 int
4483 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4484     restarter_instance_state_t old_state)
4485 {
4486 	int was_up, now_up;
4487 	int ret = 0;
4488 
4489 	assert(v->gv_type == GVT_INST);
4490 
4491 	/* Don't care if we're not going to a milestone. */
4492 	if (milestone == NULL)
4493 		return (0);
4494 
4495 	/* Don't care if we already finished coming down. */
4496 	if (non_subgraph_svcs == 0)
4497 		return (0);
4498 
4499 	/* Don't care if the service is in the subgraph. */
4500 	if (v->gv_flags & GV_INSUBGRAPH)
4501 		return (0);
4502 
4503 	/*
4504 	 * Update non_subgraph_svcs.  It is the number of non-subgraph
4505 	 * services which are in online, degraded, or offline.
4506 	 */
4507 
4508 	was_up = up_state(old_state);
4509 	now_up = up_state(v->gv_state);
4510 
4511 	if (!was_up && now_up) {
4512 		++non_subgraph_svcs;
4513 	} else if (was_up && !now_up) {
4514 		--non_subgraph_svcs;
4515 
4516 		if (non_subgraph_svcs == 0) {
4517 			if (halting != -1) {
4518 				do_uadmin();
4519 			} else if (go_single_user_mode || go_to_level1) {
4520 				(void) startd_thread_create(single_user_thread,
4521 				    NULL);
4522 			}
4523 			return (0);
4524 		}
4525 	}
4526 
4527 	/* If this service is a leaf, it should be disabled. */
4528 	if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4529 		int r;
4530 
4531 		r = disable_service_temporarily(v, h);
4532 		switch (r) {
4533 		case 0:
4534 			break;
4535 
4536 		case ECONNABORTED:
4537 			ret = ECONNABORTED;
4538 			break;
4539 
4540 		default:
4541 			bad_error("disable_service_temporarily", r);
4542 		}
4543 	}
4544 
4545 	/*
4546 	 * If the service just came down, propagate the disable to the newly
4547 	 * exposed leaves.
4548 	 */
4549 	if (was_up && !now_up)
4550 		graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4551 		    (void *)h);
4552 
4553 	return (ret);
4554 }
4555 
4556 /*
4557  * Decide whether to start up an sulogin thread after a service is
4558  * finished changing state.  Only need to do the full can_come_up()
4559  * evaluation if an instance is changing state, we're not halfway through
4560  * loading the thread, and we aren't shutting down or going to the single
4561  * user milestone.
4562  */
4563 void
4564 graph_transition_sulogin(restarter_instance_state_t state,
4565     restarter_instance_state_t old_state)
4566 {
4567 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4568 
4569 	if (state != old_state && st->st_load_complete &&
4570 	    !go_single_user_mode && !go_to_level1 &&
4571 	    halting == -1) {
4572 		if (!sulogin_thread_running && !can_come_up()) {
4573 			(void) startd_thread_create(sulogin_thread, NULL);
4574 			sulogin_thread_running = B_TRUE;
4575 		}
4576 	}
4577 }
4578 
4579 /*
4580  * Propagate a start, stop event, or a satisfiability event.
4581  *
4582  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4583  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4584  * full dependent graph to check for newly satisfied nodes.  This is
4585  * necessary for cases when non-direct dependents may be effected but direct
4586  * dependents may not (e.g. for optional_all evaluations, see the
4587  * propagate_satbility() comments).
4588  *
4589  * PROPAGATE_SAT should be used whenever a non-running service moves into
4590  * a state which can satisfy optional dependencies, like disabled or
4591  * maintenance.
4592  */
4593 void
4594 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4595     restarter_error_t rerr)
4596 {
4597 	if (type == PROPAGATE_STOP) {
4598 		graph_walk_dependents(v, propagate_stop, (void *)rerr);
4599 	} else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4600 		graph_walk_dependents(v, propagate_start, NULL);
4601 
4602 		if (type == PROPAGATE_SAT)
4603 			propagate_satbility(v);
4604 	} else {
4605 #ifndef NDEBUG
4606 		uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4607 		    __LINE__, type);
4608 #endif
4609 		abort();
4610 	}
4611 }
4612 
4613 /*
4614  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4615  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4616  * all property group dependencies, and the dependency on the restarter,
4617  * disposing of vertices as appropriate.  If other vertices depend on this
4618  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4619  * returns 0.
4620  */
4621 static int
4622 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4623 {
4624 	graph_vertex_t *v;
4625 	graph_edge_t *e;
4626 	uu_list_t *old_deps;
4627 	int err;
4628 
4629 	log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4630 
4631 	MUTEX_LOCK(&dgraph_lock);
4632 
4633 	v = vertex_get_by_name(fmri);
4634 	if (v == NULL) {
4635 		MUTEX_UNLOCK(&dgraph_lock);
4636 		return (0);
4637 	}
4638 
4639 	/* Send restarter delete event. */
4640 	if (v->gv_flags & GV_CONFIGURED)
4641 		graph_unset_restarter(v);
4642 
4643 	if (milestone > MILESTONE_NONE) {
4644 		/*
4645 		 * Make a list of v's current dependencies so we can
4646 		 * reevaluate their GV_INSUBGRAPH flags after the dependencies
4647 		 * are removed.
4648 		 */
4649 		old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4650 
4651 		err = uu_list_walk(v->gv_dependencies,
4652 		    (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4653 		assert(err == 0);
4654 	}
4655 
4656 	delete_instance_dependencies(v, B_TRUE);
4657 
4658 	/*
4659 	 * Deleting an instance can both satisfy and unsatisfy dependencies,
4660 	 * depending on their type.  First propagate the stop as a RERR_RESTART
4661 	 * event -- deletion isn't a fault, just a normal stop.  This gives
4662 	 * dependent services the chance to do a clean shutdown.  Then, mark
4663 	 * the service as unconfigured and propagate the start event for the
4664 	 * optional_all dependencies that might have become satisfied.
4665 	 */
4666 	graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
4667 
4668 	v->gv_flags &= ~GV_CONFIGURED;
4669 	v->gv_flags &= ~GV_DEATHROW;
4670 
4671 	graph_walk_dependents(v, propagate_start, NULL);
4672 	propagate_satbility(v);
4673 
4674 	/*
4675 	 * If there are no (non-service) dependents, the vertex can be
4676 	 * completely removed.
4677 	 */
4678 	if (v != milestone && v->gv_refs == 0 &&
4679 	    uu_list_numnodes(v->gv_dependents) == 1)
4680 		remove_inst_vertex(v);
4681 
4682 	if (milestone > MILESTONE_NONE) {
4683 		void *cookie = NULL;
4684 
4685 		while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
4686 			v = e->ge_vertex;
4687 
4688 			if (vertex_unref(v) == VERTEX_INUSE)
4689 				while (eval_subgraph(v, h) == ECONNABORTED)
4690 					libscf_handle_rebind(h);
4691 
4692 			startd_free(e, sizeof (*e));
4693 		}
4694 
4695 		uu_list_destroy(old_deps);
4696 	}
4697 
4698 	MUTEX_UNLOCK(&dgraph_lock);
4699 
4700 	return (0);
4701 }
4702 
4703 /*
4704  * Return the eventual (maybe current) milestone in the form of a
4705  * legacy runlevel.
4706  */
4707 static char
4708 target_milestone_as_runlevel()
4709 {
4710 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4711 
4712 	if (milestone == NULL)
4713 		return ('3');
4714 	else if (milestone == MILESTONE_NONE)
4715 		return ('0');
4716 
4717 	if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
4718 		return ('2');
4719 	else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
4720 		return ('S');
4721 	else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
4722 		return ('3');
4723 
4724 #ifndef NDEBUG
4725 	(void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
4726 	    __FILE__, __LINE__, milestone->gv_name);
4727 #endif
4728 	abort();
4729 	/* NOTREACHED */
4730 }
4731 
4732 static struct {
4733 	char	rl;
4734 	int	sig;
4735 } init_sigs[] = {
4736 	{ 'S', SIGBUS },
4737 	{ '0', SIGINT },
4738 	{ '1', SIGQUIT },
4739 	{ '2', SIGILL },
4740 	{ '3', SIGTRAP },
4741 	{ '4', SIGIOT },
4742 	{ '5', SIGEMT },
4743 	{ '6', SIGFPE },
4744 	{ 0, 0 }
4745 };
4746 
4747 static void
4748 signal_init(char rl)
4749 {
4750 	pid_t init_pid;
4751 	int i;
4752 
4753 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4754 
4755 	if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
4756 	    sizeof (init_pid)) != sizeof (init_pid)) {
4757 		log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
4758 		return;
4759 	}
4760 
4761 	for (i = 0; init_sigs[i].rl != 0; ++i)
4762 		if (init_sigs[i].rl == rl)
4763 			break;
4764 
4765 	if (init_sigs[i].rl != 0) {
4766 		if (kill(init_pid, init_sigs[i].sig) != 0) {
4767 			switch (errno) {
4768 			case EPERM:
4769 			case ESRCH:
4770 				log_error(LOG_NOTICE, "Could not signal init: "
4771 				    "%s.\n", strerror(errno));
4772 				break;
4773 
4774 			case EINVAL:
4775 			default:
4776 				bad_error("kill", errno);
4777 			}
4778 		}
4779 	}
4780 }
4781 
4782 /*
4783  * This is called when one of the major milestones changes state, or when
4784  * init is signalled and tells us it was told to change runlevel.  We wait
4785  * to reach the milestone because this allows /etc/inittab entries to retain
4786  * some boot ordering: historically, entries could place themselves before/after
4787  * the running of /sbin/rcX scripts but we can no longer make the
4788  * distinction because the /sbin/rcX scripts no longer exist as punctuation
4789  * marks in /etc/inittab.
4790  *
4791  * Also, we only trigger an update when we reach the eventual target
4792  * milestone: without this, an /etc/inittab entry marked only for
4793  * runlevel 2 would be executed for runlevel 3, which is not how
4794  * /etc/inittab entries work.
4795  *
4796  * If we're single user coming online, then we set utmpx to the target
4797  * runlevel so that legacy scripts can work as expected.
4798  */
4799 static void
4800 graph_runlevel_changed(char rl, int online)
4801 {
4802 	char trl;
4803 
4804 	assert(PTHREAD_MUTEX_HELD(&dgraph_lock));
4805 
4806 	trl = target_milestone_as_runlevel();
4807 
4808 	if (online) {
4809 		if (rl == trl) {
4810 			current_runlevel = trl;
4811 			signal_init(trl);
4812 		} else if (rl == 'S') {
4813 			/*
4814 			 * At boot, set the entry early for the benefit of the
4815 			 * legacy init scripts.
4816 			 */
4817 			utmpx_set_runlevel(trl, 'S', B_FALSE);
4818 		}
4819 	} else {
4820 		if (rl == '3' && trl == '2') {
4821 			current_runlevel = trl;
4822 			signal_init(trl);
4823 		} else if (rl == '2' && trl == 'S') {
4824 			current_runlevel = trl;
4825 			signal_init(trl);
4826 		}
4827 	}
4828 }
4829 
4830 /*
4831  * Move to a backwards-compatible runlevel by executing the appropriate
4832  * /etc/rc?.d/K* scripts and/or setting the milestone.
4833  *
4834  * Returns
4835  *   0 - success
4836  *   ECONNRESET - success, but handle was reset
4837  *   ECONNABORTED - repository connection broken
4838  *   ECANCELED - pg was deleted
4839  */
4840 static int
4841 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
4842 {
4843 	char rl;
4844 	scf_handle_t *h;
4845 	int r;
4846 	const char *ms = NULL;	/* what to commit as options/milestone */
4847 	boolean_t rebound = B_FALSE;
4848 	int mark_rl = 0;
4849 
4850 	const char * const stop = "stop";
4851 
4852 	r = libscf_extract_runlevel(prop, &rl);
4853 	switch (r) {
4854 	case 0:
4855 		break;
4856 
4857 	case ECONNABORTED:
4858 	case ECANCELED:
4859 		return (r);
4860 
4861 	case EINVAL:
4862 	case ENOENT:
4863 		log_error(LOG_WARNING, "runlevel property is misconfigured; "
4864 		    "ignoring.\n");
4865 		/* delete the bad property */
4866 		goto nolock_out;
4867 
4868 	default:
4869 		bad_error("libscf_extract_runlevel", r);
4870 	}
4871 
4872 	switch (rl) {
4873 	case 's':
4874 		rl = 'S';
4875 		/* FALLTHROUGH */
4876 
4877 	case 'S':
4878 	case '2':
4879 	case '3':
4880 		/*
4881 		 * These cases cause a milestone change, so
4882 		 * graph_runlevel_changed() will eventually deal with
4883 		 * signalling init.
4884 		 */
4885 		break;
4886 
4887 	case '0':
4888 	case '1':
4889 	case '4':
4890 	case '5':
4891 	case '6':
4892 		mark_rl = 1;
4893 		break;
4894 
4895 	default:
4896 		log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
4897 		ms = NULL;
4898 		goto nolock_out;
4899 	}
4900 
4901 	h = scf_pg_handle(pg);
4902 
4903 	MUTEX_LOCK(&dgraph_lock);
4904 
4905 	/*
4906 	 * Since this triggers no milestone changes, force it by hand.
4907 	 */
4908 	if (current_runlevel == '4' && rl == '3')
4909 		mark_rl = 1;
4910 
4911 	/*
4912 	 * 1. If we are here after an "init X":
4913 	 *
4914 	 * init X
4915 	 *	init/lscf_set_runlevel()
4916 	 *		process_pg_event()
4917 	 *		dgraph_set_runlevel()
4918 	 *
4919 	 * then we haven't passed through graph_runlevel_changed() yet,
4920 	 * therefore 'current_runlevel' has not changed for sure but 'rl' has.
4921 	 * In consequence, if 'rl' is lower than 'current_runlevel', we change
4922 	 * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
4923 	 * past this test.
4924 	 *
4925 	 * 2. On the other hand, if we are here after a "svcadm milestone":
4926 	 *
4927 	 * svcadm milestone X
4928 	 *	dgraph_set_milestone()
4929 	 *		handle_graph_update_event()
4930 	 *		dgraph_set_instance_state()
4931 	 *		graph_post_X_[online|offline]()
4932 	 *		graph_runlevel_changed()
4933 	 *		signal_init()
4934 	 *			init/lscf_set_runlevel()
4935 	 *				process_pg_event()
4936 	 *				dgraph_set_runlevel()
4937 	 *
4938 	 * then we already passed through graph_runlevel_changed() (by the way
4939 	 * of dgraph_set_milestone()) and 'current_runlevel' may have changed
4940 	 * and already be equal to 'rl' so we are going to return immediately
4941 	 * from dgraph_set_runlevel() without changing the system runlevel and
4942 	 * without executing the /etc/rc?.d/K* scripts.
4943 	 */
4944 	if (rl == current_runlevel) {
4945 		ms = NULL;
4946 		goto out;
4947 	}
4948 
4949 	log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
4950 
4951 	/*
4952 	 * Make sure stop rc scripts see the new settings via who -r.
4953 	 */
4954 	utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
4955 
4956 	/*
4957 	 * Some run levels don't have a direct correspondence to any
4958 	 * milestones, so we have to signal init directly.
4959 	 */
4960 	if (mark_rl) {
4961 		current_runlevel = rl;
4962 		signal_init(rl);
4963 	}
4964 
4965 	switch (rl) {
4966 	case 'S':
4967 		uu_warn("The system is coming down for administration.  "
4968 		    "Please wait.\n");
4969 		fork_rc_script(rl, stop, B_FALSE);
4970 		ms = single_user_fmri;
4971 		go_single_user_mode = B_TRUE;
4972 		break;
4973 
4974 	case '0':
4975 		fork_rc_script(rl, stop, B_TRUE);
4976 		halting = AD_HALT;
4977 		goto uadmin;
4978 
4979 	case '5':
4980 		fork_rc_script(rl, stop, B_TRUE);
4981 		halting = AD_POWEROFF;
4982 		goto uadmin;
4983 
4984 	case '6':
4985 		fork_rc_script(rl, stop, B_TRUE);
4986 		halting = AD_BOOT;
4987 		goto uadmin;
4988 
4989 uadmin:
4990 		uu_warn("The system is coming down.  Please wait.\n");
4991 		ms = "none";
4992 
4993 		/*
4994 		 * We can't wait until all services are offline since this
4995 		 * thread is responsible for taking them offline.  Instead we
4996 		 * set halting to the second argument for uadmin() and call
4997 		 * do_uadmin() from dgraph_set_instance_state() when
4998 		 * appropriate.
4999 		 */
5000 		break;
5001 
5002 	case '1':
5003 		if (current_runlevel != 'S') {
5004 			uu_warn("Changing to state 1.\n");
5005 			fork_rc_script(rl, stop, B_FALSE);
5006 		} else {
5007 			uu_warn("The system is coming up for administration.  "
5008 			    "Please wait.\n");
5009 		}
5010 		ms = single_user_fmri;
5011 		go_to_level1 = B_TRUE;
5012 		break;
5013 
5014 	case '2':
5015 		if (current_runlevel == '3' || current_runlevel == '4')
5016 			fork_rc_script(rl, stop, B_FALSE);
5017 		ms = multi_user_fmri;
5018 		break;
5019 
5020 	case '3':
5021 	case '4':
5022 		ms = "all";
5023 		break;
5024 
5025 	default:
5026 #ifndef NDEBUG
5027 		(void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5028 		    __FILE__, __LINE__, rl, rl);
5029 #endif
5030 		abort();
5031 	}
5032 
5033 out:
5034 	MUTEX_UNLOCK(&dgraph_lock);
5035 
5036 nolock_out:
5037 	switch (r = libscf_clear_runlevel(pg, ms)) {
5038 	case 0:
5039 		break;
5040 
5041 	case ECONNABORTED:
5042 		libscf_handle_rebind(h);
5043 		rebound = B_TRUE;
5044 		goto nolock_out;
5045 
5046 	case ECANCELED:
5047 		break;
5048 
5049 	case EPERM:
5050 	case EACCES:
5051 	case EROFS:
5052 		log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5053 		    "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5054 		break;
5055 
5056 	default:
5057 		bad_error("libscf_clear_runlevel", r);
5058 	}
5059 
5060 	return (rebound ? ECONNRESET : 0);
5061 }
5062 
5063 /*
5064  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5065  * to the instances that are supposed to go offline during an
5066  * administrative disable operation.
5067  */
5068 static int
5069 mark_subtree(graph_edge_t *e, void *arg)
5070 {
5071 	graph_vertex_t *v;
5072 	int r;
5073 
5074 	v = e->ge_vertex;
5075 
5076 	/* If it's already in the subgraph, skip. */
5077 	if (v->gv_flags & GV_TOOFFLINE)
5078 		return (UU_WALK_NEXT);
5079 
5080 	switch (v->gv_type) {
5081 	case GVT_INST:
5082 		/* If the instance is already disabled, skip it. */
5083 		if (!(v->gv_flags & GV_ENABLED))
5084 			return (UU_WALK_NEXT);
5085 
5086 		v->gv_flags |= GV_TOOFFLINE;
5087 		log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5088 		break;
5089 	case GVT_GROUP:
5090 		/*
5091 		 * Skip all excluded and optional_all dependencies and decide
5092 		 * whether to offline the service based on restart_on attribute.
5093 		 */
5094 		if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL ||
5095 		    v->gv_depgroup == DEPGRP_OPTIONAL_ALL ||
5096 		    v->gv_restart < RERR_RESTART)
5097 			return (UU_WALK_NEXT);
5098 		break;
5099 	}
5100 
5101 	r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5102 	    0);
5103 	assert(r == 0);
5104 	return (UU_WALK_NEXT);
5105 }
5106 
5107 static int
5108 mark_subgraph(graph_edge_t *e, void *arg)
5109 {
5110 	graph_vertex_t *v;
5111 	int r;
5112 	int optional = (int)arg;
5113 
5114 	v = e->ge_vertex;
5115 
5116 	/* If it's already in the subgraph, skip. */
5117 	if (v->gv_flags & GV_INSUBGRAPH)
5118 		return (UU_WALK_NEXT);
5119 
5120 	/*
5121 	 * Keep track if walk has entered an optional dependency group
5122 	 */
5123 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5124 		optional = 1;
5125 	}
5126 	/*
5127 	 * Quit if we are in an optional dependency group and the instance
5128 	 * is disabled
5129 	 */
5130 	if (optional && (v->gv_type == GVT_INST) &&
5131 	    (!(v->gv_flags & GV_ENBLD_NOOVR)))
5132 		return (UU_WALK_NEXT);
5133 
5134 	v->gv_flags |= GV_INSUBGRAPH;
5135 
5136 	/* Skip all excluded dependencies. */
5137 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5138 		return (UU_WALK_NEXT);
5139 
5140 	r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5141 	    (void *)optional, 0);
5142 	assert(r == 0);
5143 	return (UU_WALK_NEXT);
5144 }
5145 
5146 /*
5147  * Bring down all services which are not dependencies of fmri.  The
5148  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5149  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5150  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5151  * could consider "offline" to be down, and refrain from sending start
5152  * commands for such services, but that's not strictly necessary, so we'll
5153  * decline to intrude on the state machine.  It would probably confuse users
5154  * anyway.
5155  *
5156  * The services should be brought down in reverse-dependency order, so we
5157  * can't do it all at once here.  We initiate by override-disabling the leaves
5158  * of the dependency tree -- those services which are up but have no
5159  * dependents which are up.  When they come down,
5160  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5161  * exposed leaves.  Perseverance will ensure completion.
5162  *
5163  * Sometimes we need to take action when the transition is complete, like
5164  * start sulogin or halt the system.  To tell when we're done, we initialize
5165  * non_subgraph_svcs here to be the number of services which need to come
5166  * down.  As each does, we decrement the counter.  When it hits zero, we take
5167  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5168  *
5169  * In case we're coming up, we also remove any enable-overrides for the
5170  * services which are dependencies of fmri.
5171  *
5172  * If norepository is true, the function will not change the repository.
5173  *
5174  * The decision to change the system run level in accordance with the milestone
5175  * is taken in dgraph_set_runlevel().
5176  *
5177  * Returns
5178  *   0 - success
5179  *   ECONNRESET - success, but handle was rebound
5180  *   EINVAL - fmri is invalid (error is logged)
5181  *   EALREADY - the milestone is already set to fmri
5182  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5183  */
5184 static int
5185 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5186 {
5187 	const char *cfmri, *fs;
5188 	graph_vertex_t *nm, *v;
5189 	int ret = 0, r;
5190 	scf_instance_t *inst;
5191 	boolean_t isall, isnone, rebound = B_FALSE;
5192 
5193 	/* Validate fmri */
5194 	isall = (strcmp(fmri, "all") == 0);
5195 	isnone = (strcmp(fmri, "none") == 0);
5196 
5197 	if (!isall && !isnone) {
5198 		if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5199 			goto reject;
5200 
5201 		if (strcmp(cfmri, single_user_fmri) != 0 &&
5202 		    strcmp(cfmri, multi_user_fmri) != 0 &&
5203 		    strcmp(cfmri, multi_user_svr_fmri) != 0) {
5204 			startd_free((void *)cfmri, max_scf_fmri_size);
5205 reject:
5206 			log_framework(LOG_WARNING,
5207 			    "Rejecting request for invalid milestone \"%s\".\n",
5208 			    fmri);
5209 			return (EINVAL);
5210 		}
5211 	}
5212 
5213 	inst = safe_scf_instance_create(h);
5214 
5215 	MUTEX_LOCK(&dgraph_lock);
5216 
5217 	if (milestone == NULL) {
5218 		if (isall) {
5219 			log_framework(LOG_DEBUG,
5220 			    "Milestone already set to all.\n");
5221 			ret = EALREADY;
5222 			goto out;
5223 		}
5224 	} else if (milestone == MILESTONE_NONE) {
5225 		if (isnone) {
5226 			log_framework(LOG_DEBUG,
5227 			    "Milestone already set to none.\n");
5228 			ret = EALREADY;
5229 			goto out;
5230 		}
5231 	} else {
5232 		if (!isall && !isnone &&
5233 		    strcmp(cfmri, milestone->gv_name) == 0) {
5234 			log_framework(LOG_DEBUG,
5235 			    "Milestone already set to %s.\n", cfmri);
5236 			ret = EALREADY;
5237 			goto out;
5238 		}
5239 	}
5240 
5241 	if (!isall && !isnone) {
5242 		nm = vertex_get_by_name(cfmri);
5243 		if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5244 			log_framework(LOG_WARNING, "Cannot set milestone to %s "
5245 			    "because no such service exists.\n", cfmri);
5246 			ret = ENOENT;
5247 			goto out;
5248 		}
5249 	}
5250 
5251 	log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5252 
5253 	/*
5254 	 * Set milestone, removing the old one if this was the last reference.
5255 	 */
5256 	if (milestone > MILESTONE_NONE)
5257 		(void) vertex_unref(milestone);
5258 
5259 	if (isall)
5260 		milestone = NULL;
5261 	else if (isnone)
5262 		milestone = MILESTONE_NONE;
5263 	else {
5264 		milestone = nm;
5265 		/* milestone should count as a reference */
5266 		vertex_ref(milestone);
5267 	}
5268 
5269 	/* Clear all GV_INSUBGRAPH bits. */
5270 	for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5271 		v->gv_flags &= ~GV_INSUBGRAPH;
5272 
5273 	if (!isall && !isnone) {
5274 		/* Set GV_INSUBGRAPH for milestone & descendents. */
5275 		milestone->gv_flags |= GV_INSUBGRAPH;
5276 
5277 		r = uu_list_walk(milestone->gv_dependencies,
5278 		    (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5279 		assert(r == 0);
5280 	}
5281 
5282 	/* Un-override services in the subgraph & override-disable the rest. */
5283 	if (norepository)
5284 		goto out;
5285 
5286 	non_subgraph_svcs = 0;
5287 	for (v = uu_list_first(dgraph);
5288 	    v != NULL;
5289 	    v = uu_list_next(dgraph, v)) {
5290 		if (v->gv_type != GVT_INST ||
5291 		    (v->gv_flags & GV_CONFIGURED) == 0)
5292 			continue;
5293 
5294 again:
5295 		r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5296 		    NULL, NULL, SCF_DECODE_FMRI_EXACT);
5297 		if (r != 0) {
5298 			switch (scf_error()) {
5299 			case SCF_ERROR_CONNECTION_BROKEN:
5300 			default:
5301 				libscf_handle_rebind(h);
5302 				rebound = B_TRUE;
5303 				goto again;
5304 
5305 			case SCF_ERROR_NOT_FOUND:
5306 				continue;
5307 
5308 			case SCF_ERROR_HANDLE_MISMATCH:
5309 			case SCF_ERROR_INVALID_ARGUMENT:
5310 			case SCF_ERROR_CONSTRAINT_VIOLATED:
5311 			case SCF_ERROR_NOT_BOUND:
5312 				bad_error("scf_handle_decode_fmri",
5313 				    scf_error());
5314 			}
5315 		}
5316 
5317 		if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5318 			r = libscf_delete_enable_ovr(inst);
5319 			fs = "libscf_delete_enable_ovr";
5320 		} else {
5321 			assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5322 
5323 			/*
5324 			 * Services which are up need to come down before
5325 			 * we're done, but we can only disable the leaves
5326 			 * here.
5327 			 */
5328 
5329 			if (up_state(v->gv_state))
5330 				++non_subgraph_svcs;
5331 
5332 			/* If it's already disabled, don't bother. */
5333 			if ((v->gv_flags & GV_ENABLED) == 0)
5334 				continue;
5335 
5336 			if (!is_nonsubgraph_leaf(v))
5337 				continue;
5338 
5339 			r = libscf_set_enable_ovr(inst, 0);
5340 			fs = "libscf_set_enable_ovr";
5341 		}
5342 		switch (r) {
5343 		case 0:
5344 		case ECANCELED:
5345 			break;
5346 
5347 		case ECONNABORTED:
5348 			libscf_handle_rebind(h);
5349 			rebound = B_TRUE;
5350 			goto again;
5351 
5352 		case EPERM:
5353 		case EROFS:
5354 			log_error(LOG_WARNING,
5355 			    "Could not set %s/%s for %s: %s.\n",
5356 			    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5357 			    v->gv_name, strerror(r));
5358 			break;
5359 
5360 		default:
5361 			bad_error(fs, r);
5362 		}
5363 	}
5364 
5365 	if (halting != -1) {
5366 		if (non_subgraph_svcs > 1)
5367 			uu_warn("%d system services are now being stopped.\n",
5368 			    non_subgraph_svcs);
5369 		else if (non_subgraph_svcs == 1)
5370 			uu_warn("One system service is now being stopped.\n");
5371 		else if (non_subgraph_svcs == 0)
5372 			do_uadmin();
5373 	}
5374 
5375 	ret = rebound ? ECONNRESET : 0;
5376 
5377 out:
5378 	MUTEX_UNLOCK(&dgraph_lock);
5379 	if (!isall && !isnone)
5380 		startd_free((void *)cfmri, max_scf_fmri_size);
5381 	scf_instance_destroy(inst);
5382 	return (ret);
5383 }
5384 
5385 
5386 /*
5387  * Returns 0, ECONNABORTED, or EINVAL.
5388  */
5389 static int
5390 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5391 {
5392 	int r;
5393 
5394 	switch (e->gpe_type) {
5395 	case GRAPH_UPDATE_RELOAD_GRAPH:
5396 		log_error(LOG_WARNING,
5397 		    "graph_event: reload graph unimplemented\n");
5398 		break;
5399 
5400 	case GRAPH_UPDATE_STATE_CHANGE: {
5401 		protocol_states_t *states = e->gpe_data;
5402 
5403 		switch (r = dgraph_set_instance_state(h, e->gpe_inst,
5404 		    states->ps_state, states->ps_err)) {
5405 		case 0:
5406 		case ENOENT:
5407 			break;
5408 
5409 		case ECONNABORTED:
5410 			return (ECONNABORTED);
5411 
5412 		case EINVAL:
5413 		default:
5414 #ifndef NDEBUG
5415 			(void) fprintf(stderr, "dgraph_set_instance_state() "
5416 			    "failed with unexpected error %d at %s:%d.\n", r,
5417 			    __FILE__, __LINE__);
5418 #endif
5419 			abort();
5420 		}
5421 
5422 		startd_free(states, sizeof (protocol_states_t));
5423 		break;
5424 	}
5425 
5426 	default:
5427 		log_error(LOG_WARNING,
5428 		    "graph_event_loop received an unknown event: %d\n",
5429 		    e->gpe_type);
5430 		break;
5431 	}
5432 
5433 	return (0);
5434 }
5435 
5436 /*
5437  * graph_event_thread()
5438  *    Wait for state changes from the restarters.
5439  */
5440 /*ARGSUSED*/
5441 void *
5442 graph_event_thread(void *unused)
5443 {
5444 	scf_handle_t *h;
5445 	int err;
5446 
5447 	h = libscf_handle_create_bound_loop();
5448 
5449 	/*CONSTCOND*/
5450 	while (1) {
5451 		graph_protocol_event_t *e;
5452 
5453 		MUTEX_LOCK(&gu->gu_lock);
5454 
5455 		while (gu->gu_wakeup == 0)
5456 			(void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5457 
5458 		gu->gu_wakeup = 0;
5459 
5460 		while ((e = graph_event_dequeue()) != NULL) {
5461 			MUTEX_LOCK(&e->gpe_lock);
5462 			MUTEX_UNLOCK(&gu->gu_lock);
5463 
5464 			while ((err = handle_graph_update_event(h, e)) ==
5465 			    ECONNABORTED)
5466 				libscf_handle_rebind(h);
5467 
5468 			if (err == 0)
5469 				graph_event_release(e);
5470 			else
5471 				graph_event_requeue(e);
5472 
5473 			MUTEX_LOCK(&gu->gu_lock);
5474 		}
5475 
5476 		MUTEX_UNLOCK(&gu->gu_lock);
5477 	}
5478 
5479 	/*
5480 	 * Unreachable for now -- there's currently no graceful cleanup
5481 	 * called on exit().
5482 	 */
5483 	MUTEX_UNLOCK(&gu->gu_lock);
5484 	scf_handle_destroy(h);
5485 	return (NULL);
5486 }
5487 
5488 static void
5489 set_initial_milestone(scf_handle_t *h)
5490 {
5491 	scf_instance_t *inst;
5492 	char *fmri, *cfmri;
5493 	size_t sz;
5494 	int r;
5495 
5496 	inst = safe_scf_instance_create(h);
5497 	fmri = startd_alloc(max_scf_fmri_size);
5498 
5499 	/*
5500 	 * If -m milestone= was specified, we want to set options_ovr/milestone
5501 	 * to it.  Otherwise we want to read what the milestone should be set
5502 	 * to.  Either way we need our inst.
5503 	 */
5504 get_self:
5505 	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5506 	    NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5507 		switch (scf_error()) {
5508 		case SCF_ERROR_CONNECTION_BROKEN:
5509 			libscf_handle_rebind(h);
5510 			goto get_self;
5511 
5512 		case SCF_ERROR_NOT_FOUND:
5513 			if (st->st_subgraph != NULL &&
5514 			    st->st_subgraph[0] != '\0') {
5515 				sz = strlcpy(fmri, st->st_subgraph,
5516 				    max_scf_fmri_size);
5517 				assert(sz < max_scf_fmri_size);
5518 			} else {
5519 				fmri[0] = '\0';
5520 			}
5521 			break;
5522 
5523 		case SCF_ERROR_INVALID_ARGUMENT:
5524 		case SCF_ERROR_CONSTRAINT_VIOLATED:
5525 		case SCF_ERROR_HANDLE_MISMATCH:
5526 		default:
5527 			bad_error("scf_handle_decode_fmri", scf_error());
5528 		}
5529 	} else {
5530 		if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5531 			scf_propertygroup_t *pg;
5532 
5533 			pg = safe_scf_pg_create(h);
5534 
5535 			sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5536 			assert(sz < max_scf_fmri_size);
5537 
5538 			r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5539 			    SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5540 			    pg);
5541 			switch (r) {
5542 			case 0:
5543 				break;
5544 
5545 			case ECONNABORTED:
5546 				libscf_handle_rebind(h);
5547 				goto get_self;
5548 
5549 			case EPERM:
5550 			case EACCES:
5551 			case EROFS:
5552 				log_error(LOG_WARNING, "Could not set %s/%s: "
5553 				    "%s.\n", SCF_PG_OPTIONS_OVR,
5554 				    SCF_PROPERTY_MILESTONE, strerror(r));
5555 				/* FALLTHROUGH */
5556 
5557 			case ECANCELED:
5558 				sz = strlcpy(fmri, st->st_subgraph,
5559 				    max_scf_fmri_size);
5560 				assert(sz < max_scf_fmri_size);
5561 				break;
5562 
5563 			default:
5564 				bad_error("libscf_inst_get_or_add_pg", r);
5565 			}
5566 
5567 			r = libscf_clear_runlevel(pg, fmri);
5568 			switch (r) {
5569 			case 0:
5570 				break;
5571 
5572 			case ECONNABORTED:
5573 				libscf_handle_rebind(h);
5574 				goto get_self;
5575 
5576 			case EPERM:
5577 			case EACCES:
5578 			case EROFS:
5579 				log_error(LOG_WARNING, "Could not set %s/%s: "
5580 				    "%s.\n", SCF_PG_OPTIONS_OVR,
5581 				    SCF_PROPERTY_MILESTONE, strerror(r));
5582 				/* FALLTHROUGH */
5583 
5584 			case ECANCELED:
5585 				sz = strlcpy(fmri, st->st_subgraph,
5586 				    max_scf_fmri_size);
5587 				assert(sz < max_scf_fmri_size);
5588 				break;
5589 
5590 			default:
5591 				bad_error("libscf_clear_runlevel", r);
5592 			}
5593 
5594 			scf_pg_destroy(pg);
5595 		} else {
5596 			scf_property_t *prop;
5597 			scf_value_t *val;
5598 
5599 			prop = safe_scf_property_create(h);
5600 			val = safe_scf_value_create(h);
5601 
5602 			r = libscf_get_milestone(inst, prop, val, fmri,
5603 			    max_scf_fmri_size);
5604 			switch (r) {
5605 			case 0:
5606 				break;
5607 
5608 			case ECONNABORTED:
5609 				libscf_handle_rebind(h);
5610 				goto get_self;
5611 
5612 			case EINVAL:
5613 				log_error(LOG_WARNING, "Milestone property is "
5614 				    "misconfigured.  Defaulting to \"all\".\n");
5615 				/* FALLTHROUGH */
5616 
5617 			case ECANCELED:
5618 			case ENOENT:
5619 				fmri[0] = '\0';
5620 				break;
5621 
5622 			default:
5623 				bad_error("libscf_get_milestone", r);
5624 			}
5625 
5626 			scf_value_destroy(val);
5627 			scf_property_destroy(prop);
5628 		}
5629 	}
5630 
5631 	if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5632 		goto out;
5633 
5634 	if (strcmp(fmri, "none") != 0) {
5635 retry:
5636 		if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5637 		    NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5638 			switch (scf_error()) {
5639 			case SCF_ERROR_INVALID_ARGUMENT:
5640 				log_error(LOG_WARNING,
5641 				    "Requested milestone \"%s\" is invalid.  "
5642 				    "Reverting to \"all\".\n", fmri);
5643 				goto out;
5644 
5645 			case SCF_ERROR_CONSTRAINT_VIOLATED:
5646 				log_error(LOG_WARNING, "Requested milestone "
5647 				    "\"%s\" does not specify an instance.  "
5648 				    "Reverting to \"all\".\n", fmri);
5649 				goto out;
5650 
5651 			case SCF_ERROR_CONNECTION_BROKEN:
5652 				libscf_handle_rebind(h);
5653 				goto retry;
5654 
5655 			case SCF_ERROR_NOT_FOUND:
5656 				log_error(LOG_WARNING, "Requested milestone "
5657 				    "\"%s\" not in repository.  Reverting to "
5658 				    "\"all\".\n", fmri);
5659 				goto out;
5660 
5661 			case SCF_ERROR_HANDLE_MISMATCH:
5662 			default:
5663 				bad_error("scf_handle_decode_fmri",
5664 				    scf_error());
5665 			}
5666 		}
5667 
5668 		r = fmri_canonify(fmri, &cfmri, B_FALSE);
5669 		assert(r == 0);
5670 
5671 		r = dgraph_add_instance(cfmri, inst, B_TRUE);
5672 		startd_free(cfmri, max_scf_fmri_size);
5673 		switch (r) {
5674 		case 0:
5675 			break;
5676 
5677 		case ECONNABORTED:
5678 			goto retry;
5679 
5680 		case EINVAL:
5681 			log_error(LOG_WARNING,
5682 			    "Requested milestone \"%s\" is invalid.  "
5683 			    "Reverting to \"all\".\n", fmri);
5684 			goto out;
5685 
5686 		case ECANCELED:
5687 			log_error(LOG_WARNING,
5688 			    "Requested milestone \"%s\" not "
5689 			    "in repository.  Reverting to \"all\".\n",
5690 			    fmri);
5691 			goto out;
5692 
5693 		case EEXIST:
5694 		default:
5695 			bad_error("dgraph_add_instance", r);
5696 		}
5697 	}
5698 
5699 	log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
5700 
5701 	r = dgraph_set_milestone(fmri, h, B_FALSE);
5702 	switch (r) {
5703 	case 0:
5704 	case ECONNRESET:
5705 	case EALREADY:
5706 		break;
5707 
5708 	case EINVAL:
5709 	case ENOENT:
5710 	default:
5711 		bad_error("dgraph_set_milestone", r);
5712 	}
5713 
5714 out:
5715 	startd_free(fmri, max_scf_fmri_size);
5716 	scf_instance_destroy(inst);
5717 }
5718 
5719 void
5720 set_restart_milestone(scf_handle_t *h)
5721 {
5722 	scf_instance_t *inst;
5723 	scf_property_t *prop;
5724 	scf_value_t *val;
5725 	char *fmri;
5726 	int r;
5727 
5728 	inst = safe_scf_instance_create(h);
5729 
5730 get_self:
5731 	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
5732 	    inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5733 		switch (scf_error()) {
5734 		case SCF_ERROR_CONNECTION_BROKEN:
5735 			libscf_handle_rebind(h);
5736 			goto get_self;
5737 
5738 		case SCF_ERROR_NOT_FOUND:
5739 			break;
5740 
5741 		case SCF_ERROR_INVALID_ARGUMENT:
5742 		case SCF_ERROR_CONSTRAINT_VIOLATED:
5743 		case SCF_ERROR_HANDLE_MISMATCH:
5744 		default:
5745 			bad_error("scf_handle_decode_fmri", scf_error());
5746 		}
5747 
5748 		scf_instance_destroy(inst);
5749 		return;
5750 	}
5751 
5752 	prop = safe_scf_property_create(h);
5753 	val = safe_scf_value_create(h);
5754 	fmri = startd_alloc(max_scf_fmri_size);
5755 
5756 	r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
5757 	switch (r) {
5758 	case 0:
5759 		break;
5760 
5761 	case ECONNABORTED:
5762 		libscf_handle_rebind(h);
5763 		goto get_self;
5764 
5765 	case ECANCELED:
5766 	case ENOENT:
5767 	case EINVAL:
5768 		goto out;
5769 
5770 	default:
5771 		bad_error("libscf_get_milestone", r);
5772 	}
5773 
5774 	r = dgraph_set_milestone(fmri, h, B_TRUE);
5775 	switch (r) {
5776 	case 0:
5777 	case ECONNRESET:
5778 	case EALREADY:
5779 	case EINVAL:
5780 	case ENOENT:
5781 		break;
5782 
5783 	default:
5784 		bad_error("dgraph_set_milestone", r);
5785 	}
5786 
5787 out:
5788 	startd_free(fmri, max_scf_fmri_size);
5789 	scf_value_destroy(val);
5790 	scf_property_destroy(prop);
5791 	scf_instance_destroy(inst);
5792 }
5793 
5794 /*
5795  * void *graph_thread(void *)
5796  *
5797  * Graph management thread.
5798  */
5799 /*ARGSUSED*/
5800 void *
5801 graph_thread(void *arg)
5802 {
5803 	scf_handle_t *h;
5804 	int err;
5805 
5806 	h = libscf_handle_create_bound_loop();
5807 
5808 	if (st->st_initial)
5809 		set_initial_milestone(h);
5810 
5811 	MUTEX_LOCK(&dgraph_lock);
5812 	initial_milestone_set = B_TRUE;
5813 	err = pthread_cond_broadcast(&initial_milestone_cv);
5814 	assert(err == 0);
5815 	MUTEX_UNLOCK(&dgraph_lock);
5816 
5817 	libscf_populate_graph(h);
5818 
5819 	if (!st->st_initial)
5820 		set_restart_milestone(h);
5821 
5822 	MUTEX_LOCK(&st->st_load_lock);
5823 	st->st_load_complete = 1;
5824 	(void) pthread_cond_broadcast(&st->st_load_cv);
5825 	MUTEX_UNLOCK(&st->st_load_lock);
5826 
5827 	MUTEX_LOCK(&dgraph_lock);
5828 	/*
5829 	 * Now that we've set st_load_complete we need to check can_come_up()
5830 	 * since if we booted to a milestone, then there won't be any more
5831 	 * state updates.
5832 	 */
5833 	if (!go_single_user_mode && !go_to_level1 &&
5834 	    halting == -1) {
5835 		if (!sulogin_thread_running && !can_come_up()) {
5836 			(void) startd_thread_create(sulogin_thread, NULL);
5837 			sulogin_thread_running = B_TRUE;
5838 		}
5839 	}
5840 	MUTEX_UNLOCK(&dgraph_lock);
5841 
5842 	(void) pthread_mutex_lock(&gu->gu_freeze_lock);
5843 
5844 	/*CONSTCOND*/
5845 	while (1) {
5846 		(void) pthread_cond_wait(&gu->gu_freeze_cv,
5847 		    &gu->gu_freeze_lock);
5848 	}
5849 
5850 	/*
5851 	 * Unreachable for now -- there's currently no graceful cleanup
5852 	 * called on exit().
5853 	 */
5854 	(void) pthread_mutex_unlock(&gu->gu_freeze_lock);
5855 	scf_handle_destroy(h);
5856 
5857 	return (NULL);
5858 }
5859 
5860 
5861 /*
5862  * int next_action()
5863  *   Given an array of timestamps 'a' with 'num' elements, find the
5864  *   lowest non-zero timestamp and return its index. If there are no
5865  *   non-zero elements, return -1.
5866  */
5867 static int
5868 next_action(hrtime_t *a, int num)
5869 {
5870 	hrtime_t t = 0;
5871 	int i = 0, smallest = -1;
5872 
5873 	for (i = 0; i < num; i++) {
5874 		if (t == 0) {
5875 			t = a[i];
5876 			smallest = i;
5877 		} else if (a[i] != 0 && a[i] < t) {
5878 			t = a[i];
5879 			smallest = i;
5880 		}
5881 	}
5882 
5883 	if (t == 0)
5884 		return (-1);
5885 	else
5886 		return (smallest);
5887 }
5888 
5889 /*
5890  * void process_actions()
5891  *   Process actions requested by the administrator. Possibilities include:
5892  *   refresh, restart, maintenance mode off, maintenance mode on,
5893  *   maintenance mode immediate, and degraded.
5894  *
5895  *   The set of pending actions is represented in the repository as a
5896  *   per-instance property group, with each action being a single property
5897  *   in that group.  This property group is converted to an array, with each
5898  *   action type having an array slot.  The actions in the array at the
5899  *   time process_actions() is called are acted on in the order of the
5900  *   timestamp (which is the value stored in the slot).  A value of zero
5901  *   indicates that there is no pending action of the type associated with
5902  *   a particular slot.
5903  *
5904  *   Sending an action event multiple times before the restarter has a
5905  *   chance to process that action will force it to be run at the last
5906  *   timestamp where it appears in the ordering.
5907  *
5908  *   Turning maintenance mode on trumps all other actions.
5909  *
5910  *   Returns 0 or ECONNABORTED.
5911  */
5912 static int
5913 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
5914 {
5915 	scf_property_t *prop = NULL;
5916 	scf_value_t *val = NULL;
5917 	scf_type_t type;
5918 	graph_vertex_t *vertex;
5919 	admin_action_t a;
5920 	int i, ret = 0, r;
5921 	hrtime_t action_ts[NACTIONS];
5922 	char *inst_name;
5923 
5924 	r = libscf_instance_get_fmri(inst, &inst_name);
5925 	switch (r) {
5926 	case 0:
5927 		break;
5928 
5929 	case ECONNABORTED:
5930 		return (ECONNABORTED);
5931 
5932 	case ECANCELED:
5933 		return (0);
5934 
5935 	default:
5936 		bad_error("libscf_instance_get_fmri", r);
5937 	}
5938 
5939 	MUTEX_LOCK(&dgraph_lock);
5940 
5941 	vertex = vertex_get_by_name(inst_name);
5942 	if (vertex == NULL) {
5943 		MUTEX_UNLOCK(&dgraph_lock);
5944 		startd_free(inst_name, max_scf_fmri_size);
5945 		log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
5946 		    "The instance must have been removed.\n", inst_name);
5947 		return (0);
5948 	}
5949 
5950 	prop = safe_scf_property_create(h);
5951 	val = safe_scf_value_create(h);
5952 
5953 	for (i = 0; i < NACTIONS; i++) {
5954 		if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
5955 			switch (scf_error()) {
5956 			case SCF_ERROR_CONNECTION_BROKEN:
5957 			default:
5958 				ret = ECONNABORTED;
5959 				goto out;
5960 
5961 			case SCF_ERROR_DELETED:
5962 				goto out;
5963 
5964 			case SCF_ERROR_NOT_FOUND:
5965 				action_ts[i] = 0;
5966 				continue;
5967 
5968 			case SCF_ERROR_HANDLE_MISMATCH:
5969 			case SCF_ERROR_INVALID_ARGUMENT:
5970 			case SCF_ERROR_NOT_SET:
5971 				bad_error("scf_pg_get_property", scf_error());
5972 			}
5973 		}
5974 
5975 		if (scf_property_type(prop, &type) != 0) {
5976 			switch (scf_error()) {
5977 			case SCF_ERROR_CONNECTION_BROKEN:
5978 			default:
5979 				ret = ECONNABORTED;
5980 				goto out;
5981 
5982 			case SCF_ERROR_DELETED:
5983 				action_ts[i] = 0;
5984 				continue;
5985 
5986 			case SCF_ERROR_NOT_SET:
5987 				bad_error("scf_property_type", scf_error());
5988 			}
5989 		}
5990 
5991 		if (type != SCF_TYPE_INTEGER) {
5992 			action_ts[i] = 0;
5993 			continue;
5994 		}
5995 
5996 		if (scf_property_get_value(prop, val) != 0) {
5997 			switch (scf_error()) {
5998 			case SCF_ERROR_CONNECTION_BROKEN:
5999 			default:
6000 				ret = ECONNABORTED;
6001 				goto out;
6002 
6003 			case SCF_ERROR_DELETED:
6004 				goto out;
6005 
6006 			case SCF_ERROR_NOT_FOUND:
6007 			case SCF_ERROR_CONSTRAINT_VIOLATED:
6008 				action_ts[i] = 0;
6009 				continue;
6010 
6011 			case SCF_ERROR_NOT_SET:
6012 			case SCF_ERROR_PERMISSION_DENIED:
6013 				bad_error("scf_property_get_value",
6014 				    scf_error());
6015 			}
6016 		}
6017 
6018 		r = scf_value_get_integer(val, &action_ts[i]);
6019 		assert(r == 0);
6020 	}
6021 
6022 	a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6023 	if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6024 	    action_ts[ADMIN_EVENT_MAINT_ON]) {
6025 		a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6026 		    ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6027 
6028 		vertex_send_event(vertex, admin_events[a]);
6029 		r = libscf_unset_action(h, pg, a, action_ts[a]);
6030 		switch (r) {
6031 		case 0:
6032 		case EACCES:
6033 			break;
6034 
6035 		case ECONNABORTED:
6036 			ret = ECONNABORTED;
6037 			goto out;
6038 
6039 		case EPERM:
6040 			uu_die("Insufficient privilege.\n");
6041 			/* NOTREACHED */
6042 
6043 		default:
6044 			bad_error("libscf_unset_action", r);
6045 		}
6046 	}
6047 
6048 	while ((a = next_action(action_ts, NACTIONS)) != -1) {
6049 		log_framework(LOG_DEBUG,
6050 		    "Graph: processing %s action for %s.\n", admin_actions[a],
6051 		    inst_name);
6052 
6053 		if (a == ADMIN_EVENT_REFRESH) {
6054 			r = dgraph_refresh_instance(vertex, inst);
6055 			switch (r) {
6056 			case 0:
6057 			case ECANCELED:
6058 			case EINVAL:
6059 			case -1:
6060 				break;
6061 
6062 			case ECONNABORTED:
6063 				/* pg & inst are reset now, so just return. */
6064 				ret = ECONNABORTED;
6065 				goto out;
6066 
6067 			default:
6068 				bad_error("dgraph_refresh_instance", r);
6069 			}
6070 		}
6071 
6072 		vertex_send_event(vertex, admin_events[a]);
6073 
6074 		r = libscf_unset_action(h, pg, a, action_ts[a]);
6075 		switch (r) {
6076 		case 0:
6077 		case EACCES:
6078 			break;
6079 
6080 		case ECONNABORTED:
6081 			ret = ECONNABORTED;
6082 			goto out;
6083 
6084 		case EPERM:
6085 			uu_die("Insufficient privilege.\n");
6086 			/* NOTREACHED */
6087 
6088 		default:
6089 			bad_error("libscf_unset_action", r);
6090 		}
6091 
6092 		action_ts[a] = 0;
6093 	}
6094 
6095 out:
6096 	MUTEX_UNLOCK(&dgraph_lock);
6097 
6098 	scf_property_destroy(prop);
6099 	scf_value_destroy(val);
6100 	startd_free(inst_name, max_scf_fmri_size);
6101 	return (ret);
6102 }
6103 
6104 /*
6105  * inst and pg_name are scratch space, and are unset on entry.
6106  * Returns
6107  *   0 - success
6108  *   ECONNRESET - success, but repository handle rebound
6109  *   ECONNABORTED - repository connection broken
6110  */
6111 static int
6112 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6113     char *pg_name)
6114 {
6115 	int r;
6116 	scf_property_t *prop;
6117 	scf_value_t *val;
6118 	char *fmri;
6119 	boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6120 
6121 	if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6122 		switch (scf_error()) {
6123 		case SCF_ERROR_CONNECTION_BROKEN:
6124 		default:
6125 			return (ECONNABORTED);
6126 
6127 		case SCF_ERROR_DELETED:
6128 			return (0);
6129 
6130 		case SCF_ERROR_NOT_SET:
6131 			bad_error("scf_pg_get_name", scf_error());
6132 		}
6133 	}
6134 
6135 	if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6136 	    strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6137 		r = dgraph_update_general(pg);
6138 		switch (r) {
6139 		case 0:
6140 		case ENOTSUP:
6141 		case ECANCELED:
6142 			return (0);
6143 
6144 		case ECONNABORTED:
6145 			return (ECONNABORTED);
6146 
6147 		case -1:
6148 			/* Error should have been logged. */
6149 			return (0);
6150 
6151 		default:
6152 			bad_error("dgraph_update_general", r);
6153 		}
6154 	} else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6155 		if (scf_pg_get_parent_instance(pg, inst) != 0) {
6156 			switch (scf_error()) {
6157 			case SCF_ERROR_CONNECTION_BROKEN:
6158 				return (ECONNABORTED);
6159 
6160 			case SCF_ERROR_DELETED:
6161 			case SCF_ERROR_CONSTRAINT_VIOLATED:
6162 				/* Ignore commands on services. */
6163 				return (0);
6164 
6165 			case SCF_ERROR_NOT_BOUND:
6166 			case SCF_ERROR_HANDLE_MISMATCH:
6167 			case SCF_ERROR_NOT_SET:
6168 			default:
6169 				bad_error("scf_pg_get_parent_instance",
6170 				    scf_error());
6171 			}
6172 		}
6173 
6174 		return (process_actions(h, pg, inst));
6175 	}
6176 
6177 	if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6178 	    strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6179 		return (0);
6180 
6181 	/*
6182 	 * We only care about the options[_ovr] property groups of our own
6183 	 * instance, so get the fmri and compare.  Plus, once we know it's
6184 	 * correct, if the repository connection is broken we know exactly what
6185 	 * property group we were operating on, and can look it up again.
6186 	 */
6187 	if (scf_pg_get_parent_instance(pg, inst) != 0) {
6188 		switch (scf_error()) {
6189 		case SCF_ERROR_CONNECTION_BROKEN:
6190 			return (ECONNABORTED);
6191 
6192 		case SCF_ERROR_DELETED:
6193 		case SCF_ERROR_CONSTRAINT_VIOLATED:
6194 			return (0);
6195 
6196 		case SCF_ERROR_HANDLE_MISMATCH:
6197 		case SCF_ERROR_NOT_BOUND:
6198 		case SCF_ERROR_NOT_SET:
6199 		default:
6200 			bad_error("scf_pg_get_parent_instance",
6201 			    scf_error());
6202 		}
6203 	}
6204 
6205 	switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6206 	case 0:
6207 		break;
6208 
6209 	case ECONNABORTED:
6210 		return (ECONNABORTED);
6211 
6212 	case ECANCELED:
6213 		return (0);
6214 
6215 	default:
6216 		bad_error("libscf_instance_get_fmri", r);
6217 	}
6218 
6219 	if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6220 		startd_free(fmri, max_scf_fmri_size);
6221 		return (0);
6222 	}
6223 
6224 	prop = safe_scf_property_create(h);
6225 	val = safe_scf_value_create(h);
6226 
6227 	if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6228 		/* See if we need to set the runlevel. */
6229 		/* CONSTCOND */
6230 		if (0) {
6231 rebind_pg:
6232 			libscf_handle_rebind(h);
6233 			rebound = B_TRUE;
6234 
6235 			r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6236 			switch (r) {
6237 			case 0:
6238 				break;
6239 
6240 			case ECONNABORTED:
6241 				goto rebind_pg;
6242 
6243 			case ENOENT:
6244 				goto out;
6245 
6246 			case EINVAL:
6247 			case ENOTSUP:
6248 				bad_error("libscf_lookup_instance", r);
6249 			}
6250 
6251 			if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6252 				switch (scf_error()) {
6253 				case SCF_ERROR_DELETED:
6254 				case SCF_ERROR_NOT_FOUND:
6255 					goto out;
6256 
6257 				case SCF_ERROR_CONNECTION_BROKEN:
6258 					goto rebind_pg;
6259 
6260 				case SCF_ERROR_HANDLE_MISMATCH:
6261 				case SCF_ERROR_NOT_BOUND:
6262 				case SCF_ERROR_NOT_SET:
6263 				case SCF_ERROR_INVALID_ARGUMENT:
6264 				default:
6265 					bad_error("scf_instance_get_pg",
6266 					    scf_error());
6267 				}
6268 			}
6269 		}
6270 
6271 		if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6272 			r = dgraph_set_runlevel(pg, prop);
6273 			switch (r) {
6274 			case ECONNRESET:
6275 				rebound = B_TRUE;
6276 				rebind_inst = B_TRUE;
6277 				/* FALLTHROUGH */
6278 
6279 			case 0:
6280 				break;
6281 
6282 			case ECONNABORTED:
6283 				goto rebind_pg;
6284 
6285 			case ECANCELED:
6286 				goto out;
6287 
6288 			default:
6289 				bad_error("dgraph_set_runlevel", r);
6290 			}
6291 		} else {
6292 			switch (scf_error()) {
6293 			case SCF_ERROR_CONNECTION_BROKEN:
6294 			default:
6295 				goto rebind_pg;
6296 
6297 			case SCF_ERROR_DELETED:
6298 				goto out;
6299 
6300 			case SCF_ERROR_NOT_FOUND:
6301 				break;
6302 
6303 			case SCF_ERROR_INVALID_ARGUMENT:
6304 			case SCF_ERROR_HANDLE_MISMATCH:
6305 			case SCF_ERROR_NOT_BOUND:
6306 			case SCF_ERROR_NOT_SET:
6307 				bad_error("scf_pg_get_property", scf_error());
6308 			}
6309 		}
6310 	}
6311 
6312 	if (rebind_inst) {
6313 lookup_inst:
6314 		r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6315 		switch (r) {
6316 		case 0:
6317 			break;
6318 
6319 		case ECONNABORTED:
6320 			libscf_handle_rebind(h);
6321 			rebound = B_TRUE;
6322 			goto lookup_inst;
6323 
6324 		case ENOENT:
6325 			goto out;
6326 
6327 		case EINVAL:
6328 		case ENOTSUP:
6329 			bad_error("libscf_lookup_instance", r);
6330 		}
6331 	}
6332 
6333 	r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6334 	switch (r) {
6335 	case 0:
6336 		break;
6337 
6338 	case ECONNABORTED:
6339 		libscf_handle_rebind(h);
6340 		rebound = B_TRUE;
6341 		goto lookup_inst;
6342 
6343 	case EINVAL:
6344 		log_error(LOG_NOTICE,
6345 		    "%s/%s property of %s is misconfigured.\n", pg_name,
6346 		    SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6347 		/* FALLTHROUGH */
6348 
6349 	case ECANCELED:
6350 	case ENOENT:
6351 		(void) strcpy(fmri, "all");
6352 		break;
6353 
6354 	default:
6355 		bad_error("libscf_get_milestone", r);
6356 	}
6357 
6358 	r = dgraph_set_milestone(fmri, h, B_FALSE);
6359 	switch (r) {
6360 	case 0:
6361 	case ECONNRESET:
6362 	case EALREADY:
6363 		break;
6364 
6365 	case EINVAL:
6366 		log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6367 		break;
6368 
6369 	case ENOENT:
6370 		log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6371 		break;
6372 
6373 	default:
6374 		bad_error("dgraph_set_milestone", r);
6375 	}
6376 
6377 out:
6378 	startd_free(fmri, max_scf_fmri_size);
6379 	scf_value_destroy(val);
6380 	scf_property_destroy(prop);
6381 
6382 	return (rebound ? ECONNRESET : 0);
6383 }
6384 
6385 /*
6386  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6387  * instance fmri or if 'fmri' matches the 'general' property group of an
6388  * instance (or the 'general/enabled' property).
6389  *
6390  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6391  */
6392 static void
6393 process_delete(char *fmri, scf_handle_t *h)
6394 {
6395 	char *lfmri, *end_inst_fmri;
6396 	const char *inst_name = NULL;
6397 	const char *pg_name = NULL;
6398 	const char *prop_name = NULL;
6399 
6400 	lfmri = safe_strdup(fmri);
6401 
6402 	/* Determine if the FMRI is a property group or instance */
6403 	if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6404 	    &prop_name) != SCF_SUCCESS) {
6405 		log_error(LOG_WARNING,
6406 		    "Received invalid FMRI \"%s\" from repository server.\n",
6407 		    fmri);
6408 	} else if (inst_name != NULL && pg_name == NULL) {
6409 		(void) dgraph_remove_instance(fmri, h);
6410 	} else if (inst_name != NULL && pg_name != NULL) {
6411 		/*
6412 		 * If we're deleting the 'general' property group or
6413 		 * 'general/enabled' property then the whole instance
6414 		 * must be removed from the dgraph.
6415 		 */
6416 		if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6417 			free(lfmri);
6418 			return;
6419 		}
6420 
6421 		if (prop_name != NULL &&
6422 		    strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6423 			free(lfmri);
6424 			return;
6425 		}
6426 
6427 		/*
6428 		 * Because the instance has already been deleted from the
6429 		 * repository, we cannot use any scf_ functions to retrieve
6430 		 * the instance FMRI however we can easily reconstruct it
6431 		 * manually.
6432 		 */
6433 		end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6434 		if (end_inst_fmri == NULL)
6435 			bad_error("process_delete", 0);
6436 
6437 		end_inst_fmri[0] = '\0';
6438 
6439 		(void) dgraph_remove_instance(fmri, h);
6440 	}
6441 
6442 	free(lfmri);
6443 }
6444 
6445 /*ARGSUSED*/
6446 void *
6447 repository_event_thread(void *unused)
6448 {
6449 	scf_handle_t *h;
6450 	scf_propertygroup_t *pg;
6451 	scf_instance_t *inst;
6452 	char *fmri = startd_alloc(max_scf_fmri_size);
6453 	char *pg_name = startd_alloc(max_scf_value_size);
6454 	int r;
6455 
6456 	h = libscf_handle_create_bound_loop();
6457 
6458 	pg = safe_scf_pg_create(h);
6459 	inst = safe_scf_instance_create(h);
6460 
6461 retry:
6462 	if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6463 		if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6464 			libscf_handle_rebind(h);
6465 		} else {
6466 			log_error(LOG_WARNING,
6467 			    "Couldn't set up repository notification "
6468 			    "for property group type %s: %s\n",
6469 			    SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6470 
6471 			(void) sleep(1);
6472 		}
6473 
6474 		goto retry;
6475 	}
6476 
6477 	/*CONSTCOND*/
6478 	while (1) {
6479 		ssize_t res;
6480 
6481 		/* Note: fmri is only set on delete events. */
6482 		res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6483 		if (res < 0) {
6484 			libscf_handle_rebind(h);
6485 			goto retry;
6486 		} else if (res == 0) {
6487 			/*
6488 			 * property group modified.  inst and pg_name are
6489 			 * pre-allocated scratch space.
6490 			 */
6491 			if (scf_pg_update(pg) < 0) {
6492 				switch (scf_error()) {
6493 				case SCF_ERROR_DELETED:
6494 					continue;
6495 
6496 				case SCF_ERROR_CONNECTION_BROKEN:
6497 					log_error(LOG_WARNING,
6498 					    "Lost repository event due to "
6499 					    "disconnection.\n");
6500 					libscf_handle_rebind(h);
6501 					goto retry;
6502 
6503 				case SCF_ERROR_NOT_BOUND:
6504 				case SCF_ERROR_NOT_SET:
6505 				default:
6506 					bad_error("scf_pg_update", scf_error());
6507 				}
6508 			}
6509 
6510 			r = process_pg_event(h, pg, inst, pg_name);
6511 			switch (r) {
6512 			case 0:
6513 				break;
6514 
6515 			case ECONNABORTED:
6516 				log_error(LOG_WARNING, "Lost repository event "
6517 				    "due to disconnection.\n");
6518 				libscf_handle_rebind(h);
6519 				/* FALLTHROUGH */
6520 
6521 			case ECONNRESET:
6522 				goto retry;
6523 
6524 			default:
6525 				bad_error("process_pg_event", r);
6526 			}
6527 		} else {
6528 			/*
6529 			 * Service, instance, or pg deleted.
6530 			 * Don't trust fmri on return.
6531 			 */
6532 			process_delete(fmri, h);
6533 		}
6534 	}
6535 
6536 	/*NOTREACHED*/
6537 	return (NULL);
6538 }
6539 
6540 void
6541 graph_engine_start()
6542 {
6543 	int err;
6544 
6545 	(void) startd_thread_create(graph_thread, NULL);
6546 
6547 	MUTEX_LOCK(&dgraph_lock);
6548 	while (!initial_milestone_set) {
6549 		err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6550 		assert(err == 0);
6551 	}
6552 	MUTEX_UNLOCK(&dgraph_lock);
6553 
6554 	(void) startd_thread_create(repository_event_thread, NULL);
6555 	(void) startd_thread_create(graph_event_thread, NULL);
6556 }
6557