xref: /illumos-gate/usr/src/cmd/svc/startd/graph.c (revision dd72704bd9e794056c558153663c739e2012d721)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2019 Joyent, Inc.
25  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
26  * Copyright 2016 Toomas Soome <tsoome@me.com>
27  * Copyright 2016 RackTop Systems.
28  */
29 
30 /*
31  * graph.c - master restarter graph engine
32  *
33  *   The graph engine keeps a dependency graph of all service instances on the
34  *   system, as recorded in the repository.  It decides when services should
35  *   be brought up or down based on service states and dependencies and sends
36  *   commands to restarters to effect any changes.  It also executes
37  *   administrator commands sent by svcadm via the repository.
38  *
39  *   The graph is stored in uu_list_t *dgraph and its vertices are
40  *   graph_vertex_t's, each of which has a name and an integer id unique to
41  *   its name (see dict.c).  A vertex's type attribute designates the type
42  *   of object it represents: GVT_INST for service instances, GVT_SVC for
43  *   service objects (since service instances may depend on another service,
44  *   rather than service instance), GVT_FILE for files (which services may
45  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
46  *   vertices are necessary because dependency lists may have particular
47  *   grouping types (require any, require all, optional, or exclude) and
48  *   event-propagation characteristics.
49  *
50  *   The initial graph is built by libscf_populate_graph() invoking
51  *   dgraph_add_instance() for each instance in the repository.  The function
52  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
53  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
54  *   The resulting web of vertices & edges associated with an instance's vertex
55  *   includes
56  *
57  *     - an edge from the GVT_SVC vertex for the instance's service
58  *
59  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
60  *       restarter is not svc.startd
61  *
62  *     - edges from other GVT_INST vertices if the instance is a restarter
63  *
64  *     - for each dependency property group in the instance's "running"
65  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
66  *       instance and the name of the property group
67  *
68  *     - for each value of the "entities" property in each dependency property
69  *       group, an edge from the corresponding GVT_GROUP vertex to a
70  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
71  *
72  *     - edges from GVT_GROUP vertices for each dependent instance
73  *
74  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
75  *   there are problems, or if a service is mentioned in a dependency but does
76  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
77  *
78  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
79  *   See restarter.c for more information.
80  *
81  *   The properties of an instance fall into two classes: immediate and
82  *   snapshotted.  Immediate properties should have an immediate effect when
83  *   changed.  Snapshotted properties should be read from a snapshot, so they
84  *   only change when the snapshot changes.  The immediate properties used by
85  *   the graph engine are general/enabled, general/restarter, and the properties
86  *   in the restarter_actions property group.  Since they are immediate, they
87  *   are not read out of a snapshot.  The snapshotted properties used by the
88  *   graph engine are those in the property groups with type "dependency" and
89  *   are read out of the "running" snapshot.  The "running" snapshot is created
90  *   by the the graph engine as soon as possible, and it is updated, along with
91  *   in-core copies of the data (dependency information for the graph engine) on
92  *   receipt of the refresh command from svcadm.  In addition, the graph engine
93  *   updates the "start" snapshot from the "running" snapshot whenever a service
94  *   comes online.
95  *
96  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
97  *   the dependents first before shutting down the requested service.
98  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
99  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
100  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
101  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
102  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
103  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
104  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
105  *   exposed leaves. We do the same until we reach the last leaf (the one with
106  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
107  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
108  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
109  *   continue to propagate the offline event to the vertex's dependencies.
110  *
111  *
112  * SMF state transition notifications
113  *
114  *   When an instance of a service managed by SMF changes state, svc.startd may
115  *   publish a GPEC sysevent. All transitions to or from maintenance, a
116  *   transition cause by a hardware error will generate an event.
117  *   Other transitions will generate an event if there exist notification
118  *   parameter for that transition. Notification parameters are stored in the
119  *   SMF repository for the service/instance they refer to. System-wide
120  *   notification parameters are stored in the global instance.
121  *   svc.startd can be told to send events for all SMF state transitions despite
122  *   of notification parameters by setting options/info_events_all to true in
123  *   restarter:default
124  *
125  *   The set of transitions that generate events is cached in the
126  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
127  *   stn_global for the system-wide set. They are re-read when instances are
128  *   refreshed.
129  *
130  *   The GPEC events published by svc.startd are consumed by fmd(8). After
131  *   processing these events, fmd(8) publishes the processed events to
132  *   notification agents. The notification agents read the notification
133  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
134  *   the notification, or not, based on those parameters.
135  *
136  *   Subscription and publishing to the GPEC channels is done with the
137  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
138  *   fmev_shdl_(un)subscribe().
139  *
140  */
141 
142 #include <sys/uadmin.h>
143 #include <sys/wait.h>
144 
145 #include <assert.h>
146 #include <errno.h>
147 #include <fcntl.h>
148 #include <fm/libfmevent.h>
149 #include <libscf.h>
150 #include <libscf_priv.h>
151 #include <librestart.h>
152 #include <libuutil.h>
153 #include <locale.h>
154 #include <poll.h>
155 #include <pthread.h>
156 #include <signal.h>
157 #include <stddef.h>
158 #include <stdio.h>
159 #include <stdlib.h>
160 #include <string.h>
161 #include <strings.h>
162 #include <sys/statvfs.h>
163 #include <sys/uadmin.h>
164 #include <zone.h>
165 #if defined(__x86)
166 #include <libbe.h>
167 #endif	/* __x86 */
168 
169 #include "startd.h"
170 #include "protocol.h"
171 
172 
173 #define	MILESTONE_NONE	((graph_vertex_t *)1)
174 
175 #define	CONSOLE_LOGIN_FMRI	"svc:/system/console-login:default"
176 #define	FS_MINIMAL_FMRI		"svc:/system/filesystem/minimal:default"
177 
178 #define	VERTEX_REMOVED	0	/* vertex has been freed  */
179 #define	VERTEX_INUSE	1	/* vertex is still in use */
180 
181 #define	IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
182 
183 /*
184  * stn_global holds the tset for the system wide notification parameters.
185  * It is updated on refresh of svc:/system/svc/global:default
186  *
187  * There are two assumptions that relax the need for a mutex:
188  *     1. 32-bit value assignments are atomic
189  *     2. Its value is consumed only in one point at
190  *     dgraph_state_transition_notify(). There are no test and set races.
191  *
192  *     If either assumption is broken, we'll need a mutex to synchronize
193  *     access to stn_global
194  */
195 int32_t stn_global;
196 /*
197  * info_events_all holds a flag to override notification parameters and send
198  * Information events for all state transitions.
199  * same about the need of a mutex here.
200  */
201 int info_events_all;
202 
203 /*
204  * Services in these states are not considered 'down' by the
205  * milestone/shutdown code.
206  */
207 #define	up_state(state)	((state) == RESTARTER_STATE_ONLINE || \
208 	(state) == RESTARTER_STATE_DEGRADED || \
209 	(state) == RESTARTER_STATE_OFFLINE)
210 
211 #define	is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
212 	((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
213 	(v->gv_restart < RERR_RESTART)))
214 
215 #define	is_inst_bypassed(v) ((v->gv_type == GVT_INST) && \
216 	((v->gv_flags & GV_TODISABLE) || \
217 	(v->gv_flags & GV_TOOFFLINE)))
218 
219 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
220 static uu_list_t *dgraph;
221 static pthread_mutex_t dgraph_lock;
222 
223 /*
224  * milestone indicates the current subgraph.  When NULL, it is the entire
225  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
226  * services on which the target vertex depends.
227  */
228 static graph_vertex_t *milestone = NULL;
229 static boolean_t initial_milestone_set = B_FALSE;
230 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
231 
232 /* protected by dgraph_lock */
233 static boolean_t sulogin_thread_running = B_FALSE;
234 static boolean_t sulogin_running = B_FALSE;
235 static boolean_t console_login_ready = B_FALSE;
236 
237 /* Number of services to come down to complete milestone transition. */
238 static uint_t non_subgraph_svcs;
239 
240 /*
241  * These variables indicate what should be done when we reach the milestone
242  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
243  * dgraph_set_instance_state().
244  */
245 static int halting = -1;
246 static boolean_t go_single_user_mode = B_FALSE;
247 static boolean_t go_to_level1 = B_FALSE;
248 
249 /*
250  * Tracks when we started halting.
251  */
252 static time_t halting_time = 0;
253 
254 /*
255  * This tracks the legacy runlevel to ensure we signal init and manage
256  * utmpx entries correctly.
257  */
258 static char current_runlevel = '\0';
259 
260 /* Number of single user threads currently running */
261 static pthread_mutex_t single_user_thread_lock;
262 static int single_user_thread_count = 0;
263 
264 /* Statistics for dependency cycle-checking */
265 static u_longlong_t dep_inserts = 0;
266 static u_longlong_t dep_cycle_ns = 0;
267 static u_longlong_t dep_insert_ns = 0;
268 
269 
270 static const char * const emsg_invalid_restarter =
271 	"Transitioning %s to maintenance, restarter FMRI %s is invalid "
272 	"(see 'svcs -xv' for details).\n";
273 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
274 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
275 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
276 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
277 
278 
279 /*
280  * These services define the system being "up".  If none of them can come
281  * online, then we will run sulogin on the console.  Note that the install ones
282  * are for the miniroot and when installing CDs after the first.  can_come_up()
283  * does the decision making, and an sulogin_thread() runs sulogin, which can be
284  * started by dgraph_set_instance_state() or single_user_thread().
285  *
286  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
287  * entry, which is only used when booting_to_single_user (boot -s) is set.
288  * This is because when doing a "boot -s", sulogin is started from specials.c
289  * after milestone/single-user comes online, for backwards compatibility.
290  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
291  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
292  */
293 static const char * const up_svcs[] = {
294 	SCF_MILESTONE_SINGLE_USER,
295 	CONSOLE_LOGIN_FMRI,
296 	"svc:/system/install-setup:default",
297 	"svc:/system/install:default",
298 	NULL
299 };
300 
301 /* This array must have an element for each non-NULL element of up_svcs[]. */
302 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
303 
304 /* These are for seed repository magic.  See can_come_up(). */
305 static const char * const manifest_import = SCF_INSTANCE_MI;
306 static graph_vertex_t *manifest_import_p = NULL;
307 
308 
309 static char target_milestone_as_runlevel(void);
310 static void graph_runlevel_changed(char rl, int online);
311 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
312 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
313 static int mark_subtree(graph_edge_t *, void *);
314 static boolean_t insubtree_dependents_down(graph_vertex_t *);
315 
316 /*
317  * graph_vertex_compare()
318  *	This function can compare either int *id or * graph_vertex_t *gv
319  *	values, as the vertex id is always the first element of a
320  *	graph_vertex structure.
321  */
322 /* ARGSUSED */
323 static int
324 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
325 {
326 	int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
327 	int rc_id = *(int *)rc_arg;
328 
329 	if (lc_id > rc_id)
330 		return (1);
331 	if (lc_id < rc_id)
332 		return (-1);
333 	return (0);
334 }
335 
336 void
337 graph_init()
338 {
339 	graph_edge_pool = startd_list_pool_create("graph_edges",
340 	    sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
341 	    UU_LIST_POOL_DEBUG);
342 	assert(graph_edge_pool != NULL);
343 
344 	graph_vertex_pool = startd_list_pool_create("graph_vertices",
345 	    sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
346 	    graph_vertex_compare, UU_LIST_POOL_DEBUG);
347 	assert(graph_vertex_pool != NULL);
348 
349 	(void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
350 	(void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
351 	dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
352 	assert(dgraph != NULL);
353 
354 	if (!st->st_initial)
355 		current_runlevel = utmpx_get_runlevel();
356 
357 	log_framework(LOG_DEBUG, "Initialized graph\n");
358 }
359 
360 static graph_vertex_t *
361 vertex_get_by_name(const char *name)
362 {
363 	int id;
364 
365 	assert(MUTEX_HELD(&dgraph_lock));
366 
367 	id = dict_lookup_byname(name);
368 	if (id == -1)
369 		return (NULL);
370 
371 	return (uu_list_find(dgraph, &id, NULL, NULL));
372 }
373 
374 static graph_vertex_t *
375 vertex_get_by_id(int id)
376 {
377 	assert(MUTEX_HELD(&dgraph_lock));
378 
379 	if (id == -1)
380 		return (NULL);
381 
382 	return (uu_list_find(dgraph, &id, NULL, NULL));
383 }
384 
385 /*
386  * Creates a new vertex with the given name, adds it to the graph, and returns
387  * a pointer to it.  The graph lock must be held by this thread on entry.
388  */
389 static graph_vertex_t *
390 graph_add_vertex(const char *name)
391 {
392 	int id;
393 	graph_vertex_t *v;
394 	void *p;
395 	uu_list_index_t idx;
396 
397 	assert(MUTEX_HELD(&dgraph_lock));
398 
399 	id = dict_insert(name);
400 
401 	v = startd_zalloc(sizeof (*v));
402 
403 	v->gv_id = id;
404 
405 	v->gv_name = startd_alloc(strlen(name) + 1);
406 	(void) strcpy(v->gv_name, name);
407 
408 	v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
409 	v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
410 
411 	p = uu_list_find(dgraph, &id, NULL, &idx);
412 	assert(p == NULL);
413 
414 	uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
415 	uu_list_insert(dgraph, v, idx);
416 
417 	return (v);
418 }
419 
420 /*
421  * Removes v from the graph and frees it.  The graph should be locked by this
422  * thread, and v should have no edges associated with it.
423  */
424 static void
425 graph_remove_vertex(graph_vertex_t *v)
426 {
427 	assert(MUTEX_HELD(&dgraph_lock));
428 
429 	assert(uu_list_numnodes(v->gv_dependencies) == 0);
430 	assert(uu_list_numnodes(v->gv_dependents) == 0);
431 	assert(v->gv_refs == 0);
432 
433 	startd_free(v->gv_name, strlen(v->gv_name) + 1);
434 	uu_list_destroy(v->gv_dependencies);
435 	uu_list_destroy(v->gv_dependents);
436 	uu_list_remove(dgraph, v);
437 
438 	startd_free(v, sizeof (graph_vertex_t));
439 }
440 
441 static void
442 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
443 {
444 	graph_edge_t *e, *re;
445 	int r;
446 
447 	assert(MUTEX_HELD(&dgraph_lock));
448 
449 	e = startd_alloc(sizeof (graph_edge_t));
450 	re = startd_alloc(sizeof (graph_edge_t));
451 
452 	e->ge_parent = fv;
453 	e->ge_vertex = tv;
454 
455 	re->ge_parent = tv;
456 	re->ge_vertex = fv;
457 
458 	uu_list_node_init(e, &e->ge_link, graph_edge_pool);
459 	r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
460 	assert(r == 0);
461 
462 	uu_list_node_init(re, &re->ge_link, graph_edge_pool);
463 	r = uu_list_insert_before(tv->gv_dependents, NULL, re);
464 	assert(r == 0);
465 }
466 
467 static void
468 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
469 {
470 	graph_edge_t *e;
471 
472 	for (e = uu_list_first(v->gv_dependencies);
473 	    e != NULL;
474 	    e = uu_list_next(v->gv_dependencies, e)) {
475 		if (e->ge_vertex == dv) {
476 			uu_list_remove(v->gv_dependencies, e);
477 			startd_free(e, sizeof (graph_edge_t));
478 			break;
479 		}
480 	}
481 
482 	for (e = uu_list_first(dv->gv_dependents);
483 	    e != NULL;
484 	    e = uu_list_next(dv->gv_dependents, e)) {
485 		if (e->ge_vertex == v) {
486 			uu_list_remove(dv->gv_dependents, e);
487 			startd_free(e, sizeof (graph_edge_t));
488 			break;
489 		}
490 	}
491 }
492 
493 static void
494 remove_inst_vertex(graph_vertex_t *v)
495 {
496 	graph_edge_t *e;
497 	graph_vertex_t *sv;
498 	int i;
499 
500 	assert(MUTEX_HELD(&dgraph_lock));
501 	assert(uu_list_numnodes(v->gv_dependents) == 1);
502 	assert(uu_list_numnodes(v->gv_dependencies) == 0);
503 	assert(v->gv_refs == 0);
504 	assert((v->gv_flags & GV_CONFIGURED) == 0);
505 
506 	e = uu_list_first(v->gv_dependents);
507 	sv = e->ge_vertex;
508 	graph_remove_edge(sv, v);
509 
510 	for (i = 0; up_svcs[i] != NULL; ++i) {
511 		if (up_svcs_p[i] == v)
512 			up_svcs_p[i] = NULL;
513 	}
514 
515 	if (manifest_import_p == v)
516 		manifest_import_p = NULL;
517 
518 	graph_remove_vertex(v);
519 
520 	if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
521 	    uu_list_numnodes(sv->gv_dependents) == 0 &&
522 	    sv->gv_refs == 0)
523 		graph_remove_vertex(sv);
524 }
525 
526 static void
527 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
528     void *arg)
529 {
530 	graph_edge_t *e;
531 
532 	for (e = uu_list_first(v->gv_dependents);
533 	    e != NULL;
534 	    e = uu_list_next(v->gv_dependents, e))
535 		func(e->ge_vertex, arg);
536 }
537 
538 static void
539 graph_walk_dependencies(graph_vertex_t *v,
540     void (*func)(graph_vertex_t *, void *), void *arg)
541 {
542 	graph_edge_t *e;
543 
544 	assert(MUTEX_HELD(&dgraph_lock));
545 
546 	for (e = uu_list_first(v->gv_dependencies);
547 	    e != NULL;
548 	    e = uu_list_next(v->gv_dependencies, e)) {
549 
550 		func(e->ge_vertex, arg);
551 	}
552 }
553 
554 /*
555  * Generic graph walking function.
556  *
557  * Given a vertex, this function will walk either dependencies
558  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
559  * for the entire graph.  It will avoid cycles and never visit the same vertex
560  * twice.
561  *
562  * We avoid traversing exclusion dependencies, because they are allowed to
563  * create cycles in the graph.  When propagating satisfiability, there is no
564  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
565  * test for satisfiability.
566  *
567  * The walker takes two callbacks.  The first is called before examining the
568  * dependents of each vertex.  The second is called on each vertex after
569  * examining its dependents.  This allows is_path_to() to construct a path only
570  * after the target vertex has been found.
571  */
572 typedef enum {
573 	WALK_DEPENDENTS,
574 	WALK_DEPENDENCIES
575 } graph_walk_dir_t;
576 
577 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
578 
579 typedef struct graph_walk_info {
580 	graph_walk_dir_t	gi_dir;
581 	uchar_t			*gi_visited;	/* vertex bitmap */
582 	int			(*gi_pre)(graph_vertex_t *, void *);
583 	void			(*gi_post)(graph_vertex_t *, void *);
584 	void			*gi_arg;	/* callback arg */
585 	int			gi_ret;		/* return value */
586 } graph_walk_info_t;
587 
588 static int
589 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
590 {
591 	uu_list_t *list;
592 	int r;
593 	graph_vertex_t *v = e->ge_vertex;
594 	int i;
595 	uint_t b;
596 
597 	i = v->gv_id / 8;
598 	b = 1 << (v->gv_id % 8);
599 
600 	/*
601 	 * Check to see if we've visited this vertex already.
602 	 */
603 	if (gip->gi_visited[i] & b)
604 		return (UU_WALK_NEXT);
605 
606 	gip->gi_visited[i] |= b;
607 
608 	/*
609 	 * Don't follow exclusions.
610 	 */
611 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
612 		return (UU_WALK_NEXT);
613 
614 	/*
615 	 * Call pre-visit callback.  If this doesn't terminate the walk,
616 	 * continue search.
617 	 */
618 	if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
619 		/*
620 		 * Recurse using appropriate list.
621 		 */
622 		if (gip->gi_dir == WALK_DEPENDENTS)
623 			list = v->gv_dependents;
624 		else
625 			list = v->gv_dependencies;
626 
627 		r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
628 		    gip, 0);
629 		assert(r == 0);
630 	}
631 
632 	/*
633 	 * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
634 	 */
635 	assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
636 
637 	/*
638 	 * If given a post-callback, call the function for every vertex.
639 	 */
640 	if (gip->gi_post != NULL)
641 		(void) gip->gi_post(v, gip->gi_arg);
642 
643 	/*
644 	 * Preserve the callback's return value.  If the callback returns
645 	 * UU_WALK_DONE, then we propagate that to the caller in order to
646 	 * terminate the walk.
647 	 */
648 	return (gip->gi_ret);
649 }
650 
651 static void
652 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
653     int (*pre)(graph_vertex_t *, void *),
654     void (*post)(graph_vertex_t *, void *), void *arg)
655 {
656 	graph_walk_info_t gi;
657 	graph_edge_t fake;
658 	size_t sz = dictionary->dict_new_id / 8 + 1;
659 
660 	gi.gi_visited = startd_zalloc(sz);
661 	gi.gi_pre = pre;
662 	gi.gi_post = post;
663 	gi.gi_arg = arg;
664 	gi.gi_dir = dir;
665 	gi.gi_ret = 0;
666 
667 	/*
668 	 * Fake up an edge for the first iteration
669 	 */
670 	fake.ge_vertex = v;
671 	(void) graph_walk_recurse(&fake, &gi);
672 
673 	startd_free(gi.gi_visited, sz);
674 }
675 
676 typedef struct child_search {
677 	int	id;		/* id of vertex to look for */
678 	uint_t	depth;		/* recursion depth */
679 	/*
680 	 * While the vertex is not found, path is NULL.  After the search, if
681 	 * the vertex was found then path should point to a -1-terminated
682 	 * array of vertex id's which constitute the path to the vertex.
683 	 */
684 	int	*path;
685 } child_search_t;
686 
687 static int
688 child_pre(graph_vertex_t *v, void *arg)
689 {
690 	child_search_t *cs = arg;
691 
692 	cs->depth++;
693 
694 	if (v->gv_id == cs->id) {
695 		cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
696 		cs->path[cs->depth] = -1;
697 		return (UU_WALK_DONE);
698 	}
699 
700 	return (UU_WALK_NEXT);
701 }
702 
703 static void
704 child_post(graph_vertex_t *v, void *arg)
705 {
706 	child_search_t *cs = arg;
707 
708 	cs->depth--;
709 
710 	if (cs->path != NULL)
711 		cs->path[cs->depth] = v->gv_id;
712 }
713 
714 /*
715  * Look for a path from from to to.  If one exists, returns a pointer to
716  * a NULL-terminated array of pointers to the vertices along the path.  If
717  * there is no path, returns NULL.
718  */
719 static int *
720 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
721 {
722 	child_search_t cs;
723 
724 	cs.id = to->gv_id;
725 	cs.depth = 0;
726 	cs.path = NULL;
727 
728 	graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
729 
730 	return (cs.path);
731 }
732 
733 /*
734  * Given an array of int's as returned by is_path_to, allocates a string of
735  * their names joined by newlines.  Returns the size of the allocated buffer
736  * in *sz and frees path.
737  */
738 static void
739 path_to_str(int *path, char **cpp, size_t *sz)
740 {
741 	int i;
742 	graph_vertex_t *v;
743 	size_t allocd, new_allocd;
744 	char *new, *name;
745 
746 	assert(MUTEX_HELD(&dgraph_lock));
747 	assert(path[0] != -1);
748 
749 	allocd = 1;
750 	*cpp = startd_alloc(1);
751 	(*cpp)[0] = '\0';
752 
753 	for (i = 0; path[i] != -1; ++i) {
754 		name = NULL;
755 
756 		v = vertex_get_by_id(path[i]);
757 
758 		if (v == NULL)
759 			name = "<deleted>";
760 		else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
761 			name = v->gv_name;
762 
763 		if (name != NULL) {
764 			new_allocd = allocd + strlen(name) + 1;
765 			new = startd_alloc(new_allocd);
766 			(void) strcpy(new, *cpp);
767 			(void) strcat(new, name);
768 			(void) strcat(new, "\n");
769 
770 			startd_free(*cpp, allocd);
771 
772 			*cpp = new;
773 			allocd = new_allocd;
774 		}
775 	}
776 
777 	startd_free(path, sizeof (int) * (i + 1));
778 
779 	*sz = allocd;
780 }
781 
782 
783 /*
784  * This function along with run_sulogin() implements an exclusion relationship
785  * between system/console-login and sulogin.  run_sulogin() will fail if
786  * system/console-login is online, and the graph engine should call
787  * graph_clogin_start() to bring system/console-login online, which defers the
788  * start if sulogin is running.
789  */
790 static void
791 graph_clogin_start(graph_vertex_t *v)
792 {
793 	assert(MUTEX_HELD(&dgraph_lock));
794 
795 	if (sulogin_running)
796 		console_login_ready = B_TRUE;
797 	else
798 		vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
799 }
800 
801 static void
802 graph_su_start(graph_vertex_t *v)
803 {
804 	/*
805 	 * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
806 	 * entry with a runlevel of 'S', before jumping to the final
807 	 * target runlevel (as set in initdefault).  We mimic that legacy
808 	 * behavior here.
809 	 */
810 	utmpx_set_runlevel('S', '0', B_FALSE);
811 	vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
812 }
813 
814 static void
815 graph_post_su_online(void)
816 {
817 	graph_runlevel_changed('S', 1);
818 }
819 
820 static void
821 graph_post_su_disable(void)
822 {
823 	graph_runlevel_changed('S', 0);
824 }
825 
826 static void
827 graph_post_mu_online(void)
828 {
829 	graph_runlevel_changed('2', 1);
830 }
831 
832 static void
833 graph_post_mu_disable(void)
834 {
835 	graph_runlevel_changed('2', 0);
836 }
837 
838 static void
839 graph_post_mus_online(void)
840 {
841 	graph_runlevel_changed('3', 1);
842 }
843 
844 static void
845 graph_post_mus_disable(void)
846 {
847 	graph_runlevel_changed('3', 0);
848 }
849 
850 static struct special_vertex_info {
851 	const char	*name;
852 	void		(*start_f)(graph_vertex_t *);
853 	void		(*post_online_f)(void);
854 	void		(*post_disable_f)(void);
855 } special_vertices[] = {
856 	{ CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
857 	{ SCF_MILESTONE_SINGLE_USER, graph_su_start,
858 	    graph_post_su_online, graph_post_su_disable },
859 	{ SCF_MILESTONE_MULTI_USER, NULL,
860 	    graph_post_mu_online, graph_post_mu_disable },
861 	{ SCF_MILESTONE_MULTI_USER_SERVER, NULL,
862 	    graph_post_mus_online, graph_post_mus_disable },
863 	{ NULL },
864 };
865 
866 
867 void
868 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
869 {
870 	switch (e) {
871 	case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
872 		assert(v->gv_state == RESTARTER_STATE_UNINIT);
873 
874 		MUTEX_LOCK(&st->st_load_lock);
875 		st->st_load_instances++;
876 		MUTEX_UNLOCK(&st->st_load_lock);
877 		break;
878 
879 	case RESTARTER_EVENT_TYPE_ENABLE:
880 		log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
881 		assert(v->gv_state == RESTARTER_STATE_UNINIT ||
882 		    v->gv_state == RESTARTER_STATE_DISABLED ||
883 		    v->gv_state == RESTARTER_STATE_MAINT);
884 		break;
885 
886 	case RESTARTER_EVENT_TYPE_DISABLE:
887 	case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
888 		log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
889 		assert(v->gv_state != RESTARTER_STATE_DISABLED);
890 		break;
891 
892 	case RESTARTER_EVENT_TYPE_STOP_RESET:
893 	case RESTARTER_EVENT_TYPE_STOP:
894 		log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
895 		assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
896 		    v->gv_state == RESTARTER_STATE_ONLINE);
897 		break;
898 
899 	case RESTARTER_EVENT_TYPE_START:
900 		log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
901 		assert(v->gv_state == RESTARTER_STATE_OFFLINE);
902 		break;
903 
904 	case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
905 	case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
906 	case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
907 	case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
908 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
909 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
910 	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
911 	case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
912 	case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
913 		break;
914 
915 	default:
916 #ifndef NDEBUG
917 		uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
918 #endif
919 		abort();
920 	}
921 
922 	restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
923 	    v->gv_reason);
924 }
925 
926 static void
927 graph_unset_restarter(graph_vertex_t *v)
928 {
929 	assert(MUTEX_HELD(&dgraph_lock));
930 	assert(v->gv_flags & GV_CONFIGURED);
931 
932 	vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
933 
934 	if (v->gv_restarter_id != -1) {
935 		graph_vertex_t *rv;
936 
937 		rv = vertex_get_by_id(v->gv_restarter_id);
938 		graph_remove_edge(v, rv);
939 	}
940 
941 	v->gv_restarter_id = -1;
942 	v->gv_restarter_channel = NULL;
943 }
944 
945 /*
946  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
947  * dgraph otherwise return VERTEX_INUSE.
948  */
949 static int
950 free_if_unrefed(graph_vertex_t *v)
951 {
952 	assert(MUTEX_HELD(&dgraph_lock));
953 
954 	if (v->gv_refs > 0)
955 		return (VERTEX_INUSE);
956 
957 	if (v->gv_type == GVT_SVC &&
958 	    uu_list_numnodes(v->gv_dependents) == 0 &&
959 	    uu_list_numnodes(v->gv_dependencies) == 0) {
960 		graph_remove_vertex(v);
961 		return (VERTEX_REMOVED);
962 	} else if (v->gv_type == GVT_INST &&
963 	    (v->gv_flags & GV_CONFIGURED) == 0 &&
964 	    uu_list_numnodes(v->gv_dependents) == 1 &&
965 	    uu_list_numnodes(v->gv_dependencies) == 0) {
966 		remove_inst_vertex(v);
967 		return (VERTEX_REMOVED);
968 	}
969 
970 	return (VERTEX_INUSE);
971 }
972 
973 static void
974 delete_depgroup(graph_vertex_t *v)
975 {
976 	graph_edge_t *e;
977 	graph_vertex_t *dv;
978 
979 	assert(MUTEX_HELD(&dgraph_lock));
980 	assert(v->gv_type == GVT_GROUP);
981 	assert(uu_list_numnodes(v->gv_dependents) == 0);
982 
983 	while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
984 		dv = e->ge_vertex;
985 
986 		graph_remove_edge(v, dv);
987 
988 		switch (dv->gv_type) {
989 		case GVT_INST:		/* instance dependency */
990 		case GVT_SVC:		/* service dependency */
991 			(void) free_if_unrefed(dv);
992 			break;
993 
994 		case GVT_FILE:		/* file dependency */
995 			assert(uu_list_numnodes(dv->gv_dependencies) == 0);
996 			if (uu_list_numnodes(dv->gv_dependents) == 0)
997 				graph_remove_vertex(dv);
998 			break;
999 
1000 		default:
1001 #ifndef NDEBUG
1002 			uu_warn("%s:%d: Unexpected node type %d", __FILE__,
1003 			    __LINE__, dv->gv_type);
1004 #endif
1005 			abort();
1006 		}
1007 	}
1008 
1009 	graph_remove_vertex(v);
1010 }
1011 
1012 static int
1013 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1014 {
1015 	graph_vertex_t *v = ptrs[0];
1016 	boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1017 	graph_vertex_t *dv;
1018 
1019 	dv = e->ge_vertex;
1020 
1021 	/*
1022 	 * We have four possibilities here:
1023 	 *   - GVT_INST: restarter
1024 	 *   - GVT_GROUP - GVT_INST: instance dependency
1025 	 *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1026 	 *   - GVT_GROUP - GVT_FILE: file dependency
1027 	 */
1028 	switch (dv->gv_type) {
1029 	case GVT_INST:	/* restarter */
1030 		assert(dv->gv_id == v->gv_restarter_id);
1031 		if (delete_restarter_dep)
1032 			graph_remove_edge(v, dv);
1033 		break;
1034 
1035 	case GVT_GROUP:	/* pg dependency */
1036 		graph_remove_edge(v, dv);
1037 		delete_depgroup(dv);
1038 		break;
1039 
1040 	case GVT_FILE:
1041 		/* These are currently not direct dependencies */
1042 
1043 	default:
1044 #ifndef NDEBUG
1045 		uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1046 		    dv->gv_type);
1047 #endif
1048 		abort();
1049 	}
1050 
1051 	return (UU_WALK_NEXT);
1052 }
1053 
1054 static void
1055 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1056 {
1057 	void *ptrs[2];
1058 	int r;
1059 
1060 	assert(MUTEX_HELD(&dgraph_lock));
1061 	assert(v->gv_type == GVT_INST);
1062 
1063 	ptrs[0] = v;
1064 	ptrs[1] = (void *)delete_restarter_dep;
1065 
1066 	r = uu_list_walk(v->gv_dependencies,
1067 	    (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1068 	assert(r == 0);
1069 }
1070 
1071 /*
1072  * int graph_insert_vertex_unconfigured()
1073  *   Insert a vertex without sending any restarter events. If the vertex
1074  *   already exists or creation is successful, return a pointer to it in *vp.
1075  *
1076  *   If type is not GVT_GROUP, dt can remain unset.
1077  *
1078  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1079  *   doesn't agree with type, or type doesn't agree with dt).
1080  */
1081 static int
1082 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1083     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1084 {
1085 	int r;
1086 	int i;
1087 
1088 	assert(MUTEX_HELD(&dgraph_lock));
1089 
1090 	switch (type) {
1091 	case GVT_SVC:
1092 	case GVT_INST:
1093 		if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1094 			return (EINVAL);
1095 		break;
1096 
1097 	case GVT_FILE:
1098 		if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1099 			return (EINVAL);
1100 		break;
1101 
1102 	case GVT_GROUP:
1103 		if (dt <= 0 || rt < 0)
1104 			return (EINVAL);
1105 		break;
1106 
1107 	default:
1108 #ifndef NDEBUG
1109 		uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1110 #endif
1111 		abort();
1112 	}
1113 
1114 	*vp = vertex_get_by_name(fmri);
1115 	if (*vp != NULL)
1116 		return (EEXIST);
1117 
1118 	*vp = graph_add_vertex(fmri);
1119 
1120 	(*vp)->gv_type = type;
1121 	(*vp)->gv_depgroup = dt;
1122 	(*vp)->gv_restart = rt;
1123 
1124 	(*vp)->gv_flags = 0;
1125 	(*vp)->gv_state = RESTARTER_STATE_NONE;
1126 
1127 	for (i = 0; special_vertices[i].name != NULL; ++i) {
1128 		if (strcmp(fmri, special_vertices[i].name) == 0) {
1129 			(*vp)->gv_start_f = special_vertices[i].start_f;
1130 			(*vp)->gv_post_online_f =
1131 			    special_vertices[i].post_online_f;
1132 			(*vp)->gv_post_disable_f =
1133 			    special_vertices[i].post_disable_f;
1134 			break;
1135 		}
1136 	}
1137 
1138 	(*vp)->gv_restarter_id = -1;
1139 	(*vp)->gv_restarter_channel = 0;
1140 
1141 	if (type == GVT_INST) {
1142 		char *sfmri;
1143 		graph_vertex_t *sv;
1144 
1145 		sfmri = inst_fmri_to_svc_fmri(fmri);
1146 		sv = vertex_get_by_name(sfmri);
1147 		if (sv == NULL) {
1148 			r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1149 			    0, &sv);
1150 			assert(r == 0);
1151 		}
1152 		startd_free(sfmri, max_scf_fmri_size);
1153 
1154 		graph_add_edge(sv, *vp);
1155 	}
1156 
1157 	/*
1158 	 * If this vertex is in the subgraph, mark it as so, for both
1159 	 * GVT_INST and GVT_SERVICE verteces.
1160 	 * A GVT_SERVICE vertex can only be in the subgraph if another instance
1161 	 * depends on it, in which case it's already been added to the graph
1162 	 * and marked as in the subgraph (by refresh_vertex()).  If a
1163 	 * GVT_SERVICE vertex was freshly added (by the code above), it means
1164 	 * that it has no dependents, and cannot be in the subgraph.
1165 	 * Regardless of this, we still check that gv_flags includes
1166 	 * GV_INSUBGRAPH in the event that future behavior causes the above
1167 	 * code to add a GVT_SERVICE vertex which should be in the subgraph.
1168 	 */
1169 
1170 	(*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1171 
1172 	return (0);
1173 }
1174 
1175 /*
1176  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1177  */
1178 static int
1179 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1180 {
1181 	hrtime_t now;
1182 
1183 	assert(MUTEX_HELD(&dgraph_lock));
1184 
1185 	/* cycle detection */
1186 	now = gethrtime();
1187 
1188 	/* Don't follow exclusions. */
1189 	if (!(fv->gv_type == GVT_GROUP &&
1190 	    fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1191 		*pathp = is_path_to(tv, fv);
1192 		if (*pathp)
1193 			return (ELOOP);
1194 	}
1195 
1196 	dep_cycle_ns += gethrtime() - now;
1197 	++dep_inserts;
1198 	now = gethrtime();
1199 
1200 	graph_add_edge(fv, tv);
1201 
1202 	dep_insert_ns += gethrtime() - now;
1203 
1204 	/* Check if the dependency adds the "to" vertex to the subgraph */
1205 	tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1206 
1207 	return (0);
1208 }
1209 
1210 static int
1211 inst_running(graph_vertex_t *v)
1212 {
1213 	assert(v->gv_type == GVT_INST);
1214 
1215 	if (v->gv_state == RESTARTER_STATE_ONLINE ||
1216 	    v->gv_state == RESTARTER_STATE_DEGRADED)
1217 		return (1);
1218 
1219 	return (0);
1220 }
1221 
1222 /*
1223  * The dependency evaluation functions return
1224  *   1 - dependency satisfied
1225  *   0 - dependency unsatisfied
1226  *   -1 - dependency unsatisfiable (without administrator intervention)
1227  *
1228  * The functions also take a boolean satbility argument.  When true, the
1229  * functions may recurse in order to determine satisfiability.
1230  */
1231 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1232 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1233 
1234 /*
1235  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1236  * is unsatisfiable if any elements are unsatisfiable.
1237  */
1238 static int
1239 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1240 {
1241 	graph_edge_t *edge;
1242 	int i;
1243 	boolean_t any_unsatisfied;
1244 
1245 	if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1246 		return (1);
1247 
1248 	any_unsatisfied = B_FALSE;
1249 
1250 	for (edge = uu_list_first(groupv->gv_dependencies);
1251 	    edge != NULL;
1252 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1253 		i = dependency_satisfied(edge->ge_vertex, satbility);
1254 		if (i == 1)
1255 			continue;
1256 
1257 		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1258 		    "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1259 		    edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1260 
1261 		if (!satbility)
1262 			return (0);
1263 
1264 		if (i == -1)
1265 			return (-1);
1266 
1267 		any_unsatisfied = B_TRUE;
1268 	}
1269 
1270 	return (any_unsatisfied ? 0 : 1);
1271 }
1272 
1273 /*
1274  * A require_any dependency is satisfied if any element is satisfied.  It is
1275  * satisfiable if any element is satisfiable.
1276  */
1277 static int
1278 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1279 {
1280 	graph_edge_t *edge;
1281 	int s;
1282 	boolean_t satisfiable;
1283 
1284 	if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1285 		return (1);
1286 
1287 	satisfiable = B_FALSE;
1288 
1289 	for (edge = uu_list_first(groupv->gv_dependencies);
1290 	    edge != NULL;
1291 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1292 		s = dependency_satisfied(edge->ge_vertex, satbility);
1293 
1294 		if (s == 1)
1295 			return (1);
1296 
1297 		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1298 		    "require_any(%s): %s is unsatisfi%s.\n",
1299 		    groupv->gv_name, edge->ge_vertex->gv_name,
1300 		    s == 0 ? "ed" : "able");
1301 
1302 		if (satbility && s == 0)
1303 			satisfiable = B_TRUE;
1304 	}
1305 
1306 	return ((!satbility || satisfiable) ? 0 : -1);
1307 }
1308 
1309 /*
1310  * An optional_all dependency only considers elements which are configured,
1311  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1312  * is unsatisfied.
1313  *
1314  * Offline dependencies which are waiting for a dependency to come online are
1315  * unsatisfied.  Offline dependences which cannot possibly come online
1316  * (unsatisfiable) are always considered satisfied.
1317  */
1318 static int
1319 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1320 {
1321 	graph_edge_t *edge;
1322 	graph_vertex_t *v;
1323 	boolean_t any_qualified;
1324 	boolean_t any_unsatisfied;
1325 	int i;
1326 
1327 	any_qualified = B_FALSE;
1328 	any_unsatisfied = B_FALSE;
1329 
1330 	for (edge = uu_list_first(groupv->gv_dependencies);
1331 	    edge != NULL;
1332 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1333 		v = edge->ge_vertex;
1334 
1335 		switch (v->gv_type) {
1336 		case GVT_INST:
1337 			/* Skip missing instances */
1338 			if ((v->gv_flags & GV_CONFIGURED) == 0)
1339 				continue;
1340 
1341 			if (v->gv_state == RESTARTER_STATE_MAINT)
1342 				continue;
1343 
1344 			any_qualified = B_TRUE;
1345 			if (v->gv_state == RESTARTER_STATE_OFFLINE ||
1346 			    v->gv_state == RESTARTER_STATE_DISABLED) {
1347 				/*
1348 				 * For offline/disabled dependencies,
1349 				 * treat unsatisfiable as satisfied.
1350 				 */
1351 				i = dependency_satisfied(v, B_TRUE);
1352 				if (i == -1)
1353 					i = 1;
1354 			} else {
1355 				i = dependency_satisfied(v, satbility);
1356 			}
1357 			break;
1358 
1359 		case GVT_FILE:
1360 			any_qualified = B_TRUE;
1361 			i = dependency_satisfied(v, satbility);
1362 
1363 			break;
1364 
1365 		case GVT_SVC: {
1366 			any_qualified = B_TRUE;
1367 			i = optional_all_satisfied(v, satbility);
1368 
1369 			break;
1370 		}
1371 
1372 		case GVT_GROUP:
1373 		default:
1374 #ifndef NDEBUG
1375 			uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1376 			    __LINE__, v->gv_type);
1377 #endif
1378 			abort();
1379 		}
1380 
1381 		if (i == 1)
1382 			continue;
1383 
1384 		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1385 		    "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1386 		    v->gv_name, i == 0 ? "ed" : "able");
1387 
1388 		if (!satbility)
1389 			return (0);
1390 		if (i == -1)
1391 			return (-1);
1392 		any_unsatisfied = B_TRUE;
1393 	}
1394 
1395 	if (!any_qualified)
1396 		return (1);
1397 
1398 	return (any_unsatisfied ? 0 : 1);
1399 }
1400 
1401 /*
1402  * An exclude_all dependency is unsatisfied if any non-service element is
1403  * satisfied or any service instance which is configured, enabled, and not in
1404  * maintenance is satisfied.  Usually when unsatisfied, it is also
1405  * unsatisfiable.
1406  */
1407 #define	LOG_EXCLUDE(u, v)						\
1408 	log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,			\
1409 	    "exclude_all(%s): %s is satisfied.\n",			\
1410 	    (u)->gv_name, (v)->gv_name)
1411 
1412 /* ARGSUSED */
1413 static int
1414 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1415 {
1416 	graph_edge_t *edge, *e2;
1417 	graph_vertex_t *v, *v2;
1418 
1419 	for (edge = uu_list_first(groupv->gv_dependencies);
1420 	    edge != NULL;
1421 	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1422 		v = edge->ge_vertex;
1423 
1424 		switch (v->gv_type) {
1425 		case GVT_INST:
1426 			if ((v->gv_flags & GV_CONFIGURED) == 0)
1427 				continue;
1428 
1429 			switch (v->gv_state) {
1430 			case RESTARTER_STATE_ONLINE:
1431 			case RESTARTER_STATE_DEGRADED:
1432 				LOG_EXCLUDE(groupv, v);
1433 				return (v->gv_flags & GV_ENABLED ? -1 : 0);
1434 
1435 			case RESTARTER_STATE_OFFLINE:
1436 			case RESTARTER_STATE_UNINIT:
1437 				LOG_EXCLUDE(groupv, v);
1438 				return (0);
1439 
1440 			case RESTARTER_STATE_DISABLED:
1441 			case RESTARTER_STATE_MAINT:
1442 				continue;
1443 
1444 			default:
1445 #ifndef NDEBUG
1446 				uu_warn("%s:%d: Unexpected vertex state %d.\n",
1447 				    __FILE__, __LINE__, v->gv_state);
1448 #endif
1449 				abort();
1450 			}
1451 			/* NOTREACHED */
1452 
1453 		case GVT_SVC:
1454 			break;
1455 
1456 		case GVT_FILE:
1457 			if (!file_ready(v))
1458 				continue;
1459 			LOG_EXCLUDE(groupv, v);
1460 			return (-1);
1461 
1462 		case GVT_GROUP:
1463 		default:
1464 #ifndef NDEBUG
1465 			uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1466 			    __LINE__, v->gv_type);
1467 #endif
1468 			abort();
1469 		}
1470 
1471 		/* v represents a service */
1472 		if (uu_list_numnodes(v->gv_dependencies) == 0)
1473 			continue;
1474 
1475 		for (e2 = uu_list_first(v->gv_dependencies);
1476 		    e2 != NULL;
1477 		    e2 = uu_list_next(v->gv_dependencies, e2)) {
1478 			v2 = e2->ge_vertex;
1479 			assert(v2->gv_type == GVT_INST);
1480 
1481 			if ((v2->gv_flags & GV_CONFIGURED) == 0)
1482 				continue;
1483 
1484 			switch (v2->gv_state) {
1485 			case RESTARTER_STATE_ONLINE:
1486 			case RESTARTER_STATE_DEGRADED:
1487 				LOG_EXCLUDE(groupv, v2);
1488 				return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1489 
1490 			case RESTARTER_STATE_OFFLINE:
1491 			case RESTARTER_STATE_UNINIT:
1492 				LOG_EXCLUDE(groupv, v2);
1493 				return (0);
1494 
1495 			case RESTARTER_STATE_DISABLED:
1496 			case RESTARTER_STATE_MAINT:
1497 				continue;
1498 
1499 			default:
1500 #ifndef NDEBUG
1501 				uu_warn("%s:%d: Unexpected vertex type %d.\n",
1502 				    __FILE__, __LINE__, v2->gv_type);
1503 #endif
1504 				abort();
1505 			}
1506 		}
1507 	}
1508 
1509 	return (1);
1510 }
1511 
1512 /*
1513  * int instance_satisfied()
1514  *   Determine if all the dependencies are satisfied for the supplied instance
1515  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1516  *   without administrator intervention.
1517  */
1518 static int
1519 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1520 {
1521 	assert(v->gv_type == GVT_INST);
1522 	assert(!inst_running(v));
1523 
1524 	return (require_all_satisfied(v, satbility));
1525 }
1526 
1527 /*
1528  * Decide whether v can satisfy a dependency.  v can either be a child of
1529  * a group vertex, or of an instance vertex.
1530  */
1531 static int
1532 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1533 {
1534 	switch (v->gv_type) {
1535 	case GVT_INST:
1536 		if ((v->gv_flags & GV_CONFIGURED) == 0) {
1537 			if (v->gv_flags & GV_DEATHROW) {
1538 				/*
1539 				 * A dependency on an instance with GV_DEATHROW
1540 				 * flag is always considered as satisfied.
1541 				 */
1542 				return (1);
1543 			}
1544 			return (-1);
1545 		}
1546 
1547 		/*
1548 		 * Vertices may be transitioning so we try to figure out if
1549 		 * the end state is likely to satisfy the dependency instead
1550 		 * of assuming the dependency is unsatisfied/unsatisfiable.
1551 		 *
1552 		 * Support for optional_all dependencies depends on us getting
1553 		 * this right because unsatisfiable dependencies are treated
1554 		 * as being satisfied.
1555 		 */
1556 		switch (v->gv_state) {
1557 		case RESTARTER_STATE_ONLINE:
1558 		case RESTARTER_STATE_DEGRADED:
1559 			if (v->gv_flags & GV_TODISABLE)
1560 				return (-1);
1561 			if (v->gv_flags & GV_TOOFFLINE)
1562 				return (0);
1563 			return (1);
1564 
1565 		case RESTARTER_STATE_OFFLINE:
1566 			if (!satbility || v->gv_flags & GV_TODISABLE)
1567 				return (satbility ? -1 : 0);
1568 			return (instance_satisfied(v, satbility) != -1 ?
1569 			    0 : -1);
1570 
1571 		case RESTARTER_STATE_DISABLED:
1572 			if (!satbility || !(v->gv_flags & GV_ENABLED))
1573 				return (satbility ? -1 : 0);
1574 			return (instance_satisfied(v, satbility) != -1 ?
1575 			    0 : -1);
1576 
1577 		case RESTARTER_STATE_MAINT:
1578 			return (-1);
1579 
1580 		case RESTARTER_STATE_UNINIT:
1581 			return (0);
1582 
1583 		default:
1584 #ifndef NDEBUG
1585 			uu_warn("%s:%d: Unexpected vertex state %d.\n",
1586 			    __FILE__, __LINE__, v->gv_state);
1587 #endif
1588 			abort();
1589 			/* NOTREACHED */
1590 		}
1591 
1592 	case GVT_SVC:
1593 		if (uu_list_numnodes(v->gv_dependencies) == 0)
1594 			return (-1);
1595 		return (require_any_satisfied(v, satbility));
1596 
1597 	case GVT_FILE:
1598 		/* i.e., we assume files will not be automatically generated */
1599 		return (file_ready(v) ? 1 : -1);
1600 
1601 	case GVT_GROUP:
1602 		break;
1603 
1604 	default:
1605 #ifndef NDEBUG
1606 		uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1607 		    v->gv_type);
1608 #endif
1609 		abort();
1610 		/* NOTREACHED */
1611 	}
1612 
1613 	switch (v->gv_depgroup) {
1614 	case DEPGRP_REQUIRE_ANY:
1615 		return (require_any_satisfied(v, satbility));
1616 
1617 	case DEPGRP_REQUIRE_ALL:
1618 		return (require_all_satisfied(v, satbility));
1619 
1620 	case DEPGRP_OPTIONAL_ALL:
1621 		return (optional_all_satisfied(v, satbility));
1622 
1623 	case DEPGRP_EXCLUDE_ALL:
1624 		return (exclude_all_satisfied(v, satbility));
1625 
1626 	default:
1627 #ifndef NDEBUG
1628 		uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1629 		    __LINE__, v->gv_depgroup);
1630 #endif
1631 		abort();
1632 	}
1633 }
1634 
1635 void
1636 graph_start_if_satisfied(graph_vertex_t *v)
1637 {
1638 	if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1639 	    instance_satisfied(v, B_FALSE) == 1) {
1640 		if (v->gv_start_f == NULL)
1641 			vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1642 		else
1643 			v->gv_start_f(v);
1644 	}
1645 }
1646 
1647 /*
1648  * propagate_satbility()
1649  *
1650  * This function is used when the given vertex changes state in such a way that
1651  * one of its dependents may become unsatisfiable.  This happens when an
1652  * instance transitions between offline -> online, or from !running ->
1653  * maintenance, as well as when an instance is removed from the graph.
1654  *
1655  * We have to walk all the dependents, since optional_all dependencies several
1656  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1657  *
1658  *	+-----+  optional_all  +-----+  require_all  +-----+
1659  *	|  A  |--------------->|  B  |-------------->|  C  |
1660  *	+-----+                +-----+               +-----+
1661  *
1662  *	                                        offline -> maintenance
1663  *
1664  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1665  * an optional dependency, what was previously an unsatisfiable situation is now
1666  * satisfied (B will never come online, even though its state hasn't changed).
1667  *
1668  * Note that it's not necessary to continue examining dependents after reaching
1669  * an optional_all dependency.  It's not possible for an optional_all dependency
1670  * to change satisfiability without also coming online, in which case we get a
1671  * start event and propagation continues naturally.  However, it does no harm to
1672  * continue propagating satisfiability (as it is a relatively rare event), and
1673  * keeps the walker code simple and generic.
1674  */
1675 /*ARGSUSED*/
1676 static int
1677 satbility_cb(graph_vertex_t *v, void *arg)
1678 {
1679 	if (is_inst_bypassed(v))
1680 		return (UU_WALK_NEXT);
1681 
1682 	if (v->gv_type == GVT_INST)
1683 		graph_start_if_satisfied(v);
1684 
1685 	return (UU_WALK_NEXT);
1686 }
1687 
1688 static void
1689 propagate_satbility(graph_vertex_t *v)
1690 {
1691 	graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1692 }
1693 
1694 static void propagate_stop(graph_vertex_t *, void *);
1695 
1696 /*
1697  * propagate_start()
1698  *
1699  * This function is used to propagate a start event to the dependents of the
1700  * given vertex.  Any dependents that are offline but have their dependencies
1701  * satisfied are started.  Any dependents that are online and have restart_on
1702  * set to "restart" or "refresh" are restarted because their dependencies have
1703  * just changed.  This only happens with optional_all dependencies.
1704  */
1705 static void
1706 propagate_start(graph_vertex_t *v, void *arg)
1707 {
1708 	restarter_error_t err = (restarter_error_t)arg;
1709 
1710 	if (is_inst_bypassed(v))
1711 		return;
1712 
1713 	switch (v->gv_type) {
1714 	case GVT_INST:
1715 		/* Restarter */
1716 		if (inst_running(v)) {
1717 			if (err == RERR_RESTART || err == RERR_REFRESH) {
1718 				vertex_send_event(v,
1719 				    RESTARTER_EVENT_TYPE_STOP_RESET);
1720 			}
1721 		} else {
1722 			graph_start_if_satisfied(v);
1723 		}
1724 		break;
1725 
1726 	case GVT_GROUP:
1727 		if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1728 			graph_walk_dependents(v, propagate_stop,
1729 			    (void *)RERR_RESTART);
1730 			break;
1731 		}
1732 		err = v->gv_restart;
1733 		/* FALLTHROUGH */
1734 
1735 	case GVT_SVC:
1736 		graph_walk_dependents(v, propagate_start, (void *)err);
1737 		break;
1738 
1739 	case GVT_FILE:
1740 #ifndef NDEBUG
1741 		uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1742 		    __FILE__, __LINE__);
1743 #endif
1744 		abort();
1745 		/* NOTREACHED */
1746 
1747 	default:
1748 #ifndef NDEBUG
1749 		uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1750 		    v->gv_type);
1751 #endif
1752 		abort();
1753 	}
1754 }
1755 
1756 /*
1757  * propagate_stop()
1758  *
1759  * This function is used to propagate a stop event to the dependents of the
1760  * given vertex.  Any dependents that are online (or in degraded state) with
1761  * the restart_on property set to "restart" or "refresh" will be stopped as
1762  * their dependencies have just changed, propagate_start() will start them
1763  * again once their dependencies have been re-satisfied.
1764  */
1765 static void
1766 propagate_stop(graph_vertex_t *v, void *arg)
1767 {
1768 	restarter_error_t err = (restarter_error_t)arg;
1769 
1770 	if (is_inst_bypassed(v))
1771 		return;
1772 
1773 	switch (v->gv_type) {
1774 	case GVT_INST:
1775 		/* Restarter */
1776 		if (err > RERR_NONE && inst_running(v)) {
1777 			if (err == RERR_RESTART || err == RERR_REFRESH) {
1778 				vertex_send_event(v,
1779 				    RESTARTER_EVENT_TYPE_STOP_RESET);
1780 			} else {
1781 				vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1782 			}
1783 		}
1784 		break;
1785 
1786 	case GVT_SVC:
1787 		graph_walk_dependents(v, propagate_stop, arg);
1788 		break;
1789 
1790 	case GVT_FILE:
1791 #ifndef NDEBUG
1792 		uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1793 		    __FILE__, __LINE__);
1794 #endif
1795 		abort();
1796 		/* NOTREACHED */
1797 
1798 	case GVT_GROUP:
1799 		if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1800 			graph_walk_dependents(v, propagate_start,
1801 			    (void *)RERR_NONE);
1802 			break;
1803 		}
1804 
1805 		if (err == RERR_NONE || err > v->gv_restart)
1806 			break;
1807 
1808 		graph_walk_dependents(v, propagate_stop, arg);
1809 		break;
1810 
1811 	default:
1812 #ifndef NDEBUG
1813 		uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1814 		    v->gv_type);
1815 #endif
1816 		abort();
1817 	}
1818 }
1819 
1820 void
1821 offline_vertex(graph_vertex_t *v)
1822 {
1823 	scf_handle_t *h = libscf_handle_create_bound_loop();
1824 	scf_instance_t *scf_inst = safe_scf_instance_create(h);
1825 	scf_propertygroup_t *pg = safe_scf_pg_create(h);
1826 	restarter_instance_state_t state, next_state;
1827 	int r;
1828 
1829 	assert(v->gv_type == GVT_INST);
1830 
1831 	if (scf_inst == NULL)
1832 		bad_error("safe_scf_instance_create", scf_error());
1833 	if (pg == NULL)
1834 		bad_error("safe_scf_pg_create", scf_error());
1835 
1836 	/* if the vertex is already going offline, return */
1837 rep_retry:
1838 	if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1839 	    NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1840 		switch (scf_error()) {
1841 		case SCF_ERROR_CONNECTION_BROKEN:
1842 			libscf_handle_rebind(h);
1843 			goto rep_retry;
1844 
1845 		case SCF_ERROR_NOT_FOUND:
1846 			scf_pg_destroy(pg);
1847 			scf_instance_destroy(scf_inst);
1848 			(void) scf_handle_unbind(h);
1849 			scf_handle_destroy(h);
1850 			return;
1851 		}
1852 		uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1853 		    scf_strerror(scf_error()));
1854 	}
1855 
1856 	r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1857 	if (r != 0) {
1858 		switch (scf_error()) {
1859 		case SCF_ERROR_CONNECTION_BROKEN:
1860 			libscf_handle_rebind(h);
1861 			goto rep_retry;
1862 
1863 		case SCF_ERROR_NOT_SET:
1864 		case SCF_ERROR_NOT_FOUND:
1865 			scf_pg_destroy(pg);
1866 			scf_instance_destroy(scf_inst);
1867 			(void) scf_handle_unbind(h);
1868 			scf_handle_destroy(h);
1869 			return;
1870 
1871 		default:
1872 			bad_error("scf_instance_get_pg", scf_error());
1873 		}
1874 	} else {
1875 		r = libscf_read_states(pg, &state, &next_state);
1876 		if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1877 		    next_state == RESTARTER_STATE_DISABLED)) {
1878 			log_framework(LOG_DEBUG,
1879 			    "%s: instance is already going down.\n",
1880 			    v->gv_name);
1881 			scf_pg_destroy(pg);
1882 			scf_instance_destroy(scf_inst);
1883 			(void) scf_handle_unbind(h);
1884 			scf_handle_destroy(h);
1885 			return;
1886 		}
1887 	}
1888 
1889 	scf_pg_destroy(pg);
1890 	scf_instance_destroy(scf_inst);
1891 	(void) scf_handle_unbind(h);
1892 	scf_handle_destroy(h);
1893 
1894 	vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1895 }
1896 
1897 /*
1898  * void graph_enable_by_vertex()
1899  *   If admin is non-zero, this is an administrative request for change
1900  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1901  *   a plain DISABLE restarter event.
1902  */
1903 void
1904 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1905 {
1906 	graph_vertex_t *v;
1907 	int r;
1908 
1909 	assert(MUTEX_HELD(&dgraph_lock));
1910 	assert((vertex->gv_flags & GV_CONFIGURED));
1911 
1912 	vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1913 	    (enable ? GV_ENABLED : 0);
1914 
1915 	if (enable) {
1916 		if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1917 		    vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1918 		    vertex->gv_state != RESTARTER_STATE_ONLINE) {
1919 			/*
1920 			 * In case the vertex was notified to go down,
1921 			 * but now can return online, clear the _TOOFFLINE
1922 			 * and _TODISABLE flags.
1923 			 */
1924 			vertex->gv_flags &= ~GV_TOOFFLINE;
1925 			vertex->gv_flags &= ~GV_TODISABLE;
1926 
1927 			vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1928 		}
1929 
1930 		/*
1931 		 * Wait for state update from restarter before sending _START or
1932 		 * _STOP.
1933 		 */
1934 
1935 		return;
1936 	}
1937 
1938 	if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1939 		return;
1940 
1941 	if (!admin) {
1942 		vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1943 
1944 		/*
1945 		 * Wait for state update from restarter before sending _START or
1946 		 * _STOP.
1947 		 */
1948 
1949 		return;
1950 	}
1951 
1952 	/*
1953 	 * If it is a DISABLE event requested by the administrator then we are
1954 	 * offlining the dependents first.
1955 	 */
1956 
1957 	/*
1958 	 * Set GV_TOOFFLINE for the services we are offlining. We cannot
1959 	 * clear the GV_TOOFFLINE bits from all the services because
1960 	 * other DISABLE events might be handled at the same time.
1961 	 */
1962 	vertex->gv_flags |= GV_TOOFFLINE;
1963 
1964 	/* remember which vertex to disable... */
1965 	vertex->gv_flags |= GV_TODISABLE;
1966 
1967 	log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1968 	    "disabling %s.\n", vertex->gv_name);
1969 
1970 	/* set GV_TOOFFLINE for its dependents */
1971 	r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1972 	    NULL, 0);
1973 	assert(r == 0);
1974 
1975 	/* disable the instance now if there is nothing else to offline */
1976 	if (insubtree_dependents_down(vertex) == B_TRUE) {
1977 		vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1978 		return;
1979 	}
1980 
1981 	/*
1982 	 * This loop is similar to the one used for the graph reversal shutdown
1983 	 * and could be improved in term of performance for the subtree reversal
1984 	 * disable case.
1985 	 */
1986 	for (v = uu_list_first(dgraph); v != NULL;
1987 	    v = uu_list_next(dgraph, v)) {
1988 		/* skip the vertex we are disabling for now */
1989 		if (v == vertex)
1990 			continue;
1991 
1992 		if (v->gv_type != GVT_INST ||
1993 		    (v->gv_flags & GV_CONFIGURED) == 0 ||
1994 		    (v->gv_flags & GV_ENABLED) == 0 ||
1995 		    (v->gv_flags & GV_TOOFFLINE) == 0)
1996 			continue;
1997 
1998 		if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1999 		    (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2000 			/* continue if there is nothing to offline */
2001 			continue;
2002 		}
2003 
2004 		/*
2005 		 * Instances which are up need to come down before we're
2006 		 * done, but we can only offline the leaves here. An
2007 		 * instance is a leaf when all its dependents are down.
2008 		 */
2009 		if (insubtree_dependents_down(v) == B_TRUE) {
2010 			log_framework(LOG_DEBUG, "Offlining in-subtree "
2011 			    "instance %s for %s.\n",
2012 			    v->gv_name, vertex->gv_name);
2013 			offline_vertex(v);
2014 		}
2015 	}
2016 }
2017 
2018 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2019 
2020 /*
2021  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2022  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2023  * v is already configured and fmri_arg indicates the current restarter, do
2024  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2025  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2026  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2027  * ECONNABORTED if the repository connection is broken, and ELOOP
2028  * if the dependency would create a cycle.  In the last case, *pathp will
2029  * point to a -1-terminated array of ids which compose the path from v to
2030  * restarter_fmri.
2031  */
2032 int
2033 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2034     int **pathp)
2035 {
2036 	char *restarter_fmri = NULL;
2037 	graph_vertex_t *rv;
2038 	int err;
2039 	int id;
2040 
2041 	assert(MUTEX_HELD(&dgraph_lock));
2042 
2043 	if (fmri_arg[0] != '\0') {
2044 		err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2045 		if (err != 0) {
2046 			assert(err == EINVAL);
2047 			return (err);
2048 		}
2049 	}
2050 
2051 	if (restarter_fmri == NULL ||
2052 	    strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2053 		if (v->gv_flags & GV_CONFIGURED) {
2054 			if (v->gv_restarter_id == -1) {
2055 				if (restarter_fmri != NULL)
2056 					startd_free(restarter_fmri,
2057 					    max_scf_fmri_size);
2058 				return (0);
2059 			}
2060 
2061 			graph_unset_restarter(v);
2062 		}
2063 
2064 		/* Master restarter, nothing to do. */
2065 		v->gv_restarter_id = -1;
2066 		v->gv_restarter_channel = NULL;
2067 		vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2068 		return (0);
2069 	}
2070 
2071 	if (v->gv_flags & GV_CONFIGURED) {
2072 		id = dict_lookup_byname(restarter_fmri);
2073 		if (id != -1 && v->gv_restarter_id == id) {
2074 			startd_free(restarter_fmri, max_scf_fmri_size);
2075 			return (0);
2076 		}
2077 
2078 		graph_unset_restarter(v);
2079 	}
2080 
2081 	err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2082 	    RERR_NONE, &rv);
2083 	startd_free(restarter_fmri, max_scf_fmri_size);
2084 	assert(err == 0 || err == EEXIST);
2085 
2086 	if (rv->gv_delegate_initialized == 0) {
2087 		if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2088 		    rv->gv_name)) == NULL)
2089 			return (EINVAL);
2090 		rv->gv_delegate_initialized = 1;
2091 	}
2092 	v->gv_restarter_id = rv->gv_id;
2093 	v->gv_restarter_channel = rv->gv_delegate_channel;
2094 
2095 	err = graph_insert_dependency(v, rv, pathp);
2096 	if (err != 0) {
2097 		assert(err == ELOOP);
2098 		return (ELOOP);
2099 	}
2100 
2101 	vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2102 
2103 	if (!(rv->gv_flags & GV_CONFIGURED)) {
2104 		scf_instance_t *inst;
2105 
2106 		err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2107 		switch (err) {
2108 		case 0:
2109 			err = configure_vertex(rv, inst);
2110 			scf_instance_destroy(inst);
2111 			switch (err) {
2112 			case 0:
2113 			case ECANCELED:
2114 				break;
2115 
2116 			case ECONNABORTED:
2117 				return (ECONNABORTED);
2118 
2119 			default:
2120 				bad_error("configure_vertex", err);
2121 			}
2122 			break;
2123 
2124 		case ECONNABORTED:
2125 			return (ECONNABORTED);
2126 
2127 		case ENOENT:
2128 			break;
2129 
2130 		case ENOTSUP:
2131 			/*
2132 			 * The fmri doesn't specify an instance - translate
2133 			 * to EINVAL.
2134 			 */
2135 			return (EINVAL);
2136 
2137 		case EINVAL:
2138 		default:
2139 			bad_error("libscf_fmri_get_instance", err);
2140 		}
2141 	}
2142 
2143 	return (0);
2144 }
2145 
2146 
2147 /*
2148  * Add all of the instances of the service named by fmri to the graph.
2149  * Returns
2150  *   0 - success
2151  *   ENOENT - service indicated by fmri does not exist
2152  *
2153  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2154  * otherwise.
2155  */
2156 static int
2157 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2158 {
2159 	scf_service_t *svc;
2160 	scf_instance_t *inst;
2161 	scf_iter_t *iter;
2162 	char *inst_fmri;
2163 	int ret, r;
2164 
2165 	*reboundp = B_FALSE;
2166 
2167 	svc = safe_scf_service_create(h);
2168 	inst = safe_scf_instance_create(h);
2169 	iter = safe_scf_iter_create(h);
2170 	inst_fmri = startd_alloc(max_scf_fmri_size);
2171 
2172 rebound:
2173 	if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2174 	    SCF_DECODE_FMRI_EXACT) != 0) {
2175 		switch (scf_error()) {
2176 		case SCF_ERROR_CONNECTION_BROKEN:
2177 		default:
2178 			libscf_handle_rebind(h);
2179 			*reboundp = B_TRUE;
2180 			goto rebound;
2181 
2182 		case SCF_ERROR_NOT_FOUND:
2183 			ret = ENOENT;
2184 			goto out;
2185 
2186 		case SCF_ERROR_INVALID_ARGUMENT:
2187 		case SCF_ERROR_CONSTRAINT_VIOLATED:
2188 		case SCF_ERROR_NOT_BOUND:
2189 		case SCF_ERROR_HANDLE_MISMATCH:
2190 			bad_error("scf_handle_decode_fmri", scf_error());
2191 		}
2192 	}
2193 
2194 	if (scf_iter_service_instances(iter, svc) != 0) {
2195 		switch (scf_error()) {
2196 		case SCF_ERROR_CONNECTION_BROKEN:
2197 		default:
2198 			libscf_handle_rebind(h);
2199 			*reboundp = B_TRUE;
2200 			goto rebound;
2201 
2202 		case SCF_ERROR_DELETED:
2203 			ret = ENOENT;
2204 			goto out;
2205 
2206 		case SCF_ERROR_HANDLE_MISMATCH:
2207 		case SCF_ERROR_NOT_BOUND:
2208 		case SCF_ERROR_NOT_SET:
2209 			bad_error("scf_iter_service_instances", scf_error());
2210 		}
2211 	}
2212 
2213 	for (;;) {
2214 		r = scf_iter_next_instance(iter, inst);
2215 		if (r == 0)
2216 			break;
2217 		if (r != 1) {
2218 			switch (scf_error()) {
2219 			case SCF_ERROR_CONNECTION_BROKEN:
2220 			default:
2221 				libscf_handle_rebind(h);
2222 				*reboundp = B_TRUE;
2223 				goto rebound;
2224 
2225 			case SCF_ERROR_DELETED:
2226 				ret = ENOENT;
2227 				goto out;
2228 
2229 			case SCF_ERROR_HANDLE_MISMATCH:
2230 			case SCF_ERROR_NOT_BOUND:
2231 			case SCF_ERROR_NOT_SET:
2232 			case SCF_ERROR_INVALID_ARGUMENT:
2233 				bad_error("scf_iter_next_instance",
2234 				    scf_error());
2235 			}
2236 		}
2237 
2238 		if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2239 		    0) {
2240 			switch (scf_error()) {
2241 			case SCF_ERROR_CONNECTION_BROKEN:
2242 				libscf_handle_rebind(h);
2243 				*reboundp = B_TRUE;
2244 				goto rebound;
2245 
2246 			case SCF_ERROR_DELETED:
2247 				continue;
2248 
2249 			case SCF_ERROR_NOT_BOUND:
2250 			case SCF_ERROR_NOT_SET:
2251 				bad_error("scf_instance_to_fmri", scf_error());
2252 			}
2253 		}
2254 
2255 		r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2256 		switch (r) {
2257 		case 0:
2258 		case ECANCELED:
2259 			break;
2260 
2261 		case EEXIST:
2262 			continue;
2263 
2264 		case ECONNABORTED:
2265 			libscf_handle_rebind(h);
2266 			*reboundp = B_TRUE;
2267 			goto rebound;
2268 
2269 		case EINVAL:
2270 		default:
2271 			bad_error("dgraph_add_instance", r);
2272 		}
2273 	}
2274 
2275 	ret = 0;
2276 
2277 out:
2278 	startd_free(inst_fmri, max_scf_fmri_size);
2279 	scf_iter_destroy(iter);
2280 	scf_instance_destroy(inst);
2281 	scf_service_destroy(svc);
2282 	return (ret);
2283 }
2284 
2285 struct depfmri_info {
2286 	graph_vertex_t	*v;		/* GVT_GROUP vertex */
2287 	gv_type_t	type;		/* type of dependency */
2288 	const char	*inst_fmri;	/* FMRI of parental GVT_INST vert. */
2289 	const char	*pg_name;	/* Name of dependency pg */
2290 	scf_handle_t	*h;
2291 	int		err;		/* return error code */
2292 	int		**pathp;	/* return circular dependency path */
2293 };
2294 
2295 /*
2296  * Find or create a vertex for fmri and make info->v depend on it.
2297  * Returns
2298  *   0 - success
2299  *   nonzero - failure
2300  *
2301  * On failure, sets info->err to
2302  *   EINVAL - fmri is invalid
2303  *	      fmri does not match info->type
2304  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2305  *	     will point to an array of the ids of the members of the cycle.
2306  *   ECONNABORTED - repository connection was broken
2307  *   ECONNRESET - succeeded, but repository connection was reset
2308  */
2309 static int
2310 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2311 {
2312 	int err;
2313 	graph_vertex_t *depgroup_v, *v;
2314 	char *fmri_copy, *cfmri;
2315 	size_t fmri_copy_sz;
2316 	const char *scope, *service, *instance, *pg;
2317 	scf_instance_t *inst;
2318 	boolean_t rebound;
2319 
2320 	assert(MUTEX_HELD(&dgraph_lock));
2321 
2322 	/* Get or create vertex for FMRI */
2323 	depgroup_v = info->v;
2324 
2325 	if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2326 		if (info->type != GVT_FILE) {
2327 			log_framework(LOG_NOTICE,
2328 			    "FMRI \"%s\" is not allowed for the \"%s\" "
2329 			    "dependency's type of instance %s.\n", fmri,
2330 			    info->pg_name, info->inst_fmri);
2331 			return (info->err = EINVAL);
2332 		}
2333 
2334 		err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2335 		    RERR_NONE, &v);
2336 		switch (err) {
2337 		case 0:
2338 			break;
2339 
2340 		case EEXIST:
2341 			assert(v->gv_type == GVT_FILE);
2342 			break;
2343 
2344 		case EINVAL:		/* prevented above */
2345 		default:
2346 			bad_error("graph_insert_vertex_unconfigured", err);
2347 		}
2348 	} else {
2349 		if (info->type != GVT_INST) {
2350 			log_framework(LOG_NOTICE,
2351 			    "FMRI \"%s\" is not allowed for the \"%s\" "
2352 			    "dependency's type of instance %s.\n", fmri,
2353 			    info->pg_name, info->inst_fmri);
2354 			return (info->err = EINVAL);
2355 		}
2356 
2357 		/*
2358 		 * We must canonify fmri & add a vertex for it.
2359 		 */
2360 		fmri_copy_sz = strlen(fmri) + 1;
2361 		fmri_copy = startd_alloc(fmri_copy_sz);
2362 		(void) strcpy(fmri_copy, fmri);
2363 
2364 		/* Determine if the FMRI is a property group or instance */
2365 		if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2366 		    &instance, &pg, NULL) != 0) {
2367 			startd_free(fmri_copy, fmri_copy_sz);
2368 			log_framework(LOG_NOTICE,
2369 			    "Dependency \"%s\" of %s has invalid FMRI "
2370 			    "\"%s\".\n", info->pg_name, info->inst_fmri,
2371 			    fmri);
2372 			return (info->err = EINVAL);
2373 		}
2374 
2375 		if (service == NULL || pg != NULL) {
2376 			startd_free(fmri_copy, fmri_copy_sz);
2377 			log_framework(LOG_NOTICE,
2378 			    "Dependency \"%s\" of %s does not designate a "
2379 			    "service or instance.\n", info->pg_name,
2380 			    info->inst_fmri);
2381 			return (info->err = EINVAL);
2382 		}
2383 
2384 		if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2385 			cfmri = uu_msprintf("svc:/%s%s%s",
2386 			    service, instance ? ":" : "", instance ? instance :
2387 			    "");
2388 		} else {
2389 			cfmri = uu_msprintf("svc://%s/%s%s%s",
2390 			    scope, service, instance ? ":" : "", instance ?
2391 			    instance : "");
2392 		}
2393 
2394 		startd_free(fmri_copy, fmri_copy_sz);
2395 
2396 		err = graph_insert_vertex_unconfigured(cfmri, instance ?
2397 		    GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2398 		    RERR_NONE, &v);
2399 		uu_free(cfmri);
2400 		switch (err) {
2401 		case 0:
2402 			break;
2403 
2404 		case EEXIST:
2405 			/* Verify v. */
2406 			if (instance != NULL)
2407 				assert(v->gv_type == GVT_INST);
2408 			else
2409 				assert(v->gv_type == GVT_SVC);
2410 			break;
2411 
2412 		default:
2413 			bad_error("graph_insert_vertex_unconfigured", err);
2414 		}
2415 	}
2416 
2417 	/* Add dependency from depgroup_v to new vertex */
2418 	info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2419 	switch (info->err) {
2420 	case 0:
2421 		break;
2422 
2423 	case ELOOP:
2424 		return (ELOOP);
2425 
2426 	default:
2427 		bad_error("graph_insert_dependency", info->err);
2428 	}
2429 
2430 	/* This must be after we insert the dependency, to avoid looping. */
2431 	switch (v->gv_type) {
2432 	case GVT_INST:
2433 		if ((v->gv_flags & GV_CONFIGURED) != 0)
2434 			break;
2435 
2436 		inst = safe_scf_instance_create(info->h);
2437 
2438 		rebound = B_FALSE;
2439 
2440 rebound:
2441 		err = libscf_lookup_instance(v->gv_name, inst);
2442 		switch (err) {
2443 		case 0:
2444 			err = configure_vertex(v, inst);
2445 			switch (err) {
2446 			case 0:
2447 			case ECANCELED:
2448 				break;
2449 
2450 			case ECONNABORTED:
2451 				libscf_handle_rebind(info->h);
2452 				rebound = B_TRUE;
2453 				goto rebound;
2454 
2455 			default:
2456 				bad_error("configure_vertex", err);
2457 			}
2458 			break;
2459 
2460 		case ENOENT:
2461 			break;
2462 
2463 		case ECONNABORTED:
2464 			libscf_handle_rebind(info->h);
2465 			rebound = B_TRUE;
2466 			goto rebound;
2467 
2468 		case EINVAL:
2469 		case ENOTSUP:
2470 		default:
2471 			bad_error("libscf_fmri_get_instance", err);
2472 		}
2473 
2474 		scf_instance_destroy(inst);
2475 
2476 		if (rebound)
2477 			return (info->err = ECONNRESET);
2478 		break;
2479 
2480 	case GVT_SVC:
2481 		(void) add_service(v->gv_name, info->h, &rebound);
2482 		if (rebound)
2483 			return (info->err = ECONNRESET);
2484 	}
2485 
2486 	return (0);
2487 }
2488 
2489 struct deppg_info {
2490 	graph_vertex_t	*v;		/* GVT_INST vertex */
2491 	int		err;		/* return error */
2492 	int		**pathp;	/* return circular dependency path */
2493 };
2494 
2495 /*
2496  * Make info->v depend on a new GVT_GROUP node for this property group,
2497  * and then call process_dependency_fmri() for the values of the entity
2498  * property.  Return 0 on success, or if something goes wrong return nonzero
2499  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2500  * process_dependency_fmri().
2501  */
2502 static int
2503 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2504 {
2505 	scf_handle_t *h;
2506 	depgroup_type_t deptype;
2507 	restarter_error_t rerr;
2508 	struct depfmri_info linfo;
2509 	char *fmri, *pg_name;
2510 	size_t fmri_sz;
2511 	graph_vertex_t *depgrp;
2512 	scf_property_t *prop;
2513 	int err;
2514 	int empty;
2515 	scf_error_t scferr;
2516 	ssize_t len;
2517 
2518 	assert(MUTEX_HELD(&dgraph_lock));
2519 
2520 	h = scf_pg_handle(pg);
2521 
2522 	pg_name = startd_alloc(max_scf_name_size);
2523 
2524 	len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2525 	if (len < 0) {
2526 		startd_free(pg_name, max_scf_name_size);
2527 		switch (scf_error()) {
2528 		case SCF_ERROR_CONNECTION_BROKEN:
2529 		default:
2530 			return (info->err = ECONNABORTED);
2531 
2532 		case SCF_ERROR_DELETED:
2533 			return (info->err = 0);
2534 
2535 		case SCF_ERROR_NOT_SET:
2536 			bad_error("scf_pg_get_name", scf_error());
2537 		}
2538 	}
2539 
2540 	/*
2541 	 * Skip over empty dependency groups.  Since dependency property
2542 	 * groups are updated atomically, they are either empty or
2543 	 * fully populated.
2544 	 */
2545 	empty = depgroup_empty(h, pg);
2546 	if (empty < 0) {
2547 		log_error(LOG_INFO,
2548 		    "Error reading dependency group \"%s\" of %s: %s\n",
2549 		    pg_name, info->v->gv_name, scf_strerror(scf_error()));
2550 		startd_free(pg_name, max_scf_name_size);
2551 		return (info->err = EINVAL);
2552 
2553 	} else if (empty == 1) {
2554 		log_framework(LOG_DEBUG,
2555 		    "Ignoring empty dependency group \"%s\" of %s\n",
2556 		    pg_name, info->v->gv_name);
2557 		startd_free(pg_name, max_scf_name_size);
2558 		return (info->err = 0);
2559 	}
2560 
2561 	fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2562 	fmri = startd_alloc(fmri_sz);
2563 
2564 	(void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2565 	    pg_name);
2566 
2567 	/* Validate the pg before modifying the graph */
2568 	deptype = depgroup_read_grouping(h, pg);
2569 	if (deptype == DEPGRP_UNSUPPORTED) {
2570 		log_error(LOG_INFO,
2571 		    "Dependency \"%s\" of %s has an unknown grouping value.\n",
2572 		    pg_name, info->v->gv_name);
2573 		startd_free(fmri, fmri_sz);
2574 		startd_free(pg_name, max_scf_name_size);
2575 		return (info->err = EINVAL);
2576 	}
2577 
2578 	rerr = depgroup_read_restart(h, pg);
2579 	if (rerr == RERR_UNSUPPORTED) {
2580 		log_error(LOG_INFO,
2581 		    "Dependency \"%s\" of %s has an unknown restart_on value."
2582 		    "\n", pg_name, info->v->gv_name);
2583 		startd_free(fmri, fmri_sz);
2584 		startd_free(pg_name, max_scf_name_size);
2585 		return (info->err = EINVAL);
2586 	}
2587 
2588 	prop = safe_scf_property_create(h);
2589 
2590 	if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2591 		scferr = scf_error();
2592 		scf_property_destroy(prop);
2593 		if (scferr == SCF_ERROR_DELETED) {
2594 			startd_free(fmri, fmri_sz);
2595 			startd_free(pg_name, max_scf_name_size);
2596 			return (info->err = 0);
2597 		} else if (scferr != SCF_ERROR_NOT_FOUND) {
2598 			startd_free(fmri, fmri_sz);
2599 			startd_free(pg_name, max_scf_name_size);
2600 			return (info->err = ECONNABORTED);
2601 		}
2602 
2603 		log_error(LOG_INFO,
2604 		    "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2605 		    pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2606 
2607 		startd_free(fmri, fmri_sz);
2608 		startd_free(pg_name, max_scf_name_size);
2609 
2610 		return (info->err = EINVAL);
2611 	}
2612 
2613 	/* Create depgroup vertex for pg */
2614 	err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2615 	    rerr, &depgrp);
2616 	assert(err == 0);
2617 	startd_free(fmri, fmri_sz);
2618 
2619 	/* Add dependency from inst vertex to new vertex */
2620 	err = graph_insert_dependency(info->v, depgrp, info->pathp);
2621 	/* ELOOP can't happen because this should be a new vertex */
2622 	assert(err == 0);
2623 
2624 	linfo.v = depgrp;
2625 	linfo.type = depgroup_read_scheme(h, pg);
2626 	linfo.inst_fmri = info->v->gv_name;
2627 	linfo.pg_name = pg_name;
2628 	linfo.h = h;
2629 	linfo.err = 0;
2630 	linfo.pathp = info->pathp;
2631 	err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2632 	    &linfo);
2633 
2634 	scf_property_destroy(prop);
2635 	startd_free(pg_name, max_scf_name_size);
2636 
2637 	switch (err) {
2638 	case 0:
2639 	case EINTR:
2640 		return (info->err = linfo.err);
2641 
2642 	case ECONNABORTED:
2643 	case EINVAL:
2644 		return (info->err = err);
2645 
2646 	case ECANCELED:
2647 		return (info->err = 0);
2648 
2649 	case ECONNRESET:
2650 		return (info->err = ECONNABORTED);
2651 
2652 	default:
2653 		bad_error("walk_property_astrings", err);
2654 		/* NOTREACHED */
2655 	}
2656 }
2657 
2658 /*
2659  * Build the dependency info for v from the repository.  Returns 0 on success,
2660  * ECONNABORTED on repository disconnection, EINVAL if the repository
2661  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2662  * In the last case, *pathp will point to a -1-terminated array of ids which
2663  * constitute the rest of the dependency cycle.
2664  */
2665 static int
2666 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2667 {
2668 	struct deppg_info info;
2669 	int err;
2670 	uint_t old_configured;
2671 
2672 	assert(MUTEX_HELD(&dgraph_lock));
2673 
2674 	/*
2675 	 * Mark the vertex as configured during dependency insertion to avoid
2676 	 * dependency cycles (which can appear in the graph if one of the
2677 	 * vertices is an exclusion-group).
2678 	 */
2679 	old_configured = v->gv_flags & GV_CONFIGURED;
2680 	v->gv_flags |= GV_CONFIGURED;
2681 
2682 	info.err = 0;
2683 	info.v = v;
2684 	info.pathp = pathp;
2685 
2686 	err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2687 	    &info);
2688 
2689 	if (!old_configured)
2690 		v->gv_flags &= ~GV_CONFIGURED;
2691 
2692 	switch (err) {
2693 	case 0:
2694 	case EINTR:
2695 		return (info.err);
2696 
2697 	case ECONNABORTED:
2698 		return (ECONNABORTED);
2699 
2700 	case ECANCELED:
2701 		/* Should get delete event, so return 0. */
2702 		return (0);
2703 
2704 	default:
2705 		bad_error("walk_dependency_pgs", err);
2706 		/* NOTREACHED */
2707 	}
2708 }
2709 
2710 
2711 static void
2712 handle_cycle(const char *fmri, int *path)
2713 {
2714 	const char *cp;
2715 	size_t sz;
2716 
2717 	assert(MUTEX_HELD(&dgraph_lock));
2718 
2719 	path_to_str(path, (char **)&cp, &sz);
2720 
2721 	log_error(LOG_ERR, "Transitioning %s to maintenance "
2722 	    "because it completes a dependency cycle (see svcs -xv for "
2723 	    "details):\n%s", fmri ? fmri : "?", cp);
2724 
2725 	startd_free((void *)cp, sz);
2726 }
2727 
2728 /*
2729  * Increment the vertex's reference count to prevent the vertex removal
2730  * from the dgraph.
2731  */
2732 static void
2733 vertex_ref(graph_vertex_t *v)
2734 {
2735 	assert(MUTEX_HELD(&dgraph_lock));
2736 
2737 	v->gv_refs++;
2738 }
2739 
2740 /*
2741  * Decrement the vertex's reference count and remove the vertex from
2742  * the dgraph when possible.
2743  *
2744  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2745  * return VERTEX_INUSE.
2746  */
2747 static int
2748 vertex_unref(graph_vertex_t *v)
2749 {
2750 	assert(MUTEX_HELD(&dgraph_lock));
2751 	assert(v->gv_refs > 0);
2752 
2753 	v->gv_refs--;
2754 
2755 	return (free_if_unrefed(v));
2756 }
2757 
2758 /*
2759  * When run on the dependencies of a vertex, populates list with
2760  * graph_edge_t's which point to the service vertices or the instance
2761  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2762  *
2763  * Increment the vertex's reference count once the vertex is inserted
2764  * in the list. The vertex won't be able to be deleted from the dgraph
2765  * while it is referenced.
2766  */
2767 static int
2768 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2769 {
2770 	graph_vertex_t *v = e->ge_vertex;
2771 	graph_edge_t *new;
2772 	int r;
2773 
2774 	switch (v->gv_type) {
2775 	case GVT_INST:
2776 	case GVT_SVC:
2777 		break;
2778 
2779 	case GVT_GROUP:
2780 		r = uu_list_walk(v->gv_dependencies,
2781 		    (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2782 		assert(r == 0);
2783 		return (UU_WALK_NEXT);
2784 
2785 	case GVT_FILE:
2786 		return (UU_WALK_NEXT);
2787 
2788 	default:
2789 #ifndef NDEBUG
2790 		uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2791 		    __LINE__, v->gv_type);
2792 #endif
2793 		abort();
2794 	}
2795 
2796 	new = startd_alloc(sizeof (*new));
2797 	new->ge_vertex = v;
2798 	uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2799 	r = uu_list_insert_before(list, NULL, new);
2800 	assert(r == 0);
2801 
2802 	/*
2803 	 * Because we are inserting the vertex in a list, we don't want
2804 	 * the vertex to be freed while the list is in use. In order to
2805 	 * achieve that, increment the vertex's reference count.
2806 	 */
2807 	vertex_ref(v);
2808 
2809 	return (UU_WALK_NEXT);
2810 }
2811 
2812 static boolean_t
2813 should_be_in_subgraph(graph_vertex_t *v)
2814 {
2815 	graph_edge_t *e;
2816 
2817 	if (v == milestone)
2818 		return (B_TRUE);
2819 
2820 	/*
2821 	 * v is in the subgraph if any of its dependents are in the subgraph.
2822 	 * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2823 	 * count if we're enabled.
2824 	 */
2825 	for (e = uu_list_first(v->gv_dependents);
2826 	    e != NULL;
2827 	    e = uu_list_next(v->gv_dependents, e)) {
2828 		graph_vertex_t *dv = e->ge_vertex;
2829 
2830 		if (!(dv->gv_flags & GV_INSUBGRAPH))
2831 			continue;
2832 
2833 		/*
2834 		 * Don't include instances that are optional and disabled.
2835 		 */
2836 		if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2837 
2838 			int in = 0;
2839 			graph_edge_t *ee;
2840 
2841 			for (ee = uu_list_first(dv->gv_dependents);
2842 			    ee != NULL;
2843 			    ee = uu_list_next(dv->gv_dependents, ee)) {
2844 
2845 				graph_vertex_t *ddv = e->ge_vertex;
2846 
2847 				if (ddv->gv_type == GVT_GROUP &&
2848 				    ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2849 					continue;
2850 
2851 				if (ddv->gv_type == GVT_GROUP &&
2852 				    ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2853 				    !(v->gv_flags & GV_ENBLD_NOOVR))
2854 					continue;
2855 
2856 				in = 1;
2857 			}
2858 			if (!in)
2859 				continue;
2860 		}
2861 		if (v->gv_type == GVT_INST &&
2862 		    dv->gv_type == GVT_GROUP &&
2863 		    dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2864 		    !(v->gv_flags & GV_ENBLD_NOOVR))
2865 			continue;
2866 
2867 		/* Don't include excluded services and instances */
2868 		if (dv->gv_type == GVT_GROUP &&
2869 		    dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2870 			continue;
2871 
2872 		return (B_TRUE);
2873 	}
2874 
2875 	return (B_FALSE);
2876 }
2877 
2878 /*
2879  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2880  * any bits change, manipulate the repository appropriately.  Returns 0 or
2881  * ECONNABORTED.
2882  */
2883 static int
2884 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2885 {
2886 	boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2887 	boolean_t new;
2888 	graph_edge_t *e;
2889 	scf_instance_t *inst;
2890 	int ret = 0, r;
2891 
2892 	assert(milestone != NULL && milestone != MILESTONE_NONE);
2893 
2894 	new = should_be_in_subgraph(v);
2895 
2896 	if (new == old)
2897 		return (0);
2898 
2899 	log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2900 	    "Removing %s from the subgraph.\n", v->gv_name);
2901 
2902 	v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2903 	    (new ? GV_INSUBGRAPH : 0);
2904 
2905 	if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2906 		int err;
2907 
2908 get_inst:
2909 		err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2910 		if (err != 0) {
2911 			switch (err) {
2912 			case ECONNABORTED:
2913 				libscf_handle_rebind(h);
2914 				ret = ECONNABORTED;
2915 				goto get_inst;
2916 
2917 			case ENOENT:
2918 				break;
2919 
2920 			case EINVAL:
2921 			case ENOTSUP:
2922 			default:
2923 				bad_error("libscf_fmri_get_instance", err);
2924 			}
2925 		} else {
2926 			const char *f;
2927 
2928 			if (new) {
2929 				err = libscf_delete_enable_ovr(inst);
2930 				f = "libscf_delete_enable_ovr";
2931 			} else {
2932 				err = libscf_set_enable_ovr(inst, 0);
2933 				f = "libscf_set_enable_ovr";
2934 			}
2935 			scf_instance_destroy(inst);
2936 			switch (err) {
2937 			case 0:
2938 			case ECANCELED:
2939 				break;
2940 
2941 			case ECONNABORTED:
2942 				libscf_handle_rebind(h);
2943 				/*
2944 				 * We must continue so the graph is updated,
2945 				 * but we must return ECONNABORTED so any
2946 				 * libscf state held by any callers is reset.
2947 				 */
2948 				ret = ECONNABORTED;
2949 				goto get_inst;
2950 
2951 			case EROFS:
2952 			case EPERM:
2953 				log_error(LOG_WARNING,
2954 				    "Could not set %s/%s for %s: %s.\n",
2955 				    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2956 				    v->gv_name, strerror(err));
2957 				break;
2958 
2959 			default:
2960 				bad_error(f, err);
2961 			}
2962 		}
2963 	}
2964 
2965 	for (e = uu_list_first(v->gv_dependencies);
2966 	    e != NULL;
2967 	    e = uu_list_next(v->gv_dependencies, e)) {
2968 		r = eval_subgraph(e->ge_vertex, h);
2969 		if (r != 0) {
2970 			assert(r == ECONNABORTED);
2971 			ret = ECONNABORTED;
2972 		}
2973 	}
2974 
2975 	return (ret);
2976 }
2977 
2978 /*
2979  * Delete the (property group) dependencies of v & create new ones based on
2980  * inst.  If doing so would create a cycle, log a message and put the instance
2981  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2982  * ECONNABORTED.
2983  */
2984 int
2985 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2986 {
2987 	int err;
2988 	int *path;
2989 	char *fmri;
2990 	int r;
2991 	scf_handle_t *h = scf_instance_handle(inst);
2992 	uu_list_t *old_deps;
2993 	int ret = 0;
2994 	graph_edge_t *e;
2995 	graph_vertex_t *vv;
2996 
2997 	assert(MUTEX_HELD(&dgraph_lock));
2998 	assert(v->gv_type == GVT_INST);
2999 
3000 	log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3001 
3002 	if (milestone > MILESTONE_NONE) {
3003 		/*
3004 		 * In case some of v's dependencies are being deleted we must
3005 		 * make a list of them now for GV_INSUBGRAPH-flag evaluation
3006 		 * after the new dependencies are in place.
3007 		 */
3008 		old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3009 
3010 		err = uu_list_walk(v->gv_dependencies,
3011 		    (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3012 		assert(err == 0);
3013 	}
3014 
3015 	delete_instance_dependencies(v, B_FALSE);
3016 
3017 	err = set_dependencies(v, inst, &path);
3018 	switch (err) {
3019 	case 0:
3020 		break;
3021 
3022 	case ECONNABORTED:
3023 		ret = err;
3024 		goto out;
3025 
3026 	case EINVAL:
3027 	case ELOOP:
3028 		r = libscf_instance_get_fmri(inst, &fmri);
3029 		switch (r) {
3030 		case 0:
3031 			break;
3032 
3033 		case ECONNABORTED:
3034 			ret = ECONNABORTED;
3035 			goto out;
3036 
3037 		case ECANCELED:
3038 			ret = 0;
3039 			goto out;
3040 
3041 		default:
3042 			bad_error("libscf_instance_get_fmri", r);
3043 		}
3044 
3045 		if (err == EINVAL) {
3046 			log_error(LOG_ERR, "Transitioning %s "
3047 			    "to maintenance due to misconfiguration.\n",
3048 			    fmri ? fmri : "?");
3049 			vertex_send_event(v,
3050 			    RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3051 		} else {
3052 			handle_cycle(fmri, path);
3053 			vertex_send_event(v,
3054 			    RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3055 		}
3056 		startd_free(fmri, max_scf_fmri_size);
3057 		ret = 0;
3058 		goto out;
3059 
3060 	default:
3061 		bad_error("set_dependencies", err);
3062 	}
3063 
3064 	if (milestone > MILESTONE_NONE) {
3065 		boolean_t aborted = B_FALSE;
3066 
3067 		for (e = uu_list_first(old_deps);
3068 		    e != NULL;
3069 		    e = uu_list_next(old_deps, e)) {
3070 			vv = e->ge_vertex;
3071 
3072 			if (vertex_unref(vv) == VERTEX_INUSE &&
3073 			    eval_subgraph(vv, h) == ECONNABORTED)
3074 				aborted = B_TRUE;
3075 		}
3076 
3077 		for (e = uu_list_first(v->gv_dependencies);
3078 		    e != NULL;
3079 		    e = uu_list_next(v->gv_dependencies, e)) {
3080 			if (eval_subgraph(e->ge_vertex, h) ==
3081 			    ECONNABORTED)
3082 				aborted = B_TRUE;
3083 		}
3084 
3085 		if (aborted) {
3086 			ret = ECONNABORTED;
3087 			goto out;
3088 		}
3089 	}
3090 
3091 	graph_start_if_satisfied(v);
3092 
3093 	ret = 0;
3094 
3095 out:
3096 	if (milestone > MILESTONE_NONE) {
3097 		void *cookie = NULL;
3098 
3099 		while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3100 			startd_free(e, sizeof (*e));
3101 
3102 		uu_list_destroy(old_deps);
3103 	}
3104 
3105 	return (ret);
3106 }
3107 
3108 /*
3109  * Set up v according to inst.  That is, make sure it depends on its
3110  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3111  * the restarter, and send ENABLE or DISABLE as appropriate.
3112  *
3113  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3114  * ECANCELED if inst is deleted.
3115  */
3116 static int
3117 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3118 {
3119 	scf_handle_t *h;
3120 	scf_propertygroup_t *pg;
3121 	scf_snapshot_t *snap;
3122 	char *restarter_fmri = startd_alloc(max_scf_value_size);
3123 	int enabled, enabled_ovr;
3124 	int err;
3125 	int *path;
3126 	int deathrow;
3127 	int32_t tset;
3128 
3129 	restarter_fmri[0] = '\0';
3130 
3131 	assert(MUTEX_HELD(&dgraph_lock));
3132 	assert(v->gv_type == GVT_INST);
3133 	assert((v->gv_flags & GV_CONFIGURED) == 0);
3134 
3135 	/* GV_INSUBGRAPH should already be set properly. */
3136 	assert(should_be_in_subgraph(v) ==
3137 	    ((v->gv_flags & GV_INSUBGRAPH) != 0));
3138 
3139 	/*
3140 	 * If the instance fmri is in the deathrow list then set the
3141 	 * GV_DEATHROW flag on the vertex and create and set to true the
3142 	 * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3143 	 * repository for this instance fmri.
3144 	 */
3145 	if ((v->gv_flags & GV_DEATHROW) ||
3146 	    (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3147 		if ((v->gv_flags & GV_DEATHROW) == 0) {
3148 			/*
3149 			 * Set flag GV_DEATHROW, create and set to true
3150 			 * the SCF_PROPERTY_DEATHROW property in the
3151 			 * non-persistent repository for this instance fmri.
3152 			 */
3153 			v->gv_flags |= GV_DEATHROW;
3154 
3155 			switch (err = libscf_set_deathrow(inst, 1)) {
3156 			case 0:
3157 				break;
3158 
3159 			case ECONNABORTED:
3160 			case ECANCELED:
3161 				startd_free(restarter_fmri, max_scf_value_size);
3162 				return (err);
3163 
3164 			case EROFS:
3165 				log_error(LOG_WARNING, "Could not set %s/%s "
3166 				    "for deathrow %s: %s.\n",
3167 				    SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3168 				    v->gv_name, strerror(err));
3169 				break;
3170 
3171 			case EPERM:
3172 				uu_die("Permission denied.\n");
3173 				/* NOTREACHED */
3174 
3175 			default:
3176 				bad_error("libscf_set_deathrow", err);
3177 			}
3178 			log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3179 			    v->gv_name);
3180 		}
3181 		startd_free(restarter_fmri, max_scf_value_size);
3182 		return (0);
3183 	}
3184 
3185 	h = scf_instance_handle(inst);
3186 
3187 	/*
3188 	 * Using a temporary deathrow boolean property, set through
3189 	 * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3190 	 * because deathrow_fini() may already have been called, and in case
3191 	 * of a refresh, GV_DEATHROW may need to be set again.
3192 	 * libscf_get_deathrow() sets deathrow to 1 only if this instance
3193 	 * has a temporary boolean property named 'deathrow' valued true
3194 	 * in a property group 'deathrow', -1 or 0 in all other cases.
3195 	 */
3196 	err = libscf_get_deathrow(h, inst, &deathrow);
3197 	switch (err) {
3198 	case 0:
3199 		break;
3200 
3201 	case ECONNABORTED:
3202 	case ECANCELED:
3203 		startd_free(restarter_fmri, max_scf_value_size);
3204 		return (err);
3205 
3206 	default:
3207 		bad_error("libscf_get_deathrow", err);
3208 	}
3209 
3210 	if (deathrow == 1) {
3211 		v->gv_flags |= GV_DEATHROW;
3212 		startd_free(restarter_fmri, max_scf_value_size);
3213 		return (0);
3214 	}
3215 
3216 	log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3217 
3218 	/*
3219 	 * If the instance does not have a restarter property group,
3220 	 * initialize its state to uninitialized/none, in case the restarter
3221 	 * is not enabled.
3222 	 */
3223 	pg = safe_scf_pg_create(h);
3224 
3225 	if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3226 		instance_data_t idata;
3227 		uint_t count = 0, msecs = ALLOC_DELAY;
3228 
3229 		switch (scf_error()) {
3230 		case SCF_ERROR_NOT_FOUND:
3231 			break;
3232 
3233 		case SCF_ERROR_CONNECTION_BROKEN:
3234 		default:
3235 			scf_pg_destroy(pg);
3236 			startd_free(restarter_fmri, max_scf_value_size);
3237 			return (ECONNABORTED);
3238 
3239 		case SCF_ERROR_DELETED:
3240 			scf_pg_destroy(pg);
3241 			startd_free(restarter_fmri, max_scf_value_size);
3242 			return (ECANCELED);
3243 
3244 		case SCF_ERROR_NOT_SET:
3245 			bad_error("scf_instance_get_pg", scf_error());
3246 		}
3247 
3248 		switch (err = libscf_instance_get_fmri(inst,
3249 		    (char **)&idata.i_fmri)) {
3250 		case 0:
3251 			break;
3252 
3253 		case ECONNABORTED:
3254 		case ECANCELED:
3255 			scf_pg_destroy(pg);
3256 			startd_free(restarter_fmri, max_scf_value_size);
3257 			return (err);
3258 
3259 		default:
3260 			bad_error("libscf_instance_get_fmri", err);
3261 		}
3262 
3263 		idata.i_state = RESTARTER_STATE_NONE;
3264 		idata.i_next_state = RESTARTER_STATE_NONE;
3265 
3266 init_state:
3267 		switch (err = _restarter_commit_states(h, &idata,
3268 		    RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3269 		    restarter_get_str_short(restarter_str_insert_in_graph))) {
3270 		case 0:
3271 			break;
3272 
3273 		case ENOMEM:
3274 			++count;
3275 			if (count < ALLOC_RETRY) {
3276 				(void) poll(NULL, 0, msecs);
3277 				msecs *= ALLOC_DELAY_MULT;
3278 				goto init_state;
3279 			}
3280 
3281 			uu_die("Insufficient memory.\n");
3282 			/* NOTREACHED */
3283 
3284 		case ECONNABORTED:
3285 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3286 			scf_pg_destroy(pg);
3287 			startd_free(restarter_fmri, max_scf_value_size);
3288 			return (ECONNABORTED);
3289 
3290 		case ENOENT:
3291 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3292 			scf_pg_destroy(pg);
3293 			startd_free(restarter_fmri, max_scf_value_size);
3294 			return (ECANCELED);
3295 
3296 		case EPERM:
3297 		case EACCES:
3298 		case EROFS:
3299 			log_error(LOG_NOTICE, "Could not initialize state for "
3300 			    "%s: %s.\n", idata.i_fmri, strerror(err));
3301 			break;
3302 
3303 		case EINVAL:
3304 		default:
3305 			bad_error("_restarter_commit_states", err);
3306 		}
3307 
3308 		startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3309 	}
3310 
3311 	scf_pg_destroy(pg);
3312 
3313 	if (milestone != NULL) {
3314 		/*
3315 		 * Make sure the enable-override is set properly before we
3316 		 * read whether we should be enabled.
3317 		 */
3318 		if (milestone == MILESTONE_NONE ||
3319 		    !(v->gv_flags & GV_INSUBGRAPH)) {
3320 			/*
3321 			 * This might seem unjustified after the milestone
3322 			 * transition has completed (non_subgraph_svcs == 0),
3323 			 * but it's important because when we boot to
3324 			 * a milestone, we set the milestone before populating
3325 			 * the graph, and all of the new non-subgraph services
3326 			 * need to be disabled here.
3327 			 */
3328 			switch (err = libscf_set_enable_ovr(inst, 0)) {
3329 			case 0:
3330 				break;
3331 
3332 			case ECONNABORTED:
3333 			case ECANCELED:
3334 				startd_free(restarter_fmri, max_scf_value_size);
3335 				return (err);
3336 
3337 			case EROFS:
3338 				log_error(LOG_WARNING,
3339 				    "Could not set %s/%s for %s: %s.\n",
3340 				    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3341 				    v->gv_name, strerror(err));
3342 				break;
3343 
3344 			case EPERM:
3345 				uu_die("Permission denied.\n");
3346 				/* NOTREACHED */
3347 
3348 			default:
3349 				bad_error("libscf_set_enable_ovr", err);
3350 			}
3351 		} else {
3352 			assert(v->gv_flags & GV_INSUBGRAPH);
3353 			switch (err = libscf_delete_enable_ovr(inst)) {
3354 			case 0:
3355 				break;
3356 
3357 			case ECONNABORTED:
3358 			case ECANCELED:
3359 				startd_free(restarter_fmri, max_scf_value_size);
3360 				return (err);
3361 
3362 			case EPERM:
3363 				uu_die("Permission denied.\n");
3364 				/* NOTREACHED */
3365 
3366 			default:
3367 				bad_error("libscf_delete_enable_ovr", err);
3368 			}
3369 		}
3370 	}
3371 
3372 	err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3373 	    &enabled_ovr, &restarter_fmri);
3374 	switch (err) {
3375 	case 0:
3376 		break;
3377 
3378 	case ECONNABORTED:
3379 	case ECANCELED:
3380 		startd_free(restarter_fmri, max_scf_value_size);
3381 		return (err);
3382 
3383 	case ENOENT:
3384 		log_framework(LOG_DEBUG,
3385 		    "Ignoring %s because it has no general property group.\n",
3386 		    v->gv_name);
3387 		startd_free(restarter_fmri, max_scf_value_size);
3388 		return (0);
3389 
3390 	default:
3391 		bad_error("libscf_get_basic_instance_data", err);
3392 	}
3393 
3394 	if ((tset = libscf_get_stn_tset(inst)) == -1) {
3395 		log_framework(LOG_WARNING,
3396 		    "Failed to get notification parameters for %s: %s\n",
3397 		    v->gv_name, scf_strerror(scf_error()));
3398 		v->gv_stn_tset = 0;
3399 	} else {
3400 		v->gv_stn_tset = tset;
3401 	}
3402 	if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3403 		stn_global = v->gv_stn_tset;
3404 
3405 	if (enabled == -1) {
3406 		startd_free(restarter_fmri, max_scf_value_size);
3407 		return (0);
3408 	}
3409 
3410 	v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3411 	    (enabled ? GV_ENBLD_NOOVR : 0);
3412 
3413 	if (enabled_ovr != -1)
3414 		enabled = enabled_ovr;
3415 
3416 	v->gv_state = RESTARTER_STATE_UNINIT;
3417 
3418 	snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3419 	scf_snapshot_destroy(snap);
3420 
3421 	/* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3422 	err = graph_change_restarter(v, restarter_fmri, h, &path);
3423 	if (err != 0) {
3424 		instance_data_t idata;
3425 		uint_t count = 0, msecs = ALLOC_DELAY;
3426 		restarter_str_t reason;
3427 
3428 		if (err == ECONNABORTED) {
3429 			startd_free(restarter_fmri, max_scf_value_size);
3430 			return (err);
3431 		}
3432 
3433 		assert(err == EINVAL || err == ELOOP);
3434 
3435 		if (err == EINVAL) {
3436 			log_framework(LOG_ERR, emsg_invalid_restarter,
3437 			    v->gv_name, restarter_fmri);
3438 			reason = restarter_str_invalid_restarter;
3439 		} else {
3440 			handle_cycle(v->gv_name, path);
3441 			reason = restarter_str_dependency_cycle;
3442 		}
3443 
3444 		startd_free(restarter_fmri, max_scf_value_size);
3445 
3446 		/*
3447 		 * We didn't register the instance with the restarter, so we
3448 		 * must set maintenance mode ourselves.
3449 		 */
3450 		err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3451 		if (err != 0) {
3452 			assert(err == ECONNABORTED || err == ECANCELED);
3453 			return (err);
3454 		}
3455 
3456 		idata.i_state = RESTARTER_STATE_NONE;
3457 		idata.i_next_state = RESTARTER_STATE_NONE;
3458 
3459 set_maint:
3460 		switch (err = _restarter_commit_states(h, &idata,
3461 		    RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3462 		    restarter_get_str_short(reason))) {
3463 		case 0:
3464 			break;
3465 
3466 		case ENOMEM:
3467 			++count;
3468 			if (count < ALLOC_RETRY) {
3469 				(void) poll(NULL, 0, msecs);
3470 				msecs *= ALLOC_DELAY_MULT;
3471 				goto set_maint;
3472 			}
3473 
3474 			uu_die("Insufficient memory.\n");
3475 			/* NOTREACHED */
3476 
3477 		case ECONNABORTED:
3478 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3479 			return (ECONNABORTED);
3480 
3481 		case ENOENT:
3482 			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3483 			return (ECANCELED);
3484 
3485 		case EPERM:
3486 		case EACCES:
3487 		case EROFS:
3488 			log_error(LOG_NOTICE, "Could not initialize state for "
3489 			    "%s: %s.\n", idata.i_fmri, strerror(err));
3490 			break;
3491 
3492 		case EINVAL:
3493 		default:
3494 			bad_error("_restarter_commit_states", err);
3495 		}
3496 
3497 		startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3498 
3499 		v->gv_state = RESTARTER_STATE_MAINT;
3500 
3501 		goto out;
3502 	}
3503 	startd_free(restarter_fmri, max_scf_value_size);
3504 
3505 	/* Add all the other dependencies. */
3506 	err = refresh_vertex(v, inst);
3507 	if (err != 0) {
3508 		assert(err == ECONNABORTED);
3509 		return (err);
3510 	}
3511 
3512 out:
3513 	v->gv_flags |= GV_CONFIGURED;
3514 
3515 	graph_enable_by_vertex(v, enabled, 0);
3516 
3517 	return (0);
3518 }
3519 
3520 
3521 static void
3522 kill_user_procs(void)
3523 {
3524 	(void) fputs("svc.startd: Killing user processes.\n", stdout);
3525 
3526 	/*
3527 	 * Despite its name, killall's role is to get select user processes--
3528 	 * basically those representing terminal-based logins-- to die.  Victims
3529 	 * are located by killall in the utmp database.  Since these are most
3530 	 * often shell based logins, and many shells mask SIGTERM (but are
3531 	 * responsive to SIGHUP) we first HUP and then shortly thereafter
3532 	 * kill -9.
3533 	 */
3534 	(void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3535 	(void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3536 
3537 	/*
3538 	 * Note the selection of user id's 0, 1 and 15, subsequently
3539 	 * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3540 	 * kludge-- a better policy is needed.
3541 	 *
3542 	 * Note that fork_with_timeout will only wait out the 1 second
3543 	 * "grace time" if pkill actually returns 0.  So if there are
3544 	 * no matches, this will run to completion much more quickly.
3545 	 */
3546 	(void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3547 	(void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3548 }
3549 
3550 static void
3551 do_uadmin(void)
3552 {
3553 	const char * const resetting = "/etc/svc/volatile/resetting";
3554 	int fd;
3555 	struct statvfs vfs;
3556 	time_t now;
3557 	struct tm nowtm;
3558 	char down_buf[256], time_buf[256];
3559 	uintptr_t mdep;
3560 #if defined(__x86)
3561 	char *fbarg = NULL;
3562 #endif	/* __x86 */
3563 
3564 	mdep = 0;
3565 	fd = creat(resetting, 0777);
3566 	if (fd >= 0)
3567 		startd_close(fd);
3568 	else
3569 		uu_warn("Could not create \"%s\"", resetting);
3570 
3571 	/* Kill dhcpagent if we're not using nfs for root */
3572 	if ((statvfs("/", &vfs) == 0) &&
3573 	    (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3574 		fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3575 
3576 	/*
3577 	 * Call sync(2) now, before we kill off user processes.  This takes
3578 	 * advantage of the several seconds of pause we have before the
3579 	 * killalls are done.  Time we can make good use of to get pages
3580 	 * moving out to disk.
3581 	 *
3582 	 * Inside non-global zones, we don't bother, and it's better not to
3583 	 * anyway, since sync(2) can have system-wide impact.
3584 	 */
3585 	if (getzoneid() == 0)
3586 		sync();
3587 
3588 	kill_user_procs();
3589 
3590 	/*
3591 	 * Note that this must come after the killing of user procs, since
3592 	 * killall relies on utmpx, and this command affects the contents of
3593 	 * said file.
3594 	 */
3595 	if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3596 		fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3597 
3598 	/*
3599 	 * For patches which may be installed as the system is shutting
3600 	 * down, we need to ensure, one more time, that the boot archive
3601 	 * really is up to date.
3602 	 */
3603 	if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3604 		fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3605 
3606 	/*
3607 	 * Right now, fast reboot is supported only on i386.
3608 	 * scf_is_fastboot_default() should take care of it.
3609 	 * If somehow we got there on unsupported platform -
3610 	 * print warning and fall back to regular reboot.
3611 	 */
3612 	if (halting == AD_FASTREBOOT) {
3613 #if defined(__x86)
3614 		if (be_get_boot_args(&fbarg, BE_ENTRY_DEFAULT) == 0) {
3615 			mdep = (uintptr_t)fbarg;
3616 		} else {
3617 			/*
3618 			 * Failed to read BE info, fall back to normal reboot
3619 			 */
3620 			halting = AD_BOOT;
3621 			uu_warn("Failed to get fast reboot arguments.\n"
3622 			    "Falling back to regular reboot.\n");
3623 		}
3624 #else	/* __x86 */
3625 		halting = AD_BOOT;
3626 		uu_warn("Fast reboot configured, but not supported by "
3627 		    "this ISA\n");
3628 #endif	/* __x86 */
3629 	}
3630 
3631 	fork_with_timeout("/sbin/umountall -l", 0, 5);
3632 	fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3633 	    ">/dev/null 2>&1", 0, 5);
3634 
3635 	/*
3636 	 * Try to get to consistency for whatever UFS filesystems are left.
3637 	 * This is pretty expensive, so we save it for the end in the hopes of
3638 	 * minimizing what it must do.  The other option would be to start in
3639 	 * parallel with the killall's, but lockfs tends to throw out much more
3640 	 * than is needed, and so subsequent commands (like umountall) take a
3641 	 * long time to get going again.
3642 	 *
3643 	 * Inside of zones, we don't bother, since we're not about to terminate
3644 	 * the whole OS instance.
3645 	 *
3646 	 * On systems using only ZFS, this call to lockfs -fa is a no-op.
3647 	 */
3648 	if (getzoneid() == 0) {
3649 		if (access("/usr/sbin/lockfs", X_OK) == 0)
3650 			fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3651 
3652 		sync();	/* once more, with feeling */
3653 	}
3654 
3655 	fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3656 
3657 	/*
3658 	 * Construct and emit the last words from userland:
3659 	 * "<timestamp> The system is down.  Shutdown took <N> seconds."
3660 	 *
3661 	 * Normally we'd use syslog, but with /var and other things
3662 	 * potentially gone, try to minimize the external dependencies.
3663 	 */
3664 	now = time(NULL);
3665 	(void) localtime_r(&now, &nowtm);
3666 
3667 	if (strftime(down_buf, sizeof (down_buf),
3668 	    "%b %e %T The system is down.", &nowtm) == 0) {
3669 		(void) strlcpy(down_buf, "The system is down.",
3670 		    sizeof (down_buf));
3671 	}
3672 
3673 	if (halting_time != 0 && halting_time <= now) {
3674 		(void) snprintf(time_buf, sizeof (time_buf),
3675 		    "  Shutdown took %lu seconds.", now - halting_time);
3676 	} else {
3677 		time_buf[0] = '\0';
3678 	}
3679 	(void) printf("%s%s\n", down_buf, time_buf);
3680 
3681 	(void) uadmin(A_SHUTDOWN, halting, mdep);
3682 	uu_warn("uadmin() failed");
3683 
3684 #if defined(__x86)
3685 	if (halting == AD_FASTREBOOT)
3686 		free(fbarg);
3687 #endif	/* __x86 */
3688 
3689 	if (remove(resetting) != 0 && errno != ENOENT)
3690 		uu_warn("Could not remove \"%s\"", resetting);
3691 }
3692 
3693 /*
3694  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3695  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3696  */
3697 boolean_t
3698 can_come_up(void)
3699 {
3700 	int i;
3701 
3702 	assert(MUTEX_HELD(&dgraph_lock));
3703 
3704 	/*
3705 	 * If we are booting to single user (boot -s),
3706 	 * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3707 	 * spawns sulogin after single-user is online (see specials.c).
3708 	 */
3709 	i = (booting_to_single_user ? 0 : 1);
3710 
3711 	for (; up_svcs[i] != NULL; ++i) {
3712 		if (up_svcs_p[i] == NULL) {
3713 			up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3714 
3715 			if (up_svcs_p[i] == NULL)
3716 				continue;
3717 		}
3718 
3719 		/*
3720 		 * Ignore unconfigured services (the ones that have been
3721 		 * mentioned in a dependency from other services, but do
3722 		 * not exist in the repository).  Services which exist
3723 		 * in the repository but don't have general/enabled
3724 		 * property will be also ignored.
3725 		 */
3726 		if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3727 			continue;
3728 
3729 		switch (up_svcs_p[i]->gv_state) {
3730 		case RESTARTER_STATE_ONLINE:
3731 		case RESTARTER_STATE_DEGRADED:
3732 			/*
3733 			 * Deactivate verbose boot once a login service has been
3734 			 * reached.
3735 			 */
3736 			st->st_log_login_reached = 1;
3737 			/*FALLTHROUGH*/
3738 		case RESTARTER_STATE_UNINIT:
3739 			return (B_TRUE);
3740 
3741 		case RESTARTER_STATE_OFFLINE:
3742 			if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3743 				return (B_TRUE);
3744 			log_framework(LOG_DEBUG,
3745 			    "can_come_up(): %s is unsatisfiable.\n",
3746 			    up_svcs_p[i]->gv_name);
3747 			continue;
3748 
3749 		case RESTARTER_STATE_DISABLED:
3750 		case RESTARTER_STATE_MAINT:
3751 			log_framework(LOG_DEBUG,
3752 			    "can_come_up(): %s is in state %s.\n",
3753 			    up_svcs_p[i]->gv_name,
3754 			    instance_state_str[up_svcs_p[i]->gv_state]);
3755 			continue;
3756 
3757 		default:
3758 #ifndef NDEBUG
3759 			uu_warn("%s:%d: Unexpected vertex state %d.\n",
3760 			    __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3761 #endif
3762 			abort();
3763 		}
3764 	}
3765 
3766 	/*
3767 	 * In the seed repository, console-login is unsatisfiable because
3768 	 * services are missing.  To behave correctly in that case we don't want
3769 	 * to return false until manifest-import is online.
3770 	 */
3771 
3772 	if (manifest_import_p == NULL) {
3773 		manifest_import_p = vertex_get_by_name(manifest_import);
3774 
3775 		if (manifest_import_p == NULL)
3776 			return (B_FALSE);
3777 	}
3778 
3779 	switch (manifest_import_p->gv_state) {
3780 	case RESTARTER_STATE_ONLINE:
3781 	case RESTARTER_STATE_DEGRADED:
3782 	case RESTARTER_STATE_DISABLED:
3783 	case RESTARTER_STATE_MAINT:
3784 		break;
3785 
3786 	case RESTARTER_STATE_OFFLINE:
3787 		if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3788 			break;
3789 		/* FALLTHROUGH */
3790 
3791 	case RESTARTER_STATE_UNINIT:
3792 		return (B_TRUE);
3793 	}
3794 
3795 	return (B_FALSE);
3796 }
3797 
3798 /*
3799  * Runs sulogin.  Returns
3800  *   0 - success
3801  *   EALREADY - sulogin is already running
3802  *   EBUSY - console-login is running
3803  */
3804 static int
3805 run_sulogin(const char *msg)
3806 {
3807 	graph_vertex_t *v;
3808 
3809 	assert(MUTEX_HELD(&dgraph_lock));
3810 
3811 	if (sulogin_running)
3812 		return (EALREADY);
3813 
3814 	v = vertex_get_by_name(console_login_fmri);
3815 	if (v != NULL && inst_running(v))
3816 		return (EBUSY);
3817 
3818 	sulogin_running = B_TRUE;
3819 
3820 	MUTEX_UNLOCK(&dgraph_lock);
3821 
3822 	fork_sulogin(B_FALSE, msg);
3823 
3824 	MUTEX_LOCK(&dgraph_lock);
3825 
3826 	sulogin_running = B_FALSE;
3827 
3828 	if (console_login_ready) {
3829 		v = vertex_get_by_name(console_login_fmri);
3830 
3831 		if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3832 			if (v->gv_start_f == NULL)
3833 				vertex_send_event(v,
3834 				    RESTARTER_EVENT_TYPE_START);
3835 			else
3836 				v->gv_start_f(v);
3837 		}
3838 
3839 		console_login_ready = B_FALSE;
3840 	}
3841 
3842 	return (0);
3843 }
3844 
3845 /*
3846  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3847  * keeps sulogin from stepping on console-login's toes.
3848  */
3849 /* ARGSUSED */
3850 static void *
3851 sulogin_thread(void *unused)
3852 {
3853 	(void) pthread_setname_np(pthread_self(), "sulogin");
3854 
3855 	MUTEX_LOCK(&dgraph_lock);
3856 
3857 	assert(sulogin_thread_running);
3858 
3859 	do {
3860 		(void) run_sulogin("Console login service(s) cannot run\n");
3861 	} while (!can_come_up());
3862 
3863 	sulogin_thread_running = B_FALSE;
3864 	MUTEX_UNLOCK(&dgraph_lock);
3865 
3866 	return (NULL);
3867 }
3868 
3869 /* ARGSUSED */
3870 void *
3871 single_user_thread(void *unused)
3872 {
3873 	uint_t left;
3874 	scf_handle_t *h;
3875 	scf_instance_t *inst;
3876 	scf_property_t *prop;
3877 	scf_value_t *val;
3878 	const char *msg;
3879 	char *buf;
3880 	int r;
3881 
3882 	(void) pthread_setname_np(pthread_self(), "single_user");
3883 
3884 	MUTEX_LOCK(&single_user_thread_lock);
3885 	single_user_thread_count++;
3886 
3887 	if (!booting_to_single_user)
3888 		kill_user_procs();
3889 
3890 	if (go_single_user_mode || booting_to_single_user) {
3891 		msg = "SINGLE USER MODE\n";
3892 	} else {
3893 		assert(go_to_level1);
3894 
3895 		fork_rc_script('1', "start", B_TRUE);
3896 
3897 		uu_warn("The system is ready for administration.\n");
3898 
3899 		msg = "";
3900 	}
3901 
3902 	MUTEX_UNLOCK(&single_user_thread_lock);
3903 
3904 	for (;;) {
3905 		MUTEX_LOCK(&dgraph_lock);
3906 		r = run_sulogin(msg);
3907 		MUTEX_UNLOCK(&dgraph_lock);
3908 		if (r == 0)
3909 			break;
3910 
3911 		assert(r == EALREADY || r == EBUSY);
3912 
3913 		left = 3;
3914 		while (left > 0)
3915 			left = sleep(left);
3916 	}
3917 
3918 	MUTEX_LOCK(&single_user_thread_lock);
3919 
3920 	/*
3921 	 * If another single user thread has started, let it finish changing
3922 	 * the run level.
3923 	 */
3924 	if (single_user_thread_count > 1) {
3925 		single_user_thread_count--;
3926 		MUTEX_UNLOCK(&single_user_thread_lock);
3927 		return (NULL);
3928 	}
3929 
3930 	h = libscf_handle_create_bound_loop();
3931 	inst = scf_instance_create(h);
3932 	prop = safe_scf_property_create(h);
3933 	val = safe_scf_value_create(h);
3934 	buf = startd_alloc(max_scf_fmri_size);
3935 
3936 lookup:
3937 	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3938 	    NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3939 		switch (scf_error()) {
3940 		case SCF_ERROR_NOT_FOUND:
3941 			r = libscf_create_self(h);
3942 			if (r == 0)
3943 				goto lookup;
3944 			assert(r == ECONNABORTED);
3945 			/* FALLTHROUGH */
3946 
3947 		case SCF_ERROR_CONNECTION_BROKEN:
3948 			libscf_handle_rebind(h);
3949 			goto lookup;
3950 
3951 		case SCF_ERROR_INVALID_ARGUMENT:
3952 		case SCF_ERROR_CONSTRAINT_VIOLATED:
3953 		case SCF_ERROR_NOT_BOUND:
3954 		case SCF_ERROR_HANDLE_MISMATCH:
3955 		default:
3956 			bad_error("scf_handle_decode_fmri", scf_error());
3957 		}
3958 	}
3959 
3960 	MUTEX_LOCK(&dgraph_lock);
3961 
3962 	r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3963 	    SCF_PROPERTY_MILESTONE);
3964 	switch (r) {
3965 	case 0:
3966 	case ECANCELED:
3967 		break;
3968 
3969 	case ECONNABORTED:
3970 		MUTEX_UNLOCK(&dgraph_lock);
3971 		libscf_handle_rebind(h);
3972 		goto lookup;
3973 
3974 	case EPERM:
3975 	case EACCES:
3976 	case EROFS:
3977 		log_error(LOG_WARNING, "Could not clear temporary milestone: "
3978 		    "%s.\n", strerror(r));
3979 		break;
3980 
3981 	default:
3982 		bad_error("scf_instance_delete_prop", r);
3983 	}
3984 
3985 	MUTEX_UNLOCK(&dgraph_lock);
3986 
3987 	r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3988 	switch (r) {
3989 	case ECANCELED:
3990 	case ENOENT:
3991 	case EINVAL:
3992 		(void) strcpy(buf, "all");
3993 		/* FALLTHROUGH */
3994 
3995 	case 0:
3996 		uu_warn("Returning to milestone %s.\n", buf);
3997 		break;
3998 
3999 	case ECONNABORTED:
4000 		libscf_handle_rebind(h);
4001 		goto lookup;
4002 
4003 	default:
4004 		bad_error("libscf_get_milestone", r);
4005 	}
4006 
4007 	r = dgraph_set_milestone(buf, h, B_FALSE);
4008 	switch (r) {
4009 	case 0:
4010 	case ECONNRESET:
4011 	case EALREADY:
4012 	case EINVAL:
4013 	case ENOENT:
4014 		break;
4015 
4016 	default:
4017 		bad_error("dgraph_set_milestone", r);
4018 	}
4019 
4020 	/*
4021 	 * See graph_runlevel_changed().
4022 	 */
4023 	MUTEX_LOCK(&dgraph_lock);
4024 	utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4025 	MUTEX_UNLOCK(&dgraph_lock);
4026 
4027 	startd_free(buf, max_scf_fmri_size);
4028 	scf_value_destroy(val);
4029 	scf_property_destroy(prop);
4030 	scf_instance_destroy(inst);
4031 	scf_handle_destroy(h);
4032 
4033 	/*
4034 	 * We'll give ourselves 3 seconds to respond to all of the enablings
4035 	 * that setting the milestone should have created before checking
4036 	 * whether to run sulogin.
4037 	 */
4038 	left = 3;
4039 	while (left > 0)
4040 		left = sleep(left);
4041 
4042 	MUTEX_LOCK(&dgraph_lock);
4043 	/*
4044 	 * Clearing these variables will allow the sulogin thread to run.  We
4045 	 * check here in case there aren't any more state updates anytime soon.
4046 	 */
4047 	go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4048 	if (!sulogin_thread_running && !can_come_up()) {
4049 		(void) startd_thread_create(sulogin_thread, NULL);
4050 		sulogin_thread_running = B_TRUE;
4051 	}
4052 	MUTEX_UNLOCK(&dgraph_lock);
4053 	single_user_thread_count--;
4054 	MUTEX_UNLOCK(&single_user_thread_lock);
4055 	return (NULL);
4056 }
4057 
4058 
4059 /*
4060  * Dependency graph operations API.  These are handle-independent thread-safe
4061  * graph manipulation functions which are the entry points for the event
4062  * threads below.
4063  */
4064 
4065 /*
4066  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4067  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4068  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4069  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4070  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4071  * property groups and add vertices and edges as appropriate.  If anything
4072  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4073  * instance in maintenance.  Don't send _START or _STOP until we get a state
4074  * update in case we're being restarted and the service is already running.
4075  *
4076  * To support booting to a milestone, we must also make sure all dependencies
4077  * encountered are configured, if they exist in the repository.
4078  *
4079  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4080  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4081  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4082  */
4083 int
4084 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4085     boolean_t lock_graph)
4086 {
4087 	graph_vertex_t *v;
4088 	int err;
4089 
4090 	if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4091 		return (0);
4092 
4093 	/* Check for a vertex for inst_fmri. */
4094 	if (lock_graph) {
4095 		MUTEX_LOCK(&dgraph_lock);
4096 	} else {
4097 		assert(MUTEX_HELD(&dgraph_lock));
4098 	}
4099 
4100 	v = vertex_get_by_name(inst_fmri);
4101 
4102 	if (v != NULL) {
4103 		assert(v->gv_type == GVT_INST);
4104 
4105 		if (v->gv_flags & GV_CONFIGURED) {
4106 			if (lock_graph)
4107 				MUTEX_UNLOCK(&dgraph_lock);
4108 			return (EEXIST);
4109 		}
4110 	} else {
4111 		/* Add the vertex. */
4112 		err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4113 		    RERR_NONE, &v);
4114 		if (err != 0) {
4115 			assert(err == EINVAL);
4116 			if (lock_graph)
4117 				MUTEX_UNLOCK(&dgraph_lock);
4118 			return (EINVAL);
4119 		}
4120 	}
4121 
4122 	err = configure_vertex(v, inst);
4123 
4124 	if (lock_graph)
4125 		MUTEX_UNLOCK(&dgraph_lock);
4126 
4127 	return (err);
4128 }
4129 
4130 /*
4131  * Locate the vertex for this property group's instance.  If it doesn't exist
4132  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4133  * the restarter for the instance, and if it has changed, send
4134  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4135  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4136  * the new restarter.  Then fetch whether the instance should be enabled, and
4137  * if it is different from what we had, or if we changed the restarter, send
4138  * the appropriate _ENABLE or _DISABLE command.
4139  *
4140  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4141  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4142  * deleted, or -1 if the instance's general property group is deleted or if
4143  * its enabled property is misconfigured.
4144  */
4145 static int
4146 dgraph_update_general(scf_propertygroup_t *pg)
4147 {
4148 	scf_handle_t *h;
4149 	scf_instance_t *inst;
4150 	char *fmri;
4151 	char *restarter_fmri;
4152 	graph_vertex_t *v;
4153 	int err;
4154 	int enabled, enabled_ovr;
4155 	int oldflags;
4156 
4157 	/* Find the vertex for this service */
4158 	h = scf_pg_handle(pg);
4159 
4160 	inst = safe_scf_instance_create(h);
4161 
4162 	if (scf_pg_get_parent_instance(pg, inst) != 0) {
4163 		switch (scf_error()) {
4164 		case SCF_ERROR_CONSTRAINT_VIOLATED:
4165 			return (ENOTSUP);
4166 
4167 		case SCF_ERROR_CONNECTION_BROKEN:
4168 		default:
4169 			return (ECONNABORTED);
4170 
4171 		case SCF_ERROR_DELETED:
4172 			return (0);
4173 
4174 		case SCF_ERROR_NOT_SET:
4175 			bad_error("scf_pg_get_parent_instance", scf_error());
4176 		}
4177 	}
4178 
4179 	err = libscf_instance_get_fmri(inst, &fmri);
4180 	switch (err) {
4181 	case 0:
4182 		break;
4183 
4184 	case ECONNABORTED:
4185 		scf_instance_destroy(inst);
4186 		return (ECONNABORTED);
4187 
4188 	case ECANCELED:
4189 		scf_instance_destroy(inst);
4190 		return (0);
4191 
4192 	default:
4193 		bad_error("libscf_instance_get_fmri", err);
4194 	}
4195 
4196 	log_framework(LOG_DEBUG,
4197 	    "Graph engine: Reloading general properties for %s.\n", fmri);
4198 
4199 	MUTEX_LOCK(&dgraph_lock);
4200 
4201 	v = vertex_get_by_name(fmri);
4202 	if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4203 		/* Will get the up-to-date properties. */
4204 		MUTEX_UNLOCK(&dgraph_lock);
4205 		err = dgraph_add_instance(fmri, inst, B_TRUE);
4206 		startd_free(fmri, max_scf_fmri_size);
4207 		scf_instance_destroy(inst);
4208 		return (err == ECANCELED ? 0 : err);
4209 	}
4210 
4211 	/* Read enabled & restarter from repository. */
4212 	restarter_fmri = startd_alloc(max_scf_value_size);
4213 	err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4214 	    &enabled_ovr, &restarter_fmri);
4215 	if (err != 0 || enabled == -1) {
4216 		MUTEX_UNLOCK(&dgraph_lock);
4217 		scf_instance_destroy(inst);
4218 		startd_free(fmri, max_scf_fmri_size);
4219 
4220 		switch (err) {
4221 		case ENOENT:
4222 		case 0:
4223 			startd_free(restarter_fmri, max_scf_value_size);
4224 			return (-1);
4225 
4226 		case ECONNABORTED:
4227 		case ECANCELED:
4228 			startd_free(restarter_fmri, max_scf_value_size);
4229 			return (err);
4230 
4231 		default:
4232 			bad_error("libscf_get_basic_instance_data", err);
4233 		}
4234 	}
4235 
4236 	oldflags = v->gv_flags;
4237 	v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4238 	    (enabled ? GV_ENBLD_NOOVR : 0);
4239 
4240 	if (enabled_ovr != -1)
4241 		enabled = enabled_ovr;
4242 
4243 	/*
4244 	 * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4245 	 * subgraph.
4246 	 */
4247 	if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4248 		(void) eval_subgraph(v, h);
4249 
4250 	scf_instance_destroy(inst);
4251 
4252 	/* Ignore restarter change for now. */
4253 
4254 	startd_free(restarter_fmri, max_scf_value_size);
4255 	startd_free(fmri, max_scf_fmri_size);
4256 
4257 	/*
4258 	 * Always send _ENABLE or _DISABLE.  We could avoid this if the
4259 	 * restarter didn't change and the enabled value didn't change, but
4260 	 * that's not easy to check and improbable anyway, so we'll just do
4261 	 * this.
4262 	 */
4263 	graph_enable_by_vertex(v, enabled, 1);
4264 
4265 	MUTEX_UNLOCK(&dgraph_lock);
4266 
4267 	return (0);
4268 }
4269 
4270 /*
4271  * Delete all of the property group dependencies of v, update inst's running
4272  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4273  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4274  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4275  * the same for v's dependents.
4276  *
4277  * Returns
4278  *   0 - success
4279  *   ECONNABORTED - repository connection broken
4280  *   ECANCELED - inst was deleted
4281  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4282  *   -1 - libscf_snapshots_refresh() failed
4283  */
4284 static int
4285 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4286 {
4287 	int r;
4288 	int enabled;
4289 	int32_t tset;
4290 
4291 	assert(MUTEX_HELD(&dgraph_lock));
4292 	assert(v->gv_type == GVT_INST);
4293 
4294 	/* Only refresh services with valid general/enabled properties. */
4295 	r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4296 	    v->gv_name, &enabled, NULL, NULL);
4297 	switch (r) {
4298 	case 0:
4299 		break;
4300 
4301 	case ECONNABORTED:
4302 	case ECANCELED:
4303 		return (r);
4304 
4305 	case ENOENT:
4306 		log_framework(LOG_DEBUG,
4307 		    "Ignoring %s because it has no general property group.\n",
4308 		    v->gv_name);
4309 		return (EINVAL);
4310 
4311 	default:
4312 		bad_error("libscf_get_basic_instance_data", r);
4313 	}
4314 
4315 	if ((tset = libscf_get_stn_tset(inst)) == -1) {
4316 		log_framework(LOG_WARNING,
4317 		    "Failed to get notification parameters for %s: %s\n",
4318 		    v->gv_name, scf_strerror(scf_error()));
4319 		tset = 0;
4320 	}
4321 	v->gv_stn_tset = tset;
4322 	if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4323 		stn_global = tset;
4324 
4325 	if (enabled == -1)
4326 		return (EINVAL);
4327 
4328 	r = libscf_snapshots_refresh(inst, v->gv_name);
4329 	if (r != 0) {
4330 		if (r != -1)
4331 			bad_error("libscf_snapshots_refresh", r);
4332 
4333 		/* error logged */
4334 		return (r);
4335 	}
4336 
4337 	r = refresh_vertex(v, inst);
4338 	if (r != 0 && r != ECONNABORTED)
4339 		bad_error("refresh_vertex", r);
4340 	return (r);
4341 }
4342 
4343 /*
4344  * Returns true only if none of this service's dependents are 'up' -- online
4345  * or degraded (offline is considered down in this situation). This function
4346  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4347  */
4348 static boolean_t
4349 insubtree_dependents_down(graph_vertex_t *v)
4350 {
4351 	graph_vertex_t *vv;
4352 	graph_edge_t *e;
4353 
4354 	assert(MUTEX_HELD(&dgraph_lock));
4355 
4356 	for (e = uu_list_first(v->gv_dependents); e != NULL;
4357 	    e = uu_list_next(v->gv_dependents, e)) {
4358 		vv = e->ge_vertex;
4359 		if (vv->gv_type == GVT_INST) {
4360 			if ((vv->gv_flags & GV_CONFIGURED) == 0)
4361 				continue;
4362 
4363 			if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4364 				continue;
4365 
4366 			if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4367 			    (vv->gv_state == RESTARTER_STATE_DEGRADED))
4368 				return (B_FALSE);
4369 		} else {
4370 			/*
4371 			 * Skip all excluded dependents and decide whether
4372 			 * to offline the service based on the restart_on
4373 			 * attribute.
4374 			 */
4375 			if (is_depgrp_bypassed(vv))
4376 				continue;
4377 
4378 			/*
4379 			 * For dependency groups or service vertices, keep
4380 			 * traversing to see if instances are running.
4381 			 */
4382 			if (insubtree_dependents_down(vv) == B_FALSE)
4383 				return (B_FALSE);
4384 		}
4385 	}
4386 
4387 	return (B_TRUE);
4388 }
4389 
4390 /*
4391  * Returns true only if none of this service's dependents are 'up' -- online,
4392  * degraded, or offline.
4393  */
4394 static int
4395 is_nonsubgraph_leaf(graph_vertex_t *v)
4396 {
4397 	graph_vertex_t *vv;
4398 	graph_edge_t *e;
4399 
4400 	assert(MUTEX_HELD(&dgraph_lock));
4401 
4402 	for (e = uu_list_first(v->gv_dependents);
4403 	    e != NULL;
4404 	    e = uu_list_next(v->gv_dependents, e)) {
4405 
4406 		vv = e->ge_vertex;
4407 		if (vv->gv_type == GVT_INST) {
4408 			if ((vv->gv_flags & GV_CONFIGURED) == 0)
4409 				continue;
4410 
4411 			if (vv->gv_flags & GV_INSUBGRAPH)
4412 				continue;
4413 
4414 			if (up_state(vv->gv_state))
4415 				return (0);
4416 		} else {
4417 			/*
4418 			 * For dependency group or service vertices, keep
4419 			 * traversing to see if instances are running.
4420 			 *
4421 			 * We should skip exclude_all dependencies otherwise
4422 			 * the vertex will never be considered as a leaf
4423 			 * if the dependent is offline. The main reason for
4424 			 * this is that disable_nonsubgraph_leaves() skips
4425 			 * exclusion dependencies.
4426 			 */
4427 			if (vv->gv_type == GVT_GROUP &&
4428 			    vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4429 				continue;
4430 
4431 			if (!is_nonsubgraph_leaf(vv))
4432 				return (0);
4433 		}
4434 	}
4435 
4436 	return (1);
4437 }
4438 
4439 /*
4440  * Disable v temporarily.  Attempt to do this by setting its enabled override
4441  * property in the repository.  If that fails, send a _DISABLE command.
4442  * Returns 0 on success and ECONNABORTED if the repository connection is
4443  * broken.
4444  */
4445 static int
4446 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4447 {
4448 	const char * const emsg = "Could not temporarily disable %s because "
4449 	    "%s.  Will stop service anyways.  Repository status for the "
4450 	    "service may be inaccurate.\n";
4451 	const char * const emsg_cbroken =
4452 	    "the repository connection was broken";
4453 
4454 	scf_instance_t *inst;
4455 	int r;
4456 
4457 	inst = scf_instance_create(h);
4458 	if (inst == NULL) {
4459 		char buf[100];
4460 
4461 		(void) snprintf(buf, sizeof (buf),
4462 		    "scf_instance_create() failed (%s)",
4463 		    scf_strerror(scf_error()));
4464 		log_error(LOG_WARNING, emsg, v->gv_name, buf);
4465 
4466 		graph_enable_by_vertex(v, 0, 0);
4467 		return (0);
4468 	}
4469 
4470 	r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4471 	    NULL, NULL, SCF_DECODE_FMRI_EXACT);
4472 	if (r != 0) {
4473 		switch (scf_error()) {
4474 		case SCF_ERROR_CONNECTION_BROKEN:
4475 			log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4476 			graph_enable_by_vertex(v, 0, 0);
4477 			return (ECONNABORTED);
4478 
4479 		case SCF_ERROR_NOT_FOUND:
4480 			return (0);
4481 
4482 		case SCF_ERROR_HANDLE_MISMATCH:
4483 		case SCF_ERROR_INVALID_ARGUMENT:
4484 		case SCF_ERROR_CONSTRAINT_VIOLATED:
4485 		case SCF_ERROR_NOT_BOUND:
4486 		default:
4487 			bad_error("scf_handle_decode_fmri",
4488 			    scf_error());
4489 		}
4490 	}
4491 
4492 	r = libscf_set_enable_ovr(inst, 0);
4493 	switch (r) {
4494 	case 0:
4495 		scf_instance_destroy(inst);
4496 		return (0);
4497 
4498 	case ECANCELED:
4499 		scf_instance_destroy(inst);
4500 		return (0);
4501 
4502 	case ECONNABORTED:
4503 		log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4504 		graph_enable_by_vertex(v, 0, 0);
4505 		return (ECONNABORTED);
4506 
4507 	case EPERM:
4508 		log_error(LOG_WARNING, emsg, v->gv_name,
4509 		    "the repository denied permission");
4510 		graph_enable_by_vertex(v, 0, 0);
4511 		return (0);
4512 
4513 	case EROFS:
4514 		log_error(LOG_WARNING, emsg, v->gv_name,
4515 		    "the repository is read-only");
4516 		graph_enable_by_vertex(v, 0, 0);
4517 		return (0);
4518 
4519 	default:
4520 		bad_error("libscf_set_enable_ovr", r);
4521 		/* NOTREACHED */
4522 	}
4523 }
4524 
4525 /*
4526  * Of the transitive instance dependencies of v, offline those which are
4527  * in the subtree and which are leaves (i.e., have no dependents which are
4528  * "up").
4529  */
4530 void
4531 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4532 {
4533 	assert(MUTEX_HELD(&dgraph_lock));
4534 
4535 	/* If v isn't an instance, recurse on its dependencies. */
4536 	if (v->gv_type != GVT_INST) {
4537 		graph_walk_dependencies(v, offline_subtree_leaves, arg);
4538 		return;
4539 	}
4540 
4541 	/*
4542 	 * If v is not in the subtree, so should all of its dependencies,
4543 	 * so do nothing.
4544 	 */
4545 	if ((v->gv_flags & GV_TOOFFLINE) == 0)
4546 		return;
4547 
4548 	/* If v isn't a leaf because it's already down, recurse. */
4549 	if (!up_state(v->gv_state)) {
4550 		graph_walk_dependencies(v, offline_subtree_leaves, arg);
4551 		return;
4552 	}
4553 
4554 	/* if v is a leaf, offline it or disable it if it's the last one */
4555 	if (insubtree_dependents_down(v) == B_TRUE) {
4556 		if (v->gv_flags & GV_TODISABLE)
4557 			vertex_send_event(v,
4558 			    RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4559 		else
4560 			offline_vertex(v);
4561 	}
4562 }
4563 
4564 void
4565 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4566 {
4567 	graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4568 }
4569 
4570 
4571 /*
4572  * Of the transitive instance dependencies of v, disable those which are not
4573  * in the subgraph and which are leaves (i.e., have no dependents which are
4574  * "up").
4575  */
4576 static void
4577 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4578 {
4579 	assert(MUTEX_HELD(&dgraph_lock));
4580 
4581 	/*
4582 	 * We must skip exclusion dependencies because they are allowed to
4583 	 * complete dependency cycles.  This is correct because A's exclusion
4584 	 * dependency on B doesn't bear on the order in which they should be
4585 	 * stopped.  Indeed, the exclusion dependency should guarantee that
4586 	 * they are never online at the same time.
4587 	 */
4588 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4589 		return;
4590 
4591 	/* If v isn't an instance, recurse on its dependencies. */
4592 	if (v->gv_type != GVT_INST)
4593 		goto recurse;
4594 
4595 	if ((v->gv_flags & GV_CONFIGURED) == 0)
4596 		/*
4597 		 * Unconfigured instances should have no dependencies, but in
4598 		 * case they ever get them,
4599 		 */
4600 		goto recurse;
4601 
4602 	/*
4603 	 * If v is in the subgraph, so should all of its dependencies, so do
4604 	 * nothing.
4605 	 */
4606 	if (v->gv_flags & GV_INSUBGRAPH)
4607 		return;
4608 
4609 	/* If v isn't a leaf because it's already down, recurse. */
4610 	if (!up_state(v->gv_state))
4611 		goto recurse;
4612 
4613 	/* If v is disabled but not down yet, be patient. */
4614 	if ((v->gv_flags & GV_ENABLED) == 0)
4615 		return;
4616 
4617 	/* If v is a leaf, disable it. */
4618 	if (is_nonsubgraph_leaf(v))
4619 		(void) disable_service_temporarily(v, (scf_handle_t *)arg);
4620 
4621 	return;
4622 
4623 recurse:
4624 	graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4625 }
4626 
4627 static int
4628 stn_restarter_state(restarter_instance_state_t rstate)
4629 {
4630 	static const struct statemap {
4631 		restarter_instance_state_t restarter_state;
4632 		int scf_state;
4633 	} map[] = {
4634 		{ RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4635 		{ RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4636 		{ RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4637 		{ RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4638 		{ RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4639 		{ RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4640 	};
4641 
4642 	int i;
4643 
4644 	for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4645 		if (rstate == map[i].restarter_state)
4646 			return (map[i].scf_state);
4647 	}
4648 
4649 	return (-1);
4650 }
4651 
4652 /*
4653  * State transition counters
4654  * Not incremented atomically - indicative only
4655  */
4656 static uint64_t stev_ct_maint;
4657 static uint64_t stev_ct_hwerr;
4658 static uint64_t stev_ct_service;
4659 static uint64_t stev_ct_global;
4660 static uint64_t stev_ct_noprefs;
4661 static uint64_t stev_ct_from_uninit;
4662 static uint64_t stev_ct_bad_state;
4663 static uint64_t stev_ct_ovr_prefs;
4664 
4665 static void
4666 dgraph_state_transition_notify(graph_vertex_t *v,
4667     restarter_instance_state_t old_state, restarter_str_t reason)
4668 {
4669 	restarter_instance_state_t new_state = v->gv_state;
4670 	int stn_transition, maint;
4671 	int from, to;
4672 	nvlist_t *attr;
4673 	fmev_pri_t pri = FMEV_LOPRI;
4674 	int raise = 0;
4675 
4676 	if ((from = stn_restarter_state(old_state)) == -1 ||
4677 	    (to = stn_restarter_state(new_state)) == -1) {
4678 		stev_ct_bad_state++;
4679 		return;
4680 	}
4681 
4682 	stn_transition = from << 16 | to;
4683 
4684 	maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4685 
4686 	if (maint) {
4687 		/*
4688 		 * All transitions to/from maintenance state must raise
4689 		 * an event.
4690 		 */
4691 		raise++;
4692 		pri = FMEV_HIPRI;
4693 		stev_ct_maint++;
4694 	} else if (reason == restarter_str_ct_ev_hwerr) {
4695 		/*
4696 		 * All transitions caused by hardware fault must raise
4697 		 * an event
4698 		 */
4699 		raise++;
4700 		pri = FMEV_HIPRI;
4701 		stev_ct_hwerr++;
4702 	} else if (stn_transition & v->gv_stn_tset) {
4703 		/*
4704 		 * Specifically enabled event.
4705 		 */
4706 		raise++;
4707 		stev_ct_service++;
4708 	} else if (from == SCF_STATE_UNINIT) {
4709 		/*
4710 		 * Only raise these if specifically selected above.
4711 		 */
4712 		stev_ct_from_uninit++;
4713 	} else if (stn_transition & stn_global &&
4714 	    (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4715 		raise++;
4716 		stev_ct_global++;
4717 	} else {
4718 		stev_ct_noprefs++;
4719 	}
4720 
4721 	if (info_events_all) {
4722 		stev_ct_ovr_prefs++;
4723 		raise++;
4724 	}
4725 	if (!raise)
4726 		return;
4727 
4728 	if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4729 	    nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4730 	    nvlist_add_uint32(attr, "reason-version",
4731 	    restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4732 	    restarter_get_str_short(reason)) != 0 ||
4733 	    nvlist_add_string(attr, "reason-long",
4734 	    restarter_get_str_long(reason)) != 0 ||
4735 	    nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4736 		log_framework(LOG_WARNING,
4737 		    "FMEV: %s could not create nvlist for transition "
4738 		    "event: %s\n", v->gv_name, strerror(errno));
4739 		nvlist_free(attr);
4740 		return;
4741 	}
4742 
4743 	if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4744 	    instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4745 		log_framework(LOG_DEBUG,
4746 		    "FMEV: %s failed to publish transition event: %s\n",
4747 		    v->gv_name, fmev_strerror(fmev_errno));
4748 		nvlist_free(attr);
4749 	}
4750 }
4751 
4752 /*
4753  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4754  * Otherwise set its state to state.  If the instance has entered a state
4755  * which requires automatic action, take it (Uninitialized: do
4756  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4757  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4758  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4759  * _START.  Online, Degraded: if the instance wasn't running, update its start
4760  * snapshot.  Maintenance: no action.)
4761  *
4762  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4763  */
4764 static int
4765 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4766     protocol_states_t *states)
4767 {
4768 	graph_vertex_t *v;
4769 	int err = 0;
4770 	restarter_instance_state_t old_state;
4771 	restarter_instance_state_t state = states->ps_state;
4772 	restarter_error_t serr = states->ps_err;
4773 
4774 	MUTEX_LOCK(&dgraph_lock);
4775 
4776 	v = vertex_get_by_name(inst_name);
4777 	if (v == NULL) {
4778 		MUTEX_UNLOCK(&dgraph_lock);
4779 		return (ENOENT);
4780 	}
4781 
4782 	assert(v->gv_type == GVT_INST);
4783 
4784 	switch (state) {
4785 	case RESTARTER_STATE_UNINIT:
4786 	case RESTARTER_STATE_DISABLED:
4787 	case RESTARTER_STATE_OFFLINE:
4788 	case RESTARTER_STATE_ONLINE:
4789 	case RESTARTER_STATE_DEGRADED:
4790 	case RESTARTER_STATE_MAINT:
4791 		break;
4792 
4793 	default:
4794 		MUTEX_UNLOCK(&dgraph_lock);
4795 		return (EINVAL);
4796 	}
4797 
4798 	log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4799 	    instance_state_str[v->gv_state], instance_state_str[state]);
4800 
4801 	old_state = v->gv_state;
4802 	v->gv_state = state;
4803 
4804 	v->gv_reason = states->ps_reason;
4805 	err = gt_transition(h, v, serr, old_state);
4806 	if (err == 0 && v->gv_state != old_state) {
4807 		dgraph_state_transition_notify(v, old_state, states->ps_reason);
4808 	}
4809 
4810 	MUTEX_UNLOCK(&dgraph_lock);
4811 	return (err);
4812 }
4813 
4814 /*
4815  * Handle state changes during milestone shutdown.  See
4816  * dgraph_set_milestone().  If the repository connection is broken,
4817  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4818  * the vertex anyway.
4819  */
4820 int
4821 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4822     restarter_instance_state_t old_state)
4823 {
4824 	int was_up, now_up;
4825 	int ret = 0;
4826 
4827 	assert(v->gv_type == GVT_INST);
4828 
4829 	/* Don't care if we're not going to a milestone. */
4830 	if (milestone == NULL)
4831 		return (0);
4832 
4833 	/* Don't care if we already finished coming down. */
4834 	if (non_subgraph_svcs == 0)
4835 		return (0);
4836 
4837 	/* Don't care if the service is in the subgraph. */
4838 	if (v->gv_flags & GV_INSUBGRAPH)
4839 		return (0);
4840 
4841 	/*
4842 	 * Update non_subgraph_svcs.  It is the number of non-subgraph
4843 	 * services which are in online, degraded, or offline.
4844 	 */
4845 
4846 	was_up = up_state(old_state);
4847 	now_up = up_state(v->gv_state);
4848 
4849 	if (!was_up && now_up) {
4850 		++non_subgraph_svcs;
4851 	} else if (was_up && !now_up) {
4852 		--non_subgraph_svcs;
4853 
4854 		if (non_subgraph_svcs == 0) {
4855 			if (halting != -1) {
4856 				do_uadmin();
4857 			} else if (go_single_user_mode || go_to_level1) {
4858 				(void) startd_thread_create(single_user_thread,
4859 				    NULL);
4860 			}
4861 			return (0);
4862 		}
4863 	}
4864 
4865 	/* If this service is a leaf, it should be disabled. */
4866 	if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4867 		int r;
4868 
4869 		r = disable_service_temporarily(v, h);
4870 		switch (r) {
4871 		case 0:
4872 			break;
4873 
4874 		case ECONNABORTED:
4875 			ret = ECONNABORTED;
4876 			break;
4877 
4878 		default:
4879 			bad_error("disable_service_temporarily", r);
4880 		}
4881 	}
4882 
4883 	/*
4884 	 * If the service just came down, propagate the disable to the newly
4885 	 * exposed leaves.
4886 	 */
4887 	if (was_up && !now_up)
4888 		graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4889 		    (void *)h);
4890 
4891 	return (ret);
4892 }
4893 
4894 /*
4895  * Decide whether to start up an sulogin thread after a service is
4896  * finished changing state.  Only need to do the full can_come_up()
4897  * evaluation if an instance is changing state, we're not halfway through
4898  * loading the thread, and we aren't shutting down or going to the single
4899  * user milestone.
4900  */
4901 void
4902 graph_transition_sulogin(restarter_instance_state_t state,
4903     restarter_instance_state_t old_state)
4904 {
4905 	assert(MUTEX_HELD(&dgraph_lock));
4906 
4907 	if (state != old_state && st->st_load_complete &&
4908 	    !go_single_user_mode && !go_to_level1 &&
4909 	    halting == -1) {
4910 		if (!sulogin_thread_running && !can_come_up()) {
4911 			(void) startd_thread_create(sulogin_thread, NULL);
4912 			sulogin_thread_running = B_TRUE;
4913 		}
4914 	}
4915 }
4916 
4917 /*
4918  * Propagate a start, stop event, or a satisfiability event.
4919  *
4920  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4921  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4922  * full dependent graph to check for newly satisfied nodes.  This is
4923  * necessary for cases when non-direct dependents may be effected but direct
4924  * dependents may not (e.g. for optional_all evaluations, see the
4925  * propagate_satbility() comments).
4926  *
4927  * PROPAGATE_SAT should be used whenever a non-running service moves into
4928  * a state which can satisfy optional dependencies, like disabled or
4929  * maintenance.
4930  */
4931 void
4932 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4933     restarter_error_t rerr)
4934 {
4935 	if (type == PROPAGATE_STOP) {
4936 		graph_walk_dependents(v, propagate_stop, (void *)rerr);
4937 	} else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4938 		graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4939 
4940 		if (type == PROPAGATE_SAT)
4941 			propagate_satbility(v);
4942 	} else {
4943 #ifndef NDEBUG
4944 		uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4945 		    __LINE__, type);
4946 #endif
4947 		abort();
4948 	}
4949 }
4950 
4951 /*
4952  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4953  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4954  * all property group dependencies, and the dependency on the restarter,
4955  * disposing of vertices as appropriate.  If other vertices depend on this
4956  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4957  * returns 0.
4958  */
4959 static int
4960 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4961 {
4962 	graph_vertex_t *v;
4963 	graph_edge_t *e;
4964 	uu_list_t *old_deps;
4965 	int err;
4966 
4967 	log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4968 
4969 	MUTEX_LOCK(&dgraph_lock);
4970 
4971 	v = vertex_get_by_name(fmri);
4972 	if (v == NULL) {
4973 		MUTEX_UNLOCK(&dgraph_lock);
4974 		return (0);
4975 	}
4976 
4977 	/* Send restarter delete event. */
4978 	if (v->gv_flags & GV_CONFIGURED)
4979 		graph_unset_restarter(v);
4980 
4981 	if (milestone > MILESTONE_NONE) {
4982 		/*
4983 		 * Make a list of v's current dependencies so we can
4984 		 * reevaluate their GV_INSUBGRAPH flags after the dependencies
4985 		 * are removed.
4986 		 */
4987 		old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4988 
4989 		err = uu_list_walk(v->gv_dependencies,
4990 		    (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4991 		assert(err == 0);
4992 	}
4993 
4994 	delete_instance_dependencies(v, B_TRUE);
4995 
4996 	/*
4997 	 * Deleting an instance can both satisfy and unsatisfy dependencies,
4998 	 * depending on their type.  First propagate the stop as a RERR_RESTART
4999 	 * event -- deletion isn't a fault, just a normal stop.  This gives
5000 	 * dependent services the chance to do a clean shutdown.  Then, mark
5001 	 * the service as unconfigured and propagate the start event for the
5002 	 * optional_all dependencies that might have become satisfied.
5003 	 */
5004 	graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5005 
5006 	v->gv_flags &= ~GV_CONFIGURED;
5007 	v->gv_flags &= ~GV_DEATHROW;
5008 
5009 	graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5010 	propagate_satbility(v);
5011 
5012 	/*
5013 	 * If there are no (non-service) dependents, the vertex can be
5014 	 * completely removed.
5015 	 */
5016 	if (v != milestone && v->gv_refs == 0 &&
5017 	    uu_list_numnodes(v->gv_dependents) == 1)
5018 		remove_inst_vertex(v);
5019 
5020 	if (milestone > MILESTONE_NONE) {
5021 		void *cookie = NULL;
5022 
5023 		while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5024 			v = e->ge_vertex;
5025 
5026 			if (vertex_unref(v) == VERTEX_INUSE)
5027 				while (eval_subgraph(v, h) == ECONNABORTED)
5028 					libscf_handle_rebind(h);
5029 
5030 			startd_free(e, sizeof (*e));
5031 		}
5032 
5033 		uu_list_destroy(old_deps);
5034 	}
5035 
5036 	MUTEX_UNLOCK(&dgraph_lock);
5037 
5038 	return (0);
5039 }
5040 
5041 /*
5042  * Return the eventual (maybe current) milestone in the form of a
5043  * legacy runlevel.
5044  */
5045 static char
5046 target_milestone_as_runlevel()
5047 {
5048 	assert(MUTEX_HELD(&dgraph_lock));
5049 
5050 	if (milestone == NULL)
5051 		return ('3');
5052 	else if (milestone == MILESTONE_NONE)
5053 		return ('0');
5054 
5055 	if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5056 		return ('2');
5057 	else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5058 		return ('S');
5059 	else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5060 		return ('3');
5061 
5062 #ifndef NDEBUG
5063 	(void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5064 	    __FILE__, __LINE__, milestone->gv_name);
5065 #endif
5066 	abort();
5067 	/* NOTREACHED */
5068 }
5069 
5070 static struct {
5071 	char	rl;
5072 	int	sig;
5073 } init_sigs[] = {
5074 	{ 'S', SIGBUS },
5075 	{ '0', SIGINT },
5076 	{ '1', SIGQUIT },
5077 	{ '2', SIGILL },
5078 	{ '3', SIGTRAP },
5079 	{ '4', SIGIOT },
5080 	{ '5', SIGEMT },
5081 	{ '6', SIGFPE },
5082 	{ 0, 0 }
5083 };
5084 
5085 static void
5086 signal_init(char rl)
5087 {
5088 	pid_t init_pid;
5089 	int i;
5090 
5091 	assert(MUTEX_HELD(&dgraph_lock));
5092 
5093 	if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5094 	    sizeof (init_pid)) != sizeof (init_pid)) {
5095 		log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5096 		return;
5097 	}
5098 
5099 	for (i = 0; init_sigs[i].rl != 0; ++i)
5100 		if (init_sigs[i].rl == rl)
5101 			break;
5102 
5103 	if (init_sigs[i].rl != 0) {
5104 		if (kill(init_pid, init_sigs[i].sig) != 0) {
5105 			switch (errno) {
5106 			case EPERM:
5107 			case ESRCH:
5108 				log_error(LOG_NOTICE, "Could not signal init: "
5109 				    "%s.\n", strerror(errno));
5110 				break;
5111 
5112 			case EINVAL:
5113 			default:
5114 				bad_error("kill", errno);
5115 			}
5116 		}
5117 	}
5118 }
5119 
5120 /*
5121  * This is called when one of the major milestones changes state, or when
5122  * init is signalled and tells us it was told to change runlevel.  We wait
5123  * to reach the milestone because this allows /etc/inittab entries to retain
5124  * some boot ordering: historically, entries could place themselves before/after
5125  * the running of /sbin/rcX scripts but we can no longer make the
5126  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5127  * marks in /etc/inittab.
5128  *
5129  * Also, we only trigger an update when we reach the eventual target
5130  * milestone: without this, an /etc/inittab entry marked only for
5131  * runlevel 2 would be executed for runlevel 3, which is not how
5132  * /etc/inittab entries work.
5133  *
5134  * If we're single user coming online, then we set utmpx to the target
5135  * runlevel so that legacy scripts can work as expected.
5136  */
5137 static void
5138 graph_runlevel_changed(char rl, int online)
5139 {
5140 	char trl;
5141 
5142 	assert(MUTEX_HELD(&dgraph_lock));
5143 
5144 	trl = target_milestone_as_runlevel();
5145 
5146 	if (online) {
5147 		if (rl == trl) {
5148 			current_runlevel = trl;
5149 			signal_init(trl);
5150 		} else if (rl == 'S') {
5151 			/*
5152 			 * At boot, set the entry early for the benefit of the
5153 			 * legacy init scripts.
5154 			 */
5155 			utmpx_set_runlevel(trl, 'S', B_FALSE);
5156 		}
5157 	} else {
5158 		if (rl == '3' && trl == '2') {
5159 			current_runlevel = trl;
5160 			signal_init(trl);
5161 		} else if (rl == '2' && trl == 'S') {
5162 			current_runlevel = trl;
5163 			signal_init(trl);
5164 		}
5165 	}
5166 }
5167 
5168 /*
5169  * Move to a backwards-compatible runlevel by executing the appropriate
5170  * /etc/rc?.d/K* scripts and/or setting the milestone.
5171  *
5172  * Returns
5173  *   0 - success
5174  *   ECONNRESET - success, but handle was reset
5175  *   ECONNABORTED - repository connection broken
5176  *   ECANCELED - pg was deleted
5177  */
5178 static int
5179 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5180 {
5181 	char rl;
5182 	scf_handle_t *h;
5183 	int r;
5184 	const char *ms = NULL;	/* what to commit as options/milestone */
5185 	boolean_t rebound = B_FALSE;
5186 	int mark_rl = 0;
5187 
5188 	const char * const stop = "stop";
5189 
5190 	r = libscf_extract_runlevel(prop, &rl);
5191 	switch (r) {
5192 	case 0:
5193 		break;
5194 
5195 	case ECONNABORTED:
5196 	case ECANCELED:
5197 		return (r);
5198 
5199 	case EINVAL:
5200 	case ENOENT:
5201 		log_error(LOG_WARNING, "runlevel property is misconfigured; "
5202 		    "ignoring.\n");
5203 		/* delete the bad property */
5204 		goto nolock_out;
5205 
5206 	default:
5207 		bad_error("libscf_extract_runlevel", r);
5208 	}
5209 
5210 	switch (rl) {
5211 	case 's':
5212 		rl = 'S';
5213 		/* FALLTHROUGH */
5214 
5215 	case 'S':
5216 	case '2':
5217 	case '3':
5218 		/*
5219 		 * These cases cause a milestone change, so
5220 		 * graph_runlevel_changed() will eventually deal with
5221 		 * signalling init.
5222 		 */
5223 		break;
5224 
5225 	case '0':
5226 	case '1':
5227 	case '4':
5228 	case '5':
5229 	case '6':
5230 		mark_rl = 1;
5231 		break;
5232 
5233 	default:
5234 		log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5235 		ms = NULL;
5236 		goto nolock_out;
5237 	}
5238 
5239 	h = scf_pg_handle(pg);
5240 
5241 	MUTEX_LOCK(&dgraph_lock);
5242 
5243 	/*
5244 	 * Since this triggers no milestone changes, force it by hand.
5245 	 */
5246 	if (current_runlevel == '4' && rl == '3')
5247 		mark_rl = 1;
5248 
5249 	/*
5250 	 * 1. If we are here after an "init X":
5251 	 *
5252 	 * init X
5253 	 *	init/lscf_set_runlevel()
5254 	 *		process_pg_event()
5255 	 *		dgraph_set_runlevel()
5256 	 *
5257 	 * then we haven't passed through graph_runlevel_changed() yet,
5258 	 * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5259 	 * In consequence, if 'rl' is lower than 'current_runlevel', we change
5260 	 * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5261 	 * past this test.
5262 	 *
5263 	 * 2. On the other hand, if we are here after a "svcadm milestone":
5264 	 *
5265 	 * svcadm milestone X
5266 	 *	dgraph_set_milestone()
5267 	 *		handle_graph_update_event()
5268 	 *		dgraph_set_instance_state()
5269 	 *		graph_post_X_[online|offline]()
5270 	 *		graph_runlevel_changed()
5271 	 *		signal_init()
5272 	 *			init/lscf_set_runlevel()
5273 	 *				process_pg_event()
5274 	 *				dgraph_set_runlevel()
5275 	 *
5276 	 * then we already passed through graph_runlevel_changed() (by the way
5277 	 * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5278 	 * and already be equal to 'rl' so we are going to return immediately
5279 	 * from dgraph_set_runlevel() without changing the system runlevel and
5280 	 * without executing the /etc/rc?.d/K* scripts.
5281 	 */
5282 	if (rl == current_runlevel) {
5283 		ms = NULL;
5284 		goto out;
5285 	}
5286 
5287 	log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5288 
5289 	/*
5290 	 * Make sure stop rc scripts see the new settings via who -r.
5291 	 */
5292 	utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5293 
5294 	/*
5295 	 * Some run levels don't have a direct correspondence to any
5296 	 * milestones, so we have to signal init directly.
5297 	 */
5298 	if (mark_rl) {
5299 		current_runlevel = rl;
5300 		signal_init(rl);
5301 	}
5302 
5303 	switch (rl) {
5304 	case 'S':
5305 		uu_warn("The system is coming down for administration.  "
5306 		    "Please wait.\n");
5307 		fork_rc_script(rl, stop, B_FALSE);
5308 		ms = single_user_fmri;
5309 		go_single_user_mode = B_TRUE;
5310 		break;
5311 
5312 	case '0':
5313 		halting_time = time(NULL);
5314 		fork_rc_script(rl, stop, B_TRUE);
5315 		halting = AD_HALT;
5316 		goto uadmin;
5317 
5318 	case '5':
5319 		halting_time = time(NULL);
5320 		fork_rc_script(rl, stop, B_TRUE);
5321 		halting = AD_POWEROFF;
5322 		goto uadmin;
5323 
5324 	case '6':
5325 		halting_time = time(NULL);
5326 		fork_rc_script(rl, stop, B_TRUE);
5327 		if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5328 			halting = AD_FASTREBOOT;
5329 		else
5330 			halting = AD_BOOT;
5331 
5332 uadmin:
5333 		uu_warn("The system is coming down.  Please wait.\n");
5334 		ms = "none";
5335 
5336 		/*
5337 		 * We can't wait until all services are offline since this
5338 		 * thread is responsible for taking them offline.  Instead we
5339 		 * set halting to the second argument for uadmin() and call
5340 		 * do_uadmin() from dgraph_set_instance_state() when
5341 		 * appropriate.
5342 		 */
5343 		break;
5344 
5345 	case '1':
5346 		if (current_runlevel != 'S') {
5347 			uu_warn("Changing to state 1.\n");
5348 			fork_rc_script(rl, stop, B_FALSE);
5349 		} else {
5350 			uu_warn("The system is coming up for administration.  "
5351 			    "Please wait.\n");
5352 		}
5353 		ms = single_user_fmri;
5354 		go_to_level1 = B_TRUE;
5355 		break;
5356 
5357 	case '2':
5358 		if (current_runlevel == '3' || current_runlevel == '4')
5359 			fork_rc_script(rl, stop, B_FALSE);
5360 		ms = multi_user_fmri;
5361 		break;
5362 
5363 	case '3':
5364 	case '4':
5365 		ms = "all";
5366 		break;
5367 
5368 	default:
5369 #ifndef NDEBUG
5370 		(void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5371 		    __FILE__, __LINE__, rl, rl);
5372 #endif
5373 		abort();
5374 	}
5375 
5376 out:
5377 	MUTEX_UNLOCK(&dgraph_lock);
5378 
5379 nolock_out:
5380 	switch (r = libscf_clear_runlevel(pg, ms)) {
5381 	case 0:
5382 		break;
5383 
5384 	case ECONNABORTED:
5385 		libscf_handle_rebind(h);
5386 		rebound = B_TRUE;
5387 		goto nolock_out;
5388 
5389 	case ECANCELED:
5390 		break;
5391 
5392 	case EPERM:
5393 	case EACCES:
5394 	case EROFS:
5395 		log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5396 		    "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5397 		break;
5398 
5399 	default:
5400 		bad_error("libscf_clear_runlevel", r);
5401 	}
5402 
5403 	return (rebound ? ECONNRESET : 0);
5404 }
5405 
5406 /*
5407  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5408  * to the instances that are supposed to go offline during an
5409  * administrative disable operation.
5410  */
5411 static int
5412 mark_subtree(graph_edge_t *e, void *arg)
5413 {
5414 	graph_vertex_t *v;
5415 	int r;
5416 
5417 	v = e->ge_vertex;
5418 
5419 	/* If it's already in the subgraph, skip. */
5420 	if (v->gv_flags & GV_TOOFFLINE)
5421 		return (UU_WALK_NEXT);
5422 
5423 	switch (v->gv_type) {
5424 	case GVT_INST:
5425 		/* If the instance is already offline, skip it. */
5426 		if (!inst_running(v))
5427 			return (UU_WALK_NEXT);
5428 
5429 		v->gv_flags |= GV_TOOFFLINE;
5430 		log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5431 		break;
5432 	case GVT_GROUP:
5433 		/*
5434 		 * Skip all excluded dependents and decide whether to offline
5435 		 * the service based on the restart_on attribute.
5436 		 */
5437 		if (is_depgrp_bypassed(v))
5438 			return (UU_WALK_NEXT);
5439 		break;
5440 	}
5441 
5442 	r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5443 	    0);
5444 	assert(r == 0);
5445 	return (UU_WALK_NEXT);
5446 }
5447 
5448 static int
5449 mark_subgraph(graph_edge_t *e, void *arg)
5450 {
5451 	graph_vertex_t *v;
5452 	int r;
5453 	int optional = (int)arg;
5454 
5455 	v = e->ge_vertex;
5456 
5457 	/* If it's already in the subgraph, skip. */
5458 	if (v->gv_flags & GV_INSUBGRAPH)
5459 		return (UU_WALK_NEXT);
5460 
5461 	/*
5462 	 * Keep track if walk has entered an optional dependency group
5463 	 */
5464 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5465 		optional = 1;
5466 	}
5467 	/*
5468 	 * Quit if we are in an optional dependency group and the instance
5469 	 * is disabled
5470 	 */
5471 	if (optional && (v->gv_type == GVT_INST) &&
5472 	    (!(v->gv_flags & GV_ENBLD_NOOVR)))
5473 		return (UU_WALK_NEXT);
5474 
5475 	v->gv_flags |= GV_INSUBGRAPH;
5476 
5477 	/* Skip all excluded dependencies. */
5478 	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5479 		return (UU_WALK_NEXT);
5480 
5481 	r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5482 	    (void *)optional, 0);
5483 	assert(r == 0);
5484 	return (UU_WALK_NEXT);
5485 }
5486 
5487 /*
5488  * Bring down all services which are not dependencies of fmri.  The
5489  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5490  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5491  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5492  * could consider "offline" to be down, and refrain from sending start
5493  * commands for such services, but that's not strictly necessary, so we'll
5494  * decline to intrude on the state machine.  It would probably confuse users
5495  * anyway.
5496  *
5497  * The services should be brought down in reverse-dependency order, so we
5498  * can't do it all at once here.  We initiate by override-disabling the leaves
5499  * of the dependency tree -- those services which are up but have no
5500  * dependents which are up.  When they come down,
5501  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5502  * exposed leaves.  Perseverance will ensure completion.
5503  *
5504  * Sometimes we need to take action when the transition is complete, like
5505  * start sulogin or halt the system.  To tell when we're done, we initialize
5506  * non_subgraph_svcs here to be the number of services which need to come
5507  * down.  As each does, we decrement the counter.  When it hits zero, we take
5508  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5509  *
5510  * In case we're coming up, we also remove any enable-overrides for the
5511  * services which are dependencies of fmri.
5512  *
5513  * If norepository is true, the function will not change the repository.
5514  *
5515  * The decision to change the system run level in accordance with the milestone
5516  * is taken in dgraph_set_runlevel().
5517  *
5518  * Returns
5519  *   0 - success
5520  *   ECONNRESET - success, but handle was rebound
5521  *   EINVAL - fmri is invalid (error is logged)
5522  *   EALREADY - the milestone is already set to fmri
5523  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5524  */
5525 static int
5526 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5527 {
5528 	const char *cfmri, *fs;
5529 	graph_vertex_t *nm, *v;
5530 	int ret = 0, r;
5531 	scf_instance_t *inst;
5532 	boolean_t isall, isnone, rebound = B_FALSE;
5533 
5534 	/* Validate fmri */
5535 	isall = (strcmp(fmri, "all") == 0);
5536 	isnone = (strcmp(fmri, "none") == 0);
5537 
5538 	if (!isall && !isnone) {
5539 		if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5540 			goto reject;
5541 
5542 		if (strcmp(cfmri, single_user_fmri) != 0 &&
5543 		    strcmp(cfmri, multi_user_fmri) != 0 &&
5544 		    strcmp(cfmri, multi_user_svr_fmri) != 0) {
5545 			startd_free((void *)cfmri, max_scf_fmri_size);
5546 reject:
5547 			log_framework(LOG_WARNING,
5548 			    "Rejecting request for invalid milestone \"%s\".\n",
5549 			    fmri);
5550 			return (EINVAL);
5551 		}
5552 	}
5553 
5554 	inst = safe_scf_instance_create(h);
5555 
5556 	MUTEX_LOCK(&dgraph_lock);
5557 
5558 	if (milestone == NULL) {
5559 		if (isall) {
5560 			log_framework(LOG_DEBUG,
5561 			    "Milestone already set to all.\n");
5562 			ret = EALREADY;
5563 			goto out;
5564 		}
5565 	} else if (milestone == MILESTONE_NONE) {
5566 		if (isnone) {
5567 			log_framework(LOG_DEBUG,
5568 			    "Milestone already set to none.\n");
5569 			ret = EALREADY;
5570 			goto out;
5571 		}
5572 	} else {
5573 		if (!isall && !isnone &&
5574 		    strcmp(cfmri, milestone->gv_name) == 0) {
5575 			log_framework(LOG_DEBUG,
5576 			    "Milestone already set to %s.\n", cfmri);
5577 			ret = EALREADY;
5578 			goto out;
5579 		}
5580 	}
5581 
5582 	if (!isall && !isnone) {
5583 		nm = vertex_get_by_name(cfmri);
5584 		if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5585 			log_framework(LOG_WARNING, "Cannot set milestone to %s "
5586 			    "because no such service exists.\n", cfmri);
5587 			ret = ENOENT;
5588 			goto out;
5589 		}
5590 	}
5591 
5592 	log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5593 
5594 	/*
5595 	 * Set milestone, removing the old one if this was the last reference.
5596 	 */
5597 	if (milestone > MILESTONE_NONE)
5598 		(void) vertex_unref(milestone);
5599 
5600 	if (isall)
5601 		milestone = NULL;
5602 	else if (isnone)
5603 		milestone = MILESTONE_NONE;
5604 	else {
5605 		milestone = nm;
5606 		/* milestone should count as a reference */
5607 		vertex_ref(milestone);
5608 	}
5609 
5610 	/* Clear all GV_INSUBGRAPH bits. */
5611 	for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5612 		v->gv_flags &= ~GV_INSUBGRAPH;
5613 
5614 	if (!isall && !isnone) {
5615 		/* Set GV_INSUBGRAPH for milestone & descendents. */
5616 		milestone->gv_flags |= GV_INSUBGRAPH;
5617 
5618 		r = uu_list_walk(milestone->gv_dependencies,
5619 		    (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5620 		assert(r == 0);
5621 	}
5622 
5623 	/* Un-override services in the subgraph & override-disable the rest. */
5624 	if (norepository)
5625 		goto out;
5626 
5627 	non_subgraph_svcs = 0;
5628 	for (v = uu_list_first(dgraph);
5629 	    v != NULL;
5630 	    v = uu_list_next(dgraph, v)) {
5631 		if (v->gv_type != GVT_INST ||
5632 		    (v->gv_flags & GV_CONFIGURED) == 0)
5633 			continue;
5634 
5635 again:
5636 		r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5637 		    NULL, NULL, SCF_DECODE_FMRI_EXACT);
5638 		if (r != 0) {
5639 			switch (scf_error()) {
5640 			case SCF_ERROR_CONNECTION_BROKEN:
5641 			default:
5642 				libscf_handle_rebind(h);
5643 				rebound = B_TRUE;
5644 				goto again;
5645 
5646 			case SCF_ERROR_NOT_FOUND:
5647 				continue;
5648 
5649 			case SCF_ERROR_HANDLE_MISMATCH:
5650 			case SCF_ERROR_INVALID_ARGUMENT:
5651 			case SCF_ERROR_CONSTRAINT_VIOLATED:
5652 			case SCF_ERROR_NOT_BOUND:
5653 				bad_error("scf_handle_decode_fmri",
5654 				    scf_error());
5655 			}
5656 		}
5657 
5658 		if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5659 			r = libscf_delete_enable_ovr(inst);
5660 			fs = "libscf_delete_enable_ovr";
5661 		} else {
5662 			assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5663 
5664 			/*
5665 			 * Services which are up need to come down before
5666 			 * we're done, but we can only disable the leaves
5667 			 * here.
5668 			 */
5669 
5670 			if (up_state(v->gv_state))
5671 				++non_subgraph_svcs;
5672 
5673 			/* If it's already disabled, don't bother. */
5674 			if ((v->gv_flags & GV_ENABLED) == 0)
5675 				continue;
5676 
5677 			if (!is_nonsubgraph_leaf(v))
5678 				continue;
5679 
5680 			r = libscf_set_enable_ovr(inst, 0);
5681 			fs = "libscf_set_enable_ovr";
5682 		}
5683 		switch (r) {
5684 		case 0:
5685 		case ECANCELED:
5686 			break;
5687 
5688 		case ECONNABORTED:
5689 			libscf_handle_rebind(h);
5690 			rebound = B_TRUE;
5691 			goto again;
5692 
5693 		case EPERM:
5694 		case EROFS:
5695 			log_error(LOG_WARNING,
5696 			    "Could not set %s/%s for %s: %s.\n",
5697 			    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5698 			    v->gv_name, strerror(r));
5699 			break;
5700 
5701 		default:
5702 			bad_error(fs, r);
5703 		}
5704 	}
5705 
5706 	if (halting != -1) {
5707 		if (non_subgraph_svcs > 1)
5708 			uu_warn("%d system services are now being stopped.\n",
5709 			    non_subgraph_svcs);
5710 		else if (non_subgraph_svcs == 1)
5711 			uu_warn("One system service is now being stopped.\n");
5712 		else if (non_subgraph_svcs == 0)
5713 			do_uadmin();
5714 	}
5715 
5716 	ret = rebound ? ECONNRESET : 0;
5717 
5718 out:
5719 	MUTEX_UNLOCK(&dgraph_lock);
5720 	if (!isall && !isnone)
5721 		startd_free((void *)cfmri, max_scf_fmri_size);
5722 	scf_instance_destroy(inst);
5723 	return (ret);
5724 }
5725 
5726 
5727 /*
5728  * Returns 0, ECONNABORTED, or EINVAL.
5729  */
5730 static int
5731 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5732 {
5733 	int r;
5734 
5735 	switch (e->gpe_type) {
5736 	case GRAPH_UPDATE_RELOAD_GRAPH:
5737 		log_error(LOG_WARNING,
5738 		    "graph_event: reload graph unimplemented\n");
5739 		break;
5740 
5741 	case GRAPH_UPDATE_STATE_CHANGE: {
5742 		protocol_states_t *states = e->gpe_data;
5743 
5744 		switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5745 		case 0:
5746 		case ENOENT:
5747 			break;
5748 
5749 		case ECONNABORTED:
5750 			return (ECONNABORTED);
5751 
5752 		case EINVAL:
5753 		default:
5754 #ifndef NDEBUG
5755 			(void) fprintf(stderr, "dgraph_set_instance_state() "
5756 			    "failed with unexpected error %d at %s:%d.\n", r,
5757 			    __FILE__, __LINE__);
5758 #endif
5759 			abort();
5760 		}
5761 
5762 		startd_free(states, sizeof (protocol_states_t));
5763 		break;
5764 	}
5765 
5766 	default:
5767 		log_error(LOG_WARNING,
5768 		    "graph_event_loop received an unknown event: %d\n",
5769 		    e->gpe_type);
5770 		break;
5771 	}
5772 
5773 	return (0);
5774 }
5775 
5776 /*
5777  * graph_event_thread()
5778  *    Wait for state changes from the restarters.
5779  */
5780 /*ARGSUSED*/
5781 void *
5782 graph_event_thread(void *unused)
5783 {
5784 	scf_handle_t *h;
5785 	int err;
5786 
5787 	(void) pthread_setname_np(pthread_self(), "graph_event");
5788 
5789 	h = libscf_handle_create_bound_loop();
5790 
5791 	/*CONSTCOND*/
5792 	while (1) {
5793 		graph_protocol_event_t *e;
5794 
5795 		MUTEX_LOCK(&gu->gu_lock);
5796 
5797 		while (gu->gu_wakeup == 0)
5798 			(void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5799 
5800 		gu->gu_wakeup = 0;
5801 
5802 		while ((e = graph_event_dequeue()) != NULL) {
5803 			MUTEX_LOCK(&e->gpe_lock);
5804 			MUTEX_UNLOCK(&gu->gu_lock);
5805 
5806 			while ((err = handle_graph_update_event(h, e)) ==
5807 			    ECONNABORTED)
5808 				libscf_handle_rebind(h);
5809 
5810 			if (err == 0)
5811 				graph_event_release(e);
5812 			else
5813 				graph_event_requeue(e);
5814 
5815 			MUTEX_LOCK(&gu->gu_lock);
5816 		}
5817 
5818 		MUTEX_UNLOCK(&gu->gu_lock);
5819 	}
5820 }
5821 
5822 static void
5823 set_initial_milestone(scf_handle_t *h)
5824 {
5825 	scf_instance_t *inst;
5826 	char *fmri, *cfmri;
5827 	size_t sz;
5828 	int r;
5829 
5830 	inst = safe_scf_instance_create(h);
5831 	fmri = startd_alloc(max_scf_fmri_size);
5832 
5833 	/*
5834 	 * If -m milestone= was specified, we want to set options_ovr/milestone
5835 	 * to it.  Otherwise we want to read what the milestone should be set
5836 	 * to.  Either way we need our inst.
5837 	 */
5838 get_self:
5839 	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5840 	    NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5841 		switch (scf_error()) {
5842 		case SCF_ERROR_CONNECTION_BROKEN:
5843 			libscf_handle_rebind(h);
5844 			goto get_self;
5845 
5846 		case SCF_ERROR_NOT_FOUND:
5847 			if (st->st_subgraph != NULL &&
5848 			    st->st_subgraph[0] != '\0') {
5849 				sz = strlcpy(fmri, st->st_subgraph,
5850 				    max_scf_fmri_size);
5851 				assert(sz < max_scf_fmri_size);
5852 			} else {
5853 				fmri[0] = '\0';
5854 			}
5855 			break;
5856 
5857 		case SCF_ERROR_INVALID_ARGUMENT:
5858 		case SCF_ERROR_CONSTRAINT_VIOLATED:
5859 		case SCF_ERROR_HANDLE_MISMATCH:
5860 		default:
5861 			bad_error("scf_handle_decode_fmri", scf_error());
5862 		}
5863 	} else {
5864 		if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5865 			scf_propertygroup_t *pg;
5866 
5867 			pg = safe_scf_pg_create(h);
5868 
5869 			sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5870 			assert(sz < max_scf_fmri_size);
5871 
5872 			r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5873 			    SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5874 			    pg);
5875 			switch (r) {
5876 			case 0:
5877 				break;
5878 
5879 			case ECONNABORTED:
5880 				libscf_handle_rebind(h);
5881 				goto get_self;
5882 
5883 			case EPERM:
5884 			case EACCES:
5885 			case EROFS:
5886 				log_error(LOG_WARNING, "Could not set %s/%s: "
5887 				    "%s.\n", SCF_PG_OPTIONS_OVR,
5888 				    SCF_PROPERTY_MILESTONE, strerror(r));
5889 				/* FALLTHROUGH */
5890 
5891 			case ECANCELED:
5892 				sz = strlcpy(fmri, st->st_subgraph,
5893 				    max_scf_fmri_size);
5894 				assert(sz < max_scf_fmri_size);
5895 				break;
5896 
5897 			default:
5898 				bad_error("libscf_inst_get_or_add_pg", r);
5899 			}
5900 
5901 			r = libscf_clear_runlevel(pg, fmri);
5902 			switch (r) {
5903 			case 0:
5904 				break;
5905 
5906 			case ECONNABORTED:
5907 				libscf_handle_rebind(h);
5908 				goto get_self;
5909 
5910 			case EPERM:
5911 			case EACCES:
5912 			case EROFS:
5913 				log_error(LOG_WARNING, "Could not set %s/%s: "
5914 				    "%s.\n", SCF_PG_OPTIONS_OVR,
5915 				    SCF_PROPERTY_MILESTONE, strerror(r));
5916 				/* FALLTHROUGH */
5917 
5918 			case ECANCELED:
5919 				sz = strlcpy(fmri, st->st_subgraph,
5920 				    max_scf_fmri_size);
5921 				assert(sz < max_scf_fmri_size);
5922 				break;
5923 
5924 			default:
5925 				bad_error("libscf_clear_runlevel", r);
5926 			}
5927 
5928 			scf_pg_destroy(pg);
5929 		} else {
5930 			scf_property_t *prop;
5931 			scf_value_t *val;
5932 
5933 			prop = safe_scf_property_create(h);
5934 			val = safe_scf_value_create(h);
5935 
5936 			r = libscf_get_milestone(inst, prop, val, fmri,
5937 			    max_scf_fmri_size);
5938 			switch (r) {
5939 			case 0:
5940 				break;
5941 
5942 			case ECONNABORTED:
5943 				libscf_handle_rebind(h);
5944 				goto get_self;
5945 
5946 			case EINVAL:
5947 				log_error(LOG_WARNING, "Milestone property is "
5948 				    "misconfigured.  Defaulting to \"all\".\n");
5949 				/* FALLTHROUGH */
5950 
5951 			case ECANCELED:
5952 			case ENOENT:
5953 				fmri[0] = '\0';
5954 				break;
5955 
5956 			default:
5957 				bad_error("libscf_get_milestone", r);
5958 			}
5959 
5960 			scf_value_destroy(val);
5961 			scf_property_destroy(prop);
5962 		}
5963 	}
5964 
5965 	if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5966 		goto out;
5967 
5968 	if (strcmp(fmri, "none") != 0) {
5969 retry:
5970 		if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5971 		    NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5972 			switch (scf_error()) {
5973 			case SCF_ERROR_INVALID_ARGUMENT:
5974 				log_error(LOG_WARNING,
5975 				    "Requested milestone \"%s\" is invalid.  "
5976 				    "Reverting to \"all\".\n", fmri);
5977 				goto out;
5978 
5979 			case SCF_ERROR_CONSTRAINT_VIOLATED:
5980 				log_error(LOG_WARNING, "Requested milestone "
5981 				    "\"%s\" does not specify an instance.  "
5982 				    "Reverting to \"all\".\n", fmri);
5983 				goto out;
5984 
5985 			case SCF_ERROR_CONNECTION_BROKEN:
5986 				libscf_handle_rebind(h);
5987 				goto retry;
5988 
5989 			case SCF_ERROR_NOT_FOUND:
5990 				log_error(LOG_WARNING, "Requested milestone "
5991 				    "\"%s\" not in repository.  Reverting to "
5992 				    "\"all\".\n", fmri);
5993 				goto out;
5994 
5995 			case SCF_ERROR_HANDLE_MISMATCH:
5996 			default:
5997 				bad_error("scf_handle_decode_fmri",
5998 				    scf_error());
5999 			}
6000 		}
6001 
6002 		r = fmri_canonify(fmri, &cfmri, B_FALSE);
6003 		assert(r == 0);
6004 
6005 		r = dgraph_add_instance(cfmri, inst, B_TRUE);
6006 		startd_free(cfmri, max_scf_fmri_size);
6007 		switch (r) {
6008 		case 0:
6009 			break;
6010 
6011 		case ECONNABORTED:
6012 			goto retry;
6013 
6014 		case EINVAL:
6015 			log_error(LOG_WARNING,
6016 			    "Requested milestone \"%s\" is invalid.  "
6017 			    "Reverting to \"all\".\n", fmri);
6018 			goto out;
6019 
6020 		case ECANCELED:
6021 			log_error(LOG_WARNING,
6022 			    "Requested milestone \"%s\" not "
6023 			    "in repository.  Reverting to \"all\".\n",
6024 			    fmri);
6025 			goto out;
6026 
6027 		case EEXIST:
6028 		default:
6029 			bad_error("dgraph_add_instance", r);
6030 		}
6031 	}
6032 
6033 	log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6034 
6035 	r = dgraph_set_milestone(fmri, h, B_FALSE);
6036 	switch (r) {
6037 	case 0:
6038 	case ECONNRESET:
6039 	case EALREADY:
6040 		break;
6041 
6042 	case EINVAL:
6043 	case ENOENT:
6044 	default:
6045 		bad_error("dgraph_set_milestone", r);
6046 	}
6047 
6048 out:
6049 	startd_free(fmri, max_scf_fmri_size);
6050 	scf_instance_destroy(inst);
6051 }
6052 
6053 void
6054 set_restart_milestone(scf_handle_t *h)
6055 {
6056 	scf_instance_t *inst;
6057 	scf_property_t *prop;
6058 	scf_value_t *val;
6059 	char *fmri;
6060 	int r;
6061 
6062 	inst = safe_scf_instance_create(h);
6063 
6064 get_self:
6065 	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6066 	    inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6067 		switch (scf_error()) {
6068 		case SCF_ERROR_CONNECTION_BROKEN:
6069 			libscf_handle_rebind(h);
6070 			goto get_self;
6071 
6072 		case SCF_ERROR_NOT_FOUND:
6073 			break;
6074 
6075 		case SCF_ERROR_INVALID_ARGUMENT:
6076 		case SCF_ERROR_CONSTRAINT_VIOLATED:
6077 		case SCF_ERROR_HANDLE_MISMATCH:
6078 		default:
6079 			bad_error("scf_handle_decode_fmri", scf_error());
6080 		}
6081 
6082 		scf_instance_destroy(inst);
6083 		return;
6084 	}
6085 
6086 	prop = safe_scf_property_create(h);
6087 	val = safe_scf_value_create(h);
6088 	fmri = startd_alloc(max_scf_fmri_size);
6089 
6090 	r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6091 	switch (r) {
6092 	case 0:
6093 		break;
6094 
6095 	case ECONNABORTED:
6096 		libscf_handle_rebind(h);
6097 		goto get_self;
6098 
6099 	case ECANCELED:
6100 	case ENOENT:
6101 	case EINVAL:
6102 		goto out;
6103 
6104 	default:
6105 		bad_error("libscf_get_milestone", r);
6106 	}
6107 
6108 	r = dgraph_set_milestone(fmri, h, B_TRUE);
6109 	switch (r) {
6110 	case 0:
6111 	case ECONNRESET:
6112 	case EALREADY:
6113 	case EINVAL:
6114 	case ENOENT:
6115 		break;
6116 
6117 	default:
6118 		bad_error("dgraph_set_milestone", r);
6119 	}
6120 
6121 out:
6122 	startd_free(fmri, max_scf_fmri_size);
6123 	scf_value_destroy(val);
6124 	scf_property_destroy(prop);
6125 	scf_instance_destroy(inst);
6126 }
6127 
6128 /*
6129  * void *graph_thread(void *)
6130  *
6131  * Graph management thread.
6132  */
6133 /*ARGSUSED*/
6134 void *
6135 graph_thread(void *arg)
6136 {
6137 	scf_handle_t *h;
6138 	int err;
6139 
6140 	(void) pthread_setname_np(pthread_self(), "graph");
6141 
6142 	h = libscf_handle_create_bound_loop();
6143 
6144 	if (st->st_initial)
6145 		set_initial_milestone(h);
6146 
6147 	MUTEX_LOCK(&dgraph_lock);
6148 	initial_milestone_set = B_TRUE;
6149 	err = pthread_cond_broadcast(&initial_milestone_cv);
6150 	assert(err == 0);
6151 	MUTEX_UNLOCK(&dgraph_lock);
6152 
6153 	libscf_populate_graph(h);
6154 
6155 	if (!st->st_initial)
6156 		set_restart_milestone(h);
6157 
6158 	MUTEX_LOCK(&st->st_load_lock);
6159 	st->st_load_complete = 1;
6160 	(void) pthread_cond_broadcast(&st->st_load_cv);
6161 	MUTEX_UNLOCK(&st->st_load_lock);
6162 
6163 	MUTEX_LOCK(&dgraph_lock);
6164 	/*
6165 	 * Now that we've set st_load_complete we need to check can_come_up()
6166 	 * since if we booted to a milestone, then there won't be any more
6167 	 * state updates.
6168 	 */
6169 	if (!go_single_user_mode && !go_to_level1 &&
6170 	    halting == -1) {
6171 		if (!sulogin_thread_running && !can_come_up()) {
6172 			(void) startd_thread_create(sulogin_thread, NULL);
6173 			sulogin_thread_running = B_TRUE;
6174 		}
6175 	}
6176 	MUTEX_UNLOCK(&dgraph_lock);
6177 
6178 	(void) pthread_mutex_lock(&gu->gu_freeze_lock);
6179 
6180 	/*CONSTCOND*/
6181 	while (1) {
6182 		(void) pthread_cond_wait(&gu->gu_freeze_cv,
6183 		    &gu->gu_freeze_lock);
6184 	}
6185 }
6186 
6187 
6188 /*
6189  * int next_action()
6190  *   Given an array of timestamps 'a' with 'num' elements, find the
6191  *   lowest non-zero timestamp and return its index. If there are no
6192  *   non-zero elements, return -1.
6193  */
6194 static int
6195 next_action(hrtime_t *a, int num)
6196 {
6197 	hrtime_t t = 0;
6198 	int i = 0, smallest = -1;
6199 
6200 	for (i = 0; i < num; i++) {
6201 		if (t == 0) {
6202 			t = a[i];
6203 			smallest = i;
6204 		} else if (a[i] != 0 && a[i] < t) {
6205 			t = a[i];
6206 			smallest = i;
6207 		}
6208 	}
6209 
6210 	if (t == 0)
6211 		return (-1);
6212 	else
6213 		return (smallest);
6214 }
6215 
6216 /*
6217  * void process_actions()
6218  *   Process actions requested by the administrator. Possibilities include:
6219  *   refresh, restart, maintenance mode off, maintenance mode on,
6220  *   maintenance mode immediate, and degraded.
6221  *
6222  *   The set of pending actions is represented in the repository as a
6223  *   per-instance property group, with each action being a single property
6224  *   in that group.  This property group is converted to an array, with each
6225  *   action type having an array slot.  The actions in the array at the
6226  *   time process_actions() is called are acted on in the order of the
6227  *   timestamp (which is the value stored in the slot).  A value of zero
6228  *   indicates that there is no pending action of the type associated with
6229  *   a particular slot.
6230  *
6231  *   Sending an action event multiple times before the restarter has a
6232  *   chance to process that action will force it to be run at the last
6233  *   timestamp where it appears in the ordering.
6234  *
6235  *   Turning maintenance mode on trumps all other actions.
6236  *
6237  *   Returns 0 or ECONNABORTED.
6238  */
6239 static int
6240 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6241 {
6242 	scf_property_t *prop = NULL;
6243 	scf_value_t *val = NULL;
6244 	scf_type_t type;
6245 	graph_vertex_t *vertex;
6246 	admin_action_t a;
6247 	int i, ret = 0, r;
6248 	hrtime_t action_ts[NACTIONS];
6249 	char *inst_name;
6250 
6251 	r = libscf_instance_get_fmri(inst, &inst_name);
6252 	switch (r) {
6253 	case 0:
6254 		break;
6255 
6256 	case ECONNABORTED:
6257 		return (ECONNABORTED);
6258 
6259 	case ECANCELED:
6260 		return (0);
6261 
6262 	default:
6263 		bad_error("libscf_instance_get_fmri", r);
6264 	}
6265 
6266 	MUTEX_LOCK(&dgraph_lock);
6267 
6268 	vertex = vertex_get_by_name(inst_name);
6269 	if (vertex == NULL) {
6270 		MUTEX_UNLOCK(&dgraph_lock);
6271 		log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6272 		    "The instance must have been removed.\n", inst_name);
6273 		startd_free(inst_name, max_scf_fmri_size);
6274 		return (0);
6275 	}
6276 
6277 	prop = safe_scf_property_create(h);
6278 	val = safe_scf_value_create(h);
6279 
6280 	for (i = 0; i < NACTIONS; i++) {
6281 		if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6282 			switch (scf_error()) {
6283 			case SCF_ERROR_CONNECTION_BROKEN:
6284 			default:
6285 				ret = ECONNABORTED;
6286 				goto out;
6287 
6288 			case SCF_ERROR_DELETED:
6289 				goto out;
6290 
6291 			case SCF_ERROR_NOT_FOUND:
6292 				action_ts[i] = 0;
6293 				continue;
6294 
6295 			case SCF_ERROR_HANDLE_MISMATCH:
6296 			case SCF_ERROR_INVALID_ARGUMENT:
6297 			case SCF_ERROR_NOT_SET:
6298 				bad_error("scf_pg_get_property", scf_error());
6299 			}
6300 		}
6301 
6302 		if (scf_property_type(prop, &type) != 0) {
6303 			switch (scf_error()) {
6304 			case SCF_ERROR_CONNECTION_BROKEN:
6305 			default:
6306 				ret = ECONNABORTED;
6307 				goto out;
6308 
6309 			case SCF_ERROR_DELETED:
6310 				action_ts[i] = 0;
6311 				continue;
6312 
6313 			case SCF_ERROR_NOT_SET:
6314 				bad_error("scf_property_type", scf_error());
6315 			}
6316 		}
6317 
6318 		if (type != SCF_TYPE_INTEGER) {
6319 			action_ts[i] = 0;
6320 			continue;
6321 		}
6322 
6323 		if (scf_property_get_value(prop, val) != 0) {
6324 			switch (scf_error()) {
6325 			case SCF_ERROR_CONNECTION_BROKEN:
6326 			default:
6327 				ret = ECONNABORTED;
6328 				goto out;
6329 
6330 			case SCF_ERROR_DELETED:
6331 				goto out;
6332 
6333 			case SCF_ERROR_NOT_FOUND:
6334 			case SCF_ERROR_CONSTRAINT_VIOLATED:
6335 				action_ts[i] = 0;
6336 				continue;
6337 
6338 			case SCF_ERROR_NOT_SET:
6339 			case SCF_ERROR_PERMISSION_DENIED:
6340 				bad_error("scf_property_get_value",
6341 				    scf_error());
6342 			}
6343 		}
6344 
6345 		r = scf_value_get_integer(val, &action_ts[i]);
6346 		assert(r == 0);
6347 	}
6348 
6349 	a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6350 	if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6351 	    action_ts[ADMIN_EVENT_MAINT_ON]) {
6352 		a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6353 		    ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6354 
6355 		vertex_send_event(vertex, admin_events[a]);
6356 		r = libscf_unset_action(h, pg, a, action_ts[a]);
6357 		switch (r) {
6358 		case 0:
6359 		case EACCES:
6360 			break;
6361 
6362 		case ECONNABORTED:
6363 			ret = ECONNABORTED;
6364 			goto out;
6365 
6366 		case EPERM:
6367 			uu_die("Insufficient privilege.\n");
6368 			/* NOTREACHED */
6369 
6370 		default:
6371 			bad_error("libscf_unset_action", r);
6372 		}
6373 	}
6374 
6375 	while ((a = next_action(action_ts, NACTIONS)) != -1) {
6376 		log_framework(LOG_DEBUG,
6377 		    "Graph: processing %s action for %s.\n", admin_actions[a],
6378 		    inst_name);
6379 
6380 		if (a == ADMIN_EVENT_REFRESH) {
6381 			r = dgraph_refresh_instance(vertex, inst);
6382 			switch (r) {
6383 			case 0:
6384 			case ECANCELED:
6385 			case EINVAL:
6386 			case -1:
6387 				break;
6388 
6389 			case ECONNABORTED:
6390 				/* pg & inst are reset now, so just return. */
6391 				ret = ECONNABORTED;
6392 				goto out;
6393 
6394 			default:
6395 				bad_error("dgraph_refresh_instance", r);
6396 			}
6397 		}
6398 
6399 		vertex_send_event(vertex, admin_events[a]);
6400 
6401 		r = libscf_unset_action(h, pg, a, action_ts[a]);
6402 		switch (r) {
6403 		case 0:
6404 		case EACCES:
6405 			break;
6406 
6407 		case ECONNABORTED:
6408 			ret = ECONNABORTED;
6409 			goto out;
6410 
6411 		case EPERM:
6412 			uu_die("Insufficient privilege.\n");
6413 			/* NOTREACHED */
6414 
6415 		default:
6416 			bad_error("libscf_unset_action", r);
6417 		}
6418 
6419 		action_ts[a] = 0;
6420 	}
6421 
6422 out:
6423 	MUTEX_UNLOCK(&dgraph_lock);
6424 
6425 	scf_property_destroy(prop);
6426 	scf_value_destroy(val);
6427 	startd_free(inst_name, max_scf_fmri_size);
6428 	return (ret);
6429 }
6430 
6431 /*
6432  * inst and pg_name are scratch space, and are unset on entry.
6433  * Returns
6434  *   0 - success
6435  *   ECONNRESET - success, but repository handle rebound
6436  *   ECONNABORTED - repository connection broken
6437  */
6438 static int
6439 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6440     char *pg_name)
6441 {
6442 	int r;
6443 	scf_property_t *prop;
6444 	scf_value_t *val;
6445 	char *fmri;
6446 	boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6447 
6448 	if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6449 		switch (scf_error()) {
6450 		case SCF_ERROR_CONNECTION_BROKEN:
6451 		default:
6452 			return (ECONNABORTED);
6453 
6454 		case SCF_ERROR_DELETED:
6455 			return (0);
6456 
6457 		case SCF_ERROR_NOT_SET:
6458 			bad_error("scf_pg_get_name", scf_error());
6459 		}
6460 	}
6461 
6462 	if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6463 	    strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6464 		r = dgraph_update_general(pg);
6465 		switch (r) {
6466 		case 0:
6467 		case ENOTSUP:
6468 		case ECANCELED:
6469 			return (0);
6470 
6471 		case ECONNABORTED:
6472 			return (ECONNABORTED);
6473 
6474 		case -1:
6475 			/* Error should have been logged. */
6476 			return (0);
6477 
6478 		default:
6479 			bad_error("dgraph_update_general", r);
6480 		}
6481 	} else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6482 		if (scf_pg_get_parent_instance(pg, inst) != 0) {
6483 			switch (scf_error()) {
6484 			case SCF_ERROR_CONNECTION_BROKEN:
6485 				return (ECONNABORTED);
6486 
6487 			case SCF_ERROR_DELETED:
6488 			case SCF_ERROR_CONSTRAINT_VIOLATED:
6489 				/* Ignore commands on services. */
6490 				return (0);
6491 
6492 			case SCF_ERROR_NOT_BOUND:
6493 			case SCF_ERROR_HANDLE_MISMATCH:
6494 			case SCF_ERROR_NOT_SET:
6495 			default:
6496 				bad_error("scf_pg_get_parent_instance",
6497 				    scf_error());
6498 			}
6499 		}
6500 
6501 		return (process_actions(h, pg, inst));
6502 	}
6503 
6504 	if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6505 	    strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6506 		return (0);
6507 
6508 	/*
6509 	 * We only care about the options[_ovr] property groups of our own
6510 	 * instance, so get the fmri and compare.  Plus, once we know it's
6511 	 * correct, if the repository connection is broken we know exactly what
6512 	 * property group we were operating on, and can look it up again.
6513 	 */
6514 	if (scf_pg_get_parent_instance(pg, inst) != 0) {
6515 		switch (scf_error()) {
6516 		case SCF_ERROR_CONNECTION_BROKEN:
6517 			return (ECONNABORTED);
6518 
6519 		case SCF_ERROR_DELETED:
6520 		case SCF_ERROR_CONSTRAINT_VIOLATED:
6521 			return (0);
6522 
6523 		case SCF_ERROR_HANDLE_MISMATCH:
6524 		case SCF_ERROR_NOT_BOUND:
6525 		case SCF_ERROR_NOT_SET:
6526 		default:
6527 			bad_error("scf_pg_get_parent_instance",
6528 			    scf_error());
6529 		}
6530 	}
6531 
6532 	switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6533 	case 0:
6534 		break;
6535 
6536 	case ECONNABORTED:
6537 		return (ECONNABORTED);
6538 
6539 	case ECANCELED:
6540 		return (0);
6541 
6542 	default:
6543 		bad_error("libscf_instance_get_fmri", r);
6544 	}
6545 
6546 	if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6547 		startd_free(fmri, max_scf_fmri_size);
6548 		return (0);
6549 	}
6550 
6551 	/*
6552 	 * update the information events flag
6553 	 */
6554 	if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6555 		info_events_all = libscf_get_info_events_all(pg);
6556 
6557 	prop = safe_scf_property_create(h);
6558 	val = safe_scf_value_create(h);
6559 
6560 	if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6561 		/* See if we need to set the runlevel. */
6562 		/* CONSTCOND */
6563 		if (0) {
6564 rebind_pg:
6565 			libscf_handle_rebind(h);
6566 			rebound = B_TRUE;
6567 
6568 			r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6569 			switch (r) {
6570 			case 0:
6571 				break;
6572 
6573 			case ECONNABORTED:
6574 				goto rebind_pg;
6575 
6576 			case ENOENT:
6577 				goto out;
6578 
6579 			case EINVAL:
6580 			case ENOTSUP:
6581 				bad_error("libscf_lookup_instance", r);
6582 			}
6583 
6584 			if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6585 				switch (scf_error()) {
6586 				case SCF_ERROR_DELETED:
6587 				case SCF_ERROR_NOT_FOUND:
6588 					goto out;
6589 
6590 				case SCF_ERROR_CONNECTION_BROKEN:
6591 					goto rebind_pg;
6592 
6593 				case SCF_ERROR_HANDLE_MISMATCH:
6594 				case SCF_ERROR_NOT_BOUND:
6595 				case SCF_ERROR_NOT_SET:
6596 				case SCF_ERROR_INVALID_ARGUMENT:
6597 				default:
6598 					bad_error("scf_instance_get_pg",
6599 					    scf_error());
6600 				}
6601 			}
6602 		}
6603 
6604 		if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6605 			r = dgraph_set_runlevel(pg, prop);
6606 			switch (r) {
6607 			case ECONNRESET:
6608 				rebound = B_TRUE;
6609 				rebind_inst = B_TRUE;
6610 				/* FALLTHROUGH */
6611 
6612 			case 0:
6613 				break;
6614 
6615 			case ECONNABORTED:
6616 				goto rebind_pg;
6617 
6618 			case ECANCELED:
6619 				goto out;
6620 
6621 			default:
6622 				bad_error("dgraph_set_runlevel", r);
6623 			}
6624 		} else {
6625 			switch (scf_error()) {
6626 			case SCF_ERROR_CONNECTION_BROKEN:
6627 			default:
6628 				goto rebind_pg;
6629 
6630 			case SCF_ERROR_DELETED:
6631 				goto out;
6632 
6633 			case SCF_ERROR_NOT_FOUND:
6634 				break;
6635 
6636 			case SCF_ERROR_INVALID_ARGUMENT:
6637 			case SCF_ERROR_HANDLE_MISMATCH:
6638 			case SCF_ERROR_NOT_BOUND:
6639 			case SCF_ERROR_NOT_SET:
6640 				bad_error("scf_pg_get_property", scf_error());
6641 			}
6642 		}
6643 	}
6644 
6645 	if (rebind_inst) {
6646 lookup_inst:
6647 		r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6648 		switch (r) {
6649 		case 0:
6650 			break;
6651 
6652 		case ECONNABORTED:
6653 			libscf_handle_rebind(h);
6654 			rebound = B_TRUE;
6655 			goto lookup_inst;
6656 
6657 		case ENOENT:
6658 			goto out;
6659 
6660 		case EINVAL:
6661 		case ENOTSUP:
6662 			bad_error("libscf_lookup_instance", r);
6663 		}
6664 	}
6665 
6666 	r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6667 	switch (r) {
6668 	case 0:
6669 		break;
6670 
6671 	case ECONNABORTED:
6672 		libscf_handle_rebind(h);
6673 		rebound = B_TRUE;
6674 		goto lookup_inst;
6675 
6676 	case EINVAL:
6677 		log_error(LOG_NOTICE,
6678 		    "%s/%s property of %s is misconfigured.\n", pg_name,
6679 		    SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6680 		/* FALLTHROUGH */
6681 
6682 	case ECANCELED:
6683 	case ENOENT:
6684 		(void) strcpy(fmri, "all");
6685 		break;
6686 
6687 	default:
6688 		bad_error("libscf_get_milestone", r);
6689 	}
6690 
6691 	r = dgraph_set_milestone(fmri, h, B_FALSE);
6692 	switch (r) {
6693 	case 0:
6694 	case ECONNRESET:
6695 	case EALREADY:
6696 		break;
6697 
6698 	case EINVAL:
6699 		log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6700 		break;
6701 
6702 	case ENOENT:
6703 		log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6704 		break;
6705 
6706 	default:
6707 		bad_error("dgraph_set_milestone", r);
6708 	}
6709 
6710 out:
6711 	startd_free(fmri, max_scf_fmri_size);
6712 	scf_value_destroy(val);
6713 	scf_property_destroy(prop);
6714 
6715 	return (rebound ? ECONNRESET : 0);
6716 }
6717 
6718 /*
6719  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6720  * instance fmri or if 'fmri' matches the 'general' property group of an
6721  * instance (or the 'general/enabled' property).
6722  *
6723  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6724  */
6725 static void
6726 process_delete(char *fmri, scf_handle_t *h)
6727 {
6728 	char *lfmri, *end_inst_fmri;
6729 	const char *inst_name = NULL;
6730 	const char *pg_name = NULL;
6731 	const char *prop_name = NULL;
6732 
6733 	lfmri = safe_strdup(fmri);
6734 
6735 	/* Determine if the FMRI is a property group or instance */
6736 	if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6737 	    &prop_name) != SCF_SUCCESS) {
6738 		log_error(LOG_WARNING,
6739 		    "Received invalid FMRI \"%s\" from repository server.\n",
6740 		    fmri);
6741 	} else if (inst_name != NULL && pg_name == NULL) {
6742 		(void) dgraph_remove_instance(fmri, h);
6743 	} else if (inst_name != NULL && pg_name != NULL) {
6744 		/*
6745 		 * If we're deleting the 'general' property group or
6746 		 * 'general/enabled' property then the whole instance
6747 		 * must be removed from the dgraph.
6748 		 */
6749 		if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6750 			free(lfmri);
6751 			return;
6752 		}
6753 
6754 		if (prop_name != NULL &&
6755 		    strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6756 			free(lfmri);
6757 			return;
6758 		}
6759 
6760 		/*
6761 		 * Because the instance has already been deleted from the
6762 		 * repository, we cannot use any scf_ functions to retrieve
6763 		 * the instance FMRI however we can easily reconstruct it
6764 		 * manually.
6765 		 */
6766 		end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6767 		if (end_inst_fmri == NULL)
6768 			bad_error("process_delete", 0);
6769 
6770 		end_inst_fmri[0] = '\0';
6771 
6772 		(void) dgraph_remove_instance(fmri, h);
6773 	}
6774 
6775 	free(lfmri);
6776 }
6777 
6778 /*ARGSUSED*/
6779 void *
6780 repository_event_thread(void *unused)
6781 {
6782 	scf_handle_t *h;
6783 	scf_propertygroup_t *pg;
6784 	scf_instance_t *inst;
6785 	char *fmri = startd_alloc(max_scf_fmri_size);
6786 	char *pg_name = startd_alloc(max_scf_value_size);
6787 	int r;
6788 
6789 	(void) pthread_setname_np(pthread_self(), "repository_event");
6790 
6791 	h = libscf_handle_create_bound_loop();
6792 
6793 	pg = safe_scf_pg_create(h);
6794 	inst = safe_scf_instance_create(h);
6795 
6796 retry:
6797 	if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6798 		if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6799 			libscf_handle_rebind(h);
6800 		} else {
6801 			log_error(LOG_WARNING,
6802 			    "Couldn't set up repository notification "
6803 			    "for property group type %s: %s\n",
6804 			    SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6805 
6806 			(void) sleep(1);
6807 		}
6808 
6809 		goto retry;
6810 	}
6811 
6812 	/*CONSTCOND*/
6813 	while (1) {
6814 		ssize_t res;
6815 
6816 		/* Note: fmri is only set on delete events. */
6817 		res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6818 		if (res < 0) {
6819 			libscf_handle_rebind(h);
6820 			goto retry;
6821 		} else if (res == 0) {
6822 			/*
6823 			 * property group modified.  inst and pg_name are
6824 			 * pre-allocated scratch space.
6825 			 */
6826 			if (scf_pg_update(pg) < 0) {
6827 				switch (scf_error()) {
6828 				case SCF_ERROR_DELETED:
6829 					continue;
6830 
6831 				case SCF_ERROR_CONNECTION_BROKEN:
6832 					log_error(LOG_WARNING,
6833 					    "Lost repository event due to "
6834 					    "disconnection.\n");
6835 					libscf_handle_rebind(h);
6836 					goto retry;
6837 
6838 				case SCF_ERROR_NOT_BOUND:
6839 				case SCF_ERROR_NOT_SET:
6840 				default:
6841 					bad_error("scf_pg_update", scf_error());
6842 				}
6843 			}
6844 
6845 			r = process_pg_event(h, pg, inst, pg_name);
6846 			switch (r) {
6847 			case 0:
6848 				break;
6849 
6850 			case ECONNABORTED:
6851 				log_error(LOG_WARNING, "Lost repository event "
6852 				    "due to disconnection.\n");
6853 				libscf_handle_rebind(h);
6854 				/* FALLTHROUGH */
6855 
6856 			case ECONNRESET:
6857 				goto retry;
6858 
6859 			default:
6860 				bad_error("process_pg_event", r);
6861 			}
6862 		} else {
6863 			/*
6864 			 * Service, instance, or pg deleted.
6865 			 * Don't trust fmri on return.
6866 			 */
6867 			process_delete(fmri, h);
6868 		}
6869 	}
6870 
6871 	/*NOTREACHED*/
6872 	return (NULL);
6873 }
6874 
6875 void
6876 graph_engine_start()
6877 {
6878 	int err;
6879 
6880 	(void) startd_thread_create(graph_thread, NULL);
6881 
6882 	MUTEX_LOCK(&dgraph_lock);
6883 	while (!initial_milestone_set) {
6884 		err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6885 		assert(err == 0);
6886 	}
6887 	MUTEX_UNLOCK(&dgraph_lock);
6888 
6889 	(void) startd_thread_create(repository_event_thread, NULL);
6890 	(void) startd_thread_create(graph_event_thread, NULL);
6891 }
6892